Files
MeltMind/tools/run_s2_pilot_probe.py
T

61 lines
4.5 KiB
Python

"""One bounded E256 timing probe, learning geometry and synthetic zero targets only."""
import argparse
from collections import Counter
import json
from pathlib import Path
import time
import traceback
import numpy as np
import torch
from freeze_g1_components import sha
from s2_e1_supervisor import save
from s2_models.network import QueryModel
from s2_models.normalization import fit_base_standardizer
from s2_pilot_data import permitted_role
from s2_pilot_train import train_part,part_weights,SkipPolicy,common_epoch_limit
from run_s2_pilot_training import LearningBases,read_part_arrays,to_device
def run(root):
started=time.monotonic();dest=root/'timing-probe';dest.mkdir(exist_ok=False)
rows=[];policy=SkipPolicy()
try:
c=json.loads((root/'pilot-plan.json').read_text())
for p,h in c['code_sha256'].items():assert sha(p)==h,p
assert sha(c['success_manifest'])==c['success_manifest_sha256'] and sha(c['role_manifest'])==c['role_manifest_sha256']
roles=json.loads(Path(c['role_manifest']).read_text());records={r['case']:r for r in json.loads(Path(c['success_manifest']).read_text())['ready_parts']}
learning=sorted(p for p in records if permitted_role(roles,p)=='learning');probe=c['probe_cases']
assert len(probe)==len(set(probe))==20 and set(probe)<=set(learning)
torch.set_num_threads(1);torch.manual_seed(0);np.random.seed(0)
if not torch.cuda.is_available() or torch.cuda.device_count()!=1 or 'V100' not in torch.cuda.get_device_name():raise RuntimeError('Exactly one visible V100 required')
device=torch.device('cuda:0');torch.cuda.manual_seed_all(0);torch.cuda.reset_peak_memory_stats();torch.backends.cuda.matmul.allow_tf32=False
stats=fit_base_standardizer(LearningBases(records,learning,roles),learning_part_ids=learning,fold='0',pool='100')
cache={case:read_part_arrays(records[case],roles,synthetic_targets=True) for case in probe}
model=QueryModel('E',256,standardizer=stats).to(device);optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-5)
scaler=torch.amp.GradScaler('cuda');groups=Counter(records[p]['component_id'] for p in learning)
for case in probe:
torch.cuda.synchronize();start=time.monotonic();model.train();data=to_device(cache[case],device)
weights=torch.tensor(part_weights(data['counts'],groups[records[case]['component_id']],len(groups),len(learning)),dtype=torch.float32,device=device)
info={};train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True,step_info=info)
policy.observe(info['step_skipped']);del data,weights;torch.cuda.synchronize();train_seconds=time.monotonic()-start
start=time.monotonic();model.eval()
with torch.no_grad(),torch.autocast(device_type='cuda',enabled=False):
data=to_device(cache[case],device);prediction=model(data['base'],data['ids'],data['gaps'],data['edge'])
# Include prediction device-to-host cost; discard values, no metrics.
prediction.float().cpu().numpy();del prediction,data
torch.cuda.synchronize();inference_seconds=time.monotonic()-start
rows.append(dict(case=case,nodes=records[case]['nodes'],training_seconds=train_seconds,inference_seconds=inference_seconds,amp=info))
save(dest/'progress.json',dict(rows=rows,amp=policy.summary(),elapsed_seconds=time.monotonic()-started))
nodes=sum(r['nodes'] for r in rows)
timing=common_epoch_limit(sum(r['training_seconds'] for r in rows)/nodes,sum(r['inference_seconds'] for r in rows)/nodes,c['learning_nodes'],c['validation_nodes'])
result=dict(status='COMPLETE',**timing,rows=rows,probe_nodes=nodes,amp=policy.summary(),standardizer_fingerprint=stats.fingerprint,
synthetic_zero_targets=True,labels_read=False,metrics_evaluated=False,model_discarded=True,
wall_seconds=time.monotonic()-started,peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved())
save(dest/'result.json',result);print(json.dumps({k:v for k,v in result.items() if k!='rows'}),flush=True)
except Exception as exc:
save(dest/'failure.json',dict(status='FAILED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(),amp=policy.summary(),rows=rows,wall_seconds=time.monotonic()-started));raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root)