"""One bounded E256 timing probe, learning geometry and synthetic zero targets only.""" import argparse from collections import Counter import json from pathlib import Path import time import traceback import numpy as np import torch from freeze_g1_components import sha from s2_e1_supervisor import save from s2_models.network import QueryModel from s2_models.normalization import fit_base_standardizer from s2_pilot_data import permitted_role from s2_pilot_train import train_part,part_weights,SkipPolicy,common_epoch_limit from run_s2_pilot_training import LearningBases,read_part_arrays,to_device def run(root): started=time.monotonic();dest=root/'timing-probe';dest.mkdir(exist_ok=False) rows=[];policy=SkipPolicy() try: c=json.loads((root/'pilot-plan.json').read_text()) for p,h in c['code_sha256'].items():assert sha(p)==h,p assert sha(c['success_manifest'])==c['success_manifest_sha256'] and sha(c['role_manifest'])==c['role_manifest_sha256'] roles=json.loads(Path(c['role_manifest']).read_text());records={r['case']:r for r in json.loads(Path(c['success_manifest']).read_text())['ready_parts']} learning=sorted(p for p in records if permitted_role(roles,p)=='learning');probe=c['probe_cases'] assert len(probe)==len(set(probe))==20 and set(probe)<=set(learning) torch.set_num_threads(1);torch.manual_seed(0);np.random.seed(0) if not torch.cuda.is_available() or torch.cuda.device_count()!=1 or 'V100' not in torch.cuda.get_device_name():raise RuntimeError('Exactly one visible V100 required') device=torch.device('cuda:0');torch.cuda.manual_seed_all(0);torch.cuda.reset_peak_memory_stats();torch.backends.cuda.matmul.allow_tf32=False stats=fit_base_standardizer(LearningBases(records,learning,roles),learning_part_ids=learning,fold='0',pool='100') cache={case:read_part_arrays(records[case],roles,synthetic_targets=True) for case in probe} model=QueryModel('E',256,standardizer=stats).to(device);optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-5) scaler=torch.amp.GradScaler('cuda');groups=Counter(records[p]['component_id'] for p in learning) for case in probe: torch.cuda.synchronize();start=time.monotonic();model.train();data=to_device(cache[case],device) weights=torch.tensor(part_weights(data['counts'],groups[records[case]['component_id']],len(groups),len(learning)),dtype=torch.float32,device=device) info={};train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True,step_info=info) policy.observe(info['step_skipped']);del data,weights;torch.cuda.synchronize();train_seconds=time.monotonic()-start start=time.monotonic();model.eval() with torch.no_grad(),torch.autocast(device_type='cuda',enabled=False): data=to_device(cache[case],device);prediction=model(data['base'],data['ids'],data['gaps'],data['edge']) # Include prediction device-to-host cost; discard values, no metrics. prediction.float().cpu().numpy();del prediction,data torch.cuda.synchronize();inference_seconds=time.monotonic()-start rows.append(dict(case=case,nodes=records[case]['nodes'],training_seconds=train_seconds,inference_seconds=inference_seconds,amp=info)) save(dest/'progress.json',dict(rows=rows,amp=policy.summary(),elapsed_seconds=time.monotonic()-started)) nodes=sum(r['nodes'] for r in rows) timing=common_epoch_limit(sum(r['training_seconds'] for r in rows)/nodes,sum(r['inference_seconds'] for r in rows)/nodes,c['learning_nodes'],c['validation_nodes']) result=dict(status='COMPLETE',**timing,rows=rows,probe_nodes=nodes,amp=policy.summary(),standardizer_fingerprint=stats.fingerprint, synthetic_zero_targets=True,labels_read=False,metrics_evaluated=False,model_discarded=True, wall_seconds=time.monotonic()-started,peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved()) save(dest/'result.json',result);print(json.dumps({k:v for k,v in result.items() if k!='rows'}),flush=True) except Exception as exc: save(dest/'failure.json',dict(status='FAILED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(),amp=policy.summary(),rows=rows,wall_seconds=time.monotonic()-started));raise if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root)