61 lines
4.5 KiB
Python
61 lines
4.5 KiB
Python
"""One bounded E256 timing probe, learning geometry and synthetic zero targets only."""
|
|
import argparse
|
|
from collections import Counter
|
|
import json
|
|
from pathlib import Path
|
|
import time
|
|
import traceback
|
|
import numpy as np
|
|
import torch
|
|
from freeze_g1_components import sha
|
|
from s2_e1_supervisor import save
|
|
from s2_models.network import QueryModel
|
|
from s2_models.normalization import fit_base_standardizer
|
|
from s2_pilot_data import permitted_role
|
|
from s2_pilot_train import train_part,part_weights,SkipPolicy,common_epoch_limit
|
|
from run_s2_pilot_training import LearningBases,read_part_arrays,to_device
|
|
|
|
|
|
def run(root):
|
|
started=time.monotonic();dest=root/'timing-probe';dest.mkdir(exist_ok=False)
|
|
rows=[];policy=SkipPolicy()
|
|
try:
|
|
c=json.loads((root/'pilot-plan.json').read_text())
|
|
for p,h in c['code_sha256'].items():assert sha(p)==h,p
|
|
assert sha(c['success_manifest'])==c['success_manifest_sha256'] and sha(c['role_manifest'])==c['role_manifest_sha256']
|
|
roles=json.loads(Path(c['role_manifest']).read_text());records={r['case']:r for r in json.loads(Path(c['success_manifest']).read_text())['ready_parts']}
|
|
learning=sorted(p for p in records if permitted_role(roles,p)=='learning');probe=c['probe_cases']
|
|
assert len(probe)==len(set(probe))==20 and set(probe)<=set(learning)
|
|
torch.set_num_threads(1);torch.manual_seed(0);np.random.seed(0)
|
|
if not torch.cuda.is_available() or torch.cuda.device_count()!=1 or 'V100' not in torch.cuda.get_device_name():raise RuntimeError('Exactly one visible V100 required')
|
|
device=torch.device('cuda:0');torch.cuda.manual_seed_all(0);torch.cuda.reset_peak_memory_stats();torch.backends.cuda.matmul.allow_tf32=False
|
|
stats=fit_base_standardizer(LearningBases(records,learning,roles),learning_part_ids=learning,fold='0',pool='100')
|
|
cache={case:read_part_arrays(records[case],roles,synthetic_targets=True) for case in probe}
|
|
model=QueryModel('E',256,standardizer=stats).to(device);optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-5)
|
|
scaler=torch.amp.GradScaler('cuda');groups=Counter(records[p]['component_id'] for p in learning)
|
|
for case in probe:
|
|
torch.cuda.synchronize();start=time.monotonic();model.train();data=to_device(cache[case],device)
|
|
weights=torch.tensor(part_weights(data['counts'],groups[records[case]['component_id']],len(groups),len(learning)),dtype=torch.float32,device=device)
|
|
info={};train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True,step_info=info)
|
|
policy.observe(info['step_skipped']);del data,weights;torch.cuda.synchronize();train_seconds=time.monotonic()-start
|
|
start=time.monotonic();model.eval()
|
|
with torch.no_grad(),torch.autocast(device_type='cuda',enabled=False):
|
|
data=to_device(cache[case],device);prediction=model(data['base'],data['ids'],data['gaps'],data['edge'])
|
|
# Include prediction device-to-host cost; discard values, no metrics.
|
|
prediction.float().cpu().numpy();del prediction,data
|
|
torch.cuda.synchronize();inference_seconds=time.monotonic()-start
|
|
rows.append(dict(case=case,nodes=records[case]['nodes'],training_seconds=train_seconds,inference_seconds=inference_seconds,amp=info))
|
|
save(dest/'progress.json',dict(rows=rows,amp=policy.summary(),elapsed_seconds=time.monotonic()-started))
|
|
nodes=sum(r['nodes'] for r in rows)
|
|
timing=common_epoch_limit(sum(r['training_seconds'] for r in rows)/nodes,sum(r['inference_seconds'] for r in rows)/nodes,c['learning_nodes'],c['validation_nodes'])
|
|
result=dict(status='COMPLETE',**timing,rows=rows,probe_nodes=nodes,amp=policy.summary(),standardizer_fingerprint=stats.fingerprint,
|
|
synthetic_zero_targets=True,labels_read=False,metrics_evaluated=False,model_discarded=True,
|
|
wall_seconds=time.monotonic()-started,peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved())
|
|
save(dest/'result.json',result);print(json.dumps({k:v for k,v in result.items() if k!='rows'}),flush=True)
|
|
except Exception as exc:
|
|
save(dest/'failure.json',dict(status='FAILED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(),amp=policy.summary(),rows=rows,wall_seconds=time.monotonic()-started));raise
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root)
|