"""One fixed pilot arm/width. Must be launched inside the GPU resource guard.""" import argparse from collections import Counter from collections.abc import Mapping import json from pathlib import Path import random import time import traceback import numpy as np import torch from freeze_g1_components import sha from s2_e1_supervisor import save from s2_models.network import QueryModel,TRAINING_CONTRACT from s2_models.normalization import BaseStandardizer,fit_base_standardizer from s2_models.objectives import relative_l2 from s2_pilot_data import permitted_role from s2_pilot_train import part_weights,train_part,component_score,SkipPolicy class LearningBases(Mapping): def __init__(self,records,ids,roles,cache=None):self.records=records;self.ids=tuple(ids);self.roles=roles;self.cache=cache def __iter__(self):return iter(self.ids) def __len__(self):return len(self.ids) def __getitem__(self,case): if case not in self.ids or permitted_role(self.roles,case)!='learning':raise PermissionError('Learning-only normalization') if self.cache is not None:return self.cache[case]['base'] assert sha(Path(self.records[case]['preparation_root'])/'base.npy')==self.records[case]['input_sha256']['base.npy'] return np.load(Path(self.records[case]['preparation_root'])/'base.npy',allow_pickle=False,mmap_mode='r') def read_part_arrays(record,roles,*,synthetic_targets=False): case=record['case'];role=permitted_role(roles,case) if synthetic_targets and role!='learning':raise PermissionError('Timing probe accepts only learning parts') root=Path(record['preparation_root']);label=Path(record['label_path']) assert sha(root/'base.npy')==record['input_sha256']['base.npy'] assert sha(root/'queries.npz')==record['input_sha256']['queries.npz'] with np.load(root/'queries.npz',allow_pickle=False) as q: mask=q['eligible_s2'];ids=q['anchor_ids'][mask];g=q['g'][mask];edge=q['edge_summary'][mask];counts=q['multiplicity'][mask] if synthetic_targets: target=np.zeros(len(ids));seconds=None else: assert sha(label)==record['input_sha256']['labels.npz'] with np.load(label,allow_pickle=False) as z: assert np.array_equal(ids,z['anchor_ids']) and np.array_equal(counts,z['multiplicity']) target=z['target_t_star'];seconds=z['seconds'] base=np.load(root/'base.npy',allow_pickle=False) if len(base)!=record['nodes']:raise ValueError('Full-node base size mismatch') return dict(base=base,ids=ids,gaps=g,edge=edge,target=target,counts=counts,truth_seconds=seconds) def to_device(data,device): result={k:v for k,v in data.items() if k not in ['base','ids','gaps','edge','target']} for key in ['base','ids','gaps','edge','target']: result[key]=torch.as_tensor(np.array(data[key],copy=True),dtype=torch.long if key=='ids' else torch.float32,device=device) return result def load_part(record,roles,device): return to_device(read_part_arrays(record,roles),device) def run(root,index): started=time.monotonic();c=json.loads((root/'training-contract.json').read_text());run=c['runs'][index] dest=root/run['name'];dest.mkdir(exist_ok=False);save(dest/'started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'training-contract.json'))) try: for p,h in c['code_sha256'].items():assert sha(p)==h,p source=Path(c['success_manifest']);assert sha(source)==c['success_manifest_sha256'] manifest=json.loads(source.read_text());records={r['case']:r for r in manifest['ready_parts']} roles=json.loads(Path(c['role_manifest']).read_text());assert sha(c['role_manifest'])==c['role_manifest_sha256'] learning=sorted(p for p in records if permitted_role(roles,p)=='learning') validation=sorted(p for p in records if permitted_role(roles,p)=='validation') if not learning or not validation:raise ValueError('Empty successful learning/validation roster') if any(p in roles['sealed_test_parts'] for p in records):raise PermissionError('Test case present in ready manifest') assert run['seed']==0 and c['fold']==0 and c['pool']==100 torch.set_num_threads(1);random.seed(run['seed']);np.random.seed(run['seed']);torch.manual_seed(run['seed']) if not torch.cuda.is_available() or torch.cuda.device_count()!=1:raise RuntimeError('Exactly one visible CUDA device required') device=torch.device('cuda:0');torch.cuda.manual_seed_all(run['seed']);torch.cuda.reset_peak_memory_stats() torch.backends.cuda.matmul.allow_tf32=False if 'V100' not in torch.cuda.get_device_name():raise RuntimeError('Frozen pilot requires one V100') # Check hashes once, cache raw FP64 arrays, and fit only the learning subset. cache={case:read_part_arrays(records[case],roles) for case in sorted(records)} stats=fit_base_standardizer(LearningBases(records,learning,roles,cache),learning_part_ids=learning,fold='0',pool='100') assert stats.fingerprint==c['standardizer_fingerprint'],'Probe and actual runs must share learning statistics' save(dest/'standardizer.json',dict(**stats.as_dict(),fingerprint=stats.fingerprint)) shared=root/'shared-standardizer-fingerprint.json' if shared.exists():assert json.loads(shared.read_text())['fingerprint']==stats.fingerprint else:save(shared,dict(fingerprint=stats.fingerprint,learning_parts=learning)) model=QueryModel(run['arm'],run['width'],standardizer=stats).to(device) optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-5) epoch_limit=int(c['E_max']);assert 1<=epoch_limit<=200 and c['cosine_T_max']==epoch_limit assert sha(c['timing_probe_receipt'])==c['timing_probe_sha256'] scheduler=torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,T_max=epoch_limit) scaler=torch.amp.GradScaler('cuda');rng=np.random.Generator(np.random.PCG64(run['seed'])) group_counts=Counter(records[p]['component_id'] for p in learning);group_number=len(group_counts) val_groups={p:records[p]['component_id'] for p in validation} best=float('inf');best_epoch=None;stale=0;history=[];updates=0;attempted_updates=0;skip_policy=SkipPolicy() for epoch in range(epoch_limit): model.train();loss_sum=0.;skip_policy.start_epoch() order=[learning[int(j)] for j in rng.permutation(len(learning))] for j,case in enumerate(order): record=records[case];data=to_device(cache[case],device) weights=torch.tensor(part_weights(data['counts'],group_counts[record['component_id']],group_number,len(learning)),dtype=torch.float32,device=device) step_info={} loss=train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True,step_info=step_info) loss_sum+=loss/len(learning);attempted_updates+=1;updates+=int(not step_info['step_skipped']) skip_policy.observe(step_info['step_skipped']) del data,weights if j%16==0: torch.cuda.synchronize();save(dest/'progress.json',dict(stage='training',epoch=epoch,parts_done=j+1,parts_total=len(learning),updates=updates,elapsed_seconds=time.monotonic()-started, peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved())) amp_summary=skip_policy.summary();save(dest/f'epoch-{epoch:03d}-amp.json',amp_summary) skip_policy.finish_epoch(epoch) model.eval();errors={} with torch.no_grad(): for case in validation: data=to_device(cache[case],device) with torch.autocast(device_type='cuda',enabled=False): pred=model(data['base'],data['ids'],data['gaps'],data['edge'] if run['arm']=='E' else None) predicted_seconds=pred.float().cpu().numpy().astype(np.float64)*records[case]['t_set'] score=relative_l2(predicted_seconds,data['truth_seconds'],data['counts'],np.ones(len(data['counts']),bool)) errors[case]=score['value'];del data,pred value=component_score(errors,val_groups);torch.cuda.synchronize() row=dict(epoch=epoch,amp=amp_summary,attempted_updates=attempted_updates,training_global_weighted_mse=loss_sum,validation_component_equal_relative_l2=value,updates=updates,elapsed_seconds=time.monotonic()-started, peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved()) history.append(row);save(dest/'history.json',history) if value=20:break torch.cuda.synchronize() result=dict(status='COMPLETE',run=run,E_max=epoch_limit,cosine_T_max=epoch_limit,epochs=len(history),attempted_updates=attempted_updates,updates=updates,best_epoch=best_epoch,best_validation_score=best, learning_parts=len(learning),validation_parts=len(validation),learning_components=group_number,validation_components=len(set(val_groups.values())), failed_preparation_parts_dropped=manifest['failed_parts'],standardizer_fingerprint=stats.fingerprint, wall_seconds=time.monotonic()-started,peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved(), test_labels_read=False,test_predictions=False,capacity_selected=False,checkpoint_sha256=sha(dest/'best.pt')) save(dest/'result.json',result);print(json.dumps(result),flush=True);return 0 except Exception as exc: save(dest/'failure.json',dict(status='FAILED_NO_RETRY',error_type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),wall_seconds=time.monotonic()-started, epoch=locals().get('epoch'),amp=skip_policy.summary() if 'skip_policy' in locals() else None)) raise if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.index))