"""One fresh frozen matrix cell; unchanged v2 numerical training loop inside the GPU guard.""" import argparse from collections import Counter from collections.abc import Mapping import json from pathlib import Path import random import time import traceback import numpy as np import torch from freeze_g1_components import sha from s2_e1_supervisor import save from s2_models_v2.network import QueryModel,TRAINING_CONTRACT from s2_models.normalization import BaseStandardizer,fit_base_standardizer from s2_models.objectives import relative_l2 from s2_pilot_data import permitted_role from s2_main_data import cell_records from s2_pilot_train import part_weights,train_part,component_score,SkipPolicy class LearningBases(Mapping): def __init__(self,records,ids,roles,cache=None):self.records=records;self.ids=tuple(ids);self.roles=roles;self.cache=cache def __iter__(self):return iter(self.ids) def __len__(self):return len(self.ids) def __getitem__(self,case): if case not in self.ids or permitted_role(self.roles,case)!='learning':raise PermissionError('Learning-only normalization') if self.cache is not None:return self.cache[case]['base'] assert sha(Path(self.records[case]['preparation_root'])/'base.npy')==self.records[case]['input_sha256']['base.npy'] return np.load(Path(self.records[case]['preparation_root'])/'base.npy',allow_pickle=False,mmap_mode='r') def read_part_arrays(record,roles,*,synthetic_targets=False): case=record['case'];role=permitted_role(roles,case) if synthetic_targets and role!='learning':raise PermissionError('Timing probe accepts only learning parts') root=Path(record['preparation_root']);label=Path(record['label_path']) assert sha(root/'base.npy')==record['input_sha256']['base.npy'] assert sha(root/'queries.npz')==record['input_sha256']['queries.npz'] with np.load(root/'queries.npz',allow_pickle=False) as q: mask=q['eligible_s2'];ids=q['anchor_ids'][mask];g=q['g'][mask];edge=q['edge_summary'][mask];counts=q['multiplicity'][mask] if synthetic_targets: target=np.zeros(len(ids));seconds=None else: assert sha(label)==record['input_sha256']['labels.npz'] with np.load(label,allow_pickle=False) as z: assert np.array_equal(ids,z['anchor_ids']) and np.array_equal(counts,z['multiplicity']) target=z['target_t_star'];seconds=z['seconds'] base=np.load(root/'base.npy',allow_pickle=False) if len(base)!=record['nodes']:raise ValueError('Full-node base size mismatch') return dict(base=base,ids=ids,gaps=g,edge=edge,target=target,counts=counts,truth_seconds=seconds) def to_device(data,device): result={k:v for k,v in data.items() if k not in ['base','ids','gaps','edge','target']} for key in ['base','ids','gaps','edge','target']: result[key]=torch.as_tensor(np.array(data[key],copy=True),dtype=torch.long if key=='ids' else torch.float32,device=device) return result def load_part(record,roles,device): return to_device(read_part_arrays(record,roles),device) def run(root,index): started=time.monotonic();c=json.loads((root/'training-contract.json').read_text());run=c['runs'][index] dest=root/'cells'/run['name']/run['name'];dest.mkdir(exist_ok=False);save(dest/'started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'training-contract.json'))) try: for p,h in c['code_sha256'].items():assert sha(p)==h,p records,roles,manifest,target_audit=cell_records(root,c,run) save(dest/'target-access.json',dict(fold=run['fold'],pool=run['pool'],roster_sha256=c['roster_sha256'],access=target_audit,outer_fold_test_labels_read=False)) learning=sorted(p for p in records if permitted_role(roles,p)=='learning') validation=sorted(p for p in records if permitted_role(roles,p)=='validation') if not learning or not validation:raise ValueError('Empty successful learning/validation roster') if any(p in roles['sealed_test_parts'] for p in records):raise PermissionError('Test case present in ready manifest') assert run['seed'] in [0,1,2] and run['fold'] in range(5) and run['pool'] in [25,50,100] assert run['width']==128 and c['E_max']==42 and run['initialization']=='fresh_seeded' torch.set_num_threads(1);random.seed(run['seed']);np.random.seed(run['seed']);torch.manual_seed(run['seed']) if not torch.cuda.is_available() or torch.cuda.device_count()!=1:raise RuntimeError('Exactly one visible CUDA device required') device=torch.device('cuda:0');torch.cuda.manual_seed_all(run['seed']);torch.cuda.reset_peak_memory_stats() torch.backends.cuda.matmul.allow_tf32=False if 'V100' not in torch.cuda.get_device_name():raise RuntimeError('Frozen pilot requires one V100') # Check hashes once, cache raw FP64 arrays, and fit only the learning subset. cache={case:read_part_arrays(records[case],roles) for case in sorted(records)} stats=fit_base_standardizer(LearningBases(records,learning,roles,cache),learning_part_ids=learning,fold=str(run['fold']),pool=str(run['pool'])) save(dest/'standardizer.json',dict(**stats.as_dict(),fingerprint=stats.fingerprint)) model=QueryModel(run['arm'],run['width'],standardizer=stats).to(device) optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-5) epoch_limit=int(c['E_max']);assert 1<=epoch_limit<=200 and c['cosine_T_max']==epoch_limit scheduler=torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,T_max=epoch_limit) scaler=torch.amp.GradScaler('cuda');rng=np.random.Generator(np.random.PCG64(run['seed'])) group_counts=Counter(records[p]['component_id'] for p in learning);group_number=len(group_counts) val_groups={p:records[p]['component_id'] for p in validation} best=float('inf');best_epoch=None;stale=0;history=[];updates=0;attempted_updates=0;skip_policy=SkipPolicy() for epoch in range(epoch_limit): model.train();loss_sum=0.;skip_policy.start_epoch() order=[learning[int(j)] for j in rng.permutation(len(learning))] for j,case in enumerate(order): record=records[case];data=to_device(cache[case],device) weights=torch.tensor(part_weights(data['counts'],group_counts[record['component_id']],group_number,len(learning)),dtype=torch.float32,device=device) step_info={} loss=train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True,step_info=step_info) loss_sum+=loss/len(learning);attempted_updates+=1;updates+=int(not step_info['step_skipped']) skip_policy.observe(step_info['step_skipped']) del data,weights if j%16==0: torch.cuda.synchronize();save(dest/'progress.json',dict(stage='training',epoch=epoch,parts_done=j+1,parts_total=len(learning),updates=updates,elapsed_seconds=time.monotonic()-started, peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved())) amp_summary=skip_policy.summary();save(dest/f'epoch-{epoch:03d}-amp.json',amp_summary) skip_policy.finish_epoch(epoch) model.eval();errors={} with torch.no_grad(): for case in validation: data=to_device(cache[case],device) with torch.autocast(device_type='cuda',enabled=False): pred=model(data['base'],data['ids'],data['gaps'],data['edge'] if run['arm']=='E' else None) predicted_seconds=pred.float().cpu().numpy().astype(np.float64)*records[case]['t_set'] score=relative_l2(predicted_seconds,data['truth_seconds'],data['counts'],np.ones(len(data['counts']),bool)) errors[case]=score['value'];del data,pred value=component_score(errors,val_groups);torch.cuda.synchronize() row=dict(epoch=epoch,amp=amp_summary,attempted_updates=attempted_updates,training_global_weighted_mse=loss_sum,validation_component_equal_relative_l2=value,updates=updates,elapsed_seconds=time.monotonic()-started, peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved()) history.append(row);save(dest/'history.json',history) if value=20:break torch.cuda.synchronize() result=dict(status='COMPLETE',run=run,E_max=epoch_limit,cosine_T_max=epoch_limit,epochs=len(history),attempted_updates=attempted_updates,updates=updates,best_epoch=best_epoch,best_validation_score=best, learning_parts=len(learning),validation_parts=len(validation),learning_components=group_number,validation_components=len(set(val_groups.values())), roster_sha256=c['roster_sha256'],fold=run['fold'],pool=run['pool'],fresh_initialization=True,failed_preparation_parts_dropped=manifest['failed_parts'],standardizer_fingerprint=stats.fingerprint, wall_seconds=time.monotonic()-started,peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved(), test_labels_read=False,test_predictions=False,capacity_selected=False,checkpoint_sha256=sha(dest/'best.pt')) save(dest/'result.json',result);print(json.dumps(result),flush=True);return 0 except Exception as exc: save(dest/'failure.json',dict(status='FAILED_NO_RETRY',error_type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),wall_seconds=time.monotonic()-started, peak_allocated_bytes=torch.cuda.max_memory_allocated() if torch.cuda.is_initialized() else 0, peak_reserved_bytes=torch.cuda.max_memory_reserved() if torch.cuda.is_initialized() else 0, epoch=locals().get('epoch'),amp=skip_policy.summary() if 'skip_policy' in locals() else None)) raise if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.index))