"""Four fixed fresh fits; common step trajectory; no test prediction/target access.""" import os,sys,time,traceback,collections from pathlib import Path import numpy as np import torch from phase3_data_sufficiency_common import config,view_for,lr_step,first_epoch_release from phase2_h2_model import H2Geo,optimizer,lr_epoch from phase2_h2_data import standardization,load_part,to_device,arguments,cached from phase2_h2_pilot_worker import seed,peaks,train_step from phase2_residual_math import residual_target from phase2_h1_worker_v2 import save,sha def run(root,jobid): c,pilot=config(root);j=c['jobs'][jobid];view=view_for(c,pilot,j);f=view['folds'][0];out=root/'jobs'/str(jobid)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir() os.sched_setaffinity(0,j['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1) start=time.monotonic();cpu=time.process_time();steps=0;epoch=0;stage='SETUP';exposures=collections.Counter();nodes_exposed=0;edges_exposed=0;checkpoints=[];val_checks=0 def record(extra,final=False): d=dict(stage=stage,job=jobid,pool=j['pool'],mode=j['mode'],epoch=epoch,optimizer_steps=steps,nodes_exposed=nodes_exposed,edges_exposed=edges_exposed,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**extra,**peaks());save(out/'progress.json',d) if final:save(out/'history'/f'{epoch:04d}.json',d) return d def checkpoint(tag): path=out/(tag+'.pt');torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=0,pool=j['pool'],mode=j['mode'],contract_sha256=sha(root/'contract.json'),role=tag),path) checkpoints.append(dict(tag=tag,steps=steps,path=str(path),sha256=sha(path),standardization=str(out/'standardization.json'),standardization_sha256=sha(out/'standardization.json'))) def validation(): nonlocal val_checks begin=time.monotonic();model.eval();loss=0.;n=0 with torch.no_grad(): for case in f['validation']: d=to_device(load_part(view,case,stats,'validation',out),'cuda');p=model(*arguments(d),checkpoint_blocks=False) if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output') with (out/'validation-target-access.jsonl').open('a') as log: import json log.write(json.dumps(dict(case=case,steps=steps,role='validation_diagnostic_only',target_sha256=view['parts'][case]['output_sha256']['target.npy']))+'\n') target=residual_target(cached(view,case,'target'),d['F'],f['c'],fit_dtype='float32');delta=p.cpu().numpy().astype(np.float64)-target;loss+=float(delta@delta);n+=d['nodes'];del d,p torch.cuda.synchronize() if n!=f['validation_nodes']:raise ValueError('Validation coverage') val_checks+=1;v=dict(steps=steps,validation_mse=loss/n,validation_nodes=n,validation_seconds=time.monotonic()-begin,validation_used_for_selection=False);save(out/f'validation-{steps:06d}.json',v);return v try: if os.environ.get('CUDA_VISIBLE_DEVICES')!=j['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU allocator identity') stats=standardization(view,out) torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False;torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed();model=H2Geo().cuda();opt=optimizer(model) save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=j['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),seed=0,fresh_initialization=True,model_dtype='FP32',autocast=False)) while steps