"""One fresh seed0 fold0 fit, exactly200 epochs; no test prediction or labels.""" import json,os,sys,time,traceback from pathlib import Path import numpy as np import torch from phase2_h2_s1_common import config,stats_from_pilot,selected_checkpoint from phase2_h2_data import load_part,to_device,arguments,cached from phase2_h2_model import H2Geo,optimizer,lr_epoch from phase2_h2_pilot_worker import seed,peaks,train_step from phase2_residual_math import residual_target from phase2_h1_worker_v2 import save,sha def run(root): c,old=config(root);out=root/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir() os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1) start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0 def record(item,epoch_file=False): item=dict(stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks()) with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n') save(out/'progress.json',item) if epoch_file:save(out/'history'/f'{epoch:03d}.json',item) return item try: if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity') stats=stats_from_pilot(c,old);save(out/'standardization.json',stats) torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed() model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes'] save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=c['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters()))) for epoch in range(1,201): stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0. if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch') order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts) for index in order: case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda') result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss'] record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip'])) del data if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch') metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch)) if epoch%10==0: stage='VALIDATION_DIAGNOSTIC';model.eval();val_loss=0.;val_nodes=0;vstart=time.monotonic() with torch.no_grad(): for case in fold['validation']: data=to_device(load_part(old,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False) if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output') with (out/'validation-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,epoch=epoch,role='validation_diagnostic_only',target_sha256=old['parts'][case]['output_sha256']['target.npy']))+'\n') target=residual_target(cached(old,case,'target'),data['F'],fold['c'],fit_dtype='float32') delta=p.cpu().numpy().astype(np.float64)-target;val_loss+=float(delta@delta);val_nodes+=data['nodes'];del data,p torch.cuda.synchronize() if val_nodes!=fold['validation_nodes']:raise ValueError('Validation coverage') metrics.update(validation_mse=val_loss/val_nodes,validation_nodes=val_nodes,validation_seconds=time.monotonic()-vstart,validation_used_for_selection=False) stage='EPOCH_COMPLETE';record(metrics,epoch_file=True) if epoch<200: for group in opt.param_groups:group['lr']=lr_epoch(epoch+1) stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch) torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=0,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name) save(out/'result.json',dict(status='COMPLETE',completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks())) except BaseException as exc: memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0) save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise if __name__=='__main__':run(Path(sys.argv[1]))