"""One fixed H2 pilot: representative steps, full fold0 epoch, inference timing.""" import gc import json import os from pathlib import Path import random import sys import time import traceback import numpy as np import torch from phase2_h2_data import config,standardization,load_part,to_device,arguments,projection from phase2_h2_model import H2Geo,optimizer,weighted_loss,lr_epoch from phase2_h1_worker_v2 import save,sha from phase2_residual_math import decode_seconds def peaks(): return dict(peak_allocated_bytes=int(torch.cuda.max_memory_allocated()),peak_reserved_bytes=int(torch.cuda.max_memory_reserved())) def seed(): random.seed(0);np.random.seed(0);torch.manual_seed(0);torch.cuda.manual_seed_all(0) def train_step(model,opt,data,parts,total): model.train();opt.zero_grad(set_to_none=True) prediction=model(*arguments(data));loss=weighted_loss(prediction,data['target'],parts,data['nodes'],total) if not bool(torch.isfinite(loss)):raise ValueError('Nonfinite loss') loss.backward() norm=torch.nn.utils.clip_grad_norm_(model.parameters(),1.,error_if_nonfinite=True) opt.step() if any(not bool(torch.isfinite(p).all()) for p in model.parameters()):raise ValueError('Nonfinite parameter') torch.cuda.synchronize();return dict(loss=float(loss.detach()),gradient_norm_before_clip=float(norm),optimizer_steps=1,**peaks()) def run(root): c=config(root);out=root/'worker';out.mkdir(exist_ok=False);os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1) start=time.monotonic();cpu=time.process_time();stage='SETUP';records=[];history_index=0 (out/'history').mkdir() def record(value): nonlocal history_index item=dict(unix=time.time(),stage=stage,elapsed_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**value) with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n') save(out/'history'/f'{history_index:05d}.json',item);history_index+=1 save(out/'progress.json',item);return item try: if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator binding differs') stats=standardization(c,out);torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed() props=torch.cuda.get_device_properties(0) save(out/'environment.json',dict(torch=torch.__version__,cuda=torch.version.cuda,gpu_name=props.name,gpu_uuid=c['gpu_uuid'],allocator=os.environ['PYTORCH_CUDA_ALLOC_CONF'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),autocast=False,grad_scaler=False,model_dtype='float32',affinity=sorted(os.sched_getaffinity(0)))) fold=c['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes'];stage='REPRESENTATIVE_FORWARD_BACKWARD' for case in c['representatives']: seed();model=H2Geo().cuda();opt=optimizer(model);begin=time.monotonic();data=to_device(load_part(c,case,stats,'learning',out),'cuda');torch.cuda.synchronize() for step in range(2): tick=time.monotonic();r=train_step(model,opt,data,parts,total);record(dict(case=case,probe_step=step+1,step_seconds=time.monotonic()-tick,nodes=data['nodes'],edges=data['directed_edges'],physical_components=data['physical_components'],**r)) records.append(dict(case=case,nodes=data['nodes'],edges=data['directed_edges'],seconds=time.monotonic()-begin,steps=2,**peaks())) del data,opt,model;gc.collect();torch.cuda.empty_cache() save(out/'representative-timings.json',records) stage='FULL_FOLD0_EPOCH1';seed();model=H2Geo().cuda();opt=optimizer(model);epoch_start=time.monotonic();order=np.random.default_rng(np.random.SeedSequence([0,1])).permutation(parts);trained=[] for index in order: case=fold['learning'][int(index)];begin=time.monotonic();data=to_device(load_part(c,case,stats,'learning',out),'cuda');r=train_step(model,opt,data,parts,total) trained.append(record(dict(case=case,epoch=1,epoch_parts=len(trained)+1,part_seconds=time.monotonic()-begin,nodes=data['nodes'],edges=data['directed_edges'],physical_components=data['physical_components'],lr=opt.param_groups[0]['lr'],**r)));del data torch.cuda.synchronize();epoch_seconds=time.monotonic()-epoch_start if len(trained)!=parts or sum(r['nodes'] for r in trained)!=total:raise ValueError('Incomplete full learning epoch') for group in opt.param_groups:group['lr']=lr_epoch(2) torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=1,seed=0,contract_sha256=sha(root/'contract.json'),role='pilot_only_not_formal_checkpoint'),out/'pilot-epoch1.pt') save(out/'epoch1.json',dict(parts=parts,nodes=total,optimizer_steps=parts,seconds=epoch_seconds,records=trained,checkpoint_sha256=sha(out/'pilot-epoch1.pt'))) stage='VALIDATION_FORWARD_ONLY';model.eval();validation=[];validation_start=time.monotonic() with torch.no_grad(): for case in fold['validation']: begin=time.monotonic();data=to_device(load_part(c,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False) if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output') seconds=decode_seconds(p.cpu().numpy(),data['F'],fold['c'],data['t_set']);torch.cuda.synchronize() validation.append(record(dict(case=case,validation_parts=len(validation)+1,part_seconds=time.monotonic()-begin,nodes=data['nodes'],edges=data['directed_edges'],finite_seconds=bool(np.isfinite(seconds).all()),validation_targets_read=False,**peaks())));del data,p,seconds duration=time.monotonic()-validation_start;elapsed=time.monotonic()-start projected=projection(c,epoch_seconds,duration,elapsed+c['pilot_setup_charge_seconds']);save(out/'projection.json',projected) save(out/'result.json',dict(status='COMPLETE',decision=projected['decision'],pilot_only=True,parameter_count=sum(p.numel() for p in model.parameters()),representatives=records,full_epoch_parts=parts,full_epoch_seconds=epoch_seconds,validation_parts=len(validation),validation_seconds=duration,worker_wall_seconds=elapsed,worker_cpu_seconds=time.process_time()-cpu,projection=projected,standardization_sha256=sha(out/'standardization.json'),validation_targets_read=False,test_targets_read=False,full_run_started=False,**peaks())) except BaseException as exc: memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0) save(out/'failure.json',dict(status='FAILED',stage=stage,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise if __name__=='__main__':run(Path(sys.argv[1]))