"""One new S2 fold, fixed item c loss with seed0/200epochs; no test prediction or target reads.""" import json,os,sys,time,traceback,random from pathlib import Path import numpy as np import torch from phase2_h2_s1_common import selected_checkpoint from phase3_item_c_common import config,fold_view,first_epoch_release,assignment as assigned_job,read from phase3_item_c_data import prepare_queries,query_data from phase3_item_c_loss import objective from phase3_shared_capacity import process_identity from phase2_h2_data import standardization from phase2_h2_data import load_part,to_device,arguments,cached from phase2_h2_model import H2Geo,optimizer,lr_epoch from phase2_h2_pilot_worker import peaks from phase2_residual_math import residual_target from phase2_h1_worker_v2 import save,sha def train_step(model,opt,data,parts,total,query,c): model.train();opt.zero_grad(set_to_none=True) prediction=model(*arguments(data)) prior=torch.as_tensor(c*data['F'],dtype=torch.float64,device=prediction.device) indices=torch.as_tensor(query['indices'],dtype=torch.int64,device=prediction.device) distances=torch.as_tensor(query['distances'],dtype=torch.float64,device=prediction.device) loss,regression,location=objective(prediction,data['target'],parts,data['nodes'],total,prior,indices,distances) if not bool(torch.isfinite(loss)):raise ValueError('Nonfinite loss') loss.backward();norm=torch.nn.utils.clip_grad_norm_(model.parameters(),1.,error_if_nonfinite=True);opt.step() if any(not bool(torch.isfinite(p).all()) for p in model.parameters()):raise ValueError('Nonfinite parameter') torch.cuda.synchronize() return dict(loss=float(loss.detach()),regression_loss=float(regression.detach()),location_loss=float(location.detach()),gradient_norm_before_clip=float(norm),**peaks()) def run(root,fold_id,assignment_sha): c,pilot=config(root);old=fold_view(pilot,fold_id);job=assigned_job(root,c,fold_id,assignment_sha);out=root/'jobs'/str(fold_id)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir() os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1) start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0 save(out/'worker-identity.json',process_identity(os.getpid())) def record(item,epoch_file=False): item=dict(fold=fold_id,stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks()) with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n') save(out/'progress.json',item) if epoch_file:save(out/'history'/f'{epoch:03d}.json',item) return item try: if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity') stage='LEARNING_ONLY_NORMALIZATION';normal=out/'normalization';normal.mkdir();stats=standardization(old,normal);save(out/'standardization.json',stats);save(out/'fold-identity.json',dict(actual_fold=fold_id,learning=old['folds'][0]['learning'],validation=old['folds'][0]['validation'],test=old['folds'][0]['test'],clock_sha256=old['folds'][0]['clock_sha256'])) stage='LEARNING_QUERY_PREPARATION';queries=prepare_queries(old,read(c['query_roster_contract'])['rows'],out/'queries') torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();random.seed(0);np.random.seed(0);torch.manual_seed(0);torch.cuda.manual_seed_all(0) model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes'] save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=job['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters()))) for epoch in range(1,201): stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0. if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch') order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts) for index in order: case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda') query=query_data(old,case,queries);result=train_step(model,opt,data,parts,total,query,fold['c']);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss'] record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],regression_loss=result['regression_loss'],location_loss=result['location_loss'],gradient_norm_before_clip=result['gradient_norm_before_clip'])) del data if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch') metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch)) stage='EPOCH_COMPLETE';record(metrics,epoch_file=True) if epoch==1: stage='WAIT_THIS_FOLD_FIRST_EPOCH_COST_GATE';record(dict(epoch_parts=seen,epoch_nodes=nodes)) while not first_epoch_release(root,fold_id):time.sleep(.5) if epoch<200: for group in opt.param_groups:group['lr']=lr_epoch(epoch+1) stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch) torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=fold_id,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name) save(out/'result.json',dict(status='COMPLETE',fold=fold_id,completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,validation_targets_read=False,query_manifest_sha256=sha(out/'queries/manifest.json'),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks())) except BaseException as exc: memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0) save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise if __name__=='__main__': import ctypes,signal from phase3_shared_capacity import alive expected=read(Path(sys.argv[1])/'jobs'/sys.argv[2]/'lease-ready.json')['owner'] parent=os.getppid() if parent!=expected['pid'] or not alive(expected):raise RuntimeError('Pinned guard lost before worker import completed') if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGKILL,0,0,0)!=0:raise OSError('PDEATHSIG') if os.getppid()!=parent:raise RuntimeError('Guard exited before worker setup') run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3])