Files
MeltMind/tools/phase2_h2_pilot_worker.py

81 lines
6.9 KiB
Python

"""One fixed H2 pilot: representative steps, full fold0 epoch, inference timing."""
import gc
import json
import os
from pathlib import Path
import random
import sys
import time
import traceback
import numpy as np
import torch
from phase2_h2_data import config,standardization,load_part,to_device,arguments,projection
from phase2_h2_model import H2Geo,optimizer,weighted_loss,lr_epoch
from phase2_h1_worker_v2 import save,sha
from phase2_residual_math import decode_seconds
def peaks():
return dict(peak_allocated_bytes=int(torch.cuda.max_memory_allocated()),peak_reserved_bytes=int(torch.cuda.max_memory_reserved()))
def seed():
random.seed(0);np.random.seed(0);torch.manual_seed(0);torch.cuda.manual_seed_all(0)
def train_step(model,opt,data,parts,total):
model.train();opt.zero_grad(set_to_none=True)
prediction=model(*arguments(data));loss=weighted_loss(prediction,data['target'],parts,data['nodes'],total)
if not bool(torch.isfinite(loss)):raise ValueError('Nonfinite loss')
loss.backward()
norm=torch.nn.utils.clip_grad_norm_(model.parameters(),1.,error_if_nonfinite=True)
opt.step()
if any(not bool(torch.isfinite(p).all()) for p in model.parameters()):raise ValueError('Nonfinite parameter')
torch.cuda.synchronize();return dict(loss=float(loss.detach()),gradient_norm_before_clip=float(norm),optimizer_steps=1,**peaks())
def run(root):
c=config(root);out=root/'worker';out.mkdir(exist_ok=False);os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
start=time.monotonic();cpu=time.process_time();stage='SETUP';records=[];history_index=0
(out/'history').mkdir()
def record(value):
nonlocal history_index
item=dict(unix=time.time(),stage=stage,elapsed_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**value)
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
save(out/'history'/f'{history_index:05d}.json',item);history_index+=1
save(out/'progress.json',item);return item
try:
if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator binding differs')
stats=standardization(c,out);torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed()
props=torch.cuda.get_device_properties(0)
save(out/'environment.json',dict(torch=torch.__version__,cuda=torch.version.cuda,gpu_name=props.name,gpu_uuid=c['gpu_uuid'],allocator=os.environ['PYTORCH_CUDA_ALLOC_CONF'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),autocast=False,grad_scaler=False,model_dtype='float32',affinity=sorted(os.sched_getaffinity(0))))
fold=c['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes'];stage='REPRESENTATIVE_FORWARD_BACKWARD'
for case in c['representatives']:
seed();model=H2Geo().cuda();opt=optimizer(model);begin=time.monotonic();data=to_device(load_part(c,case,stats,'learning',out),'cuda');torch.cuda.synchronize()
for step in range(2):
tick=time.monotonic();r=train_step(model,opt,data,parts,total);record(dict(case=case,probe_step=step+1,step_seconds=time.monotonic()-tick,nodes=data['nodes'],edges=data['directed_edges'],physical_components=data['physical_components'],**r))
records.append(dict(case=case,nodes=data['nodes'],edges=data['directed_edges'],seconds=time.monotonic()-begin,steps=2,**peaks()))
del data,opt,model;gc.collect();torch.cuda.empty_cache()
save(out/'representative-timings.json',records)
stage='FULL_FOLD0_EPOCH1';seed();model=H2Geo().cuda();opt=optimizer(model);epoch_start=time.monotonic();order=np.random.default_rng(np.random.SeedSequence([0,1])).permutation(parts);trained=[]
for index in order:
case=fold['learning'][int(index)];begin=time.monotonic();data=to_device(load_part(c,case,stats,'learning',out),'cuda');r=train_step(model,opt,data,parts,total)
trained.append(record(dict(case=case,epoch=1,epoch_parts=len(trained)+1,part_seconds=time.monotonic()-begin,nodes=data['nodes'],edges=data['directed_edges'],physical_components=data['physical_components'],lr=opt.param_groups[0]['lr'],**r)));del data
torch.cuda.synchronize();epoch_seconds=time.monotonic()-epoch_start
if len(trained)!=parts or sum(r['nodes'] for r in trained)!=total:raise ValueError('Incomplete full learning epoch')
for group in opt.param_groups:group['lr']=lr_epoch(2)
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=1,seed=0,contract_sha256=sha(root/'contract.json'),role='pilot_only_not_formal_checkpoint'),out/'pilot-epoch1.pt')
save(out/'epoch1.json',dict(parts=parts,nodes=total,optimizer_steps=parts,seconds=epoch_seconds,records=trained,checkpoint_sha256=sha(out/'pilot-epoch1.pt')))
stage='VALIDATION_FORWARD_ONLY';model.eval();validation=[];validation_start=time.monotonic()
with torch.no_grad():
for case in fold['validation']:
begin=time.monotonic();data=to_device(load_part(c,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False)
if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output')
seconds=decode_seconds(p.cpu().numpy(),data['F'],fold['c'],data['t_set']);torch.cuda.synchronize()
validation.append(record(dict(case=case,validation_parts=len(validation)+1,part_seconds=time.monotonic()-begin,nodes=data['nodes'],edges=data['directed_edges'],finite_seconds=bool(np.isfinite(seconds).all()),validation_targets_read=False,**peaks())));del data,p,seconds
duration=time.monotonic()-validation_start;elapsed=time.monotonic()-start
projected=projection(c,epoch_seconds,duration,elapsed+c['pilot_setup_charge_seconds']);save(out/'projection.json',projected)
save(out/'result.json',dict(status='COMPLETE',decision=projected['decision'],pilot_only=True,parameter_count=sum(p.numel() for p in model.parameters()),representatives=records,full_epoch_parts=parts,full_epoch_seconds=epoch_seconds,validation_parts=len(validation),validation_seconds=duration,worker_wall_seconds=elapsed,worker_cpu_seconds=time.process_time()-cpu,projection=projected,standardization_sha256=sha(out/'standardization.json'),validation_targets_read=False,test_targets_read=False,full_run_started=False,**peaks()))
except BaseException as exc:
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
save(out/'failure.json',dict(status='FAILED',stage=stage,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
if __name__=='__main__':run(Path(sys.argv[1]))