81 lines
6.9 KiB
Python
81 lines
6.9 KiB
Python
"""One fixed H2 pilot: representative steps, full fold0 epoch, inference timing."""
|
|
import gc
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import random
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import numpy as np
|
|
import torch
|
|
from phase2_h2_data import config,standardization,load_part,to_device,arguments,projection
|
|
from phase2_h2_model import H2Geo,optimizer,weighted_loss,lr_epoch
|
|
from phase2_h1_worker_v2 import save,sha
|
|
from phase2_residual_math import decode_seconds
|
|
|
|
def peaks():
|
|
return dict(peak_allocated_bytes=int(torch.cuda.max_memory_allocated()),peak_reserved_bytes=int(torch.cuda.max_memory_reserved()))
|
|
|
|
def seed():
|
|
random.seed(0);np.random.seed(0);torch.manual_seed(0);torch.cuda.manual_seed_all(0)
|
|
|
|
def train_step(model,opt,data,parts,total):
|
|
model.train();opt.zero_grad(set_to_none=True)
|
|
prediction=model(*arguments(data));loss=weighted_loss(prediction,data['target'],parts,data['nodes'],total)
|
|
if not bool(torch.isfinite(loss)):raise ValueError('Nonfinite loss')
|
|
loss.backward()
|
|
norm=torch.nn.utils.clip_grad_norm_(model.parameters(),1.,error_if_nonfinite=True)
|
|
opt.step()
|
|
if any(not bool(torch.isfinite(p).all()) for p in model.parameters()):raise ValueError('Nonfinite parameter')
|
|
torch.cuda.synchronize();return dict(loss=float(loss.detach()),gradient_norm_before_clip=float(norm),optimizer_steps=1,**peaks())
|
|
|
|
def run(root):
|
|
c=config(root);out=root/'worker';out.mkdir(exist_ok=False);os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
|
|
start=time.monotonic();cpu=time.process_time();stage='SETUP';records=[];history_index=0
|
|
(out/'history').mkdir()
|
|
def record(value):
|
|
nonlocal history_index
|
|
item=dict(unix=time.time(),stage=stage,elapsed_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**value)
|
|
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
|
|
save(out/'history'/f'{history_index:05d}.json',item);history_index+=1
|
|
save(out/'progress.json',item);return item
|
|
try:
|
|
if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator binding differs')
|
|
stats=standardization(c,out);torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
|
|
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed()
|
|
props=torch.cuda.get_device_properties(0)
|
|
save(out/'environment.json',dict(torch=torch.__version__,cuda=torch.version.cuda,gpu_name=props.name,gpu_uuid=c['gpu_uuid'],allocator=os.environ['PYTORCH_CUDA_ALLOC_CONF'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),autocast=False,grad_scaler=False,model_dtype='float32',affinity=sorted(os.sched_getaffinity(0))))
|
|
fold=c['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes'];stage='REPRESENTATIVE_FORWARD_BACKWARD'
|
|
for case in c['representatives']:
|
|
seed();model=H2Geo().cuda();opt=optimizer(model);begin=time.monotonic();data=to_device(load_part(c,case,stats,'learning',out),'cuda');torch.cuda.synchronize()
|
|
for step in range(2):
|
|
tick=time.monotonic();r=train_step(model,opt,data,parts,total);record(dict(case=case,probe_step=step+1,step_seconds=time.monotonic()-tick,nodes=data['nodes'],edges=data['directed_edges'],physical_components=data['physical_components'],**r))
|
|
records.append(dict(case=case,nodes=data['nodes'],edges=data['directed_edges'],seconds=time.monotonic()-begin,steps=2,**peaks()))
|
|
del data,opt,model;gc.collect();torch.cuda.empty_cache()
|
|
save(out/'representative-timings.json',records)
|
|
stage='FULL_FOLD0_EPOCH1';seed();model=H2Geo().cuda();opt=optimizer(model);epoch_start=time.monotonic();order=np.random.default_rng(np.random.SeedSequence([0,1])).permutation(parts);trained=[]
|
|
for index in order:
|
|
case=fold['learning'][int(index)];begin=time.monotonic();data=to_device(load_part(c,case,stats,'learning',out),'cuda');r=train_step(model,opt,data,parts,total)
|
|
trained.append(record(dict(case=case,epoch=1,epoch_parts=len(trained)+1,part_seconds=time.monotonic()-begin,nodes=data['nodes'],edges=data['directed_edges'],physical_components=data['physical_components'],lr=opt.param_groups[0]['lr'],**r)));del data
|
|
torch.cuda.synchronize();epoch_seconds=time.monotonic()-epoch_start
|
|
if len(trained)!=parts or sum(r['nodes'] for r in trained)!=total:raise ValueError('Incomplete full learning epoch')
|
|
for group in opt.param_groups:group['lr']=lr_epoch(2)
|
|
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=1,seed=0,contract_sha256=sha(root/'contract.json'),role='pilot_only_not_formal_checkpoint'),out/'pilot-epoch1.pt')
|
|
save(out/'epoch1.json',dict(parts=parts,nodes=total,optimizer_steps=parts,seconds=epoch_seconds,records=trained,checkpoint_sha256=sha(out/'pilot-epoch1.pt')))
|
|
stage='VALIDATION_FORWARD_ONLY';model.eval();validation=[];validation_start=time.monotonic()
|
|
with torch.no_grad():
|
|
for case in fold['validation']:
|
|
begin=time.monotonic();data=to_device(load_part(c,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False)
|
|
if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output')
|
|
seconds=decode_seconds(p.cpu().numpy(),data['F'],fold['c'],data['t_set']);torch.cuda.synchronize()
|
|
validation.append(record(dict(case=case,validation_parts=len(validation)+1,part_seconds=time.monotonic()-begin,nodes=data['nodes'],edges=data['directed_edges'],finite_seconds=bool(np.isfinite(seconds).all()),validation_targets_read=False,**peaks())));del data,p,seconds
|
|
duration=time.monotonic()-validation_start;elapsed=time.monotonic()-start
|
|
projected=projection(c,epoch_seconds,duration,elapsed+c['pilot_setup_charge_seconds']);save(out/'projection.json',projected)
|
|
save(out/'result.json',dict(status='COMPLETE',decision=projected['decision'],pilot_only=True,parameter_count=sum(p.numel() for p in model.parameters()),representatives=records,full_epoch_parts=parts,full_epoch_seconds=epoch_seconds,validation_parts=len(validation),validation_seconds=duration,worker_wall_seconds=elapsed,worker_cpu_seconds=time.process_time()-cpu,projection=projected,standardization_sha256=sha(out/'standardization.json'),validation_targets_read=False,test_targets_read=False,full_run_started=False,**peaks()))
|
|
except BaseException as exc:
|
|
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
|
|
save(out/'failure.json',dict(status='FAILED',stage=stage,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
|
|
|
|
if __name__=='__main__':run(Path(sys.argv[1]))
|