72 lines
6.4 KiB
Python
72 lines
6.4 KiB
Python
"""Six fixed fresh fits; each update budget has a complete schedule; no test prediction/target access."""
|
|
import os,sys,time,traceback,collections
|
|
from pathlib import Path
|
|
import numpy as np
|
|
import torch
|
|
from phase3_data_sufficiency_v2_common import config,view_for,lr_step,first_epoch_release
|
|
from phase2_h2_model import H2Geo,optimizer,lr_epoch
|
|
from phase2_h2_data import standardization,load_part,to_device,arguments,cached
|
|
from phase2_h2_pilot_worker import seed,peaks,train_step
|
|
from phase2_residual_math import residual_target
|
|
from phase2_h1_worker_v2 import save,sha
|
|
|
|
def run(root,jobid):
|
|
c,pilot=config(root);j=c['jobs'][jobid];view=view_for(c,pilot,j);f=view['folds'][0];out=root/'jobs'/str(jobid)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir()
|
|
os.sched_setaffinity(0,j['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
|
|
start=time.monotonic();cpu=time.process_time();steps=0;epoch=0;stage='SETUP';exposures=collections.Counter();nodes_exposed=0;edges_exposed=0;checkpoints=[];val_checks=0
|
|
def record(extra,final=False):
|
|
d=dict(stage=stage,job=jobid,pool=j['pool'],mode=j['mode'],epoch=epoch,optimizer_steps=steps,nodes_exposed=nodes_exposed,edges_exposed=edges_exposed,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**extra,**peaks());save(out/'progress.json',d)
|
|
if final:save(out/'history'/f'{epoch:04d}.json',d)
|
|
return d
|
|
def checkpoint(tag):
|
|
path=out/(tag+'.pt');torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=0,pool=j['pool'],mode=j['mode'],contract_sha256=sha(root/'contract.json'),role=tag),path)
|
|
checkpoints.append(dict(tag=tag,steps=steps,path=str(path),sha256=sha(path),standardization=str(out/'standardization.json'),standardization_sha256=sha(out/'standardization.json')))
|
|
def validation():
|
|
nonlocal val_checks
|
|
begin=time.monotonic();model.eval();loss=0.;n=0
|
|
with torch.no_grad():
|
|
for case in f['validation']:
|
|
d=to_device(load_part(view,case,stats,'validation',out),'cuda');p=model(*arguments(d),checkpoint_blocks=False)
|
|
if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output')
|
|
with (out/'validation-target-access.jsonl').open('a') as log:
|
|
import json
|
|
log.write(json.dumps(dict(case=case,steps=steps,role='validation_diagnostic_only',target_sha256=view['parts'][case]['output_sha256']['target.npy']))+'\n')
|
|
target=residual_target(cached(view,case,'target'),d['F'],f['c'],fit_dtype='float32');delta=p.cpu().numpy().astype(np.float64)-target;loss+=float(delta@delta);n+=d['nodes'];del d,p
|
|
torch.cuda.synchronize()
|
|
if n!=f['validation_nodes']:raise ValueError('Validation coverage')
|
|
val_checks+=1;v=dict(steps=steps,validation_mse=loss/n,validation_nodes=n,validation_seconds=time.monotonic()-begin,validation_used_for_selection=False);save(out/f'validation-{steps:06d}.json',v);return v
|
|
try:
|
|
if os.environ.get('CUDA_VISIBLE_DEVICES')!=j['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU allocator identity')
|
|
stats=standardization(view,out)
|
|
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False;torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed();model=H2Geo().cuda();opt=optimizer(model)
|
|
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=j['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),seed=0,fresh_initialization=True,model_dtype='FP32',autocast=False))
|
|
while steps<j['target_steps']:
|
|
epoch+=1;stage='TRAIN';begin=time.monotonic();seen=0;enodes=0;loss_sum=0.;validation_seconds=0.
|
|
order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(len(f['learning']))
|
|
for index in order:
|
|
case=f['learning'][int(index)];lr=lr_epoch(epoch) if j['mode']=='epoch' else lr_step(steps+1,total=j['target_steps'])
|
|
for group in opt.param_groups:group['lr']=lr
|
|
d=to_device(load_part(view,case,stats,'learning',out),'cuda');res=train_step(model,opt,d,len(f['learning']),f['learning_nodes']);steps+=1;seen+=1;enodes+=d['nodes'];nodes_exposed+=d['nodes'];edges_exposed+=d['directed_edges'];exposures[case]+=1;loss_sum+=res['loss'];del d
|
|
record(dict(case=case,epoch_parts=seen,epoch_nodes=enodes,lr=lr,loss=res['loss']))
|
|
if j['mode']=='step' and steps==j['target_steps']:checkpoint('U' if steps==c['U'] else '2U')
|
|
if j['mode']=='step' and steps%3200==0:validation_seconds+=validation()['validation_seconds']
|
|
if steps==j['target_steps']:break
|
|
train_seconds=time.monotonic()-begin-validation_seconds
|
|
if j['mode']=='epoch' and epoch%10==0:validation_seconds+=validation()['validation_seconds']
|
|
full_epoch=seen==len(f['learning'])
|
|
if full_epoch and enodes!=f['learning_nodes']:raise ValueError('Epoch full-node identity')
|
|
stage='EPOCH_COMPLETE';record(dict(epoch_parts=seen,epoch_nodes=enodes,full_epoch=full_epoch,train_epoch_seconds=train_seconds,validation_seconds=validation_seconds,train_online_mean_weighted_loss=loss_sum/seen,not_frozen_model_training_domain_error=True),final=True)
|
|
if epoch==1:
|
|
stage='WAIT_FIRST_EPOCH_PROJECTION'
|
|
while not first_epoch_release(root,jobid):
|
|
record(dict(waiting=True));time.sleep(1)
|
|
if (root/'global-stop.json').exists():raise RuntimeError('Global item1 stop')
|
|
if j['mode']=='epoch':checkpoint('epoch200')
|
|
expected=['epoch200'] if j['mode']=='epoch' else ['U' if j['target_steps']==c['U'] else '2U']
|
|
if [x['tag'] for x in checkpoints]!=expected:raise ValueError('Checkpoint coverage')
|
|
save(out/'exposures.json',dict(per_part=dict(exposures),nodes_exposed=nodes_exposed,edges_exposed=edges_exposed,steps=steps,common_steps_not_equal_compute=True))
|
|
save(out/'result.json',dict(status='COMPLETE',job=jobid,pool=j['pool'],mode=j['mode'],completed_epochs=epoch,optimizer_steps=steps,checkpoints=checkpoints,validation_checks=val_checks,standardization_sha256=sha(out/'standardization.json'),exposures_sha256=sha(out/'exposures.json'),test_targets_read=False,test_prediction_started=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks()))
|
|
except BaseException as exc:
|
|
save(out/'failure.json',dict(status='FAILED',job=jobid,stage=stage,epoch=epoch,optimizer_steps=steps,reason=str(exc),traceback=traceback.format_exc(),**(peaks() if torch.cuda.is_initialized() else {})));raise
|
|
if __name__=='__main__':run(Path(sys.argv[1]),int(sys.argv[2]))
|