Files
MeltMind/tools/phase3_seed1_worker.py
T

68 lines
6.6 KiB
Python

"""One new S2 fold, same frozen recipe with seed1/200epochs; no test prediction or target reads."""
import json,os,sys,time,traceback,random
from pathlib import Path
import numpy as np
import torch
from phase2_h2_s1_common import selected_checkpoint
from phase3_seed1_common import config,fold_view,first_epoch_release,assigned_job
from phase2_h2_data import standardization
from phase2_h2_data import load_part,to_device,arguments,cached
from phase2_h2_model import H2Geo,optimizer,lr_epoch
from phase2_h2_pilot_worker import peaks,train_step
from phase2_residual_math import residual_target
from phase2_h1_worker_v2 import save,sha
def run(root,fold_id,assignment_sha):
c,pilot=config(root);old=fold_view(pilot,fold_id);job=assigned_job(root,c,fold_id,assignment_sha);out=root/'jobs'/str(fold_id)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir()
os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0
def record(item,epoch_file=False):
item=dict(fold=fold_id,stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks())
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
save(out/'progress.json',item)
if epoch_file:save(out/'history'/f'{epoch:03d}.json',item)
return item
try:
if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity')
stage='LEARNING_ONLY_NORMALIZATION';normal=out/'normalization';normal.mkdir();stats=standardization(old,normal);save(out/'standardization.json',stats);save(out/'fold-identity.json',dict(actual_fold=fold_id,learning=old['folds'][0]['learning'],validation=old['folds'][0]['validation'],test=old['folds'][0]['test'],clock_sha256=old['folds'][0]['clock_sha256']))
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();random.seed(1);np.random.seed(1);torch.manual_seed(1);torch.cuda.manual_seed_all(1)
model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes']
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=job['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=1,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters())))
for epoch in range(1,201):
stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0.
if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch')
order=np.random.default_rng(np.random.SeedSequence([1,epoch])).permutation(parts)
for index in order:
case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda')
result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss']
record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip']))
del data
if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch')
metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch))
if epoch%10==0:
stage='VALIDATION_DIAGNOSTIC';model.eval();val_loss=0.;val_nodes=0;vstart=time.monotonic()
with torch.no_grad():
for case in fold['validation']:
data=to_device(load_part(old,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False)
if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output')
with (out/'validation-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,epoch=epoch,role='validation_diagnostic_only',target_sha256=old['parts'][case]['output_sha256']['target.npy']))+'\n')
target=residual_target(cached(old,case,'target'),data['F'],fold['c'],fit_dtype='float32')
delta=p.cpu().numpy().astype(np.float64)-target;val_loss+=float(delta@delta);val_nodes+=data['nodes'];del data,p
torch.cuda.synchronize()
if val_nodes!=fold['validation_nodes']:raise ValueError('Validation coverage')
metrics.update(validation_mse=val_loss/val_nodes,validation_nodes=val_nodes,validation_seconds=time.monotonic()-vstart,validation_used_for_selection=False)
stage='EPOCH_COMPLETE';record(metrics,epoch_file=True)
if epoch==1:
stage='WAIT_THIS_FOLD_FIRST_EPOCH_COST_GATE';record(dict(epoch_parts=seen,epoch_nodes=nodes))
while not first_epoch_release(root,fold_id):time.sleep(.5)
if epoch<200:
for group in opt.param_groups:group['lr']=lr_epoch(epoch+1)
stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch)
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=1,fold=fold_id,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name)
save(out/'result.json',dict(status='COMPLETE',fold=fold_id,completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks()))
except BaseException as exc:
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
if __name__=='__main__':run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3])