83 lines
7.4 KiB
Python
83 lines
7.4 KiB
Python
"""One new S2 fold, fixed item c loss with seed0/200epochs; no test prediction or target reads."""
|
|
import json,os,sys,time,traceback,random
|
|
from pathlib import Path
|
|
import numpy as np
|
|
import torch
|
|
from phase2_h2_s1_common import selected_checkpoint
|
|
from phase3_item_c_common import config,fold_view,first_epoch_release,assignment as assigned_job,read
|
|
from phase3_item_c_data import prepare_queries,query_data
|
|
from phase3_item_c_loss import objective
|
|
from phase3_shared_capacity import process_identity
|
|
from phase2_h2_data import standardization
|
|
from phase2_h2_data import load_part,to_device,arguments,cached
|
|
from phase2_h2_model import H2Geo,optimizer,lr_epoch
|
|
from phase2_h2_pilot_worker import peaks
|
|
from phase2_residual_math import residual_target
|
|
from phase2_h1_worker_v2 import save,sha
|
|
|
|
def train_step(model,opt,data,parts,total,query,c):
|
|
model.train();opt.zero_grad(set_to_none=True)
|
|
prediction=model(*arguments(data))
|
|
prior=torch.as_tensor(c*data['F'],dtype=torch.float64,device=prediction.device)
|
|
indices=torch.as_tensor(query['indices'],dtype=torch.int64,device=prediction.device)
|
|
distances=torch.as_tensor(query['distances'],dtype=torch.float64,device=prediction.device)
|
|
loss,regression,location=objective(prediction,data['target'],parts,data['nodes'],total,prior,indices,distances)
|
|
if not bool(torch.isfinite(loss)):raise ValueError('Nonfinite loss')
|
|
loss.backward();norm=torch.nn.utils.clip_grad_norm_(model.parameters(),1.,error_if_nonfinite=True);opt.step()
|
|
if any(not bool(torch.isfinite(p).all()) for p in model.parameters()):raise ValueError('Nonfinite parameter')
|
|
torch.cuda.synchronize()
|
|
return dict(loss=float(loss.detach()),regression_loss=float(regression.detach()),location_loss=float(location.detach()),gradient_norm_before_clip=float(norm),**peaks())
|
|
|
|
|
|
def run(root,fold_id,assignment_sha):
|
|
c,pilot=config(root);old=fold_view(pilot,fold_id);job=assigned_job(root,c,fold_id,assignment_sha);out=root/'jobs'/str(fold_id)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir()
|
|
os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
|
|
start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0
|
|
save(out/'worker-identity.json',process_identity(os.getpid()))
|
|
def record(item,epoch_file=False):
|
|
item=dict(fold=fold_id,stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks())
|
|
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
|
|
save(out/'progress.json',item)
|
|
if epoch_file:save(out/'history'/f'{epoch:03d}.json',item)
|
|
return item
|
|
try:
|
|
if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity')
|
|
stage='LEARNING_ONLY_NORMALIZATION';normal=out/'normalization';normal.mkdir();stats=standardization(old,normal);save(out/'standardization.json',stats);save(out/'fold-identity.json',dict(actual_fold=fold_id,learning=old['folds'][0]['learning'],validation=old['folds'][0]['validation'],test=old['folds'][0]['test'],clock_sha256=old['folds'][0]['clock_sha256']))
|
|
stage='LEARNING_QUERY_PREPARATION';queries=prepare_queries(old,read(c['query_roster_contract'])['rows'],out/'queries')
|
|
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
|
|
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();random.seed(0);np.random.seed(0);torch.manual_seed(0);torch.cuda.manual_seed_all(0)
|
|
model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes']
|
|
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=job['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters())))
|
|
for epoch in range(1,201):
|
|
stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0.
|
|
if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch')
|
|
order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts)
|
|
for index in order:
|
|
case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda')
|
|
query=query_data(old,case,queries);result=train_step(model,opt,data,parts,total,query,fold['c']);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss']
|
|
record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],regression_loss=result['regression_loss'],location_loss=result['location_loss'],gradient_norm_before_clip=result['gradient_norm_before_clip']))
|
|
del data
|
|
if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch')
|
|
metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch))
|
|
stage='EPOCH_COMPLETE';record(metrics,epoch_file=True)
|
|
if epoch==1:
|
|
stage='WAIT_THIS_FOLD_FIRST_EPOCH_COST_GATE';record(dict(epoch_parts=seen,epoch_nodes=nodes))
|
|
while not first_epoch_release(root,fold_id):time.sleep(.5)
|
|
if epoch<200:
|
|
for group in opt.param_groups:group['lr']=lr_epoch(epoch+1)
|
|
stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch)
|
|
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=fold_id,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name)
|
|
save(out/'result.json',dict(status='COMPLETE',fold=fold_id,completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,validation_targets_read=False,query_manifest_sha256=sha(out/'queries/manifest.json'),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks()))
|
|
except BaseException as exc:
|
|
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
|
|
save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
|
|
if __name__=='__main__':
|
|
import ctypes,signal
|
|
from phase3_shared_capacity import alive
|
|
expected=read(Path(sys.argv[1])/'jobs'/sys.argv[2]/'lease-ready.json')['owner']
|
|
parent=os.getppid()
|
|
if parent!=expected['pid'] or not alive(expected):raise RuntimeError('Pinned guard lost before worker import completed')
|
|
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGKILL,0,0,0)!=0:raise OSError('PDEATHSIG')
|
|
if os.getppid()!=parent:raise RuntimeError('Guard exited before worker setup')
|
|
run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3])
|