Files
MeltMind/tools/track_b_gino_greif_worker.py
T

229 lines
11 KiB
Python

"""One guarded Greif job. Only learning tensors are opened by the fit paths."""
import argparse
import ctypes
import json
import os
from pathlib import Path
import random
import signal
import sys
import time
from types import FunctionType, SimpleNamespace
import numpy as np
import torch
from phase2_h2_data import cached, standardization
from phase2_h2_model import lr_epoch, optimizer, weighted_loss
from phase2_residual_math import decode_seconds, residual_target
from s2_custody_common import prediction_inputs
import track_b_gino_formal_worker as release
import track_b_gino_greif_entry as entry
from track_b_gino_greif_adapter import GreifGINO, stencil
from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha
from track_b_gino_shapenet_worker import cuda_peaks, test_environment
def fold_view(c, fold):
from track_b_gino_greif_prepare import PILOT, PINS
pilot=read(checked(PILOT,PINS[str(PILOT)]))
row=pilot['folds'][fold]
fixed=c['folds'][str(fold)]
if any(row[role]!=cases for role,cases in fixed['parts'].items()) or row['clock_sha256']!=fixed['clock_sha256']:
raise ValueError('Learning, validation and test roles or original clock differ')
return dict(pilot,folds=[row])
def model_input(pilot, case, stats, role):
f=pilot['folds'][0]
if role not in ('learning','test') or case not in f[role]:
raise PermissionError('Original fold role required before reading inputs')
pin=pilot['geometry'][case]
with np.load(checked(pin['path'],pin['sha256']),allow_pickle=False) as archive:
points,raw,mass=(archive[k].copy() for k in ('points','raw_ids','mass'))
if len(raw)!=pilot['parts'][case]['nodes'] or not np.array_equal(raw,cached(pilot,case,'raw_ids')) or not np.array_equal(mass,cached(pilot,case,'mass')):
raise ValueError('Original full node identity differs')
feat=np.asarray(cached(pilot,case,'features'),np.float64)
x=((feat-np.asarray(stats['mean']))/np.asarray(stats['scale'])).astype(np.float32)
fvec=cached(pilot,case,'Vf')
if x.shape!=(len(raw),19) or not np.isfinite(x).all() or fvec.dtype!=np.float64 or fvec.shape!=(len(raw),):
raise ValueError('Original nineteen columns or FP64 clock differ')
corners,weights=stencil(points,raw)
args=(torch.from_numpy(x),torch.from_numpy(mass.astype(np.float32)),corners,weights)
return args,raw,fvec
def checkpoint(path, model, opt, epoch, fold, seed, stats):
temp=path.with_suffix('.tmp')
with temp.open('xb') as stream:
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),epochs=epoch,fold=fold,seed=seed,
stats=stats,torch_cpu_rng=torch.get_rng_state(),torch_cuda_rng=torch.cuda.get_rng_state(),
numpy_rng=np.random.get_state(),python_rng=random.getstate(),
contract_sha256=sha(entry.ROOT/'contract.json')),stream)
temp.rename(path)
def restore(artifact, fold, seed, epochs):
state=torch.load(checked(artifact['path'],artifact['sha256']),map_location='cpu')
if (state['fold'],state['seed'],state['epochs'],state['contract_sha256']) != (
fold,seed,epochs,sha(entry.ROOT/'contract.json')):
raise ValueError('Checkpoint identity differs')
return state
def train(c, fold, seed, phase, root, records):
p=fold_view(c,fold)
f=p['folds'][0]
rng_seed=seed
random.seed(rng_seed)
np.random.seed(rng_seed)
torch.manual_seed(rng_seed)
torch.cuda.manual_seed_all(rng_seed)
model=GreifGINO().cuda()
opt=optimizer(model)
start, end=1,c['epochs']
if phase=='first':
if seed!=0:
raise PermissionError('First full epoch is seed 0 only')
end=1
normalization=root/'norm'
normalization.mkdir(exist_ok=False)
stats=standardization(p,normalization)
else:
cost=read(entry.ROOT/'first-epoch-cost-gate.json')
if not cost['release_epoch2']:
raise PermissionError('Both measured full epochs must pass the cost gate')
earlier=entry.stage_terminal(fold,0,'first')['files'][0]
state=restore(earlier,fold,0,1)
stats=state['stats']
if seed==0:
start=2
model.load_state_dict(state['model'],strict=True)
opt.load_state_dict(state['optimizer'])
torch.set_rng_state(state['torch_cpu_rng'])
torch.cuda.set_rng_state(state['torch_cuda_rng'])
np.random.set_state(state['numpy_rng'])
random.setstate(state['python_rng'])
del state
if stats['parts']!=f['learning'] or stats['nodes']!=f['learning_nodes']:
raise ValueError('Only the learning normalization of this fold is valid')
ncases,nnodes=len(f['learning']),f['learning_nodes']
duration=None
with (root/'learning-access.jsonl').open('x') as access:
for epoch in range(start,end+1):
started=time.monotonic()
model.train()
for group in opt.param_groups:
group['lr']=lr_epoch(epoch)
seen=0
order=np.random.default_rng(np.random.SeedSequence([seed,epoch])).permutation(ncases)
for index in order:
case=f['learning'][int(index)]
args,raw,fvec=model_input(p,case,stats,'learning')
access.write(json.dumps(dict(case=case,fold=fold,seed=seed,epoch=epoch,role='learning',
sha256=p['parts'][case]['output_sha256']['target.npy']))+'\n')
access.flush()
target=residual_target(cached(p,case,'target'),fvec,f['c'],fit_dtype='float32')
opt.zero_grad(set_to_none=True)
prediction=model(*(a.cuda() for a in args))
loss=weighted_loss(prediction,torch.from_numpy(target).cuda(),ncases,len(raw),nnodes)
if not bool(torch.isfinite(loss)):
raise ValueError('Nonfinite full-node H2-weighted loss')
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(),1.,error_if_nonfinite=True)
opt.step()
if any(not bool(torch.isfinite(value).all()) for value in model.parameters()):
raise ValueError('Nonfinite model weight after update')
torch.cuda.synchronize()
seen+=len(raw)
del prediction,loss,args,raw,fvec,target
if seen!=nnodes:
raise ValueError('Incomplete full-node learning epoch')
duration=time.monotonic()-started
save(records/'progress.json',dict(stage='EPOCH_COMPLETE',epoch=epoch,
learning_cases=ncases,learning_nodes=seen,seconds=duration,**cuda_peaks(torch)))
output=root/f'epoch-{end}.pt'
checkpoint(output,model,opt,end,fold,seed,stats)
return dict(epochs=end,learning_cases=ncases,learning_nodes=nnodes,
first_epoch_seconds=duration if phase=='first' else None,files=[
dict(path=str(output),sha256=sha(output),fold=fold,seed=seed)])
def predict(c, fold, seed, root, records):
from track_b_gino_greif_prepare import QUERY, PINS
p=fold_view(c,fold)
f=p['folds'][0]
models=read(entry.ROOT/'models-frozen.json')
by={(r['fold'],r['seed']):r for r in models}
if len(models)!=4 or set(by)!=set(entry.JOBS):
raise PermissionError('Predict only after four frozen endpoints')
state=restore(by[fold,seed],fold,seed,c['epochs'])
stats=state['stats']
model=GreifGINO().cuda()
model.load_state_dict(state['model'],strict=True)
model.eval()
del state
if stats['parts']!=f['learning'] or stats['nodes']!=f['learning_nodes']:
raise ValueError('Original fold normalization differs')
query={r['case']:r for r in read(checked(QUERY,PINS[str(QUERY)]))['rows'] if r['included']}
entries=[]
with torch.no_grad():
for case in f['test']:
args,raw,fvec=model_input(p,case,stats,'test')
q=prediction_inputs(query[case])
ix=np.searchsorted(raw,q['raw_ids'])
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):
raise ValueError('Frozen PPS query-to-node map differs')
residual=model(*(a.cuda() for a in args)).cpu().numpy()
seconds=decode_seconds(residual,fvec,f['c'],p['parts'][case]['t_set'])
item=root/(case+'.npz')
with item.open('xb') as file:
np.savez_compressed(file,seconds=seconds,raw_ids=raw,query_raw_ids=q['raw_ids'],
counts=q['counts'],near_wall=q['near_wall'],coordinates_over_Rg=q['coordinates_over_Rg'])
entries.append(dict(case=case,fold=fold,seed=seed,family=query[case]['component_id'],
path=str(item),sha256=sha(item)))
save(records/'progress.json',dict(stage='PREDICT_NO_TARGETS',cases=len(entries),**cuda_peaks(torch)))
return dict(files=entries)
def run(fold,seed,phase):
c=entry.contract()
auth,_=entry.authorize(c)
root=entry.path(fold,seed,phase)
ticket=read(root/'dispatch.json')
parent=os.getppid()
if (ticket['coordinator']!=process_identity(parent) or (ticket['fold'],ticket['seed'],ticket['phase'])!=(fold,seed,phase)
or ticket['contract_sha256']!=sha(entry.ROOT/'contract.json')
or time.monotonic()>=ticket['start_by_monotonic'] or time.time()>=auth['dispatch_before_unix']
or str(Path(sys.executable).absolute())!=c['environment']['python']):
raise PermissionError('Worker not attached to the reviewed guard')
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGKILL,0,0,0) or os.getppid()!=parent:
raise RuntimeError('Could not bind worker lifetime to the guard')
test_environment(c)
torch.set_num_threads(1)
torch.set_num_interop_threads(1)
device=entry.DEVICES[int(ticket['ordinal'])]
binder=FunctionType(release.bind_device.__code__,dict(release.bind_device.__globals__,
formal=SimpleNamespace(DEVICES={str(seed):device})),release.bind_device.__name__)
binder(root,c,ticket,torch)
torch.backends.cuda.matmul.allow_tf32=False
torch.backends.cudnn.allow_tf32=False
torch.cuda.reset_peak_memory_stats()
records=root/'worker-records'
records.mkdir(exist_ok=False)
result=predict(c,fold,seed,root,records) if phase=='predict' else train(c,fold,seed,phase,root,records)
save(records/'result.json',dict(result,status='COMPLETE',fold=fold,seed=seed,phase=phase,
test_targets_read=False,**cuda_peaks(torch)))
if __name__=='__main__':
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('fold',type=int,choices=(0,1))
parser.add_argument('seed',type=int,choices=(0,1))
parser.add_argument('phase',choices=('first','fit','predict'))
args=parser.parse_args()
run(args.fold,args.seed,args.phase)