229 lines
11 KiB
Python
229 lines
11 KiB
Python
"""One guarded Greif job. Only learning tensors are opened by the fit paths."""
|
|
|
|
import argparse
|
|
import ctypes
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import random
|
|
import signal
|
|
import sys
|
|
import time
|
|
from types import FunctionType, SimpleNamespace
|
|
|
|
import numpy as np
|
|
import torch
|
|
|
|
from phase2_h2_data import cached, standardization
|
|
from phase2_h2_model import lr_epoch, optimizer, weighted_loss
|
|
from phase2_residual_math import decode_seconds, residual_target
|
|
from s2_custody_common import prediction_inputs
|
|
import track_b_gino_formal_worker as release
|
|
import track_b_gino_greif_entry as entry
|
|
from track_b_gino_greif_adapter import GreifGINO, stencil
|
|
from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha
|
|
from track_b_gino_shapenet_worker import cuda_peaks, test_environment
|
|
|
|
|
|
def fold_view(c, fold):
|
|
from track_b_gino_greif_prepare import PILOT, PINS
|
|
|
|
pilot=read(checked(PILOT,PINS[str(PILOT)]))
|
|
row=pilot['folds'][fold]
|
|
fixed=c['folds'][str(fold)]
|
|
if any(row[role]!=cases for role,cases in fixed['parts'].items()) or row['clock_sha256']!=fixed['clock_sha256']:
|
|
raise ValueError('Learning, validation and test roles or original clock differ')
|
|
return dict(pilot,folds=[row])
|
|
|
|
|
|
def model_input(pilot, case, stats, role):
|
|
f=pilot['folds'][0]
|
|
if role not in ('learning','test') or case not in f[role]:
|
|
raise PermissionError('Original fold role required before reading inputs')
|
|
pin=pilot['geometry'][case]
|
|
with np.load(checked(pin['path'],pin['sha256']),allow_pickle=False) as archive:
|
|
points,raw,mass=(archive[k].copy() for k in ('points','raw_ids','mass'))
|
|
if len(raw)!=pilot['parts'][case]['nodes'] or not np.array_equal(raw,cached(pilot,case,'raw_ids')) or not np.array_equal(mass,cached(pilot,case,'mass')):
|
|
raise ValueError('Original full node identity differs')
|
|
feat=np.asarray(cached(pilot,case,'features'),np.float64)
|
|
x=((feat-np.asarray(stats['mean']))/np.asarray(stats['scale'])).astype(np.float32)
|
|
fvec=cached(pilot,case,'Vf')
|
|
if x.shape!=(len(raw),19) or not np.isfinite(x).all() or fvec.dtype!=np.float64 or fvec.shape!=(len(raw),):
|
|
raise ValueError('Original nineteen columns or FP64 clock differ')
|
|
corners,weights=stencil(points,raw)
|
|
args=(torch.from_numpy(x),torch.from_numpy(mass.astype(np.float32)),corners,weights)
|
|
return args,raw,fvec
|
|
|
|
|
|
def checkpoint(path, model, opt, epoch, fold, seed, stats):
|
|
temp=path.with_suffix('.tmp')
|
|
with temp.open('xb') as stream:
|
|
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),epochs=epoch,fold=fold,seed=seed,
|
|
stats=stats,torch_cpu_rng=torch.get_rng_state(),torch_cuda_rng=torch.cuda.get_rng_state(),
|
|
numpy_rng=np.random.get_state(),python_rng=random.getstate(),
|
|
contract_sha256=sha(entry.ROOT/'contract.json')),stream)
|
|
temp.rename(path)
|
|
|
|
|
|
def restore(artifact, fold, seed, epochs):
|
|
state=torch.load(checked(artifact['path'],artifact['sha256']),map_location='cpu')
|
|
if (state['fold'],state['seed'],state['epochs'],state['contract_sha256']) != (
|
|
fold,seed,epochs,sha(entry.ROOT/'contract.json')):
|
|
raise ValueError('Checkpoint identity differs')
|
|
return state
|
|
|
|
|
|
def train(c, fold, seed, phase, root, records):
|
|
p=fold_view(c,fold)
|
|
f=p['folds'][0]
|
|
rng_seed=seed
|
|
random.seed(rng_seed)
|
|
np.random.seed(rng_seed)
|
|
torch.manual_seed(rng_seed)
|
|
torch.cuda.manual_seed_all(rng_seed)
|
|
model=GreifGINO().cuda()
|
|
opt=optimizer(model)
|
|
start, end=1,c['epochs']
|
|
if phase=='first':
|
|
if seed!=0:
|
|
raise PermissionError('First full epoch is seed 0 only')
|
|
end=1
|
|
normalization=root/'norm'
|
|
normalization.mkdir(exist_ok=False)
|
|
stats=standardization(p,normalization)
|
|
else:
|
|
cost=read(entry.ROOT/'first-epoch-cost-gate.json')
|
|
if not cost['release_epoch2']:
|
|
raise PermissionError('Both measured full epochs must pass the cost gate')
|
|
earlier=entry.stage_terminal(fold,0,'first')['files'][0]
|
|
state=restore(earlier,fold,0,1)
|
|
stats=state['stats']
|
|
if seed==0:
|
|
start=2
|
|
model.load_state_dict(state['model'],strict=True)
|
|
opt.load_state_dict(state['optimizer'])
|
|
torch.set_rng_state(state['torch_cpu_rng'])
|
|
torch.cuda.set_rng_state(state['torch_cuda_rng'])
|
|
np.random.set_state(state['numpy_rng'])
|
|
random.setstate(state['python_rng'])
|
|
del state
|
|
if stats['parts']!=f['learning'] or stats['nodes']!=f['learning_nodes']:
|
|
raise ValueError('Only the learning normalization of this fold is valid')
|
|
ncases,nnodes=len(f['learning']),f['learning_nodes']
|
|
duration=None
|
|
with (root/'learning-access.jsonl').open('x') as access:
|
|
for epoch in range(start,end+1):
|
|
started=time.monotonic()
|
|
model.train()
|
|
for group in opt.param_groups:
|
|
group['lr']=lr_epoch(epoch)
|
|
seen=0
|
|
order=np.random.default_rng(np.random.SeedSequence([seed,epoch])).permutation(ncases)
|
|
for index in order:
|
|
case=f['learning'][int(index)]
|
|
args,raw,fvec=model_input(p,case,stats,'learning')
|
|
access.write(json.dumps(dict(case=case,fold=fold,seed=seed,epoch=epoch,role='learning',
|
|
sha256=p['parts'][case]['output_sha256']['target.npy']))+'\n')
|
|
access.flush()
|
|
target=residual_target(cached(p,case,'target'),fvec,f['c'],fit_dtype='float32')
|
|
opt.zero_grad(set_to_none=True)
|
|
prediction=model(*(a.cuda() for a in args))
|
|
loss=weighted_loss(prediction,torch.from_numpy(target).cuda(),ncases,len(raw),nnodes)
|
|
if not bool(torch.isfinite(loss)):
|
|
raise ValueError('Nonfinite full-node H2-weighted loss')
|
|
loss.backward()
|
|
torch.nn.utils.clip_grad_norm_(model.parameters(),1.,error_if_nonfinite=True)
|
|
opt.step()
|
|
if any(not bool(torch.isfinite(value).all()) for value in model.parameters()):
|
|
raise ValueError('Nonfinite model weight after update')
|
|
torch.cuda.synchronize()
|
|
seen+=len(raw)
|
|
del prediction,loss,args,raw,fvec,target
|
|
if seen!=nnodes:
|
|
raise ValueError('Incomplete full-node learning epoch')
|
|
duration=time.monotonic()-started
|
|
save(records/'progress.json',dict(stage='EPOCH_COMPLETE',epoch=epoch,
|
|
learning_cases=ncases,learning_nodes=seen,seconds=duration,**cuda_peaks(torch)))
|
|
output=root/f'epoch-{end}.pt'
|
|
checkpoint(output,model,opt,end,fold,seed,stats)
|
|
return dict(epochs=end,learning_cases=ncases,learning_nodes=nnodes,
|
|
first_epoch_seconds=duration if phase=='first' else None,files=[
|
|
dict(path=str(output),sha256=sha(output),fold=fold,seed=seed)])
|
|
|
|
|
|
def predict(c, fold, seed, root, records):
|
|
from track_b_gino_greif_prepare import QUERY, PINS
|
|
|
|
p=fold_view(c,fold)
|
|
f=p['folds'][0]
|
|
models=read(entry.ROOT/'models-frozen.json')
|
|
by={(r['fold'],r['seed']):r for r in models}
|
|
if len(models)!=4 or set(by)!=set(entry.JOBS):
|
|
raise PermissionError('Predict only after four frozen endpoints')
|
|
state=restore(by[fold,seed],fold,seed,c['epochs'])
|
|
stats=state['stats']
|
|
model=GreifGINO().cuda()
|
|
model.load_state_dict(state['model'],strict=True)
|
|
model.eval()
|
|
del state
|
|
if stats['parts']!=f['learning'] or stats['nodes']!=f['learning_nodes']:
|
|
raise ValueError('Original fold normalization differs')
|
|
query={r['case']:r for r in read(checked(QUERY,PINS[str(QUERY)]))['rows'] if r['included']}
|
|
entries=[]
|
|
with torch.no_grad():
|
|
for case in f['test']:
|
|
args,raw,fvec=model_input(p,case,stats,'test')
|
|
q=prediction_inputs(query[case])
|
|
ix=np.searchsorted(raw,q['raw_ids'])
|
|
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):
|
|
raise ValueError('Frozen PPS query-to-node map differs')
|
|
residual=model(*(a.cuda() for a in args)).cpu().numpy()
|
|
seconds=decode_seconds(residual,fvec,f['c'],p['parts'][case]['t_set'])
|
|
item=root/(case+'.npz')
|
|
with item.open('xb') as file:
|
|
np.savez_compressed(file,seconds=seconds,raw_ids=raw,query_raw_ids=q['raw_ids'],
|
|
counts=q['counts'],near_wall=q['near_wall'],coordinates_over_Rg=q['coordinates_over_Rg'])
|
|
entries.append(dict(case=case,fold=fold,seed=seed,family=query[case]['component_id'],
|
|
path=str(item),sha256=sha(item)))
|
|
save(records/'progress.json',dict(stage='PREDICT_NO_TARGETS',cases=len(entries),**cuda_peaks(torch)))
|
|
return dict(files=entries)
|
|
|
|
|
|
def run(fold,seed,phase):
|
|
c=entry.contract()
|
|
auth,_=entry.authorize(c)
|
|
root=entry.path(fold,seed,phase)
|
|
ticket=read(root/'dispatch.json')
|
|
parent=os.getppid()
|
|
if (ticket['coordinator']!=process_identity(parent) or (ticket['fold'],ticket['seed'],ticket['phase'])!=(fold,seed,phase)
|
|
or ticket['contract_sha256']!=sha(entry.ROOT/'contract.json')
|
|
or time.monotonic()>=ticket['start_by_monotonic'] or time.time()>=auth['dispatch_before_unix']
|
|
or str(Path(sys.executable).absolute())!=c['environment']['python']):
|
|
raise PermissionError('Worker not attached to the reviewed guard')
|
|
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGKILL,0,0,0) or os.getppid()!=parent:
|
|
raise RuntimeError('Could not bind worker lifetime to the guard')
|
|
test_environment(c)
|
|
torch.set_num_threads(1)
|
|
torch.set_num_interop_threads(1)
|
|
device=entry.DEVICES[int(ticket['ordinal'])]
|
|
binder=FunctionType(release.bind_device.__code__,dict(release.bind_device.__globals__,
|
|
formal=SimpleNamespace(DEVICES={str(seed):device})),release.bind_device.__name__)
|
|
binder(root,c,ticket,torch)
|
|
torch.backends.cuda.matmul.allow_tf32=False
|
|
torch.backends.cudnn.allow_tf32=False
|
|
torch.cuda.reset_peak_memory_stats()
|
|
records=root/'worker-records'
|
|
records.mkdir(exist_ok=False)
|
|
result=predict(c,fold,seed,root,records) if phase=='predict' else train(c,fold,seed,phase,root,records)
|
|
save(records/'result.json',dict(result,status='COMPLETE',fold=fold,seed=seed,phase=phase,
|
|
test_targets_read=False,**cuda_peaks(torch)))
|
|
|
|
|
|
if __name__=='__main__':
|
|
parser=argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('fold',type=int,choices=(0,1))
|
|
parser.add_argument('seed',type=int,choices=(0,1))
|
|
parser.add_argument('phase',choices=('first','fit','predict'))
|
|
args=parser.parse_args()
|
|
run(args.fold,args.seed,args.phase)
|