"""One guarded Greif job. Only learning tensors are opened by the fit paths.""" import argparse import ctypes import json import os from pathlib import Path import random import signal import sys import time from types import FunctionType, SimpleNamespace import numpy as np import torch from phase2_h2_data import cached, standardization from phase2_h2_model import lr_epoch, optimizer, weighted_loss from phase2_residual_math import decode_seconds, residual_target from s2_custody_common import prediction_inputs import track_b_gino_formal_worker as release import track_b_gino_greif_entry as entry from track_b_gino_greif_adapter import GreifGINO, stencil from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha from track_b_gino_shapenet_worker import cuda_peaks, test_environment def fold_view(c, fold): from track_b_gino_greif_prepare import PILOT, PINS pilot=read(checked(PILOT,PINS[str(PILOT)])) row=pilot['folds'][fold] fixed=c['folds'][str(fold)] if any(row[role]!=cases for role,cases in fixed['parts'].items()) or row['clock_sha256']!=fixed['clock_sha256']: raise ValueError('Learning, validation and test roles or original clock differ') return dict(pilot,folds=[row]) def model_input(pilot, case, stats, role): f=pilot['folds'][0] if role not in ('learning','test') or case not in f[role]: raise PermissionError('Original fold role required before reading inputs') pin=pilot['geometry'][case] with np.load(checked(pin['path'],pin['sha256']),allow_pickle=False) as archive: points,raw,mass=(archive[k].copy() for k in ('points','raw_ids','mass')) if len(raw)!=pilot['parts'][case]['nodes'] or not np.array_equal(raw,cached(pilot,case,'raw_ids')) or not np.array_equal(mass,cached(pilot,case,'mass')): raise ValueError('Original full node identity differs') feat=np.asarray(cached(pilot,case,'features'),np.float64) x=((feat-np.asarray(stats['mean']))/np.asarray(stats['scale'])).astype(np.float32) fvec=cached(pilot,case,'Vf') if x.shape!=(len(raw),19) or not np.isfinite(x).all() or fvec.dtype!=np.float64 or fvec.shape!=(len(raw),): raise ValueError('Original nineteen columns or FP64 clock differ') corners,weights=stencil(points,raw) args=(torch.from_numpy(x),torch.from_numpy(mass.astype(np.float32)),corners,weights) return args,raw,fvec def checkpoint(path, model, opt, epoch, fold, seed, stats): temp=path.with_suffix('.tmp') with temp.open('xb') as stream: torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),epochs=epoch,fold=fold,seed=seed, stats=stats,torch_cpu_rng=torch.get_rng_state(),torch_cuda_rng=torch.cuda.get_rng_state(), numpy_rng=np.random.get_state(),python_rng=random.getstate(), contract_sha256=sha(entry.ROOT/'contract.json')),stream) temp.rename(path) def restore(artifact, fold, seed, epochs): state=torch.load(checked(artifact['path'],artifact['sha256']),map_location='cpu') if (state['fold'],state['seed'],state['epochs'],state['contract_sha256']) != ( fold,seed,epochs,sha(entry.ROOT/'contract.json')): raise ValueError('Checkpoint identity differs') return state def train(c, fold, seed, phase, root, records): p=fold_view(c,fold) f=p['folds'][0] rng_seed=seed random.seed(rng_seed) np.random.seed(rng_seed) torch.manual_seed(rng_seed) torch.cuda.manual_seed_all(rng_seed) model=GreifGINO().cuda() opt=optimizer(model) start, end=1,c['epochs'] if phase=='first': if seed!=0: raise PermissionError('First full epoch is seed 0 only') end=1 normalization=root/'norm' normalization.mkdir(exist_ok=False) stats=standardization(p,normalization) else: cost=read(entry.ROOT/'first-epoch-cost-gate.json') if not cost['release_epoch2']: raise PermissionError('Both measured full epochs must pass the cost gate') earlier=entry.stage_terminal(fold,0,'first')['files'][0] state=restore(earlier,fold,0,1) stats=state['stats'] if seed==0: start=2 model.load_state_dict(state['model'],strict=True) opt.load_state_dict(state['optimizer']) torch.set_rng_state(state['torch_cpu_rng']) torch.cuda.set_rng_state(state['torch_cuda_rng']) np.random.set_state(state['numpy_rng']) random.setstate(state['python_rng']) del state if stats['parts']!=f['learning'] or stats['nodes']!=f['learning_nodes']: raise ValueError('Only the learning normalization of this fold is valid') ncases,nnodes=len(f['learning']),f['learning_nodes'] duration=None with (root/'learning-access.jsonl').open('x') as access: for epoch in range(start,end+1): started=time.monotonic() model.train() for group in opt.param_groups: group['lr']=lr_epoch(epoch) seen=0 order=np.random.default_rng(np.random.SeedSequence([seed,epoch])).permutation(ncases) for index in order: case=f['learning'][int(index)] args,raw,fvec=model_input(p,case,stats,'learning') access.write(json.dumps(dict(case=case,fold=fold,seed=seed,epoch=epoch,role='learning', sha256=p['parts'][case]['output_sha256']['target.npy']))+'\n') access.flush() target=residual_target(cached(p,case,'target'),fvec,f['c'],fit_dtype='float32') opt.zero_grad(set_to_none=True) prediction=model(*(a.cuda() for a in args)) loss=weighted_loss(prediction,torch.from_numpy(target).cuda(),ncases,len(raw),nnodes) if not bool(torch.isfinite(loss)): raise ValueError('Nonfinite full-node H2-weighted loss') loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(),1.,error_if_nonfinite=True) opt.step() if any(not bool(torch.isfinite(value).all()) for value in model.parameters()): raise ValueError('Nonfinite model weight after update') torch.cuda.synchronize() seen+=len(raw) del prediction,loss,args,raw,fvec,target if seen!=nnodes: raise ValueError('Incomplete full-node learning epoch') duration=time.monotonic()-started save(records/'progress.json',dict(stage='EPOCH_COMPLETE',epoch=epoch, learning_cases=ncases,learning_nodes=seen,seconds=duration,**cuda_peaks(torch))) output=root/f'epoch-{end}.pt' checkpoint(output,model,opt,end,fold,seed,stats) return dict(epochs=end,learning_cases=ncases,learning_nodes=nnodes, first_epoch_seconds=duration if phase=='first' else None,files=[ dict(path=str(output),sha256=sha(output),fold=fold,seed=seed)]) def predict(c, fold, seed, root, records): from track_b_gino_greif_prepare import QUERY, PINS p=fold_view(c,fold) f=p['folds'][0] models=read(entry.ROOT/'models-frozen.json') by={(r['fold'],r['seed']):r for r in models} if len(models)!=4 or set(by)!=set(entry.JOBS): raise PermissionError('Predict only after four frozen endpoints') state=restore(by[fold,seed],fold,seed,c['epochs']) stats=state['stats'] model=GreifGINO().cuda() model.load_state_dict(state['model'],strict=True) model.eval() del state if stats['parts']!=f['learning'] or stats['nodes']!=f['learning_nodes']: raise ValueError('Original fold normalization differs') query={r['case']:r for r in read(checked(QUERY,PINS[str(QUERY)]))['rows'] if r['included']} entries=[] with torch.no_grad(): for case in f['test']: args,raw,fvec=model_input(p,case,stats,'test') q=prediction_inputs(query[case]) ix=np.searchsorted(raw,q['raw_ids']) if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']): raise ValueError('Frozen PPS query-to-node map differs') residual=model(*(a.cuda() for a in args)).cpu().numpy() seconds=decode_seconds(residual,fvec,f['c'],p['parts'][case]['t_set']) item=root/(case+'.npz') with item.open('xb') as file: np.savez_compressed(file,seconds=seconds,raw_ids=raw,query_raw_ids=q['raw_ids'], counts=q['counts'],near_wall=q['near_wall'],coordinates_over_Rg=q['coordinates_over_Rg']) entries.append(dict(case=case,fold=fold,seed=seed,family=query[case]['component_id'], path=str(item),sha256=sha(item))) save(records/'progress.json',dict(stage='PREDICT_NO_TARGETS',cases=len(entries),**cuda_peaks(torch))) return dict(files=entries) def run(fold,seed,phase): c=entry.contract() auth,_=entry.authorize(c) root=entry.path(fold,seed,phase) ticket=read(root/'dispatch.json') parent=os.getppid() if (ticket['coordinator']!=process_identity(parent) or (ticket['fold'],ticket['seed'],ticket['phase'])!=(fold,seed,phase) or ticket['contract_sha256']!=sha(entry.ROOT/'contract.json') or time.monotonic()>=ticket['start_by_monotonic'] or time.time()>=auth['dispatch_before_unix'] or str(Path(sys.executable).absolute())!=c['environment']['python']): raise PermissionError('Worker not attached to the reviewed guard') if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGKILL,0,0,0) or os.getppid()!=parent: raise RuntimeError('Could not bind worker lifetime to the guard') test_environment(c) torch.set_num_threads(1) torch.set_num_interop_threads(1) device=entry.DEVICES[int(ticket['ordinal'])] binder=FunctionType(release.bind_device.__code__,dict(release.bind_device.__globals__, formal=SimpleNamespace(DEVICES={str(seed):device})),release.bind_device.__name__) binder(root,c,ticket,torch) torch.backends.cuda.matmul.allow_tf32=False torch.backends.cudnn.allow_tf32=False torch.cuda.reset_peak_memory_stats() records=root/'worker-records' records.mkdir(exist_ok=False) result=predict(c,fold,seed,root,records) if phase=='predict' else train(c,fold,seed,phase,root,records) save(records/'result.json',dict(result,status='COMPLETE',fold=fold,seed=seed,phase=phase, test_targets_read=False,**cuda_peaks(torch))) if __name__=='__main__': parser=argparse.ArgumentParser(description=__doc__) parser.add_argument('fold',type=int,choices=(0,1)) parser.add_argument('seed',type=int,choices=(0,1)) parser.add_argument('phase',choices=('first','fit','predict')) args=parser.parse_args() run(args.fold,args.seed,args.phase)