124 lines
13 KiB
Python
124 lines
13 KiB
Python
"""Frozen final-only H2-S1 prediction/score; no fitting or additional GPU stages."""
|
|
import argparse,fcntl,json,os,resource,sys,time
|
|
from pathlib import Path
|
|
import numpy as np
|
|
from phase2_fixed_io import BASE
|
|
from phase2_h1_worker_v2 import checked,sha,save
|
|
from phase2_home_freeze_io import verify,freeze
|
|
from phase2_gate_outcome_review import validate_review
|
|
from phase2_h2_s1_common import gate
|
|
from phase2_residual_math import decode_seconds
|
|
from s2_custody_common import prediction_inputs
|
|
from s2_custody_statistics import part_metrics
|
|
from phase2_b13_e1b import pooled
|
|
from phase2_b1213_features import cached
|
|
from r4bc_guard import supervise
|
|
|
|
def read(p):return json.loads(Path(p).read_text())
|
|
def h2_execution_command(args):
|
|
return any(Path(x).name in {'phase2_h2_s1_guard.py','phase2_h2_s1_worker.py','phase2_h2_s1_postfit.py'} for x in args)
|
|
def config(root):
|
|
c=read(root/'contract.json');verify(root,'implementation');verify(Path(c['checkpoint_root']),'checkpoint')
|
|
if c['approval']!='1794921374.599419' or c['prediction_gpu_seconds']!=600 or c['postfit_cpu_seconds']!=3600 or c['total_h2_gpu_seconds']!=144000 or c['new_fits']!=0:raise PermissionError('Fixed H2-S1 scope')
|
|
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():checked(p,h)
|
|
pilot=read(c['pilot_contract']);train=read(c['training_contract']);done=read(c['training_complete']);result=read(c['training_result'])
|
|
if done['status']!='PASS' or done['reasons'] or result['completed_epochs']!=200 or result['optimizer_steps']!=64000 or result['checkpoint_sha256']!=c['checkpoint_sha256']:raise PermissionError('Original last200 training PASS required')
|
|
if done['cumulative_h2_gpu_seconds']!=c['prior_gpu_seconds'] or c['prior_gpu_seconds']+600>144000:raise ValueError('Original pilot/train cumulative cost')
|
|
stats=read(c['standardization']);fold=pilot['folds'][0]
|
|
if stats['parts']!=fold['learning'] or stats['nodes']!=fold['learning_nodes'] or sha(c['standardization'])!=train['standardization_sha256']:raise ValueError('Training normalization identity')
|
|
if len(c['rows'])!=121 or len({r['case'] for r in c['rows']})!=121 or {r['case'] for r in c['rows']}!=set(fold['test']):raise ValueError('Original fold0 test121')
|
|
if len({r['component_id'] for r in c['rows']})!=37:raise ValueError('Original37 test components')
|
|
for row in c['rows']:
|
|
case=row['case'];pr=pilot['parts'][case];tests=[a['fold'] for a in row['assignments'] if a['role']=='test']
|
|
if tests!=[0] or row['component_id']!=pr['clock_group'] or row['v5_cache']['output_sha256']!=pr['output_sha256'] or row['preparation']['t_set']!=pr['t_set']:raise ValueError('H2/A cached role identity')
|
|
return c,pilot,stats
|
|
|
|
def query_index(raw,q,baseline):
|
|
if raw.ndim!=1 or np.any(np.diff(raw)<=0) or q['raw_ids'].ndim!=1 or not len(q['raw_ids']) or np.any(np.diff(q['raw_ids'])<=0):raise ValueError('Ascending unique raw/query IDs required')
|
|
ix=np.searchsorted(raw,q['raw_ids'])
|
|
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):raise ValueError('Query outside original full nodes')
|
|
for name in ['raw_ids','counts','coordinates_over_Rg','near_wall']:
|
|
if not np.array_equal(q[name],baseline[name]):raise ValueError('Original A query identity: '+name)
|
|
return ix
|
|
|
|
def checkpoint_identity(blob,c):
|
|
if blob['completed_epochs']!=200 or blob['optimizer_steps']!=64000 or blob['seed']!=0 or blob['fold']!=0 or blob['role']!='last_epoch_only' or blob['contract_sha256']!=c['source_sha256'][c['training_contract']]:raise ValueError('Wrong final checkpoint, not pilot/best epoch')
|
|
|
|
def predict(root):
|
|
import torch
|
|
from phase2_h2_model import H2Geo
|
|
from phase2_h2_data import to_device,arguments
|
|
from phase2_h2_s1_test_data import load_test_part
|
|
c,pilot,stats=config(root);validate_review(root);verify(root,'model');dest=root/'predictions';dest.mkdir(exist_ok=False)
|
|
if torch.__version__!=pilot['versions']['torch'] or np.__version__!=pilot['versions']['numpy']:raise ValueError('Scientific library version')
|
|
if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator binding')
|
|
os.sched_setaffinity(0,[0,1]);torch.set_num_threads(1);torch.set_num_interop_threads(1);torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
|
|
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();start=time.monotonic();cpu=time.process_time();entries=[]
|
|
def peaks():return dict(peak_allocated_bytes=int(torch.cuda.max_memory_allocated()),peak_reserved_bytes=int(torch.cuda.max_memory_reserved()))
|
|
try:
|
|
blob=torch.load(checked(c['checkpoint'],c['checkpoint_sha256']),map_location='cpu',weights_only=False);checkpoint_identity(blob,c);model=H2Geo().cuda();model.load_state_dict(blob['model'],strict=True);model.eval();del blob
|
|
save(dest/'environment.json',dict(torch=torch.__version__,gpu_uuid=c['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,checkpoint_sha256=c['checkpoint_sha256']))
|
|
with torch.no_grad():
|
|
for row in c['rows']:
|
|
case=row['case'];data=to_device(load_test_part(pilot,case,stats),'cuda');residual=model(*arguments(data),checkpoint_blocks=False)
|
|
if not bool(torch.isfinite(residual).all()):raise ValueError('Nonfinite H2 residual')
|
|
seconds=decode_seconds(residual.cpu().numpy(),data['F'],pilot['folds'][0]['c'],data['t_set']);torch.cuda.synchronize();q=prediction_inputs(row);a=c['A_predictions'][case]
|
|
with np.load(checked(a['path'],a['sha256']),allow_pickle=False) as old:
|
|
ix=query_index(data['raw_ids'],q,old);A=old['A'].copy()
|
|
if not np.isfinite(A).all() or len(seconds)!=row['nodes']:raise ValueError('Full-node coverage or A prediction')
|
|
path=dest/(case+'.npz');np.savez_compressed(path,seconds=seconds,raw_ids=data['raw_ids'],pred=seconds[ix],A=A,query_raw_ids=q['raw_ids'],counts=q['counts'],coordinates_over_Rg=q['coordinates_over_Rg'],near_wall=q['near_wall'])
|
|
entries.append(dict(case=case,component_id=row['component_id'],fold=0,path=str(path),sha256=sha(path),nodes=row['nodes'],query_nodes=len(ix),A_source_sha256=a['sha256'],negative_full_predictions=int(np.sum(seconds<0)),negative_query_predictions=int(np.sum(seconds[ix]<0))))
|
|
save(dest/'progress.json',dict(stage='TEST_PREDICTION_NO_TARGETS',parts=len(entries),wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()));del data,residual,seconds
|
|
save(dest/'manifest.json',dict(entries=entries,test_targets_read=False,new_fits=0,full_nodes=sum(e['nodes'] for e in entries)))
|
|
save(dest/'result.json',dict(status='COMPLETE',parts=len(entries),test_targets_read=False,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()))
|
|
except BaseException as exc:
|
|
save(dest/'failure.json',dict(status='FAILED',reason=type(exc).__name__+': '+str(exc),completed_parts=len(entries),wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()));raise
|
|
|
|
def score(root):
|
|
c,pilot,stats=config(root);verify(root,'predictions');validate_review(root) # Must precede every target and old-score read.
|
|
dest=root/'score';dest.mkdir(exist_ok=False);manifest=read(root/'predictions-manifest.json')['entries'];by={r['case']:r for r in c['rows']}
|
|
if len(manifest)!=121 or {e['case'] for e in manifest}!=set(by):raise ValueError('All121 frozen predictions required')
|
|
save(dest/'review-consumed.json',dict(unix=time.time(),review_sha256=sha(root/'review.json'),verdict='PASS',before_target_access=True))
|
|
prior={r['case']:r for r in read(checked(c['A_score'],c['source_sha256'][c['A_score']]))['rows']};rows=[];access=[]
|
|
for e in manifest:
|
|
row=by[e['case']];case=row['case'];old=prior[case]
|
|
if e['fold']!=0 or old['fold']!=0 or e['component_id']!=row['component_id'] or old['component_id']!=row['component_id']:raise ValueError('Paired fold/component')
|
|
access.append(dict(case=case,fold=0,role='test',target_sha256=row['v5_cache']['output_sha256']['seconds.npy']));save(dest/'target-access.json',access)
|
|
truth=cached(row,'seconds');raw=cached(row,'raw_ids');base=c['A_predictions'][case]
|
|
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z,np.load(checked(base['path'],base['sha256']),allow_pickle=False) as original:
|
|
q={k:z['query_raw_ids' if k=='raw_ids' else k] for k in ['raw_ids','counts','coordinates_over_Rg','near_wall']};ix=query_index(raw,q,original)
|
|
if not np.array_equal(z['raw_ids'],raw) or not np.array_equal(z['pred'],z['seconds'][ix]) or not np.array_equal(z['A'],original['A']):raise ValueError('Full/query/A frozen array identity')
|
|
m=part_metrics(z['pred'],truth[ix],q['counts'],q['near_wall'],q['raw_ids'],q['coordinates_over_Rg'],int(case.split('_')[0]))
|
|
rows.append(dict(case=case,component_id=row['component_id'],fold=0,new=m,A=old['A']));save(dest/'progress.json',dict(parts=len(rows)))
|
|
metrics=pooled(rows);eligible=gate(metrics['relative_l2']['new_minus_A']['high'],metrics['last_location']['new_minus_A']['high'])
|
|
save(dest/'result.json',dict(status='COMPLETE',parts=121,components=37,metrics=metrics,rows=rows,s2_eligible_by_original_gate=eligible,gate='field H2-A CI upper<+.005 OR last-location/Rg H2-A CI upper<0; strict bounds',s2_started=False,data_sufficiency_started=False,checkpoint_epoch=200,validation_used_for_selection=False,negative_predictions_clipped=False,new_decodes=0,new_fits=0,review_sha256=sha(root/'review.json'),interpretation='Single fixed fold/seed conditional component uncertainty; not pure connectivity causality'))
|
|
|
|
def run(root):
|
|
start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);c,pilot,stats=config(root);validate_review(root);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if (root/'started.json').exists():raise FileExistsError('Single H2-S1 postfit attempt')
|
|
import psutil
|
|
for p in psutil.process_iter(['pid','cmdline']):
|
|
args=p.info['cmdline'] or []
|
|
if p.pid!=os.getpid() and h2_execution_command(args):raise RuntimeError('Existing H2 train/postfit process')
|
|
os.sched_setaffinity(0,[2]);os.environ['PYTORCH_CUDA_ALLOC_CONF']='expandable_segments:True';save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),training_complete_sha256=c['source_sha256'][c['training_complete']],checkpoint_sha256=c['checkpoint_sha256'],new_fits=0));gpu=0.;charged=0.;records=[];prediction_start=None
|
|
def cpu():
|
|
u=resource.getrusage(resource.RUSAGE_CHILDREN);return c['setup_cpu_seconds']+time.process_time()-start+max(charged,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)
|
|
try:
|
|
if cpu()+1200+600+120>3600:raise RuntimeError('Postfit CPU reservation')
|
|
from phase2_h2_s1_prediction_guard import run_prediction
|
|
prediction_start=time.monotonic();receipt=run_prediction(root,c,1200);gpu=receipt['gpu_seconds_charged'];prediction_start=None;charged+=receipt['worker']['cpu_seconds'] if receipt['worker'] else 0.;records.append(dict(mode='predict',receipt=receipt))
|
|
if receipt['status']!='PASS' or gpu>600 or c['prior_gpu_seconds']+gpu>144000:raise RuntimeError('Prediction not PASS')
|
|
m=read(root/'predictions/manifest.json');assert len(m['entries'])==121 and m['full_nodes']==pilot['folds'][0]['test_nodes']
|
|
for e in m['entries']:checked(e['path'],e['sha256'])
|
|
save(root/'predictions-manifest.json',m);freeze(root,'predictions',[root/'contract.json',root/'predictions-manifest.json',root/'prediction-guard.json',*[Path(p) for p in c['code_sha256']]])
|
|
if cpu()+600+120>3600:raise RuntimeError('Score CPU reservation')
|
|
g=supervise([sys.executable,'tools/phase2_h2_s1_postfit.py','score',str(root)],root/'score-guard',dict(cpu_seconds=600,wall_seconds=780,rss_bytes=16*2**30,disk_bytes=2*2**30));charged+=g['cpu_seconds'];records.append(dict(mode='score',guard=g))
|
|
if g['status']!='PASS' or cpu()>3600:raise RuntimeError('Score not PASS')
|
|
save(root/'complete.json',dict(status='COMPLETE',records=records,postfit_cpu_seconds=cpu(),prediction_gpu_seconds=gpu,cumulative_h2_gpu_seconds=c['prior_gpu_seconds']+gpu,prior_pilot_and_train_gpu_once=c['prior_gpu_seconds'],s2_eligible_by_original_gate=read(root/'score/result.json')['s2_eligible_by_original_gate'],new_fits=0,s2_started=False,data_sufficiency_started=False,retry=False))
|
|
freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'score/result.json',root/'review.json']);save(root/'post-terminal-cost.json',dict(postfit_cpu_seconds=cpu(),prediction_gpu_seconds=gpu,cumulative_h2_gpu_seconds=c['prior_gpu_seconds']+gpu,within_cap=cpu()<=3600 and gpu<=600 and c['prior_gpu_seconds']+gpu<=144000))
|
|
except BaseException as exc:
|
|
if prediction_start is not None:gpu=max(gpu,60+time.monotonic()-prediction_start)
|
|
save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),records=records,postfit_cpu_seconds=cpu(),prediction_gpu_seconds=gpu,cumulative_h2_gpu_seconds=c['prior_gpu_seconds']+gpu,new_fits=0,retry=False));raise
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('action',choices=['run','predict','score']);p.add_argument('root',type=Path);a=p.parse_args();{'run':run,'predict':predict,'score':score}[a.action](a.root)
|