126 lines
14 KiB
Python
126 lines
14 KiB
Python
"""Single S2 postfit: four frozen-checkpoint predictions then exact-reviewed pooled score."""
|
|
import argparse,fcntl,os,resource,sys,time
|
|
from pathlib import Path
|
|
import numpy as np
|
|
import psutil
|
|
from phase2_h1_worker_v2 import save,sha,checked
|
|
from phase2_home_freeze_io import freeze,verify
|
|
from phase2_gate_outcome_review import validate_review
|
|
from phase2_h2_s2_postfit_common import read,config,terminal_inputs,adopted_inputs,checkpoint_identity,comparisons,decision,require_budget,roster,reuse_fold0_metric
|
|
from phase2_h2_s2_common import fold_view
|
|
from phase2_h2_s1_postfit import query_index
|
|
from phase2_residual_math import decode_seconds
|
|
from s2_custody_common import prediction_inputs
|
|
from phase2_b1213_features import cached
|
|
from s2_custody_statistics import part_metrics
|
|
from r4bc_guard import supervise
|
|
|
|
def predict(root,fold):
|
|
import torch
|
|
from phase2_h2_model import H2Geo
|
|
from phase2_h2_data import to_device,arguments
|
|
from phase2_h2_s1_test_data import load_test_part
|
|
if fold not in (1,2,3,4):raise PermissionError('Fold0 predictions are hash reuse only')
|
|
c,pilot,tc=config(root);validate_review(root);adopted=adopted_inputs(root,c,pilot,tc);m=next(m for m in adopted['models'] if m['fold']==fold);view=fold_view(pilot,fold);f=view['folds'][0];job=tc['jobs'][fold-1];stats=read(m['standardization']);by=roster(c,pilot)
|
|
dest=root/'folds'/str(fold)/'predictions';dest.mkdir(exist_ok=False)
|
|
if torch.__version__!=pilot['versions']['torch'] or np.__version__!=pilot['versions']['numpy']:raise ValueError('Original scientific library versions')
|
|
if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity')
|
|
os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1);torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
|
|
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();start=time.monotonic();cpu=time.process_time();entries=[]
|
|
def peaks():return dict(peak_allocated_bytes=int(torch.cuda.max_memory_allocated()),peak_reserved_bytes=int(torch.cuda.max_memory_reserved()))
|
|
try:
|
|
blob=torch.load(checked(m['checkpoint'],m['checkpoint_sha256']),map_location='cpu',weights_only=False);checkpoint_identity(blob,c,fold,len(f['learning']));model=H2Geo().cuda();model.load_state_dict(blob['model'],strict=True);model.eval();del blob
|
|
save(dest/'environment.json',dict(fold=fold,torch=torch.__version__,gpu_uuid=job['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,checkpoint_sha256=m['checkpoint_sha256']))
|
|
with torch.no_grad():
|
|
for case in f['test']:
|
|
row=by[case];data=to_device(load_test_part(view,case,stats),'cuda');residual=model(*arguments(data),checkpoint_blocks=False)
|
|
if not bool(torch.isfinite(residual).all()):raise ValueError('Nonfinite H2 residual')
|
|
seconds=decode_seconds(residual.cpu().numpy(),data['F'],f['c'],data['t_set']);torch.cuda.synchronize();q=prediction_inputs(row);a=c['baseline_predictions'][case]
|
|
with np.load(checked(a['path'],a['sha256']),allow_pickle=False) as old:
|
|
ix=query_index(data['raw_ids'],q,old);A=old['A'].copy();G=old['pred'].copy()
|
|
if not np.isfinite(A).all() or not np.isfinite(G).all() or len(seconds)!=row['nodes']:raise ValueError('Full-node/baseline coverage')
|
|
path=dest/(case+'.npz');np.savez_compressed(path,seconds=seconds,raw_ids=data['raw_ids'],pred=seconds[ix],A=A,B13_E1b=G,query_raw_ids=q['raw_ids'],counts=q['counts'],coordinates_over_Rg=q['coordinates_over_Rg'],near_wall=q['near_wall'])
|
|
entries.append(dict(case=case,component_id=row['component_id'],fold=fold,path=str(path),sha256=sha(path),nodes=row['nodes'],query_nodes=len(ix),A_source_sha256=a['sha256'],negative_full_predictions=int(np.sum(seconds<0)),negative_query_predictions=int(np.sum(seconds[ix]<0)),origin='S2_FROZEN_CHECKPOINT'))
|
|
save(dest/'progress.json',dict(stage='TEST_PREDICTION_NO_TARGETS',fold=fold,parts=len(entries),wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()));del data,residual,seconds
|
|
save(dest/'manifest.json',dict(entries=entries,test_targets_read=False,new_fits=0,full_nodes=sum(e['nodes'] for e in entries)))
|
|
save(dest/'result.json',dict(status='COMPLETE',fold=fold,parts=len(entries),test_targets_read=False,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()))
|
|
except BaseException as exc:
|
|
save(dest/'failure.json',dict(status='FAILED',fold=fold,reason=type(exc).__name__+': '+str(exc),completed_parts=len(entries),wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()));raise
|
|
|
|
def manifest_identity(entries,c,pilot):
|
|
by=roster(c,pilot);old={e['case']:e for e in read(c['fold0_manifest'])['entries']}
|
|
if len(entries)!=552 or len({e['case'] for e in entries})!=552 or {e['case'] for e in entries}!=set(by):raise ValueError('All552 distinct predictions required')
|
|
for e in entries:
|
|
row=by[e['case']];fold=next(a['fold'] for a in row['assignments'] if a['role']=='test')
|
|
if e['fold']!=fold or e['component_id']!=row['component_id'] or e['nodes']!=row['nodes']:raise ValueError('Prediction roster identity')
|
|
if fold==0:
|
|
if e.get('origin')!='S1_HASH_REUSE' or {k:v for k,v in e.items() if k!='origin'}!=old[e['case']]:raise ValueError('Fold0 entry must be exact original hash reuse')
|
|
elif e.get('origin')!='S2_FROZEN_CHECKPOINT' or e['A_source_sha256']!=c['baseline_predictions'][e['case']]['sha256'] or Path(e['path']).resolve()!=(Path(c['postfit_root'])/'folds'/str(fold)/'predictions'/(e['case']+'.npz')).resolve():raise ValueError('New prediction source identity')
|
|
for key,limit in [('negative_full_predictions',e['nodes']),('negative_query_predictions',e['query_nodes'])]:
|
|
if isinstance(e[key],bool) or not isinstance(e[key],int) or not 0<=e[key]<=limit:raise ValueError('Invalid unclipped negative count')
|
|
return by
|
|
|
|
def score(root):
|
|
c,pilot,tc=config(root);validate_review(root);adopted_inputs(root,c,pilot,tc);verify(root,'predictions')
|
|
entries=read(root/'predictions-manifest.json')['entries'];by=manifest_identity(entries,c,pilot);dest=root/'score';dest.mkdir(exist_ok=False)
|
|
save(dest/'review-consumed.json',dict(unix=time.time(),review_sha256=sha(root/'review.json'),before_old_score_or_target_values=True))
|
|
oldA={r['case']:r for r in read(checked(c['baseline_score'],c['source_sha256'][c['baseline_score']]))['rows']};oldH={r['case']:r for r in read(checked(c['fold0_score'],c['source_sha256'][c['fold0_score']]))['rows']};rows=[];access=[]
|
|
if len(oldH)!=121 or set(oldH)!=set(pilot['folds'][0]['test']) or len(oldA)!=552 or set(oldA)!=set(by):raise ValueError('Frozen reference score coverage')
|
|
for e in entries:
|
|
case=e['case'];row=by[case];old=oldA[case];checked(e['path'],e['sha256'])
|
|
if old['fold']!=e['fold'] or old['component_id']!=e['component_id']:raise ValueError('Baseline score pairing')
|
|
if e['fold']==0:
|
|
metric=reuse_fold0_metric(oldH[case],old,e) # No fold0 target read, prediction, or per-part metric recomputation.
|
|
else:
|
|
access.append(dict(case=case,fold=e['fold'],role='test',target_sha256=row['v5_cache']['output_sha256']['seconds.npy']));save(dest/'target-access.json',access)
|
|
truth=cached(row,'seconds');raw=cached(row,'raw_ids');base=c['baseline_predictions'][case]
|
|
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z,np.load(checked(base['path'],base['sha256']),allow_pickle=False) as original:
|
|
q={k:z['query_raw_ids' if k=='raw_ids' else k] for k in ['raw_ids','counts','coordinates_over_Rg','near_wall']};ix=query_index(raw,q,original)
|
|
if not np.array_equal(z['raw_ids'],raw) or not np.array_equal(z['pred'],z['seconds'][ix]) or not np.array_equal(z['A'],original['A']) or not np.array_equal(z['B13_E1b'],original['pred']):raise ValueError('Full/query/reference prediction identity')
|
|
if len(z['seconds'])!=e['nodes'] or len(ix)!=e['query_nodes'] or int(np.sum(z['seconds']<0))!=e['negative_full_predictions'] or int(np.sum(z['pred']<0))!=e['negative_query_predictions']:raise ValueError('Negative counts or coverage differ')
|
|
metric=part_metrics(z['pred'],truth[ix],q['counts'],q['near_wall'],q['raw_ids'],q['coordinates_over_Rg'],int(case.split('_')[0]))
|
|
rows.append(dict(case=case,component_id=e['component_id'],fold=e['fold'],H2=metric,A=old['A'],B13_E1b=old['new'],baseline_G_valid=old['G_valid'],H2_origin=e['origin']));save(dest/'progress.json',dict(parts=len(rows),new_test_target_reads=len(access)))
|
|
if len(access)!=431 or len({x['case'] for x in access})!=431:raise ValueError('Exactly431 new held-out targets; fold0 untouched')
|
|
metrics=comparisons(rows);result=decision(metrics,len(rows),len({r['component_id'] for r in rows}))
|
|
save(dest/'result.json',dict(status='COMPLETE',decision=result,parts=552,components=204,comparisons=metrics,per_fold={str(f):comparisons([r for r in rows if r['fold']==f]) for f in range(5)},rows=rows,confirmation_rule='pooled H2-A field CI upper<0, complete552/204; B13/last/tau report only',bootstrap=dict(replicates=2000,seed=2026111408),negative_predictions=dict(full_nodes=sum(e['negative_full_predictions'] for e in entries),eligible_unique_query_nodes=sum(e['negative_query_predictions'] for e in entries),clipped=False),full_nodes=sum(e['nodes'] for e in entries),eligible_unique_query_nodes=sum(e['query_nodes'] for e in entries),baseline_G_invalid_parts=sum(not r['baseline_G_valid'] for r in rows),fold0_score_reused=121,new_test_target_reads=431,new_decodes=0,new_fits=0,review_sha256=sha(root/'review.json'),data_sufficiency_started=False,interpretation='Five-fold recipe confirmation after observing fold0; no pure connectivity causality or untouched-five-fold preregistration claim'))
|
|
|
|
def run(root):
|
|
start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);c,pilot,tc=config(root);validate_review(root);adopted=terminal_inputs(c,pilot,tc);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if (root/'started.json').exists():raise FileExistsError('Single pooled postfit attempt')
|
|
for p in psutil.process_iter(['pid','cmdline']):
|
|
if p.pid!=os.getpid() and any(Path(a).name=='phase2_h2_s2_postfit.py' for a in p.info['cmdline'] or []):raise RuntimeError('Existing S2 postfit process')
|
|
os.sched_setaffinity(0,[12,13]);os.environ['PYTORCH_CUDA_ALLOC_CONF']='expandable_segments:True';gpu=0.;charged=0.;records=[];prediction_start=None;prior=adopted['prior_gpu_seconds']
|
|
def cpu():
|
|
u=resource.getrusage(resource.RUSAGE_CHILDREN);return 360+time.process_time()-start+max(charged,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)
|
|
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),prior_gpu_once=prior,old_s1_gpu_once=adopted['old_s1_gpu_once'],new_training_gpu_once=adopted['new_training_gpu_once'],review_sha256=sha(root/'review.json'),new_fits=0,fold0_reexecution=False))
|
|
try:
|
|
require_budget(prior,gpu,cpu(),4);save(root/'models-adopted.json',adopted);train=Path(c['training_root'])
|
|
freeze(root,'models',[root/'contract.json',root/'models-adopted.json',train/'complete.json',train/'models-terminal.json',train/'models-git-freeze.json',train/'terminal-git-freeze.json',*[Path(m['checkpoint']) for m in adopted['models']],*[Path(m['standardization']) for m in adopted['models']]])
|
|
entries=[dict(e,origin='S1_HASH_REUSE') for e in read(c['fold0_manifest'])['entries']]
|
|
for e in entries:checked(e['path'],e['sha256'])
|
|
from phase2_h2_s2_prediction_guard import run_prediction
|
|
for fold in range(1,5):
|
|
require_budget(prior,gpu,cpu(),5-fold);(root/'folds'/str(fold)).mkdir(parents=True,exist_ok=False);prediction_start=time.monotonic();receipt=run_prediction(root,c,pilot,fold,prior+gpu,600)
|
|
observed=60+time.monotonic()-prediction_start;prediction_start=None;cost=max(receipt['gpu_seconds_charged'],observed);gpu+=cost;charged+=receipt['worker']['cpu_seconds'] if receipt['worker'] else 0.;records.append(dict(mode='predict',fold=fold,charged_gpu_seconds=cost,guard=receipt))
|
|
save(root/'resource-progress.json',dict(records=records,prediction_gpu_seconds=gpu,postfit_cpu_seconds=cpu(),cumulative_h2_gpu_seconds=prior+gpu))
|
|
if receipt['status']!='PASS' or cost>600 or gpu>2400 or prior+gpu>144000:raise RuntimeError('Prediction not PASS or cumulative GPU limit')
|
|
m=read(root/'folds'/str(fold)/'predictions/manifest.json')
|
|
if len(m['entries'])!=len(pilot['folds'][fold]['test']) or m['full_nodes']!=pilot['folds'][fold]['test_nodes']:raise ValueError('Full-fold prediction coverage')
|
|
for e in m['entries']:checked(e['path'],e['sha256'])
|
|
entries.extend(m['entries'])
|
|
manifest_identity(entries,c,pilot);save(root/'predictions-manifest.json',dict(entries=entries,new_fits=0,fold0_reused=121,new_predictions=431,test_targets_read=False));freeze(root,'predictions',[root/'contract.json',root/'predictions-manifest.json',root/'models-adopted.json',root/'resource-progress.json',*[Path(p) for p in c['code_sha256']]])
|
|
require_budget(prior,gpu,cpu(),0)
|
|
g=supervise([sys.executable,'tools/phase2_h2_s2_postfit.py','score',str(root)],root/'score-guard',dict(cpu_seconds=600,wall_seconds=780,rss_bytes=16*2**30,disk_bytes=2*2**30));charged+=g['cpu_seconds'];records.append(dict(mode='score',guard=g))
|
|
if g['status']!='PASS' or cpu()>3600:raise RuntimeError('Score not PASS or CPU limit')
|
|
save(root/'complete.json',dict(status='COMPLETE',records=records,prediction_gpu_seconds=gpu,postfit_cpu_seconds=cpu(),prior_gpu_once=prior,cumulative_h2_gpu_seconds=prior+gpu,decision=read(root/'score/result.json')['decision'],new_fits=0,fold0_reexecution=False,data_sufficiency_started=False,no_retry=True));freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'score/result.json',root/'review.json'])
|
|
save(root/'post-terminal-cost.json',dict(postfit_cpu_seconds=cpu(),prediction_gpu_seconds=gpu,cumulative_h2_gpu_seconds=prior+gpu,within_cap=cpu()<=3600 and gpu<=2400 and prior+gpu<=144000))
|
|
except BaseException as exc:
|
|
if prediction_start is not None:gpu+=60+time.monotonic()-prediction_start
|
|
save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),records=records,postfit_cpu_seconds=cpu(),prediction_gpu_seconds=gpu,prior_gpu_once=prior,cumulative_h2_gpu_seconds=prior+gpu,new_fits=0,no_retry=True));raise
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('action',choices=['run','predict','score']);p.add_argument('root',type=Path);p.add_argument('fold',type=int,nargs='?');a=p.parse_args()
|
|
if a.action=='predict':predict(a.root,a.fold)
|
|
elif a.fold is not None:p.error('Fold allowed only for predict')
|
|
elif a.action=='score':score(a.root)
|
|
else:run(a.root)
|