Fix bounded S3-A postfit chain before test prediction access
This commit is contained in:
1 parent
4d76d35fc2
commit
58e4e4c5af
6 files changed
+355
No files matched your search
@@ -0,0 +1,132 @@
|
||||
"""Wait for the sole approved recovery; freeze, predict and score once within its budget."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import psutil
|
||||
from freeze_g1_components import sha
|
||||
from s2_custody_common import read_json,checked
|
||||
from s2_recovery_supervisor import save
|
||||
from s3a_recovery_common import state
|
||||
from s3a_postfit import validate_terminal
|
||||
|
||||
|
||||
REPO=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8')
|
||||
BRANCH='refs/heads/codex/s2-execution-diff-20261114'
|
||||
|
||||
|
||||
def freeze_files(paths,message):
|
||||
# This checkout is the existing publication checkout, never the unborn root .git.
|
||||
status=subprocess.check_output(['git','status','--porcelain','--untracked-files=no'],cwd=REPO,text=True)
|
||||
if status.strip():raise RuntimeError('Publication checkout has unrelated tracked changes')
|
||||
for relative in paths:
|
||||
p=Path(relative);dest=REPO/p
|
||||
if p.is_dir():shutil.copytree(p,dest,dirs_exist_ok=True)
|
||||
else:dest.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(p,dest)
|
||||
subprocess.run(['git','add','-f','--',*[str(p) for p in paths]],cwd=REPO,check=True)
|
||||
subprocess.run(['git','diff','--cached','--check','--','.',':(exclude)*.log'],cwd=REPO,check=True)
|
||||
subprocess.run(['git','-c','gc.auto=0','commit','-q','-m',message],cwd=REPO,check=True)
|
||||
return subprocess.check_output(['git','rev-parse','HEAD'],cwd=REPO,text=True).strip()
|
||||
|
||||
|
||||
def run(root,chain):
|
||||
chain.mkdir(exist_ok=True)
|
||||
with (chain/'lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (chain/'started.json').exists():raise FileExistsError('Postfit chain already attempted')
|
||||
c,scope,old,passed=state(root)
|
||||
code=read_json(root/'postfit-code-freeze.json')
|
||||
for p,h in code['source_sha256'].items():checked(p,h)
|
||||
save(chain/'started.json',dict(unix=time.time(),pid=os.getpid(),root=str(root),postfit_code_freeze=code,
|
||||
no_fits_launched_by_this_chain=True,existing_recovery_only=True))
|
||||
def mark(stage,**extra):save(chain/'progress.json',dict(stage=stage,unix=time.time(),**extra))
|
||||
stage='WAIT_EXISTING_RECOVERY'
|
||||
try:
|
||||
mark(stage)
|
||||
while not (root/'batch-complete.json').exists():time.sleep(5)
|
||||
batch=read_json(root/'batch-complete.json')
|
||||
if batch['status']!='ALL_S3A_FITS_TERMINAL':raise RuntimeError('Recovery stopped incomplete; no retry or prediction')
|
||||
frozen=read_json(root/'terminal-freeze.json');validate_terminal(batch,frozen)
|
||||
start=read_json(root/'batch-started.json')
|
||||
# A finished receipt can precede the coordinator's last exit by a moment.
|
||||
for _ in range(24):
|
||||
if not psutil.pid_exists(start['pid']):break
|
||||
p=psutil.Process(start['pid'])
|
||||
if str(root) not in p.cmdline():raise RuntimeError('Coordinator PID reused')
|
||||
time.sleep(5)
|
||||
else:raise RuntimeError('Coordinator still alive after terminal receipt')
|
||||
for p in psutil.process_iter(['pid','cmdline']):
|
||||
cmd=p.info['cmdline'] or []
|
||||
if str(root) in cmd and any(Path(v).name=='run_s3a_fit.py' for v in cmd):
|
||||
raise RuntimeError('Fit worker remains alive')
|
||||
stage='FREEZE_TERMINAL_MODELS';mark(stage)
|
||||
terminal_commit=freeze_files([str(root),scope['original_root']],
|
||||
'Freeze complete two-source S3-A fits before test predictions')
|
||||
save(root/'terminal-git-freeze.json',dict(git_commit=terminal_commit,
|
||||
terminal_freeze_sha256=sha(root/'terminal-freeze.json'),before_predictions=True,
|
||||
remote_publication_separate=True))
|
||||
stage='PREDICT_ONCE';mark(stage,terminal_commit=terminal_commit)
|
||||
env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',
|
||||
NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
||||
with (chain/'predict-launch.log').open('x') as log:
|
||||
subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'predict'],env=env,
|
||||
stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920)
|
||||
if read_json(root/'postfit-predict-guard.json')['worker']['status']!='PASS':
|
||||
raise RuntimeError('Prediction guard failed; no retry/scoring')
|
||||
written=read_json(root/'predictions-written.json')
|
||||
if written['status']!='ALL_S3A_PREDICTIONS_WRITTEN':raise RuntimeError('Predictions not complete')
|
||||
stage='FREEZE_PREDICTIONS';mark(stage)
|
||||
prediction_commit=freeze_files([str(root)],'Freeze S3-A predictions and exact manifest before scoring')
|
||||
save(root/'predictions-complete.json',dict(status='ALL_S3A_PREDICTIONS_FROZEN',git_commit=prediction_commit,
|
||||
contract_sha256=sha(root/'contract.json'),terminal_freeze_sha256=sha(root/'terminal-freeze.json'),
|
||||
prediction_index_sha256=sha(root/'prediction-index.json'),prediction_files=written['prediction_files'],
|
||||
no_new_label_decode=True,remote_publication_separate=True))
|
||||
stage='SCORE_ONCE';mark(stage,prediction_commit=prediction_commit)
|
||||
with (chain/'score-launch.log').open('x') as log:
|
||||
subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'score'],env=env,
|
||||
stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920)
|
||||
guard=read_json(root/'postfit-score-guard.json')
|
||||
if guard['worker']['status']!='PASS':raise RuntimeError('Score guard failed; no retry')
|
||||
result=read_json(root/'scores/result.json')
|
||||
if result['status']!='S3A_SCORED':raise RuntimeError('Score result missing')
|
||||
for name,digest in result['output_sha256'].items():checked(root/'scores'/name,digest)
|
||||
save(chain/'scientific-complete.json',dict(status='S3A_SCORED',root=str(root),
|
||||
terminal_commit=terminal_commit,prediction_commit=prediction_commit,
|
||||
result_sha256=sha(root/'scores/result.json'),guard=guard))
|
||||
stage='FREEZE_RESULTS';mark(stage)
|
||||
result_commit=freeze_files([str(root),str(chain)],'Preserve complete S3-A adaptive follow-up results and cumulative accounting')
|
||||
stage='WAIT_EXISTING_PUBLICATION';mark(stage,result_commit=result_commit)
|
||||
publication=Path('artifacts/publication-after-naive-2026-11-15/status.json')
|
||||
deadline=time.monotonic()+10800
|
||||
while True:
|
||||
pub=read_json(publication)
|
||||
if pub['status']=='PUBLICATION_COMPLETE':break
|
||||
if pub['status']=='PUBLICATION_STOPPED':raise RuntimeError('Earlier publication stopped; preserve results locally')
|
||||
if time.monotonic()>deadline:raise TimeoutError('Earlier publication still pending')
|
||||
time.sleep(5)
|
||||
remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0]
|
||||
if remote!=pub['target']:raise RuntimeError('Concurrent remote branch change; do not overwrite')
|
||||
stage='PUBLISH_RESULTS';mark(stage,result_commit=result_commit)
|
||||
with (chain/'publication.log').open('x') as log:
|
||||
subprocess.run(['git','-c','pack.window=0','-c','pack.threads=4','-c','core.compression=1',
|
||||
'push','origin',result_commit+':'+BRANCH],cwd=REPO,stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10800)
|
||||
remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0]
|
||||
if remote!=result_commit:raise RuntimeError('Publication readback differs')
|
||||
save(chain/'complete.json',dict(status='S3A_SCIENCE_AND_PUBLICATION_COMPLETE',result_commit=result_commit,
|
||||
result_sha256=sha(root/'scores/result.json'),cumulative_worker_CPU_s=guard['cumulative_worker_CPU_s']))
|
||||
mark('COMPLETE',result_commit=result_commit)
|
||||
return 0
|
||||
except Exception as exc:
|
||||
save(chain/'stopped.json',dict(stage=stage,error=repr(exc),unix=time.time(),no_automatic_retry=True))
|
||||
mark('STOPPED',failed_stage=stage,error=repr(exc))
|
||||
raise
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('chain',type=Path);a=p.parse_args()
|
||||
raise SystemExit(run(a.root,a.chain))
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Bounded prediction then existing frozen scorer; no target IO during prediction."""
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import joblib
|
||||
import numpy as np
|
||||
import psutil
|
||||
from threadpoolctl import threadpool_limits
|
||||
from freeze_g1_components import sha
|
||||
from s2_custody_common import checked,read_json,prediction_inputs
|
||||
from s2_recovery_supervisor import save,supervise,stop_group
|
||||
from s2_recovery_monitoring import disk_usage
|
||||
from run_s3a_score import expected_predictions,run as score
|
||||
from s3a_postfit import gate,budget,predict_seconds
|
||||
|
||||
|
||||
def predict(root):
|
||||
c,scope,batch,frozen=gate(root)
|
||||
if (root/'predictions-started.json').exists() or (root/'predictions').exists():
|
||||
raise FileExistsError('No repeat prediction phase')
|
||||
start=time.process_time()
|
||||
save(root/'predictions-started.json',dict(unix=time.time(),pid=os.getpid(),
|
||||
terminal_freeze_sha256=sha(root/'terminal-freeze.json'),postfit_code_freeze=read_json(root/'postfit-code-freeze.json')))
|
||||
roster=read_json(checked(c['roster'],c['roster_sha256']))['parts']
|
||||
expected=expected_predictions(root,c,frozen['final_fits'],roster)
|
||||
models={t['cell']:joblib.load(checked(t['model_path'],t['model_sha256']))
|
||||
for t in frozen['final_fits'] if t['status']=='PASS'}
|
||||
by_case={}
|
||||
for path,(cell,row) in expected.items():by_case.setdefault(row['case'],[]).append((path,cell,row))
|
||||
index={}
|
||||
with threadpool_limits(limits=1):
|
||||
for n,(case,items) in enumerate(sorted(by_case.items()),1):
|
||||
d=prediction_inputs(items[0][2])
|
||||
for path,cell,row in items:
|
||||
values=predict_seconds(models[cell['index']],d,cell['arm'])
|
||||
p=Path(path);p.parent.mkdir(parents=True,exist_ok=True)
|
||||
if p.exists():raise FileExistsError('Duplicate prediction')
|
||||
np.savez_compressed(p,seconds=values,raw_ids=d['raw_ids'],counts=d['counts'])
|
||||
index[str(p)]=sha(p)
|
||||
save(root/'prediction-progress.json',dict(parts=n,predictions=len(index),worker_CPU_s=time.process_time()-start))
|
||||
if set(index)!=set(expected):raise ValueError('Missing/extra predicted keys')
|
||||
save(root/'prediction-index.json',index)
|
||||
save(root/'predictions-written.json',dict(status='ALL_S3A_PREDICTIONS_WRITTEN',contract_sha256=sha(root/'contract.json'),
|
||||
terminal_freeze_sha256=sha(root/'terminal-freeze.json'),prediction_index_sha256=sha(root/'prediction-index.json'),
|
||||
prediction_files=len(index),test_parts=len(by_case),CPU_s=time.process_time()-start,
|
||||
new_label_decodes=0,test_labels_read=False,new_fits=0,finite_negative_predictions_retained=True))
|
||||
return 0
|
||||
|
||||
|
||||
def guarded(root,phase):
|
||||
c,scope,batch,frozen=gate(root)
|
||||
if phase=='predict':
|
||||
if (root/'postfit-started.json').exists():raise FileExistsError('No repeated postfit start')
|
||||
save(root/'postfit-started.json',dict(unix=time.time(),includes_interphase_idle_wall=True))
|
||||
begun=read_json(root/'postfit-started.json')['unix']
|
||||
remaining_wall=c['limits']['prediction_scoring_wall_seconds']-(time.time()-begun)
|
||||
if remaining_wall<=0:raise ValueError('Combined prediction/scoring wall cap exhausted')
|
||||
previous=0.
|
||||
if phase=='score':
|
||||
old=read_json(root/'postfit-predict-guard.json')
|
||||
if old['worker']['status']!='PASS':raise ValueError('Prediction guard failed')
|
||||
previous=old['worker']['cpu_seconds']
|
||||
cap=budget(batch['worker_cpu_seconds'],previous,c['limits'])
|
||||
os.sched_setaffinity(0,[24])
|
||||
command=[sys.executable,__file__,str(root),phase,'--worker']
|
||||
done=threading.Event();reasons=[]
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
try:
|
||||
if disk_usage(root)+disk_usage(Path(scope['original_root']))>c['limits']['disk_bytes']:
|
||||
reasons.append('CUMULATIVE_DISK_CAP')
|
||||
except Exception as exc:reasons.append('DISK_MONITOR_ERROR:'+repr(exc))
|
||||
if reasons:
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:stop_group(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
done.wait(1)
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
try:
|
||||
w=supervise(command,root/'postfit-guards'/phase,'worker',dict(total_cpu_seconds=cap,
|
||||
job_cpu_seconds=cap,job_wall_seconds=remaining_wall,
|
||||
tree_rss_bytes=c['limits']['per_fit_tree_rss_bytes'],disk_bytes=c['limits']['disk_bytes'],max_processes=1),poll=.2)
|
||||
finally:done.set();thread.join(timeout=3)
|
||||
if reasons:w.update(status='STOPPED_OR_FAILED',stop_reason=reasons[0])
|
||||
save(root/('postfit-'+phase+'-guard.json'),dict(worker=w,prior_fit_worker_CPU_s=batch['worker_cpu_seconds'],
|
||||
prior_postfit_worker_CPU_s=previous,postfit_worker_CPU_s=previous+w['cpu_seconds'],
|
||||
cumulative_worker_CPU_s=batch['worker_cpu_seconds']+previous+w['cpu_seconds'],
|
||||
total_CPU_cap=c['limits']['worker_cpu_seconds'],postfit_CPU_cap=c['limits']['postfit_reserved_cpu_seconds'],
|
||||
guard_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2])))
|
||||
return 0 if w['status']=='PASS' else 1
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['predict','score']);p.add_argument('--worker',action='store_true');a=p.parse_args()
|
||||
if a.worker:
|
||||
if a.phase=='predict':raise SystemExit(predict(a.root))
|
||||
gate(a.root);score(a.root,a.root)
|
||||
else:raise SystemExit(guarded(a.root,a.phase))
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Postfit gates and existing-budget accounting for the approved S3-A continuation."""
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
from freeze_g1_components import sha
|
||||
from s2_custody_common import read_json,checked
|
||||
from s3a_data import features,columns
|
||||
from s3a_recovery_common import state
|
||||
|
||||
|
||||
def validate_terminal(batch,frozen):
|
||||
ids=[r['index'] for r in batch['jobs']]
|
||||
if (batch['status']!='ALL_S3A_FITS_TERMINAL' or batch['stop_reason'] is not None
|
||||
or not batch['accounting_complete'] or len(ids)!=390 or set(ids)!=set(range(390))):
|
||||
raise ValueError('All390 canonical terminal jobs required')
|
||||
if frozen['status']!='ALL_S3A_FITS_TERMINAL' or frozen['hpo_terminal_indices']!=list(range(360)):
|
||||
raise ValueError('360 HPO terminal freeze required')
|
||||
if len(frozen['final_fits'])!=30 or frozen['final_fits']!=batch['final_fits']:
|
||||
raise ValueError('Final checkpoint freeze differs')
|
||||
|
||||
|
||||
def budget(fit_seconds,already_postfit_seconds,limits):
|
||||
if fit_seconds<0 or fit_seconds>limits['fitting_worker_cpu_seconds'] or already_postfit_seconds<0:
|
||||
raise ValueError('Invalid completed CPU ledger')
|
||||
remaining=min(limits['postfit_reserved_cpu_seconds']-already_postfit_seconds,
|
||||
limits['worker_cpu_seconds']-fit_seconds-already_postfit_seconds)
|
||||
if remaining<=0:raise ValueError('Postfit budget exhausted')
|
||||
return remaining
|
||||
|
||||
|
||||
def predict_seconds(model,d,arm):
|
||||
x=features(d,arm)
|
||||
if model.n_features_in_!=len(columns(arm)):raise ValueError('Model feature width differs')
|
||||
values=np.asarray(model.predict(x),dtype=np.float64)*d['t_set']
|
||||
if values.shape!=(len(d['raw_ids']),) or not np.isfinite(values).all():
|
||||
raise ValueError('Invalid prediction shape/values')
|
||||
return values # Finite negative predictions are retained, never clipped.
|
||||
|
||||
|
||||
def gate(root):
|
||||
root=Path(root);c,scope,old,passed=state(root)
|
||||
batch=read_json(root/'batch-complete.json');frozen=read_json(root/'terminal-freeze.json')
|
||||
validate_terminal(batch,frozen)
|
||||
checked(frozen['batch_receipt'],frozen['batch_receipt_sha256'])
|
||||
manifest=read_json(checked(root/'two-source-manifest.json',batch['two_source_manifest_sha256']))
|
||||
if len(manifest['records'])!=390 or {r['index'] for r in manifest['records']}!=set(range(390)):
|
||||
raise ValueError('Two-source canonical manifest incomplete')
|
||||
if manifest['cumulative_worker_CPU_s']!=batch['worker_cpu_seconds']:
|
||||
raise ValueError('Two-source cumulative accounting differs')
|
||||
for entry in manifest['records']:
|
||||
if entry['job']!=c['jobs'][entry['index']]:raise ValueError('Job specification mutation')
|
||||
if entry['guard_path']:checked(entry['guard_path'],entry['guard_sha256'])
|
||||
if entry['result_path']:checked(entry['result_path'],entry['result_sha256'])
|
||||
code=read_json(root/'postfit-code-freeze.json')
|
||||
if not code['git_commit'] or code['contract_sha256']!=sha(root/'contract.json'):
|
||||
raise ValueError('Postfit implementation not Git fixed')
|
||||
for p,h in code['source_sha256'].items():checked(p,h)
|
||||
terminal_git=read_json(root/'terminal-git-freeze.json')
|
||||
if not terminal_git['git_commit'] or terminal_git['terminal_freeze_sha256']!=sha(root/'terminal-freeze.json'):
|
||||
raise ValueError('Terminal checkpoints must be Git fixed before predictions')
|
||||
return c,scope,batch,frozen
|
||||
@@ -0,0 +1,42 @@
|
||||
import unittest
|
||||
import numpy as np
|
||||
from s3a_postfit import budget,predict_seconds,validate_terminal
|
||||
|
||||
|
||||
class Model:
|
||||
n_features_in_=24
|
||||
def predict(self,x):return -np.ones(len(x))
|
||||
|
||||
|
||||
class Postfit(unittest.TestCase):
|
||||
def test_cumulative_budget_never_resets(self):
|
||||
l=dict(fitting_worker_cpu_seconds=64800,postfit_reserved_cpu_seconds=7200,worker_cpu_seconds=72000)
|
||||
self.assertEqual(budget(64800,100,l),7100)
|
||||
self.assertEqual(budget(20000,100,l),7100)
|
||||
with self.assertRaises(ValueError):budget(64801,0,l)
|
||||
with self.assertRaises(ValueError):budget(20000,7200,l)
|
||||
|
||||
def test_finite_negative_prediction_retained(self):
|
||||
d=dict(x=np.zeros((2,20)),edge=np.zeros((2,4)),raw_ids=np.array([9,10]),t_set=2.)
|
||||
np.testing.assert_array_equal(predict_seconds(Model(),d,'R2_E4'),[-2.,-2.])
|
||||
with self.assertRaises(ValueError):predict_seconds(Model(),dict(d,edge=np.zeros((2,3))),'R2_E4')
|
||||
|
||||
def test_width_and_nonfinite_rejected(self):
|
||||
d=dict(x=np.zeros((2,20)),edge=np.zeros((2,4)),raw_ids=np.array([9,10]),t_set=2.)
|
||||
m=Model();m.n_features_in_=22
|
||||
with self.assertRaises(ValueError):predict_seconds(m,d,'R2_E4')
|
||||
m=Model();m.predict=lambda x:np.full(len(x),np.nan)
|
||||
with self.assertRaises(ValueError):predict_seconds(m,d,'R2_E4')
|
||||
|
||||
def test_full_terminal_gate(self):
|
||||
final=[dict(cell=i,status='PASS') for i in range(30)]
|
||||
b=dict(status='ALL_S3A_FITS_TERMINAL',stop_reason=None,accounting_complete=True,jobs=[dict(index=i) for i in range(390)],final_fits=final)
|
||||
f=dict(status='ALL_S3A_FITS_TERMINAL',hpo_terminal_indices=list(range(360)),final_fits=final)
|
||||
validate_terminal(b,f)
|
||||
with self.assertRaises(ValueError):validate_terminal(dict(b,jobs=b['jobs'][:-1]),f)
|
||||
with self.assertRaises(ValueError):validate_terminal(dict(b,jobs=b['jobs'][:-1]+[b['jobs'][0]]),f)
|
||||
with self.assertRaises(ValueError):validate_terminal(b,dict(f,hpo_terminal_indices=list(range(390))))
|
||||
with self.assertRaises(ValueError):validate_terminal(dict(b,stop_reason='INTERRUPTED'),f)
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in new issue
Block a user