Fix bounded S3-A postfit chain before test prediction access

This commit is contained in:
Frank14f committed 2026-11-15 18:24:05 +08:00
1 parent 4d76d35fc2
commit 58e4e4c5af
6 files changed
+355

No files matched your search

+132
View File
@@ -0,0 +1,132 @@
"""Wait for the sole approved recovery; freeze, predict and score once within its budget."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import time
import psutil
from freeze_g1_components import sha
from s2_custody_common import read_json,checked
from s2_recovery_supervisor import save
from s3a_recovery_common import state
from s3a_postfit import validate_terminal
REPO=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8')
BRANCH='refs/heads/codex/s2-execution-diff-20261114'
def freeze_files(paths,message):
# This checkout is the existing publication checkout, never the unborn root .git.
status=subprocess.check_output(['git','status','--porcelain','--untracked-files=no'],cwd=REPO,text=True)
if status.strip():raise RuntimeError('Publication checkout has unrelated tracked changes')
for relative in paths:
p=Path(relative);dest=REPO/p
if p.is_dir():shutil.copytree(p,dest,dirs_exist_ok=True)
else:dest.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(p,dest)
subprocess.run(['git','add','-f','--',*[str(p) for p in paths]],cwd=REPO,check=True)
subprocess.run(['git','diff','--cached','--check','--','.',':(exclude)*.log'],cwd=REPO,check=True)
subprocess.run(['git','-c','gc.auto=0','commit','-q','-m',message],cwd=REPO,check=True)
return subprocess.check_output(['git','rev-parse','HEAD'],cwd=REPO,text=True).strip()
def run(root,chain):
chain.mkdir(exist_ok=True)
with (chain/'lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (chain/'started.json').exists():raise FileExistsError('Postfit chain already attempted')
c,scope,old,passed=state(root)
code=read_json(root/'postfit-code-freeze.json')
for p,h in code['source_sha256'].items():checked(p,h)
save(chain/'started.json',dict(unix=time.time(),pid=os.getpid(),root=str(root),postfit_code_freeze=code,
no_fits_launched_by_this_chain=True,existing_recovery_only=True))
def mark(stage,**extra):save(chain/'progress.json',dict(stage=stage,unix=time.time(),**extra))
stage='WAIT_EXISTING_RECOVERY'
try:
mark(stage)
while not (root/'batch-complete.json').exists():time.sleep(5)
batch=read_json(root/'batch-complete.json')
if batch['status']!='ALL_S3A_FITS_TERMINAL':raise RuntimeError('Recovery stopped incomplete; no retry or prediction')
frozen=read_json(root/'terminal-freeze.json');validate_terminal(batch,frozen)
start=read_json(root/'batch-started.json')
# A finished receipt can precede the coordinator's last exit by a moment.
for _ in range(24):
if not psutil.pid_exists(start['pid']):break
p=psutil.Process(start['pid'])
if str(root) not in p.cmdline():raise RuntimeError('Coordinator PID reused')
time.sleep(5)
else:raise RuntimeError('Coordinator still alive after terminal receipt')
for p in psutil.process_iter(['pid','cmdline']):
cmd=p.info['cmdline'] or []
if str(root) in cmd and any(Path(v).name=='run_s3a_fit.py' for v in cmd):
raise RuntimeError('Fit worker remains alive')
stage='FREEZE_TERMINAL_MODELS';mark(stage)
terminal_commit=freeze_files([str(root),scope['original_root']],
'Freeze complete two-source S3-A fits before test predictions')
save(root/'terminal-git-freeze.json',dict(git_commit=terminal_commit,
terminal_freeze_sha256=sha(root/'terminal-freeze.json'),before_predictions=True,
remote_publication_separate=True))
stage='PREDICT_ONCE';mark(stage,terminal_commit=terminal_commit)
env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',
NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
with (chain/'predict-launch.log').open('x') as log:
subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'predict'],env=env,
stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920)
if read_json(root/'postfit-predict-guard.json')['worker']['status']!='PASS':
raise RuntimeError('Prediction guard failed; no retry/scoring')
written=read_json(root/'predictions-written.json')
if written['status']!='ALL_S3A_PREDICTIONS_WRITTEN':raise RuntimeError('Predictions not complete')
stage='FREEZE_PREDICTIONS';mark(stage)
prediction_commit=freeze_files([str(root)],'Freeze S3-A predictions and exact manifest before scoring')
save(root/'predictions-complete.json',dict(status='ALL_S3A_PREDICTIONS_FROZEN',git_commit=prediction_commit,
contract_sha256=sha(root/'contract.json'),terminal_freeze_sha256=sha(root/'terminal-freeze.json'),
prediction_index_sha256=sha(root/'prediction-index.json'),prediction_files=written['prediction_files'],
no_new_label_decode=True,remote_publication_separate=True))
stage='SCORE_ONCE';mark(stage,prediction_commit=prediction_commit)
with (chain/'score-launch.log').open('x') as log:
subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'score'],env=env,
stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920)
guard=read_json(root/'postfit-score-guard.json')
if guard['worker']['status']!='PASS':raise RuntimeError('Score guard failed; no retry')
result=read_json(root/'scores/result.json')
if result['status']!='S3A_SCORED':raise RuntimeError('Score result missing')
for name,digest in result['output_sha256'].items():checked(root/'scores'/name,digest)
save(chain/'scientific-complete.json',dict(status='S3A_SCORED',root=str(root),
terminal_commit=terminal_commit,prediction_commit=prediction_commit,
result_sha256=sha(root/'scores/result.json'),guard=guard))
stage='FREEZE_RESULTS';mark(stage)
result_commit=freeze_files([str(root),str(chain)],'Preserve complete S3-A adaptive follow-up results and cumulative accounting')
stage='WAIT_EXISTING_PUBLICATION';mark(stage,result_commit=result_commit)
publication=Path('artifacts/publication-after-naive-2026-11-15/status.json')
deadline=time.monotonic()+10800
while True:
pub=read_json(publication)
if pub['status']=='PUBLICATION_COMPLETE':break
if pub['status']=='PUBLICATION_STOPPED':raise RuntimeError('Earlier publication stopped; preserve results locally')
if time.monotonic()>deadline:raise TimeoutError('Earlier publication still pending')
time.sleep(5)
remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0]
if remote!=pub['target']:raise RuntimeError('Concurrent remote branch change; do not overwrite')
stage='PUBLISH_RESULTS';mark(stage,result_commit=result_commit)
with (chain/'publication.log').open('x') as log:
subprocess.run(['git','-c','pack.window=0','-c','pack.threads=4','-c','core.compression=1',
'push','origin',result_commit+':'+BRANCH],cwd=REPO,stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10800)
remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0]
if remote!=result_commit:raise RuntimeError('Publication readback differs')
save(chain/'complete.json',dict(status='S3A_SCIENCE_AND_PUBLICATION_COMPLETE',result_commit=result_commit,
result_sha256=sha(root/'scores/result.json'),cumulative_worker_CPU_s=guard['cumulative_worker_CPU_s']))
mark('COMPLETE',result_commit=result_commit)
return 0
except Exception as exc:
save(chain/'stopped.json',dict(stage=stage,error=repr(exc),unix=time.time(),no_automatic_retry=True))
mark('STOPPED',failed_stage=stage,error=repr(exc))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('chain',type=Path);a=p.parse_args()
raise SystemExit(run(a.root,a.chain))
+103
View File
@@ -0,0 +1,103 @@
"""Bounded prediction then existing frozen scorer; no target IO during prediction."""
import argparse
import os
from pathlib import Path
import resource
import sys
import threading
import time
import joblib
import numpy as np
import psutil
from threadpoolctl import threadpool_limits
from freeze_g1_components import sha
from s2_custody_common import checked,read_json,prediction_inputs
from s2_recovery_supervisor import save,supervise,stop_group
from s2_recovery_monitoring import disk_usage
from run_s3a_score import expected_predictions,run as score
from s3a_postfit import gate,budget,predict_seconds
def predict(root):
c,scope,batch,frozen=gate(root)
if (root/'predictions-started.json').exists() or (root/'predictions').exists():
raise FileExistsError('No repeat prediction phase')
start=time.process_time()
save(root/'predictions-started.json',dict(unix=time.time(),pid=os.getpid(),
terminal_freeze_sha256=sha(root/'terminal-freeze.json'),postfit_code_freeze=read_json(root/'postfit-code-freeze.json')))
roster=read_json(checked(c['roster'],c['roster_sha256']))['parts']
expected=expected_predictions(root,c,frozen['final_fits'],roster)
models={t['cell']:joblib.load(checked(t['model_path'],t['model_sha256']))
for t in frozen['final_fits'] if t['status']=='PASS'}
by_case={}
for path,(cell,row) in expected.items():by_case.setdefault(row['case'],[]).append((path,cell,row))
index={}
with threadpool_limits(limits=1):
for n,(case,items) in enumerate(sorted(by_case.items()),1):
d=prediction_inputs(items[0][2])
for path,cell,row in items:
values=predict_seconds(models[cell['index']],d,cell['arm'])
p=Path(path);p.parent.mkdir(parents=True,exist_ok=True)
if p.exists():raise FileExistsError('Duplicate prediction')
np.savez_compressed(p,seconds=values,raw_ids=d['raw_ids'],counts=d['counts'])
index[str(p)]=sha(p)
save(root/'prediction-progress.json',dict(parts=n,predictions=len(index),worker_CPU_s=time.process_time()-start))
if set(index)!=set(expected):raise ValueError('Missing/extra predicted keys')
save(root/'prediction-index.json',index)
save(root/'predictions-written.json',dict(status='ALL_S3A_PREDICTIONS_WRITTEN',contract_sha256=sha(root/'contract.json'),
terminal_freeze_sha256=sha(root/'terminal-freeze.json'),prediction_index_sha256=sha(root/'prediction-index.json'),
prediction_files=len(index),test_parts=len(by_case),CPU_s=time.process_time()-start,
new_label_decodes=0,test_labels_read=False,new_fits=0,finite_negative_predictions_retained=True))
return 0
def guarded(root,phase):
c,scope,batch,frozen=gate(root)
if phase=='predict':
if (root/'postfit-started.json').exists():raise FileExistsError('No repeated postfit start')
save(root/'postfit-started.json',dict(unix=time.time(),includes_interphase_idle_wall=True))
begun=read_json(root/'postfit-started.json')['unix']
remaining_wall=c['limits']['prediction_scoring_wall_seconds']-(time.time()-begun)
if remaining_wall<=0:raise ValueError('Combined prediction/scoring wall cap exhausted')
previous=0.
if phase=='score':
old=read_json(root/'postfit-predict-guard.json')
if old['worker']['status']!='PASS':raise ValueError('Prediction guard failed')
previous=old['worker']['cpu_seconds']
cap=budget(batch['worker_cpu_seconds'],previous,c['limits'])
os.sched_setaffinity(0,[24])
command=[sys.executable,__file__,str(root),phase,'--worker']
done=threading.Event();reasons=[]
def watch():
while not done.is_set():
try:
if disk_usage(root)+disk_usage(Path(scope['original_root']))>c['limits']['disk_bytes']:
reasons.append('CUMULATIVE_DISK_CAP')
except Exception as exc:reasons.append('DISK_MONITOR_ERROR:'+repr(exc))
if reasons:
for p in psutil.Process().children(recursive=True):
try:
if p.cmdline()==command:stop_group(p.pid)
except psutil.NoSuchProcess:pass
done.wait(1)
thread=threading.Thread(target=watch,daemon=True);thread.start()
try:
w=supervise(command,root/'postfit-guards'/phase,'worker',dict(total_cpu_seconds=cap,
job_cpu_seconds=cap,job_wall_seconds=remaining_wall,
tree_rss_bytes=c['limits']['per_fit_tree_rss_bytes'],disk_bytes=c['limits']['disk_bytes'],max_processes=1),poll=.2)
finally:done.set();thread.join(timeout=3)
if reasons:w.update(status='STOPPED_OR_FAILED',stop_reason=reasons[0])
save(root/('postfit-'+phase+'-guard.json'),dict(worker=w,prior_fit_worker_CPU_s=batch['worker_cpu_seconds'],
prior_postfit_worker_CPU_s=previous,postfit_worker_CPU_s=previous+w['cpu_seconds'],
cumulative_worker_CPU_s=batch['worker_cpu_seconds']+previous+w['cpu_seconds'],
total_CPU_cap=c['limits']['worker_cpu_seconds'],postfit_CPU_cap=c['limits']['postfit_reserved_cpu_seconds'],
guard_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2])))
return 0 if w['status']=='PASS' else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['predict','score']);p.add_argument('--worker',action='store_true');a=p.parse_args()
if a.worker:
if a.phase=='predict':raise SystemExit(predict(a.root))
gate(a.root);score(a.root,a.root)
else:raise SystemExit(guarded(a.root,a.phase))
+60
View File
@@ -0,0 +1,60 @@
"""Postfit gates and existing-budget accounting for the approved S3-A continuation."""
from pathlib import Path
import numpy as np
from freeze_g1_components import sha
from s2_custody_common import read_json,checked
from s3a_data import features,columns
from s3a_recovery_common import state
def validate_terminal(batch,frozen):
ids=[r['index'] for r in batch['jobs']]
if (batch['status']!='ALL_S3A_FITS_TERMINAL' or batch['stop_reason'] is not None
or not batch['accounting_complete'] or len(ids)!=390 or set(ids)!=set(range(390))):
raise ValueError('All390 canonical terminal jobs required')
if frozen['status']!='ALL_S3A_FITS_TERMINAL' or frozen['hpo_terminal_indices']!=list(range(360)):
raise ValueError('360 HPO terminal freeze required')
if len(frozen['final_fits'])!=30 or frozen['final_fits']!=batch['final_fits']:
raise ValueError('Final checkpoint freeze differs')
def budget(fit_seconds,already_postfit_seconds,limits):
if fit_seconds<0 or fit_seconds>limits['fitting_worker_cpu_seconds'] or already_postfit_seconds<0:
raise ValueError('Invalid completed CPU ledger')
remaining=min(limits['postfit_reserved_cpu_seconds']-already_postfit_seconds,
limits['worker_cpu_seconds']-fit_seconds-already_postfit_seconds)
if remaining<=0:raise ValueError('Postfit budget exhausted')
return remaining
def predict_seconds(model,d,arm):
x=features(d,arm)
if model.n_features_in_!=len(columns(arm)):raise ValueError('Model feature width differs')
values=np.asarray(model.predict(x),dtype=np.float64)*d['t_set']
if values.shape!=(len(d['raw_ids']),) or not np.isfinite(values).all():
raise ValueError('Invalid prediction shape/values')
return values # Finite negative predictions are retained, never clipped.
def gate(root):
root=Path(root);c,scope,old,passed=state(root)
batch=read_json(root/'batch-complete.json');frozen=read_json(root/'terminal-freeze.json')
validate_terminal(batch,frozen)
checked(frozen['batch_receipt'],frozen['batch_receipt_sha256'])
manifest=read_json(checked(root/'two-source-manifest.json',batch['two_source_manifest_sha256']))
if len(manifest['records'])!=390 or {r['index'] for r in manifest['records']}!=set(range(390)):
raise ValueError('Two-source canonical manifest incomplete')
if manifest['cumulative_worker_CPU_s']!=batch['worker_cpu_seconds']:
raise ValueError('Two-source cumulative accounting differs')
for entry in manifest['records']:
if entry['job']!=c['jobs'][entry['index']]:raise ValueError('Job specification mutation')
if entry['guard_path']:checked(entry['guard_path'],entry['guard_sha256'])
if entry['result_path']:checked(entry['result_path'],entry['result_sha256'])
code=read_json(root/'postfit-code-freeze.json')
if not code['git_commit'] or code['contract_sha256']!=sha(root/'contract.json'):
raise ValueError('Postfit implementation not Git fixed')
for p,h in code['source_sha256'].items():checked(p,h)
terminal_git=read_json(root/'terminal-git-freeze.json')
if not terminal_git['git_commit'] or terminal_git['terminal_freeze_sha256']!=sha(root/'terminal-freeze.json'):
raise ValueError('Terminal checkpoints must be Git fixed before predictions')
return c,scope,batch,frozen
+42
View File
@@ -0,0 +1,42 @@
import unittest
import numpy as np
from s3a_postfit import budget,predict_seconds,validate_terminal
class Model:
n_features_in_=24
def predict(self,x):return -np.ones(len(x))
class Postfit(unittest.TestCase):
def test_cumulative_budget_never_resets(self):
l=dict(fitting_worker_cpu_seconds=64800,postfit_reserved_cpu_seconds=7200,worker_cpu_seconds=72000)
self.assertEqual(budget(64800,100,l),7100)
self.assertEqual(budget(20000,100,l),7100)
with self.assertRaises(ValueError):budget(64801,0,l)
with self.assertRaises(ValueError):budget(20000,7200,l)
def test_finite_negative_prediction_retained(self):
d=dict(x=np.zeros((2,20)),edge=np.zeros((2,4)),raw_ids=np.array([9,10]),t_set=2.)
np.testing.assert_array_equal(predict_seconds(Model(),d,'R2_E4'),[-2.,-2.])
with self.assertRaises(ValueError):predict_seconds(Model(),dict(d,edge=np.zeros((2,3))),'R2_E4')
def test_width_and_nonfinite_rejected(self):
d=dict(x=np.zeros((2,20)),edge=np.zeros((2,4)),raw_ids=np.array([9,10]),t_set=2.)
m=Model();m.n_features_in_=22
with self.assertRaises(ValueError):predict_seconds(m,d,'R2_E4')
m=Model();m.predict=lambda x:np.full(len(x),np.nan)
with self.assertRaises(ValueError):predict_seconds(m,d,'R2_E4')
def test_full_terminal_gate(self):
final=[dict(cell=i,status='PASS') for i in range(30)]
b=dict(status='ALL_S3A_FITS_TERMINAL',stop_reason=None,accounting_complete=True,jobs=[dict(index=i) for i in range(390)],final_fits=final)
f=dict(status='ALL_S3A_FITS_TERMINAL',hpo_terminal_indices=list(range(360)),final_fits=final)
validate_terminal(b,f)
with self.assertRaises(ValueError):validate_terminal(dict(b,jobs=b['jobs'][:-1]),f)
with self.assertRaises(ValueError):validate_terminal(dict(b,jobs=b['jobs'][:-1]+[b['jobs'][0]]),f)
with self.assertRaises(ValueError):validate_terminal(b,dict(f,hpo_terminal_indices=list(range(390))))
with self.assertRaises(ValueError):validate_terminal(dict(b,stop_reason='INTERRUPTED'),f)
if __name__=='__main__':unittest.main()