Fix bounded S3-A postfit chain before test prediction access

This commit is contained in:
Frank14f committed 2026-11-15 18:24:05 +08:00
1 parent 4d76d35fc2
commit 58e4e4c5af
6 files changed
+355

No files matched your search

@@ -0,0 +1,7 @@
{
"status": "PASS",
"synthetic_tests": 4,
"cases": "cumulative20h/2h accounting; finite-negative preservation; wrong width/nonfinite rejection; all360HPO+30final unique terminal gate",
"no_real_models_loaded": true,
"fit_worker_sources_unchanged": true
}
@@ -0,0 +1,11 @@
# S3-A既定预测/评分接续实现
沿原主审1794732829与恢复裁量1794736905,在唯一159项接续达到390项canonical终态、全部最终checkpoint固定后,才进行原已获准的预测与评分。没有新增模型/臂/训练预算或标签解码。旧run_s3a_fit/data/statistics/run_s3a_score来源完全不动。
四项新增验收通过:总20h与预留postfit2h不重置、有限负预测不裁剪、列宽/非有限预测拒绝、360HPO+30final完整唯一终态门。两源来源逐项spec/guard/result校验、原source pins继续核;拟合中断或预算终止则链停,不重试。
接续链只等待现有recovery,不启动任何fit。先Git固定两源全部模型和终态,按同一617roster及30个final细胞生成精确预测清单(仅PASS模型,NA保留原理由),固定全部预测与清单Git提交后,才调用原冻结评分器读取S2已有query标签缓存。baseline15R2模型、预测、1851分数复用。保留finite negatives,无新目标解码。
预测/评分共享7200 worker CPU秒并加入原所有fit累计;合并wall3h包括阶段间等待,16GiB树RSS、单CPU、旧新目录累计100GiB。每段单次guard,失败原样留下且链停。进程树/累计磁盘检查守住执行边界。模型输出文件和两源manifest在评分前后均核hash。
科学结果提交与远端发布分开:Git本地不可变提交先固定;发布等待原087f63b大包及固定4d76d35后续提交完成,核远端ref后才按正常非强推发最终结果。网络传输不改变科学预算/冻结,不对尚未完成的上传宣称远端可读。当前这里只完成实现及合成验收,预测/评分尚未执行。
+132
View File
@@ -0,0 +1,132 @@
"""Wait for the sole approved recovery; freeze, predict and score once within its budget."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import time
import psutil
from freeze_g1_components import sha
from s2_custody_common import read_json,checked
from s2_recovery_supervisor import save
from s3a_recovery_common import state
from s3a_postfit import validate_terminal
REPO=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8')
BRANCH='refs/heads/codex/s2-execution-diff-20261114'
def freeze_files(paths,message):
# This checkout is the existing publication checkout, never the unborn root .git.
status=subprocess.check_output(['git','status','--porcelain','--untracked-files=no'],cwd=REPO,text=True)
if status.strip():raise RuntimeError('Publication checkout has unrelated tracked changes')
for relative in paths:
p=Path(relative);dest=REPO/p
if p.is_dir():shutil.copytree(p,dest,dirs_exist_ok=True)
else:dest.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(p,dest)
subprocess.run(['git','add','-f','--',*[str(p) for p in paths]],cwd=REPO,check=True)
subprocess.run(['git','diff','--cached','--check','--','.',':(exclude)*.log'],cwd=REPO,check=True)
subprocess.run(['git','-c','gc.auto=0','commit','-q','-m',message],cwd=REPO,check=True)
return subprocess.check_output(['git','rev-parse','HEAD'],cwd=REPO,text=True).strip()
def run(root,chain):
chain.mkdir(exist_ok=True)
with (chain/'lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (chain/'started.json').exists():raise FileExistsError('Postfit chain already attempted')
c,scope,old,passed=state(root)
code=read_json(root/'postfit-code-freeze.json')
for p,h in code['source_sha256'].items():checked(p,h)
save(chain/'started.json',dict(unix=time.time(),pid=os.getpid(),root=str(root),postfit_code_freeze=code,
no_fits_launched_by_this_chain=True,existing_recovery_only=True))
def mark(stage,**extra):save(chain/'progress.json',dict(stage=stage,unix=time.time(),**extra))
stage='WAIT_EXISTING_RECOVERY'
try:
mark(stage)
while not (root/'batch-complete.json').exists():time.sleep(5)
batch=read_json(root/'batch-complete.json')
if batch['status']!='ALL_S3A_FITS_TERMINAL':raise RuntimeError('Recovery stopped incomplete; no retry or prediction')
frozen=read_json(root/'terminal-freeze.json');validate_terminal(batch,frozen)
start=read_json(root/'batch-started.json')
# A finished receipt can precede the coordinator's last exit by a moment.
for _ in range(24):
if not psutil.pid_exists(start['pid']):break
p=psutil.Process(start['pid'])
if str(root) not in p.cmdline():raise RuntimeError('Coordinator PID reused')
time.sleep(5)
else:raise RuntimeError('Coordinator still alive after terminal receipt')
for p in psutil.process_iter(['pid','cmdline']):
cmd=p.info['cmdline'] or []
if str(root) in cmd and any(Path(v).name=='run_s3a_fit.py' for v in cmd):
raise RuntimeError('Fit worker remains alive')
stage='FREEZE_TERMINAL_MODELS';mark(stage)
terminal_commit=freeze_files([str(root),scope['original_root']],
'Freeze complete two-source S3-A fits before test predictions')
save(root/'terminal-git-freeze.json',dict(git_commit=terminal_commit,
terminal_freeze_sha256=sha(root/'terminal-freeze.json'),before_predictions=True,
remote_publication_separate=True))
stage='PREDICT_ONCE';mark(stage,terminal_commit=terminal_commit)
env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',
NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
with (chain/'predict-launch.log').open('x') as log:
subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'predict'],env=env,
stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920)
if read_json(root/'postfit-predict-guard.json')['worker']['status']!='PASS':
raise RuntimeError('Prediction guard failed; no retry/scoring')
written=read_json(root/'predictions-written.json')
if written['status']!='ALL_S3A_PREDICTIONS_WRITTEN':raise RuntimeError('Predictions not complete')
stage='FREEZE_PREDICTIONS';mark(stage)
prediction_commit=freeze_files([str(root)],'Freeze S3-A predictions and exact manifest before scoring')
save(root/'predictions-complete.json',dict(status='ALL_S3A_PREDICTIONS_FROZEN',git_commit=prediction_commit,
contract_sha256=sha(root/'contract.json'),terminal_freeze_sha256=sha(root/'terminal-freeze.json'),
prediction_index_sha256=sha(root/'prediction-index.json'),prediction_files=written['prediction_files'],
no_new_label_decode=True,remote_publication_separate=True))
stage='SCORE_ONCE';mark(stage,prediction_commit=prediction_commit)
with (chain/'score-launch.log').open('x') as log:
subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'score'],env=env,
stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920)
guard=read_json(root/'postfit-score-guard.json')
if guard['worker']['status']!='PASS':raise RuntimeError('Score guard failed; no retry')
result=read_json(root/'scores/result.json')
if result['status']!='S3A_SCORED':raise RuntimeError('Score result missing')
for name,digest in result['output_sha256'].items():checked(root/'scores'/name,digest)
save(chain/'scientific-complete.json',dict(status='S3A_SCORED',root=str(root),
terminal_commit=terminal_commit,prediction_commit=prediction_commit,
result_sha256=sha(root/'scores/result.json'),guard=guard))
stage='FREEZE_RESULTS';mark(stage)
result_commit=freeze_files([str(root),str(chain)],'Preserve complete S3-A adaptive follow-up results and cumulative accounting')
stage='WAIT_EXISTING_PUBLICATION';mark(stage,result_commit=result_commit)
publication=Path('artifacts/publication-after-naive-2026-11-15/status.json')
deadline=time.monotonic()+10800
while True:
pub=read_json(publication)
if pub['status']=='PUBLICATION_COMPLETE':break
if pub['status']=='PUBLICATION_STOPPED':raise RuntimeError('Earlier publication stopped; preserve results locally')
if time.monotonic()>deadline:raise TimeoutError('Earlier publication still pending')
time.sleep(5)
remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0]
if remote!=pub['target']:raise RuntimeError('Concurrent remote branch change; do not overwrite')
stage='PUBLISH_RESULTS';mark(stage,result_commit=result_commit)
with (chain/'publication.log').open('x') as log:
subprocess.run(['git','-c','pack.window=0','-c','pack.threads=4','-c','core.compression=1',
'push','origin',result_commit+':'+BRANCH],cwd=REPO,stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10800)
remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0]
if remote!=result_commit:raise RuntimeError('Publication readback differs')
save(chain/'complete.json',dict(status='S3A_SCIENCE_AND_PUBLICATION_COMPLETE',result_commit=result_commit,
result_sha256=sha(root/'scores/result.json'),cumulative_worker_CPU_s=guard['cumulative_worker_CPU_s']))
mark('COMPLETE',result_commit=result_commit)
return 0
except Exception as exc:
save(chain/'stopped.json',dict(stage=stage,error=repr(exc),unix=time.time(),no_automatic_retry=True))
mark('STOPPED',failed_stage=stage,error=repr(exc))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('chain',type=Path);a=p.parse_args()
raise SystemExit(run(a.root,a.chain))
+103
View File
@@ -0,0 +1,103 @@
"""Bounded prediction then existing frozen scorer; no target IO during prediction."""
import argparse
import os
from pathlib import Path
import resource
import sys
import threading
import time
import joblib
import numpy as np
import psutil
from threadpoolctl import threadpool_limits
from freeze_g1_components import sha
from s2_custody_common import checked,read_json,prediction_inputs
from s2_recovery_supervisor import save,supervise,stop_group
from s2_recovery_monitoring import disk_usage
from run_s3a_score import expected_predictions,run as score
from s3a_postfit import gate,budget,predict_seconds
def predict(root):
c,scope,batch,frozen=gate(root)
if (root/'predictions-started.json').exists() or (root/'predictions').exists():
raise FileExistsError('No repeat prediction phase')
start=time.process_time()
save(root/'predictions-started.json',dict(unix=time.time(),pid=os.getpid(),
terminal_freeze_sha256=sha(root/'terminal-freeze.json'),postfit_code_freeze=read_json(root/'postfit-code-freeze.json')))
roster=read_json(checked(c['roster'],c['roster_sha256']))['parts']
expected=expected_predictions(root,c,frozen['final_fits'],roster)
models={t['cell']:joblib.load(checked(t['model_path'],t['model_sha256']))
for t in frozen['final_fits'] if t['status']=='PASS'}
by_case={}
for path,(cell,row) in expected.items():by_case.setdefault(row['case'],[]).append((path,cell,row))
index={}
with threadpool_limits(limits=1):
for n,(case,items) in enumerate(sorted(by_case.items()),1):
d=prediction_inputs(items[0][2])
for path,cell,row in items:
values=predict_seconds(models[cell['index']],d,cell['arm'])
p=Path(path);p.parent.mkdir(parents=True,exist_ok=True)
if p.exists():raise FileExistsError('Duplicate prediction')
np.savez_compressed(p,seconds=values,raw_ids=d['raw_ids'],counts=d['counts'])
index[str(p)]=sha(p)
save(root/'prediction-progress.json',dict(parts=n,predictions=len(index),worker_CPU_s=time.process_time()-start))
if set(index)!=set(expected):raise ValueError('Missing/extra predicted keys')
save(root/'prediction-index.json',index)
save(root/'predictions-written.json',dict(status='ALL_S3A_PREDICTIONS_WRITTEN',contract_sha256=sha(root/'contract.json'),
terminal_freeze_sha256=sha(root/'terminal-freeze.json'),prediction_index_sha256=sha(root/'prediction-index.json'),
prediction_files=len(index),test_parts=len(by_case),CPU_s=time.process_time()-start,
new_label_decodes=0,test_labels_read=False,new_fits=0,finite_negative_predictions_retained=True))
return 0
def guarded(root,phase):
c,scope,batch,frozen=gate(root)
if phase=='predict':
if (root/'postfit-started.json').exists():raise FileExistsError('No repeated postfit start')
save(root/'postfit-started.json',dict(unix=time.time(),includes_interphase_idle_wall=True))
begun=read_json(root/'postfit-started.json')['unix']
remaining_wall=c['limits']['prediction_scoring_wall_seconds']-(time.time()-begun)
if remaining_wall<=0:raise ValueError('Combined prediction/scoring wall cap exhausted')
previous=0.
if phase=='score':
old=read_json(root/'postfit-predict-guard.json')
if old['worker']['status']!='PASS':raise ValueError('Prediction guard failed')
previous=old['worker']['cpu_seconds']
cap=budget(batch['worker_cpu_seconds'],previous,c['limits'])
os.sched_setaffinity(0,[24])
command=[sys.executable,__file__,str(root),phase,'--worker']
done=threading.Event();reasons=[]
def watch():
while not done.is_set():
try:
if disk_usage(root)+disk_usage(Path(scope['original_root']))>c['limits']['disk_bytes']:
reasons.append('CUMULATIVE_DISK_CAP')
except Exception as exc:reasons.append('DISK_MONITOR_ERROR:'+repr(exc))
if reasons:
for p in psutil.Process().children(recursive=True):
try:
if p.cmdline()==command:stop_group(p.pid)
except psutil.NoSuchProcess:pass
done.wait(1)
thread=threading.Thread(target=watch,daemon=True);thread.start()
try:
w=supervise(command,root/'postfit-guards'/phase,'worker',dict(total_cpu_seconds=cap,
job_cpu_seconds=cap,job_wall_seconds=remaining_wall,
tree_rss_bytes=c['limits']['per_fit_tree_rss_bytes'],disk_bytes=c['limits']['disk_bytes'],max_processes=1),poll=.2)
finally:done.set();thread.join(timeout=3)
if reasons:w.update(status='STOPPED_OR_FAILED',stop_reason=reasons[0])
save(root/('postfit-'+phase+'-guard.json'),dict(worker=w,prior_fit_worker_CPU_s=batch['worker_cpu_seconds'],
prior_postfit_worker_CPU_s=previous,postfit_worker_CPU_s=previous+w['cpu_seconds'],
cumulative_worker_CPU_s=batch['worker_cpu_seconds']+previous+w['cpu_seconds'],
total_CPU_cap=c['limits']['worker_cpu_seconds'],postfit_CPU_cap=c['limits']['postfit_reserved_cpu_seconds'],
guard_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2])))
return 0 if w['status']=='PASS' else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['predict','score']);p.add_argument('--worker',action='store_true');a=p.parse_args()
if a.worker:
if a.phase=='predict':raise SystemExit(predict(a.root))
gate(a.root);score(a.root,a.root)
else:raise SystemExit(guarded(a.root,a.phase))
+60
View File
@@ -0,0 +1,60 @@
"""Postfit gates and existing-budget accounting for the approved S3-A continuation."""
from pathlib import Path
import numpy as np
from freeze_g1_components import sha
from s2_custody_common import read_json,checked
from s3a_data import features,columns
from s3a_recovery_common import state
def validate_terminal(batch,frozen):
ids=[r['index'] for r in batch['jobs']]
if (batch['status']!='ALL_S3A_FITS_TERMINAL' or batch['stop_reason'] is not None
or not batch['accounting_complete'] or len(ids)!=390 or set(ids)!=set(range(390))):
raise ValueError('All390 canonical terminal jobs required')
if frozen['status']!='ALL_S3A_FITS_TERMINAL' or frozen['hpo_terminal_indices']!=list(range(360)):
raise ValueError('360 HPO terminal freeze required')
if len(frozen['final_fits'])!=30 or frozen['final_fits']!=batch['final_fits']:
raise ValueError('Final checkpoint freeze differs')
def budget(fit_seconds,already_postfit_seconds,limits):
if fit_seconds<0 or fit_seconds>limits['fitting_worker_cpu_seconds'] or already_postfit_seconds<0:
raise ValueError('Invalid completed CPU ledger')
remaining=min(limits['postfit_reserved_cpu_seconds']-already_postfit_seconds,
limits['worker_cpu_seconds']-fit_seconds-already_postfit_seconds)
if remaining<=0:raise ValueError('Postfit budget exhausted')
return remaining
def predict_seconds(model,d,arm):
x=features(d,arm)
if model.n_features_in_!=len(columns(arm)):raise ValueError('Model feature width differs')
values=np.asarray(model.predict(x),dtype=np.float64)*d['t_set']
if values.shape!=(len(d['raw_ids']),) or not np.isfinite(values).all():
raise ValueError('Invalid prediction shape/values')
return values # Finite negative predictions are retained, never clipped.
def gate(root):
root=Path(root);c,scope,old,passed=state(root)
batch=read_json(root/'batch-complete.json');frozen=read_json(root/'terminal-freeze.json')
validate_terminal(batch,frozen)
checked(frozen['batch_receipt'],frozen['batch_receipt_sha256'])
manifest=read_json(checked(root/'two-source-manifest.json',batch['two_source_manifest_sha256']))
if len(manifest['records'])!=390 or {r['index'] for r in manifest['records']}!=set(range(390)):
raise ValueError('Two-source canonical manifest incomplete')
if manifest['cumulative_worker_CPU_s']!=batch['worker_cpu_seconds']:
raise ValueError('Two-source cumulative accounting differs')
for entry in manifest['records']:
if entry['job']!=c['jobs'][entry['index']]:raise ValueError('Job specification mutation')
if entry['guard_path']:checked(entry['guard_path'],entry['guard_sha256'])
if entry['result_path']:checked(entry['result_path'],entry['result_sha256'])
code=read_json(root/'postfit-code-freeze.json')
if not code['git_commit'] or code['contract_sha256']!=sha(root/'contract.json'):
raise ValueError('Postfit implementation not Git fixed')
for p,h in code['source_sha256'].items():checked(p,h)
terminal_git=read_json(root/'terminal-git-freeze.json')
if not terminal_git['git_commit'] or terminal_git['terminal_freeze_sha256']!=sha(root/'terminal-freeze.json'):
raise ValueError('Terminal checkpoints must be Git fixed before predictions')
return c,scope,batch,frozen
+42
View File
@@ -0,0 +1,42 @@
import unittest
import numpy as np
from s3a_postfit import budget,predict_seconds,validate_terminal
class Model:
n_features_in_=24
def predict(self,x):return -np.ones(len(x))
class Postfit(unittest.TestCase):
def test_cumulative_budget_never_resets(self):
l=dict(fitting_worker_cpu_seconds=64800,postfit_reserved_cpu_seconds=7200,worker_cpu_seconds=72000)
self.assertEqual(budget(64800,100,l),7100)
self.assertEqual(budget(20000,100,l),7100)
with self.assertRaises(ValueError):budget(64801,0,l)
with self.assertRaises(ValueError):budget(20000,7200,l)
def test_finite_negative_prediction_retained(self):
d=dict(x=np.zeros((2,20)),edge=np.zeros((2,4)),raw_ids=np.array([9,10]),t_set=2.)
np.testing.assert_array_equal(predict_seconds(Model(),d,'R2_E4'),[-2.,-2.])
with self.assertRaises(ValueError):predict_seconds(Model(),dict(d,edge=np.zeros((2,3))),'R2_E4')
def test_width_and_nonfinite_rejected(self):
d=dict(x=np.zeros((2,20)),edge=np.zeros((2,4)),raw_ids=np.array([9,10]),t_set=2.)
m=Model();m.n_features_in_=22
with self.assertRaises(ValueError):predict_seconds(m,d,'R2_E4')
m=Model();m.predict=lambda x:np.full(len(x),np.nan)
with self.assertRaises(ValueError):predict_seconds(m,d,'R2_E4')
def test_full_terminal_gate(self):
final=[dict(cell=i,status='PASS') for i in range(30)]
b=dict(status='ALL_S3A_FITS_TERMINAL',stop_reason=None,accounting_complete=True,jobs=[dict(index=i) for i in range(390)],final_fits=final)
f=dict(status='ALL_S3A_FITS_TERMINAL',hpo_terminal_indices=list(range(360)),final_fits=final)
validate_terminal(b,f)
with self.assertRaises(ValueError):validate_terminal(dict(b,jobs=b['jobs'][:-1]),f)
with self.assertRaises(ValueError):validate_terminal(dict(b,jobs=b['jobs'][:-1]+[b['jobs'][0]]),f)
with self.assertRaises(ValueError):validate_terminal(b,dict(f,hpo_terminal_indices=list(range(390))))
with self.assertRaises(ValueError):validate_terminal(dict(b,stop_reason='INTERRUPTED'),f)
if __name__=='__main__':unittest.main()