From 58e4e4c5af31e5a340cc75a7fbf9f6ae63ff7e1b Mon Sep 17 00:00:00 2001 From: Frank14f <1515444314@qq.com> Date: Sun, 15 Nov 2026 18:24:05 +0800 Subject: [PATCH] Fix bounded S3-A postfit chain before test prediction access --- .../postfit-synthetic-acceptance.json | 7 + docs/s3a-postfit-continuation-2026-11-15.md | 11 ++ tools/continue_s3a_postfit.py | 132 ++++++++++++++++++ tools/run_s3a_postfit.py | 103 ++++++++++++++ tools/s3a_postfit.py | 60 ++++++++ tools/test_s3a_postfit.py | 42 ++++++ 6 files changed, 355 insertions(+) create mode 100644 artifacts/s3a-recovery-2026-11-15/postfit-synthetic-acceptance.json create mode 100644 docs/s3a-postfit-continuation-2026-11-15.md create mode 100644 tools/continue_s3a_postfit.py create mode 100644 tools/run_s3a_postfit.py create mode 100644 tools/s3a_postfit.py create mode 100644 tools/test_s3a_postfit.py diff --git a/artifacts/s3a-recovery-2026-11-15/postfit-synthetic-acceptance.json b/artifacts/s3a-recovery-2026-11-15/postfit-synthetic-acceptance.json new file mode 100644 index 000000000..677fd6703 --- /dev/null +++ b/artifacts/s3a-recovery-2026-11-15/postfit-synthetic-acceptance.json @@ -0,0 +1,7 @@ +{ + "status": "PASS", + "synthetic_tests": 4, + "cases": "cumulative20h/2h accounting; finite-negative preservation; wrong width/nonfinite rejection; all360HPO+30final unique terminal gate", + "no_real_models_loaded": true, + "fit_worker_sources_unchanged": true +} diff --git a/docs/s3a-postfit-continuation-2026-11-15.md b/docs/s3a-postfit-continuation-2026-11-15.md new file mode 100644 index 000000000..c406d5fad --- /dev/null +++ b/docs/s3a-postfit-continuation-2026-11-15.md @@ -0,0 +1,11 @@ +# S3-A既定预测/评分接续实现 + +沿原主审1794732829与恢复裁量1794736905,在唯一159项接续达到390项canonical终态、全部最终checkpoint固定后,才进行原已获准的预测与评分。没有新增模型/臂/训练预算或标签解码。旧run_s3a_fit/data/statistics/run_s3a_score来源完全不动。 + +四项新增验收通过:总20h与预留postfit2h不重置、有限负预测不裁剪、列宽/非有限预测拒绝、360HPO+30final完整唯一终态门。两源来源逐项spec/guard/result校验、原source pins继续核;拟合中断或预算终止则链停,不重试。 + +接续链只等待现有recovery,不启动任何fit。先Git固定两源全部模型和终态,按同一617roster及30个final细胞生成精确预测清单(仅PASS模型,NA保留原理由),固定全部预测与清单Git提交后,才调用原冻结评分器读取S2已有query标签缓存。baseline15R2模型、预测、1851分数复用。保留finite negatives,无新目标解码。 + +预测/评分共享7200 worker CPU秒并加入原所有fit累计;合并wall3h包括阶段间等待,16GiB树RSS、单CPU、旧新目录累计100GiB。每段单次guard,失败原样留下且链停。进程树/累计磁盘检查守住执行边界。模型输出文件和两源manifest在评分前后均核hash。 + +科学结果提交与远端发布分开:Git本地不可变提交先固定;发布等待原087f63b大包及固定4d76d35后续提交完成,核远端ref后才按正常非强推发最终结果。网络传输不改变科学预算/冻结,不对尚未完成的上传宣称远端可读。当前这里只完成实现及合成验收,预测/评分尚未执行。 diff --git a/tools/continue_s3a_postfit.py b/tools/continue_s3a_postfit.py new file mode 100644 index 000000000..16d6fcf00 --- /dev/null +++ b/tools/continue_s3a_postfit.py @@ -0,0 +1,132 @@ +"""Wait for the sole approved recovery; freeze, predict and score once within its budget.""" +import argparse +import fcntl +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import time +import psutil +from freeze_g1_components import sha +from s2_custody_common import read_json,checked +from s2_recovery_supervisor import save +from s3a_recovery_common import state +from s3a_postfit import validate_terminal + + +REPO=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8') +BRANCH='refs/heads/codex/s2-execution-diff-20261114' + + +def freeze_files(paths,message): + # This checkout is the existing publication checkout, never the unborn root .git. + status=subprocess.check_output(['git','status','--porcelain','--untracked-files=no'],cwd=REPO,text=True) + if status.strip():raise RuntimeError('Publication checkout has unrelated tracked changes') + for relative in paths: + p=Path(relative);dest=REPO/p + if p.is_dir():shutil.copytree(p,dest,dirs_exist_ok=True) + else:dest.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(p,dest) + subprocess.run(['git','add','-f','--',*[str(p) for p in paths]],cwd=REPO,check=True) + subprocess.run(['git','diff','--cached','--check','--','.',':(exclude)*.log'],cwd=REPO,check=True) + subprocess.run(['git','-c','gc.auto=0','commit','-q','-m',message],cwd=REPO,check=True) + return subprocess.check_output(['git','rev-parse','HEAD'],cwd=REPO,text=True).strip() + + +def run(root,chain): + chain.mkdir(exist_ok=True) + with (chain/'lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + if (chain/'started.json').exists():raise FileExistsError('Postfit chain already attempted') + c,scope,old,passed=state(root) + code=read_json(root/'postfit-code-freeze.json') + for p,h in code['source_sha256'].items():checked(p,h) + save(chain/'started.json',dict(unix=time.time(),pid=os.getpid(),root=str(root),postfit_code_freeze=code, + no_fits_launched_by_this_chain=True,existing_recovery_only=True)) + def mark(stage,**extra):save(chain/'progress.json',dict(stage=stage,unix=time.time(),**extra)) + stage='WAIT_EXISTING_RECOVERY' + try: + mark(stage) + while not (root/'batch-complete.json').exists():time.sleep(5) + batch=read_json(root/'batch-complete.json') + if batch['status']!='ALL_S3A_FITS_TERMINAL':raise RuntimeError('Recovery stopped incomplete; no retry or prediction') + frozen=read_json(root/'terminal-freeze.json');validate_terminal(batch,frozen) + start=read_json(root/'batch-started.json') + # A finished receipt can precede the coordinator's last exit by a moment. + for _ in range(24): + if not psutil.pid_exists(start['pid']):break + p=psutil.Process(start['pid']) + if str(root) not in p.cmdline():raise RuntimeError('Coordinator PID reused') + time.sleep(5) + else:raise RuntimeError('Coordinator still alive after terminal receipt') + for p in psutil.process_iter(['pid','cmdline']): + cmd=p.info['cmdline'] or [] + if str(root) in cmd and any(Path(v).name=='run_s3a_fit.py' for v in cmd): + raise RuntimeError('Fit worker remains alive') + stage='FREEZE_TERMINAL_MODELS';mark(stage) + terminal_commit=freeze_files([str(root),scope['original_root']], + 'Freeze complete two-source S3-A fits before test predictions') + save(root/'terminal-git-freeze.json',dict(git_commit=terminal_commit, + terminal_freeze_sha256=sha(root/'terminal-freeze.json'),before_predictions=True, + remote_publication_separate=True)) + stage='PREDICT_ONCE';mark(stage,terminal_commit=terminal_commit) + env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1', + NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='') + with (chain/'predict-launch.log').open('x') as log: + subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'predict'],env=env, + stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920) + if read_json(root/'postfit-predict-guard.json')['worker']['status']!='PASS': + raise RuntimeError('Prediction guard failed; no retry/scoring') + written=read_json(root/'predictions-written.json') + if written['status']!='ALL_S3A_PREDICTIONS_WRITTEN':raise RuntimeError('Predictions not complete') + stage='FREEZE_PREDICTIONS';mark(stage) + prediction_commit=freeze_files([str(root)],'Freeze S3-A predictions and exact manifest before scoring') + save(root/'predictions-complete.json',dict(status='ALL_S3A_PREDICTIONS_FROZEN',git_commit=prediction_commit, + contract_sha256=sha(root/'contract.json'),terminal_freeze_sha256=sha(root/'terminal-freeze.json'), + prediction_index_sha256=sha(root/'prediction-index.json'),prediction_files=written['prediction_files'], + no_new_label_decode=True,remote_publication_separate=True)) + stage='SCORE_ONCE';mark(stage,prediction_commit=prediction_commit) + with (chain/'score-launch.log').open('x') as log: + subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'score'],env=env, + stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920) + guard=read_json(root/'postfit-score-guard.json') + if guard['worker']['status']!='PASS':raise RuntimeError('Score guard failed; no retry') + result=read_json(root/'scores/result.json') + if result['status']!='S3A_SCORED':raise RuntimeError('Score result missing') + for name,digest in result['output_sha256'].items():checked(root/'scores'/name,digest) + save(chain/'scientific-complete.json',dict(status='S3A_SCORED',root=str(root), + terminal_commit=terminal_commit,prediction_commit=prediction_commit, + result_sha256=sha(root/'scores/result.json'),guard=guard)) + stage='FREEZE_RESULTS';mark(stage) + result_commit=freeze_files([str(root),str(chain)],'Preserve complete S3-A adaptive follow-up results and cumulative accounting') + stage='WAIT_EXISTING_PUBLICATION';mark(stage,result_commit=result_commit) + publication=Path('artifacts/publication-after-naive-2026-11-15/status.json') + deadline=time.monotonic()+10800 + while True: + pub=read_json(publication) + if pub['status']=='PUBLICATION_COMPLETE':break + if pub['status']=='PUBLICATION_STOPPED':raise RuntimeError('Earlier publication stopped; preserve results locally') + if time.monotonic()>deadline:raise TimeoutError('Earlier publication still pending') + time.sleep(5) + remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0] + if remote!=pub['target']:raise RuntimeError('Concurrent remote branch change; do not overwrite') + stage='PUBLISH_RESULTS';mark(stage,result_commit=result_commit) + with (chain/'publication.log').open('x') as log: + subprocess.run(['git','-c','pack.window=0','-c','pack.threads=4','-c','core.compression=1', + 'push','origin',result_commit+':'+BRANCH],cwd=REPO,stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10800) + remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0] + if remote!=result_commit:raise RuntimeError('Publication readback differs') + save(chain/'complete.json',dict(status='S3A_SCIENCE_AND_PUBLICATION_COMPLETE',result_commit=result_commit, + result_sha256=sha(root/'scores/result.json'),cumulative_worker_CPU_s=guard['cumulative_worker_CPU_s'])) + mark('COMPLETE',result_commit=result_commit) + return 0 + except Exception as exc: + save(chain/'stopped.json',dict(stage=stage,error=repr(exc),unix=time.time(),no_automatic_retry=True)) + mark('STOPPED',failed_stage=stage,error=repr(exc)) + raise + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('chain',type=Path);a=p.parse_args() + raise SystemExit(run(a.root,a.chain)) diff --git a/tools/run_s3a_postfit.py b/tools/run_s3a_postfit.py new file mode 100644 index 000000000..8d52a7f07 --- /dev/null +++ b/tools/run_s3a_postfit.py @@ -0,0 +1,103 @@ +"""Bounded prediction then existing frozen scorer; no target IO during prediction.""" +import argparse +import os +from pathlib import Path +import resource +import sys +import threading +import time +import joblib +import numpy as np +import psutil +from threadpoolctl import threadpool_limits +from freeze_g1_components import sha +from s2_custody_common import checked,read_json,prediction_inputs +from s2_recovery_supervisor import save,supervise,stop_group +from s2_recovery_monitoring import disk_usage +from run_s3a_score import expected_predictions,run as score +from s3a_postfit import gate,budget,predict_seconds + + +def predict(root): + c,scope,batch,frozen=gate(root) + if (root/'predictions-started.json').exists() or (root/'predictions').exists(): + raise FileExistsError('No repeat prediction phase') + start=time.process_time() + save(root/'predictions-started.json',dict(unix=time.time(),pid=os.getpid(), + terminal_freeze_sha256=sha(root/'terminal-freeze.json'),postfit_code_freeze=read_json(root/'postfit-code-freeze.json'))) + roster=read_json(checked(c['roster'],c['roster_sha256']))['parts'] + expected=expected_predictions(root,c,frozen['final_fits'],roster) + models={t['cell']:joblib.load(checked(t['model_path'],t['model_sha256'])) + for t in frozen['final_fits'] if t['status']=='PASS'} + by_case={} + for path,(cell,row) in expected.items():by_case.setdefault(row['case'],[]).append((path,cell,row)) + index={} + with threadpool_limits(limits=1): + for n,(case,items) in enumerate(sorted(by_case.items()),1): + d=prediction_inputs(items[0][2]) + for path,cell,row in items: + values=predict_seconds(models[cell['index']],d,cell['arm']) + p=Path(path);p.parent.mkdir(parents=True,exist_ok=True) + if p.exists():raise FileExistsError('Duplicate prediction') + np.savez_compressed(p,seconds=values,raw_ids=d['raw_ids'],counts=d['counts']) + index[str(p)]=sha(p) + save(root/'prediction-progress.json',dict(parts=n,predictions=len(index),worker_CPU_s=time.process_time()-start)) + if set(index)!=set(expected):raise ValueError('Missing/extra predicted keys') + save(root/'prediction-index.json',index) + save(root/'predictions-written.json',dict(status='ALL_S3A_PREDICTIONS_WRITTEN',contract_sha256=sha(root/'contract.json'), + terminal_freeze_sha256=sha(root/'terminal-freeze.json'),prediction_index_sha256=sha(root/'prediction-index.json'), + prediction_files=len(index),test_parts=len(by_case),CPU_s=time.process_time()-start, + new_label_decodes=0,test_labels_read=False,new_fits=0,finite_negative_predictions_retained=True)) + return 0 + + +def guarded(root,phase): + c,scope,batch,frozen=gate(root) + if phase=='predict': + if (root/'postfit-started.json').exists():raise FileExistsError('No repeated postfit start') + save(root/'postfit-started.json',dict(unix=time.time(),includes_interphase_idle_wall=True)) + begun=read_json(root/'postfit-started.json')['unix'] + remaining_wall=c['limits']['prediction_scoring_wall_seconds']-(time.time()-begun) + if remaining_wall<=0:raise ValueError('Combined prediction/scoring wall cap exhausted') + previous=0. + if phase=='score': + old=read_json(root/'postfit-predict-guard.json') + if old['worker']['status']!='PASS':raise ValueError('Prediction guard failed') + previous=old['worker']['cpu_seconds'] + cap=budget(batch['worker_cpu_seconds'],previous,c['limits']) + os.sched_setaffinity(0,[24]) + command=[sys.executable,__file__,str(root),phase,'--worker'] + done=threading.Event();reasons=[] + def watch(): + while not done.is_set(): + try: + if disk_usage(root)+disk_usage(Path(scope['original_root']))>c['limits']['disk_bytes']: + reasons.append('CUMULATIVE_DISK_CAP') + except Exception as exc:reasons.append('DISK_MONITOR_ERROR:'+repr(exc)) + if reasons: + for p in psutil.Process().children(recursive=True): + try: + if p.cmdline()==command:stop_group(p.pid) + except psutil.NoSuchProcess:pass + done.wait(1) + thread=threading.Thread(target=watch,daemon=True);thread.start() + try: + w=supervise(command,root/'postfit-guards'/phase,'worker',dict(total_cpu_seconds=cap, + job_cpu_seconds=cap,job_wall_seconds=remaining_wall, + tree_rss_bytes=c['limits']['per_fit_tree_rss_bytes'],disk_bytes=c['limits']['disk_bytes'],max_processes=1),poll=.2) + finally:done.set();thread.join(timeout=3) + if reasons:w.update(status='STOPPED_OR_FAILED',stop_reason=reasons[0]) + save(root/('postfit-'+phase+'-guard.json'),dict(worker=w,prior_fit_worker_CPU_s=batch['worker_cpu_seconds'], + prior_postfit_worker_CPU_s=previous,postfit_worker_CPU_s=previous+w['cpu_seconds'], + cumulative_worker_CPU_s=batch['worker_cpu_seconds']+previous+w['cpu_seconds'], + total_CPU_cap=c['limits']['worker_cpu_seconds'],postfit_CPU_cap=c['limits']['postfit_reserved_cpu_seconds'], + guard_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]))) + return 0 if w['status']=='PASS' else 1 + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['predict','score']);p.add_argument('--worker',action='store_true');a=p.parse_args() + if a.worker: + if a.phase=='predict':raise SystemExit(predict(a.root)) + gate(a.root);score(a.root,a.root) + else:raise SystemExit(guarded(a.root,a.phase)) diff --git a/tools/s3a_postfit.py b/tools/s3a_postfit.py new file mode 100644 index 000000000..c0d7284fd --- /dev/null +++ b/tools/s3a_postfit.py @@ -0,0 +1,60 @@ +"""Postfit gates and existing-budget accounting for the approved S3-A continuation.""" +from pathlib import Path +import numpy as np +from freeze_g1_components import sha +from s2_custody_common import read_json,checked +from s3a_data import features,columns +from s3a_recovery_common import state + + +def validate_terminal(batch,frozen): + ids=[r['index'] for r in batch['jobs']] + if (batch['status']!='ALL_S3A_FITS_TERMINAL' or batch['stop_reason'] is not None + or not batch['accounting_complete'] or len(ids)!=390 or set(ids)!=set(range(390))): + raise ValueError('All390 canonical terminal jobs required') + if frozen['status']!='ALL_S3A_FITS_TERMINAL' or frozen['hpo_terminal_indices']!=list(range(360)): + raise ValueError('360 HPO terminal freeze required') + if len(frozen['final_fits'])!=30 or frozen['final_fits']!=batch['final_fits']: + raise ValueError('Final checkpoint freeze differs') + + +def budget(fit_seconds,already_postfit_seconds,limits): + if fit_seconds<0 or fit_seconds>limits['fitting_worker_cpu_seconds'] or already_postfit_seconds<0: + raise ValueError('Invalid completed CPU ledger') + remaining=min(limits['postfit_reserved_cpu_seconds']-already_postfit_seconds, + limits['worker_cpu_seconds']-fit_seconds-already_postfit_seconds) + if remaining<=0:raise ValueError('Postfit budget exhausted') + return remaining + + +def predict_seconds(model,d,arm): + x=features(d,arm) + if model.n_features_in_!=len(columns(arm)):raise ValueError('Model feature width differs') + values=np.asarray(model.predict(x),dtype=np.float64)*d['t_set'] + if values.shape!=(len(d['raw_ids']),) or not np.isfinite(values).all(): + raise ValueError('Invalid prediction shape/values') + return values # Finite negative predictions are retained, never clipped. + + +def gate(root): + root=Path(root);c,scope,old,passed=state(root) + batch=read_json(root/'batch-complete.json');frozen=read_json(root/'terminal-freeze.json') + validate_terminal(batch,frozen) + checked(frozen['batch_receipt'],frozen['batch_receipt_sha256']) + manifest=read_json(checked(root/'two-source-manifest.json',batch['two_source_manifest_sha256'])) + if len(manifest['records'])!=390 or {r['index'] for r in manifest['records']}!=set(range(390)): + raise ValueError('Two-source canonical manifest incomplete') + if manifest['cumulative_worker_CPU_s']!=batch['worker_cpu_seconds']: + raise ValueError('Two-source cumulative accounting differs') + for entry in manifest['records']: + if entry['job']!=c['jobs'][entry['index']]:raise ValueError('Job specification mutation') + if entry['guard_path']:checked(entry['guard_path'],entry['guard_sha256']) + if entry['result_path']:checked(entry['result_path'],entry['result_sha256']) + code=read_json(root/'postfit-code-freeze.json') + if not code['git_commit'] or code['contract_sha256']!=sha(root/'contract.json'): + raise ValueError('Postfit implementation not Git fixed') + for p,h in code['source_sha256'].items():checked(p,h) + terminal_git=read_json(root/'terminal-git-freeze.json') + if not terminal_git['git_commit'] or terminal_git['terminal_freeze_sha256']!=sha(root/'terminal-freeze.json'): + raise ValueError('Terminal checkpoints must be Git fixed before predictions') + return c,scope,batch,frozen diff --git a/tools/test_s3a_postfit.py b/tools/test_s3a_postfit.py new file mode 100644 index 000000000..088699057 --- /dev/null +++ b/tools/test_s3a_postfit.py @@ -0,0 +1,42 @@ +import unittest +import numpy as np +from s3a_postfit import budget,predict_seconds,validate_terminal + + +class Model: + n_features_in_=24 + def predict(self,x):return -np.ones(len(x)) + + +class Postfit(unittest.TestCase): + def test_cumulative_budget_never_resets(self): + l=dict(fitting_worker_cpu_seconds=64800,postfit_reserved_cpu_seconds=7200,worker_cpu_seconds=72000) + self.assertEqual(budget(64800,100,l),7100) + self.assertEqual(budget(20000,100,l),7100) + with self.assertRaises(ValueError):budget(64801,0,l) + with self.assertRaises(ValueError):budget(20000,7200,l) + + def test_finite_negative_prediction_retained(self): + d=dict(x=np.zeros((2,20)),edge=np.zeros((2,4)),raw_ids=np.array([9,10]),t_set=2.) + np.testing.assert_array_equal(predict_seconds(Model(),d,'R2_E4'),[-2.,-2.]) + with self.assertRaises(ValueError):predict_seconds(Model(),dict(d,edge=np.zeros((2,3))),'R2_E4') + + def test_width_and_nonfinite_rejected(self): + d=dict(x=np.zeros((2,20)),edge=np.zeros((2,4)),raw_ids=np.array([9,10]),t_set=2.) + m=Model();m.n_features_in_=22 + with self.assertRaises(ValueError):predict_seconds(m,d,'R2_E4') + m=Model();m.predict=lambda x:np.full(len(x),np.nan) + with self.assertRaises(ValueError):predict_seconds(m,d,'R2_E4') + + def test_full_terminal_gate(self): + final=[dict(cell=i,status='PASS') for i in range(30)] + b=dict(status='ALL_S3A_FITS_TERMINAL',stop_reason=None,accounting_complete=True,jobs=[dict(index=i) for i in range(390)],final_fits=final) + f=dict(status='ALL_S3A_FITS_TERMINAL',hpo_terminal_indices=list(range(360)),final_fits=final) + validate_terminal(b,f) + with self.assertRaises(ValueError):validate_terminal(dict(b,jobs=b['jobs'][:-1]),f) + with self.assertRaises(ValueError):validate_terminal(dict(b,jobs=b['jobs'][:-1]+[b['jobs'][0]]),f) + with self.assertRaises(ValueError):validate_terminal(b,dict(f,hpo_terminal_indices=list(range(390)))) + with self.assertRaises(ValueError):validate_terminal(dict(b,stop_reason='INTERRUPTED'),f) + + +if __name__=='__main__':unittest.main()