From cb786589448447be1fba86baed9020b8b6c52456 Mon Sep 17 00:00:00 2001 From: Frank14f <1515444314@qq.com> Date: Sun, 15 Nov 2026 18:40:42 +0800 Subject: [PATCH] Gate final process-count stops on Claude ruling without changing fit workers --- .../retirement-for-final-policy.json | 20 ++ .../synthetic-acceptance.json | 7 + .../cell26-hpo-audit.json | 185 ++++++++++++++++++ docs/s3a-final-process-policy-2026-11-15.md | 11 ++ tools/continue_s3a_final_policy_postfit.py | 144 ++++++++++++++ tools/s3a_final_process_policy.py | 34 ++++ tools/test_s3a_final_process_policy.py | 32 +++ 7 files changed, 433 insertions(+) create mode 100644 artifacts/s3a-postfit-chain-2026-11-15/retirement-for-final-policy.json create mode 100644 artifacts/s3a-postfit-final-policy-chain-2026-11-15/synthetic-acceptance.json create mode 100644 artifacts/status-readback-2026-11-15/cell26-hpo-audit.json create mode 100644 docs/s3a-final-process-policy-2026-11-15.md create mode 100644 tools/continue_s3a_final_policy_postfit.py create mode 100644 tools/s3a_final_process_policy.py create mode 100644 tools/test_s3a_final_process_policy.py diff --git a/artifacts/s3a-postfit-chain-2026-11-15/retirement-for-final-policy.json b/artifacts/s3a-postfit-chain-2026-11-15/retirement-for-final-policy.json new file mode 100644 index 000000000..5f8ad9a12 --- /dev/null +++ b/artifacts/s3a-postfit-chain-2026-11-15/retirement-for-final-policy.json @@ -0,0 +1,20 @@ +{ + "unix": 1794739122.4651701, + "retired_pid": 465797, + "cmdline": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "-u", + "tools/continue_s3a_postfit.py", + "artifacts/s3a-recovery-2026-11-15", + "artifacts/s3a-postfit-chain-2026-11-15" + ], + "prior_progress": { + "stage": "WAIT_EXISTING_RECOVERY", + "unix": 1794738262.3016055 + }, + "reason": "Claude1794738923 requires final-process-count review before accepting whole-cell NA; replace idle waiter only", + "no_children": true, + "prediction_not_started": true, + "recovery_PID464772_not_signalled": true, + "exit_verified": true +} diff --git a/artifacts/s3a-postfit-final-policy-chain-2026-11-15/synthetic-acceptance.json b/artifacts/s3a-postfit-final-policy-chain-2026-11-15/synthetic-acceptance.json new file mode 100644 index 000000000..1cb9a6f27 --- /dev/null +++ b/artifacts/s3a-postfit-final-policy-chain-2026-11-15/synthetic-acceptance.json @@ -0,0 +1,7 @@ +{ + "status": "PASS", + "tests": 4, + "ruling": "1794738923.689619", + "old_waiter_retired": "artifacts/s3a-postfit-chain-2026-11-15/retirement-for-final-policy.json", + "scope": "Only final process_count_limit blocks adoption of whole-cell NA and predictions/scoring; HPO policy unchanged; no recovery automatically authorized" +} diff --git a/artifacts/status-readback-2026-11-15/cell26-hpo-audit.json b/artifacts/status-readback-2026-11-15/cell26-hpo-audit.json new file mode 100644 index 000000000..24956a3ec --- /dev/null +++ b/artifacts/status-readback-2026-11-15/cell26-hpo-audit.json @@ -0,0 +1,185 @@ +{ + "unix": 1794739015.6367815, + "cell": 26, + "arm": "R2_E4", + "fold": 4, + "pool": 50, + "terminal_HPO": 12, + "usable_finished_configurations": 11, + "unavailable_configuration": { + "job": 323, + "config_index": 11, + "reason": "process_count_limit", + "numeric_failure": false, + "child_identity": "unknown", + "no_retry": true + }, + "trials": [ + { + "index": 312, + "config_index": 0, + "guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-312/guard-result.json", + "guard_sha256": "6b0d67446f079a53fc6aa7bcb3b8700301e57009208ce688e822ff09ba3398e3", + "status": "PASS", + "stop_reason": null, + "returncode": 0, + "eligible_for_selection": true, + "validation_score": 0.07428104382661917, + "result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-312/output/result.json", + "result_sha256": "c0959b8a3d4b6c8b74c6f7a469d2a29a032024286d861a9b44adc625ec425fc2", + "model_sha256": "d4b6025a8641aa15ca2cb32f4896e9963fd443f884e55054bf0bbf760e4485f6" + }, + { + "index": 313, + "config_index": 1, + "guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-313/guard-result.json", + "guard_sha256": "4bafc13be24b164ff04c35bdb3ec6af8d0a2dac697daec77e059da569ef50ad1", + "status": "PASS", + "stop_reason": null, + "returncode": 0, + "eligible_for_selection": true, + "validation_score": 0.07561749254650058, + "result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-313/output/result.json", + "result_sha256": "db53b2f3495b4a8a416215fed910dd3a2418c41fd433aff6ea358423024eda6c", + "model_sha256": "3fe662cb5d712eae1fa0da9640edd8cf739c41652f4a11af3b37a67e380ecaae" + }, + { + "index": 314, + "config_index": 2, + "guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-314/guard-result.json", + "guard_sha256": "64c0b54c7adad1a19fca2ad14906c9d16e5fde2f7f4307e75f9486e7f8b767dc", + "status": "PASS", + "stop_reason": null, + "returncode": 0, + "eligible_for_selection": true, + "validation_score": 0.07641824353110742, + "result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-314/output/result.json", + "result_sha256": "13cf434440e66995f7294f7a1b1ff56b0fada8f1a40566ad71d24fd6a0dee604", + "model_sha256": "ab370bbe3f47f69bef5d44141abfcee8e157cee1d8fe3531ea105769be7dfab5" + }, + { + "index": 315, + "config_index": 3, + "guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-315/guard-result.json", + "guard_sha256": "9e6b4a5f493619b7eaa2c0d30c30f8818afee6c0e4a6e291ba68ac24e2024dd5", + "status": "PASS", + "stop_reason": null, + "returncode": 0, + "eligible_for_selection": true, + "validation_score": 0.07429812958903101, + "result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-315/output/result.json", + "result_sha256": "87bf28e13b813422889b3a64b930f249faf31da04fccce81efb9588f1294c08c", + "model_sha256": "d3f3fc6d0aa068d397bf4284634cccf0abdd99f587333b71d2a0cb7a9263d4a8" + }, + { + "index": 316, + "config_index": 4, + "guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-316/guard-result.json", + "guard_sha256": "728c586480af55a35b4887a01bd362c251895039eb640572646d63303500f0ef", + "status": "PASS", + "stop_reason": null, + "returncode": 0, + "eligible_for_selection": true, + "validation_score": 0.07518402096943289, + "result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-316/output/result.json", + "result_sha256": "2e877ec07958060682b7e09467517e6febef30a7198303d1a1c0d375a2520d08", + "model_sha256": "7022615bf7eafd04ce189fc5e997c13cdc4482f6ee4adcb16d37a1e4649352b7" + }, + { + "index": 317, + "config_index": 5, + "guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-317/guard-result.json", + "guard_sha256": "c33ec446c3d0096aea2d5c49bc208e3d75b1cb25ebee1a90be8038e29243a4f5", + "status": "PASS", + "stop_reason": null, + "returncode": 0, + "eligible_for_selection": true, + "validation_score": 0.07579071136290132, + "result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-317/output/result.json", + "result_sha256": "43a26ba6446b6cd8e311db724d9310d196b6b6147ada22f61f21ca0afb8ba9e7", + "model_sha256": "be5afb20e3ef1a8d10cfd804c270299f560a30c72d9d14acf19e48e45f3df718" + }, + { + "index": 318, + "config_index": 6, + "guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-318/guard-result.json", + "guard_sha256": "5d6759182ed22a8d9b7b5acada033215c353a9d77d1e6e3a8947949d4143c0fe", + "status": "PASS", + "stop_reason": null, + "returncode": 0, + "eligible_for_selection": true, + "validation_score": 0.07558271091917644, + "result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-318/output/result.json", + "result_sha256": "6721769d5a5e9ed25074f601e3471dfb277d7656e0c6b4140dc6dca1d8458795", + "model_sha256": "97a1424b5e8c2987cbe0507c1e6401f2e27d6cb32ef618da8630564a3c45d546" + }, + { + "index": 319, + "config_index": 7, + "guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-319/guard-result.json", + "guard_sha256": "5b9f9f4b9e1db4b7f65c2852a0928b706fa3c01a4cce8910c3db753e16469a1e", + "status": "PASS", + "stop_reason": null, + "returncode": 0, + "eligible_for_selection": true, + "validation_score": 0.07688896201981775, + "result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-319/output/result.json", + "result_sha256": "de746031e96dd71864e182afed7ee5ecccfe88cfb3f840d1923332f4bc554338", + "model_sha256": "556d30ee92e63012b8c11bff367a8a461cd5fbc87b1a887f030a4d46cdba641b" + }, + { + "index": 320, + "config_index": 8, + "guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-320/guard-result.json", + "guard_sha256": "fbbf9def5e8d1886e5a091ce9cc5e2f6f6f4a6b86d0169f90deded0da15d1df3", + "status": "PASS", + "stop_reason": null, + "returncode": 0, + "eligible_for_selection": true, + "validation_score": 0.07705623996408809, + "result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-320/output/result.json", + "result_sha256": "83c0721e3ede6ae19b14c2528bf1198c2c894ac94ba9415b6449cd5dd58c08e9", + "model_sha256": "11f606c0e06355587372356497b6782453eded2db2ce2a31758eb679eaf745b4" + }, + { + "index": 321, + "config_index": 9, + "guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-321/guard-result.json", + "guard_sha256": "5d350c5b56b630e063126d91004988634e223a2461ec3ffa126ff519ba6ba27b", + "status": "PASS", + "stop_reason": null, + "returncode": 0, + "eligible_for_selection": true, + "validation_score": 0.07589428974158799, + "result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-321/output/result.json", + "result_sha256": "910c5fb1179c062c44a65c3fbb3c2e788da6f3fe293dc5263dd05787c1b7ada7", + "model_sha256": "1fe6260455fbc5cf6f2ccb13ee544b0faabef49921c6875709fed5fdda294831" + }, + { + "index": 322, + "config_index": 10, + "guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-322/guard-result.json", + "guard_sha256": "bf13b9b0f21eb4fd34ade94451bcd1af3bfb4824ac1f9cfc0cb14b63178ed065", + "status": "PASS", + "stop_reason": null, + "returncode": 0, + "eligible_for_selection": true, + "validation_score": 0.0764765430827128, + "result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-322/output/result.json", + "result_sha256": "d1be4c6a6ff371c3ec40fe6643f5319452ff13a28a6c61dfaf6dd1d40dd5c4bd", + "model_sha256": "fb9fe1fd9ab604027c7c518f6c374eee0f634cdbec689d273c25a4ec07e304c4" + }, + { + "index": 323, + "config_index": 11, + "guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-323/guard-result.json", + "guard_sha256": "eb4a85962d90f08b05f94b6167f256b387b6cf4051d349c6a5e8d4cf05acac98", + "status": "STOPPED_OR_FAILED", + "stop_reason": "process_count_limit", + "returncode": -9, + "eligible_for_selection": false + } + ], + "actual_selection_exists": false, + "no_selection_written_or_changed": true +} diff --git a/docs/s3a-final-process-policy-2026-11-15.md b/docs/s3a-final-process-policy-2026-11-15.md new file mode 100644 index 000000000..0f46959dd --- /dev/null +++ b/docs/s3a-final-process-policy-2026-11-15.md @@ -0,0 +1,11 @@ +# 最终拟合进程数边界:执行Claude1794738923裁量 + +cell26(fold4/p50/R2_E4)的12个HPO均终态:312–322的11项PASS,完整result/model hash逐一通过;323/config11因process_count_limit结束,returncode−9、观测2进程,子进程身份未知,无数值结果。保留原回执,按原失败配置规则为不可选/NA,不把整个cell判NA、不重试。 + +Claude1794738923明确区分最终拟合:若30个final中任一触发process_count_limit,须先向Claude报告并另行裁量,才决定整个cell是否NA;“一次恢复可能合理”不是预先的恢复授权。 + +为避免旧自动等待器直接接受这种最终NA,已只退役尚未执行且无子进程的81572/PID465797;退役时WAIT_EXISTING_RECOVERY,预测尚未开始,未向恢复7921/PID464772或其worker发信号。原等待源码、原回执、原postfit源与原worker全部保留不改。 + +唯一替代等待器使用新文件continue_s3a_final_policy_postfit.py。等待期间检查final守卫,完整终态后、checkpoint固定前、预测前、评分前重复拦截这一明确条件。命中后写awaiting-final-infra-ruling.json并停止后处理;仍保留原队列可能写出的NA原始字段,但不采纳为科学cell排除决定,不自行恢复。其它HPO/final条件沿原裁量。四项测试验证HPO不误挡、final必须等待且不授权恢复、其它失败不重分类、改spec/重复回执拒绝。 + +科学合同、已有fits、累计20h/2hpostfit预算和统计口径均不变。新等待器只替代已退役等待器,不增加训练任务。当前执行状态以新started/progress及原恢复回执为准。 diff --git a/tools/continue_s3a_final_policy_postfit.py b/tools/continue_s3a_final_policy_postfit.py new file mode 100644 index 000000000..4ee91a68e --- /dev/null +++ b/tools/continue_s3a_final_policy_postfit.py @@ -0,0 +1,144 @@ +"""Final-policy replacement for the retired idle waiter; original fit/postfit sources unchanged.""" +import argparse +import fcntl +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import time +import psutil +from freeze_g1_components import sha +from s2_custody_common import read_json,checked +from s2_recovery_supervisor import save +from s3a_recovery_common import state +from s3a_postfit import validate_terminal +from s3a_final_process_policy import guard_final_policy + + +REPO=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8') +BRANCH='refs/heads/codex/s2-execution-diff-20261114' + + +def freeze_files(paths,message): + # This checkout is the existing publication checkout, never the unborn root .git. + status=subprocess.check_output(['git','status','--porcelain','--untracked-files=no'],cwd=REPO,text=True) + if status.strip():raise RuntimeError('Publication checkout has unrelated tracked changes') + for relative in paths: + p=Path(relative);dest=REPO/p + if p.is_dir():shutil.copytree(p,dest,dirs_exist_ok=True) + else:dest.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(p,dest) + subprocess.run(['git','add','-f','--',*[str(p) for p in paths]],cwd=REPO,check=True) + subprocess.run(['git','diff','--cached','--check','--','.',':(exclude)*.log'],cwd=REPO,check=True) + subprocess.run(['git','-c','gc.auto=0','commit','-q','-m',message],cwd=REPO,check=True) + return subprocess.check_output(['git','rev-parse','HEAD'],cwd=REPO,text=True).strip() + + +def run(root,chain): + chain.mkdir(exist_ok=True) + with (chain/'lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + if (chain/'started.json').exists():raise FileExistsError('Postfit chain already attempted') + c,scope,old,passed=state(root) + policy=read_json(chain/'policy-code-freeze.json') + if not policy.get('git_commit'):raise ValueError('Final policy must be Git fixed') + for p,h in policy['source_sha256'].items():checked(p,h) + retired=read_json(Path('artifacts/s3a-postfit-chain-2026-11-15/retirement-for-final-policy.json')) + if not retired['exit_verified'] or psutil.pid_exists(retired['retired_pid']):raise RuntimeError('Old waiter still active') + code=read_json(root/'postfit-code-freeze.json') + for p,h in code['source_sha256'].items():checked(p,h) + save(chain/'started.json',dict(unix=time.time(),pid=os.getpid(),root=str(root),postfit_code_freeze=code,policy_code_freeze=policy, + no_fits_launched_by_this_chain=True,existing_recovery_only=True)) + def mark(stage,**extra):save(chain/'progress.json',dict(stage=stage,unix=time.time(),**extra)) + stage='WAIT_EXISTING_RECOVERY' + try: + mark(stage) + while not (root/'batch-complete.json').exists(): + guard_final_policy(root,chain,c) + time.sleep(5) + guard_final_policy(root,chain,c) + batch=read_json(root/'batch-complete.json') + if batch['status']!='ALL_S3A_FITS_TERMINAL':raise RuntimeError('Recovery stopped incomplete; no retry or prediction') + frozen=read_json(root/'terminal-freeze.json');validate_terminal(batch,frozen) + start=read_json(root/'batch-started.json') + # A finished receipt can precede the coordinator's last exit by a moment. + for _ in range(24): + if not psutil.pid_exists(start['pid']):break + p=psutil.Process(start['pid']) + if str(root) not in p.cmdline():raise RuntimeError('Coordinator PID reused') + time.sleep(5) + else:raise RuntimeError('Coordinator still alive after terminal receipt') + for p in psutil.process_iter(['pid','cmdline']): + cmd=p.info['cmdline'] or [] + if str(root) in cmd and any(Path(v).name=='run_s3a_fit.py' for v in cmd): + raise RuntimeError('Fit worker remains alive') + guard_final_policy(root,chain,c) + stage='FREEZE_TERMINAL_MODELS';mark(stage) + terminal_commit=freeze_files([str(root),scope['original_root']], + 'Freeze complete two-source S3-A fits before test predictions') + save(root/'terminal-git-freeze.json',dict(git_commit=terminal_commit, + terminal_freeze_sha256=sha(root/'terminal-freeze.json'),before_predictions=True, + remote_publication_separate=True)) + guard_final_policy(root,chain,c) + stage='PREDICT_ONCE';mark(stage,terminal_commit=terminal_commit) + env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1', + NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='') + with (chain/'predict-launch.log').open('x') as log: + subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'predict'],env=env, + stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920) + if read_json(root/'postfit-predict-guard.json')['worker']['status']!='PASS': + raise RuntimeError('Prediction guard failed; no retry/scoring') + written=read_json(root/'predictions-written.json') + if written['status']!='ALL_S3A_PREDICTIONS_WRITTEN':raise RuntimeError('Predictions not complete') + stage='FREEZE_PREDICTIONS';mark(stage) + prediction_commit=freeze_files([str(root)],'Freeze S3-A predictions and exact manifest before scoring') + save(root/'predictions-complete.json',dict(status='ALL_S3A_PREDICTIONS_FROZEN',git_commit=prediction_commit, + contract_sha256=sha(root/'contract.json'),terminal_freeze_sha256=sha(root/'terminal-freeze.json'), + prediction_index_sha256=sha(root/'prediction-index.json'),prediction_files=written['prediction_files'], + no_new_label_decode=True,remote_publication_separate=True)) + guard_final_policy(root,chain,c) + stage='SCORE_ONCE';mark(stage,prediction_commit=prediction_commit) + with (chain/'score-launch.log').open('x') as log: + subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'score'],env=env, + stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920) + guard=read_json(root/'postfit-score-guard.json') + if guard['worker']['status']!='PASS':raise RuntimeError('Score guard failed; no retry') + result=read_json(root/'scores/result.json') + if result['status']!='S3A_SCORED':raise RuntimeError('Score result missing') + for name,digest in result['output_sha256'].items():checked(root/'scores'/name,digest) + save(chain/'scientific-complete.json',dict(status='S3A_SCORED',root=str(root), + terminal_commit=terminal_commit,prediction_commit=prediction_commit, + result_sha256=sha(root/'scores/result.json'),guard=guard)) + stage='FREEZE_RESULTS';mark(stage) + result_commit=freeze_files([str(root),str(chain)],'Preserve complete S3-A adaptive follow-up results and cumulative accounting') + stage='WAIT_EXISTING_PUBLICATION';mark(stage,result_commit=result_commit) + publication=Path('artifacts/publication-after-naive-2026-11-15/status.json') + deadline=time.monotonic()+10800 + while True: + pub=read_json(publication) + if pub['status']=='PUBLICATION_COMPLETE':break + if pub['status']=='PUBLICATION_STOPPED':raise RuntimeError('Earlier publication stopped; preserve results locally') + if time.monotonic()>deadline:raise TimeoutError('Earlier publication still pending') + time.sleep(5) + remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0] + if remote!=policy['remote_base_for_results']:raise RuntimeError('Concurrent remote branch change; do not overwrite') + stage='PUBLISH_RESULTS';mark(stage,result_commit=result_commit) + with (chain/'publication.log').open('x') as log: + subprocess.run(['git','-c','pack.window=0','-c','pack.threads=4','-c','core.compression=1', + 'push','origin',result_commit+':'+BRANCH],cwd=REPO,stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10800) + remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0] + if remote!=result_commit:raise RuntimeError('Publication readback differs') + save(chain/'complete.json',dict(status='S3A_SCIENCE_AND_PUBLICATION_COMPLETE',result_commit=result_commit, + result_sha256=sha(root/'scores/result.json'),cumulative_worker_CPU_s=guard['cumulative_worker_CPU_s'])) + mark('COMPLETE',result_commit=result_commit) + return 0 + except Exception as exc: + save(chain/'stopped.json',dict(stage=stage,error=repr(exc),unix=time.time(),no_automatic_retry=True)) + mark('STOPPED',failed_stage=stage,error=repr(exc)) + raise + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('chain',type=Path);a=p.parse_args() + raise SystemExit(run(a.root,a.chain)) diff --git a/tools/s3a_final_process_policy.py b/tools/s3a_final_process_policy.py new file mode 100644 index 000000000..25c978a37 --- /dev/null +++ b/tools/s3a_final_process_policy.py @@ -0,0 +1,34 @@ +"""Claude1794738923: final process-count stops require a separate decision.""" +from pathlib import Path +from freeze_g1_components import sha +from s2_custody_common import read_json +from s2_recovery_supervisor import save + + +def pending_finals(contract, receipts): + pending=[] + seen=set() + for row in receipts: + job=row['job'];i=job['index'] + if i in seen or job!=contract['jobs'][i]:raise ValueError('Receipt duplicate or changed specification') + seen.add(i) + if job['kind']=='FINAL' and row['worker']['stop_reason']=='process_count_limit': + pending.append(dict(index=i,cell=job['cell'],arm=job['arm'],fold=job['fold'],pool=job['pool'], + worker=row['worker'],classification='FINAL_INFRASTRUCTURE_REVIEW_PENDING', + scientific_cell_NA_adopted=False,recovery_authorized=False)) + return pending + + +def guard_final_policy(root,chain,contract): + receipts=[];sources={} + for job in contract['jobs']: + if job['kind']!='FINAL':continue + p=Path(root)/'jobs'/f"job-{job['index']:03d}"/'guard-result.json' + if p.exists():receipts.append(read_json(p));sources[str(p)]=sha(p) + pending=pending_finals(contract,receipts) + if pending: + save(Path(chain)/'awaiting-final-infra-ruling.json',dict(ruling='1794738923.689619',pending=pending, + receipt_sha256=sources,original_raw_receipts_unchanged=True, + raw_batch_NA_is_not_an_adopted_scientific_NA=True,predictions_and_scoring_blocked=True, + no_fit_restart=True,no_recovery_without_further_explicit_ruling=True)) + raise RuntimeError('FINAL_PROCESS_COUNT_REQUIRES_CLAUDE_RULING_BEFORE_PREDICTION_OR_SCORING') diff --git a/tools/test_s3a_final_process_policy.py b/tools/test_s3a_final_process_policy.py new file mode 100644 index 000000000..5f04b3ff9 --- /dev/null +++ b/tools/test_s3a_final_process_policy.py @@ -0,0 +1,32 @@ +import unittest +from s3a_final_process_policy import pending_finals + + +def fixtures(index,reason): + jobs=[dict(index=i,kind='HPO' if i<360 else 'FINAL',cell=i-360 if i>=360 else i//12, + arm='R2_E4',fold=4,pool=50) for i in range(390)] + row=dict(job=jobs[index],worker=dict(status='STOPPED_OR_FAILED',stop_reason=reason,returncode=-9)) + return dict(jobs=jobs),row + + +class Policy(unittest.TestCase): + def test_hpo323_keeps_existing_failed_config_rule(self): + c,r=fixtures(323,'process_count_limit');self.assertEqual(pending_finals(c,[r]),[]) + + def test_final_process_count_holds_not_recovery(self): + c,r=fixtures(386,'process_count_limit');p=pending_finals(c,[r]) + self.assertEqual(p[0]['cell'],26) + self.assertFalse(p[0]['scientific_cell_NA_adopted']) + self.assertFalse(p[0]['recovery_authorized']) + + def test_unrelated_final_failure_not_reclassified(self): + c,r=fixtures(386,'wall_limit');self.assertEqual(pending_finals(c,[r]),[]) + + def test_changed_or_duplicate_spec_refused(self): + c,r=fixtures(386,'process_count_limit') + with self.assertRaises(ValueError):pending_finals(c,[r,r]) + r=dict(r,job=dict(r['job'],cell=99)) + with self.assertRaises(ValueError):pending_finals(c,[r]) + + +if __name__=='__main__':unittest.main()