Gate final process-count stops on Claude ruling without changing fit workers
This commit is contained in:
1 parent
df56d52997
commit
cb78658944
7 files changed
+433
No files matched your search
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"unix": 1794739122.4651701,
|
||||
"retired_pid": 465797,
|
||||
"cmdline": [
|
||||
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
||||
"-u",
|
||||
"tools/continue_s3a_postfit.py",
|
||||
"artifacts/s3a-recovery-2026-11-15",
|
||||
"artifacts/s3a-postfit-chain-2026-11-15"
|
||||
],
|
||||
"prior_progress": {
|
||||
"stage": "WAIT_EXISTING_RECOVERY",
|
||||
"unix": 1794738262.3016055
|
||||
},
|
||||
"reason": "Claude1794738923 requires final-process-count review before accepting whole-cell NA; replace idle waiter only",
|
||||
"no_children": true,
|
||||
"prediction_not_started": true,
|
||||
"recovery_PID464772_not_signalled": true,
|
||||
"exit_verified": true
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"tests": 4,
|
||||
"ruling": "1794738923.689619",
|
||||
"old_waiter_retired": "artifacts/s3a-postfit-chain-2026-11-15/retirement-for-final-policy.json",
|
||||
"scope": "Only final process_count_limit blocks adoption of whole-cell NA and predictions/scoring; HPO policy unchanged; no recovery automatically authorized"
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
{
|
||||
"unix": 1794739015.6367815,
|
||||
"cell": 26,
|
||||
"arm": "R2_E4",
|
||||
"fold": 4,
|
||||
"pool": 50,
|
||||
"terminal_HPO": 12,
|
||||
"usable_finished_configurations": 11,
|
||||
"unavailable_configuration": {
|
||||
"job": 323,
|
||||
"config_index": 11,
|
||||
"reason": "process_count_limit",
|
||||
"numeric_failure": false,
|
||||
"child_identity": "unknown",
|
||||
"no_retry": true
|
||||
},
|
||||
"trials": [
|
||||
{
|
||||
"index": 312,
|
||||
"config_index": 0,
|
||||
"guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-312/guard-result.json",
|
||||
"guard_sha256": "6b0d67446f079a53fc6aa7bcb3b8700301e57009208ce688e822ff09ba3398e3",
|
||||
"status": "PASS",
|
||||
"stop_reason": null,
|
||||
"returncode": 0,
|
||||
"eligible_for_selection": true,
|
||||
"validation_score": 0.07428104382661917,
|
||||
"result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-312/output/result.json",
|
||||
"result_sha256": "c0959b8a3d4b6c8b74c6f7a469d2a29a032024286d861a9b44adc625ec425fc2",
|
||||
"model_sha256": "d4b6025a8641aa15ca2cb32f4896e9963fd443f884e55054bf0bbf760e4485f6"
|
||||
},
|
||||
{
|
||||
"index": 313,
|
||||
"config_index": 1,
|
||||
"guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-313/guard-result.json",
|
||||
"guard_sha256": "4bafc13be24b164ff04c35bdb3ec6af8d0a2dac697daec77e059da569ef50ad1",
|
||||
"status": "PASS",
|
||||
"stop_reason": null,
|
||||
"returncode": 0,
|
||||
"eligible_for_selection": true,
|
||||
"validation_score": 0.07561749254650058,
|
||||
"result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-313/output/result.json",
|
||||
"result_sha256": "db53b2f3495b4a8a416215fed910dd3a2418c41fd433aff6ea358423024eda6c",
|
||||
"model_sha256": "3fe662cb5d712eae1fa0da9640edd8cf739c41652f4a11af3b37a67e380ecaae"
|
||||
},
|
||||
{
|
||||
"index": 314,
|
||||
"config_index": 2,
|
||||
"guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-314/guard-result.json",
|
||||
"guard_sha256": "64c0b54c7adad1a19fca2ad14906c9d16e5fde2f7f4307e75f9486e7f8b767dc",
|
||||
"status": "PASS",
|
||||
"stop_reason": null,
|
||||
"returncode": 0,
|
||||
"eligible_for_selection": true,
|
||||
"validation_score": 0.07641824353110742,
|
||||
"result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-314/output/result.json",
|
||||
"result_sha256": "13cf434440e66995f7294f7a1b1ff56b0fada8f1a40566ad71d24fd6a0dee604",
|
||||
"model_sha256": "ab370bbe3f47f69bef5d44141abfcee8e157cee1d8fe3531ea105769be7dfab5"
|
||||
},
|
||||
{
|
||||
"index": 315,
|
||||
"config_index": 3,
|
||||
"guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-315/guard-result.json",
|
||||
"guard_sha256": "9e6b4a5f493619b7eaa2c0d30c30f8818afee6c0e4a6e291ba68ac24e2024dd5",
|
||||
"status": "PASS",
|
||||
"stop_reason": null,
|
||||
"returncode": 0,
|
||||
"eligible_for_selection": true,
|
||||
"validation_score": 0.07429812958903101,
|
||||
"result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-315/output/result.json",
|
||||
"result_sha256": "87bf28e13b813422889b3a64b930f249faf31da04fccce81efb9588f1294c08c",
|
||||
"model_sha256": "d3f3fc6d0aa068d397bf4284634cccf0abdd99f587333b71d2a0cb7a9263d4a8"
|
||||
},
|
||||
{
|
||||
"index": 316,
|
||||
"config_index": 4,
|
||||
"guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-316/guard-result.json",
|
||||
"guard_sha256": "728c586480af55a35b4887a01bd362c251895039eb640572646d63303500f0ef",
|
||||
"status": "PASS",
|
||||
"stop_reason": null,
|
||||
"returncode": 0,
|
||||
"eligible_for_selection": true,
|
||||
"validation_score": 0.07518402096943289,
|
||||
"result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-316/output/result.json",
|
||||
"result_sha256": "2e877ec07958060682b7e09467517e6febef30a7198303d1a1c0d375a2520d08",
|
||||
"model_sha256": "7022615bf7eafd04ce189fc5e997c13cdc4482f6ee4adcb16d37a1e4649352b7"
|
||||
},
|
||||
{
|
||||
"index": 317,
|
||||
"config_index": 5,
|
||||
"guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-317/guard-result.json",
|
||||
"guard_sha256": "c33ec446c3d0096aea2d5c49bc208e3d75b1cb25ebee1a90be8038e29243a4f5",
|
||||
"status": "PASS",
|
||||
"stop_reason": null,
|
||||
"returncode": 0,
|
||||
"eligible_for_selection": true,
|
||||
"validation_score": 0.07579071136290132,
|
||||
"result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-317/output/result.json",
|
||||
"result_sha256": "43a26ba6446b6cd8e311db724d9310d196b6b6147ada22f61f21ca0afb8ba9e7",
|
||||
"model_sha256": "be5afb20e3ef1a8d10cfd804c270299f560a30c72d9d14acf19e48e45f3df718"
|
||||
},
|
||||
{
|
||||
"index": 318,
|
||||
"config_index": 6,
|
||||
"guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-318/guard-result.json",
|
||||
"guard_sha256": "5d6759182ed22a8d9b7b5acada033215c353a9d77d1e6e3a8947949d4143c0fe",
|
||||
"status": "PASS",
|
||||
"stop_reason": null,
|
||||
"returncode": 0,
|
||||
"eligible_for_selection": true,
|
||||
"validation_score": 0.07558271091917644,
|
||||
"result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-318/output/result.json",
|
||||
"result_sha256": "6721769d5a5e9ed25074f601e3471dfb277d7656e0c6b4140dc6dca1d8458795",
|
||||
"model_sha256": "97a1424b5e8c2987cbe0507c1e6401f2e27d6cb32ef618da8630564a3c45d546"
|
||||
},
|
||||
{
|
||||
"index": 319,
|
||||
"config_index": 7,
|
||||
"guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-319/guard-result.json",
|
||||
"guard_sha256": "5b9f9f4b9e1db4b7f65c2852a0928b706fa3c01a4cce8910c3db753e16469a1e",
|
||||
"status": "PASS",
|
||||
"stop_reason": null,
|
||||
"returncode": 0,
|
||||
"eligible_for_selection": true,
|
||||
"validation_score": 0.07688896201981775,
|
||||
"result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-319/output/result.json",
|
||||
"result_sha256": "de746031e96dd71864e182afed7ee5ecccfe88cfb3f840d1923332f4bc554338",
|
||||
"model_sha256": "556d30ee92e63012b8c11bff367a8a461cd5fbc87b1a887f030a4d46cdba641b"
|
||||
},
|
||||
{
|
||||
"index": 320,
|
||||
"config_index": 8,
|
||||
"guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-320/guard-result.json",
|
||||
"guard_sha256": "fbbf9def5e8d1886e5a091ce9cc5e2f6f6f4a6b86d0169f90deded0da15d1df3",
|
||||
"status": "PASS",
|
||||
"stop_reason": null,
|
||||
"returncode": 0,
|
||||
"eligible_for_selection": true,
|
||||
"validation_score": 0.07705623996408809,
|
||||
"result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-320/output/result.json",
|
||||
"result_sha256": "83c0721e3ede6ae19b14c2528bf1198c2c894ac94ba9415b6449cd5dd58c08e9",
|
||||
"model_sha256": "11f606c0e06355587372356497b6782453eded2db2ce2a31758eb679eaf745b4"
|
||||
},
|
||||
{
|
||||
"index": 321,
|
||||
"config_index": 9,
|
||||
"guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-321/guard-result.json",
|
||||
"guard_sha256": "5d350c5b56b630e063126d91004988634e223a2461ec3ffa126ff519ba6ba27b",
|
||||
"status": "PASS",
|
||||
"stop_reason": null,
|
||||
"returncode": 0,
|
||||
"eligible_for_selection": true,
|
||||
"validation_score": 0.07589428974158799,
|
||||
"result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-321/output/result.json",
|
||||
"result_sha256": "910c5fb1179c062c44a65c3fbb3c2e788da6f3fe293dc5263dd05787c1b7ada7",
|
||||
"model_sha256": "1fe6260455fbc5cf6f2ccb13ee544b0faabef49921c6875709fed5fdda294831"
|
||||
},
|
||||
{
|
||||
"index": 322,
|
||||
"config_index": 10,
|
||||
"guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-322/guard-result.json",
|
||||
"guard_sha256": "bf13b9b0f21eb4fd34ade94451bcd1af3bfb4824ac1f9cfc0cb14b63178ed065",
|
||||
"status": "PASS",
|
||||
"stop_reason": null,
|
||||
"returncode": 0,
|
||||
"eligible_for_selection": true,
|
||||
"validation_score": 0.0764765430827128,
|
||||
"result_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-322/output/result.json",
|
||||
"result_sha256": "d1be4c6a6ff371c3ec40fe6643f5319452ff13a28a6c61dfaf6dd1d40dd5c4bd",
|
||||
"model_sha256": "fb9fe1fd9ab604027c7c518f6c374eee0f634cdbec689d273c25a4ec07e304c4"
|
||||
},
|
||||
{
|
||||
"index": 323,
|
||||
"config_index": 11,
|
||||
"guard_path": "artifacts/s3a-recovery-2026-11-15/jobs/job-323/guard-result.json",
|
||||
"guard_sha256": "eb4a85962d90f08b05f94b6167f256b387b6cf4051d349c6a5e8d4cf05acac98",
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"stop_reason": "process_count_limit",
|
||||
"returncode": -9,
|
||||
"eligible_for_selection": false
|
||||
}
|
||||
],
|
||||
"actual_selection_exists": false,
|
||||
"no_selection_written_or_changed": true
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
# 最终拟合进程数边界:执行Claude1794738923裁量
|
||||
|
||||
cell26(fold4/p50/R2_E4)的12个HPO均终态:312–322的11项PASS,完整result/model hash逐一通过;323/config11因process_count_limit结束,returncode−9、观测2进程,子进程身份未知,无数值结果。保留原回执,按原失败配置规则为不可选/NA,不把整个cell判NA、不重试。
|
||||
|
||||
Claude1794738923明确区分最终拟合:若30个final中任一触发process_count_limit,须先向Claude报告并另行裁量,才决定整个cell是否NA;“一次恢复可能合理”不是预先的恢复授权。
|
||||
|
||||
为避免旧自动等待器直接接受这种最终NA,已只退役尚未执行且无子进程的81572/PID465797;退役时WAIT_EXISTING_RECOVERY,预测尚未开始,未向恢复7921/PID464772或其worker发信号。原等待源码、原回执、原postfit源与原worker全部保留不改。
|
||||
|
||||
唯一替代等待器使用新文件continue_s3a_final_policy_postfit.py。等待期间检查final守卫,完整终态后、checkpoint固定前、预测前、评分前重复拦截这一明确条件。命中后写awaiting-final-infra-ruling.json并停止后处理;仍保留原队列可能写出的NA原始字段,但不采纳为科学cell排除决定,不自行恢复。其它HPO/final条件沿原裁量。四项测试验证HPO不误挡、final必须等待且不授权恢复、其它失败不重分类、改spec/重复回执拒绝。
|
||||
|
||||
科学合同、已有fits、累计20h/2hpostfit预算和统计口径均不变。新等待器只替代已退役等待器,不增加训练任务。当前执行状态以新started/progress及原恢复回执为准。
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Final-policy replacement for the retired idle waiter; original fit/postfit sources unchanged."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import psutil
|
||||
from freeze_g1_components import sha
|
||||
from s2_custody_common import read_json,checked
|
||||
from s2_recovery_supervisor import save
|
||||
from s3a_recovery_common import state
|
||||
from s3a_postfit import validate_terminal
|
||||
from s3a_final_process_policy import guard_final_policy
|
||||
|
||||
|
||||
REPO=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8')
|
||||
BRANCH='refs/heads/codex/s2-execution-diff-20261114'
|
||||
|
||||
|
||||
def freeze_files(paths,message):
|
||||
# This checkout is the existing publication checkout, never the unborn root .git.
|
||||
status=subprocess.check_output(['git','status','--porcelain','--untracked-files=no'],cwd=REPO,text=True)
|
||||
if status.strip():raise RuntimeError('Publication checkout has unrelated tracked changes')
|
||||
for relative in paths:
|
||||
p=Path(relative);dest=REPO/p
|
||||
if p.is_dir():shutil.copytree(p,dest,dirs_exist_ok=True)
|
||||
else:dest.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(p,dest)
|
||||
subprocess.run(['git','add','-f','--',*[str(p) for p in paths]],cwd=REPO,check=True)
|
||||
subprocess.run(['git','diff','--cached','--check','--','.',':(exclude)*.log'],cwd=REPO,check=True)
|
||||
subprocess.run(['git','-c','gc.auto=0','commit','-q','-m',message],cwd=REPO,check=True)
|
||||
return subprocess.check_output(['git','rev-parse','HEAD'],cwd=REPO,text=True).strip()
|
||||
|
||||
|
||||
def run(root,chain):
|
||||
chain.mkdir(exist_ok=True)
|
||||
with (chain/'lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (chain/'started.json').exists():raise FileExistsError('Postfit chain already attempted')
|
||||
c,scope,old,passed=state(root)
|
||||
policy=read_json(chain/'policy-code-freeze.json')
|
||||
if not policy.get('git_commit'):raise ValueError('Final policy must be Git fixed')
|
||||
for p,h in policy['source_sha256'].items():checked(p,h)
|
||||
retired=read_json(Path('artifacts/s3a-postfit-chain-2026-11-15/retirement-for-final-policy.json'))
|
||||
if not retired['exit_verified'] or psutil.pid_exists(retired['retired_pid']):raise RuntimeError('Old waiter still active')
|
||||
code=read_json(root/'postfit-code-freeze.json')
|
||||
for p,h in code['source_sha256'].items():checked(p,h)
|
||||
save(chain/'started.json',dict(unix=time.time(),pid=os.getpid(),root=str(root),postfit_code_freeze=code,policy_code_freeze=policy,
|
||||
no_fits_launched_by_this_chain=True,existing_recovery_only=True))
|
||||
def mark(stage,**extra):save(chain/'progress.json',dict(stage=stage,unix=time.time(),**extra))
|
||||
stage='WAIT_EXISTING_RECOVERY'
|
||||
try:
|
||||
mark(stage)
|
||||
while not (root/'batch-complete.json').exists():
|
||||
guard_final_policy(root,chain,c)
|
||||
time.sleep(5)
|
||||
guard_final_policy(root,chain,c)
|
||||
batch=read_json(root/'batch-complete.json')
|
||||
if batch['status']!='ALL_S3A_FITS_TERMINAL':raise RuntimeError('Recovery stopped incomplete; no retry or prediction')
|
||||
frozen=read_json(root/'terminal-freeze.json');validate_terminal(batch,frozen)
|
||||
start=read_json(root/'batch-started.json')
|
||||
# A finished receipt can precede the coordinator's last exit by a moment.
|
||||
for _ in range(24):
|
||||
if not psutil.pid_exists(start['pid']):break
|
||||
p=psutil.Process(start['pid'])
|
||||
if str(root) not in p.cmdline():raise RuntimeError('Coordinator PID reused')
|
||||
time.sleep(5)
|
||||
else:raise RuntimeError('Coordinator still alive after terminal receipt')
|
||||
for p in psutil.process_iter(['pid','cmdline']):
|
||||
cmd=p.info['cmdline'] or []
|
||||
if str(root) in cmd and any(Path(v).name=='run_s3a_fit.py' for v in cmd):
|
||||
raise RuntimeError('Fit worker remains alive')
|
||||
guard_final_policy(root,chain,c)
|
||||
stage='FREEZE_TERMINAL_MODELS';mark(stage)
|
||||
terminal_commit=freeze_files([str(root),scope['original_root']],
|
||||
'Freeze complete two-source S3-A fits before test predictions')
|
||||
save(root/'terminal-git-freeze.json',dict(git_commit=terminal_commit,
|
||||
terminal_freeze_sha256=sha(root/'terminal-freeze.json'),before_predictions=True,
|
||||
remote_publication_separate=True))
|
||||
guard_final_policy(root,chain,c)
|
||||
stage='PREDICT_ONCE';mark(stage,terminal_commit=terminal_commit)
|
||||
env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',
|
||||
NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
||||
with (chain/'predict-launch.log').open('x') as log:
|
||||
subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'predict'],env=env,
|
||||
stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920)
|
||||
if read_json(root/'postfit-predict-guard.json')['worker']['status']!='PASS':
|
||||
raise RuntimeError('Prediction guard failed; no retry/scoring')
|
||||
written=read_json(root/'predictions-written.json')
|
||||
if written['status']!='ALL_S3A_PREDICTIONS_WRITTEN':raise RuntimeError('Predictions not complete')
|
||||
stage='FREEZE_PREDICTIONS';mark(stage)
|
||||
prediction_commit=freeze_files([str(root)],'Freeze S3-A predictions and exact manifest before scoring')
|
||||
save(root/'predictions-complete.json',dict(status='ALL_S3A_PREDICTIONS_FROZEN',git_commit=prediction_commit,
|
||||
contract_sha256=sha(root/'contract.json'),terminal_freeze_sha256=sha(root/'terminal-freeze.json'),
|
||||
prediction_index_sha256=sha(root/'prediction-index.json'),prediction_files=written['prediction_files'],
|
||||
no_new_label_decode=True,remote_publication_separate=True))
|
||||
guard_final_policy(root,chain,c)
|
||||
stage='SCORE_ONCE';mark(stage,prediction_commit=prediction_commit)
|
||||
with (chain/'score-launch.log').open('x') as log:
|
||||
subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'score'],env=env,
|
||||
stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920)
|
||||
guard=read_json(root/'postfit-score-guard.json')
|
||||
if guard['worker']['status']!='PASS':raise RuntimeError('Score guard failed; no retry')
|
||||
result=read_json(root/'scores/result.json')
|
||||
if result['status']!='S3A_SCORED':raise RuntimeError('Score result missing')
|
||||
for name,digest in result['output_sha256'].items():checked(root/'scores'/name,digest)
|
||||
save(chain/'scientific-complete.json',dict(status='S3A_SCORED',root=str(root),
|
||||
terminal_commit=terminal_commit,prediction_commit=prediction_commit,
|
||||
result_sha256=sha(root/'scores/result.json'),guard=guard))
|
||||
stage='FREEZE_RESULTS';mark(stage)
|
||||
result_commit=freeze_files([str(root),str(chain)],'Preserve complete S3-A adaptive follow-up results and cumulative accounting')
|
||||
stage='WAIT_EXISTING_PUBLICATION';mark(stage,result_commit=result_commit)
|
||||
publication=Path('artifacts/publication-after-naive-2026-11-15/status.json')
|
||||
deadline=time.monotonic()+10800
|
||||
while True:
|
||||
pub=read_json(publication)
|
||||
if pub['status']=='PUBLICATION_COMPLETE':break
|
||||
if pub['status']=='PUBLICATION_STOPPED':raise RuntimeError('Earlier publication stopped; preserve results locally')
|
||||
if time.monotonic()>deadline:raise TimeoutError('Earlier publication still pending')
|
||||
time.sleep(5)
|
||||
remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0]
|
||||
if remote!=policy['remote_base_for_results']:raise RuntimeError('Concurrent remote branch change; do not overwrite')
|
||||
stage='PUBLISH_RESULTS';mark(stage,result_commit=result_commit)
|
||||
with (chain/'publication.log').open('x') as log:
|
||||
subprocess.run(['git','-c','pack.window=0','-c','pack.threads=4','-c','core.compression=1',
|
||||
'push','origin',result_commit+':'+BRANCH],cwd=REPO,stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10800)
|
||||
remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0]
|
||||
if remote!=result_commit:raise RuntimeError('Publication readback differs')
|
||||
save(chain/'complete.json',dict(status='S3A_SCIENCE_AND_PUBLICATION_COMPLETE',result_commit=result_commit,
|
||||
result_sha256=sha(root/'scores/result.json'),cumulative_worker_CPU_s=guard['cumulative_worker_CPU_s']))
|
||||
mark('COMPLETE',result_commit=result_commit)
|
||||
return 0
|
||||
except Exception as exc:
|
||||
save(chain/'stopped.json',dict(stage=stage,error=repr(exc),unix=time.time(),no_automatic_retry=True))
|
||||
mark('STOPPED',failed_stage=stage,error=repr(exc))
|
||||
raise
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('chain',type=Path);a=p.parse_args()
|
||||
raise SystemExit(run(a.root,a.chain))
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Claude1794738923: final process-count stops require a separate decision."""
|
||||
from pathlib import Path
|
||||
from freeze_g1_components import sha
|
||||
from s2_custody_common import read_json
|
||||
from s2_recovery_supervisor import save
|
||||
|
||||
|
||||
def pending_finals(contract, receipts):
|
||||
pending=[]
|
||||
seen=set()
|
||||
for row in receipts:
|
||||
job=row['job'];i=job['index']
|
||||
if i in seen or job!=contract['jobs'][i]:raise ValueError('Receipt duplicate or changed specification')
|
||||
seen.add(i)
|
||||
if job['kind']=='FINAL' and row['worker']['stop_reason']=='process_count_limit':
|
||||
pending.append(dict(index=i,cell=job['cell'],arm=job['arm'],fold=job['fold'],pool=job['pool'],
|
||||
worker=row['worker'],classification='FINAL_INFRASTRUCTURE_REVIEW_PENDING',
|
||||
scientific_cell_NA_adopted=False,recovery_authorized=False))
|
||||
return pending
|
||||
|
||||
|
||||
def guard_final_policy(root,chain,contract):
|
||||
receipts=[];sources={}
|
||||
for job in contract['jobs']:
|
||||
if job['kind']!='FINAL':continue
|
||||
p=Path(root)/'jobs'/f"job-{job['index']:03d}"/'guard-result.json'
|
||||
if p.exists():receipts.append(read_json(p));sources[str(p)]=sha(p)
|
||||
pending=pending_finals(contract,receipts)
|
||||
if pending:
|
||||
save(Path(chain)/'awaiting-final-infra-ruling.json',dict(ruling='1794738923.689619',pending=pending,
|
||||
receipt_sha256=sources,original_raw_receipts_unchanged=True,
|
||||
raw_batch_NA_is_not_an_adopted_scientific_NA=True,predictions_and_scoring_blocked=True,
|
||||
no_fit_restart=True,no_recovery_without_further_explicit_ruling=True))
|
||||
raise RuntimeError('FINAL_PROCESS_COUNT_REQUIRES_CLAUDE_RULING_BEFORE_PREDICTION_OR_SCORING')
|
||||
@@ -0,0 +1,32 @@
|
||||
import unittest
|
||||
from s3a_final_process_policy import pending_finals
|
||||
|
||||
|
||||
def fixtures(index,reason):
|
||||
jobs=[dict(index=i,kind='HPO' if i<360 else 'FINAL',cell=i-360 if i>=360 else i//12,
|
||||
arm='R2_E4',fold=4,pool=50) for i in range(390)]
|
||||
row=dict(job=jobs[index],worker=dict(status='STOPPED_OR_FAILED',stop_reason=reason,returncode=-9))
|
||||
return dict(jobs=jobs),row
|
||||
|
||||
|
||||
class Policy(unittest.TestCase):
|
||||
def test_hpo323_keeps_existing_failed_config_rule(self):
|
||||
c,r=fixtures(323,'process_count_limit');self.assertEqual(pending_finals(c,[r]),[])
|
||||
|
||||
def test_final_process_count_holds_not_recovery(self):
|
||||
c,r=fixtures(386,'process_count_limit');p=pending_finals(c,[r])
|
||||
self.assertEqual(p[0]['cell'],26)
|
||||
self.assertFalse(p[0]['scientific_cell_NA_adopted'])
|
||||
self.assertFalse(p[0]['recovery_authorized'])
|
||||
|
||||
def test_unrelated_final_failure_not_reclassified(self):
|
||||
c,r=fixtures(386,'wall_limit');self.assertEqual(pending_finals(c,[r]),[])
|
||||
|
||||
def test_changed_or_duplicate_spec_refused(self):
|
||||
c,r=fixtures(386,'process_count_limit')
|
||||
with self.assertRaises(ValueError):pending_finals(c,[r,r])
|
||||
r=dict(r,job=dict(r['job'],cell=99))
|
||||
with self.assertRaises(ValueError):pending_finals(c,[r])
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in new issue
Block a user