Fix worker-exit guard race and pin authorized seven-part continuation

This commit is contained in:
Frank14f committed 2026-11-15 21:13:49 +08:00
1 parent a73e639702
commit 8dc4857c47
17 files changed
+1611

No files matched your search

File diff suppressed because it is too large. Load diff
@@ -0,0 +1,48 @@
{
"version": "s3b-v04-reference-cost-gates-v1",
"ruling": "1794745781.398039",
"clarifications": [
"1794745880.964439",
"1794747272.597049"
],
"ten_root": "artifacts/s3b-a-v04-continuation-2026-11-15",
"stage_guard_root": "artifacts/s3b-a-v04-stage-guard-v2-2026-11-15",
"additional_synthetic_CPU_s": 0.0,
"code_sha256": {
"tools/s3b_v04_gates.py": "fbe3a17c3b0016c998c6432848f83443c8580bb8f3ff65efe60f6ddc52580f51",
"tools/run_s3b_v04_gates.py": "806ecdf50d400325439477e736f8fd1dd844c45cfeddf70fe2a2ec8ab75aa7b8",
"tools/test_s3b_v04_gates.py": "93e12478ad444e4b7c566a1dc5ccfbb562b90da22c89c6002a11140fb1f24a25",
"tools/s3b_runtime.py": "f41057db3cbe38850ab65178c6b44be6b8fc3e97bf559769fbc17592a297971d",
"tools/naive_ordering_math.py": "8a71dfb139f58c523077e8e94d7cbe7ad34eb7680b6b2535bfcf7a94001ad86c",
"tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
"tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e"
},
"source_sha256": {
"artifacts/s3b-a-v04-ten-2026-11-15/contract.json": "c9d7c08f22c1cf77f2ac1d72b1a4602f06f614c246cae8eb3b7ad235b04ac4a3",
"artifacts/s3b-a-v04-ten-2026-11-15/code-freeze.json": "bb6bedab545ee83fb8883178ba61dc9291437244c09f52fe903fcaa08950b634",
"artifacts/s3b-a-v04-stage-guard-tests-2026-11-15/attempt-01/synthetic.receipt.json": "32c790eb3af0862594938842334cededbd6da6870745597d515c51d79469f8b4",
"artifacts/s3b-a-v04-gate-tests-2026-11-15/attempt-01/synthetic.receipt.json": "9c7ed5c1855522fde9010ba95e79dc61ea11eea51c3edd00c3c1e1223ae3b1b7",
"artifacts/s3b-a-v04-gate-tests-2026-11-15/attempt-02/synthetic.receipt.json": "2de6a160666b38348c905cdbbef17ab5fcc03ef5425c521997c4c825736470d4",
"artifacts/s3b-a-v04-continuation-2026-11-15/contract.json": "a39dc2947e6cc8dea3721a48a0a5bce777daec82f19ad6d2c83693e9e45a3a0f"
},
"stage_order": [
"wait existing ten and external guard terminal",
"three exact references",
"count-only all617 cost census",
"frozen max-rate gate"
],
"per_gate": {
"CPU_s": 1800,
"wall_s": 1800,
"RSS_bytes": 4294967296,
"workers": 1,
"CPU": 16
},
"no_labels": true,
"no_solver_switch": true,
"no_full617_launch": true,
"fixed_overhead_rule": "max over ten of guard CPU minus event CPU and positive-query CPU, times617; zero-query CPU stays fixed overhead",
"projection": "1.25*(max eventsCPU/nodes*full617nodes + max midpointCPU/positivequeries*full617queries) + fixed overhead + all settled worker CPU including gates +7200 <=72000; ten included conservatively but never rerun",
"engineering_ruling": "1794748036.544479",
"ledger_note": "All old/new synthetic costs, including failed attempt, are already in continuation ten ledger; do not add again"
}
Whitespace-only changes.
@@ -0,0 +1,10 @@
{
"cpu_seconds_charged": 0.22703799999999996,
"attempts": [
{
"name": "synthetic",
"status": "STOPPED_OR_FAILED",
"cpu_seconds": 0.22703799999999996
}
]
}
@@ -0,0 +1,32 @@
test_actual_worker_exit_before_parent_reap (__main__.RecoveryTests) ... ok
test_existing_spend_counts_against_stage_full_job_reservation (__main__.RecoveryTests) ... /home/frank14f/optane/MeltMind/tools/test_s3b_v04_continuation.py:59: ResourceWarning: unclosed file <_io.TextIOWrapper name='/tmp/tmpta77_p9s/batch.lock' mode='a' encoding='UTF-8'>
code=run(root)
ResourceWarning: Enable tracemalloc to get the object allocation traceback
ok
test_identity_with_completed_result_before_wait4_receipt (__main__.RecoveryTests) ... ok
test_live_expected_worker_and_known_gone (__main__.RecoveryTests) ... ok
test_only_seven_unattempted_are_dispatched (__main__.RecoveryTests) ... V04_TERMINAL method=splu index=3 status=PASS
V04_TERMINAL method=splu index=4 status=PASS
V04_TERMINAL method=splu index=5 status=PASS
V04_TERMINAL method=splu index=6 status=PASS
V04_TERMINAL method=splu index=7 status=PASS
V04_TERMINAL method=splu index=8 status=PASS
V04_TERMINAL method=splu index=9 status=PASS
/home/frank14f/optane/MeltMind/tools/test_s3b_v04_continuation.py:59: ResourceWarning: unclosed file <_io.TextIOWrapper name='/tmp/tmp5rhml_xc/batch.lock' mode='a' encoding='UTF-8'>
code=run(root)
ResourceWarning: Enable tracemalloc to get the object allocation traceback
ok
test_wait4_replaces_live_cpu_once (__main__.RecoveryTests) ... FAIL
======================================================================
FAIL: test_wait4_replaces_live_cpu_once (__main__.RecoveryTests)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/home/frank14f/optane/MeltMind/tools/test_s3b_v04_continuation.py", line 43, in test_wait4_replaces_live_cpu_once
self.assertEqual(stage_charge({'part-02':565.748434},{'part-02':565.54},3.893839),569.642273)
AssertionError: 569.6422729999999 != 569.642273
----------------------------------------------------------------------
Ran 6 tests in 0.077s
FAILED (failures=1)
@@ -0,0 +1,22 @@
{
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/test_s3b_v04_continuation.py"
],
"returncode": 1,
"stop_reason": null,
"cpu_seconds": 0.22703799999999996,
"wall_seconds": 0.24932852300116792,
"peak_tree_rss_bytes": 33337344,
"peak_process_count": 2,
"output_disk_bytes_before_receipt": 8192,
"max_poll_gap_seconds": 0.020950617967173457,
"poll_target_seconds": 0.02,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
}
@@ -0,0 +1,15 @@
{
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/test_s3b_v04_continuation.py"
],
"limits": {
"total_cpu_seconds": 60,
"job_cpu_seconds": 60,
"job_wall_seconds": 90,
"tree_rss_bytes": 1073741824,
"disk_bytes": 20971520,
"max_processes": 2
},
"started_unix": 1794748267.8378913
}
Whitespace-only changes.
@@ -0,0 +1,10 @@
{
"cpu_seconds_charged": 0.218599,
"attempts": [
{
"name": "synthetic",
"status": "PASS",
"cpu_seconds": 0.218599
}
]
}
@@ -0,0 +1,18 @@
test_actual_worker_exit_before_parent_reap (__main__.RecoveryTests) ... ok
test_existing_spend_counts_against_stage_full_job_reservation (__main__.RecoveryTests) ... ok
test_identity_with_completed_result_before_wait4_receipt (__main__.RecoveryTests) ... ok
test_live_expected_worker_and_known_gone (__main__.RecoveryTests) ... ok
test_only_seven_unattempted_are_dispatched (__main__.RecoveryTests) ... V04_TERMINAL method=splu index=3 status=PASS
V04_TERMINAL method=splu index=4 status=PASS
V04_TERMINAL method=splu index=5 status=PASS
V04_TERMINAL method=splu index=6 status=PASS
V04_TERMINAL method=splu index=7 status=PASS
V04_TERMINAL method=splu index=8 status=PASS
V04_TERMINAL method=splu index=9 status=PASS
ok
test_wait4_replaces_live_cpu_once (__main__.RecoveryTests) ... ok
----------------------------------------------------------------------
Ran 6 tests in 0.072s
OK
@@ -0,0 +1,22 @@
{
"status": "PASS",
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/test_s3b_v04_continuation.py"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 0.218599,
"wall_seconds": 0.2285621860064566,
"peak_tree_rss_bytes": 33513472,
"peak_process_count": 2,
"output_disk_bytes_before_receipt": 8192,
"max_poll_gap_seconds": 0.020923480042256415,
"poll_target_seconds": 0.02,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
}
@@ -0,0 +1,15 @@
{
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/test_s3b_v04_continuation.py"
],
"limits": {
"total_cpu_seconds": 60,
"job_cpu_seconds": 60,
"job_wall_seconds": 90,
"tree_rss_bytes": 1073741824,
"disk_bytes": 20971520,
"max_processes": 2
},
"started_unix": 1794748317.2518542
}
@@ -0,0 +1,15 @@
# v0.4 守卫竞态修复与获批七件接续
Claude1794748036.544479实际核读后批准:保留原基础设施停止及费用,复用三个PASS,仅splu按原顺序继续七个未尝试件,累计10800候选帽及同一20h账,不因本次事故换solver。未改变原数学模块、part worker、批量0.005M、路由、求解器参数或参考门。原事故已固定a73e639702f277c69e62215a5373cd5c1aeb80bc并发布,原根与失败文件不改写。
新独立stage_guard_v2在identity不符时,先重新读取wait4/result/协调器是否已进入下一件;已退出、zombie/dead/gone或有完成证据时,保留此前CPU高水位并等待最终回执。只有无法解释的身份变化停止。每次身份异常记录pid、status、cmdline、ppid、create_time及退出/派发证据;资源触发也保存当时观察。协调器消失的查询竞态单独处理。该修复并不追认原事故已实测到zombie,原事故缺失的状态证据仍缺失。
新的continue协调器从原三个已结算guard receipt开始,科学源仍847a6f8;只派原索引3–9,原固定次序。每次派发同时检查72000共享账/7200保留和10800候选完整单件预留;外部guard再监控实际CPU到界。原perpart1800CPU/wall/4GiB/单线程不变。候选阶段预留不足直接保留资源停止,不靠本事故启动备用;若未来发生原来已批的求解器归因单件失败,仍只允许固定GMRES+spilu同全十件路径。所有新/复用逐件回执列pressure_calls、early_resolves、dead_end_share,不改变路由。
6项修复合成含真实子进程exit-before-parent-reap、result-before-wait4、next-dispatch证据、未知身份拒绝、正常活进程、CPU回执替代、只派七件以及继承旧费用阻止超阶段派发(一个测试含多个断言)。attempt-01因账本565.748434+3.893839的浮点严格相等断言失败,CPU0.227038;未改变账本算法,测试改assertAlmostEqual九位小数。attempt-02全部6PASS,CPU0.218599。原失败测试完整保留,两次0.445637都计账,无科学件重试。
接续根artifacts/s3b-a-v04-continuation-2026-11-15;合同a39dc2947e6cc8dea3721a48a0a5bce777daec82f19ad6d2c83693e9e45a3a0f。三个已完成part目录逐文件复制并hash核验,只复用字节不重新计算,合同保留原guard路径/哈希。全部v0.4准备/合成费用6.244713秒(含所有旧失败测试),原三件科学963.629082秒,原exact账3239.962177秒,启动前累计4209.835972秒;splu阶段969.873795/10800,扣7200后计算余60590.164028秒。原管理CPU仍另列。
新的门接续只更换输入根/守卫根和重复计费设置,源8a7965d保持不变;gate合同57f290d0f27dfd1b6e7498ff14d03e4a428b8142a7128a924a9ee97ca96b40b7。additional_synthetic_CPU_s=0,因为新ten合同已包含全部6.244713测试CPU,不能再次加旧2.177133。十件+守卫正常终态后仍三参考tau→成本,门失败停报,无full617自动启动。
Claude明确842的2330/2332提前重解与565.748434CPU秒高于原exact310.038647,应视为本件退化到近逐事件重解的成本证据。该观察不授权调ΔV或路由;每件真实比例和最终十件成本门决定。此文记录实现/合成/固定合同,实际新PID与首个未尝试件进度另附启动审计。
+65
View File
@@ -0,0 +1,65 @@
"""Pin approved engineering continuation; copy three PASS bytes, never rerun."""
from pathlib import Path
import copy
import shutil
from s3b_v04_runtime import contract,read,sha,checked
from s2_recovery_supervisor import save
ROOT=Path('artifacts/s3b-a-v04-continuation-2026-11-15')
GATES=Path('artifacts/s3b-a-v04-gates-continuation-2026-11-15')
GUARD=Path('artifacts/s3b-a-v04-stage-guard-v2-2026-11-15')
def main():
if any(p.exists() for p in [ROOT,GATES,GUARD]):raise FileExistsError('Continuation already prepared')
old=Path('artifacts/s3b-a-v04-ten-2026-11-15');original=contract(old);terminal=read(old/'complete.json')
if terminal['stop_reason']!='COORDINATOR_INTERRUPTED' or terminal['attempts']!=3 or not terminal['account_complete']:
raise ValueError('Unexpected original terminal')
receipts=[];copies={}
for i in range(3):
p=old/'candidates/splu/parts'/f'part-{i:02d}';w=read(p/'guard-result.json');r=read(p/'output/result.json')
if w['classification']['category']!='PASS' or r['status']!='COMPLETE':raise ValueError('Not reusable')
checked(p/'output/result.json',w['files']['result']['sha256'])
for n,h in r['output_sha256'].items():checked(p/'output'/n,h)
w.update(reused_original=True,original_guard_result=str(p/'guard-result.json'),
original_guard_result_sha256=sha(p/'guard-result.json'),pressure_calls=r['pressure_calls'],
early_resolves=r['early_resolves'],dead_end_share=r['early_resolves']/r['pressure_calls'])
receipts.append(w);copies[i]=p
all_tests=[]
for name in ['s3b-a-v04-synthetic-2026-11-15','s3b-a-v04-stage-guard-tests-2026-11-15',
's3b-a-v04-gate-tests-2026-11-15','s3b-a-v04-recovery-tests-2026-11-15']:
paths=sorted((Path('artifacts')/name).glob('attempt-*/synthetic.receipt.json'))
if not paths or read(paths[-1])['status']!='PASS':raise ValueError('Final test acceptance missing')
all_tests.extend(paths)
c=copy.deepcopy(original)
c.update(version='S3B-a-v04-approved-seven-part-infrastructure-continuation',
engineering_ruling='1794748036.544479',stage_cap_ruling='1794747272.597049',
reused_receipts=receipts,new_synthetic_worker_CPU_s=sum(read(p)['cpu_seconds'] for p in all_tests),
inherited_original_stop=str(old/'complete.json'),inherited_original_stop_sha256=sha(old/'complete.json'),
never_attempted_indices=list(range(3,10)),scientific_implementation_unchanged='847a6f8ad2f87e9b09e1c5ad1720d200ce5e57e6',
candidate_stage_CPU_cap=10800,
stage='reuse first three PASS; only remaining seven same splu; original attributed solver fallback retained; stage reservation failure stops')
for n in ['run_s3b_v04_continue.py','s3b_v04_stage_guard_v2.py','prepare_s3b_v04_continuation.py','test_s3b_v04_continuation.py']:
p=Path('tools')/n;c['code_sha256'][str(p)]=sha(p)
sources=[old/'complete.json',Path('artifacts/s3b-a-v04-stage-guard-2026-11-15/failure.json'),
Path('artifacts/s3b-a-v04-gates-2026-11-15/complete.json'),*all_tests]
for p in sources:c['source_sha256'][str(p)]=sha(p)
ROOT.mkdir()
for i,p in copies.items():
dest=ROOT/'candidates/splu/parts'/f'part-{i:02d}';dest.parent.mkdir(parents=True,exist_ok=True)
shutil.copytree(p,dest)
for f in p.rglob('*'):
if f.is_file():checked(dest/f.relative_to(p),sha(f))
save(ROOT/'contract.json',c)
gc=copy.deepcopy(read('artifacts/s3b-a-v04-gates-2026-11-15/contract.json'))
gc.update(ten_root=str(ROOT),stage_guard_root=str(GUARD),additional_synthetic_CPU_s=0.,
engineering_ruling='1794748036.544479',ledger_note='All old/new synthetic costs, including failed attempt, are already in continuation ten ledger; do not add again')
gc['source_sha256'][str(ROOT/'contract.json')]=sha(ROOT/'contract.json')
GATES.mkdir();save(GATES/'contract.json',gc)
previous_science=sum(r['worker']['cpu_seconds'] for r in receipts)
print(dict(root=str(ROOT),contract_sha256=sha(ROOT/'contract.json'),gate_contract_sha256=sha(GATES/'contract.json'),
tests_CPU_s=c['new_synthetic_worker_CPU_s'],reused_science_CPU_s=previous_science,
cumulative_worker_CPU_s=c['old_worker_CPU_s']+c['new_synthetic_worker_CPU_s']+previous_science))
if __name__=='__main__':main()
+101
View File
@@ -0,0 +1,101 @@
"""Authorized seven-part infrastructure continuation; original three PASS reused."""
import argparse
import fcntl
import os
from pathlib import Path
import resource
import signal
import sys
import time
import psutil
from s2_recovery_supervisor import save, supervise, stop_group
from s2_recovery_monitoring import disk_usage
from s3b_v04_runtime import contract, read, sha, reserve_job, failure_classification, may_switch
def run(root):
lock=(root/'batch.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
c=contract(root);lim=c['limits']
if (root/'started.json').exists():raise FileExistsError('Never restart a candidate measurement')
os.sched_setaffinity(0,c['coordinator_cpus'])
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),
code_freeze=read(root/'code-freeze.json'),stage='V04_FIXED_TEN',labels_read=False))
interrupted=False
def stop(*_):
nonlocal interrupted
interrupted=True
for child in psutil.Process(os.getpid()).children():stop_group(child.pid)
signal.signal(signal.SIGTERM,stop);signal.signal(signal.SIGINT,stop)
start=time.monotonic();done=list(c['reused_receipts']);candidates=[];reason=None;active=None;selected=None
def spent():return c['new_synthetic_worker_CPU_s']+sum(r['worker']['cpu_seconds'] for r in done)
try:
for method in c['candidate_order']:
candidate=[r for r in done if r['method']==method];classification=None
for index,spec in enumerate(c['ten_parts']):
if index in {r['index'] for r in candidate}:continue
contract(root)
if interrupted:reason='COORDINATOR_INTERRUPTED';break
if not reserve_job(spent(),lim['part_CPU_s']):reason='SHARED_FULL_JOB_RESERVATION';break
stage_spent=(c['new_synthetic_worker_CPU_s'] if method=='splu' else 0.)+sum(r['worker']['cpu_seconds'] for r in done if r['method']==method)
if stage_spent+lim['part_CPU_s']>10800:reason='CANDIDATE_STAGE_FULL_JOB_RESERVATION';break
if disk_usage(root)>=lim['ten_disk_bytes']:reason='TEN_DISK_LIMIT';break
dest=root/'candidates'/method/'parts'/f'part-{index:02d}'
dest.mkdir(parents=True,exist_ok=False);active=dict(method=method,index=index)
save(root/'progress.json',dict(stage='V04_FIXED_TEN_RUNNING',**active,case=spec['case'],
completed_this_candidate=len(candidate),all_attempts=len(done),
new_settled_worker_CPU_s=spent(),old_worker_CPU_s=c['old_worker_CPU_s'],
remaining_compute_CPU_s=lim['total_worker_CPU_s']-lim['postfit_reserve_CPU_s']-c['old_worker_CPU_s']-spent()))
w=supervise(['taskset','-c',str(c['worker_cpu']),sys.executable,'tools/run_s3b_v04_part.py',str(root),str(index),method],
dest,'worker',dict(total_cpu_seconds=lim['part_CPU_s']+1,job_cpu_seconds=lim['part_CPU_s'],
job_wall_seconds=lim['part_wall_s'],tree_rss_bytes=lim['part_RSS_bytes'],
disk_bytes=lim['ten_disk_bytes']-disk_usage(root),max_processes=1),poll=.2)
paths={n:dest/'output'/f'{n}.json' for n in ['result','failure','progress']}
data={n:read(p) if p.exists() else None for n,p in paths.items()}
classification=failure_classification(w,data['failure'],data['progress'],data['result'])
result_data=data['result'] or {}
receipt=dict(method=method,index=index,case=spec['case'],worker=w,classification=classification,
pressure_calls=result_data.get('pressure_calls'),early_resolves=result_data.get('early_resolves'),
dead_end_share=(result_data['early_resolves']/result_data['pressure_calls']) if result_data.get('pressure_calls') else None,
files={n:dict(path=str(p),sha256=sha(p)) for n,p in paths.items() if p.exists()})
save(dest/'guard-result.json',receipt);done.append(receipt);candidate.append(receipt);active=None
print(f'V04_TERMINAL method={method} index={index} status={classification["category"]}',flush=True)
if classification['category']!='PASS':reason=classification['category'];break
candidate_pass=len(candidate)==10 and all(r['classification']['category']=='PASS' for r in candidate)
summary=dict(method=method,status='PASS' if candidate_pass else 'STOPPED',reason=reason,
classification=classification,attempts=candidate,
not_run_cases=[r['case'] for i,r in enumerate(c['ten_parts']) if i not in {v['index'] for v in candidate}])
candidates.append(summary);save(root/f'candidate-{method}.json',summary)
if candidate_pass:
selected=method;reason=None
save(root/'selected-solver.json',dict(method=selected,contract_sha256=sha(root/'contract.json'),
candidate_summary_sha256=sha(root/f'candidate-{method}.json'),
selection_basis='all ten complete within resource and numerical checks; no accuracy/labels read',
reuse_ten=True,reference_gate_not_yet_run=True))
break
if interrupted or reason in {'SHARED_FULL_JOB_RESERVATION','TEN_DISK_LIMIT','CANDIDATE_STAGE_FULL_JOB_RESERVATION'}:break
if classification and may_switch(method,classification):
save(root/'fallback-decision.json',dict(from_method=method,to_method='gmres_ilu',
classification=classification,new_worker_CPU_s=spent(),restart_same_fixed_ten_from_index=0))
reason=None
else:break
except Exception as exc:
reason='COORDINATOR_EXCEPTION'
save(root/'coordinator-failure.json',dict(type=type(exc).__name__,message=str(exc),active=active))
raise
finally:
report=dict(status='TEN_PASS_AWAITING_REFERENCE_AND_COST' if selected else 'TEN_STOPPED',
stop_reason=reason,selected_solver=selected,candidates=candidates,attempts=len(done),
new_worker_CPU_s=spent(),old_worker_CPU_s=c['old_worker_CPU_s'],
cumulative_worker_CPU_s=c['old_worker_CPU_s']+spent(),postfit_reserve_CPU_s=lim['postfit_reserve_CPU_s'],
remaining_compute_CPU_s=lim['total_worker_CPU_s']-lim['postfit_reserve_CPU_s']-c['old_worker_CPU_s']-spent(),
coordinator_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_s=time.monotonic()-start,
account_complete=active is None,no_silent_retries=True,labels_read=False,reused_original_PASS=3,new_attempts=len(done)-3,
candidate_CPU_s={method:(c['new_synthetic_worker_CPU_s'] if method=='splu' else 0.)+sum(r['worker']['cpu_seconds'] for r in done if r['method']==method) for method in c['candidate_order']},
note='Reference and full-count cost gates still required; no full617 launch by this coordinator.')
save(root/'complete.json',report);save(root/'progress.json',report)
lock.close()
return 0 if selected else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();raise SystemExit(run(a.root))
+122
View File
@@ -0,0 +1,122 @@
"""External sampled candidate CPU cap; never changes the active implementation."""
import argparse
import fcntl
import os
from pathlib import Path
import signal
import time
import psutil
from s2_recovery_supervisor import save, tree_usage
from s3b_v04_runtime import read, sha, contract
CAP=10800.
def stage_charge(receipts,observed,preparation):
# A settled wait4 receipt replaces, never adds to, its sampled live CPU.
keys=set(receipts)|set(observed)
return preparation+sum(max(receipts.get(k,0.),observed.get(k,0.)) for k in keys)
def stage_stop(charged,all_terminal):
return charged>CAP or (charged>=CAP and not all_terminal)
def identity_decision(observation, expected_parent, expected_tail, receipt, terminal_result, advanced):
if (observation.get('ppid')==expected_parent and observation.get('cmdline',[])[-len(expected_tail):]==expected_tail
and observation.get('status') not in ('zombie','dead','gone')):
return 'ACTIVE'
if receipt or terminal_result or advanced or observation.get('status') in ('zombie','dead','gone'):
return 'EXITED_AWAITING_RECEIPT'
return 'UNEXPLAINED_IDENTITY_CHANGE'
def run(root,dest):
dest.mkdir(parents=True,exist_ok=False)
lock=(dest/'execution.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
c=contract(root);start=read(root/'started.json');pid=start['pid']
process=psutil.Process(pid);created=process.create_time()
if 'tools/run_s3b_v04_continue.py' not in process.cmdline():raise ValueError('Coordinator identity differs')
extra=0.
prep=c['new_synthetic_worker_CPU_s']
save(dest/'started.json',dict(pid=os.getpid(),coordinator_pid=pid,coordinator_create_time=created,
unix=time.time(),ruling='1794748036.544479',per_candidate_CPU_cap=CAP,
common_preparation_CPU_s_charged_to_splu=prep,additional_guard_test_CPU_s=extra,
source_sha256=sha(__file__),active_source_unchanged=True,
method='sampled CPU cap, 0.2s target; SIGTERM only verified coordinator at boundary; retain any measured overshoot'))
seen={'splu':{},'gmres_ilu':{}};trigger=None;last_identity=None;began=time.monotonic();last=time.monotonic();max_gap=0.
try:
while True:
now=time.monotonic();max_gap=max(max_gap,now-last);last=now
try:
current=psutil.Process(pid);alive=current.create_time()==created
except psutil.NoSuchProcess:current=None;alive=False
summaries={}
for method in seen:
receipts={};completed=0;observations={}
for part in sorted((root/'candidates'/method/'parts').glob('part-*')):
key=part.name;receipt=part/'worker.receipt.json'
if receipt.exists():
receipts[key]=read(receipt)['cpu_seconds'];completed+=1
else:
started=part/'output/started.json'
if started.exists():
info=read(started)
observation=dict(pid=info['pid'],status=None,cmdline=[],ppid=None,create_time=None)
try:
worker=psutil.Process(info['pid'])
observation['status']=worker.status()
observation['ppid']=worker.ppid()
observation['create_time']=worker.create_time()
observation['cmdline']=worker.cmdline()
except psutil.NoSuchProcess as exc:
observation['last_observed_status']=observation['status']
observation['status']='gone';observation['snapshot_exception']=type(exc).__name__
observations[key]=observation
# Re-read terminal/dispatch evidence after observing identity,
# closing the result-before-wait4 window without altering receipts.
result_path=part/'output/result.json'
terminal_result=result_path.exists() and read(result_path).get('status') in ('COMPLETE','BLOCKED')
qp=read(root/'progress.json') if (root/'progress.json').exists() else {}
advanced=(qp.get('method')==method and qp.get('index',-1)>int(key.split('-')[-1])) or (method=='splu' and qp.get('method')=='gmres_ilu')
expected=['tools/run_s3b_v04_part.py',str(root),str(int(key.split('-')[-1])),method]
decision=identity_decision(observation,pid,expected,receipt.exists(),terminal_result,advanced)
if decision!='ACTIVE':
last_identity=dict(unix=time.time(),method=method,part=key,observation=observation,
receipt_present=receipt.exists(),terminal_result=terminal_result,coordinator_advanced=advanced,decision=decision)
with (dest/'identity-observations.jsonl').open('a') as log:
import json
log.write(json.dumps(last_identity)+'\n')
if decision=='UNEXPLAINED_IDENTITY_CHANGE':raise ValueError('Unexplained active worker identity change')
if decision=='ACTIVE':
cpu,_,_=tree_usage(info['pid'])
seen[method][key]=max(seen[method].get(key,0.),cpu)
charged=stage_charge(receipts,seen[method],prep if method=='splu' else 0.)
summaries[method]=dict(charged_CPU_s=charged,settled_attempts=completed,
live_high_water_CPU_s=seen[method],settled_CPU_s=receipts,active_observations=observations)
if trigger is None and stage_stop(charged,completed==10):
trigger=dict(reason='CANDIDATE_STAGE_CPU_CAP',method=method,charged_CPU_s=charged,
cap_CPU_s=CAP,all_ten_terminal=completed==10,unix=time.time(),observations=observations)
save(dest/'trigger.json',trigger)
if alive:current.send_signal(signal.SIGTERM)
save(dest/'progress.json',dict(unix=time.time(),coordinator_alive=alive,candidates=summaries,
trigger=trigger,max_poll_gap_seconds=max_gap,monitor_CPU_s=time.process_time(),
supervision_CPU_separate_from_scientific_worker_ledger=True))
if (root/'complete.json').exists():
save(dest/'complete.json',dict(status='STAGE_CAP_STOP' if trigger else 'OBSERVED_QUEUE_TERMINAL',
trigger=trigger,candidates=summaries,queue_complete_sha256=sha(root/'complete.json'),
monitor_CPU_s=time.process_time(),wall_s=time.monotonic()-began,max_poll_gap_seconds=max_gap))
return 1 if trigger else 0
if not alive:raise RuntimeError('Coordinator exited without complete receipt')
time.sleep(.2)
except Exception as exc:
save(dest/'failure.json',dict(type=type(exc).__name__,message=str(exc),unix=time.time(),last_identity=last_identity))
if psutil.pid_exists(pid):
current=psutil.Process(pid)
if current.create_time()==created:current.send_signal(signal.SIGTERM)
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('dest',type=Path)
a=p.parse_args();raise SystemExit(run(a.root,a.dest))
+76
View File
@@ -0,0 +1,76 @@
"""Approved exit-before-receipt repair and seven-only dispatch checks."""
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import time
import unittest
from unittest.mock import patch
import psutil
from s3b_v04_stage_guard_v2 import identity_decision,stage_charge
from run_s3b_v04_continue import run
from s2_recovery_supervisor import save
from s3b_runtime import read
class RecoveryTests(unittest.TestCase):
def test_identity_with_completed_result_before_wait4_receipt(self):
observation=dict(pid=123,ppid=7,status='running',cmdline=[])
self.assertEqual(identity_decision(observation,7,['worker'],False,True,False),'EXITED_AWAITING_RECEIPT')
self.assertEqual(identity_decision(observation,7,['worker'],True,False,False),'EXITED_AWAITING_RECEIPT')
self.assertEqual(identity_decision(observation,7,['worker'],False,False,True),'EXITED_AWAITING_RECEIPT')
self.assertEqual(identity_decision(observation,7,['worker'],False,False,False),'UNEXPLAINED_IDENTITY_CHANGE')
def test_actual_worker_exit_before_parent_reap(self):
with tempfile.TemporaryDirectory() as temp:
result=Path(temp)/'result.json'
proc=subprocess.Popen([sys.executable,'-c','from pathlib import Path; import sys; Path(sys.argv[1]).write_text("{}")',str(result)])
try:
worker=psutil.Process(proc.pid);until=time.monotonic()+5
while worker.status()!=psutil.STATUS_ZOMBIE and time.monotonic()<until:time.sleep(.01)
observation=dict(pid=proc.pid,ppid=worker.ppid(),status=worker.status(),cmdline=[])
self.assertTrue(result.exists());self.assertFalse((Path(temp)/'worker.receipt.json').exists())
self.assertEqual(identity_decision(observation,os.getpid(),['worker'],False,True,False),'EXITED_AWAITING_RECEIPT')
finally:proc.wait(timeout=5)
def test_live_expected_worker_and_known_gone(self):
o=dict(ppid=7,status='running',cmdline=['python','worker','root','3','splu'])
self.assertEqual(identity_decision(o,7,['worker','root','3','splu'],False,False,False),'ACTIVE')
self.assertEqual(identity_decision(dict(o,status='gone',cmdline=[]),7,['worker'],False,False,False),'EXITED_AWAITING_RECEIPT')
def test_wait4_replaces_live_cpu_once(self):
self.assertAlmostEqual(stage_charge({'part-02':565.748434},{'part-02':565.54},3.893839),569.642273,places=9)
def fixture(self,prior_cpu=2.):
calls=[]
with tempfile.TemporaryDirectory() as temp:
root=Path(temp);save(root/'contract.json',{});save(root/'code-freeze.json',dict(git_commit='synthetic'))
prior=[dict(method='splu',index=i,case=str(i),worker=dict(cpu_seconds=prior_cpu),classification=dict(category='PASS')) for i in range(3)]
c=dict(coordinator_cpus=[0],worker_cpu=0,old_worker_CPU_s=3239.962177,new_synthetic_worker_CPU_s=3.,
candidate_order=['splu','gmres_ilu'],reused_receipts=prior,ten_parts=[dict(case=str(i)) for i in range(10)],
limits=dict(part_CPU_s=1800,part_wall_s=1800,part_RSS_bytes=4*2**30,ten_disk_bytes=5*2**30,
total_worker_CPU_s=72000,postfit_reserve_CPU_s=7200))
def fake(command,dest,name,limits,**kw):
calls.append((command[-1],int(command[-2])));(dest/'output').mkdir()
save(dest/'output/result.json',dict(status='COMPLETE',pressure_calls=10,early_resolves=9))
return dict(status='PASS',monitor_errors=0,stop_reason=None,cpu_seconds=2.)
with patch('run_s3b_v04_continue.contract',return_value=c),patch('run_s3b_v04_continue.supervise',side_effect=fake),patch('run_s3b_v04_continue.os.sched_setaffinity'),patch('run_s3b_v04_continue.signal.signal'):
code=run(root)
return code,calls,read(root/'complete.json')
def test_only_seven_unattempted_are_dispatched(self):
code,calls,r=self.fixture()
self.assertEqual(code,0);self.assertEqual(calls,[('splu',i) for i in range(3,10)])
self.assertEqual(r['new_attempts'],7);self.assertEqual(r['new_worker_CPU_s'],23.)
self.assertEqual(r['candidate_CPU_s']['splu'],23.)
self.assertEqual(r['candidates'][0]['attempts'][-1]['dead_end_share'],.9)
def test_existing_spend_counts_against_stage_full_job_reservation(self):
code,calls,r=self.fixture(prior_cpu=3001.)
self.assertEqual(code,1);self.assertEqual(calls,[])
self.assertEqual(r['stop_reason'],'CANDIDATE_STAGE_FULL_JOB_RESERVATION')
self.assertIsNone(r['selected_solver'])
if __name__=='__main__':unittest.main(verbosity=2)