From 48748f59f696ed53a39c02869b0950040a5ddf59 Mon Sep 17 00:00:00 2001 From: Frank14f <1515444314@qq.com> Date: Sun, 15 Nov 2026 20:56:50 +0800 Subject: [PATCH] Add independent per-candidate stage CPU guard under Claude clarification --- .../attempt-01/execution.lock | 0 .../attempt-01/ledger.json | 10 +++ .../attempt-01/synthetic.log | 9 ++ .../attempt-01/synthetic.receipt.json | 22 +++++ .../attempt-01/synthetic.started.json | 15 ++++ tools/s3b_v04_stage_guard.py | 90 +++++++++++++++++++ tools/test_s3b_v04_stage_guard.py | 20 +++++ 7 files changed, 166 insertions(+) create mode 100644 artifacts/s3b-a-v04-stage-guard-tests-2026-11-15/attempt-01/execution.lock create mode 100644 artifacts/s3b-a-v04-stage-guard-tests-2026-11-15/attempt-01/ledger.json create mode 100644 artifacts/s3b-a-v04-stage-guard-tests-2026-11-15/attempt-01/synthetic.log create mode 100644 artifacts/s3b-a-v04-stage-guard-tests-2026-11-15/attempt-01/synthetic.receipt.json create mode 100644 artifacts/s3b-a-v04-stage-guard-tests-2026-11-15/attempt-01/synthetic.started.json create mode 100644 tools/s3b_v04_stage_guard.py create mode 100644 tools/test_s3b_v04_stage_guard.py diff --git a/artifacts/s3b-a-v04-stage-guard-tests-2026-11-15/attempt-01/execution.lock b/artifacts/s3b-a-v04-stage-guard-tests-2026-11-15/attempt-01/execution.lock new file mode 100644 index 000000000..e69de29bb diff --git a/artifacts/s3b-a-v04-stage-guard-tests-2026-11-15/attempt-01/ledger.json b/artifacts/s3b-a-v04-stage-guard-tests-2026-11-15/attempt-01/ledger.json new file mode 100644 index 000000000..10e9609e2 --- /dev/null +++ b/artifacts/s3b-a-v04-stage-guard-tests-2026-11-15/attempt-01/ledger.json @@ -0,0 +1,10 @@ +{ + "cpu_seconds_charged": 0.27189599999999997, + "attempts": [ + { + "name": "synthetic", + "status": "PASS", + "cpu_seconds": 0.27189599999999997 + } + ] +} diff --git a/artifacts/s3b-a-v04-stage-guard-tests-2026-11-15/attempt-01/synthetic.log b/artifacts/s3b-a-v04-stage-guard-tests-2026-11-15/attempt-01/synthetic.log new file mode 100644 index 000000000..a8bd9552e --- /dev/null +++ b/artifacts/s3b-a-v04-stage-guard-tests-2026-11-15/attempt-01/synthetic.log @@ -0,0 +1,9 @@ +test_exact_stage_boundary_and_terminal_overshoot (__main__.GuardTests) ... ok +test_exit_gap_preserves_observed_cpu (__main__.GuardTests) ... ok +test_receipt_lower_than_observation_never_reduces_charge (__main__.GuardTests) ... ok +test_settled_replaces_live_not_double_counted (__main__.GuardTests) ... ok + +---------------------------------------------------------------------- +Ran 4 tests in 0.000s + +OK diff --git a/artifacts/s3b-a-v04-stage-guard-tests-2026-11-15/attempt-01/synthetic.receipt.json b/artifacts/s3b-a-v04-stage-guard-tests-2026-11-15/attempt-01/synthetic.receipt.json new file mode 100644 index 000000000..2c38d70e1 --- /dev/null +++ b/artifacts/s3b-a-v04-stage-guard-tests-2026-11-15/attempt-01/synthetic.receipt.json @@ -0,0 +1,22 @@ +{ + "status": "PASS", + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/test_s3b_v04_stage_guard.py" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 0.27189599999999997, + "wall_seconds": 0.2914889520034194, + "peak_tree_rss_bytes": 20049920, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 8192, + "max_poll_gap_seconds": 0.16636546200606972, + "poll_target_seconds": 0.02, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null +} diff --git a/artifacts/s3b-a-v04-stage-guard-tests-2026-11-15/attempt-01/synthetic.started.json b/artifacts/s3b-a-v04-stage-guard-tests-2026-11-15/attempt-01/synthetic.started.json new file mode 100644 index 000000000..46d8f3f9a --- /dev/null +++ b/artifacts/s3b-a-v04-stage-guard-tests-2026-11-15/attempt-01/synthetic.started.json @@ -0,0 +1,15 @@ +{ + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/test_s3b_v04_stage_guard.py" + ], + "limits": { + "total_cpu_seconds": 30, + "job_cpu_seconds": 30, + "job_wall_seconds": 60, + "tree_rss_bytes": 1073741824, + "disk_bytes": 20971520, + "max_processes": 1 + }, + "started_unix": 1794747385.506015 +} diff --git a/tools/s3b_v04_stage_guard.py b/tools/s3b_v04_stage_guard.py new file mode 100644 index 000000000..7b8c8aa66 --- /dev/null +++ b/tools/s3b_v04_stage_guard.py @@ -0,0 +1,90 @@ +"""External sampled candidate CPU cap; never changes the active implementation.""" +import argparse +import fcntl +import os +from pathlib import Path +import signal +import time +import psutil +from s2_recovery_supervisor import save, tree_usage +from s3b_v04_runtime import read, sha, contract + +CAP=10800. + + +def stage_charge(receipts,observed,preparation): + # A settled wait4 receipt replaces, never adds to, its sampled live CPU. + keys=set(receipts)|set(observed) + return preparation+sum(max(receipts.get(k,0.),observed.get(k,0.)) for k in keys) + + +def stage_stop(charged,all_terminal): + return charged>CAP or (charged>=CAP and not all_terminal) + + +def run(root,dest): + dest.mkdir(parents=True,exist_ok=False) + lock=(dest/'execution.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + c=contract(root);start=read(root/'started.json');pid=start['pid'] + process=psutil.Process(pid);created=process.create_time() + if 'tools/run_s3b_v04_ten.py' not in process.cmdline():raise ValueError('Coordinator identity differs') + extra=sum(read(p)['cpu_seconds'] for p in Path('artifacts/s3b-a-v04-stage-guard-tests-2026-11-15').glob('attempt-*/synthetic.receipt.json')) + prep=c['new_synthetic_worker_CPU_s']+extra + save(dest/'started.json',dict(pid=os.getpid(),coordinator_pid=pid,coordinator_create_time=created, + unix=time.time(),ruling='1794747272.597049',per_candidate_CPU_cap=CAP, + common_preparation_CPU_s_charged_to_splu=prep,additional_guard_test_CPU_s=extra, + source_sha256=sha(__file__),active_source_unchanged=True, + method='sampled CPU cap, 0.2s target; SIGTERM only verified coordinator at boundary; retain any measured overshoot')) + seen={'splu':{},'gmres_ilu':{}};trigger=None;began=time.monotonic();last=time.monotonic();max_gap=0. + try: + while True: + now=time.monotonic();max_gap=max(max_gap,now-last);last=now + current=psutil.Process(pid) if psutil.pid_exists(pid) else None + alive=current is not None and current.create_time()==created + summaries={} + for method in seen: + receipts={};completed=0 + for part in sorted((root/'candidates'/method/'parts').glob('part-*')): + key=part.name;receipt=part/'worker.receipt.json' + if receipt.exists(): + receipts[key]=read(receipt)['cpu_seconds'];completed+=1 + else: + started=part/'output/started.json' + if started.exists(): + info=read(started) + try: + worker=psutil.Process(info['pid']) + if worker.ppid()!=pid or 'tools/run_s3b_v04_part.py' not in worker.cmdline(): + raise ValueError('Active worker identity differs') + cpu,_,_=tree_usage(worker.pid) + seen[method][key]=max(seen[method].get(key,0.),cpu) + except psutil.NoSuchProcess:pass + charged=stage_charge(receipts,seen[method],prep if method=='splu' else 0.) + summaries[method]=dict(charged_CPU_s=charged,settled_attempts=completed, + live_high_water_CPU_s=seen[method],settled_CPU_s=receipts) + if trigger is None and stage_stop(charged,completed==10): + trigger=dict(reason='CANDIDATE_STAGE_CPU_CAP',method=method,charged_CPU_s=charged, + cap_CPU_s=CAP,all_ten_terminal=completed==10,unix=time.time()) + save(dest/'trigger.json',trigger) + if alive:current.send_signal(signal.SIGTERM) + save(dest/'progress.json',dict(unix=time.time(),coordinator_alive=alive,candidates=summaries, + trigger=trigger,max_poll_gap_seconds=max_gap,monitor_CPU_s=time.process_time(), + supervision_CPU_separate_from_scientific_worker_ledger=True)) + if (root/'complete.json').exists(): + save(dest/'complete.json',dict(status='STAGE_CAP_STOP' if trigger else 'OBSERVED_QUEUE_TERMINAL', + trigger=trigger,candidates=summaries,queue_complete_sha256=sha(root/'complete.json'), + monitor_CPU_s=time.process_time(),wall_s=time.monotonic()-began,max_poll_gap_seconds=max_gap)) + return 1 if trigger else 0 + if not alive:raise RuntimeError('Coordinator exited without complete receipt') + time.sleep(.2) + except Exception as exc: + save(dest/'failure.json',dict(type=type(exc).__name__,message=str(exc),unix=time.time())) + if psutil.pid_exists(pid): + current=psutil.Process(pid) + if current.create_time()==created:current.send_signal(signal.SIGTERM) + raise + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('dest',type=Path) + a=p.parse_args();raise SystemExit(run(a.root,a.dest)) diff --git a/tools/test_s3b_v04_stage_guard.py b/tools/test_s3b_v04_stage_guard.py new file mode 100644 index 000000000..fb635af60 --- /dev/null +++ b/tools/test_s3b_v04_stage_guard.py @@ -0,0 +1,20 @@ +import unittest +from s3b_v04_stage_guard import stage_charge,stage_stop + + +class GuardTests(unittest.TestCase): + def test_settled_replaces_live_not_double_counted(self): + self.assertEqual(stage_charge({'part-00':12.},{'part-00':11.,'part-01':7.},3.),22.) + + def test_exit_gap_preserves_observed_cpu(self): + self.assertEqual(stage_charge({}, {'part-00':8.},1.),9.) + + def test_receipt_lower_than_observation_never_reduces_charge(self): + self.assertEqual(stage_charge({'part-00':9.},{'part-00':10.},0.),10.) + + def test_exact_stage_boundary_and_terminal_overshoot(self): + self.assertFalse(stage_stop(10799.9,False));self.assertTrue(stage_stop(10800.,False)) + self.assertFalse(stage_stop(10800.,True));self.assertTrue(stage_stop(10800.01,True)) + + +if __name__=='__main__':unittest.main(verbosity=2)