Add independent per-candidate stage CPU guard under Claude clarification
This commit is contained in:
1 parent
04f8317e93
commit
48748f59f6
7 files changed
+166
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"cpu_seconds_charged": 0.27189599999999997,
|
||||
"attempts": [
|
||||
{
|
||||
"name": "synthetic",
|
||||
"status": "PASS",
|
||||
"cpu_seconds": 0.27189599999999997
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
test_exact_stage_boundary_and_terminal_overshoot (__main__.GuardTests) ... ok
|
||||
test_exit_gap_preserves_observed_cpu (__main__.GuardTests) ... ok
|
||||
test_receipt_lower_than_observation_never_reduces_charge (__main__.GuardTests) ... ok
|
||||
test_settled_replaces_live_not_double_counted (__main__.GuardTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 4 tests in 0.000s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
||||
"tools/test_s3b_v04_stage_guard.py"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 0.27189599999999997,
|
||||
"wall_seconds": 0.2914889520034194,
|
||||
"peak_tree_rss_bytes": 20049920,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 8192,
|
||||
"max_poll_gap_seconds": 0.16636546200606972,
|
||||
"poll_target_seconds": 0.02,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"command": [
|
||||
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
||||
"tools/test_s3b_v04_stage_guard.py"
|
||||
],
|
||||
"limits": {
|
||||
"total_cpu_seconds": 30,
|
||||
"job_cpu_seconds": 30,
|
||||
"job_wall_seconds": 60,
|
||||
"tree_rss_bytes": 1073741824,
|
||||
"disk_bytes": 20971520,
|
||||
"max_processes": 1
|
||||
},
|
||||
"started_unix": 1794747385.506015
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
"""External sampled candidate CPU cap; never changes the active implementation."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import time
|
||||
import psutil
|
||||
from s2_recovery_supervisor import save, tree_usage
|
||||
from s3b_v04_runtime import read, sha, contract
|
||||
|
||||
CAP=10800.
|
||||
|
||||
|
||||
def stage_charge(receipts,observed,preparation):
|
||||
# A settled wait4 receipt replaces, never adds to, its sampled live CPU.
|
||||
keys=set(receipts)|set(observed)
|
||||
return preparation+sum(max(receipts.get(k,0.),observed.get(k,0.)) for k in keys)
|
||||
|
||||
|
||||
def stage_stop(charged,all_terminal):
|
||||
return charged>CAP or (charged>=CAP and not all_terminal)
|
||||
|
||||
|
||||
def run(root,dest):
|
||||
dest.mkdir(parents=True,exist_ok=False)
|
||||
lock=(dest/'execution.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
c=contract(root);start=read(root/'started.json');pid=start['pid']
|
||||
process=psutil.Process(pid);created=process.create_time()
|
||||
if 'tools/run_s3b_v04_ten.py' not in process.cmdline():raise ValueError('Coordinator identity differs')
|
||||
extra=sum(read(p)['cpu_seconds'] for p in Path('artifacts/s3b-a-v04-stage-guard-tests-2026-11-15').glob('attempt-*/synthetic.receipt.json'))
|
||||
prep=c['new_synthetic_worker_CPU_s']+extra
|
||||
save(dest/'started.json',dict(pid=os.getpid(),coordinator_pid=pid,coordinator_create_time=created,
|
||||
unix=time.time(),ruling='1794747272.597049',per_candidate_CPU_cap=CAP,
|
||||
common_preparation_CPU_s_charged_to_splu=prep,additional_guard_test_CPU_s=extra,
|
||||
source_sha256=sha(__file__),active_source_unchanged=True,
|
||||
method='sampled CPU cap, 0.2s target; SIGTERM only verified coordinator at boundary; retain any measured overshoot'))
|
||||
seen={'splu':{},'gmres_ilu':{}};trigger=None;began=time.monotonic();last=time.monotonic();max_gap=0.
|
||||
try:
|
||||
while True:
|
||||
now=time.monotonic();max_gap=max(max_gap,now-last);last=now
|
||||
current=psutil.Process(pid) if psutil.pid_exists(pid) else None
|
||||
alive=current is not None and current.create_time()==created
|
||||
summaries={}
|
||||
for method in seen:
|
||||
receipts={};completed=0
|
||||
for part in sorted((root/'candidates'/method/'parts').glob('part-*')):
|
||||
key=part.name;receipt=part/'worker.receipt.json'
|
||||
if receipt.exists():
|
||||
receipts[key]=read(receipt)['cpu_seconds'];completed+=1
|
||||
else:
|
||||
started=part/'output/started.json'
|
||||
if started.exists():
|
||||
info=read(started)
|
||||
try:
|
||||
worker=psutil.Process(info['pid'])
|
||||
if worker.ppid()!=pid or 'tools/run_s3b_v04_part.py' not in worker.cmdline():
|
||||
raise ValueError('Active worker identity differs')
|
||||
cpu,_,_=tree_usage(worker.pid)
|
||||
seen[method][key]=max(seen[method].get(key,0.),cpu)
|
||||
except psutil.NoSuchProcess:pass
|
||||
charged=stage_charge(receipts,seen[method],prep if method=='splu' else 0.)
|
||||
summaries[method]=dict(charged_CPU_s=charged,settled_attempts=completed,
|
||||
live_high_water_CPU_s=seen[method],settled_CPU_s=receipts)
|
||||
if trigger is None and stage_stop(charged,completed==10):
|
||||
trigger=dict(reason='CANDIDATE_STAGE_CPU_CAP',method=method,charged_CPU_s=charged,
|
||||
cap_CPU_s=CAP,all_ten_terminal=completed==10,unix=time.time())
|
||||
save(dest/'trigger.json',trigger)
|
||||
if alive:current.send_signal(signal.SIGTERM)
|
||||
save(dest/'progress.json',dict(unix=time.time(),coordinator_alive=alive,candidates=summaries,
|
||||
trigger=trigger,max_poll_gap_seconds=max_gap,monitor_CPU_s=time.process_time(),
|
||||
supervision_CPU_separate_from_scientific_worker_ledger=True))
|
||||
if (root/'complete.json').exists():
|
||||
save(dest/'complete.json',dict(status='STAGE_CAP_STOP' if trigger else 'OBSERVED_QUEUE_TERMINAL',
|
||||
trigger=trigger,candidates=summaries,queue_complete_sha256=sha(root/'complete.json'),
|
||||
monitor_CPU_s=time.process_time(),wall_s=time.monotonic()-began,max_poll_gap_seconds=max_gap))
|
||||
return 1 if trigger else 0
|
||||
if not alive:raise RuntimeError('Coordinator exited without complete receipt')
|
||||
time.sleep(.2)
|
||||
except Exception as exc:
|
||||
save(dest/'failure.json',dict(type=type(exc).__name__,message=str(exc),unix=time.time()))
|
||||
if psutil.pid_exists(pid):
|
||||
current=psutil.Process(pid)
|
||||
if current.create_time()==created:current.send_signal(signal.SIGTERM)
|
||||
raise
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('dest',type=Path)
|
||||
a=p.parse_args();raise SystemExit(run(a.root,a.dest))
|
||||
@@ -0,0 +1,20 @@
|
||||
import unittest
|
||||
from s3b_v04_stage_guard import stage_charge,stage_stop
|
||||
|
||||
|
||||
class GuardTests(unittest.TestCase):
|
||||
def test_settled_replaces_live_not_double_counted(self):
|
||||
self.assertEqual(stage_charge({'part-00':12.},{'part-00':11.,'part-01':7.},3.),22.)
|
||||
|
||||
def test_exit_gap_preserves_observed_cpu(self):
|
||||
self.assertEqual(stage_charge({}, {'part-00':8.},1.),9.)
|
||||
|
||||
def test_receipt_lower_than_observation_never_reduces_charge(self):
|
||||
self.assertEqual(stage_charge({'part-00':9.},{'part-00':10.},0.),10.)
|
||||
|
||||
def test_exact_stage_boundary_and_terminal_overshoot(self):
|
||||
self.assertFalse(stage_stop(10799.9,False));self.assertTrue(stage_stop(10800.,False))
|
||||
self.assertFalse(stage_stop(10800.,True));self.assertTrue(stage_stop(10800.01,True))
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user