Enforce running coordinator CPU allowance for S2 diagnostic

This commit is contained in:
MeltAstra committed 2026-11-18 10:27:28 +08:00
1 parent b97674a1d4
commit 1ed44f70fc
21 files changed
+19182 -15

No files matched your search

+2 -1
View File
@@ -21,9 +21,10 @@ def main():
parts.append(dict(case=case,component_id=e['component_id'],fold=e['fold'],nodes=e['nodes'],t_set=cache['t_set'],sources=sources))
spec=BASE/'references/2026-11-18/phase3-item-c/mold-tail-volume-peak-stability-spec.md'
if sha(spec)!='05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521':raise ValueError('Spec identity')
names=['s2_tail_diagnostic_math.py','s2_tail_diagnostic.py','prepare_s2_tail_diagnostic.py','test_s2_tail_diagnostic.py','r4bc_guard.py','s2_recovery_supervisor.py','s2_recovery_monitoring.py']
names=['s2_tail_diagnostic_math.py','s2_tail_diagnostic.py','s2_tail_diagnostic_guard.py','prepare_s2_tail_diagnostic.py','test_s2_tail_diagnostic.py','r4bc_guard.py','s2_recovery_supervisor.py','s2_recovery_monitoring.py']
codes={str(BASE/'tools'/n):sha(BASE/'tools'/n) for n in names};codes[str(BASE/'docs/s2-tail-diagnostic-plan-2026-11-18.md')]=sha(BASE/'docs/s2-tail-diagnostic-plan-2026-11-18.md')
ROOT.mkdir();plan=dict(approval='1794966078.809549',spec_path=str(spec),spec_sha256=sha(spec),descriptive_only=True,gate=None,cpu_cap=3600,worker_cpu_cap=3300,setup_reserve=180,worker_cpu=24,coordinator_cpu=25,worker_rss_bytes=4*2**30,worker_wall_seconds=7200,parts=parts,code_sha256=codes,S2_score=dict(path=str(s/'score/result.json'),sha256=sha(s/'score/result.json')),metadata_sources={str(q):sha(q) for q in [s/'contract.json',s/'predictions-manifest.json',s/'predictions-git-freeze.json',s/'complete.json',Path(c['pilot_contract']),BASE/'artifacts/phase2-b13-e1b-2026-11-17/contract.json']},new_fits=0,new_predictions=0,no_retry=True,light_review_before_targets=True,original_S2_decision_unchanged=True)
plan.update(coordinator_cpu_limit=90,shutdown_cpu_reserve=30)
save(ROOT/'plan.json',plan)
save(ROOT/'preparation.json',dict(status='PREPARED_NO_TARGET_ARRAYS_READ',parts=552,components=204,source_file_existence_checked=True,array_values_read=False,source_hashes_rechecked_at_execution=True,review_pending=True,launched=False,preparation_cpu_seconds=time.process_time()-start,unix=time.time()))
print(dict(root=str(ROOT),plan_sha256=sha(ROOT/'plan.json'),code_files=len(codes),parts=552,components=204,review_pending=True))
+2 -2
View File
@@ -101,13 +101,13 @@ def worker(root):
def run(root):
import fcntl,psutil
from r4bc_guard import supervise
from s2_tail_diagnostic_guard import supervise
start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);c=config(root);os.sched_setaffinity(0,[25]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('One diagnostic attempt; no duplicate or retry')
for p in psutil.process_iter(['pid','cmdline']):
if p.pid!=os.getpid() and any(Path(x).name=='s2_tail_diagnostic.py' for x in p.info['cmdline'] or []):raise RuntimeError('Existing diagnostic')
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),cpu_cap=3600,setup_reserve=180,descriptive_only=True,review_sha256=sha(root/'review-accepted.json'),cpu_start=dict(self=start,child_user=u0.ru_utime,child_system=u0.ru_stime)))
guard=supervise([sys.executable,str(BASE/'tools/s2_tail_diagnostic.py'),'worker',str(root)],root/'guard',dict(cpu_seconds=3300,wall_seconds=7200,rss_bytes=4*2**30,disk_bytes=2**30))
guard=supervise([sys.executable,str(BASE/'tools/s2_tail_diagnostic.py'),'worker',str(root)],root/'guard',dict(cpu_seconds=3300,wall_seconds=7200,rss_bytes=4*2**30,disk_bytes=2**30),coordinator_cpu_start=start,coordinator_cpu_limit=90.)
u=resource.getrusage(resource.RUSAGE_CHILDREN);charge=180+time.process_time()-start+max(guard['cpu_seconds'],u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)
ok=guard['status']=='PASS' and not guard['monitor_errors'] and charge<=3600
if ok:
+102
View File
@@ -0,0 +1,102 @@
"""Diagnostic-only copy of r4bc guard, adding a running coordinator CPU allowance.
Shared guards and all scientific definitions stay unchanged.
"""
import argparse
import ctypes
import fcntl
import json
import math
import os
from pathlib import Path
import signal
import subprocess
import sys
import time
from s2_recovery_supervisor import save, tree_usage, stop_group
from s2_recovery_monitoring import disk_usage
def supervise(command, dest, limits, *, coordinator_cpu_start, coordinator_cpu_limit=90., global_stop=None, poll=.2):
if not math.isfinite(coordinator_cpu_start) or not math.isfinite(coordinator_cpu_limit) or coordinator_cpu_limit <= 0:
raise ValueError('Invalid coordinator CPU allowance')
dest = Path(dest); dest.mkdir(parents=True, exist_ok=True)
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
raise ValueError('Invalid guard limit')
with (dest/'execution.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (dest/'worker.started.json').exists() or (dest/'worker.receipt.json').exists():
raise FileExistsError('Single attempt; no restart')
save(dest/'worker.started.json', dict(command=command, limits=limits, unix=time.time()))
requested = [None]
def stop(signum, frame):
requested[0] = 'GUARD_SIGNAL_'+str(signum)
prior = {s: signal.signal(s, stop) for s in (signal.SIGTERM, signal.SIGINT)}
libc = ctypes.CDLL(None, use_errno=True); previous = ctypes.c_int()
if libc.prctl(37, ctypes.byref(previous), 0, 0, 0) or libc.prctl(36, 1, 0, 0, 0):
raise OSError('Cannot enable descendant accounting')
start = time.monotonic(); last = start; reason = None; errors = []
cpu_peak = rss_peak = proc_peak = gap = 0; usage = None; exit_status = None
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', MKL_NUM_THREADS='1',
NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
try:
with (dest/'worker.log').open('x') as log:
child = subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT,
start_new_session=True)
save(dest/'worker-pid.json', dict(pid=child.pid, unix=time.time(), guard_pid=os.getpid()))
try:
while True:
now = time.monotonic(); gap = max(gap, now-last); last = now
try:
cpu, rss, count = tree_usage(child.pid)
disk = disk_usage(dest)
cpu_peak = max(cpu_peak, cpu); rss_peak = max(rss_peak, rss); proc_peak = max(proc_peak, count)
except Exception as exc:
errors.append(dict(type=type(exc).__name__, message=str(exc)))
reason = 'MONITOR_ERROR'
if time.process_time()-coordinator_cpu_start >= coordinator_cpu_limit:
reason = reason or 'coordinator_cpu_limit'
if requested[0]: reason = reason or requested[0]
if global_stop and Path(global_stop).exists(): reason = reason or 'GLOBAL_STOP_REQUEST'
if now-start >= limits['wall_seconds']: reason = reason or 'wall_limit'
if cpu_peak >= limits['cpu_seconds']: reason = reason or 'cpu_limit'
if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit'
if proc_peak > 1: reason = reason or 'process_count_limit'
if not errors and disk >= limits['disk_bytes']: reason = reason or 'disk_limit'
if reason: stop_group(child.pid)
pid, status, current_usage = os.wait4(child.pid, 0 if reason else os.WNOHANG)
if pid:
usage = current_usage; exit_status = status; break
time.sleep(poll)
finally:
stop_group(child.pid)
if exit_status is None:
_, exit_status, usage = os.wait4(child.pid, 0)
child.returncode = os.waitstatus_to_exitcode(exit_status)
orphan_cpu = 0.; orphan_rss = 0
while True:
try: _, _, u = os.wait4(-child.pid, 0)
except ChildProcessError: break
orphan_cpu += u.ru_utime + u.ru_stime
orphan_rss = max(orphan_rss, u.ru_maxrss*1024)
wall = time.monotonic()-start
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
rss_peak = max(rss_peak, usage.ru_maxrss*1024, orphan_rss)
# Inspect final peaks even when a numerical exception returned nonzero.
if charged > limits['cpu_seconds']: reason = reason or 'cpu_limit_after_exit'
if wall > limits['wall_seconds']: reason = reason or 'wall_limit_after_exit'
if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit_after_exit'
coordinator_cpu = time.process_time()-coordinator_cpu_start
if coordinator_cpu >= coordinator_cpu_limit: reason = reason or 'coordinator_cpu_limit_after_exit'
result = dict(coordinator_cpu_seconds=coordinator_cpu, coordinator_cpu_limit=coordinator_cpu_limit,
status='PASS' if child.returncode == 0 and reason is None else 'FAILED',
returncode=child.returncode, stop_reason=reason, cpu_seconds=charged,
wall_seconds=wall, peak_tree_rss_bytes=rss_peak, peak_process_count=proc_peak,
monitor_errors=errors, max_poll_gap_seconds=gap, reaped_orphan_cpu_seconds=orphan_cpu,
limits=limits, retry=False, gpu_visible=False)
save(dest/'worker.receipt.json', result)
return result
finally:
libc.prctl(36, previous.value, 0, 0, 0)
for signum, handler in prior.items(): signal.signal(signum, handler)
+28 -1
View File
@@ -1,4 +1,4 @@
import unittest,tempfile,json
import unittest,tempfile,json,time,sys,os
from pathlib import Path
from unittest.mock import patch
import numpy as np
@@ -6,6 +6,33 @@ from s2_tail_diagnostic_math import weights,tail_membership,tail_error,peak_diag
from s2_tail_diagnostic import config
class TailTests(unittest.TestCase):
def test_coordinator_budget_stops_and_reaps_before_worker_cap(self):
import s2_tail_diagnostic_guard as guard
original_usage=guard.tree_usage
def charged_monitor(pid):
until=time.process_time()+.04
while time.process_time()<until:pass
return original_usage(pid)
with tempfile.TemporaryDirectory() as d:
# Inject 89.98 seconds of prior coordinator work, then consume real CPU
# in the first monitor poll while a real synthetic child remains alive.
start=time.process_time()-89.98
with patch.object(guard,'tree_usage',side_effect=charged_monitor):
result=guard.supervise([sys.executable,'-c','import time; time.sleep(30)'],d,
dict(cpu_seconds=3300,wall_seconds=10,rss_bytes=2**28,disk_bytes=2**20),
coordinator_cpu_start=start,coordinator_cpu_limit=90.)
pid=json.loads((Path(d)/'worker-pid.json').read_text())['pid']
self.assertEqual(result['status'],'FAILED')
self.assertEqual(result['stop_reason'],'coordinator_cpu_limit')
self.assertGreaterEqual(result['coordinator_cpu_seconds'],90.)
self.assertLess(result['cpu_seconds'],3300.)
self.assertFalse(result['monitor_errors'])
self.assertLess(180+result['coordinator_cpu_seconds']+result['cpu_seconds'],3600)
with self.assertRaises(ProcessLookupError):os.kill(pid,0)
with self.assertRaises(ChildProcessError):os.waitpid(pid,os.WNOHANG)
self.assertEqual(json.loads((Path(d)/'worker.receipt.json').read_text()),result)
if os.environ.get('S2_TAIL_TEST_RECEIPT'):
Path(os.environ['S2_TAIL_TEST_RECEIPT']).write_text(json.dumps(dict(synthetic=True,injected_prior_coordinator_cpu_seconds=89.98,worker_is_sleep_only=True,child_killed_and_reaped=True,receipt=result),indent=2)+'\n')
def test_missing_light_review_refuses(self):
with tempfile.TemporaryDirectory() as d:
r=Path(d);(r/'plan.json').write_text(json.dumps(dict(approval='1794966078.809549',cpu_cap=3600,spec_sha256='05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521',spec_path='synthetic',code_sha256={})))