Enforce running coordinator CPU allowance for S2 diagnostic
This commit is contained in:
1 parent
b97674a1d4
commit
1ed44f70fc
21 files changed
+19182
-15
No files matched your search
@@ -21,9 +21,10 @@ def main():
|
||||
parts.append(dict(case=case,component_id=e['component_id'],fold=e['fold'],nodes=e['nodes'],t_set=cache['t_set'],sources=sources))
|
||||
spec=BASE/'references/2026-11-18/phase3-item-c/mold-tail-volume-peak-stability-spec.md'
|
||||
if sha(spec)!='05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521':raise ValueError('Spec identity')
|
||||
names=['s2_tail_diagnostic_math.py','s2_tail_diagnostic.py','prepare_s2_tail_diagnostic.py','test_s2_tail_diagnostic.py','r4bc_guard.py','s2_recovery_supervisor.py','s2_recovery_monitoring.py']
|
||||
names=['s2_tail_diagnostic_math.py','s2_tail_diagnostic.py','s2_tail_diagnostic_guard.py','prepare_s2_tail_diagnostic.py','test_s2_tail_diagnostic.py','r4bc_guard.py','s2_recovery_supervisor.py','s2_recovery_monitoring.py']
|
||||
codes={str(BASE/'tools'/n):sha(BASE/'tools'/n) for n in names};codes[str(BASE/'docs/s2-tail-diagnostic-plan-2026-11-18.md')]=sha(BASE/'docs/s2-tail-diagnostic-plan-2026-11-18.md')
|
||||
ROOT.mkdir();plan=dict(approval='1794966078.809549',spec_path=str(spec),spec_sha256=sha(spec),descriptive_only=True,gate=None,cpu_cap=3600,worker_cpu_cap=3300,setup_reserve=180,worker_cpu=24,coordinator_cpu=25,worker_rss_bytes=4*2**30,worker_wall_seconds=7200,parts=parts,code_sha256=codes,S2_score=dict(path=str(s/'score/result.json'),sha256=sha(s/'score/result.json')),metadata_sources={str(q):sha(q) for q in [s/'contract.json',s/'predictions-manifest.json',s/'predictions-git-freeze.json',s/'complete.json',Path(c['pilot_contract']),BASE/'artifacts/phase2-b13-e1b-2026-11-17/contract.json']},new_fits=0,new_predictions=0,no_retry=True,light_review_before_targets=True,original_S2_decision_unchanged=True)
|
||||
plan.update(coordinator_cpu_limit=90,shutdown_cpu_reserve=30)
|
||||
save(ROOT/'plan.json',plan)
|
||||
save(ROOT/'preparation.json',dict(status='PREPARED_NO_TARGET_ARRAYS_READ',parts=552,components=204,source_file_existence_checked=True,array_values_read=False,source_hashes_rechecked_at_execution=True,review_pending=True,launched=False,preparation_cpu_seconds=time.process_time()-start,unix=time.time()))
|
||||
print(dict(root=str(ROOT),plan_sha256=sha(ROOT/'plan.json'),code_files=len(codes),parts=552,components=204,review_pending=True))
|
||||
|
||||
@@ -101,13 +101,13 @@ def worker(root):
|
||||
|
||||
def run(root):
|
||||
import fcntl,psutil
|
||||
from r4bc_guard import supervise
|
||||
from s2_tail_diagnostic_guard import supervise
|
||||
start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);c=config(root);os.sched_setaffinity(0,[25]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():raise FileExistsError('One diagnostic attempt; no duplicate or retry')
|
||||
for p in psutil.process_iter(['pid','cmdline']):
|
||||
if p.pid!=os.getpid() and any(Path(x).name=='s2_tail_diagnostic.py' for x in p.info['cmdline'] or []):raise RuntimeError('Existing diagnostic')
|
||||
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),cpu_cap=3600,setup_reserve=180,descriptive_only=True,review_sha256=sha(root/'review-accepted.json'),cpu_start=dict(self=start,child_user=u0.ru_utime,child_system=u0.ru_stime)))
|
||||
guard=supervise([sys.executable,str(BASE/'tools/s2_tail_diagnostic.py'),'worker',str(root)],root/'guard',dict(cpu_seconds=3300,wall_seconds=7200,rss_bytes=4*2**30,disk_bytes=2**30))
|
||||
guard=supervise([sys.executable,str(BASE/'tools/s2_tail_diagnostic.py'),'worker',str(root)],root/'guard',dict(cpu_seconds=3300,wall_seconds=7200,rss_bytes=4*2**30,disk_bytes=2**30),coordinator_cpu_start=start,coordinator_cpu_limit=90.)
|
||||
u=resource.getrusage(resource.RUSAGE_CHILDREN);charge=180+time.process_time()-start+max(guard['cpu_seconds'],u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)
|
||||
ok=guard['status']=='PASS' and not guard['monitor_errors'] and charge<=3600
|
||||
if ok:
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Diagnostic-only copy of r4bc guard, adding a running coordinator CPU allowance.
|
||||
Shared guards and all scientific definitions stay unchanged.
|
||||
"""
|
||||
import argparse
|
||||
import ctypes
|
||||
import fcntl
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from s2_recovery_supervisor import save, tree_usage, stop_group
|
||||
from s2_recovery_monitoring import disk_usage
|
||||
|
||||
|
||||
def supervise(command, dest, limits, *, coordinator_cpu_start, coordinator_cpu_limit=90., global_stop=None, poll=.2):
|
||||
if not math.isfinite(coordinator_cpu_start) or not math.isfinite(coordinator_cpu_limit) or coordinator_cpu_limit <= 0:
|
||||
raise ValueError('Invalid coordinator CPU allowance')
|
||||
dest = Path(dest); dest.mkdir(parents=True, exist_ok=True)
|
||||
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
|
||||
raise ValueError('Invalid guard limit')
|
||||
with (dest/'execution.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if (dest/'worker.started.json').exists() or (dest/'worker.receipt.json').exists():
|
||||
raise FileExistsError('Single attempt; no restart')
|
||||
save(dest/'worker.started.json', dict(command=command, limits=limits, unix=time.time()))
|
||||
requested = [None]
|
||||
def stop(signum, frame):
|
||||
requested[0] = 'GUARD_SIGNAL_'+str(signum)
|
||||
prior = {s: signal.signal(s, stop) for s in (signal.SIGTERM, signal.SIGINT)}
|
||||
libc = ctypes.CDLL(None, use_errno=True); previous = ctypes.c_int()
|
||||
if libc.prctl(37, ctypes.byref(previous), 0, 0, 0) or libc.prctl(36, 1, 0, 0, 0):
|
||||
raise OSError('Cannot enable descendant accounting')
|
||||
start = time.monotonic(); last = start; reason = None; errors = []
|
||||
cpu_peak = rss_peak = proc_peak = gap = 0; usage = None; exit_status = None
|
||||
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', MKL_NUM_THREADS='1',
|
||||
NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
|
||||
try:
|
||||
with (dest/'worker.log').open('x') as log:
|
||||
child = subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT,
|
||||
start_new_session=True)
|
||||
save(dest/'worker-pid.json', dict(pid=child.pid, unix=time.time(), guard_pid=os.getpid()))
|
||||
try:
|
||||
while True:
|
||||
now = time.monotonic(); gap = max(gap, now-last); last = now
|
||||
try:
|
||||
cpu, rss, count = tree_usage(child.pid)
|
||||
disk = disk_usage(dest)
|
||||
cpu_peak = max(cpu_peak, cpu); rss_peak = max(rss_peak, rss); proc_peak = max(proc_peak, count)
|
||||
except Exception as exc:
|
||||
errors.append(dict(type=type(exc).__name__, message=str(exc)))
|
||||
reason = 'MONITOR_ERROR'
|
||||
if time.process_time()-coordinator_cpu_start >= coordinator_cpu_limit:
|
||||
reason = reason or 'coordinator_cpu_limit'
|
||||
if requested[0]: reason = reason or requested[0]
|
||||
if global_stop and Path(global_stop).exists(): reason = reason or 'GLOBAL_STOP_REQUEST'
|
||||
if now-start >= limits['wall_seconds']: reason = reason or 'wall_limit'
|
||||
if cpu_peak >= limits['cpu_seconds']: reason = reason or 'cpu_limit'
|
||||
if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit'
|
||||
if proc_peak > 1: reason = reason or 'process_count_limit'
|
||||
if not errors and disk >= limits['disk_bytes']: reason = reason or 'disk_limit'
|
||||
if reason: stop_group(child.pid)
|
||||
pid, status, current_usage = os.wait4(child.pid, 0 if reason else os.WNOHANG)
|
||||
if pid:
|
||||
usage = current_usage; exit_status = status; break
|
||||
time.sleep(poll)
|
||||
finally:
|
||||
stop_group(child.pid)
|
||||
if exit_status is None:
|
||||
_, exit_status, usage = os.wait4(child.pid, 0)
|
||||
child.returncode = os.waitstatus_to_exitcode(exit_status)
|
||||
orphan_cpu = 0.; orphan_rss = 0
|
||||
while True:
|
||||
try: _, _, u = os.wait4(-child.pid, 0)
|
||||
except ChildProcessError: break
|
||||
orphan_cpu += u.ru_utime + u.ru_stime
|
||||
orphan_rss = max(orphan_rss, u.ru_maxrss*1024)
|
||||
wall = time.monotonic()-start
|
||||
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
|
||||
rss_peak = max(rss_peak, usage.ru_maxrss*1024, orphan_rss)
|
||||
# Inspect final peaks even when a numerical exception returned nonzero.
|
||||
if charged > limits['cpu_seconds']: reason = reason or 'cpu_limit_after_exit'
|
||||
if wall > limits['wall_seconds']: reason = reason or 'wall_limit_after_exit'
|
||||
if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit_after_exit'
|
||||
coordinator_cpu = time.process_time()-coordinator_cpu_start
|
||||
if coordinator_cpu >= coordinator_cpu_limit: reason = reason or 'coordinator_cpu_limit_after_exit'
|
||||
result = dict(coordinator_cpu_seconds=coordinator_cpu, coordinator_cpu_limit=coordinator_cpu_limit,
|
||||
status='PASS' if child.returncode == 0 and reason is None else 'FAILED',
|
||||
returncode=child.returncode, stop_reason=reason, cpu_seconds=charged,
|
||||
wall_seconds=wall, peak_tree_rss_bytes=rss_peak, peak_process_count=proc_peak,
|
||||
monitor_errors=errors, max_poll_gap_seconds=gap, reaped_orphan_cpu_seconds=orphan_cpu,
|
||||
limits=limits, retry=False, gpu_visible=False)
|
||||
save(dest/'worker.receipt.json', result)
|
||||
return result
|
||||
finally:
|
||||
libc.prctl(36, previous.value, 0, 0, 0)
|
||||
for signum, handler in prior.items(): signal.signal(signum, handler)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import unittest,tempfile,json
|
||||
import unittest,tempfile,json,time,sys,os
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
import numpy as np
|
||||
@@ -6,6 +6,33 @@ from s2_tail_diagnostic_math import weights,tail_membership,tail_error,peak_diag
|
||||
from s2_tail_diagnostic import config
|
||||
|
||||
class TailTests(unittest.TestCase):
|
||||
def test_coordinator_budget_stops_and_reaps_before_worker_cap(self):
|
||||
import s2_tail_diagnostic_guard as guard
|
||||
original_usage=guard.tree_usage
|
||||
def charged_monitor(pid):
|
||||
until=time.process_time()+.04
|
||||
while time.process_time()<until:pass
|
||||
return original_usage(pid)
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
# Inject 89.98 seconds of prior coordinator work, then consume real CPU
|
||||
# in the first monitor poll while a real synthetic child remains alive.
|
||||
start=time.process_time()-89.98
|
||||
with patch.object(guard,'tree_usage',side_effect=charged_monitor):
|
||||
result=guard.supervise([sys.executable,'-c','import time; time.sleep(30)'],d,
|
||||
dict(cpu_seconds=3300,wall_seconds=10,rss_bytes=2**28,disk_bytes=2**20),
|
||||
coordinator_cpu_start=start,coordinator_cpu_limit=90.)
|
||||
pid=json.loads((Path(d)/'worker-pid.json').read_text())['pid']
|
||||
self.assertEqual(result['status'],'FAILED')
|
||||
self.assertEqual(result['stop_reason'],'coordinator_cpu_limit')
|
||||
self.assertGreaterEqual(result['coordinator_cpu_seconds'],90.)
|
||||
self.assertLess(result['cpu_seconds'],3300.)
|
||||
self.assertFalse(result['monitor_errors'])
|
||||
self.assertLess(180+result['coordinator_cpu_seconds']+result['cpu_seconds'],3600)
|
||||
with self.assertRaises(ProcessLookupError):os.kill(pid,0)
|
||||
with self.assertRaises(ChildProcessError):os.waitpid(pid,os.WNOHANG)
|
||||
self.assertEqual(json.loads((Path(d)/'worker.receipt.json').read_text()),result)
|
||||
if os.environ.get('S2_TAIL_TEST_RECEIPT'):
|
||||
Path(os.environ['S2_TAIL_TEST_RECEIPT']).write_text(json.dumps(dict(synthetic=True,injected_prior_coordinator_cpu_seconds=89.98,worker_is_sleep_only=True,child_killed_and_reaped=True,receipt=result),indent=2)+'\n')
|
||||
def test_missing_light_review_refuses(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
r=Path(d);(r/'plan.json').write_text(json.dumps(dict(approval='1794966078.809549',cpu_cap=3600,spec_sha256='05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521',spec_path='synthetic',code_sha256={})))
|
||||
|
||||
Reference in new issue
Block a user