Enforce running coordinator CPU allowance for S2 diagnostic

This commit is contained in:
MeltAstra committed 2026-11-18 10:27:28 +08:00
1 parent b97674a1d4
commit 1ed44f70fc
21 files changed
+19182 -15

No files matched your search

@@ -1,8 +1,9 @@
{
"status": "PASS",
"tests": 16,
"scope": "Synthetic arrays only; no scientific target or prediction values",
"log_sha256": "3d12870ac905775ea6a50a861ef2d64e297422975a29ff59c980daf1e8c0343f",
"tests": 17,
"scope": "Synthetic arrays and sleep-only child with injected prior coordinator CPU; no real scientific values",
"log_sha256": "5bb929569e925ef83030b7e755f370253b57c77751fd3c9e5eaee3742122b5e0",
"budget_negative_receipt_sha256": "100875a1205eaa63a6da81a80b118681fd2b42c2c2ff4d4cc7b304354606705f",
"real_data_read": false,
"unix": 1794968093.7768
"unix": 1794968825.439787
}
@@ -0,0 +1,8 @@
{
"status": "PASS",
"tests": 16,
"scope": "Synthetic arrays only; no scientific target or prediction values",
"log_sha256": "3d12870ac905775ea6a50a861ef2d64e297422975a29ff59c980daf1e8c0343f",
"real_data_read": false,
"unix": 1794968093.7768
}
@@ -0,0 +1,52 @@
{
"reviewer": "MeltAssist",
"kind": "light_code_spec_review",
"verdict": "BLOCKED",
"unix": 1794968645.5496745,
"implementation_reported": "b97674a1d481049363c78bbe647756ac40822e73",
"plan_sha256": "30e845417d3840ea194e791470eab0dc6821a819c6432b41c5766819606b015d",
"spec_sha256": "05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521",
"code_sha256": {
"/home/frank14f/optane/MeltMind/tools/s2_tail_diagnostic_math.py": "e7a4abcaa3001234c5244162c893711d54e8766e97db13476335dc1243a30bd9",
"/home/frank14f/optane/MeltMind/tools/s2_tail_diagnostic.py": "a9e93379c813124b5d596963e693fd8d675b92d1cd73c63fbf8dc9620811e6a5",
"/home/frank14f/optane/MeltMind/tools/prepare_s2_tail_diagnostic.py": "97200ce86bdec7867a69b681ec5092649fb49aba663d8476ea958d1d88c44270",
"/home/frank14f/optane/MeltMind/tools/test_s2_tail_diagnostic.py": "21b0c8d4070d440a4567c6d0f2307f75fb71c31170067b6f8aeac8210951c723",
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "437a756ddf10b125dce450f463e76e28df52724f8c5766e0a989dfa536805628",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"/home/frank14f/optane/MeltMind/docs/s2-tail-diagnostic-plan-2026-11-18.md": "18bb5bb782615203e939a615a9fbc8e75d2383eb4706ab1e8e273954dbe0b248"
},
"sources_read": [
"Claude1794966078.809549/h-c907cdaf",
"Melt1794968475.536719",
"full Mold05344bac specification",
"four diagnostic source/test files",
"r4bc_guard.supervise and its resource helpers",
"existing synthetic acceptance and attempt3 log"
],
"checks": {
"mathematical_spec": "PASS_STATIC",
"input_metadata": "552 unique parts / 204 components; 3312 source path/hash metadata matches to canonical S2/H1/pilot sources; metadata source hashes match",
"output_scope": "PASS_STATIC: original query scores retained; full-volume tails and same-domain certificates; no new fit/prediction/decode; descriptive only",
"synthetic_tests": "Existing 16 PASS log read, not executed by reviewer",
"runtime_monitor_errors": "Nonempty guard monitor_errors rejects completion"
},
"findings": [
{
"priority": "P2",
"blocking": true,
"file": "tools/s2_tail_diagnostic.py",
"line": 111,
"title": "Total CPU cap is checked only after supervised work exits",
"detail": "run passes only cpu_seconds=3300 for the worker to r4bc_guard.supervise. The supervisor samples child-tree CPU only. Coordinator/guard CPU is not limited while supervise runs. After return, run computes 180 + coordinator CPU + max(worker sampled CPU, reaped child CPU) and compares to 3600. Thus coordinator CPU can exceed the remaining 120 seconds while work continues, and a later STOPPED cannot enforce the authorized total cap. This is a static boundary gap, not an observed overspend.",
"requested_change": "Add a diagnostic-scoped running total CPU stop (or an enforced coordinator allowance with cleanup), retaining the existing worker cap, no retry, cumulative accounting and modest shutdown/receipt reserve. Add a small synthetic boundary check for coordinator cost causing the stop before worker3300; no real-target run required. Only the changed budget path and new source/plan hashes need follow-up review."
}
],
"limits": [
"No target or prediction arrays read or hashed by reviewer",
"No project imports, synthetic tests, model loading, diagnostic run or scientific recomputation",
"Input array identity checks were inspected statically; they remain runtime checks",
"Reported publication commit not independently fetched; review identifies actual local source bytes through plan hashes"
],
"scope_unchanged": "Existing item1 and postfit untouched; no five-field binding procedure added"
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,30 @@
"""Metadata-only preparation for Claude6078; never opens target arrays."""
import time
from pathlib import Path
from s2_tail_diagnostic import BASE,ROOT,read,save,sha
def main():
start=time.process_time()
if ROOT.exists():raise FileExistsError('Existing diagnostic root: inspect before any action')
s=Path('/home/frank14f/MeltMind-runs/phase2-h2-s2-postfit-2026-11-17');c=read(s/'contract.json');p=read(c['pilot_contract']);m=read(s/'predictions-manifest.json')['entries'];a=read(BASE/'artifacts/phase2-b13-e1b-2026-11-17/contract.json')['A_predictions'];oldrows={r['case']:r for r in c['rows']}
if read(s/'complete.json')['status']!='COMPLETE' or len(m)!=552 or len({e['case'] for e in m})!=552 or len({e['component_id'] for e in m})!=204:raise ValueError('Original completed S2 roster')
proof=read(s/'predictions-git-freeze.json')
if proof['files'][str(s/'predictions-manifest.json')]!=sha(s/'predictions-manifest.json'):raise ValueError('Original frozen prediction manifest')
parts=[]
for e in sorted(m,key=lambda e:e['case']):
case=e['case'];cache=p['parts'][case];aa=a[case];g=p['geometry'][case];row=oldrows[case]
if aa['protocol']!='grouped' or aa['pool']!=100 or aa['fold']!=e['fold'] or aa['nodes']!=e['nodes'] or g['nodes']!=e['nodes'] or row['component_id']!=e['component_id']:raise ValueError('A/H2 same domain and role')
sources={name:dict(path=d['path'],sha256=d['sha256']) for name,d in [('A',aa),('H2',e),('geometry',g)]}
for key,name in [('truth','seconds'),('mass','mass'),('raw_ids','raw_ids')]:sources[key]=dict(path=str((BASE/Path(cache['output'])/(name+'.npy')).resolve()),sha256=cache['output_sha256'][name+'.npy'])
for v in sources.values():
if not Path(v['path']).is_file():raise FileNotFoundError(v['path'])
parts.append(dict(case=case,component_id=e['component_id'],fold=e['fold'],nodes=e['nodes'],t_set=cache['t_set'],sources=sources))
spec=BASE/'references/2026-11-18/phase3-item-c/mold-tail-volume-peak-stability-spec.md'
if sha(spec)!='05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521':raise ValueError('Spec identity')
names=['s2_tail_diagnostic_math.py','s2_tail_diagnostic.py','prepare_s2_tail_diagnostic.py','test_s2_tail_diagnostic.py','r4bc_guard.py','s2_recovery_supervisor.py','s2_recovery_monitoring.py']
codes={str(BASE/'tools'/n):sha(BASE/'tools'/n) for n in names};codes[str(BASE/'docs/s2-tail-diagnostic-plan-2026-11-18.md')]=sha(BASE/'docs/s2-tail-diagnostic-plan-2026-11-18.md')
ROOT.mkdir();plan=dict(approval='1794966078.809549',spec_path=str(spec),spec_sha256=sha(spec),descriptive_only=True,gate=None,cpu_cap=3600,worker_cpu_cap=3300,setup_reserve=180,worker_cpu=24,coordinator_cpu=25,worker_rss_bytes=4*2**30,worker_wall_seconds=7200,parts=parts,code_sha256=codes,S2_score=dict(path=str(s/'score/result.json'),sha256=sha(s/'score/result.json')),metadata_sources={str(q):sha(q) for q in [s/'contract.json',s/'predictions-manifest.json',s/'predictions-git-freeze.json',s/'complete.json',Path(c['pilot_contract']),BASE/'artifacts/phase2-b13-e1b-2026-11-17/contract.json']},new_fits=0,new_predictions=0,no_retry=True,light_review_before_targets=True,original_S2_decision_unchanged=True)
save(ROOT/'plan.json',plan)
save(ROOT/'preparation.json',dict(status='PREPARED_NO_TARGET_ARRAYS_READ',parts=552,components=204,source_file_existence_checked=True,array_values_read=False,source_hashes_rechecked_at_execution=True,review_pending=True,launched=False,preparation_cpu_seconds=time.process_time()-start,unix=time.time()))
print(dict(root=str(ROOT),plan_sha256=sha(ROOT/'plan.json'),code_files=len(codes),parts=552,components=204,review_pending=True))
if __name__=='__main__':main()
@@ -0,0 +1,118 @@
"""Single attempt CPU guard; global stop requests retain wait4 accounting."""
import argparse
import ctypes
import fcntl
import json
import math
import os
from pathlib import Path
import signal
import subprocess
import sys
import time
from s2_recovery_supervisor import save, tree_usage, stop_group
from s2_recovery_monitoring import disk_usage
def supervise(command, dest, limits, *, global_stop=None, poll=.2):
dest = Path(dest); dest.mkdir(parents=True, exist_ok=True)
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
raise ValueError('Invalid guard limit')
with (dest/'execution.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (dest/'worker.started.json').exists() or (dest/'worker.receipt.json').exists():
raise FileExistsError('Single attempt; no restart')
save(dest/'worker.started.json', dict(command=command, limits=limits, unix=time.time()))
requested = [None]
def stop(signum, frame):
requested[0] = 'GUARD_SIGNAL_'+str(signum)
prior = {s: signal.signal(s, stop) for s in (signal.SIGTERM, signal.SIGINT)}
libc = ctypes.CDLL(None, use_errno=True); previous = ctypes.c_int()
if libc.prctl(37, ctypes.byref(previous), 0, 0, 0) or libc.prctl(36, 1, 0, 0, 0):
raise OSError('Cannot enable descendant accounting')
start = time.monotonic(); last = start; reason = None; errors = []
cpu_peak = rss_peak = proc_peak = gap = 0; usage = None; exit_status = None
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', MKL_NUM_THREADS='1',
NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
try:
with (dest/'worker.log').open('x') as log:
child = subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT,
start_new_session=True)
save(dest/'worker-pid.json', dict(pid=child.pid, unix=time.time(), guard_pid=os.getpid()))
try:
while True:
now = time.monotonic(); gap = max(gap, now-last); last = now
try:
cpu, rss, count = tree_usage(child.pid)
disk = disk_usage(dest)
cpu_peak = max(cpu_peak, cpu); rss_peak = max(rss_peak, rss); proc_peak = max(proc_peak, count)
except Exception as exc:
errors.append(dict(type=type(exc).__name__, message=str(exc)))
reason = 'MONITOR_ERROR'
if requested[0]: reason = reason or requested[0]
if global_stop and Path(global_stop).exists(): reason = reason or 'GLOBAL_STOP_REQUEST'
if now-start >= limits['wall_seconds']: reason = reason or 'wall_limit'
if cpu_peak >= limits['cpu_seconds']: reason = reason or 'cpu_limit'
if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit'
if proc_peak > 1: reason = reason or 'process_count_limit'
if not errors and disk >= limits['disk_bytes']: reason = reason or 'disk_limit'
if reason: stop_group(child.pid)
pid, status, current_usage = os.wait4(child.pid, 0 if reason else os.WNOHANG)
if pid:
usage = current_usage; exit_status = status; break
time.sleep(poll)
finally:
stop_group(child.pid)
if exit_status is None:
_, exit_status, usage = os.wait4(child.pid, 0)
child.returncode = os.waitstatus_to_exitcode(exit_status)
orphan_cpu = 0.; orphan_rss = 0
while True:
try: _, _, u = os.wait4(-child.pid, 0)
except ChildProcessError: break
orphan_cpu += u.ru_utime + u.ru_stime
orphan_rss = max(orphan_rss, u.ru_maxrss*1024)
wall = time.monotonic()-start
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
rss_peak = max(rss_peak, usage.ru_maxrss*1024, orphan_rss)
# Inspect final peaks even when a numerical exception returned nonzero.
if charged > limits['cpu_seconds']: reason = reason or 'cpu_limit_after_exit'
if wall > limits['wall_seconds']: reason = reason or 'wall_limit_after_exit'
if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit_after_exit'
result = dict(status='PASS' if child.returncode == 0 and reason is None else 'FAILED',
returncode=child.returncode, stop_reason=reason, cpu_seconds=charged,
wall_seconds=wall, peak_tree_rss_bytes=rss_peak, peak_process_count=proc_peak,
monitor_errors=errors, max_poll_gap_seconds=gap, reaped_orphan_cpu_seconds=orphan_cpu,
limits=limits, retry=False, gpu_visible=False)
save(dest/'worker.receipt.json', result)
return result
finally:
libc.prctl(36, previous.value, 0, 0, 0)
for signum, handler in prior.items(): signal.signal(signum, handler)
def run(root, index, slot):
from r4bc_common import contract, job_dir, read, sha
c = contract(root); os.sched_setaffinity(0, [c['worker_cpus'][slot]])
dest = job_dir(root, index)
receipt = supervise([sys.executable, 'tools/r4bc_fit.py', str(root), str(index)], dest,
c['fit_limits'], global_stop=Path(root)/'global-stop.json')
result_path = dest/'output/result.json'; failure_path = dest/'output/failure.json'
if receipt['status'] == 'PASS' and not result_path.exists():
receipt.update(status='FAILED', stop_reason='MISSING_RESULT')
failure = read(failure_path) if failure_path.exists() else None
global_failure = bool(receipt['monitor_errors']) or bool(failure and failure.get('global_stop'))
if global_failure and not (Path(root)/'global-stop.json').exists():
save(Path(root)/'global-stop.json', dict(reason='WORKER_INPUT_OR_MONITOR_FAILURE', job=index, failure=failure))
record = dict(index=index, job=c['jobs'][index], worker=receipt, global_failure=global_failure,
guard_cpu_seconds=time.process_time(), result_path=str(result_path) if result_path.exists() else None,
result_sha256=sha(result_path) if result_path.exists() else None)
save(dest/'guard-result.json', record)
return 0 if receipt['status'] == 'PASS' else 1
if __name__ == '__main__':
parser = argparse.ArgumentParser(); parser.add_argument('root', type=Path)
parser.add_argument('index', type=int); parser.add_argument('slot', type=int)
args = parser.parse_args(); raise SystemExit(run(args.root, args.index, args.slot))
@@ -0,0 +1,17 @@
# Existing S2 A/H2 tail diagnostic — light review before execution
Claude1794966078.809549/h-c907cdaf authorizes this descriptive-only CPU diagnostic on the already seen552 parts/204 components, under3600 CPU-seconds. It has no gate, does not reverse the S2 decision, starts no item c fit, and changes neither item1 nor its postfit barriers. Mold's full05344bac specification is normative; future c also retains that specification.
Entry `tools/s2_tail_diagnostic.py run artifacts/phase2-h2-s2-tail-diagnostic-2026-11-18`, original reproduction venv, project cwd. `prepare_s2_tail_diagnostic.py` reads metadata only. Before any target computation, MeltAssist performs a light code/spec check. An executor receipt records that actual review, source receipt hash, current plan hash and source map; the entry verifies it. No five-field implementation/proof ceremony or new checkpoint review. The unmodified shared CPU guard is reused, one attempt, CPU24 worker/25 coordinator, no GPU/model loading/inference or labels decoded from archives.
H2 full fields are the canonical frozen S2 manifest (including exact historical S1 fold0). Full A comes from the corresponding frozen H1 `A_seconds` fields already used to form the S2 baseline. Every case verifies both raw-ID arrays and complete node counts, geometry and cached lumped mass equality, original query membership, and exact A/H2 query values against stored S2 values before reading the existing seconds.npy target once. Input hashes are checked at execution. No predictor is run. The old S2 query last-location values are copied unchanged for comparison, separately from new geometry-domain diagnostics.
Each case normalizes time by original t_set, keeps negative predictions, obtains mass w from existing m_i=Σ|K|/4, and computes true/A/H2 fractional last1% volume tails. Raw-ID ordering does not cut a plateau. Stable compensated sums determine plateau mass, residual and total tail mass;10^-12 correction/volume bounds are enforced and recorded. `E_tail=(Miss+False)/2` is computed with the equivalent `1−I/.01` value and floating-point identity discrepancy also retained; min intersection ensures identical fractional tails have exactly zero directional loss. Constant truth stays in the roster with its marker. No query multiplicity enters volume measure.
Peak certificates and categories follow the same full/query domains, ε, r=.10Rg, strictγ>2ε, 64u near-boundary rule, exact raw-ID peak ties, peak-set/tail-support distances and true-time drop. Positive-infiniteγ from an empty exterior is stored as null plus explicit gamma_positive_infinity/trivial_radius=true so JSON contains no invalid infinity. The strict certificate and its boundary-robust interpretation are both kept. Failure of a certificate does not imply a branch error; no branch is identified from Euclidean distance.
Outputs include552 per-part JSON records, both arm tail membership threshold/volume/tie summaries, Miss/False/E_tail, query/full diagnostics, original query last scores, negative counts and target-access log. Summaries give component-equal paired H2−A differences and2000 bootstrap intervals/seed2026111408 for tail, original point and continuous diagnostics, shared draws on identical component support. Undefined values exclude the whole affected component for that metric with denominators reported; no missing component becomes a zero. Categories and certificates are reported as part counts with denominators; categories additionally have component-equal fractions. These are exploratory descriptions, not a decision rule or proof of mechanism. Full predictions are reused, not copied or rewritten.
Budgets:3600 total CPU-s, including180 setup reserve (covers metadata preparation and synthetic tests), a3300-CPU-s serial worker and actual coordinator/reaped-child accounting. The worker has7200 wall-seconds and4GiB RSS; guard output is capped at1GiB. Per-part diagnostic output is small; no full-field copies or pairwise distance matrices. A numerical/input/hash error stops the batch and preserves partial records; no silent retry or fabricated552 table. Review and publication overhead are disclosed separately from the scientific worker; they run no scientific computation.
Synthetic acceptance is required before review: fractional identical/disjoint tails, constant truth, one heavy node/nonuniform weights, ID permutation, monotone transformation, duplicate query records, zero-remainder boundary, strict certificate equality, sufficient-certificate/nonconverse/empty exterior, exact other-peak and near-tail categories, plus missing/mismatched review rejection. Synthetic arrays only; no real target values before the light review. No existing training or postfit source is edited.
@@ -0,0 +1,40 @@
"""Race-tolerant sampled infrastructure monitoring, without worker changes."""
import json
from pathlib import Path
import stat
import sys
import time
def disk_usage(root):
logical=allocated=0
for p in Path(root).rglob('*'):
try:s=p.stat(follow_symlinks=False)
except FileNotFoundError:continue # atomic progress replacement / vanished entry
if stat.S_ISREG(s.st_mode):
logical+=s.st_size;allocated+=s.st_blocks*512
return max(logical,allocated)
class Monitor:
def __init__(self,path):self.path=Path(path);self.errors=0;self.last_error=None
def record(self,operation,exc):
self.errors+=1;self.last_error=dict(unix=time.time(),operation=operation,type=type(exc).__name__,message=str(exc))
try:
with self.path.open('a') as f:f.write(json.dumps(self.last_error)+'\n')
except Exception as log_exc:
print(json.dumps(dict(monitor_error=self.last_error,logging_error=str(log_exc))),file=sys.stderr,flush=True)
def sample(self,operation,fn,previous):
try:return fn(),True
except Exception as exc:self.record(operation,exc);return previous,False
def retry(self,operation,fn,poll):
while True:
value,ok=self.sample(operation,fn,None)
if ok:return value
time.sleep(poll)
def budget_reason(prior_cpu,settled_cpu,live_cpu,disk_bytes,*,cpu_limit=60*3600,disk_limit=100*2**30):
if prior_cpu+settled_cpu+live_cpu>=cpu_limit:return 'GLOBAL_CPU_LIMIT'
if disk_bytes>=disk_limit:return 'GLOBAL_DISK_LIMIT'
return None
@@ -0,0 +1,158 @@
"""Sequential E1 job guard: tree CPU/RSS, wall, process count and disk receipts.
Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot
is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged.
"""
import fcntl
import ctypes
import json
import math
import os
from pathlib import Path
import signal
import subprocess
import time
import psutil
from s2_recovery_monitoring import disk_usage,Monitor
def save(path, data):
path = Path(path)
tmp = path.with_suffix(path.suffix + '.tmp')
tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
tmp.replace(path)
def tree_usage(pid):
try:
root = psutil.Process(pid)
processes = [root] + root.children(recursive=True)
except psutil.NoSuchProcess:
return 0., 0, 0
cpu = 0.
rss = count = 0
for p in processes:
try:
t = p.cpu_times()
# Linux child times include reaped descendants, not live children.
cpu += t.user + t.system + t.children_user + t.children_system
rss += p.memory_info().rss
count += 1
except psutil.NoSuchProcess:
pass
return cpu, rss, count
def stop_group(pid):
try:
os.killpg(pid, signal.SIGKILL)
except ProcessLookupError:
pass
def supervise(command, root, name, limits, *, poll=.02):
"""One locked attempt; no automatic retry/replacement; account failed CPU."""
root = Path(root)
root.mkdir(parents=True, exist_ok=True)
monitor=Monitor(root/(name+'.monitor-errors.jsonl'))
with (root/'execution.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
started_path = root/(name+'.started.json')
receipt_path = root/(name+'.receipt.json')
if started_path.exists() or receipt_path.exists():
raise FileExistsError('Attempt already exists; no silent retry')
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
raise ValueError('Every limit must be finite and positive')
ledger_path = root/'ledger.json'
ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]}
if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']:
raise RuntimeError('Total CPU exhausted before spawn')
if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']:
raise RuntimeError('Disk exhausted before spawn')
save(started_path, {'command':command,'limits':limits,'started_unix':time.time()})
start = last = time.monotonic()
cpu_peak = rss_peak = max_processes = max_gap = 0
cpu=rss=count=used_disk=0
reason = None
exit_status = usage = None
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1',
MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
with (root/(name+'.log')).open('x') as log:
libc = ctypes.CDLL(None, use_errno=True)
previous_subreaper = ctypes.c_int()
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping')
proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT,
env=env, start_new_session=True)
try:
while True:
now = time.monotonic()
max_gap = max(max_gap, now-last)
last = now
(cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count))
cpu_peak = max(cpu_peak,cpu)
rss_peak = max(rss_peak,rss)
max_processes = max(max_processes,count)
used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk)
if now-start >= limits['job_wall_seconds']:
reason = 'wall_limit'
elif rss > limits['tree_rss_bytes']:
reason = 'tree_rss_limit'
elif count > limits['max_processes']:
reason = 'process_count_limit'
elif used_disk >= limits['disk_bytes']:
reason = 'disk_limit'
elif cpu_peak >= limits['job_cpu_seconds']:
reason = 'job_cpu_limit'
elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']:
reason = 'total_cpu_limit'
if reason:
stop_group(proc.pid)
pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG)
if pid:
exit_status, usage = status, child_usage
break
time.sleep(poll)
finally:
# No detached work may outlive a supervised attempt.
stop_group(proc.pid)
if exit_status is None:
_, exit_status, usage = os.wait4(proc.pid, 0)
proc.returncode = os.waitstatus_to_exitcode(exit_status)
orphan_cpu = 0.
orphan_peak = 0
# Killed descendants are adopted here rather than losing their
# final CPU usage when the root worker cannot reap them itself.
while True:
try:
_, _, orphan_usage = os.wait4(-proc.pid, 0)
except ChildProcessError:
break
orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime
orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024)
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state')
elapsed = time.monotonic()-start
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak)
used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll)
if reason is None:
if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit'
elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit'
elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit'
elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit'
elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit'
receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED',
command=command,returncode=proc.returncode,stop_reason=reason,
cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak,
peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk,
max_poll_gap_seconds=max_gap,poll_target_seconds=poll,
reaped_orphan_cpu_seconds=orphan_cpu,
supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure',
gpu_visible=False,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error)
ledger['cpu_seconds_charged'] += charged
ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged})
monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll)
monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll)
return receipt
@@ -0,0 +1,122 @@
"""One reviewed descriptive CPU pass over existing S2 predictions; no inference."""
import hashlib,json,math,os,resource,sys,time
from pathlib import Path
import numpy as np
from s2_tail_diagnostic_math import weights,tail_membership,tail_error,peak_diagnostic
BASE=Path('/home/frank14f/optane/MeltMind')
ROOT=BASE/'artifacts/phase2-h2-s2-tail-diagnostic-2026-11-18'
def sha(path):
h=hashlib.sha256()
with Path(path).open('rb') as f:
for b in iter(lambda:f.read(2**20),b''):h.update(b)
return h.hexdigest()
def read(path):return json.loads(Path(path).read_text())
def save(path,data):
p=Path(path);p.parent.mkdir(parents=True,exist_ok=True);tmp=p.with_suffix(p.suffix+'.tmp');tmp.write_text(json.dumps(data,indent=2,allow_nan=False)+'\n');tmp.replace(p)
def checked(path,h):
p=Path(path)
if sha(p)!=h:raise ValueError('Source hash mismatch: '+str(p))
return p
def config(root,review=True):
c=read(root/'plan.json')
if c['approval']!='1794966078.809549' or c['cpu_cap']!=3600 or c['spec_sha256']!='05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521':raise PermissionError('Fixed descriptive scope')
for p,h in c['code_sha256'].items():checked(p,h)
checked(c['spec_path'],c['spec_sha256'])
if review:
a=read(root/'review-accepted.json')
if a['verdict']!='PASS' or a['plan_sha256']!=sha(root/'plan.json') or a['code_sha256']!=c['code_sha256']:raise PermissionError('Actual lightweight review required')
checked(a['source_receipt'],a['source_receipt_sha256'])
return c
def field(row,arm,name):
if name in ['E_tail','Miss','False_alarm']:return row['arms'][arm]['tail'][name]
if name=='original_query_last':return row['arms'][arm]['original_query_last']
domain,key=name.split('.',1);return row['arms'][arm][domain][key]
def summarize(rows):
metrics=['E_tail','Miss','False_alarm','original_query_last']+[d+'.'+k for d in ['query','full'] for k in ['peak_distance_over_Rg','d_peakset_over_Rg','epsilon','true_time_drop_at_predicted_peak']]+['full.d_tail_over_Rg']
groups={}
for row in rows:groups.setdefault(row['component_id'],[]).append(row)
result={};all_ids=sorted(groups)
for metric in metrics:
ids=[];values={'A':[],'H2':[]};excluded=[]
for cid in all_ids:
part={arm:[field(r,arm,metric) for r in groups[cid]] for arm in values}
if any(v is None or not math.isfinite(v) for vs in part.values() for v in vs):excluded.append(cid);continue
ids.append(cid)
for arm in values:values[arm].append(float(np.mean(part[arm])))
if not ids:
result[metric]=dict(A=None,H2=None,difference=None,low=None,high=None,components=0,excluded=excluded);continue
a,h=np.array(values['A']),np.array(values['H2']);delta=h-a
rng=np.random.Generator(np.random.PCG64(np.random.SeedSequence(2026111408)));draw=rng.integers(0,len(ids),size=(2000,len(ids)))
boot=delta[draw].mean(1);lo,hi=np.percentile(boot,[2.5,97.5],method='linear')
result[metric]=dict(A=float(a.mean()),H2=float(h.mean()),difference=float(delta.mean()),low=float(lo),high=float(hi),components=len(ids),paired_parts=sum(len(groups[i]) for i in ids),excluded=excluded,undefined_bootstrap=int(np.sum(~np.isfinite(boot))))
diagnostics={}
for domain in ['query','full']:
diagnostics[domain]={}
for arm in ['A','H2']:
rs=[r['arms'][arm][domain] for r in rows];categories=sorted({r['category'] for r in rs})
diagnostics[domain][arm]=dict(parts=len(rs),certified=sum(r['certified'] for r in rs),stable_certificate=sum(r['stable_certificate'] for r in rs),near_boundary=sum(r['near_boundary'] for r in rs),trivial_radius=sum(r['trivial_radius'] for r in rs),constant_truth=sum(r['constant_truth'] for r in rs),categories={k:dict(parts=sum(r['category']==k for r in rs),component_mean_fraction=float(np.mean([np.mean([r['arms'][arm][domain]['category']==k for r in gr]) for gr in groups.values()]))) for k in categories})
ties=dict(truth_threshold_tied_parts=sum(r['truth_tail']['plateau_nodes']>1 for r in rows),arms={a:dict(prediction_threshold_tied_parts=sum(r['arms'][a]['prediction_tail']['plateau_nodes']>1 for r in rows),max_threshold_mass_correction=max(abs(r['arms'][a]['prediction_tail']['mass_correction']) for r in rows)) for a in ['A','H2']})
return dict(descriptive_only=True,gate=None,S2_decision_unchanged=True,parts=len(rows),components=len(groups),metrics=result,diagnostics=diagnostics,bootstrap=dict(replicates=2000,seed=2026111408,joint_indices_when_same_component_support=True,unit='evaluation component, not training randomness'),constant_truth_parts=sum(r['truth_tail']['constant_field'] for r in rows),tie_coverage=ties)
def worker(root):
c=config(root);os.sched_setaffinity(0,[24])
if os.environ.get('CUDA_VISIBLE_DEVICES','')!='':raise PermissionError('CPU only')
out=root/'output';out.mkdir(exist_ok=False);cpu=time.process_time();rows=[]
# Existing single-point scores are retained, not replaced by the new endpoint.
old={r['case']:r for r in read(checked(c['S2_score']['path'],c['S2_score']['sha256']))['rows']}
save(out/'review-consumed.json',dict(plan_sha256=sha(root/'plan.json'),review_acceptance_sha256=sha(root/'review-accepted.json'),before_target_arrays=True,unix=time.time()))
for item in c['parts']:
case=item['case'];source=item['sources']
for rec in source.values():checked(rec['path'],rec['sha256'])
with np.load(source['H2']['path'],allow_pickle=False) as z:
raw=z['raw_ids'].copy();h2=z['seconds'].astype(np.float64);qid=z['query_raw_ids'].copy();query_a=z['A'].copy();query_h=z['pred'].copy()
with np.load(source['A']['path'],allow_pickle=False) as z:
if not np.array_equal(raw,z['raw_ids']):raise ValueError('Full A/H2 raw-ID identity')
a=z['A_seconds'].astype(np.float64)
with np.load(source['geometry']['path'],allow_pickle=False) as z:
if not np.array_equal(raw,z['raw_ids']):raise ValueError('Geometry prediction raw-ID identity')
points=z['points'].copy();mass=z['mass'].copy();rg=float(z['rg'])
cache_raw=np.load(source['raw_ids']['path'],allow_pickle=False);cache_mass=np.load(source['mass']['path'],allow_pickle=False)
if not np.array_equal(raw,cache_raw) or not np.array_equal(mass,cache_mass) or len(raw)!=item['nodes'] or np.any(np.diff(raw)<=0):raise ValueError('Complete retained-tet domain/mass')
ix=np.searchsorted(raw,qid)
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],qid) or len(np.unique(qid))!=len(qid):raise ValueError('Exact unique query identity')
if not np.array_equal(a[ix],query_a) or not np.array_equal(h2[ix],query_h):raise ValueError('Original S2 A/H2 query values differ from reused full fields')
if not math.isfinite(item['t_set']) or item['t_set']<=0 or not np.isfinite(a).all() or not np.isfinite(h2).all():raise ValueError('Finite prediction / time denominator')
with (out/'target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,path=source['truth']['path'],sha256=source['truth']['sha256'],unix=time.time()))+'\n')
truth=np.load(source['truth']['path'],allow_pickle=False).astype(np.float64)/item['t_set']
if truth.shape!=raw.shape or not np.isfinite(truth).all():raise ValueError('Full target domain finite values')
w=weights(mass);ta,td=tail_membership(truth,w);arms={}
for arm,pred in [('A',a/item['t_set']),('H2',h2/item['t_set'])]:
pa,pd=tail_membership(pred,w)
arms[arm]=dict(tail=tail_error(ta,pa,w),prediction_tail=pd,full=peak_diagnostic(truth,pred,points,raw,rg,w=w,true_tail=ta),query=peak_diagnostic(truth[ix],pred[ix],points[ix],raw[ix],rg),original_query_last=old[case][arm]['last_location']['value'],negative_full_predictions=int(np.sum(pred<0)))
row=dict(case=case,component_id=item['component_id'],fold=item['fold'],nodes=len(raw),query_nodes=len(ix),t_set=item['t_set'],Rg=rg,truth_tail=td,arms=arms)
save(out/'parts'/(case+'.json'),row);rows.append(row);save(out/'progress.json',dict(parts=len(rows),worker_cpu_seconds=time.process_time()-cpu,unix=time.time()))
if len(rows)!=552 or len({r['component_id'] for r in rows})!=204:raise ValueError('Original complete S2 denominator')
summary=summarize(rows);save(out/'result.json',dict(status='COMPLETE',summary=summary,rows=rows,new_fits=0,new_predictions=0,target_arrays_read=552,new_decodes=0,negative_clipping=False,worker_cpu_seconds=time.process_time()-cpu))
def run(root):
import fcntl,psutil
from r4bc_guard import supervise
start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);c=config(root);os.sched_setaffinity(0,[25]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('One diagnostic attempt; no duplicate or retry')
for p in psutil.process_iter(['pid','cmdline']):
if p.pid!=os.getpid() and any(Path(x).name=='s2_tail_diagnostic.py' for x in p.info['cmdline'] or []):raise RuntimeError('Existing diagnostic')
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),cpu_cap=3600,setup_reserve=180,descriptive_only=True,review_sha256=sha(root/'review-accepted.json'),cpu_start=dict(self=start,child_user=u0.ru_utime,child_system=u0.ru_stime)))
guard=supervise([sys.executable,str(BASE/'tools/s2_tail_diagnostic.py'),'worker',str(root)],root/'guard',dict(cpu_seconds=3300,wall_seconds=7200,rss_bytes=4*2**30,disk_bytes=2**30))
u=resource.getrusage(resource.RUSAGE_CHILDREN);charge=180+time.process_time()-start+max(guard['cpu_seconds'],u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)
ok=guard['status']=='PASS' and not guard['monitor_errors'] and charge<=3600
if ok:
result=read(root/'output/result.json');ok=result['status']=='COMPLETE' and result['summary']['parts']==552 and result['summary']['components']==204
save(root/'complete.json',dict(status='COMPLETE' if ok else 'STOPPED',guard=guard,cumulative_cpu_seconds=charge,cpu_cap=3600,new_fits=0,new_predictions=0,descriptive_only=True,no_gate=True,S2_decision_unchanged=True,retry=False,output_sha256=sha(root/'output/result.json') if (root/'output/result.json').exists() else None))
if not ok:raise RuntimeError('Diagnostic guard/coverage failed; preserve receipts')
if __name__=='__main__':
root=Path(sys.argv[2]) if len(sys.argv)>2 else ROOT
if sys.argv[1]=='worker':worker(root)
elif sys.argv[1]=='run':run(root)
else:raise ValueError('worker or run')
@@ -0,0 +1,109 @@
"""05344bac discrete volume tail and same-domain peak certificates; CPU only."""
import math
import numpy as np
ALPHA = .01
RADIUS = .10
MASS_TOL = 1e-12
def weights(mass):
m = np.asarray(mass, np.float64)
if m.ndim != 1 or not len(m) or not np.isfinite(m).all() or np.any(m <= 0):
raise ValueError('Finite positive lumped mass required')
total = math.fsum(m)
if not math.isfinite(total) or total <= 0:
raise ValueError('Invalid volume denominator')
return m / total
def tail_membership(values, w):
s = np.asarray(values, np.float64)
if s.ndim != 1 or s.shape != w.shape or not np.isfinite(s).all():
raise ValueError('Finite full-domain field required')
# Locate a candidate boundary; all final plateau masses use compensated sums.
order = np.argsort(-s, kind='stable')
at = min(int(np.searchsorted(np.cumsum(w[order]), ALPHA)), len(s)-1)
q = float(s[order[at]])
above, tied = s > q, s == q
high, equal = math.fsum(w[above]), math.fsum(w[tied])
remainder = ALPHA-high
fixed = min(max(remainder, 0.), equal)
correction = fixed-remainder
if equal <= 0 or abs(correction) > MASS_TOL:
raise ValueError('Tail boundary exceeds fixed mass correction tolerance')
fraction = fixed/equal
a = above.astype(np.float64)
a[tied] = fraction
actual = math.fsum(w*a)
if abs(actual-ALPHA) > MASS_TOL:
raise ValueError('Tail volume differs from alpha')
return a, dict(threshold=q, plateau_volume=equal, plateau_nodes=int(np.sum(tied)), plateau_fraction=fraction,
mass_correction=correction, tail_volume=actual,
constant_field=bool(np.all(s == s[0])))
def tail_error(a, b, w):
intersection = math.fsum(w*np.minimum(a, b))
miss = math.fsum(w*np.maximum(a-b, 0.))/ALPHA
false = math.fsum(w*np.maximum(b-a, 0.))/ALPHA
# Mean directional loss is numerically robust at identical fractional tails.
error = (miss+false)/2
identity = 1-intersection/ALPHA
if abs(error-identity) > 2*MASS_TOL/ALPHA or abs(miss-false) > 2*MASS_TOL/ALPHA:
raise ValueError('Equal tail mass / intersection identity')
if not 0 <= error <= 1+2*MASS_TOL/ALPHA:
raise ValueError('Tail error outside range')
return dict(E_tail=error, Miss=miss, False_alarm=false, intersection=intersection,
intersection_formula=identity, finite_precision_identity_difference=error-identity)
def peak_diagnostic(truth, pred, points, raw, rg, *, w=None, true_tail=None):
t, p, x = np.asarray(truth, np.float64), np.asarray(pred, np.float64), np.asarray(points, np.float64)
raw = np.asarray(raw)
if t.ndim != 1 or p.shape != t.shape or raw.shape != t.shape or x.shape != (len(t),3) or not len(t):
raise ValueError('Peak domain shape')
if not all(np.isfinite(v).all() for v in [t,p,x]) or not math.isfinite(rg) or rg <= 0:
raise ValueError('Peak finite domain and positive Rg')
# Query duplicate records must not change maxima, epsilon or distances.
ids, first, inverse = np.unique(raw, return_index=True, return_inverse=True)
if len(ids) != len(raw):
if w is not None or true_tail is not None:
raise ValueError('Full volume domain requires unique IDs')
if not np.array_equal(t, t[first][inverse]) or not np.array_equal(p, p[first][inverse]) or not np.array_equal(x, x[first][inverse]):
raise ValueError('Conflicting duplicated query identity')
t,p,x,raw = t[first],p[first],x[first],ids
top, predicted_top = np.flatnonzero(t == t.max()), np.flatnonzero(p == p.max())
y = int(top[np.argmin(raw[top])]); yh = int(predicted_top[np.argmin(raw[predicted_top])])
epsilon = float(np.max(np.abs(p-t)))
distances = np.linalg.norm(x-x[y], axis=1)/rg
exterior = distances > RADIUS
trivial = not bool(exterior.any())
gamma = None if trivial else float(t[y]-np.max(t[exterior]))
margin = None if trivial else gamma-2*epsilon
eta = float(64*np.finfo(np.float64).eps*max(1.,float(np.max(np.abs(t))),float(np.max(np.abs(p))),2*epsilon))
certified = trivial or gamma > 2*epsilon
near = bool(not trivial and abs(margin) <= eta)
distance = float(distances[yh])
if certified and distance > RADIUS:
raise ValueError('Certificate contradicts observed peak distance')
peakset = float(np.min(np.linalg.norm(x[top]-x[yh], axis=1))/rg)
taildist = None if true_tail is None else float(np.min(np.linalg.norm(x[true_tail > 0]-x[yh], axis=1))/rg)
if distance > RADIUS and peakset == 0:
category = 'OTHER_EXACT_TRUE_PEAK'
elif peakset <= RADIUS:
category = 'NEAR_TRUE_PEAK'
elif taildist is None:
category = 'FAR_TRUE_PEAK_QUERY'
elif taildist <= RADIUS:
category = 'NEAR_TRUE_TAIL_AWAY_FROM_PEAK'
else:
category = 'FAR_TRUE_TAIL'
return dict(nodes=len(t),epsilon=epsilon,gamma=gamma,gamma_positive_infinity=trivial,
gamma_minus_2epsilon=margin,eta=eta,near_boundary=near,trivial_radius=trivial,
certified=certified,stable_certificate=certified and not near,
peak_distance_over_Rg=distance,d_peakset_over_Rg=peakset,d_tail_over_Rg=taildist,
true_peak_raw_id=int(raw[y]),predicted_peak_raw_id=int(raw[yh]),
true_peak_count=len(top),predicted_peak_count=len(predicted_top),
true_peak_volume=None if w is None else math.fsum(w[top]),
predicted_peak_volume=None if w is None else math.fsum(w[predicted_top]),
true_time_drop_at_predicted_peak=float(t[y]-t[yh]),
predicted_minus_true_peak_time=float(p[yh]-t[y]),
constant_truth=bool(np.all(t==t[0])),category=category,
topology_branch_identified=False)
@@ -0,0 +1,58 @@
import unittest,tempfile,json
from pathlib import Path
from unittest.mock import patch
import numpy as np
from s2_tail_diagnostic_math import weights,tail_membership,tail_error,peak_diagnostic
from s2_tail_diagnostic import config
class TailTests(unittest.TestCase):
def test_missing_light_review_refuses(self):
with tempfile.TemporaryDirectory() as d:
r=Path(d);(r/'plan.json').write_text(json.dumps(dict(approval='1794966078.809549',cpu_cap=3600,spec_sha256='05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521',spec_path='synthetic',code_sha256={})))
with patch('s2_tail_diagnostic.checked'):
with self.assertRaises(FileNotFoundError):config(r)
def test_wrong_light_review_plan_refuses(self):
with tempfile.TemporaryDirectory() as d:
r=Path(d);(r/'plan.json').write_text(json.dumps(dict(approval='1794966078.809549',cpu_cap=3600,spec_sha256='05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521',spec_path='synthetic',code_sha256={})));(r/'review-accepted.json').write_text(json.dumps(dict(verdict='PASS',plan_sha256='wrong',code_sha256={})))
with patch('s2_tail_diagnostic.checked'):
with self.assertRaises(PermissionError):config(r)
def test_identical_fractional_plateau(self):
w=weights([1,1,2]);a,d=tail_membership(np.array([0,1,1]),w)
self.assertEqual(a[1],a[2]);self.assertEqual(tail_error(a,a,w)['E_tail'],0)
def test_disjoint_tails(self):
w=weights(np.ones(200));a,_=tail_membership(np.arange(200),w);b,_=tail_membership(-np.arange(200),w)
self.assertAlmostEqual(tail_error(a,b,w)['E_tail'],1)
def test_constant_truth_and_heavy_node(self):
w=weights([9,1]);a,d=tail_membership(np.ones(2),w);np.testing.assert_allclose(a,.01);self.assertTrue(d['constant_field'])
b,_=tail_membership(np.array([1,0]),w);self.assertAlmostEqual(b[0],.01/.9);self.assertAlmostEqual(np.dot(w,b),.01)
def test_nonuniform_and_ID_reorder(self):
s=np.array([2.,3.,3.,1.]);w=weights([.1,.7,.1,.1]);a,_=tail_membership(s,w);perm=np.array([3,1,0,2]);b,_=tail_membership(s[perm],w[perm]);np.testing.assert_array_equal(a[perm],b)
def test_zero_remainder_boundary(self):
w=weights([.01,.02,.97]);a,d=tail_membership(np.array([2.,1.,0.]),w);np.testing.assert_allclose(a,[1,0,0]);self.assertAlmostEqual(d['tail_volume'],.01)
def test_invalid_mass_and_nan(self):
for mass in [[0,1],[-1,2],[float('nan'),1],[]]:
with self.assertRaises(ValueError):weights(mass)
with self.assertRaises(ValueError):tail_membership(np.array([np.nan]),weights([1]))
def diag(self,t,p,raw=None,x=None):
return peak_diagnostic(t,p,np.array(x if x is not None else [[0,0,0],[1,0,0]]),np.array(raw if raw is not None else [0,1]),1.)
def test_strict_equality_not_certified(self):
d=self.diag([1,0],[.5,.5]);self.assertFalse(d['certified']);self.assertTrue(d['near_boundary'])
def test_certificate_and_nonconverse(self):
d=self.diag([1,0],[.9,.1]);self.assertTrue(d['certified']);self.assertLessEqual(d['peak_distance_over_Rg'],.1)
d=self.diag([1,0],[4,0]);self.assertFalse(d['certified']);self.assertEqual(d['peak_distance_over_Rg'],0)
def test_query_duplication(self):
a=self.diag([1,0],[.9,.1]);b=self.diag([1,0,1],[.9,.1,.9],raw=[0,1,0],x=[[0,0,0],[1,0,0],[0,0,0]]);self.assertEqual(a,b)
def test_empty_exterior(self):
d=self.diag([1,0],[0,1],x=[[0,0,0],[.01,0,0]]);self.assertTrue(d['trivial_radius']);self.assertIsNone(d['gamma']);self.assertTrue(d['certified'])
def test_strict_JSON_output(self):
for d in [self.diag([1,0],[.5,.5]),self.diag([1,0],[0,1],x=[[0,0,0],[.01,0,0]])]:self.assertEqual(json.loads(json.dumps(d,allow_nan=False)),d)
def test_other_exact_peak(self):
d=self.diag([1,1],[0,1]);self.assertEqual(d['category'],'OTHER_EXACT_TRUE_PEAK');self.assertEqual(d['d_peakset_over_Rg'],0)
def test_near_tail_without_branch_claim(self):
t=np.array([1,.99,0.]);p=np.array([0,1,0.]);w=weights([.005,.005,.99]);a,_=tail_membership(t,w)
d=peak_diagnostic(t,p,np.array([[0,0,0],[1,0,0],[2,0,0]]),np.arange(3),1,w=w,true_tail=a)
self.assertEqual(d['category'],'NEAR_TRUE_TAIL_AWAY_FROM_PEAK');self.assertFalse(d['topology_branch_identified'])
def test_monotone_tail_invariance(self):
s=np.array([0.,1.,2.,2.]);w=weights([3,4,5,6]);a,_=tail_membership(s,w);b,_=tail_membership(np.exp(s),w);np.testing.assert_array_equal(a,b)
if __name__=='__main__':unittest.main()
@@ -0,0 +1,28 @@
{
"synthetic": true,
"injected_prior_coordinator_cpu_seconds": 89.98,
"worker_is_sleep_only": true,
"child_killed_and_reaped": true,
"receipt": {
"coordinator_cpu_seconds": 90.022846584,
"coordinator_cpu_limit": 90.0,
"status": "FAILED",
"returncode": -9,
"stop_reason": "coordinator_cpu_limit",
"cpu_seconds": 0.020076999999999998,
"wall_seconds": 0.042679114965721965,
"peak_tree_rss_bytes": 41156608,
"peak_process_count": 1,
"monitor_errors": [],
"max_poll_gap_seconds": 0.0011497379746288061,
"reaped_orphan_cpu_seconds": 0.0,
"limits": {
"cpu_seconds": 3300,
"wall_seconds": 10,
"rss_bytes": 268435456,
"disk_bytes": 1048576
},
"retry": false,
"gpu_visible": false
}
}
@@ -0,0 +1,10 @@
{
"status": "AWAITING_ACTUAL_LIGHT_REVIEW",
"implementation": "b97674a1d481049363c78bbe647756ac40822e73",
"plan_sha256": "30e845417d3840ea194e791470eab0dc6821a819c6432b41c5766819606b015d",
"request_slack": "1794968475.536719",
"approval_slack": "1794966078.809549",
"tests_pass": 16,
"real_diagnostic_started": false,
"unix": 1794968532.7050326
}
@@ -18231,13 +18231,14 @@
],
"code_sha256": {
"/home/frank14f/optane/MeltMind/tools/s2_tail_diagnostic_math.py": "e7a4abcaa3001234c5244162c893711d54e8766e97db13476335dc1243a30bd9",
"/home/frank14f/optane/MeltMind/tools/s2_tail_diagnostic.py": "a9e93379c813124b5d596963e693fd8d675b92d1cd73c63fbf8dc9620811e6a5",
"/home/frank14f/optane/MeltMind/tools/prepare_s2_tail_diagnostic.py": "97200ce86bdec7867a69b681ec5092649fb49aba663d8476ea958d1d88c44270",
"/home/frank14f/optane/MeltMind/tools/test_s2_tail_diagnostic.py": "21b0c8d4070d440a4567c6d0f2307f75fb71c31170067b6f8aeac8210951c723",
"/home/frank14f/optane/MeltMind/tools/s2_tail_diagnostic.py": "e1b25766c6927f7b5457e7ca3643c0008032568055faf6b767cc0c4b94911ecc",
"/home/frank14f/optane/MeltMind/tools/prepare_s2_tail_diagnostic.py": "fdbaf358d4453dee127533b530e23aa9d869f08f9002128daa5daffd4e9da208",
"/home/frank14f/optane/MeltMind/tools/test_s2_tail_diagnostic.py": "76535cfcdb971726ef13032ef26d0690ed9d7a9930f2f216d6db7634d782b0ff",
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "437a756ddf10b125dce450f463e76e28df52724f8c5766e0a989dfa536805628",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"/home/frank14f/optane/MeltMind/docs/s2-tail-diagnostic-plan-2026-11-18.md": "18bb5bb782615203e939a615a9fbc8e75d2383eb4706ab1e8e273954dbe0b248"
"/home/frank14f/optane/MeltMind/docs/s2-tail-diagnostic-plan-2026-11-18.md": "e1e661bb4a8ca7a7cc6e767bd07e82a075e01db6beacfa04ea7698f1a1c0016f",
"/home/frank14f/optane/MeltMind/tools/s2_tail_diagnostic_guard.py": "d190b705764980d169c861290def90bb4bc39eed10135c57898f098e744f013e"
},
"S2_score": {
"path": "/home/frank14f/MeltMind-runs/phase2-h2-s2-postfit-2026-11-17/score/result.json",
@@ -18255,5 +18256,7 @@
"new_predictions": 0,
"no_retry": true,
"light_review_before_targets": true,
"original_S2_decision_unchanged": true
"original_S2_decision_unchanged": true,
"coordinator_cpu_limit": 90,
"shutdown_cpu_reserve": 30
}
@@ -0,0 +1,22 @@
test_certificate_and_nonconverse (tools.test_s2_tail_diagnostic.TailTests) ... ok
test_constant_truth_and_heavy_node (tools.test_s2_tail_diagnostic.TailTests) ... ok
test_coordinator_budget_stops_and_reaps_before_worker_cap (tools.test_s2_tail_diagnostic.TailTests) ... ok
test_disjoint_tails (tools.test_s2_tail_diagnostic.TailTests) ... ok
test_empty_exterior (tools.test_s2_tail_diagnostic.TailTests) ... ok
test_identical_fractional_plateau (tools.test_s2_tail_diagnostic.TailTests) ... ok
test_invalid_mass_and_nan (tools.test_s2_tail_diagnostic.TailTests) ... ok
test_missing_light_review_refuses (tools.test_s2_tail_diagnostic.TailTests) ... ok
test_monotone_tail_invariance (tools.test_s2_tail_diagnostic.TailTests) ... ok
test_near_tail_without_branch_claim (tools.test_s2_tail_diagnostic.TailTests) ... ok
test_nonuniform_and_ID_reorder (tools.test_s2_tail_diagnostic.TailTests) ... ok
test_other_exact_peak (tools.test_s2_tail_diagnostic.TailTests) ... ok
test_query_duplication (tools.test_s2_tail_diagnostic.TailTests) ... ok
test_strict_JSON_output (tools.test_s2_tail_diagnostic.TailTests) ... ok
test_strict_equality_not_certified (tools.test_s2_tail_diagnostic.TailTests) ... ok
test_wrong_light_review_plan_refuses (tools.test_s2_tail_diagnostic.TailTests) ... ok
test_zero_remainder_boundary (tools.test_s2_tail_diagnostic.TailTests) ... ok
----------------------------------------------------------------------
Ran 17 tests in 0.099s
OK
+4 -2
View File
@@ -2,7 +2,7 @@
Claude1794966078.809549/h-c907cdaf authorizes this descriptive-only CPU diagnostic on the already seen552 parts/204 components, under3600 CPU-seconds. It has no gate, does not reverse the S2 decision, starts no item c fit, and changes neither item1 nor its postfit barriers. Mold's full05344bac specification is normative; future c also retains that specification.
Entry `tools/s2_tail_diagnostic.py run artifacts/phase2-h2-s2-tail-diagnostic-2026-11-18`, original reproduction venv, project cwd. `prepare_s2_tail_diagnostic.py` reads metadata only. Before any target computation, MeltAssist performs a light code/spec check. An executor receipt records that actual review, source receipt hash, current plan hash and source map; the entry verifies it. No five-field implementation/proof ceremony or new checkpoint review. The unmodified shared CPU guard is reused, one attempt, CPU24 worker/25 coordinator, no GPU/model loading/inference or labels decoded from archives.
Entry `tools/s2_tail_diagnostic.py run artifacts/phase2-h2-s2-tail-diagnostic-2026-11-18`, original reproduction venv, project cwd. `prepare_s2_tail_diagnostic.py` reads metadata only. Before any target computation, MeltAssist performs a light code/spec check. An executor receipt records that actual review, source receipt hash, current plan hash and source map; the entry verifies it. No five-field implementation/proof ceremony or new checkpoint review. A diagnostic-local copy of the shared CPU guard adds a running coordinator allowance; the shared guard remains unchanged. One attempt, CPU24 worker/25 coordinator, no GPU/model loading/inference or labels decoded from archives.
H2 full fields are the canonical frozen S2 manifest (including exact historical S1 fold0). Full A comes from the corresponding frozen H1 `A_seconds` fields already used to form the S2 baseline. Every case verifies both raw-ID arrays and complete node counts, geometry and cached lumped mass equality, original query membership, and exact A/H2 query values against stored S2 values before reading the existing seconds.npy target once. Input hashes are checked at execution. No predictor is run. The old S2 query last-location values are copied unchanged for comparison, separately from new geometry-domain diagnostics.
@@ -12,6 +12,8 @@ Peak certificates and categories follow the same full/query domains, ε, r=.10Rg
Outputs include552 per-part JSON records, both arm tail membership threshold/volume/tie summaries, Miss/False/E_tail, query/full diagnostics, original query last scores, negative counts and target-access log. Summaries give component-equal paired H2−A differences and2000 bootstrap intervals/seed2026111408 for tail, original point and continuous diagnostics, shared draws on identical component support. Undefined values exclude the whole affected component for that metric with denominators reported; no missing component becomes a zero. Categories and certificates are reported as part counts with denominators; categories additionally have component-equal fractions. These are exploratory descriptions, not a decision rule or proof of mechanism. Full predictions are reused, not copied or rewritten.
Budgets:3600 total CPU-s, including180 setup reserve (covers metadata preparation and synthetic tests), a3300-CPU-s serial worker and actual coordinator/reaped-child accounting. The worker has7200 wall-seconds and4GiB RSS; guard output is capped at1GiB. Per-part diagnostic output is small; no full-field copies or pairwise distance matrices. A numerical/input/hash error stops the batch and preserves partial records; no silent retry or fabricated552 table. Review and publication overhead are disclosed separately from the scientific worker; they run no scientific computation.
Budgets:3600 total CPU-s, including180 setup reserve (covers metadata preparation and synthetic tests), a3300-CPU-s serial worker and actual coordinator/reaped-child accounting. The diagnostic-local guard samples coordinator CPU from entry start on each live poll and stops/kills/reaps the worker at90 coordinator CPU-s, retaining the full guard failure receipt. Thus180+3300+90=3570 leaves30 CPU-s for sampled-limit overshoot, cleanup and final receipt/accounting within3600; final accounting still rejects any overspend. The coordinator measurement includes configuration, polling and guard work, not just the scientific worker. This is a sampled CPU boundary, with actual polling gap retained; no automatic retry follows a stop. The worker has7200 wall-seconds and4GiB RSS; guard output is capped at1GiB. Per-part diagnostic output is small; no full-field copies or pairwise distance matrices. A numerical/input/hash error stops the batch and preserves partial records; no silent retry or fabricated552 table. Review and publication overhead are disclosed separately from the scientific worker; they run no scientific computation.
Synthetic acceptance is required before review: fractional identical/disjoint tails, constant truth, one heavy node/nonuniform weights, ID permutation, monotone transformation, duplicate query records, zero-remainder boundary, strict certificate equality, sufficient-certificate/nonconverse/empty exterior, exact other-peak and near-tail categories, plus missing/mismatched review rejection. Synthetic arrays only; no real target values before the light review. No existing training or postfit source is edited.
Light review1794968673 was BLOCKED solely because the prior total CPU check occurred after exit. Its original plan30e84541 and all source bytes are retained under `before-budget-fix`, and the actual BLOCKED receipt is preserved. The local budget fix adds a synthetic real-child negative test: injected prior coordinator usage plus actual polling CPU crosses90, stops while worker usage is below3300, kills/reaps the sleeping child and verifies the failure receipt. No scientific arrays are read by this test. Only this budget change and refreshed source/plan identities require the same light review follow-up.
+2 -1
View File
@@ -21,9 +21,10 @@ def main():
parts.append(dict(case=case,component_id=e['component_id'],fold=e['fold'],nodes=e['nodes'],t_set=cache['t_set'],sources=sources))
spec=BASE/'references/2026-11-18/phase3-item-c/mold-tail-volume-peak-stability-spec.md'
if sha(spec)!='05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521':raise ValueError('Spec identity')
names=['s2_tail_diagnostic_math.py','s2_tail_diagnostic.py','prepare_s2_tail_diagnostic.py','test_s2_tail_diagnostic.py','r4bc_guard.py','s2_recovery_supervisor.py','s2_recovery_monitoring.py']
names=['s2_tail_diagnostic_math.py','s2_tail_diagnostic.py','s2_tail_diagnostic_guard.py','prepare_s2_tail_diagnostic.py','test_s2_tail_diagnostic.py','r4bc_guard.py','s2_recovery_supervisor.py','s2_recovery_monitoring.py']
codes={str(BASE/'tools'/n):sha(BASE/'tools'/n) for n in names};codes[str(BASE/'docs/s2-tail-diagnostic-plan-2026-11-18.md')]=sha(BASE/'docs/s2-tail-diagnostic-plan-2026-11-18.md')
ROOT.mkdir();plan=dict(approval='1794966078.809549',spec_path=str(spec),spec_sha256=sha(spec),descriptive_only=True,gate=None,cpu_cap=3600,worker_cpu_cap=3300,setup_reserve=180,worker_cpu=24,coordinator_cpu=25,worker_rss_bytes=4*2**30,worker_wall_seconds=7200,parts=parts,code_sha256=codes,S2_score=dict(path=str(s/'score/result.json'),sha256=sha(s/'score/result.json')),metadata_sources={str(q):sha(q) for q in [s/'contract.json',s/'predictions-manifest.json',s/'predictions-git-freeze.json',s/'complete.json',Path(c['pilot_contract']),BASE/'artifacts/phase2-b13-e1b-2026-11-17/contract.json']},new_fits=0,new_predictions=0,no_retry=True,light_review_before_targets=True,original_S2_decision_unchanged=True)
plan.update(coordinator_cpu_limit=90,shutdown_cpu_reserve=30)
save(ROOT/'plan.json',plan)
save(ROOT/'preparation.json',dict(status='PREPARED_NO_TARGET_ARRAYS_READ',parts=552,components=204,source_file_existence_checked=True,array_values_read=False,source_hashes_rechecked_at_execution=True,review_pending=True,launched=False,preparation_cpu_seconds=time.process_time()-start,unix=time.time()))
print(dict(root=str(ROOT),plan_sha256=sha(ROOT/'plan.json'),code_files=len(codes),parts=552,components=204,review_pending=True))
+2 -2
View File
@@ -101,13 +101,13 @@ def worker(root):
def run(root):
import fcntl,psutil
from r4bc_guard import supervise
from s2_tail_diagnostic_guard import supervise
start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);c=config(root);os.sched_setaffinity(0,[25]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('One diagnostic attempt; no duplicate or retry')
for p in psutil.process_iter(['pid','cmdline']):
if p.pid!=os.getpid() and any(Path(x).name=='s2_tail_diagnostic.py' for x in p.info['cmdline'] or []):raise RuntimeError('Existing diagnostic')
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),cpu_cap=3600,setup_reserve=180,descriptive_only=True,review_sha256=sha(root/'review-accepted.json'),cpu_start=dict(self=start,child_user=u0.ru_utime,child_system=u0.ru_stime)))
guard=supervise([sys.executable,str(BASE/'tools/s2_tail_diagnostic.py'),'worker',str(root)],root/'guard',dict(cpu_seconds=3300,wall_seconds=7200,rss_bytes=4*2**30,disk_bytes=2**30))
guard=supervise([sys.executable,str(BASE/'tools/s2_tail_diagnostic.py'),'worker',str(root)],root/'guard',dict(cpu_seconds=3300,wall_seconds=7200,rss_bytes=4*2**30,disk_bytes=2**30),coordinator_cpu_start=start,coordinator_cpu_limit=90.)
u=resource.getrusage(resource.RUSAGE_CHILDREN);charge=180+time.process_time()-start+max(guard['cpu_seconds'],u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)
ok=guard['status']=='PASS' and not guard['monitor_errors'] and charge<=3600
if ok:
+102
View File
@@ -0,0 +1,102 @@
"""Diagnostic-only copy of r4bc guard, adding a running coordinator CPU allowance.
Shared guards and all scientific definitions stay unchanged.
"""
import argparse
import ctypes
import fcntl
import json
import math
import os
from pathlib import Path
import signal
import subprocess
import sys
import time
from s2_recovery_supervisor import save, tree_usage, stop_group
from s2_recovery_monitoring import disk_usage
def supervise(command, dest, limits, *, coordinator_cpu_start, coordinator_cpu_limit=90., global_stop=None, poll=.2):
if not math.isfinite(coordinator_cpu_start) or not math.isfinite(coordinator_cpu_limit) or coordinator_cpu_limit <= 0:
raise ValueError('Invalid coordinator CPU allowance')
dest = Path(dest); dest.mkdir(parents=True, exist_ok=True)
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
raise ValueError('Invalid guard limit')
with (dest/'execution.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (dest/'worker.started.json').exists() or (dest/'worker.receipt.json').exists():
raise FileExistsError('Single attempt; no restart')
save(dest/'worker.started.json', dict(command=command, limits=limits, unix=time.time()))
requested = [None]
def stop(signum, frame):
requested[0] = 'GUARD_SIGNAL_'+str(signum)
prior = {s: signal.signal(s, stop) for s in (signal.SIGTERM, signal.SIGINT)}
libc = ctypes.CDLL(None, use_errno=True); previous = ctypes.c_int()
if libc.prctl(37, ctypes.byref(previous), 0, 0, 0) or libc.prctl(36, 1, 0, 0, 0):
raise OSError('Cannot enable descendant accounting')
start = time.monotonic(); last = start; reason = None; errors = []
cpu_peak = rss_peak = proc_peak = gap = 0; usage = None; exit_status = None
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', MKL_NUM_THREADS='1',
NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
try:
with (dest/'worker.log').open('x') as log:
child = subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT,
start_new_session=True)
save(dest/'worker-pid.json', dict(pid=child.pid, unix=time.time(), guard_pid=os.getpid()))
try:
while True:
now = time.monotonic(); gap = max(gap, now-last); last = now
try:
cpu, rss, count = tree_usage(child.pid)
disk = disk_usage(dest)
cpu_peak = max(cpu_peak, cpu); rss_peak = max(rss_peak, rss); proc_peak = max(proc_peak, count)
except Exception as exc:
errors.append(dict(type=type(exc).__name__, message=str(exc)))
reason = 'MONITOR_ERROR'
if time.process_time()-coordinator_cpu_start >= coordinator_cpu_limit:
reason = reason or 'coordinator_cpu_limit'
if requested[0]: reason = reason or requested[0]
if global_stop and Path(global_stop).exists(): reason = reason or 'GLOBAL_STOP_REQUEST'
if now-start >= limits['wall_seconds']: reason = reason or 'wall_limit'
if cpu_peak >= limits['cpu_seconds']: reason = reason or 'cpu_limit'
if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit'
if proc_peak > 1: reason = reason or 'process_count_limit'
if not errors and disk >= limits['disk_bytes']: reason = reason or 'disk_limit'
if reason: stop_group(child.pid)
pid, status, current_usage = os.wait4(child.pid, 0 if reason else os.WNOHANG)
if pid:
usage = current_usage; exit_status = status; break
time.sleep(poll)
finally:
stop_group(child.pid)
if exit_status is None:
_, exit_status, usage = os.wait4(child.pid, 0)
child.returncode = os.waitstatus_to_exitcode(exit_status)
orphan_cpu = 0.; orphan_rss = 0
while True:
try: _, _, u = os.wait4(-child.pid, 0)
except ChildProcessError: break
orphan_cpu += u.ru_utime + u.ru_stime
orphan_rss = max(orphan_rss, u.ru_maxrss*1024)
wall = time.monotonic()-start
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
rss_peak = max(rss_peak, usage.ru_maxrss*1024, orphan_rss)
# Inspect final peaks even when a numerical exception returned nonzero.
if charged > limits['cpu_seconds']: reason = reason or 'cpu_limit_after_exit'
if wall > limits['wall_seconds']: reason = reason or 'wall_limit_after_exit'
if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit_after_exit'
coordinator_cpu = time.process_time()-coordinator_cpu_start
if coordinator_cpu >= coordinator_cpu_limit: reason = reason or 'coordinator_cpu_limit_after_exit'
result = dict(coordinator_cpu_seconds=coordinator_cpu, coordinator_cpu_limit=coordinator_cpu_limit,
status='PASS' if child.returncode == 0 and reason is None else 'FAILED',
returncode=child.returncode, stop_reason=reason, cpu_seconds=charged,
wall_seconds=wall, peak_tree_rss_bytes=rss_peak, peak_process_count=proc_peak,
monitor_errors=errors, max_poll_gap_seconds=gap, reaped_orphan_cpu_seconds=orphan_cpu,
limits=limits, retry=False, gpu_visible=False)
save(dest/'worker.receipt.json', result)
return result
finally:
libc.prctl(36, previous.value, 0, 0, 0)
for signum, handler in prior.items(): signal.signal(signum, handler)
+28 -1
View File
@@ -1,4 +1,4 @@
import unittest,tempfile,json
import unittest,tempfile,json,time,sys,os
from pathlib import Path
from unittest.mock import patch
import numpy as np
@@ -6,6 +6,33 @@ from s2_tail_diagnostic_math import weights,tail_membership,tail_error,peak_diag
from s2_tail_diagnostic import config
class TailTests(unittest.TestCase):
def test_coordinator_budget_stops_and_reaps_before_worker_cap(self):
import s2_tail_diagnostic_guard as guard
original_usage=guard.tree_usage
def charged_monitor(pid):
until=time.process_time()+.04
while time.process_time()<until:pass
return original_usage(pid)
with tempfile.TemporaryDirectory() as d:
# Inject 89.98 seconds of prior coordinator work, then consume real CPU
# in the first monitor poll while a real synthetic child remains alive.
start=time.process_time()-89.98
with patch.object(guard,'tree_usage',side_effect=charged_monitor):
result=guard.supervise([sys.executable,'-c','import time; time.sleep(30)'],d,
dict(cpu_seconds=3300,wall_seconds=10,rss_bytes=2**28,disk_bytes=2**20),
coordinator_cpu_start=start,coordinator_cpu_limit=90.)
pid=json.loads((Path(d)/'worker-pid.json').read_text())['pid']
self.assertEqual(result['status'],'FAILED')
self.assertEqual(result['stop_reason'],'coordinator_cpu_limit')
self.assertGreaterEqual(result['coordinator_cpu_seconds'],90.)
self.assertLess(result['cpu_seconds'],3300.)
self.assertFalse(result['monitor_errors'])
self.assertLess(180+result['coordinator_cpu_seconds']+result['cpu_seconds'],3600)
with self.assertRaises(ProcessLookupError):os.kill(pid,0)
with self.assertRaises(ChildProcessError):os.waitpid(pid,os.WNOHANG)
self.assertEqual(json.loads((Path(d)/'worker.receipt.json').read_text()),result)
if os.environ.get('S2_TAIL_TEST_RECEIPT'):
Path(os.environ['S2_TAIL_TEST_RECEIPT']).write_text(json.dumps(dict(synthetic=True,injected_prior_coordinator_cpu_seconds=89.98,worker_is_sleep_only=True,child_killed_and_reaped=True,receipt=result),indent=2)+'\n')
def test_missing_light_review_refuses(self):
with tempfile.TemporaryDirectory() as d:
r=Path(d);(r/'plan.json').write_text(json.dumps(dict(approval='1794966078.809549',cpu_cap=3600,spec_sha256='05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521',spec_path='synthetic',code_sha256={})))