Enforce running coordinator CPU allowance for S2 diagnostic
This commit is contained in:
1 parent
b97674a1d4
commit
1ed44f70fc
21 files changed
+19182
-15
No files matched your search
@@ -1,8 +1,9 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"tests": 16,
|
||||
"scope": "Synthetic arrays only; no scientific target or prediction values",
|
||||
"log_sha256": "3d12870ac905775ea6a50a861ef2d64e297422975a29ff59c980daf1e8c0343f",
|
||||
"tests": 17,
|
||||
"scope": "Synthetic arrays and sleep-only child with injected prior coordinator CPU; no real scientific values",
|
||||
"log_sha256": "5bb929569e925ef83030b7e755f370253b57c77751fd3c9e5eaee3742122b5e0",
|
||||
"budget_negative_receipt_sha256": "100875a1205eaa63a6da81a80b118681fd2b42c2c2ff4d4cc7b304354606705f",
|
||||
"real_data_read": false,
|
||||
"unix": 1794968093.7768
|
||||
"unix": 1794968825.439787
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"tests": 16,
|
||||
"scope": "Synthetic arrays only; no scientific target or prediction values",
|
||||
"log_sha256": "3d12870ac905775ea6a50a861ef2d64e297422975a29ff59c980daf1e8c0343f",
|
||||
"real_data_read": false,
|
||||
"unix": 1794968093.7768
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"reviewer": "MeltAssist",
|
||||
"kind": "light_code_spec_review",
|
||||
"verdict": "BLOCKED",
|
||||
"unix": 1794968645.5496745,
|
||||
"implementation_reported": "b97674a1d481049363c78bbe647756ac40822e73",
|
||||
"plan_sha256": "30e845417d3840ea194e791470eab0dc6821a819c6432b41c5766819606b015d",
|
||||
"spec_sha256": "05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521",
|
||||
"code_sha256": {
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_tail_diagnostic_math.py": "e7a4abcaa3001234c5244162c893711d54e8766e97db13476335dc1243a30bd9",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_tail_diagnostic.py": "a9e93379c813124b5d596963e693fd8d675b92d1cd73c63fbf8dc9620811e6a5",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_s2_tail_diagnostic.py": "97200ce86bdec7867a69b681ec5092649fb49aba663d8476ea958d1d88c44270",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_s2_tail_diagnostic.py": "21b0c8d4070d440a4567c6d0f2307f75fb71c31170067b6f8aeac8210951c723",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "437a756ddf10b125dce450f463e76e28df52724f8c5766e0a989dfa536805628",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/docs/s2-tail-diagnostic-plan-2026-11-18.md": "18bb5bb782615203e939a615a9fbc8e75d2383eb4706ab1e8e273954dbe0b248"
|
||||
},
|
||||
"sources_read": [
|
||||
"Claude1794966078.809549/h-c907cdaf",
|
||||
"Melt1794968475.536719",
|
||||
"full Mold05344bac specification",
|
||||
"four diagnostic source/test files",
|
||||
"r4bc_guard.supervise and its resource helpers",
|
||||
"existing synthetic acceptance and attempt3 log"
|
||||
],
|
||||
"checks": {
|
||||
"mathematical_spec": "PASS_STATIC",
|
||||
"input_metadata": "552 unique parts / 204 components; 3312 source path/hash metadata matches to canonical S2/H1/pilot sources; metadata source hashes match",
|
||||
"output_scope": "PASS_STATIC: original query scores retained; full-volume tails and same-domain certificates; no new fit/prediction/decode; descriptive only",
|
||||
"synthetic_tests": "Existing 16 PASS log read, not executed by reviewer",
|
||||
"runtime_monitor_errors": "Nonempty guard monitor_errors rejects completion"
|
||||
},
|
||||
"findings": [
|
||||
{
|
||||
"priority": "P2",
|
||||
"blocking": true,
|
||||
"file": "tools/s2_tail_diagnostic.py",
|
||||
"line": 111,
|
||||
"title": "Total CPU cap is checked only after supervised work exits",
|
||||
"detail": "run passes only cpu_seconds=3300 for the worker to r4bc_guard.supervise. The supervisor samples child-tree CPU only. Coordinator/guard CPU is not limited while supervise runs. After return, run computes 180 + coordinator CPU + max(worker sampled CPU, reaped child CPU) and compares to 3600. Thus coordinator CPU can exceed the remaining 120 seconds while work continues, and a later STOPPED cannot enforce the authorized total cap. This is a static boundary gap, not an observed overspend.",
|
||||
"requested_change": "Add a diagnostic-scoped running total CPU stop (or an enforced coordinator allowance with cleanup), retaining the existing worker cap, no retry, cumulative accounting and modest shutdown/receipt reserve. Add a small synthetic boundary check for coordinator cost causing the stop before worker3300; no real-target run required. Only the changed budget path and new source/plan hashes need follow-up review."
|
||||
}
|
||||
],
|
||||
"limits": [
|
||||
"No target or prediction arrays read or hashed by reviewer",
|
||||
"No project imports, synthetic tests, model loading, diagnostic run or scientific recomputation",
|
||||
"Input array identity checks were inspected statically; they remain runtime checks",
|
||||
"Reported publication commit not independently fetched; review identifies actual local source bytes through plan hashes"
|
||||
],
|
||||
"scope_unchanged": "Existing item1 and postfit untouched; no five-field binding procedure added"
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
+30
@@ -0,0 +1,30 @@
|
||||
"""Metadata-only preparation for Claude6078; never opens target arrays."""
|
||||
import time
|
||||
from pathlib import Path
|
||||
from s2_tail_diagnostic import BASE,ROOT,read,save,sha
|
||||
|
||||
def main():
|
||||
start=time.process_time()
|
||||
if ROOT.exists():raise FileExistsError('Existing diagnostic root: inspect before any action')
|
||||
s=Path('/home/frank14f/MeltMind-runs/phase2-h2-s2-postfit-2026-11-17');c=read(s/'contract.json');p=read(c['pilot_contract']);m=read(s/'predictions-manifest.json')['entries'];a=read(BASE/'artifacts/phase2-b13-e1b-2026-11-17/contract.json')['A_predictions'];oldrows={r['case']:r for r in c['rows']}
|
||||
if read(s/'complete.json')['status']!='COMPLETE' or len(m)!=552 or len({e['case'] for e in m})!=552 or len({e['component_id'] for e in m})!=204:raise ValueError('Original completed S2 roster')
|
||||
proof=read(s/'predictions-git-freeze.json')
|
||||
if proof['files'][str(s/'predictions-manifest.json')]!=sha(s/'predictions-manifest.json'):raise ValueError('Original frozen prediction manifest')
|
||||
parts=[]
|
||||
for e in sorted(m,key=lambda e:e['case']):
|
||||
case=e['case'];cache=p['parts'][case];aa=a[case];g=p['geometry'][case];row=oldrows[case]
|
||||
if aa['protocol']!='grouped' or aa['pool']!=100 or aa['fold']!=e['fold'] or aa['nodes']!=e['nodes'] or g['nodes']!=e['nodes'] or row['component_id']!=e['component_id']:raise ValueError('A/H2 same domain and role')
|
||||
sources={name:dict(path=d['path'],sha256=d['sha256']) for name,d in [('A',aa),('H2',e),('geometry',g)]}
|
||||
for key,name in [('truth','seconds'),('mass','mass'),('raw_ids','raw_ids')]:sources[key]=dict(path=str((BASE/Path(cache['output'])/(name+'.npy')).resolve()),sha256=cache['output_sha256'][name+'.npy'])
|
||||
for v in sources.values():
|
||||
if not Path(v['path']).is_file():raise FileNotFoundError(v['path'])
|
||||
parts.append(dict(case=case,component_id=e['component_id'],fold=e['fold'],nodes=e['nodes'],t_set=cache['t_set'],sources=sources))
|
||||
spec=BASE/'references/2026-11-18/phase3-item-c/mold-tail-volume-peak-stability-spec.md'
|
||||
if sha(spec)!='05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521':raise ValueError('Spec identity')
|
||||
names=['s2_tail_diagnostic_math.py','s2_tail_diagnostic.py','prepare_s2_tail_diagnostic.py','test_s2_tail_diagnostic.py','r4bc_guard.py','s2_recovery_supervisor.py','s2_recovery_monitoring.py']
|
||||
codes={str(BASE/'tools'/n):sha(BASE/'tools'/n) for n in names};codes[str(BASE/'docs/s2-tail-diagnostic-plan-2026-11-18.md')]=sha(BASE/'docs/s2-tail-diagnostic-plan-2026-11-18.md')
|
||||
ROOT.mkdir();plan=dict(approval='1794966078.809549',spec_path=str(spec),spec_sha256=sha(spec),descriptive_only=True,gate=None,cpu_cap=3600,worker_cpu_cap=3300,setup_reserve=180,worker_cpu=24,coordinator_cpu=25,worker_rss_bytes=4*2**30,worker_wall_seconds=7200,parts=parts,code_sha256=codes,S2_score=dict(path=str(s/'score/result.json'),sha256=sha(s/'score/result.json')),metadata_sources={str(q):sha(q) for q in [s/'contract.json',s/'predictions-manifest.json',s/'predictions-git-freeze.json',s/'complete.json',Path(c['pilot_contract']),BASE/'artifacts/phase2-b13-e1b-2026-11-17/contract.json']},new_fits=0,new_predictions=0,no_retry=True,light_review_before_targets=True,original_S2_decision_unchanged=True)
|
||||
save(ROOT/'plan.json',plan)
|
||||
save(ROOT/'preparation.json',dict(status='PREPARED_NO_TARGET_ARRAYS_READ',parts=552,components=204,source_file_existence_checked=True,array_values_read=False,source_hashes_rechecked_at_execution=True,review_pending=True,launched=False,preparation_cpu_seconds=time.process_time()-start,unix=time.time()))
|
||||
print(dict(root=str(ROOT),plan_sha256=sha(ROOT/'plan.json'),code_files=len(codes),parts=552,components=204,review_pending=True))
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Single attempt CPU guard; global stop requests retain wait4 accounting."""
|
||||
import argparse
|
||||
import ctypes
|
||||
import fcntl
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from s2_recovery_supervisor import save, tree_usage, stop_group
|
||||
from s2_recovery_monitoring import disk_usage
|
||||
|
||||
|
||||
def supervise(command, dest, limits, *, global_stop=None, poll=.2):
|
||||
dest = Path(dest); dest.mkdir(parents=True, exist_ok=True)
|
||||
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
|
||||
raise ValueError('Invalid guard limit')
|
||||
with (dest/'execution.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if (dest/'worker.started.json').exists() or (dest/'worker.receipt.json').exists():
|
||||
raise FileExistsError('Single attempt; no restart')
|
||||
save(dest/'worker.started.json', dict(command=command, limits=limits, unix=time.time()))
|
||||
requested = [None]
|
||||
def stop(signum, frame):
|
||||
requested[0] = 'GUARD_SIGNAL_'+str(signum)
|
||||
prior = {s: signal.signal(s, stop) for s in (signal.SIGTERM, signal.SIGINT)}
|
||||
libc = ctypes.CDLL(None, use_errno=True); previous = ctypes.c_int()
|
||||
if libc.prctl(37, ctypes.byref(previous), 0, 0, 0) or libc.prctl(36, 1, 0, 0, 0):
|
||||
raise OSError('Cannot enable descendant accounting')
|
||||
start = time.monotonic(); last = start; reason = None; errors = []
|
||||
cpu_peak = rss_peak = proc_peak = gap = 0; usage = None; exit_status = None
|
||||
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', MKL_NUM_THREADS='1',
|
||||
NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
|
||||
try:
|
||||
with (dest/'worker.log').open('x') as log:
|
||||
child = subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT,
|
||||
start_new_session=True)
|
||||
save(dest/'worker-pid.json', dict(pid=child.pid, unix=time.time(), guard_pid=os.getpid()))
|
||||
try:
|
||||
while True:
|
||||
now = time.monotonic(); gap = max(gap, now-last); last = now
|
||||
try:
|
||||
cpu, rss, count = tree_usage(child.pid)
|
||||
disk = disk_usage(dest)
|
||||
cpu_peak = max(cpu_peak, cpu); rss_peak = max(rss_peak, rss); proc_peak = max(proc_peak, count)
|
||||
except Exception as exc:
|
||||
errors.append(dict(type=type(exc).__name__, message=str(exc)))
|
||||
reason = 'MONITOR_ERROR'
|
||||
if requested[0]: reason = reason or requested[0]
|
||||
if global_stop and Path(global_stop).exists(): reason = reason or 'GLOBAL_STOP_REQUEST'
|
||||
if now-start >= limits['wall_seconds']: reason = reason or 'wall_limit'
|
||||
if cpu_peak >= limits['cpu_seconds']: reason = reason or 'cpu_limit'
|
||||
if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit'
|
||||
if proc_peak > 1: reason = reason or 'process_count_limit'
|
||||
if not errors and disk >= limits['disk_bytes']: reason = reason or 'disk_limit'
|
||||
if reason: stop_group(child.pid)
|
||||
pid, status, current_usage = os.wait4(child.pid, 0 if reason else os.WNOHANG)
|
||||
if pid:
|
||||
usage = current_usage; exit_status = status; break
|
||||
time.sleep(poll)
|
||||
finally:
|
||||
stop_group(child.pid)
|
||||
if exit_status is None:
|
||||
_, exit_status, usage = os.wait4(child.pid, 0)
|
||||
child.returncode = os.waitstatus_to_exitcode(exit_status)
|
||||
orphan_cpu = 0.; orphan_rss = 0
|
||||
while True:
|
||||
try: _, _, u = os.wait4(-child.pid, 0)
|
||||
except ChildProcessError: break
|
||||
orphan_cpu += u.ru_utime + u.ru_stime
|
||||
orphan_rss = max(orphan_rss, u.ru_maxrss*1024)
|
||||
wall = time.monotonic()-start
|
||||
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
|
||||
rss_peak = max(rss_peak, usage.ru_maxrss*1024, orphan_rss)
|
||||
# Inspect final peaks even when a numerical exception returned nonzero.
|
||||
if charged > limits['cpu_seconds']: reason = reason or 'cpu_limit_after_exit'
|
||||
if wall > limits['wall_seconds']: reason = reason or 'wall_limit_after_exit'
|
||||
if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit_after_exit'
|
||||
result = dict(status='PASS' if child.returncode == 0 and reason is None else 'FAILED',
|
||||
returncode=child.returncode, stop_reason=reason, cpu_seconds=charged,
|
||||
wall_seconds=wall, peak_tree_rss_bytes=rss_peak, peak_process_count=proc_peak,
|
||||
monitor_errors=errors, max_poll_gap_seconds=gap, reaped_orphan_cpu_seconds=orphan_cpu,
|
||||
limits=limits, retry=False, gpu_visible=False)
|
||||
save(dest/'worker.receipt.json', result)
|
||||
return result
|
||||
finally:
|
||||
libc.prctl(36, previous.value, 0, 0, 0)
|
||||
for signum, handler in prior.items(): signal.signal(signum, handler)
|
||||
|
||||
|
||||
def run(root, index, slot):
|
||||
from r4bc_common import contract, job_dir, read, sha
|
||||
c = contract(root); os.sched_setaffinity(0, [c['worker_cpus'][slot]])
|
||||
dest = job_dir(root, index)
|
||||
receipt = supervise([sys.executable, 'tools/r4bc_fit.py', str(root), str(index)], dest,
|
||||
c['fit_limits'], global_stop=Path(root)/'global-stop.json')
|
||||
result_path = dest/'output/result.json'; failure_path = dest/'output/failure.json'
|
||||
if receipt['status'] == 'PASS' and not result_path.exists():
|
||||
receipt.update(status='FAILED', stop_reason='MISSING_RESULT')
|
||||
failure = read(failure_path) if failure_path.exists() else None
|
||||
global_failure = bool(receipt['monitor_errors']) or bool(failure and failure.get('global_stop'))
|
||||
if global_failure and not (Path(root)/'global-stop.json').exists():
|
||||
save(Path(root)/'global-stop.json', dict(reason='WORKER_INPUT_OR_MONITOR_FAILURE', job=index, failure=failure))
|
||||
record = dict(index=index, job=c['jobs'][index], worker=receipt, global_failure=global_failure,
|
||||
guard_cpu_seconds=time.process_time(), result_path=str(result_path) if result_path.exists() else None,
|
||||
result_sha256=sha(result_path) if result_path.exists() else None)
|
||||
save(dest/'guard-result.json', record)
|
||||
return 0 if receipt['status'] == 'PASS' else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(); parser.add_argument('root', type=Path)
|
||||
parser.add_argument('index', type=int); parser.add_argument('slot', type=int)
|
||||
args = parser.parse_args(); raise SystemExit(run(args.root, args.index, args.slot))
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
# Existing S2 A/H2 tail diagnostic — light review before execution
|
||||
|
||||
Claude1794966078.809549/h-c907cdaf authorizes this descriptive-only CPU diagnostic on the already seen552 parts/204 components, under3600 CPU-seconds. It has no gate, does not reverse the S2 decision, starts no item c fit, and changes neither item1 nor its postfit barriers. Mold's full05344bac specification is normative; future c also retains that specification.
|
||||
|
||||
Entry `tools/s2_tail_diagnostic.py run artifacts/phase2-h2-s2-tail-diagnostic-2026-11-18`, original reproduction venv, project cwd. `prepare_s2_tail_diagnostic.py` reads metadata only. Before any target computation, MeltAssist performs a light code/spec check. An executor receipt records that actual review, source receipt hash, current plan hash and source map; the entry verifies it. No five-field implementation/proof ceremony or new checkpoint review. The unmodified shared CPU guard is reused, one attempt, CPU24 worker/25 coordinator, no GPU/model loading/inference or labels decoded from archives.
|
||||
|
||||
H2 full fields are the canonical frozen S2 manifest (including exact historical S1 fold0). Full A comes from the corresponding frozen H1 `A_seconds` fields already used to form the S2 baseline. Every case verifies both raw-ID arrays and complete node counts, geometry and cached lumped mass equality, original query membership, and exact A/H2 query values against stored S2 values before reading the existing seconds.npy target once. Input hashes are checked at execution. No predictor is run. The old S2 query last-location values are copied unchanged for comparison, separately from new geometry-domain diagnostics.
|
||||
|
||||
Each case normalizes time by original t_set, keeps negative predictions, obtains mass w from existing m_i=Σ|K|/4, and computes true/A/H2 fractional last1% volume tails. Raw-ID ordering does not cut a plateau. Stable compensated sums determine plateau mass, residual and total tail mass;10^-12 correction/volume bounds are enforced and recorded. `E_tail=(Miss+False)/2` is computed with the equivalent `1−I/.01` value and floating-point identity discrepancy also retained; min intersection ensures identical fractional tails have exactly zero directional loss. Constant truth stays in the roster with its marker. No query multiplicity enters volume measure.
|
||||
|
||||
Peak certificates and categories follow the same full/query domains, ε, r=.10Rg, strictγ>2ε, 64u near-boundary rule, exact raw-ID peak ties, peak-set/tail-support distances and true-time drop. Positive-infiniteγ from an empty exterior is stored as null plus explicit gamma_positive_infinity/trivial_radius=true so JSON contains no invalid infinity. The strict certificate and its boundary-robust interpretation are both kept. Failure of a certificate does not imply a branch error; no branch is identified from Euclidean distance.
|
||||
|
||||
Outputs include552 per-part JSON records, both arm tail membership threshold/volume/tie summaries, Miss/False/E_tail, query/full diagnostics, original query last scores, negative counts and target-access log. Summaries give component-equal paired H2−A differences and2000 bootstrap intervals/seed2026111408 for tail, original point and continuous diagnostics, shared draws on identical component support. Undefined values exclude the whole affected component for that metric with denominators reported; no missing component becomes a zero. Categories and certificates are reported as part counts with denominators; categories additionally have component-equal fractions. These are exploratory descriptions, not a decision rule or proof of mechanism. Full predictions are reused, not copied or rewritten.
|
||||
|
||||
Budgets:3600 total CPU-s, including180 setup reserve (covers metadata preparation and synthetic tests), a3300-CPU-s serial worker and actual coordinator/reaped-child accounting. The worker has7200 wall-seconds and4GiB RSS; guard output is capped at1GiB. Per-part diagnostic output is small; no full-field copies or pairwise distance matrices. A numerical/input/hash error stops the batch and preserves partial records; no silent retry or fabricated552 table. Review and publication overhead are disclosed separately from the scientific worker; they run no scientific computation.
|
||||
|
||||
Synthetic acceptance is required before review: fractional identical/disjoint tails, constant truth, one heavy node/nonuniform weights, ID permutation, monotone transformation, duplicate query records, zero-remainder boundary, strict certificate equality, sufficient-certificate/nonconverse/empty exterior, exact other-peak and near-tail categories, plus missing/mismatched review rejection. Synthetic arrays only; no real target values before the light review. No existing training or postfit source is edited.
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
"""Race-tolerant sampled infrastructure monitoring, without worker changes."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import stat
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def disk_usage(root):
|
||||
logical=allocated=0
|
||||
for p in Path(root).rglob('*'):
|
||||
try:s=p.stat(follow_symlinks=False)
|
||||
except FileNotFoundError:continue # atomic progress replacement / vanished entry
|
||||
if stat.S_ISREG(s.st_mode):
|
||||
logical+=s.st_size;allocated+=s.st_blocks*512
|
||||
return max(logical,allocated)
|
||||
|
||||
|
||||
class Monitor:
|
||||
def __init__(self,path):self.path=Path(path);self.errors=0;self.last_error=None
|
||||
def record(self,operation,exc):
|
||||
self.errors+=1;self.last_error=dict(unix=time.time(),operation=operation,type=type(exc).__name__,message=str(exc))
|
||||
try:
|
||||
with self.path.open('a') as f:f.write(json.dumps(self.last_error)+'\n')
|
||||
except Exception as log_exc:
|
||||
print(json.dumps(dict(monitor_error=self.last_error,logging_error=str(log_exc))),file=sys.stderr,flush=True)
|
||||
def sample(self,operation,fn,previous):
|
||||
try:return fn(),True
|
||||
except Exception as exc:self.record(operation,exc);return previous,False
|
||||
def retry(self,operation,fn,poll):
|
||||
while True:
|
||||
value,ok=self.sample(operation,fn,None)
|
||||
if ok:return value
|
||||
time.sleep(poll)
|
||||
|
||||
|
||||
def budget_reason(prior_cpu,settled_cpu,live_cpu,disk_bytes,*,cpu_limit=60*3600,disk_limit=100*2**30):
|
||||
if prior_cpu+settled_cpu+live_cpu>=cpu_limit:return 'GLOBAL_CPU_LIMIT'
|
||||
if disk_bytes>=disk_limit:return 'GLOBAL_DISK_LIMIT'
|
||||
return None
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
"""Sequential E1 job guard: tree CPU/RSS, wall, process count and disk receipts.
|
||||
|
||||
Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot
|
||||
is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged.
|
||||
"""
|
||||
import fcntl
|
||||
import ctypes
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
import psutil
|
||||
from s2_recovery_monitoring import disk_usage,Monitor
|
||||
|
||||
|
||||
def save(path, data):
|
||||
path = Path(path)
|
||||
tmp = path.with_suffix(path.suffix + '.tmp')
|
||||
tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def tree_usage(pid):
|
||||
try:
|
||||
root = psutil.Process(pid)
|
||||
processes = [root] + root.children(recursive=True)
|
||||
except psutil.NoSuchProcess:
|
||||
return 0., 0, 0
|
||||
cpu = 0.
|
||||
rss = count = 0
|
||||
for p in processes:
|
||||
try:
|
||||
t = p.cpu_times()
|
||||
# Linux child times include reaped descendants, not live children.
|
||||
cpu += t.user + t.system + t.children_user + t.children_system
|
||||
rss += p.memory_info().rss
|
||||
count += 1
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
return cpu, rss, count
|
||||
|
||||
|
||||
def stop_group(pid):
|
||||
try:
|
||||
os.killpg(pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
def supervise(command, root, name, limits, *, poll=.02):
|
||||
"""One locked attempt; no automatic retry/replacement; account failed CPU."""
|
||||
root = Path(root)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
monitor=Monitor(root/(name+'.monitor-errors.jsonl'))
|
||||
with (root/'execution.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
started_path = root/(name+'.started.json')
|
||||
receipt_path = root/(name+'.receipt.json')
|
||||
if started_path.exists() or receipt_path.exists():
|
||||
raise FileExistsError('Attempt already exists; no silent retry')
|
||||
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
|
||||
raise ValueError('Every limit must be finite and positive')
|
||||
ledger_path = root/'ledger.json'
|
||||
ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]}
|
||||
if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']:
|
||||
raise RuntimeError('Total CPU exhausted before spawn')
|
||||
if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']:
|
||||
raise RuntimeError('Disk exhausted before spawn')
|
||||
save(started_path, {'command':command,'limits':limits,'started_unix':time.time()})
|
||||
start = last = time.monotonic()
|
||||
cpu_peak = rss_peak = max_processes = max_gap = 0
|
||||
cpu=rss=count=used_disk=0
|
||||
reason = None
|
||||
exit_status = usage = None
|
||||
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1',
|
||||
MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
|
||||
with (root/(name+'.log')).open('x') as log:
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
previous_subreaper = ctypes.c_int()
|
||||
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping')
|
||||
proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT,
|
||||
env=env, start_new_session=True)
|
||||
try:
|
||||
while True:
|
||||
now = time.monotonic()
|
||||
max_gap = max(max_gap, now-last)
|
||||
last = now
|
||||
(cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count))
|
||||
cpu_peak = max(cpu_peak,cpu)
|
||||
rss_peak = max(rss_peak,rss)
|
||||
max_processes = max(max_processes,count)
|
||||
used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk)
|
||||
if now-start >= limits['job_wall_seconds']:
|
||||
reason = 'wall_limit'
|
||||
elif rss > limits['tree_rss_bytes']:
|
||||
reason = 'tree_rss_limit'
|
||||
elif count > limits['max_processes']:
|
||||
reason = 'process_count_limit'
|
||||
elif used_disk >= limits['disk_bytes']:
|
||||
reason = 'disk_limit'
|
||||
elif cpu_peak >= limits['job_cpu_seconds']:
|
||||
reason = 'job_cpu_limit'
|
||||
elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']:
|
||||
reason = 'total_cpu_limit'
|
||||
if reason:
|
||||
stop_group(proc.pid)
|
||||
pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG)
|
||||
if pid:
|
||||
exit_status, usage = status, child_usage
|
||||
break
|
||||
time.sleep(poll)
|
||||
finally:
|
||||
# No detached work may outlive a supervised attempt.
|
||||
stop_group(proc.pid)
|
||||
if exit_status is None:
|
||||
_, exit_status, usage = os.wait4(proc.pid, 0)
|
||||
proc.returncode = os.waitstatus_to_exitcode(exit_status)
|
||||
orphan_cpu = 0.
|
||||
orphan_peak = 0
|
||||
# Killed descendants are adopted here rather than losing their
|
||||
# final CPU usage when the root worker cannot reap them itself.
|
||||
while True:
|
||||
try:
|
||||
_, _, orphan_usage = os.wait4(-proc.pid, 0)
|
||||
except ChildProcessError:
|
||||
break
|
||||
orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime
|
||||
orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024)
|
||||
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state')
|
||||
elapsed = time.monotonic()-start
|
||||
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
|
||||
rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak)
|
||||
used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll)
|
||||
if reason is None:
|
||||
if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit'
|
||||
elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit'
|
||||
elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit'
|
||||
elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit'
|
||||
elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit'
|
||||
receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED',
|
||||
command=command,returncode=proc.returncode,stop_reason=reason,
|
||||
cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak,
|
||||
peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk,
|
||||
max_poll_gap_seconds=max_gap,poll_target_seconds=poll,
|
||||
reaped_orphan_cpu_seconds=orphan_cpu,
|
||||
supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure',
|
||||
gpu_visible=False,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error)
|
||||
ledger['cpu_seconds_charged'] += charged
|
||||
ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged})
|
||||
monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll)
|
||||
monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll)
|
||||
return receipt
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
"""One reviewed descriptive CPU pass over existing S2 predictions; no inference."""
|
||||
import hashlib,json,math,os,resource,sys,time
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
from s2_tail_diagnostic_math import weights,tail_membership,tail_error,peak_diagnostic
|
||||
|
||||
BASE=Path('/home/frank14f/optane/MeltMind')
|
||||
ROOT=BASE/'artifacts/phase2-h2-s2-tail-diagnostic-2026-11-18'
|
||||
|
||||
def sha(path):
|
||||
h=hashlib.sha256()
|
||||
with Path(path).open('rb') as f:
|
||||
for b in iter(lambda:f.read(2**20),b''):h.update(b)
|
||||
return h.hexdigest()
|
||||
def read(path):return json.loads(Path(path).read_text())
|
||||
def save(path,data):
|
||||
p=Path(path);p.parent.mkdir(parents=True,exist_ok=True);tmp=p.with_suffix(p.suffix+'.tmp');tmp.write_text(json.dumps(data,indent=2,allow_nan=False)+'\n');tmp.replace(p)
|
||||
def checked(path,h):
|
||||
p=Path(path)
|
||||
if sha(p)!=h:raise ValueError('Source hash mismatch: '+str(p))
|
||||
return p
|
||||
|
||||
def config(root,review=True):
|
||||
c=read(root/'plan.json')
|
||||
if c['approval']!='1794966078.809549' or c['cpu_cap']!=3600 or c['spec_sha256']!='05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521':raise PermissionError('Fixed descriptive scope')
|
||||
for p,h in c['code_sha256'].items():checked(p,h)
|
||||
checked(c['spec_path'],c['spec_sha256'])
|
||||
if review:
|
||||
a=read(root/'review-accepted.json')
|
||||
if a['verdict']!='PASS' or a['plan_sha256']!=sha(root/'plan.json') or a['code_sha256']!=c['code_sha256']:raise PermissionError('Actual lightweight review required')
|
||||
checked(a['source_receipt'],a['source_receipt_sha256'])
|
||||
return c
|
||||
|
||||
def field(row,arm,name):
|
||||
if name in ['E_tail','Miss','False_alarm']:return row['arms'][arm]['tail'][name]
|
||||
if name=='original_query_last':return row['arms'][arm]['original_query_last']
|
||||
domain,key=name.split('.',1);return row['arms'][arm][domain][key]
|
||||
|
||||
def summarize(rows):
|
||||
metrics=['E_tail','Miss','False_alarm','original_query_last']+[d+'.'+k for d in ['query','full'] for k in ['peak_distance_over_Rg','d_peakset_over_Rg','epsilon','true_time_drop_at_predicted_peak']]+['full.d_tail_over_Rg']
|
||||
groups={}
|
||||
for row in rows:groups.setdefault(row['component_id'],[]).append(row)
|
||||
result={};all_ids=sorted(groups)
|
||||
for metric in metrics:
|
||||
ids=[];values={'A':[],'H2':[]};excluded=[]
|
||||
for cid in all_ids:
|
||||
part={arm:[field(r,arm,metric) for r in groups[cid]] for arm in values}
|
||||
if any(v is None or not math.isfinite(v) for vs in part.values() for v in vs):excluded.append(cid);continue
|
||||
ids.append(cid)
|
||||
for arm in values:values[arm].append(float(np.mean(part[arm])))
|
||||
if not ids:
|
||||
result[metric]=dict(A=None,H2=None,difference=None,low=None,high=None,components=0,excluded=excluded);continue
|
||||
a,h=np.array(values['A']),np.array(values['H2']);delta=h-a
|
||||
rng=np.random.Generator(np.random.PCG64(np.random.SeedSequence(2026111408)));draw=rng.integers(0,len(ids),size=(2000,len(ids)))
|
||||
boot=delta[draw].mean(1);lo,hi=np.percentile(boot,[2.5,97.5],method='linear')
|
||||
result[metric]=dict(A=float(a.mean()),H2=float(h.mean()),difference=float(delta.mean()),low=float(lo),high=float(hi),components=len(ids),paired_parts=sum(len(groups[i]) for i in ids),excluded=excluded,undefined_bootstrap=int(np.sum(~np.isfinite(boot))))
|
||||
diagnostics={}
|
||||
for domain in ['query','full']:
|
||||
diagnostics[domain]={}
|
||||
for arm in ['A','H2']:
|
||||
rs=[r['arms'][arm][domain] for r in rows];categories=sorted({r['category'] for r in rs})
|
||||
diagnostics[domain][arm]=dict(parts=len(rs),certified=sum(r['certified'] for r in rs),stable_certificate=sum(r['stable_certificate'] for r in rs),near_boundary=sum(r['near_boundary'] for r in rs),trivial_radius=sum(r['trivial_radius'] for r in rs),constant_truth=sum(r['constant_truth'] for r in rs),categories={k:dict(parts=sum(r['category']==k for r in rs),component_mean_fraction=float(np.mean([np.mean([r['arms'][arm][domain]['category']==k for r in gr]) for gr in groups.values()]))) for k in categories})
|
||||
ties=dict(truth_threshold_tied_parts=sum(r['truth_tail']['plateau_nodes']>1 for r in rows),arms={a:dict(prediction_threshold_tied_parts=sum(r['arms'][a]['prediction_tail']['plateau_nodes']>1 for r in rows),max_threshold_mass_correction=max(abs(r['arms'][a]['prediction_tail']['mass_correction']) for r in rows)) for a in ['A','H2']})
|
||||
return dict(descriptive_only=True,gate=None,S2_decision_unchanged=True,parts=len(rows),components=len(groups),metrics=result,diagnostics=diagnostics,bootstrap=dict(replicates=2000,seed=2026111408,joint_indices_when_same_component_support=True,unit='evaluation component, not training randomness'),constant_truth_parts=sum(r['truth_tail']['constant_field'] for r in rows),tie_coverage=ties)
|
||||
|
||||
def worker(root):
|
||||
c=config(root);os.sched_setaffinity(0,[24])
|
||||
if os.environ.get('CUDA_VISIBLE_DEVICES','')!='':raise PermissionError('CPU only')
|
||||
out=root/'output';out.mkdir(exist_ok=False);cpu=time.process_time();rows=[]
|
||||
# Existing single-point scores are retained, not replaced by the new endpoint.
|
||||
old={r['case']:r for r in read(checked(c['S2_score']['path'],c['S2_score']['sha256']))['rows']}
|
||||
save(out/'review-consumed.json',dict(plan_sha256=sha(root/'plan.json'),review_acceptance_sha256=sha(root/'review-accepted.json'),before_target_arrays=True,unix=time.time()))
|
||||
for item in c['parts']:
|
||||
case=item['case'];source=item['sources']
|
||||
for rec in source.values():checked(rec['path'],rec['sha256'])
|
||||
with np.load(source['H2']['path'],allow_pickle=False) as z:
|
||||
raw=z['raw_ids'].copy();h2=z['seconds'].astype(np.float64);qid=z['query_raw_ids'].copy();query_a=z['A'].copy();query_h=z['pred'].copy()
|
||||
with np.load(source['A']['path'],allow_pickle=False) as z:
|
||||
if not np.array_equal(raw,z['raw_ids']):raise ValueError('Full A/H2 raw-ID identity')
|
||||
a=z['A_seconds'].astype(np.float64)
|
||||
with np.load(source['geometry']['path'],allow_pickle=False) as z:
|
||||
if not np.array_equal(raw,z['raw_ids']):raise ValueError('Geometry prediction raw-ID identity')
|
||||
points=z['points'].copy();mass=z['mass'].copy();rg=float(z['rg'])
|
||||
cache_raw=np.load(source['raw_ids']['path'],allow_pickle=False);cache_mass=np.load(source['mass']['path'],allow_pickle=False)
|
||||
if not np.array_equal(raw,cache_raw) or not np.array_equal(mass,cache_mass) or len(raw)!=item['nodes'] or np.any(np.diff(raw)<=0):raise ValueError('Complete retained-tet domain/mass')
|
||||
ix=np.searchsorted(raw,qid)
|
||||
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],qid) or len(np.unique(qid))!=len(qid):raise ValueError('Exact unique query identity')
|
||||
if not np.array_equal(a[ix],query_a) or not np.array_equal(h2[ix],query_h):raise ValueError('Original S2 A/H2 query values differ from reused full fields')
|
||||
if not math.isfinite(item['t_set']) or item['t_set']<=0 or not np.isfinite(a).all() or not np.isfinite(h2).all():raise ValueError('Finite prediction / time denominator')
|
||||
with (out/'target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,path=source['truth']['path'],sha256=source['truth']['sha256'],unix=time.time()))+'\n')
|
||||
truth=np.load(source['truth']['path'],allow_pickle=False).astype(np.float64)/item['t_set']
|
||||
if truth.shape!=raw.shape or not np.isfinite(truth).all():raise ValueError('Full target domain finite values')
|
||||
w=weights(mass);ta,td=tail_membership(truth,w);arms={}
|
||||
for arm,pred in [('A',a/item['t_set']),('H2',h2/item['t_set'])]:
|
||||
pa,pd=tail_membership(pred,w)
|
||||
arms[arm]=dict(tail=tail_error(ta,pa,w),prediction_tail=pd,full=peak_diagnostic(truth,pred,points,raw,rg,w=w,true_tail=ta),query=peak_diagnostic(truth[ix],pred[ix],points[ix],raw[ix],rg),original_query_last=old[case][arm]['last_location']['value'],negative_full_predictions=int(np.sum(pred<0)))
|
||||
row=dict(case=case,component_id=item['component_id'],fold=item['fold'],nodes=len(raw),query_nodes=len(ix),t_set=item['t_set'],Rg=rg,truth_tail=td,arms=arms)
|
||||
save(out/'parts'/(case+'.json'),row);rows.append(row);save(out/'progress.json',dict(parts=len(rows),worker_cpu_seconds=time.process_time()-cpu,unix=time.time()))
|
||||
if len(rows)!=552 or len({r['component_id'] for r in rows})!=204:raise ValueError('Original complete S2 denominator')
|
||||
summary=summarize(rows);save(out/'result.json',dict(status='COMPLETE',summary=summary,rows=rows,new_fits=0,new_predictions=0,target_arrays_read=552,new_decodes=0,negative_clipping=False,worker_cpu_seconds=time.process_time()-cpu))
|
||||
|
||||
def run(root):
|
||||
import fcntl,psutil
|
||||
from r4bc_guard import supervise
|
||||
start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);c=config(root);os.sched_setaffinity(0,[25]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():raise FileExistsError('One diagnostic attempt; no duplicate or retry')
|
||||
for p in psutil.process_iter(['pid','cmdline']):
|
||||
if p.pid!=os.getpid() and any(Path(x).name=='s2_tail_diagnostic.py' for x in p.info['cmdline'] or []):raise RuntimeError('Existing diagnostic')
|
||||
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),cpu_cap=3600,setup_reserve=180,descriptive_only=True,review_sha256=sha(root/'review-accepted.json'),cpu_start=dict(self=start,child_user=u0.ru_utime,child_system=u0.ru_stime)))
|
||||
guard=supervise([sys.executable,str(BASE/'tools/s2_tail_diagnostic.py'),'worker',str(root)],root/'guard',dict(cpu_seconds=3300,wall_seconds=7200,rss_bytes=4*2**30,disk_bytes=2**30))
|
||||
u=resource.getrusage(resource.RUSAGE_CHILDREN);charge=180+time.process_time()-start+max(guard['cpu_seconds'],u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)
|
||||
ok=guard['status']=='PASS' and not guard['monitor_errors'] and charge<=3600
|
||||
if ok:
|
||||
result=read(root/'output/result.json');ok=result['status']=='COMPLETE' and result['summary']['parts']==552 and result['summary']['components']==204
|
||||
save(root/'complete.json',dict(status='COMPLETE' if ok else 'STOPPED',guard=guard,cumulative_cpu_seconds=charge,cpu_cap=3600,new_fits=0,new_predictions=0,descriptive_only=True,no_gate=True,S2_decision_unchanged=True,retry=False,output_sha256=sha(root/'output/result.json') if (root/'output/result.json').exists() else None))
|
||||
if not ok:raise RuntimeError('Diagnostic guard/coverage failed; preserve receipts')
|
||||
|
||||
if __name__=='__main__':
|
||||
root=Path(sys.argv[2]) if len(sys.argv)>2 else ROOT
|
||||
if sys.argv[1]=='worker':worker(root)
|
||||
elif sys.argv[1]=='run':run(root)
|
||||
else:raise ValueError('worker or run')
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
"""05344bac discrete volume tail and same-domain peak certificates; CPU only."""
|
||||
import math
|
||||
import numpy as np
|
||||
|
||||
ALPHA = .01
|
||||
RADIUS = .10
|
||||
MASS_TOL = 1e-12
|
||||
|
||||
def weights(mass):
|
||||
m = np.asarray(mass, np.float64)
|
||||
if m.ndim != 1 or not len(m) or not np.isfinite(m).all() or np.any(m <= 0):
|
||||
raise ValueError('Finite positive lumped mass required')
|
||||
total = math.fsum(m)
|
||||
if not math.isfinite(total) or total <= 0:
|
||||
raise ValueError('Invalid volume denominator')
|
||||
return m / total
|
||||
|
||||
def tail_membership(values, w):
|
||||
s = np.asarray(values, np.float64)
|
||||
if s.ndim != 1 or s.shape != w.shape or not np.isfinite(s).all():
|
||||
raise ValueError('Finite full-domain field required')
|
||||
# Locate a candidate boundary; all final plateau masses use compensated sums.
|
||||
order = np.argsort(-s, kind='stable')
|
||||
at = min(int(np.searchsorted(np.cumsum(w[order]), ALPHA)), len(s)-1)
|
||||
q = float(s[order[at]])
|
||||
above, tied = s > q, s == q
|
||||
high, equal = math.fsum(w[above]), math.fsum(w[tied])
|
||||
remainder = ALPHA-high
|
||||
fixed = min(max(remainder, 0.), equal)
|
||||
correction = fixed-remainder
|
||||
if equal <= 0 or abs(correction) > MASS_TOL:
|
||||
raise ValueError('Tail boundary exceeds fixed mass correction tolerance')
|
||||
fraction = fixed/equal
|
||||
a = above.astype(np.float64)
|
||||
a[tied] = fraction
|
||||
actual = math.fsum(w*a)
|
||||
if abs(actual-ALPHA) > MASS_TOL:
|
||||
raise ValueError('Tail volume differs from alpha')
|
||||
return a, dict(threshold=q, plateau_volume=equal, plateau_nodes=int(np.sum(tied)), plateau_fraction=fraction,
|
||||
mass_correction=correction, tail_volume=actual,
|
||||
constant_field=bool(np.all(s == s[0])))
|
||||
|
||||
def tail_error(a, b, w):
|
||||
intersection = math.fsum(w*np.minimum(a, b))
|
||||
miss = math.fsum(w*np.maximum(a-b, 0.))/ALPHA
|
||||
false = math.fsum(w*np.maximum(b-a, 0.))/ALPHA
|
||||
# Mean directional loss is numerically robust at identical fractional tails.
|
||||
error = (miss+false)/2
|
||||
identity = 1-intersection/ALPHA
|
||||
if abs(error-identity) > 2*MASS_TOL/ALPHA or abs(miss-false) > 2*MASS_TOL/ALPHA:
|
||||
raise ValueError('Equal tail mass / intersection identity')
|
||||
if not 0 <= error <= 1+2*MASS_TOL/ALPHA:
|
||||
raise ValueError('Tail error outside range')
|
||||
return dict(E_tail=error, Miss=miss, False_alarm=false, intersection=intersection,
|
||||
intersection_formula=identity, finite_precision_identity_difference=error-identity)
|
||||
|
||||
def peak_diagnostic(truth, pred, points, raw, rg, *, w=None, true_tail=None):
|
||||
t, p, x = np.asarray(truth, np.float64), np.asarray(pred, np.float64), np.asarray(points, np.float64)
|
||||
raw = np.asarray(raw)
|
||||
if t.ndim != 1 or p.shape != t.shape or raw.shape != t.shape or x.shape != (len(t),3) or not len(t):
|
||||
raise ValueError('Peak domain shape')
|
||||
if not all(np.isfinite(v).all() for v in [t,p,x]) or not math.isfinite(rg) or rg <= 0:
|
||||
raise ValueError('Peak finite domain and positive Rg')
|
||||
# Query duplicate records must not change maxima, epsilon or distances.
|
||||
ids, first, inverse = np.unique(raw, return_index=True, return_inverse=True)
|
||||
if len(ids) != len(raw):
|
||||
if w is not None or true_tail is not None:
|
||||
raise ValueError('Full volume domain requires unique IDs')
|
||||
if not np.array_equal(t, t[first][inverse]) or not np.array_equal(p, p[first][inverse]) or not np.array_equal(x, x[first][inverse]):
|
||||
raise ValueError('Conflicting duplicated query identity')
|
||||
t,p,x,raw = t[first],p[first],x[first],ids
|
||||
top, predicted_top = np.flatnonzero(t == t.max()), np.flatnonzero(p == p.max())
|
||||
y = int(top[np.argmin(raw[top])]); yh = int(predicted_top[np.argmin(raw[predicted_top])])
|
||||
epsilon = float(np.max(np.abs(p-t)))
|
||||
distances = np.linalg.norm(x-x[y], axis=1)/rg
|
||||
exterior = distances > RADIUS
|
||||
trivial = not bool(exterior.any())
|
||||
gamma = None if trivial else float(t[y]-np.max(t[exterior]))
|
||||
margin = None if trivial else gamma-2*epsilon
|
||||
eta = float(64*np.finfo(np.float64).eps*max(1.,float(np.max(np.abs(t))),float(np.max(np.abs(p))),2*epsilon))
|
||||
certified = trivial or gamma > 2*epsilon
|
||||
near = bool(not trivial and abs(margin) <= eta)
|
||||
distance = float(distances[yh])
|
||||
if certified and distance > RADIUS:
|
||||
raise ValueError('Certificate contradicts observed peak distance')
|
||||
peakset = float(np.min(np.linalg.norm(x[top]-x[yh], axis=1))/rg)
|
||||
taildist = None if true_tail is None else float(np.min(np.linalg.norm(x[true_tail > 0]-x[yh], axis=1))/rg)
|
||||
if distance > RADIUS and peakset == 0:
|
||||
category = 'OTHER_EXACT_TRUE_PEAK'
|
||||
elif peakset <= RADIUS:
|
||||
category = 'NEAR_TRUE_PEAK'
|
||||
elif taildist is None:
|
||||
category = 'FAR_TRUE_PEAK_QUERY'
|
||||
elif taildist <= RADIUS:
|
||||
category = 'NEAR_TRUE_TAIL_AWAY_FROM_PEAK'
|
||||
else:
|
||||
category = 'FAR_TRUE_TAIL'
|
||||
return dict(nodes=len(t),epsilon=epsilon,gamma=gamma,gamma_positive_infinity=trivial,
|
||||
gamma_minus_2epsilon=margin,eta=eta,near_boundary=near,trivial_radius=trivial,
|
||||
certified=certified,stable_certificate=certified and not near,
|
||||
peak_distance_over_Rg=distance,d_peakset_over_Rg=peakset,d_tail_over_Rg=taildist,
|
||||
true_peak_raw_id=int(raw[y]),predicted_peak_raw_id=int(raw[yh]),
|
||||
true_peak_count=len(top),predicted_peak_count=len(predicted_top),
|
||||
true_peak_volume=None if w is None else math.fsum(w[top]),
|
||||
predicted_peak_volume=None if w is None else math.fsum(w[predicted_top]),
|
||||
true_time_drop_at_predicted_peak=float(t[y]-t[yh]),
|
||||
predicted_minus_true_peak_time=float(p[yh]-t[y]),
|
||||
constant_truth=bool(np.all(t==t[0])),category=category,
|
||||
topology_branch_identified=False)
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
import unittest,tempfile,json
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
import numpy as np
|
||||
from s2_tail_diagnostic_math import weights,tail_membership,tail_error,peak_diagnostic
|
||||
from s2_tail_diagnostic import config
|
||||
|
||||
class TailTests(unittest.TestCase):
|
||||
def test_missing_light_review_refuses(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
r=Path(d);(r/'plan.json').write_text(json.dumps(dict(approval='1794966078.809549',cpu_cap=3600,spec_sha256='05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521',spec_path='synthetic',code_sha256={})))
|
||||
with patch('s2_tail_diagnostic.checked'):
|
||||
with self.assertRaises(FileNotFoundError):config(r)
|
||||
def test_wrong_light_review_plan_refuses(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
r=Path(d);(r/'plan.json').write_text(json.dumps(dict(approval='1794966078.809549',cpu_cap=3600,spec_sha256='05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521',spec_path='synthetic',code_sha256={})));(r/'review-accepted.json').write_text(json.dumps(dict(verdict='PASS',plan_sha256='wrong',code_sha256={})))
|
||||
with patch('s2_tail_diagnostic.checked'):
|
||||
with self.assertRaises(PermissionError):config(r)
|
||||
def test_identical_fractional_plateau(self):
|
||||
w=weights([1,1,2]);a,d=tail_membership(np.array([0,1,1]),w)
|
||||
self.assertEqual(a[1],a[2]);self.assertEqual(tail_error(a,a,w)['E_tail'],0)
|
||||
def test_disjoint_tails(self):
|
||||
w=weights(np.ones(200));a,_=tail_membership(np.arange(200),w);b,_=tail_membership(-np.arange(200),w)
|
||||
self.assertAlmostEqual(tail_error(a,b,w)['E_tail'],1)
|
||||
def test_constant_truth_and_heavy_node(self):
|
||||
w=weights([9,1]);a,d=tail_membership(np.ones(2),w);np.testing.assert_allclose(a,.01);self.assertTrue(d['constant_field'])
|
||||
b,_=tail_membership(np.array([1,0]),w);self.assertAlmostEqual(b[0],.01/.9);self.assertAlmostEqual(np.dot(w,b),.01)
|
||||
def test_nonuniform_and_ID_reorder(self):
|
||||
s=np.array([2.,3.,3.,1.]);w=weights([.1,.7,.1,.1]);a,_=tail_membership(s,w);perm=np.array([3,1,0,2]);b,_=tail_membership(s[perm],w[perm]);np.testing.assert_array_equal(a[perm],b)
|
||||
def test_zero_remainder_boundary(self):
|
||||
w=weights([.01,.02,.97]);a,d=tail_membership(np.array([2.,1.,0.]),w);np.testing.assert_allclose(a,[1,0,0]);self.assertAlmostEqual(d['tail_volume'],.01)
|
||||
def test_invalid_mass_and_nan(self):
|
||||
for mass in [[0,1],[-1,2],[float('nan'),1],[]]:
|
||||
with self.assertRaises(ValueError):weights(mass)
|
||||
with self.assertRaises(ValueError):tail_membership(np.array([np.nan]),weights([1]))
|
||||
def diag(self,t,p,raw=None,x=None):
|
||||
return peak_diagnostic(t,p,np.array(x if x is not None else [[0,0,0],[1,0,0]]),np.array(raw if raw is not None else [0,1]),1.)
|
||||
def test_strict_equality_not_certified(self):
|
||||
d=self.diag([1,0],[.5,.5]);self.assertFalse(d['certified']);self.assertTrue(d['near_boundary'])
|
||||
def test_certificate_and_nonconverse(self):
|
||||
d=self.diag([1,0],[.9,.1]);self.assertTrue(d['certified']);self.assertLessEqual(d['peak_distance_over_Rg'],.1)
|
||||
d=self.diag([1,0],[4,0]);self.assertFalse(d['certified']);self.assertEqual(d['peak_distance_over_Rg'],0)
|
||||
def test_query_duplication(self):
|
||||
a=self.diag([1,0],[.9,.1]);b=self.diag([1,0,1],[.9,.1,.9],raw=[0,1,0],x=[[0,0,0],[1,0,0],[0,0,0]]);self.assertEqual(a,b)
|
||||
def test_empty_exterior(self):
|
||||
d=self.diag([1,0],[0,1],x=[[0,0,0],[.01,0,0]]);self.assertTrue(d['trivial_radius']);self.assertIsNone(d['gamma']);self.assertTrue(d['certified'])
|
||||
def test_strict_JSON_output(self):
|
||||
for d in [self.diag([1,0],[.5,.5]),self.diag([1,0],[0,1],x=[[0,0,0],[.01,0,0]])]:self.assertEqual(json.loads(json.dumps(d,allow_nan=False)),d)
|
||||
def test_other_exact_peak(self):
|
||||
d=self.diag([1,1],[0,1]);self.assertEqual(d['category'],'OTHER_EXACT_TRUE_PEAK');self.assertEqual(d['d_peakset_over_Rg'],0)
|
||||
def test_near_tail_without_branch_claim(self):
|
||||
t=np.array([1,.99,0.]);p=np.array([0,1,0.]);w=weights([.005,.005,.99]);a,_=tail_membership(t,w)
|
||||
d=peak_diagnostic(t,p,np.array([[0,0,0],[1,0,0],[2,0,0]]),np.arange(3),1,w=w,true_tail=a)
|
||||
self.assertEqual(d['category'],'NEAR_TRUE_TAIL_AWAY_FROM_PEAK');self.assertFalse(d['topology_branch_identified'])
|
||||
def test_monotone_tail_invariance(self):
|
||||
s=np.array([0.,1.,2.,2.]);w=weights([3,4,5,6]);a,_=tail_membership(s,w);b,_=tail_membership(np.exp(s),w);np.testing.assert_array_equal(a,b)
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"synthetic": true,
|
||||
"injected_prior_coordinator_cpu_seconds": 89.98,
|
||||
"worker_is_sleep_only": true,
|
||||
"child_killed_and_reaped": true,
|
||||
"receipt": {
|
||||
"coordinator_cpu_seconds": 90.022846584,
|
||||
"coordinator_cpu_limit": 90.0,
|
||||
"status": "FAILED",
|
||||
"returncode": -9,
|
||||
"stop_reason": "coordinator_cpu_limit",
|
||||
"cpu_seconds": 0.020076999999999998,
|
||||
"wall_seconds": 0.042679114965721965,
|
||||
"peak_tree_rss_bytes": 41156608,
|
||||
"peak_process_count": 1,
|
||||
"monitor_errors": [],
|
||||
"max_poll_gap_seconds": 0.0011497379746288061,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"limits": {
|
||||
"cpu_seconds": 3300,
|
||||
"wall_seconds": 10,
|
||||
"rss_bytes": 268435456,
|
||||
"disk_bytes": 1048576
|
||||
},
|
||||
"retry": false,
|
||||
"gpu_visible": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"status": "AWAITING_ACTUAL_LIGHT_REVIEW",
|
||||
"implementation": "b97674a1d481049363c78bbe647756ac40822e73",
|
||||
"plan_sha256": "30e845417d3840ea194e791470eab0dc6821a819c6432b41c5766819606b015d",
|
||||
"request_slack": "1794968475.536719",
|
||||
"approval_slack": "1794966078.809549",
|
||||
"tests_pass": 16,
|
||||
"real_diagnostic_started": false,
|
||||
"unix": 1794968532.7050326
|
||||
}
|
||||
@@ -18231,13 +18231,14 @@
|
||||
],
|
||||
"code_sha256": {
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_tail_diagnostic_math.py": "e7a4abcaa3001234c5244162c893711d54e8766e97db13476335dc1243a30bd9",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_tail_diagnostic.py": "a9e93379c813124b5d596963e693fd8d675b92d1cd73c63fbf8dc9620811e6a5",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_s2_tail_diagnostic.py": "97200ce86bdec7867a69b681ec5092649fb49aba663d8476ea958d1d88c44270",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_s2_tail_diagnostic.py": "21b0c8d4070d440a4567c6d0f2307f75fb71c31170067b6f8aeac8210951c723",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_tail_diagnostic.py": "e1b25766c6927f7b5457e7ca3643c0008032568055faf6b767cc0c4b94911ecc",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_s2_tail_diagnostic.py": "fdbaf358d4453dee127533b530e23aa9d869f08f9002128daa5daffd4e9da208",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_s2_tail_diagnostic.py": "76535cfcdb971726ef13032ef26d0690ed9d7a9930f2f216d6db7634d782b0ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "437a756ddf10b125dce450f463e76e28df52724f8c5766e0a989dfa536805628",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/docs/s2-tail-diagnostic-plan-2026-11-18.md": "18bb5bb782615203e939a615a9fbc8e75d2383eb4706ab1e8e273954dbe0b248"
|
||||
"/home/frank14f/optane/MeltMind/docs/s2-tail-diagnostic-plan-2026-11-18.md": "e1e661bb4a8ca7a7cc6e767bd07e82a075e01db6beacfa04ea7698f1a1c0016f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_tail_diagnostic_guard.py": "d190b705764980d169c861290def90bb4bc39eed10135c57898f098e744f013e"
|
||||
},
|
||||
"S2_score": {
|
||||
"path": "/home/frank14f/MeltMind-runs/phase2-h2-s2-postfit-2026-11-17/score/result.json",
|
||||
@@ -18255,5 +18256,7 @@
|
||||
"new_predictions": 0,
|
||||
"no_retry": true,
|
||||
"light_review_before_targets": true,
|
||||
"original_S2_decision_unchanged": true
|
||||
"original_S2_decision_unchanged": true,
|
||||
"coordinator_cpu_limit": 90,
|
||||
"shutdown_cpu_reserve": 30
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
test_certificate_and_nonconverse (tools.test_s2_tail_diagnostic.TailTests) ... ok
|
||||
test_constant_truth_and_heavy_node (tools.test_s2_tail_diagnostic.TailTests) ... ok
|
||||
test_coordinator_budget_stops_and_reaps_before_worker_cap (tools.test_s2_tail_diagnostic.TailTests) ... ok
|
||||
test_disjoint_tails (tools.test_s2_tail_diagnostic.TailTests) ... ok
|
||||
test_empty_exterior (tools.test_s2_tail_diagnostic.TailTests) ... ok
|
||||
test_identical_fractional_plateau (tools.test_s2_tail_diagnostic.TailTests) ... ok
|
||||
test_invalid_mass_and_nan (tools.test_s2_tail_diagnostic.TailTests) ... ok
|
||||
test_missing_light_review_refuses (tools.test_s2_tail_diagnostic.TailTests) ... ok
|
||||
test_monotone_tail_invariance (tools.test_s2_tail_diagnostic.TailTests) ... ok
|
||||
test_near_tail_without_branch_claim (tools.test_s2_tail_diagnostic.TailTests) ... ok
|
||||
test_nonuniform_and_ID_reorder (tools.test_s2_tail_diagnostic.TailTests) ... ok
|
||||
test_other_exact_peak (tools.test_s2_tail_diagnostic.TailTests) ... ok
|
||||
test_query_duplication (tools.test_s2_tail_diagnostic.TailTests) ... ok
|
||||
test_strict_JSON_output (tools.test_s2_tail_diagnostic.TailTests) ... ok
|
||||
test_strict_equality_not_certified (tools.test_s2_tail_diagnostic.TailTests) ... ok
|
||||
test_wrong_light_review_plan_refuses (tools.test_s2_tail_diagnostic.TailTests) ... ok
|
||||
test_zero_remainder_boundary (tools.test_s2_tail_diagnostic.TailTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 17 tests in 0.099s
|
||||
|
||||
OK
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
Claude1794966078.809549/h-c907cdaf authorizes this descriptive-only CPU diagnostic on the already seen552 parts/204 components, under3600 CPU-seconds. It has no gate, does not reverse the S2 decision, starts no item c fit, and changes neither item1 nor its postfit barriers. Mold's full05344bac specification is normative; future c also retains that specification.
|
||||
|
||||
Entry `tools/s2_tail_diagnostic.py run artifacts/phase2-h2-s2-tail-diagnostic-2026-11-18`, original reproduction venv, project cwd. `prepare_s2_tail_diagnostic.py` reads metadata only. Before any target computation, MeltAssist performs a light code/spec check. An executor receipt records that actual review, source receipt hash, current plan hash and source map; the entry verifies it. No five-field implementation/proof ceremony or new checkpoint review. The unmodified shared CPU guard is reused, one attempt, CPU24 worker/25 coordinator, no GPU/model loading/inference or labels decoded from archives.
|
||||
Entry `tools/s2_tail_diagnostic.py run artifacts/phase2-h2-s2-tail-diagnostic-2026-11-18`, original reproduction venv, project cwd. `prepare_s2_tail_diagnostic.py` reads metadata only. Before any target computation, MeltAssist performs a light code/spec check. An executor receipt records that actual review, source receipt hash, current plan hash and source map; the entry verifies it. No five-field implementation/proof ceremony or new checkpoint review. A diagnostic-local copy of the shared CPU guard adds a running coordinator allowance; the shared guard remains unchanged. One attempt, CPU24 worker/25 coordinator, no GPU/model loading/inference or labels decoded from archives.
|
||||
|
||||
H2 full fields are the canonical frozen S2 manifest (including exact historical S1 fold0). Full A comes from the corresponding frozen H1 `A_seconds` fields already used to form the S2 baseline. Every case verifies both raw-ID arrays and complete node counts, geometry and cached lumped mass equality, original query membership, and exact A/H2 query values against stored S2 values before reading the existing seconds.npy target once. Input hashes are checked at execution. No predictor is run. The old S2 query last-location values are copied unchanged for comparison, separately from new geometry-domain diagnostics.
|
||||
|
||||
@@ -12,6 +12,8 @@ Peak certificates and categories follow the same full/query domains, ε, r=.10Rg
|
||||
|
||||
Outputs include552 per-part JSON records, both arm tail membership threshold/volume/tie summaries, Miss/False/E_tail, query/full diagnostics, original query last scores, negative counts and target-access log. Summaries give component-equal paired H2−A differences and2000 bootstrap intervals/seed2026111408 for tail, original point and continuous diagnostics, shared draws on identical component support. Undefined values exclude the whole affected component for that metric with denominators reported; no missing component becomes a zero. Categories and certificates are reported as part counts with denominators; categories additionally have component-equal fractions. These are exploratory descriptions, not a decision rule or proof of mechanism. Full predictions are reused, not copied or rewritten.
|
||||
|
||||
Budgets:3600 total CPU-s, including180 setup reserve (covers metadata preparation and synthetic tests), a3300-CPU-s serial worker and actual coordinator/reaped-child accounting. The worker has7200 wall-seconds and4GiB RSS; guard output is capped at1GiB. Per-part diagnostic output is small; no full-field copies or pairwise distance matrices. A numerical/input/hash error stops the batch and preserves partial records; no silent retry or fabricated552 table. Review and publication overhead are disclosed separately from the scientific worker; they run no scientific computation.
|
||||
Budgets:3600 total CPU-s, including180 setup reserve (covers metadata preparation and synthetic tests), a3300-CPU-s serial worker and actual coordinator/reaped-child accounting. The diagnostic-local guard samples coordinator CPU from entry start on each live poll and stops/kills/reaps the worker at90 coordinator CPU-s, retaining the full guard failure receipt. Thus180+3300+90=3570 leaves30 CPU-s for sampled-limit overshoot, cleanup and final receipt/accounting within3600; final accounting still rejects any overspend. The coordinator measurement includes configuration, polling and guard work, not just the scientific worker. This is a sampled CPU boundary, with actual polling gap retained; no automatic retry follows a stop. The worker has7200 wall-seconds and4GiB RSS; guard output is capped at1GiB. Per-part diagnostic output is small; no full-field copies or pairwise distance matrices. A numerical/input/hash error stops the batch and preserves partial records; no silent retry or fabricated552 table. Review and publication overhead are disclosed separately from the scientific worker; they run no scientific computation.
|
||||
|
||||
Synthetic acceptance is required before review: fractional identical/disjoint tails, constant truth, one heavy node/nonuniform weights, ID permutation, monotone transformation, duplicate query records, zero-remainder boundary, strict certificate equality, sufficient-certificate/nonconverse/empty exterior, exact other-peak and near-tail categories, plus missing/mismatched review rejection. Synthetic arrays only; no real target values before the light review. No existing training or postfit source is edited.
|
||||
|
||||
Light review1794968673 was BLOCKED solely because the prior total CPU check occurred after exit. Its original plan30e84541 and all source bytes are retained under `before-budget-fix`, and the actual BLOCKED receipt is preserved. The local budget fix adds a synthetic real-child negative test: injected prior coordinator usage plus actual polling CPU crosses90, stops while worker usage is below3300, kills/reaps the sleeping child and verifies the failure receipt. No scientific arrays are read by this test. Only this budget change and refreshed source/plan identities require the same light review follow-up.
|
||||
@@ -21,9 +21,10 @@ def main():
|
||||
parts.append(dict(case=case,component_id=e['component_id'],fold=e['fold'],nodes=e['nodes'],t_set=cache['t_set'],sources=sources))
|
||||
spec=BASE/'references/2026-11-18/phase3-item-c/mold-tail-volume-peak-stability-spec.md'
|
||||
if sha(spec)!='05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521':raise ValueError('Spec identity')
|
||||
names=['s2_tail_diagnostic_math.py','s2_tail_diagnostic.py','prepare_s2_tail_diagnostic.py','test_s2_tail_diagnostic.py','r4bc_guard.py','s2_recovery_supervisor.py','s2_recovery_monitoring.py']
|
||||
names=['s2_tail_diagnostic_math.py','s2_tail_diagnostic.py','s2_tail_diagnostic_guard.py','prepare_s2_tail_diagnostic.py','test_s2_tail_diagnostic.py','r4bc_guard.py','s2_recovery_supervisor.py','s2_recovery_monitoring.py']
|
||||
codes={str(BASE/'tools'/n):sha(BASE/'tools'/n) for n in names};codes[str(BASE/'docs/s2-tail-diagnostic-plan-2026-11-18.md')]=sha(BASE/'docs/s2-tail-diagnostic-plan-2026-11-18.md')
|
||||
ROOT.mkdir();plan=dict(approval='1794966078.809549',spec_path=str(spec),spec_sha256=sha(spec),descriptive_only=True,gate=None,cpu_cap=3600,worker_cpu_cap=3300,setup_reserve=180,worker_cpu=24,coordinator_cpu=25,worker_rss_bytes=4*2**30,worker_wall_seconds=7200,parts=parts,code_sha256=codes,S2_score=dict(path=str(s/'score/result.json'),sha256=sha(s/'score/result.json')),metadata_sources={str(q):sha(q) for q in [s/'contract.json',s/'predictions-manifest.json',s/'predictions-git-freeze.json',s/'complete.json',Path(c['pilot_contract']),BASE/'artifacts/phase2-b13-e1b-2026-11-17/contract.json']},new_fits=0,new_predictions=0,no_retry=True,light_review_before_targets=True,original_S2_decision_unchanged=True)
|
||||
plan.update(coordinator_cpu_limit=90,shutdown_cpu_reserve=30)
|
||||
save(ROOT/'plan.json',plan)
|
||||
save(ROOT/'preparation.json',dict(status='PREPARED_NO_TARGET_ARRAYS_READ',parts=552,components=204,source_file_existence_checked=True,array_values_read=False,source_hashes_rechecked_at_execution=True,review_pending=True,launched=False,preparation_cpu_seconds=time.process_time()-start,unix=time.time()))
|
||||
print(dict(root=str(ROOT),plan_sha256=sha(ROOT/'plan.json'),code_files=len(codes),parts=552,components=204,review_pending=True))
|
||||
|
||||
@@ -101,13 +101,13 @@ def worker(root):
|
||||
|
||||
def run(root):
|
||||
import fcntl,psutil
|
||||
from r4bc_guard import supervise
|
||||
from s2_tail_diagnostic_guard import supervise
|
||||
start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);c=config(root);os.sched_setaffinity(0,[25]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():raise FileExistsError('One diagnostic attempt; no duplicate or retry')
|
||||
for p in psutil.process_iter(['pid','cmdline']):
|
||||
if p.pid!=os.getpid() and any(Path(x).name=='s2_tail_diagnostic.py' for x in p.info['cmdline'] or []):raise RuntimeError('Existing diagnostic')
|
||||
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),cpu_cap=3600,setup_reserve=180,descriptive_only=True,review_sha256=sha(root/'review-accepted.json'),cpu_start=dict(self=start,child_user=u0.ru_utime,child_system=u0.ru_stime)))
|
||||
guard=supervise([sys.executable,str(BASE/'tools/s2_tail_diagnostic.py'),'worker',str(root)],root/'guard',dict(cpu_seconds=3300,wall_seconds=7200,rss_bytes=4*2**30,disk_bytes=2**30))
|
||||
guard=supervise([sys.executable,str(BASE/'tools/s2_tail_diagnostic.py'),'worker',str(root)],root/'guard',dict(cpu_seconds=3300,wall_seconds=7200,rss_bytes=4*2**30,disk_bytes=2**30),coordinator_cpu_start=start,coordinator_cpu_limit=90.)
|
||||
u=resource.getrusage(resource.RUSAGE_CHILDREN);charge=180+time.process_time()-start+max(guard['cpu_seconds'],u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)
|
||||
ok=guard['status']=='PASS' and not guard['monitor_errors'] and charge<=3600
|
||||
if ok:
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Diagnostic-only copy of r4bc guard, adding a running coordinator CPU allowance.
|
||||
Shared guards and all scientific definitions stay unchanged.
|
||||
"""
|
||||
import argparse
|
||||
import ctypes
|
||||
import fcntl
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from s2_recovery_supervisor import save, tree_usage, stop_group
|
||||
from s2_recovery_monitoring import disk_usage
|
||||
|
||||
|
||||
def supervise(command, dest, limits, *, coordinator_cpu_start, coordinator_cpu_limit=90., global_stop=None, poll=.2):
|
||||
if not math.isfinite(coordinator_cpu_start) or not math.isfinite(coordinator_cpu_limit) or coordinator_cpu_limit <= 0:
|
||||
raise ValueError('Invalid coordinator CPU allowance')
|
||||
dest = Path(dest); dest.mkdir(parents=True, exist_ok=True)
|
||||
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
|
||||
raise ValueError('Invalid guard limit')
|
||||
with (dest/'execution.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if (dest/'worker.started.json').exists() or (dest/'worker.receipt.json').exists():
|
||||
raise FileExistsError('Single attempt; no restart')
|
||||
save(dest/'worker.started.json', dict(command=command, limits=limits, unix=time.time()))
|
||||
requested = [None]
|
||||
def stop(signum, frame):
|
||||
requested[0] = 'GUARD_SIGNAL_'+str(signum)
|
||||
prior = {s: signal.signal(s, stop) for s in (signal.SIGTERM, signal.SIGINT)}
|
||||
libc = ctypes.CDLL(None, use_errno=True); previous = ctypes.c_int()
|
||||
if libc.prctl(37, ctypes.byref(previous), 0, 0, 0) or libc.prctl(36, 1, 0, 0, 0):
|
||||
raise OSError('Cannot enable descendant accounting')
|
||||
start = time.monotonic(); last = start; reason = None; errors = []
|
||||
cpu_peak = rss_peak = proc_peak = gap = 0; usage = None; exit_status = None
|
||||
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', MKL_NUM_THREADS='1',
|
||||
NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
|
||||
try:
|
||||
with (dest/'worker.log').open('x') as log:
|
||||
child = subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT,
|
||||
start_new_session=True)
|
||||
save(dest/'worker-pid.json', dict(pid=child.pid, unix=time.time(), guard_pid=os.getpid()))
|
||||
try:
|
||||
while True:
|
||||
now = time.monotonic(); gap = max(gap, now-last); last = now
|
||||
try:
|
||||
cpu, rss, count = tree_usage(child.pid)
|
||||
disk = disk_usage(dest)
|
||||
cpu_peak = max(cpu_peak, cpu); rss_peak = max(rss_peak, rss); proc_peak = max(proc_peak, count)
|
||||
except Exception as exc:
|
||||
errors.append(dict(type=type(exc).__name__, message=str(exc)))
|
||||
reason = 'MONITOR_ERROR'
|
||||
if time.process_time()-coordinator_cpu_start >= coordinator_cpu_limit:
|
||||
reason = reason or 'coordinator_cpu_limit'
|
||||
if requested[0]: reason = reason or requested[0]
|
||||
if global_stop and Path(global_stop).exists(): reason = reason or 'GLOBAL_STOP_REQUEST'
|
||||
if now-start >= limits['wall_seconds']: reason = reason or 'wall_limit'
|
||||
if cpu_peak >= limits['cpu_seconds']: reason = reason or 'cpu_limit'
|
||||
if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit'
|
||||
if proc_peak > 1: reason = reason or 'process_count_limit'
|
||||
if not errors and disk >= limits['disk_bytes']: reason = reason or 'disk_limit'
|
||||
if reason: stop_group(child.pid)
|
||||
pid, status, current_usage = os.wait4(child.pid, 0 if reason else os.WNOHANG)
|
||||
if pid:
|
||||
usage = current_usage; exit_status = status; break
|
||||
time.sleep(poll)
|
||||
finally:
|
||||
stop_group(child.pid)
|
||||
if exit_status is None:
|
||||
_, exit_status, usage = os.wait4(child.pid, 0)
|
||||
child.returncode = os.waitstatus_to_exitcode(exit_status)
|
||||
orphan_cpu = 0.; orphan_rss = 0
|
||||
while True:
|
||||
try: _, _, u = os.wait4(-child.pid, 0)
|
||||
except ChildProcessError: break
|
||||
orphan_cpu += u.ru_utime + u.ru_stime
|
||||
orphan_rss = max(orphan_rss, u.ru_maxrss*1024)
|
||||
wall = time.monotonic()-start
|
||||
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
|
||||
rss_peak = max(rss_peak, usage.ru_maxrss*1024, orphan_rss)
|
||||
# Inspect final peaks even when a numerical exception returned nonzero.
|
||||
if charged > limits['cpu_seconds']: reason = reason or 'cpu_limit_after_exit'
|
||||
if wall > limits['wall_seconds']: reason = reason or 'wall_limit_after_exit'
|
||||
if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit_after_exit'
|
||||
coordinator_cpu = time.process_time()-coordinator_cpu_start
|
||||
if coordinator_cpu >= coordinator_cpu_limit: reason = reason or 'coordinator_cpu_limit_after_exit'
|
||||
result = dict(coordinator_cpu_seconds=coordinator_cpu, coordinator_cpu_limit=coordinator_cpu_limit,
|
||||
status='PASS' if child.returncode == 0 and reason is None else 'FAILED',
|
||||
returncode=child.returncode, stop_reason=reason, cpu_seconds=charged,
|
||||
wall_seconds=wall, peak_tree_rss_bytes=rss_peak, peak_process_count=proc_peak,
|
||||
monitor_errors=errors, max_poll_gap_seconds=gap, reaped_orphan_cpu_seconds=orphan_cpu,
|
||||
limits=limits, retry=False, gpu_visible=False)
|
||||
save(dest/'worker.receipt.json', result)
|
||||
return result
|
||||
finally:
|
||||
libc.prctl(36, previous.value, 0, 0, 0)
|
||||
for signum, handler in prior.items(): signal.signal(signum, handler)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import unittest,tempfile,json
|
||||
import unittest,tempfile,json,time,sys,os
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
import numpy as np
|
||||
@@ -6,6 +6,33 @@ from s2_tail_diagnostic_math import weights,tail_membership,tail_error,peak_diag
|
||||
from s2_tail_diagnostic import config
|
||||
|
||||
class TailTests(unittest.TestCase):
|
||||
def test_coordinator_budget_stops_and_reaps_before_worker_cap(self):
|
||||
import s2_tail_diagnostic_guard as guard
|
||||
original_usage=guard.tree_usage
|
||||
def charged_monitor(pid):
|
||||
until=time.process_time()+.04
|
||||
while time.process_time()<until:pass
|
||||
return original_usage(pid)
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
# Inject 89.98 seconds of prior coordinator work, then consume real CPU
|
||||
# in the first monitor poll while a real synthetic child remains alive.
|
||||
start=time.process_time()-89.98
|
||||
with patch.object(guard,'tree_usage',side_effect=charged_monitor):
|
||||
result=guard.supervise([sys.executable,'-c','import time; time.sleep(30)'],d,
|
||||
dict(cpu_seconds=3300,wall_seconds=10,rss_bytes=2**28,disk_bytes=2**20),
|
||||
coordinator_cpu_start=start,coordinator_cpu_limit=90.)
|
||||
pid=json.loads((Path(d)/'worker-pid.json').read_text())['pid']
|
||||
self.assertEqual(result['status'],'FAILED')
|
||||
self.assertEqual(result['stop_reason'],'coordinator_cpu_limit')
|
||||
self.assertGreaterEqual(result['coordinator_cpu_seconds'],90.)
|
||||
self.assertLess(result['cpu_seconds'],3300.)
|
||||
self.assertFalse(result['monitor_errors'])
|
||||
self.assertLess(180+result['coordinator_cpu_seconds']+result['cpu_seconds'],3600)
|
||||
with self.assertRaises(ProcessLookupError):os.kill(pid,0)
|
||||
with self.assertRaises(ChildProcessError):os.waitpid(pid,os.WNOHANG)
|
||||
self.assertEqual(json.loads((Path(d)/'worker.receipt.json').read_text()),result)
|
||||
if os.environ.get('S2_TAIL_TEST_RECEIPT'):
|
||||
Path(os.environ['S2_TAIL_TEST_RECEIPT']).write_text(json.dumps(dict(synthetic=True,injected_prior_coordinator_cpu_seconds=89.98,worker_is_sleep_only=True,child_killed_and_reaped=True,receipt=result),indent=2)+'\n')
|
||||
def test_missing_light_review_refuses(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
r=Path(d);(r/'plan.json').write_text(json.dumps(dict(approval='1794966078.809549',cpu_cap=3600,spec_sha256='05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521',spec_path='synthetic',code_sha256={})))
|
||||
|
||||
Reference in new issue
Block a user