Fix sampled monitoring races in isolated recovery infrastructure

This commit is contained in:
Frank14f committed 2026-11-15 05:03:54 +08:00
1 parent 7f53f1b422
commit f9eeb1262f
9 files changed
+626

No files matched your search

@@ -0,0 +1,10 @@
test_coordinator_repolls_fault_with_active_workers_without_killing_them (__main__.InfrastructureTests) ... ok
test_cumulative_cpu_counts_incident_once_and_real_caps_stop (__main__.InfrastructureTests) ... ok
test_live_guard_monitor_exception_does_not_kill_child (__main__.InfrastructureTests) ... ok
test_old_tmp_replace_race_reproduced_new_scan_survives (__main__.InfrastructureTests) ... ok
test_wall_cap_still_stops_with_monitor_faults (__main__.InfrastructureTests) ... ok
----------------------------------------------------------------------
Ran 5 tests in 0.315s
OK
@@ -0,0 +1,18 @@
{
"status": "PASS_SYNTHETIC_INFRASTRUCTURE_FIX",
"tests": 5,
"passed": 5,
"source_sha256": {
"tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
"tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"tools/run_s2_recovery_preparation.py": "c135a8f718468b5b1286924d45b83597bf6cf8785af53c70c53fc074787730f1",
"tools/test_s2_recovery_infrastructure.py": "3263b2c143d1bac908093acff15a77ffe5914edd40654297b69e8ad47e2fc829"
},
"original_worker_and_dependencies_pins_intact": true,
"real_recovery_started": false,
"initial_test_failure": "Coordinator fixture omitted contract.json; 4 checks passed, fixture corrected, final 5/5 PASS. Initial log retained.",
"final_small_change": "Preserve an already-recorded real global budget stop instead of replacing its reason with subsequent externally killed worker status.",
"test_log_sha256": "7e657ff3c93e2b206204635e4e0197136196f1ebe9c8a69c962e5027072838f7",
"ruling": "1794689807.792819"
}
@@ -0,0 +1,26 @@
test_coordinator_repolls_fault_with_active_workers_without_killing_them (__main__.InfrastructureTests) ... ERROR
test_cumulative_cpu_counts_incident_once_and_real_caps_stop (__main__.InfrastructureTests) ... ok
test_live_guard_monitor_exception_does_not_kill_child (__main__.InfrastructureTests) ... ok
test_old_tmp_replace_race_reproduced_new_scan_survives (__main__.InfrastructureTests) ... ok
test_wall_cap_still_stops_with_monitor_faults (__main__.InfrastructureTests) ... ok
======================================================================
ERROR: test_coordinator_repolls_fault_with_active_workers_without_killing_them (__main__.InfrastructureTests)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/home/frank14f/optane/MeltMind/tools/test_s2_recovery_infrastructure.py", line 81, in test_coordinator_repolls_fault_with_active_workers_without_killing_them
coordinator.run(root)
File "/home/frank14f/optane/MeltMind/tools/run_s2_recovery_preparation.py", line 40, in run
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root)
File "/home/frank14f/optane/MeltMind/tools/run_s2_recovery_preparation.py", line 49, in run_locked
save(root/'batch-started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json')))
File "/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py", line 35, in sha
with Path(path).open('rb') as f:
File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/pathlib.py", line 1119, in open
return self._accessor.open(self, mode, buffering, encoding, errors,
FileNotFoundError: [Errno 2] No such file or directory: '/tmp/tmpfzpikjip/contract.json'
----------------------------------------------------------------------
Ran 5 tests in 0.400s
FAILED (errors=1)
@@ -0,0 +1,9 @@
# S2 准备协调器基础设施修复
原16682在disk_usage扫描progress.json.tmp时遭遇原子替换竞态,异常进入finally并中止16个worker;未产生原batch-complete。遵循Claude 1794689807.792819,独立版本修复基础设施,原文件及worker数值代码不改。
新disk_usage每条目录项只做一次不跟随symlink的stat,FileNotFoundError表示该项已消失,跳过后下轮继续统计;其他监测/记账异常记录并重采样。协调器不在记账不新鲜的周期派新任务,但现有worker继续受独立wall/CPU/RSS监督;真实CPU/磁盘超限仍停止。worker监督器也记录并重采样监测异常,墙钟判断始终执行,最终CPU/HWM依wait4计入。CPU/GPU两种监督基础设施均有独立修复副本,原实现和历史pin保留。
恢复协调器仅接受新134件契约、≤16workers、单件1800s/1.25GiB/0 retries,无新50屏障。累计CPU=事故381个原worker已用量(包含16中止)+恢复批实际量;不把中止CPU再加一次。累计原始/隔离/恢复输出磁盘≤100GiB。真实预算停止原因优先于随后被全局停止清理的worker信号。
5项合成验收通过:原.tmp竞态复现与修复扫描、监测异常时子进程继续、监测故障时真实wall上限仍停、累计预算判断、134个模拟任务下协调器遇活跃期记账故障后重采样且不中止worker。初次协调器测试漏建契约文件,失败原记录保留;补齐测试夹具后5/5通过。没有重跑科学准备或GPU。后续事故账本、隔离、严格恢复集合与三来源合并是独立交付,不属于已执行计算。
+120
View File
@@ -0,0 +1,120 @@
"""One authorized 134-part recovery; independent pins and cumulative accounting."""
import argparse
import fcntl
import json
import math
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
from freeze_g1_components import sha
from s2_recovery_supervisor import supervise,save,tree_usage
from s2_recovery_monitoring import disk_usage,Monitor,budget_reason
from run_s2_pilot_preparation import terminate
def contract(root):
c=json.loads((root/'contract.json').read_text())
for p,h in c['code_sha256'].items():assert sha(p)==h,p
assert sha(c['incident_ledger'])==c['incident_ledger_sha256']
assert sha(c['original_snapshot'])==c['original_snapshot_sha256']
return c
def guard(root,index):
c=contract(root);job=root/'jobs'/f'job-{index:04d}';job.mkdir(parents=True,exist_ok=True)
r=supervise([sys.executable,'tools/s2_pilot_prepare.py',str(root),str(index)],job,'worker',
dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=1800,tree_rss_bytes=1342177280,
disk_bytes=100*2**30,max_processes=1),poll=.25)
monitor=Monitor(job/'accounting-errors.jsonl')
monitor.retry('guard_receipt',lambda:save(job/'guard-result.json',dict(worker=r,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
original_index=c['original_indices'][index],labelled_infrastructure_recovery=c['original_indices'][index] in c['infra_aborted_indices'])),.25)
return 0 if r['status']=='PASS' else 1
def run(root):
with (root/'batch.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root)
def run_locked(root):
c=contract(root);assert len(c['cases'])==len(c['original_indices'])==134
assert not (root/'batch-started.json').exists(),'One recovery batch only'
assert c['projected_cumulative_worker_cpu_hours']<=60 and c['projected_cumulative_gib']<=100
assert c['limits']['worker_cpu_hours']==60 and c['limits']['disk_bytes']==100*2**30
monitor=Monitor(root/'monitor-errors.jsonl');start=time.monotonic()
save(root/'batch-started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json')))
active={};done=[];next_index=0;reason=None;live_cpu=live_rss=used=peak_rss=0;fresh=False
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
def interrupted(*_):
nonlocal reason
reason='RECOVERY_COORDINATOR_INTERRUPTED'
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
def read_account(path):
r=json.loads(path.read_text());w=r['worker']
assert math.isfinite(w['cpu_seconds']) and w['cpu_seconds']>=0
assert math.isfinite(r['supervisor_cpu_seconds']) and r['supervisor_cpu_seconds']>=0
assert isinstance(w['returncode'],int) and isinstance(w['status'],str)
assert 'stop_reason' in w
return r
try:
while active or (next_index<len(c['cases']) and reason is None):
cycle_ok=True
for i,(p,log) in list(active.items()):
code,ok=monitor.sample(f'poll_{i}',p.poll,None);cycle_ok &= ok
if ok and code is not None:
path=root/'jobs'/f'job-{i:04d}'/'guard-result.json'
r,ok=monitor.sample(f'account_{i}',lambda path=path:read_account(path),None);cycle_ok &= ok
if not ok:continue
done.append(dict(index=i,case=c['cases'][i]['case'],returncode=code,**r));del active[i];log.close()
if reason is None and r['worker']['returncode']<0 and r['worker']['stop_reason'] is None:reason='UNEXPLAINED_RECOVERY_WORKER_SIGNAL'
print(json.dumps(dict(event='FINISHED',index=i,original_index=c['original_indices'][i],completed=len(done),status=r['worker']['status'])),flush=True)
def sample_trees():
values=[tree_usage(p.pid) for p,_ in active.values()]
return sum(v[0] for v in values),sum(v[1] for v in values)
(live_cpu,live_rss),ok=monitor.sample('aggregate_live_usage',sample_trees,(live_cpu,live_rss));cycle_ok &= ok
peak_rss=max(peak_rss,live_rss)
used,ok=monitor.sample('aggregate_disk',lambda:sum(disk_usage(p) for p in c['budget_roots']),used);cycle_ok &= ok
spent=sum(r['worker']['cpu_seconds'] for r in done)
over=budget_reason(c['prior_worker_cpu_seconds'],spent,live_cpu,used)
if over:reason=over
if reason:terminate(active)
progress=dict(completed=len(done),active=list(active),next_index=next_index,stop_reason=reason,
prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],recovery_worker_cpu_seconds=spent,
cumulative_settled_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,
live_guard_and_worker_cpu_seconds=live_cpu,active_tree_rss_bytes=live_rss,disk_bytes=used,
elapsed_seconds=time.monotonic()-start,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,
fresh_accounting=cycle_ok,no_new_first50_barrier=True)
_,ok=monitor.sample('progress_write',lambda:save(root/'progress.json',progress),None);cycle_ok &= ok
# Do not dispatch new work on a stale accounting cycle. Existing
# workers keep their independent wall/CPU/RSS guards while re-polling.
while cycle_ok and reason is None and len(active)<16 and next_index<len(c['cases']):
i=next_index;job=root/'jobs'/f'job-{i:04d}';job.mkdir(parents=True,exist_ok=True)
log=(job/'guard.log').open('x')
p=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(i)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
active[i]=(p,log);next_index+=1
if active or (not cycle_ok and reason is None):time.sleep(.5)
finally:
terminate(active)
for p,log in active.values():
p.wait();log.close()
used=monitor.retry('final_disk',lambda:sum(disk_usage(p) for p in c['budget_roots']),.5)
spent=sum(r['worker']['cpu_seconds'] for r in done)
if c['prior_worker_cpu_seconds']+spent>60*3600:reason='GLOBAL_CPU_LIMIT_AFTER_EXIT'
if used>100*2**30:reason='GLOBAL_DISK_LIMIT_AFTER_EXIT'
result=dict(status='ATTEMPTS_COMPLETE' if len(done)==134 and reason is None else 'STOPPED',stop_reason=reason,jobs=done,
not_run_indices=list(range(next_index,134)),prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],worker_cpu_seconds=spent,
cumulative_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done),
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-start,
peak_aggregate_rss_bytes=peak_rss,disk_bytes=used,failed=sum(r['worker']['status']!='PASS' for r in done),
monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,retry=False,replacement=False,test_labels_read=False)
monitor.retry('batch_receipt',lambda:save(root/'batch-complete.json',result),.5)
print(json.dumps({k:v for k,v in result.items() if k!='jobs'}),flush=True)
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);a=p.parse_args()
sys.exit(guard(a.root,a.guard) if a.guard is not None else run(a.root))
+158
View File
@@ -0,0 +1,158 @@
"""Pilot process guard derived from frozen E1 guard; explicit CUDA visibility.
Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot
is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged.
"""
import fcntl
import ctypes
import json
import math
import os
from pathlib import Path
import signal
import subprocess
import time
import psutil
from s2_recovery_monitoring import disk_usage,Monitor
def save(path, data):
path = Path(path)
tmp = path.with_suffix(path.suffix + '.tmp')
tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
tmp.replace(path)
def tree_usage(pid):
try:
root = psutil.Process(pid)
processes = [root] + root.children(recursive=True)
except psutil.NoSuchProcess:
return 0., 0, 0
cpu = 0.
rss = count = 0
for p in processes:
try:
t = p.cpu_times()
# Linux child times include reaped descendants, not live children.
cpu += t.user + t.system + t.children_user + t.children_system
rss += p.memory_info().rss
count += 1
except psutil.NoSuchProcess:
pass
return cpu, rss, count
def stop_group(pid):
try:
os.killpg(pid, signal.SIGKILL)
except ProcessLookupError:
pass
def supervise(command, root, name, limits, *, poll=.02, gpu_uuid=None):
"""One locked attempt; no automatic retry/replacement; account failed CPU."""
root = Path(root)
root.mkdir(parents=True, exist_ok=True)
monitor=Monitor(root/(name+'.monitor-errors.jsonl'))
with (root/'execution.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
started_path = root/(name+'.started.json')
receipt_path = root/(name+'.receipt.json')
if started_path.exists() or receipt_path.exists():
raise FileExistsError('Attempt already exists; no silent retry')
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
raise ValueError('Every limit must be finite and positive')
ledger_path = root/'ledger.json'
ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]}
if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']:
raise RuntimeError('Total CPU exhausted before spawn')
if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']:
raise RuntimeError('Disk exhausted before spawn')
save(started_path, {'command':command,'limits':limits,'started_unix':time.time()})
start = last = time.monotonic()
cpu_peak = rss_peak = max_processes = max_gap = 0
cpu=rss=count=used_disk=0
reason = None
exit_status = usage = None
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1',
MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES=str(gpu_uuid or ''))
with (root/(name+'.log')).open('x') as log:
libc = ctypes.CDLL(None, use_errno=True)
previous_subreaper = ctypes.c_int()
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping')
proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT,
env=env, start_new_session=True)
try:
while True:
now = time.monotonic()
max_gap = max(max_gap, now-last)
last = now
(cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count))
cpu_peak = max(cpu_peak,cpu)
rss_peak = max(rss_peak,rss)
max_processes = max(max_processes,count)
used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk)
if now-start >= limits['job_wall_seconds']:
reason = 'wall_limit'
elif rss > limits['tree_rss_bytes']:
reason = 'tree_rss_limit'
elif count > limits['max_processes']:
reason = 'process_count_limit'
elif used_disk >= limits['disk_bytes']:
reason = 'disk_limit'
elif cpu_peak >= limits['job_cpu_seconds']:
reason = 'job_cpu_limit'
elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']:
reason = 'total_cpu_limit'
if reason:
stop_group(proc.pid)
pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG)
if pid:
exit_status, usage = status, child_usage
break
time.sleep(poll)
finally:
# No detached work may outlive a supervised attempt.
stop_group(proc.pid)
if exit_status is None:
_, exit_status, usage = os.wait4(proc.pid, 0)
proc.returncode = os.waitstatus_to_exitcode(exit_status)
orphan_cpu = 0.
orphan_peak = 0
# Killed descendants are adopted here rather than losing their
# final CPU usage when the root worker cannot reap them itself.
while True:
try:
_, _, orphan_usage = os.wait4(-proc.pid, 0)
except ChildProcessError:
break
orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime
orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024)
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state')
elapsed = time.monotonic()-start
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak)
used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll)
if reason is None:
if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit'
elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit'
elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit'
elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit'
elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit'
receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED',
command=command,returncode=proc.returncode,stop_reason=reason,
cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak,
peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk,
max_poll_gap_seconds=max_gap,poll_target_seconds=poll,
reaped_orphan_cpu_seconds=orphan_cpu,
supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure',
gpu_visible=bool(gpu_uuid),gpu_uuid=gpu_uuid,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error)
ledger['cpu_seconds_charged'] += charged
ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged})
monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll)
monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll)
return receipt
+40
View File
@@ -0,0 +1,40 @@
"""Race-tolerant sampled infrastructure monitoring, without worker changes."""
import json
from pathlib import Path
import stat
import sys
import time
def disk_usage(root):
logical=allocated=0
for p in Path(root).rglob('*'):
try:s=p.stat(follow_symlinks=False)
except FileNotFoundError:continue # atomic progress replacement / vanished entry
if stat.S_ISREG(s.st_mode):
logical+=s.st_size;allocated+=s.st_blocks*512
return max(logical,allocated)
class Monitor:
def __init__(self,path):self.path=Path(path);self.errors=0;self.last_error=None
def record(self,operation,exc):
self.errors+=1;self.last_error=dict(unix=time.time(),operation=operation,type=type(exc).__name__,message=str(exc))
try:
with self.path.open('a') as f:f.write(json.dumps(self.last_error)+'\n')
except Exception as log_exc:
print(json.dumps(dict(monitor_error=self.last_error,logging_error=str(log_exc))),file=sys.stderr,flush=True)
def sample(self,operation,fn,previous):
try:return fn(),True
except Exception as exc:self.record(operation,exc);return previous,False
def retry(self,operation,fn,poll):
while True:
value,ok=self.sample(operation,fn,None)
if ok:return value
time.sleep(poll)
def budget_reason(prior_cpu,settled_cpu,live_cpu,disk_bytes,*,cpu_limit=60*3600,disk_limit=100*2**30):
if prior_cpu+settled_cpu+live_cpu>=cpu_limit:return 'GLOBAL_CPU_LIMIT'
if disk_bytes>=disk_limit:return 'GLOBAL_DISK_LIMIT'
return None
+158
View File
@@ -0,0 +1,158 @@
"""Sequential E1 job guard: tree CPU/RSS, wall, process count and disk receipts.
Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot
is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged.
"""
import fcntl
import ctypes
import json
import math
import os
from pathlib import Path
import signal
import subprocess
import time
import psutil
from s2_recovery_monitoring import disk_usage,Monitor
def save(path, data):
path = Path(path)
tmp = path.with_suffix(path.suffix + '.tmp')
tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
tmp.replace(path)
def tree_usage(pid):
try:
root = psutil.Process(pid)
processes = [root] + root.children(recursive=True)
except psutil.NoSuchProcess:
return 0., 0, 0
cpu = 0.
rss = count = 0
for p in processes:
try:
t = p.cpu_times()
# Linux child times include reaped descendants, not live children.
cpu += t.user + t.system + t.children_user + t.children_system
rss += p.memory_info().rss
count += 1
except psutil.NoSuchProcess:
pass
return cpu, rss, count
def stop_group(pid):
try:
os.killpg(pid, signal.SIGKILL)
except ProcessLookupError:
pass
def supervise(command, root, name, limits, *, poll=.02):
"""One locked attempt; no automatic retry/replacement; account failed CPU."""
root = Path(root)
root.mkdir(parents=True, exist_ok=True)
monitor=Monitor(root/(name+'.monitor-errors.jsonl'))
with (root/'execution.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
started_path = root/(name+'.started.json')
receipt_path = root/(name+'.receipt.json')
if started_path.exists() or receipt_path.exists():
raise FileExistsError('Attempt already exists; no silent retry')
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
raise ValueError('Every limit must be finite and positive')
ledger_path = root/'ledger.json'
ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]}
if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']:
raise RuntimeError('Total CPU exhausted before spawn')
if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']:
raise RuntimeError('Disk exhausted before spawn')
save(started_path, {'command':command,'limits':limits,'started_unix':time.time()})
start = last = time.monotonic()
cpu_peak = rss_peak = max_processes = max_gap = 0
cpu=rss=count=used_disk=0
reason = None
exit_status = usage = None
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1',
MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
with (root/(name+'.log')).open('x') as log:
libc = ctypes.CDLL(None, use_errno=True)
previous_subreaper = ctypes.c_int()
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping')
proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT,
env=env, start_new_session=True)
try:
while True:
now = time.monotonic()
max_gap = max(max_gap, now-last)
last = now
(cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count))
cpu_peak = max(cpu_peak,cpu)
rss_peak = max(rss_peak,rss)
max_processes = max(max_processes,count)
used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk)
if now-start >= limits['job_wall_seconds']:
reason = 'wall_limit'
elif rss > limits['tree_rss_bytes']:
reason = 'tree_rss_limit'
elif count > limits['max_processes']:
reason = 'process_count_limit'
elif used_disk >= limits['disk_bytes']:
reason = 'disk_limit'
elif cpu_peak >= limits['job_cpu_seconds']:
reason = 'job_cpu_limit'
elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']:
reason = 'total_cpu_limit'
if reason:
stop_group(proc.pid)
pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG)
if pid:
exit_status, usage = status, child_usage
break
time.sleep(poll)
finally:
# No detached work may outlive a supervised attempt.
stop_group(proc.pid)
if exit_status is None:
_, exit_status, usage = os.wait4(proc.pid, 0)
proc.returncode = os.waitstatus_to_exitcode(exit_status)
orphan_cpu = 0.
orphan_peak = 0
# Killed descendants are adopted here rather than losing their
# final CPU usage when the root worker cannot reap them itself.
while True:
try:
_, _, orphan_usage = os.wait4(-proc.pid, 0)
except ChildProcessError:
break
orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime
orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024)
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state')
elapsed = time.monotonic()-start
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak)
used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll)
if reason is None:
if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit'
elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit'
elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit'
elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit'
elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit'
receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED',
command=command,returncode=proc.returncode,stop_reason=reason,
cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak,
peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk,
max_poll_gap_seconds=max_gap,poll_target_seconds=poll,
reaped_orphan_cpu_seconds=orphan_cpu,
supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure',
gpu_visible=False,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error)
ledger['cpu_seconds_charged'] += charged
ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged})
monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll)
monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll)
return receipt
+87
View File
@@ -0,0 +1,87 @@
import json
from pathlib import Path
import sys
import tempfile
import unittest
from unittest.mock import patch
from s2_recovery_monitoring import disk_usage,budget_reason
import s2_e1_supervisor as old
import s2_recovery_supervisor as fixed
import run_s2_recovery_preparation as coordinator
class InfrastructureTests(unittest.TestCase):
def test_old_tmp_replace_race_reproduced_new_scan_survives(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);pending=root/'progress.json.tmp';final=root/'progress.json';pending.write_text('new');final.write_text('old')
stat=Path.stat;calls=0
def racing_stat(path,*a,**kw):
nonlocal calls
if path==pending:
calls+=1
if calls==2:pending.replace(final)
return stat(path,*a,**kw)
with patch.object(Path,'rglob',return_value=iter([pending,final])),patch.object(Path,'stat',racing_stat):
with self.assertRaises(FileNotFoundError):old.disk_usage(root)
pending.write_text('replacement')
def disappearing_stat(path,*a,**kw):
if path==pending:pending.replace(final)
return stat(path,*a,**kw)
with patch.object(Path,'rglob',return_value=iter([pending,final])),patch.object(Path,'stat',disappearing_stat):
measured=disk_usage(root)
self.assertEqual(measured,max(final.stat().st_size,final.stat().st_blocks*512))
def test_live_guard_monitor_exception_does_not_kill_child(self):
with tempfile.TemporaryDirectory() as tmp:
real=fixed.disk_usage;calls=0
def once(root):
nonlocal calls
calls+=1
if calls==2:raise OSError('synthetic live monitoring failure')
return real(root)
limits=dict(total_cpu_seconds=3,job_cpu_seconds=2,job_wall_seconds=2,tree_rss_bytes=1024*2**20,disk_bytes=2**20,max_processes=1)
with patch.object(fixed,'disk_usage',once):r=fixed.supervise([sys.executable,'-c','import time; time.sleep(.12)'],tmp,'worker',limits,poll=.01)
self.assertEqual(r['status'],'PASS');self.assertEqual(r['monitor_errors'],1);self.assertEqual(r['returncode'],0)
def test_wall_cap_still_stops_with_monitor_faults(self):
with tempfile.TemporaryDirectory() as tmp:
real=fixed.disk_usage;calls=0
def intermittent(root):
nonlocal calls
calls+=1
if calls in [2,3]:raise OSError('synthetic sample failure')
return real(root)
limits=dict(total_cpu_seconds=3,job_cpu_seconds=2,job_wall_seconds=.08,tree_rss_bytes=1024*2**20,disk_bytes=2**20,max_processes=1)
with patch.object(fixed,'disk_usage',intermittent):r=fixed.supervise([sys.executable,'-c','import time; time.sleep(2)'],tmp,'worker',limits,poll=.01)
self.assertEqual(r['stop_reason'],'wall_limit');self.assertLess(r['returncode'],0);self.assertGreater(r['monitor_errors'],0)
def test_cumulative_cpu_counts_incident_once_and_real_caps_stop(self):
self.assertIsNone(budget_reason(23.78094*3600,3600,100,100))
self.assertEqual(budget_reason(23.78094*3600,36.21906*3600,1,0),'GLOBAL_CPU_LIMIT')
self.assertEqual(budget_reason(0,0,0,100*2**30),'GLOBAL_DISK_LIMIT')
def test_coordinator_repolls_fault_with_active_workers_without_killing_them(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);spawned=[];terminations=[];fault=False
c=dict(cases=[dict(case=str(i)) for i in range(134)],original_indices=list(range(134)),
projected_cumulative_worker_cpu_hours=1,projected_cumulative_gib=1,prior_worker_cpu_seconds=0.,
limits=dict(worker_cpu_hours=60,disk_bytes=100*2**30),budget_roots=[str(root)])
(root/'contract.json').write_text(json.dumps(c))
class FakeProcess:
def __init__(self,index):self.index=index;self.pid=1000000+index;self.polls=0
def poll(self):
self.polls+=1
if self.polls==1:return None
path=root/'jobs'/f'job-{self.index:04d}'/'guard-result.json'
path.write_text(json.dumps(dict(worker=dict(status='PASS',returncode=0,stop_reason=None,cpu_seconds=.01),supervisor_cpu_seconds=0.)))
return 0
def spawn(command,**kwargs):
p=FakeProcess(int(command[-1]));spawned.append(p);return p
def disk(_):
nonlocal fault
if spawned and not fault:
fault=True;raise OSError('synthetic coordinator accounting failure')
return 100
with patch.object(coordinator,'contract',return_value=c),patch.object(coordinator.subprocess,'Popen',spawn),patch.object(coordinator,'disk_usage',disk),patch.object(coordinator,'tree_usage',return_value=(.01,100,1)),patch.object(coordinator.time,'sleep'),patch.object(coordinator,'terminate',side_effect=lambda active:terminations.append(list(active))),patch('builtins.print'):
coordinator.run(root)
r=json.loads((root/'batch-complete.json').read_text());self.assertEqual(r['status'],'ATTEMPTS_COMPLETE')
self.assertEqual(len(r['jobs']),134);self.assertEqual(r['monitor_errors'],1);self.assertTrue(all(not ids for ids in terminations))
if __name__=='__main__':unittest.main(verbosity=2)