Fix sampled monitoring races in isolated recovery infrastructure
This commit is contained in:
1 parent
7f53f1b422
commit
f9eeb1262f
9 files changed
+626
No files matched your search
@@ -0,0 +1,10 @@
|
||||
test_coordinator_repolls_fault_with_active_workers_without_killing_them (__main__.InfrastructureTests) ... ok
|
||||
test_cumulative_cpu_counts_incident_once_and_real_caps_stop (__main__.InfrastructureTests) ... ok
|
||||
test_live_guard_monitor_exception_does_not_kill_child (__main__.InfrastructureTests) ... ok
|
||||
test_old_tmp_replace_race_reproduced_new_scan_survives (__main__.InfrastructureTests) ... ok
|
||||
test_wall_cap_still_stops_with_monitor_faults (__main__.InfrastructureTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 5 tests in 0.315s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"status": "PASS_SYNTHETIC_INFRASTRUCTURE_FIX",
|
||||
"tests": 5,
|
||||
"passed": 5,
|
||||
"source_sha256": {
|
||||
"tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
|
||||
"tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"tools/run_s2_recovery_preparation.py": "c135a8f718468b5b1286924d45b83597bf6cf8785af53c70c53fc074787730f1",
|
||||
"tools/test_s2_recovery_infrastructure.py": "3263b2c143d1bac908093acff15a77ffe5914edd40654297b69e8ad47e2fc829"
|
||||
},
|
||||
"original_worker_and_dependencies_pins_intact": true,
|
||||
"real_recovery_started": false,
|
||||
"initial_test_failure": "Coordinator fixture omitted contract.json; 4 checks passed, fixture corrected, final 5/5 PASS. Initial log retained.",
|
||||
"final_small_change": "Preserve an already-recorded real global budget stop instead of replacing its reason with subsequent externally killed worker status.",
|
||||
"test_log_sha256": "7e657ff3c93e2b206204635e4e0197136196f1ebe9c8a69c962e5027072838f7",
|
||||
"ruling": "1794689807.792819"
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
test_coordinator_repolls_fault_with_active_workers_without_killing_them (__main__.InfrastructureTests) ... ERROR
|
||||
test_cumulative_cpu_counts_incident_once_and_real_caps_stop (__main__.InfrastructureTests) ... ok
|
||||
test_live_guard_monitor_exception_does_not_kill_child (__main__.InfrastructureTests) ... ok
|
||||
test_old_tmp_replace_race_reproduced_new_scan_survives (__main__.InfrastructureTests) ... ok
|
||||
test_wall_cap_still_stops_with_monitor_faults (__main__.InfrastructureTests) ... ok
|
||||
|
||||
======================================================================
|
||||
ERROR: test_coordinator_repolls_fault_with_active_workers_without_killing_them (__main__.InfrastructureTests)
|
||||
----------------------------------------------------------------------
|
||||
Traceback (most recent call last):
|
||||
File "/home/frank14f/optane/MeltMind/tools/test_s2_recovery_infrastructure.py", line 81, in test_coordinator_repolls_fault_with_active_workers_without_killing_them
|
||||
coordinator.run(root)
|
||||
File "/home/frank14f/optane/MeltMind/tools/run_s2_recovery_preparation.py", line 40, in run
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root)
|
||||
File "/home/frank14f/optane/MeltMind/tools/run_s2_recovery_preparation.py", line 49, in run_locked
|
||||
save(root/'batch-started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json')))
|
||||
File "/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py", line 35, in sha
|
||||
with Path(path).open('rb') as f:
|
||||
File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/pathlib.py", line 1119, in open
|
||||
return self._accessor.open(self, mode, buffering, encoding, errors,
|
||||
FileNotFoundError: [Errno 2] No such file or directory: '/tmp/tmpfzpikjip/contract.json'
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 5 tests in 0.400s
|
||||
|
||||
FAILED (errors=1)
|
||||
@@ -0,0 +1,9 @@
|
||||
# S2 准备协调器基础设施修复
|
||||
|
||||
原16682在disk_usage扫描progress.json.tmp时遭遇原子替换竞态,异常进入finally并中止16个worker;未产生原batch-complete。遵循Claude 1794689807.792819,独立版本修复基础设施,原文件及worker数值代码不改。
|
||||
|
||||
新disk_usage每条目录项只做一次不跟随symlink的stat,FileNotFoundError表示该项已消失,跳过后下轮继续统计;其他监测/记账异常记录并重采样。协调器不在记账不新鲜的周期派新任务,但现有worker继续受独立wall/CPU/RSS监督;真实CPU/磁盘超限仍停止。worker监督器也记录并重采样监测异常,墙钟判断始终执行,最终CPU/HWM依wait4计入。CPU/GPU两种监督基础设施均有独立修复副本,原实现和历史pin保留。
|
||||
|
||||
恢复协调器仅接受新134件契约、≤16workers、单件1800s/1.25GiB/0 retries,无新50屏障。累计CPU=事故381个原worker已用量(包含16中止)+恢复批实际量;不把中止CPU再加一次。累计原始/隔离/恢复输出磁盘≤100GiB。真实预算停止原因优先于随后被全局停止清理的worker信号。
|
||||
|
||||
5项合成验收通过:原.tmp竞态复现与修复扫描、监测异常时子进程继续、监测故障时真实wall上限仍停、累计预算判断、134个模拟任务下协调器遇活跃期记账故障后重采样且不中止worker。初次协调器测试漏建契约文件,失败原记录保留;补齐测试夹具后5/5通过。没有重跑科学准备或GPU。后续事故账本、隔离、严格恢复集合与三来源合并是独立交付,不属于已执行计算。
|
||||
@@ -0,0 +1,120 @@
|
||||
"""One authorized 134-part recovery; independent pins and cumulative accounting."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from freeze_g1_components import sha
|
||||
from s2_recovery_supervisor import supervise,save,tree_usage
|
||||
from s2_recovery_monitoring import disk_usage,Monitor,budget_reason
|
||||
from run_s2_pilot_preparation import terminate
|
||||
|
||||
|
||||
def contract(root):
|
||||
c=json.loads((root/'contract.json').read_text())
|
||||
for p,h in c['code_sha256'].items():assert sha(p)==h,p
|
||||
assert sha(c['incident_ledger'])==c['incident_ledger_sha256']
|
||||
assert sha(c['original_snapshot'])==c['original_snapshot_sha256']
|
||||
return c
|
||||
|
||||
|
||||
def guard(root,index):
|
||||
c=contract(root);job=root/'jobs'/f'job-{index:04d}';job.mkdir(parents=True,exist_ok=True)
|
||||
r=supervise([sys.executable,'tools/s2_pilot_prepare.py',str(root),str(index)],job,'worker',
|
||||
dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=1800,tree_rss_bytes=1342177280,
|
||||
disk_bytes=100*2**30,max_processes=1),poll=.25)
|
||||
monitor=Monitor(job/'accounting-errors.jsonl')
|
||||
monitor.retry('guard_receipt',lambda:save(job/'guard-result.json',dict(worker=r,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
||||
original_index=c['original_indices'][index],labelled_infrastructure_recovery=c['original_indices'][index] in c['infra_aborted_indices'])),.25)
|
||||
return 0 if r['status']=='PASS' else 1
|
||||
|
||||
|
||||
def run(root):
|
||||
with (root/'batch.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root)
|
||||
|
||||
|
||||
def run_locked(root):
|
||||
c=contract(root);assert len(c['cases'])==len(c['original_indices'])==134
|
||||
assert not (root/'batch-started.json').exists(),'One recovery batch only'
|
||||
assert c['projected_cumulative_worker_cpu_hours']<=60 and c['projected_cumulative_gib']<=100
|
||||
assert c['limits']['worker_cpu_hours']==60 and c['limits']['disk_bytes']==100*2**30
|
||||
monitor=Monitor(root/'monitor-errors.jsonl');start=time.monotonic()
|
||||
save(root/'batch-started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json')))
|
||||
active={};done=[];next_index=0;reason=None;live_cpu=live_rss=used=peak_rss=0;fresh=False
|
||||
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
||||
def interrupted(*_):
|
||||
nonlocal reason
|
||||
reason='RECOVERY_COORDINATOR_INTERRUPTED'
|
||||
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
|
||||
def read_account(path):
|
||||
r=json.loads(path.read_text());w=r['worker']
|
||||
assert math.isfinite(w['cpu_seconds']) and w['cpu_seconds']>=0
|
||||
assert math.isfinite(r['supervisor_cpu_seconds']) and r['supervisor_cpu_seconds']>=0
|
||||
assert isinstance(w['returncode'],int) and isinstance(w['status'],str)
|
||||
assert 'stop_reason' in w
|
||||
return r
|
||||
try:
|
||||
while active or (next_index<len(c['cases']) and reason is None):
|
||||
cycle_ok=True
|
||||
for i,(p,log) in list(active.items()):
|
||||
code,ok=monitor.sample(f'poll_{i}',p.poll,None);cycle_ok &= ok
|
||||
if ok and code is not None:
|
||||
path=root/'jobs'/f'job-{i:04d}'/'guard-result.json'
|
||||
r,ok=monitor.sample(f'account_{i}',lambda path=path:read_account(path),None);cycle_ok &= ok
|
||||
if not ok:continue
|
||||
done.append(dict(index=i,case=c['cases'][i]['case'],returncode=code,**r));del active[i];log.close()
|
||||
if reason is None and r['worker']['returncode']<0 and r['worker']['stop_reason'] is None:reason='UNEXPLAINED_RECOVERY_WORKER_SIGNAL'
|
||||
print(json.dumps(dict(event='FINISHED',index=i,original_index=c['original_indices'][i],completed=len(done),status=r['worker']['status'])),flush=True)
|
||||
def sample_trees():
|
||||
values=[tree_usage(p.pid) for p,_ in active.values()]
|
||||
return sum(v[0] for v in values),sum(v[1] for v in values)
|
||||
(live_cpu,live_rss),ok=monitor.sample('aggregate_live_usage',sample_trees,(live_cpu,live_rss));cycle_ok &= ok
|
||||
peak_rss=max(peak_rss,live_rss)
|
||||
used,ok=monitor.sample('aggregate_disk',lambda:sum(disk_usage(p) for p in c['budget_roots']),used);cycle_ok &= ok
|
||||
spent=sum(r['worker']['cpu_seconds'] for r in done)
|
||||
over=budget_reason(c['prior_worker_cpu_seconds'],spent,live_cpu,used)
|
||||
if over:reason=over
|
||||
if reason:terminate(active)
|
||||
progress=dict(completed=len(done),active=list(active),next_index=next_index,stop_reason=reason,
|
||||
prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],recovery_worker_cpu_seconds=spent,
|
||||
cumulative_settled_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,
|
||||
live_guard_and_worker_cpu_seconds=live_cpu,active_tree_rss_bytes=live_rss,disk_bytes=used,
|
||||
elapsed_seconds=time.monotonic()-start,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,
|
||||
fresh_accounting=cycle_ok,no_new_first50_barrier=True)
|
||||
_,ok=monitor.sample('progress_write',lambda:save(root/'progress.json',progress),None);cycle_ok &= ok
|
||||
# Do not dispatch new work on a stale accounting cycle. Existing
|
||||
# workers keep their independent wall/CPU/RSS guards while re-polling.
|
||||
while cycle_ok and reason is None and len(active)<16 and next_index<len(c['cases']):
|
||||
i=next_index;job=root/'jobs'/f'job-{i:04d}';job.mkdir(parents=True,exist_ok=True)
|
||||
log=(job/'guard.log').open('x')
|
||||
p=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(i)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
||||
active[i]=(p,log);next_index+=1
|
||||
if active or (not cycle_ok and reason is None):time.sleep(.5)
|
||||
finally:
|
||||
terminate(active)
|
||||
for p,log in active.values():
|
||||
p.wait();log.close()
|
||||
used=monitor.retry('final_disk',lambda:sum(disk_usage(p) for p in c['budget_roots']),.5)
|
||||
spent=sum(r['worker']['cpu_seconds'] for r in done)
|
||||
if c['prior_worker_cpu_seconds']+spent>60*3600:reason='GLOBAL_CPU_LIMIT_AFTER_EXIT'
|
||||
if used>100*2**30:reason='GLOBAL_DISK_LIMIT_AFTER_EXIT'
|
||||
result=dict(status='ATTEMPTS_COMPLETE' if len(done)==134 and reason is None else 'STOPPED',stop_reason=reason,jobs=done,
|
||||
not_run_indices=list(range(next_index,134)),prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],worker_cpu_seconds=spent,
|
||||
cumulative_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done),
|
||||
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-start,
|
||||
peak_aggregate_rss_bytes=peak_rss,disk_bytes=used,failed=sum(r['worker']['status']!='PASS' for r in done),
|
||||
monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,retry=False,replacement=False,test_labels_read=False)
|
||||
monitor.retry('batch_receipt',lambda:save(root/'batch-complete.json',result),.5)
|
||||
print(json.dumps({k:v for k,v in result.items() if k!='jobs'}),flush=True)
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);a=p.parse_args()
|
||||
sys.exit(guard(a.root,a.guard) if a.guard is not None else run(a.root))
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Pilot process guard derived from frozen E1 guard; explicit CUDA visibility.
|
||||
|
||||
Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot
|
||||
is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged.
|
||||
"""
|
||||
import fcntl
|
||||
import ctypes
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
import psutil
|
||||
from s2_recovery_monitoring import disk_usage,Monitor
|
||||
|
||||
|
||||
def save(path, data):
|
||||
path = Path(path)
|
||||
tmp = path.with_suffix(path.suffix + '.tmp')
|
||||
tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def tree_usage(pid):
|
||||
try:
|
||||
root = psutil.Process(pid)
|
||||
processes = [root] + root.children(recursive=True)
|
||||
except psutil.NoSuchProcess:
|
||||
return 0., 0, 0
|
||||
cpu = 0.
|
||||
rss = count = 0
|
||||
for p in processes:
|
||||
try:
|
||||
t = p.cpu_times()
|
||||
# Linux child times include reaped descendants, not live children.
|
||||
cpu += t.user + t.system + t.children_user + t.children_system
|
||||
rss += p.memory_info().rss
|
||||
count += 1
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
return cpu, rss, count
|
||||
|
||||
|
||||
def stop_group(pid):
|
||||
try:
|
||||
os.killpg(pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
def supervise(command, root, name, limits, *, poll=.02, gpu_uuid=None):
|
||||
"""One locked attempt; no automatic retry/replacement; account failed CPU."""
|
||||
root = Path(root)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
monitor=Monitor(root/(name+'.monitor-errors.jsonl'))
|
||||
with (root/'execution.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
started_path = root/(name+'.started.json')
|
||||
receipt_path = root/(name+'.receipt.json')
|
||||
if started_path.exists() or receipt_path.exists():
|
||||
raise FileExistsError('Attempt already exists; no silent retry')
|
||||
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
|
||||
raise ValueError('Every limit must be finite and positive')
|
||||
ledger_path = root/'ledger.json'
|
||||
ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]}
|
||||
if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']:
|
||||
raise RuntimeError('Total CPU exhausted before spawn')
|
||||
if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']:
|
||||
raise RuntimeError('Disk exhausted before spawn')
|
||||
save(started_path, {'command':command,'limits':limits,'started_unix':time.time()})
|
||||
start = last = time.monotonic()
|
||||
cpu_peak = rss_peak = max_processes = max_gap = 0
|
||||
cpu=rss=count=used_disk=0
|
||||
reason = None
|
||||
exit_status = usage = None
|
||||
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1',
|
||||
MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES=str(gpu_uuid or ''))
|
||||
with (root/(name+'.log')).open('x') as log:
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
previous_subreaper = ctypes.c_int()
|
||||
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping')
|
||||
proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT,
|
||||
env=env, start_new_session=True)
|
||||
try:
|
||||
while True:
|
||||
now = time.monotonic()
|
||||
max_gap = max(max_gap, now-last)
|
||||
last = now
|
||||
(cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count))
|
||||
cpu_peak = max(cpu_peak,cpu)
|
||||
rss_peak = max(rss_peak,rss)
|
||||
max_processes = max(max_processes,count)
|
||||
used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk)
|
||||
if now-start >= limits['job_wall_seconds']:
|
||||
reason = 'wall_limit'
|
||||
elif rss > limits['tree_rss_bytes']:
|
||||
reason = 'tree_rss_limit'
|
||||
elif count > limits['max_processes']:
|
||||
reason = 'process_count_limit'
|
||||
elif used_disk >= limits['disk_bytes']:
|
||||
reason = 'disk_limit'
|
||||
elif cpu_peak >= limits['job_cpu_seconds']:
|
||||
reason = 'job_cpu_limit'
|
||||
elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']:
|
||||
reason = 'total_cpu_limit'
|
||||
if reason:
|
||||
stop_group(proc.pid)
|
||||
pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG)
|
||||
if pid:
|
||||
exit_status, usage = status, child_usage
|
||||
break
|
||||
time.sleep(poll)
|
||||
finally:
|
||||
# No detached work may outlive a supervised attempt.
|
||||
stop_group(proc.pid)
|
||||
if exit_status is None:
|
||||
_, exit_status, usage = os.wait4(proc.pid, 0)
|
||||
proc.returncode = os.waitstatus_to_exitcode(exit_status)
|
||||
orphan_cpu = 0.
|
||||
orphan_peak = 0
|
||||
# Killed descendants are adopted here rather than losing their
|
||||
# final CPU usage when the root worker cannot reap them itself.
|
||||
while True:
|
||||
try:
|
||||
_, _, orphan_usage = os.wait4(-proc.pid, 0)
|
||||
except ChildProcessError:
|
||||
break
|
||||
orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime
|
||||
orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024)
|
||||
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state')
|
||||
elapsed = time.monotonic()-start
|
||||
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
|
||||
rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak)
|
||||
used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll)
|
||||
if reason is None:
|
||||
if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit'
|
||||
elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit'
|
||||
elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit'
|
||||
elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit'
|
||||
elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit'
|
||||
receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED',
|
||||
command=command,returncode=proc.returncode,stop_reason=reason,
|
||||
cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak,
|
||||
peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk,
|
||||
max_poll_gap_seconds=max_gap,poll_target_seconds=poll,
|
||||
reaped_orphan_cpu_seconds=orphan_cpu,
|
||||
supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure',
|
||||
gpu_visible=bool(gpu_uuid),gpu_uuid=gpu_uuid,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error)
|
||||
ledger['cpu_seconds_charged'] += charged
|
||||
ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged})
|
||||
monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll)
|
||||
monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll)
|
||||
return receipt
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Race-tolerant sampled infrastructure monitoring, without worker changes."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import stat
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def disk_usage(root):
|
||||
logical=allocated=0
|
||||
for p in Path(root).rglob('*'):
|
||||
try:s=p.stat(follow_symlinks=False)
|
||||
except FileNotFoundError:continue # atomic progress replacement / vanished entry
|
||||
if stat.S_ISREG(s.st_mode):
|
||||
logical+=s.st_size;allocated+=s.st_blocks*512
|
||||
return max(logical,allocated)
|
||||
|
||||
|
||||
class Monitor:
|
||||
def __init__(self,path):self.path=Path(path);self.errors=0;self.last_error=None
|
||||
def record(self,operation,exc):
|
||||
self.errors+=1;self.last_error=dict(unix=time.time(),operation=operation,type=type(exc).__name__,message=str(exc))
|
||||
try:
|
||||
with self.path.open('a') as f:f.write(json.dumps(self.last_error)+'\n')
|
||||
except Exception as log_exc:
|
||||
print(json.dumps(dict(monitor_error=self.last_error,logging_error=str(log_exc))),file=sys.stderr,flush=True)
|
||||
def sample(self,operation,fn,previous):
|
||||
try:return fn(),True
|
||||
except Exception as exc:self.record(operation,exc);return previous,False
|
||||
def retry(self,operation,fn,poll):
|
||||
while True:
|
||||
value,ok=self.sample(operation,fn,None)
|
||||
if ok:return value
|
||||
time.sleep(poll)
|
||||
|
||||
|
||||
def budget_reason(prior_cpu,settled_cpu,live_cpu,disk_bytes,*,cpu_limit=60*3600,disk_limit=100*2**30):
|
||||
if prior_cpu+settled_cpu+live_cpu>=cpu_limit:return 'GLOBAL_CPU_LIMIT'
|
||||
if disk_bytes>=disk_limit:return 'GLOBAL_DISK_LIMIT'
|
||||
return None
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Sequential E1 job guard: tree CPU/RSS, wall, process count and disk receipts.
|
||||
|
||||
Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot
|
||||
is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged.
|
||||
"""
|
||||
import fcntl
|
||||
import ctypes
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
import psutil
|
||||
from s2_recovery_monitoring import disk_usage,Monitor
|
||||
|
||||
|
||||
def save(path, data):
|
||||
path = Path(path)
|
||||
tmp = path.with_suffix(path.suffix + '.tmp')
|
||||
tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def tree_usage(pid):
|
||||
try:
|
||||
root = psutil.Process(pid)
|
||||
processes = [root] + root.children(recursive=True)
|
||||
except psutil.NoSuchProcess:
|
||||
return 0., 0, 0
|
||||
cpu = 0.
|
||||
rss = count = 0
|
||||
for p in processes:
|
||||
try:
|
||||
t = p.cpu_times()
|
||||
# Linux child times include reaped descendants, not live children.
|
||||
cpu += t.user + t.system + t.children_user + t.children_system
|
||||
rss += p.memory_info().rss
|
||||
count += 1
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
return cpu, rss, count
|
||||
|
||||
|
||||
def stop_group(pid):
|
||||
try:
|
||||
os.killpg(pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
def supervise(command, root, name, limits, *, poll=.02):
|
||||
"""One locked attempt; no automatic retry/replacement; account failed CPU."""
|
||||
root = Path(root)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
monitor=Monitor(root/(name+'.monitor-errors.jsonl'))
|
||||
with (root/'execution.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
started_path = root/(name+'.started.json')
|
||||
receipt_path = root/(name+'.receipt.json')
|
||||
if started_path.exists() or receipt_path.exists():
|
||||
raise FileExistsError('Attempt already exists; no silent retry')
|
||||
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
|
||||
raise ValueError('Every limit must be finite and positive')
|
||||
ledger_path = root/'ledger.json'
|
||||
ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]}
|
||||
if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']:
|
||||
raise RuntimeError('Total CPU exhausted before spawn')
|
||||
if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']:
|
||||
raise RuntimeError('Disk exhausted before spawn')
|
||||
save(started_path, {'command':command,'limits':limits,'started_unix':time.time()})
|
||||
start = last = time.monotonic()
|
||||
cpu_peak = rss_peak = max_processes = max_gap = 0
|
||||
cpu=rss=count=used_disk=0
|
||||
reason = None
|
||||
exit_status = usage = None
|
||||
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1',
|
||||
MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
|
||||
with (root/(name+'.log')).open('x') as log:
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
previous_subreaper = ctypes.c_int()
|
||||
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping')
|
||||
proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT,
|
||||
env=env, start_new_session=True)
|
||||
try:
|
||||
while True:
|
||||
now = time.monotonic()
|
||||
max_gap = max(max_gap, now-last)
|
||||
last = now
|
||||
(cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count))
|
||||
cpu_peak = max(cpu_peak,cpu)
|
||||
rss_peak = max(rss_peak,rss)
|
||||
max_processes = max(max_processes,count)
|
||||
used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk)
|
||||
if now-start >= limits['job_wall_seconds']:
|
||||
reason = 'wall_limit'
|
||||
elif rss > limits['tree_rss_bytes']:
|
||||
reason = 'tree_rss_limit'
|
||||
elif count > limits['max_processes']:
|
||||
reason = 'process_count_limit'
|
||||
elif used_disk >= limits['disk_bytes']:
|
||||
reason = 'disk_limit'
|
||||
elif cpu_peak >= limits['job_cpu_seconds']:
|
||||
reason = 'job_cpu_limit'
|
||||
elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']:
|
||||
reason = 'total_cpu_limit'
|
||||
if reason:
|
||||
stop_group(proc.pid)
|
||||
pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG)
|
||||
if pid:
|
||||
exit_status, usage = status, child_usage
|
||||
break
|
||||
time.sleep(poll)
|
||||
finally:
|
||||
# No detached work may outlive a supervised attempt.
|
||||
stop_group(proc.pid)
|
||||
if exit_status is None:
|
||||
_, exit_status, usage = os.wait4(proc.pid, 0)
|
||||
proc.returncode = os.waitstatus_to_exitcode(exit_status)
|
||||
orphan_cpu = 0.
|
||||
orphan_peak = 0
|
||||
# Killed descendants are adopted here rather than losing their
|
||||
# final CPU usage when the root worker cannot reap them itself.
|
||||
while True:
|
||||
try:
|
||||
_, _, orphan_usage = os.wait4(-proc.pid, 0)
|
||||
except ChildProcessError:
|
||||
break
|
||||
orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime
|
||||
orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024)
|
||||
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state')
|
||||
elapsed = time.monotonic()-start
|
||||
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
|
||||
rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak)
|
||||
used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll)
|
||||
if reason is None:
|
||||
if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit'
|
||||
elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit'
|
||||
elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit'
|
||||
elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit'
|
||||
elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit'
|
||||
receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED',
|
||||
command=command,returncode=proc.returncode,stop_reason=reason,
|
||||
cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak,
|
||||
peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk,
|
||||
max_poll_gap_seconds=max_gap,poll_target_seconds=poll,
|
||||
reaped_orphan_cpu_seconds=orphan_cpu,
|
||||
supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure',
|
||||
gpu_visible=False,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error)
|
||||
ledger['cpu_seconds_charged'] += charged
|
||||
ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged})
|
||||
monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll)
|
||||
monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll)
|
||||
return receipt
|
||||
@@ -0,0 +1,87 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from s2_recovery_monitoring import disk_usage,budget_reason
|
||||
import s2_e1_supervisor as old
|
||||
import s2_recovery_supervisor as fixed
|
||||
import run_s2_recovery_preparation as coordinator
|
||||
|
||||
|
||||
class InfrastructureTests(unittest.TestCase):
|
||||
def test_old_tmp_replace_race_reproduced_new_scan_survives(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);pending=root/'progress.json.tmp';final=root/'progress.json';pending.write_text('new');final.write_text('old')
|
||||
stat=Path.stat;calls=0
|
||||
def racing_stat(path,*a,**kw):
|
||||
nonlocal calls
|
||||
if path==pending:
|
||||
calls+=1
|
||||
if calls==2:pending.replace(final)
|
||||
return stat(path,*a,**kw)
|
||||
with patch.object(Path,'rglob',return_value=iter([pending,final])),patch.object(Path,'stat',racing_stat):
|
||||
with self.assertRaises(FileNotFoundError):old.disk_usage(root)
|
||||
pending.write_text('replacement')
|
||||
def disappearing_stat(path,*a,**kw):
|
||||
if path==pending:pending.replace(final)
|
||||
return stat(path,*a,**kw)
|
||||
with patch.object(Path,'rglob',return_value=iter([pending,final])),patch.object(Path,'stat',disappearing_stat):
|
||||
measured=disk_usage(root)
|
||||
self.assertEqual(measured,max(final.stat().st_size,final.stat().st_blocks*512))
|
||||
def test_live_guard_monitor_exception_does_not_kill_child(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
real=fixed.disk_usage;calls=0
|
||||
def once(root):
|
||||
nonlocal calls
|
||||
calls+=1
|
||||
if calls==2:raise OSError('synthetic live monitoring failure')
|
||||
return real(root)
|
||||
limits=dict(total_cpu_seconds=3,job_cpu_seconds=2,job_wall_seconds=2,tree_rss_bytes=1024*2**20,disk_bytes=2**20,max_processes=1)
|
||||
with patch.object(fixed,'disk_usage',once):r=fixed.supervise([sys.executable,'-c','import time; time.sleep(.12)'],tmp,'worker',limits,poll=.01)
|
||||
self.assertEqual(r['status'],'PASS');self.assertEqual(r['monitor_errors'],1);self.assertEqual(r['returncode'],0)
|
||||
def test_wall_cap_still_stops_with_monitor_faults(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
real=fixed.disk_usage;calls=0
|
||||
def intermittent(root):
|
||||
nonlocal calls
|
||||
calls+=1
|
||||
if calls in [2,3]:raise OSError('synthetic sample failure')
|
||||
return real(root)
|
||||
limits=dict(total_cpu_seconds=3,job_cpu_seconds=2,job_wall_seconds=.08,tree_rss_bytes=1024*2**20,disk_bytes=2**20,max_processes=1)
|
||||
with patch.object(fixed,'disk_usage',intermittent):r=fixed.supervise([sys.executable,'-c','import time; time.sleep(2)'],tmp,'worker',limits,poll=.01)
|
||||
self.assertEqual(r['stop_reason'],'wall_limit');self.assertLess(r['returncode'],0);self.assertGreater(r['monitor_errors'],0)
|
||||
def test_cumulative_cpu_counts_incident_once_and_real_caps_stop(self):
|
||||
self.assertIsNone(budget_reason(23.78094*3600,3600,100,100))
|
||||
self.assertEqual(budget_reason(23.78094*3600,36.21906*3600,1,0),'GLOBAL_CPU_LIMIT')
|
||||
self.assertEqual(budget_reason(0,0,0,100*2**30),'GLOBAL_DISK_LIMIT')
|
||||
def test_coordinator_repolls_fault_with_active_workers_without_killing_them(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);spawned=[];terminations=[];fault=False
|
||||
c=dict(cases=[dict(case=str(i)) for i in range(134)],original_indices=list(range(134)),
|
||||
projected_cumulative_worker_cpu_hours=1,projected_cumulative_gib=1,prior_worker_cpu_seconds=0.,
|
||||
limits=dict(worker_cpu_hours=60,disk_bytes=100*2**30),budget_roots=[str(root)])
|
||||
(root/'contract.json').write_text(json.dumps(c))
|
||||
class FakeProcess:
|
||||
def __init__(self,index):self.index=index;self.pid=1000000+index;self.polls=0
|
||||
def poll(self):
|
||||
self.polls+=1
|
||||
if self.polls==1:return None
|
||||
path=root/'jobs'/f'job-{self.index:04d}'/'guard-result.json'
|
||||
path.write_text(json.dumps(dict(worker=dict(status='PASS',returncode=0,stop_reason=None,cpu_seconds=.01),supervisor_cpu_seconds=0.)))
|
||||
return 0
|
||||
def spawn(command,**kwargs):
|
||||
p=FakeProcess(int(command[-1]));spawned.append(p);return p
|
||||
def disk(_):
|
||||
nonlocal fault
|
||||
if spawned and not fault:
|
||||
fault=True;raise OSError('synthetic coordinator accounting failure')
|
||||
return 100
|
||||
with patch.object(coordinator,'contract',return_value=c),patch.object(coordinator.subprocess,'Popen',spawn),patch.object(coordinator,'disk_usage',disk),patch.object(coordinator,'tree_usage',return_value=(.01,100,1)),patch.object(coordinator.time,'sleep'),patch.object(coordinator,'terminate',side_effect=lambda active:terminations.append(list(active))),patch('builtins.print'):
|
||||
coordinator.run(root)
|
||||
r=json.loads((root/'batch-complete.json').read_text());self.assertEqual(r['status'],'ATTEMPTS_COMPLETE')
|
||||
self.assertEqual(len(r['jobs']),134);self.assertEqual(r['monitor_errors'],1);self.assertTrue(all(not ids for ids in terminations))
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user