diff --git a/artifacts/s2-recovery-implementation-2026-11-15/infrastructure-final-tests.txt b/artifacts/s2-recovery-implementation-2026-11-15/infrastructure-final-tests.txt new file mode 100644 index 000000000..ba7dc839d --- /dev/null +++ b/artifacts/s2-recovery-implementation-2026-11-15/infrastructure-final-tests.txt @@ -0,0 +1,10 @@ +test_coordinator_repolls_fault_with_active_workers_without_killing_them (__main__.InfrastructureTests) ... ok +test_cumulative_cpu_counts_incident_once_and_real_caps_stop (__main__.InfrastructureTests) ... ok +test_live_guard_monitor_exception_does_not_kill_child (__main__.InfrastructureTests) ... ok +test_old_tmp_replace_race_reproduced_new_scan_survives (__main__.InfrastructureTests) ... ok +test_wall_cap_still_stops_with_monitor_faults (__main__.InfrastructureTests) ... ok + +---------------------------------------------------------------------- +Ran 5 tests in 0.315s + +OK diff --git a/artifacts/s2-recovery-implementation-2026-11-15/infrastructure-receipt.json b/artifacts/s2-recovery-implementation-2026-11-15/infrastructure-receipt.json new file mode 100644 index 000000000..44bf10e04 --- /dev/null +++ b/artifacts/s2-recovery-implementation-2026-11-15/infrastructure-receipt.json @@ -0,0 +1,18 @@ +{ + "status": "PASS_SYNTHETIC_INFRASTRUCTURE_FIX", + "tests": 5, + "passed": 5, + "source_sha256": { + "tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919", + "tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", + "tools/run_s2_recovery_preparation.py": "c135a8f718468b5b1286924d45b83597bf6cf8785af53c70c53fc074787730f1", + "tools/test_s2_recovery_infrastructure.py": "3263b2c143d1bac908093acff15a77ffe5914edd40654297b69e8ad47e2fc829" + }, + "original_worker_and_dependencies_pins_intact": true, + "real_recovery_started": false, + "initial_test_failure": "Coordinator fixture omitted contract.json; 4 checks passed, fixture corrected, final 5/5 PASS. Initial log retained.", + "final_small_change": "Preserve an already-recorded real global budget stop instead of replacing its reason with subsequent externally killed worker status.", + "test_log_sha256": "7e657ff3c93e2b206204635e4e0197136196f1ebe9c8a69c962e5027072838f7", + "ruling": "1794689807.792819" +} diff --git a/artifacts/s2-recovery-implementation-2026-11-15/infrastructure-tests.txt b/artifacts/s2-recovery-implementation-2026-11-15/infrastructure-tests.txt new file mode 100644 index 000000000..a5fc2fcc1 --- /dev/null +++ b/artifacts/s2-recovery-implementation-2026-11-15/infrastructure-tests.txt @@ -0,0 +1,26 @@ +test_coordinator_repolls_fault_with_active_workers_without_killing_them (__main__.InfrastructureTests) ... ERROR +test_cumulative_cpu_counts_incident_once_and_real_caps_stop (__main__.InfrastructureTests) ... ok +test_live_guard_monitor_exception_does_not_kill_child (__main__.InfrastructureTests) ... ok +test_old_tmp_replace_race_reproduced_new_scan_survives (__main__.InfrastructureTests) ... ok +test_wall_cap_still_stops_with_monitor_faults (__main__.InfrastructureTests) ... ok + +====================================================================== +ERROR: test_coordinator_repolls_fault_with_active_workers_without_killing_them (__main__.InfrastructureTests) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/home/frank14f/optane/MeltMind/tools/test_s2_recovery_infrastructure.py", line 81, in test_coordinator_repolls_fault_with_active_workers_without_killing_them + coordinator.run(root) + File "/home/frank14f/optane/MeltMind/tools/run_s2_recovery_preparation.py", line 40, in run + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root) + File "/home/frank14f/optane/MeltMind/tools/run_s2_recovery_preparation.py", line 49, in run_locked + save(root/'batch-started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json'))) + File "/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py", line 35, in sha + with Path(path).open('rb') as f: + File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/pathlib.py", line 1119, in open + return self._accessor.open(self, mode, buffering, encoding, errors, +FileNotFoundError: [Errno 2] No such file or directory: '/tmp/tmpfzpikjip/contract.json' + +---------------------------------------------------------------------- +Ran 5 tests in 0.400s + +FAILED (errors=1) diff --git a/docs/s2-infrastructure-recovery-fix-2026-11-15.md b/docs/s2-infrastructure-recovery-fix-2026-11-15.md new file mode 100644 index 000000000..405af3b34 --- /dev/null +++ b/docs/s2-infrastructure-recovery-fix-2026-11-15.md @@ -0,0 +1,9 @@ +# S2 准备协调器基础设施修复 + +原16682在disk_usage扫描progress.json.tmp时遭遇原子替换竞态,异常进入finally并中止16个worker;未产生原batch-complete。遵循Claude 1794689807.792819,独立版本修复基础设施,原文件及worker数值代码不改。 + +新disk_usage每条目录项只做一次不跟随symlink的stat,FileNotFoundError表示该项已消失,跳过后下轮继续统计;其他监测/记账异常记录并重采样。协调器不在记账不新鲜的周期派新任务,但现有worker继续受独立wall/CPU/RSS监督;真实CPU/磁盘超限仍停止。worker监督器也记录并重采样监测异常,墙钟判断始终执行,最终CPU/HWM依wait4计入。CPU/GPU两种监督基础设施均有独立修复副本,原实现和历史pin保留。 + +恢复协调器仅接受新134件契约、≤16workers、单件1800s/1.25GiB/0 retries,无新50屏障。累计CPU=事故381个原worker已用量(包含16中止)+恢复批实际量;不把中止CPU再加一次。累计原始/隔离/恢复输出磁盘≤100GiB。真实预算停止原因优先于随后被全局停止清理的worker信号。 + +5项合成验收通过:原.tmp竞态复现与修复扫描、监测异常时子进程继续、监测故障时真实wall上限仍停、累计预算判断、134个模拟任务下协调器遇活跃期记账故障后重采样且不中止worker。初次协调器测试漏建契约文件,失败原记录保留;补齐测试夹具后5/5通过。没有重跑科学准备或GPU。后续事故账本、隔离、严格恢复集合与三来源合并是独立交付,不属于已执行计算。 diff --git a/tools/run_s2_recovery_preparation.py b/tools/run_s2_recovery_preparation.py new file mode 100644 index 000000000..57e1f39a4 --- /dev/null +++ b/tools/run_s2_recovery_preparation.py @@ -0,0 +1,120 @@ +"""One authorized 134-part recovery; independent pins and cumulative accounting.""" +import argparse +import fcntl +import json +import math +import os +from pathlib import Path +import resource +import signal +import subprocess +import sys +import time +from freeze_g1_components import sha +from s2_recovery_supervisor import supervise,save,tree_usage +from s2_recovery_monitoring import disk_usage,Monitor,budget_reason +from run_s2_pilot_preparation import terminate + + +def contract(root): + c=json.loads((root/'contract.json').read_text()) + for p,h in c['code_sha256'].items():assert sha(p)==h,p + assert sha(c['incident_ledger'])==c['incident_ledger_sha256'] + assert sha(c['original_snapshot'])==c['original_snapshot_sha256'] + return c + + +def guard(root,index): + c=contract(root);job=root/'jobs'/f'job-{index:04d}';job.mkdir(parents=True,exist_ok=True) + r=supervise([sys.executable,'tools/s2_pilot_prepare.py',str(root),str(index)],job,'worker', + dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=1800,tree_rss_bytes=1342177280, + disk_bytes=100*2**30,max_processes=1),poll=.25) + monitor=Monitor(job/'accounting-errors.jsonl') + monitor.retry('guard_receipt',lambda:save(job/'guard-result.json',dict(worker=r,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]), + original_index=c['original_indices'][index],labelled_infrastructure_recovery=c['original_indices'][index] in c['infra_aborted_indices'])),.25) + return 0 if r['status']=='PASS' else 1 + + +def run(root): + with (root/'batch.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root) + + +def run_locked(root): + c=contract(root);assert len(c['cases'])==len(c['original_indices'])==134 + assert not (root/'batch-started.json').exists(),'One recovery batch only' + assert c['projected_cumulative_worker_cpu_hours']<=60 and c['projected_cumulative_gib']<=100 + assert c['limits']['worker_cpu_hours']==60 and c['limits']['disk_bytes']==100*2**30 + monitor=Monitor(root/'monitor-errors.jsonl');start=time.monotonic() + save(root/'batch-started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json'))) + active={};done=[];next_index=0;reason=None;live_cpu=live_rss=used=peak_rss=0;fresh=False + env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='') + def interrupted(*_): + nonlocal reason + reason='RECOVERY_COORDINATOR_INTERRUPTED' + signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted) + def read_account(path): + r=json.loads(path.read_text());w=r['worker'] + assert math.isfinite(w['cpu_seconds']) and w['cpu_seconds']>=0 + assert math.isfinite(r['supervisor_cpu_seconds']) and r['supervisor_cpu_seconds']>=0 + assert isinstance(w['returncode'],int) and isinstance(w['status'],str) + assert 'stop_reason' in w + return r + try: + while active or (next_index60*3600:reason='GLOBAL_CPU_LIMIT_AFTER_EXIT' + if used>100*2**30:reason='GLOBAL_DISK_LIMIT_AFTER_EXIT' + result=dict(status='ATTEMPTS_COMPLETE' if len(done)==134 and reason is None else 'STOPPED',stop_reason=reason,jobs=done, + not_run_indices=list(range(next_index,134)),prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],worker_cpu_seconds=spent, + cumulative_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done), + coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-start, + peak_aggregate_rss_bytes=peak_rss,disk_bytes=used,failed=sum(r['worker']['status']!='PASS' for r in done), + monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,retry=False,replacement=False,test_labels_read=False) + monitor.retry('batch_receipt',lambda:save(root/'batch-complete.json',result),.5) + print(json.dumps({k:v for k,v in result.items() if k!='jobs'}),flush=True) + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);a=p.parse_args() + sys.exit(guard(a.root,a.guard) if a.guard is not None else run(a.root)) diff --git a/tools/s2_recovery_gpu_supervisor.py b/tools/s2_recovery_gpu_supervisor.py new file mode 100644 index 000000000..6e6930171 --- /dev/null +++ b/tools/s2_recovery_gpu_supervisor.py @@ -0,0 +1,158 @@ +"""Pilot process guard derived from frozen E1 guard; explicit CUDA visibility. + +Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot +is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged. +""" +import fcntl +import ctypes +import json +import math +import os +from pathlib import Path +import signal +import subprocess +import time + +import psutil +from s2_recovery_monitoring import disk_usage,Monitor + + +def save(path, data): + path = Path(path) + tmp = path.with_suffix(path.suffix + '.tmp') + tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n') + tmp.replace(path) + + +def tree_usage(pid): + try: + root = psutil.Process(pid) + processes = [root] + root.children(recursive=True) + except psutil.NoSuchProcess: + return 0., 0, 0 + cpu = 0. + rss = count = 0 + for p in processes: + try: + t = p.cpu_times() + # Linux child times include reaped descendants, not live children. + cpu += t.user + t.system + t.children_user + t.children_system + rss += p.memory_info().rss + count += 1 + except psutil.NoSuchProcess: + pass + return cpu, rss, count + + +def stop_group(pid): + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +def supervise(command, root, name, limits, *, poll=.02, gpu_uuid=None): + """One locked attempt; no automatic retry/replacement; account failed CPU.""" + root = Path(root) + root.mkdir(parents=True, exist_ok=True) + monitor=Monitor(root/(name+'.monitor-errors.jsonl')) + with (root/'execution.lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + started_path = root/(name+'.started.json') + receipt_path = root/(name+'.receipt.json') + if started_path.exists() or receipt_path.exists(): + raise FileExistsError('Attempt already exists; no silent retry') + if any(not math.isfinite(v) or v <= 0 for v in limits.values()): + raise ValueError('Every limit must be finite and positive') + ledger_path = root/'ledger.json' + ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]} + if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']: + raise RuntimeError('Total CPU exhausted before spawn') + if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']: + raise RuntimeError('Disk exhausted before spawn') + save(started_path, {'command':command,'limits':limits,'started_unix':time.time()}) + start = last = time.monotonic() + cpu_peak = rss_peak = max_processes = max_gap = 0 + cpu=rss=count=used_disk=0 + reason = None + exit_status = usage = None + env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', + MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES=str(gpu_uuid or '')) + with (root/(name+'.log')).open('x') as log: + libc = ctypes.CDLL(None, use_errno=True) + previous_subreaper = ctypes.c_int() + if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping') + proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, + env=env, start_new_session=True) + try: + while True: + now = time.monotonic() + max_gap = max(max_gap, now-last) + last = now + (cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count)) + cpu_peak = max(cpu_peak,cpu) + rss_peak = max(rss_peak,rss) + max_processes = max(max_processes,count) + used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk) + if now-start >= limits['job_wall_seconds']: + reason = 'wall_limit' + elif rss > limits['tree_rss_bytes']: + reason = 'tree_rss_limit' + elif count > limits['max_processes']: + reason = 'process_count_limit' + elif used_disk >= limits['disk_bytes']: + reason = 'disk_limit' + elif cpu_peak >= limits['job_cpu_seconds']: + reason = 'job_cpu_limit' + elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']: + reason = 'total_cpu_limit' + if reason: + stop_group(proc.pid) + pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG) + if pid: + exit_status, usage = status, child_usage + break + time.sleep(poll) + finally: + # No detached work may outlive a supervised attempt. + stop_group(proc.pid) + if exit_status is None: + _, exit_status, usage = os.wait4(proc.pid, 0) + proc.returncode = os.waitstatus_to_exitcode(exit_status) + orphan_cpu = 0. + orphan_peak = 0 + # Killed descendants are adopted here rather than losing their + # final CPU usage when the root worker cannot reap them itself. + while True: + try: + _, _, orphan_usage = os.wait4(-proc.pid, 0) + except ChildProcessError: + break + orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime + orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024) + if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state') + elapsed = time.monotonic()-start + charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu) + rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak) + used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll) + if reason is None: + if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit' + elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit' + elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit' + elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit' + elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit' + receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED', + command=command,returncode=proc.returncode,stop_reason=reason, + cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak, + peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk, + max_poll_gap_seconds=max_gap,poll_target_seconds=poll, + reaped_orphan_cpu_seconds=orphan_cpu, + supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure', + gpu_visible=bool(gpu_uuid),gpu_uuid=gpu_uuid,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error) + ledger['cpu_seconds_charged'] += charged + ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged}) + monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll) + monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll) + return receipt diff --git a/tools/s2_recovery_monitoring.py b/tools/s2_recovery_monitoring.py new file mode 100644 index 000000000..b1c7ea515 --- /dev/null +++ b/tools/s2_recovery_monitoring.py @@ -0,0 +1,40 @@ +"""Race-tolerant sampled infrastructure monitoring, without worker changes.""" +import json +from pathlib import Path +import stat +import sys +import time + + +def disk_usage(root): + logical=allocated=0 + for p in Path(root).rglob('*'): + try:s=p.stat(follow_symlinks=False) + except FileNotFoundError:continue # atomic progress replacement / vanished entry + if stat.S_ISREG(s.st_mode): + logical+=s.st_size;allocated+=s.st_blocks*512 + return max(logical,allocated) + + +class Monitor: + def __init__(self,path):self.path=Path(path);self.errors=0;self.last_error=None + def record(self,operation,exc): + self.errors+=1;self.last_error=dict(unix=time.time(),operation=operation,type=type(exc).__name__,message=str(exc)) + try: + with self.path.open('a') as f:f.write(json.dumps(self.last_error)+'\n') + except Exception as log_exc: + print(json.dumps(dict(monitor_error=self.last_error,logging_error=str(log_exc))),file=sys.stderr,flush=True) + def sample(self,operation,fn,previous): + try:return fn(),True + except Exception as exc:self.record(operation,exc);return previous,False + def retry(self,operation,fn,poll): + while True: + value,ok=self.sample(operation,fn,None) + if ok:return value + time.sleep(poll) + + +def budget_reason(prior_cpu,settled_cpu,live_cpu,disk_bytes,*,cpu_limit=60*3600,disk_limit=100*2**30): + if prior_cpu+settled_cpu+live_cpu>=cpu_limit:return 'GLOBAL_CPU_LIMIT' + if disk_bytes>=disk_limit:return 'GLOBAL_DISK_LIMIT' + return None diff --git a/tools/s2_recovery_supervisor.py b/tools/s2_recovery_supervisor.py new file mode 100644 index 000000000..ad96cd89c --- /dev/null +++ b/tools/s2_recovery_supervisor.py @@ -0,0 +1,158 @@ +"""Sequential E1 job guard: tree CPU/RSS, wall, process count and disk receipts. + +Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot +is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged. +""" +import fcntl +import ctypes +import json +import math +import os +from pathlib import Path +import signal +import subprocess +import time + +import psutil +from s2_recovery_monitoring import disk_usage,Monitor + + +def save(path, data): + path = Path(path) + tmp = path.with_suffix(path.suffix + '.tmp') + tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n') + tmp.replace(path) + + +def tree_usage(pid): + try: + root = psutil.Process(pid) + processes = [root] + root.children(recursive=True) + except psutil.NoSuchProcess: + return 0., 0, 0 + cpu = 0. + rss = count = 0 + for p in processes: + try: + t = p.cpu_times() + # Linux child times include reaped descendants, not live children. + cpu += t.user + t.system + t.children_user + t.children_system + rss += p.memory_info().rss + count += 1 + except psutil.NoSuchProcess: + pass + return cpu, rss, count + + +def stop_group(pid): + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +def supervise(command, root, name, limits, *, poll=.02): + """One locked attempt; no automatic retry/replacement; account failed CPU.""" + root = Path(root) + root.mkdir(parents=True, exist_ok=True) + monitor=Monitor(root/(name+'.monitor-errors.jsonl')) + with (root/'execution.lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + started_path = root/(name+'.started.json') + receipt_path = root/(name+'.receipt.json') + if started_path.exists() or receipt_path.exists(): + raise FileExistsError('Attempt already exists; no silent retry') + if any(not math.isfinite(v) or v <= 0 for v in limits.values()): + raise ValueError('Every limit must be finite and positive') + ledger_path = root/'ledger.json' + ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]} + if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']: + raise RuntimeError('Total CPU exhausted before spawn') + if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']: + raise RuntimeError('Disk exhausted before spawn') + save(started_path, {'command':command,'limits':limits,'started_unix':time.time()}) + start = last = time.monotonic() + cpu_peak = rss_peak = max_processes = max_gap = 0 + cpu=rss=count=used_disk=0 + reason = None + exit_status = usage = None + env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', + MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='') + with (root/(name+'.log')).open('x') as log: + libc = ctypes.CDLL(None, use_errno=True) + previous_subreaper = ctypes.c_int() + if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping') + proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, + env=env, start_new_session=True) + try: + while True: + now = time.monotonic() + max_gap = max(max_gap, now-last) + last = now + (cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count)) + cpu_peak = max(cpu_peak,cpu) + rss_peak = max(rss_peak,rss) + max_processes = max(max_processes,count) + used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk) + if now-start >= limits['job_wall_seconds']: + reason = 'wall_limit' + elif rss > limits['tree_rss_bytes']: + reason = 'tree_rss_limit' + elif count > limits['max_processes']: + reason = 'process_count_limit' + elif used_disk >= limits['disk_bytes']: + reason = 'disk_limit' + elif cpu_peak >= limits['job_cpu_seconds']: + reason = 'job_cpu_limit' + elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']: + reason = 'total_cpu_limit' + if reason: + stop_group(proc.pid) + pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG) + if pid: + exit_status, usage = status, child_usage + break + time.sleep(poll) + finally: + # No detached work may outlive a supervised attempt. + stop_group(proc.pid) + if exit_status is None: + _, exit_status, usage = os.wait4(proc.pid, 0) + proc.returncode = os.waitstatus_to_exitcode(exit_status) + orphan_cpu = 0. + orphan_peak = 0 + # Killed descendants are adopted here rather than losing their + # final CPU usage when the root worker cannot reap them itself. + while True: + try: + _, _, orphan_usage = os.wait4(-proc.pid, 0) + except ChildProcessError: + break + orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime + orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024) + if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state') + elapsed = time.monotonic()-start + charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu) + rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak) + used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll) + if reason is None: + if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit' + elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit' + elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit' + elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit' + elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit' + receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED', + command=command,returncode=proc.returncode,stop_reason=reason, + cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak, + peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk, + max_poll_gap_seconds=max_gap,poll_target_seconds=poll, + reaped_orphan_cpu_seconds=orphan_cpu, + supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure', + gpu_visible=False,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error) + ledger['cpu_seconds_charged'] += charged + ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged}) + monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll) + monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll) + return receipt diff --git a/tools/test_s2_recovery_infrastructure.py b/tools/test_s2_recovery_infrastructure.py new file mode 100644 index 000000000..bb4f83c5e --- /dev/null +++ b/tools/test_s2_recovery_infrastructure.py @@ -0,0 +1,87 @@ +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch +from s2_recovery_monitoring import disk_usage,budget_reason +import s2_e1_supervisor as old +import s2_recovery_supervisor as fixed +import run_s2_recovery_preparation as coordinator + + +class InfrastructureTests(unittest.TestCase): + def test_old_tmp_replace_race_reproduced_new_scan_survives(self): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);pending=root/'progress.json.tmp';final=root/'progress.json';pending.write_text('new');final.write_text('old') + stat=Path.stat;calls=0 + def racing_stat(path,*a,**kw): + nonlocal calls + if path==pending: + calls+=1 + if calls==2:pending.replace(final) + return stat(path,*a,**kw) + with patch.object(Path,'rglob',return_value=iter([pending,final])),patch.object(Path,'stat',racing_stat): + with self.assertRaises(FileNotFoundError):old.disk_usage(root) + pending.write_text('replacement') + def disappearing_stat(path,*a,**kw): + if path==pending:pending.replace(final) + return stat(path,*a,**kw) + with patch.object(Path,'rglob',return_value=iter([pending,final])),patch.object(Path,'stat',disappearing_stat): + measured=disk_usage(root) + self.assertEqual(measured,max(final.stat().st_size,final.stat().st_blocks*512)) + def test_live_guard_monitor_exception_does_not_kill_child(self): + with tempfile.TemporaryDirectory() as tmp: + real=fixed.disk_usage;calls=0 + def once(root): + nonlocal calls + calls+=1 + if calls==2:raise OSError('synthetic live monitoring failure') + return real(root) + limits=dict(total_cpu_seconds=3,job_cpu_seconds=2,job_wall_seconds=2,tree_rss_bytes=1024*2**20,disk_bytes=2**20,max_processes=1) + with patch.object(fixed,'disk_usage',once):r=fixed.supervise([sys.executable,'-c','import time; time.sleep(.12)'],tmp,'worker',limits,poll=.01) + self.assertEqual(r['status'],'PASS');self.assertEqual(r['monitor_errors'],1);self.assertEqual(r['returncode'],0) + def test_wall_cap_still_stops_with_monitor_faults(self): + with tempfile.TemporaryDirectory() as tmp: + real=fixed.disk_usage;calls=0 + def intermittent(root): + nonlocal calls + calls+=1 + if calls in [2,3]:raise OSError('synthetic sample failure') + return real(root) + limits=dict(total_cpu_seconds=3,job_cpu_seconds=2,job_wall_seconds=.08,tree_rss_bytes=1024*2**20,disk_bytes=2**20,max_processes=1) + with patch.object(fixed,'disk_usage',intermittent):r=fixed.supervise([sys.executable,'-c','import time; time.sleep(2)'],tmp,'worker',limits,poll=.01) + self.assertEqual(r['stop_reason'],'wall_limit');self.assertLess(r['returncode'],0);self.assertGreater(r['monitor_errors'],0) + def test_cumulative_cpu_counts_incident_once_and_real_caps_stop(self): + self.assertIsNone(budget_reason(23.78094*3600,3600,100,100)) + self.assertEqual(budget_reason(23.78094*3600,36.21906*3600,1,0),'GLOBAL_CPU_LIMIT') + self.assertEqual(budget_reason(0,0,0,100*2**30),'GLOBAL_DISK_LIMIT') + def test_coordinator_repolls_fault_with_active_workers_without_killing_them(self): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);spawned=[];terminations=[];fault=False + c=dict(cases=[dict(case=str(i)) for i in range(134)],original_indices=list(range(134)), + projected_cumulative_worker_cpu_hours=1,projected_cumulative_gib=1,prior_worker_cpu_seconds=0., + limits=dict(worker_cpu_hours=60,disk_bytes=100*2**30),budget_roots=[str(root)]) + (root/'contract.json').write_text(json.dumps(c)) + class FakeProcess: + def __init__(self,index):self.index=index;self.pid=1000000+index;self.polls=0 + def poll(self): + self.polls+=1 + if self.polls==1:return None + path=root/'jobs'/f'job-{self.index:04d}'/'guard-result.json' + path.write_text(json.dumps(dict(worker=dict(status='PASS',returncode=0,stop_reason=None,cpu_seconds=.01),supervisor_cpu_seconds=0.))) + return 0 + def spawn(command,**kwargs): + p=FakeProcess(int(command[-1]));spawned.append(p);return p + def disk(_): + nonlocal fault + if spawned and not fault: + fault=True;raise OSError('synthetic coordinator accounting failure') + return 100 + with patch.object(coordinator,'contract',return_value=c),patch.object(coordinator.subprocess,'Popen',spawn),patch.object(coordinator,'disk_usage',disk),patch.object(coordinator,'tree_usage',return_value=(.01,100,1)),patch.object(coordinator.time,'sleep'),patch.object(coordinator,'terminate',side_effect=lambda active:terminations.append(list(active))),patch('builtins.print'): + coordinator.run(root) + r=json.loads((root/'batch-complete.json').read_text());self.assertEqual(r['status'],'ATTEMPTS_COMPLETE') + self.assertEqual(len(r['jobs']),134);self.assertEqual(r['monitor_errors'],1);self.assertTrue(all(not ids for ids in terminations)) + + +if __name__=='__main__':unittest.main(verbosity=2)