"""One authorized 134-part recovery; independent pins and cumulative accounting.""" import argparse import fcntl import json import math import os from pathlib import Path import resource import signal import subprocess import sys import time from freeze_g1_components import sha from s2_recovery_supervisor import supervise,save,tree_usage from s2_recovery_monitoring import disk_usage,Monitor,budget_reason from run_s2_pilot_preparation import terminate def contract(root): c=json.loads((root/'contract.json').read_text()) for p,h in c['code_sha256'].items():assert sha(p)==h,p assert sha(c['incident_ledger'])==c['incident_ledger_sha256'] assert sha(c['original_snapshot'])==c['original_snapshot_sha256'] return c def guard(root,index): c=contract(root);job=root/'jobs'/f'job-{index:04d}';job.mkdir(parents=True,exist_ok=True) r=supervise([sys.executable,'tools/s2_pilot_prepare.py',str(root),str(index)],job,'worker', dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=1800,tree_rss_bytes=1342177280, disk_bytes=100*2**30,max_processes=1),poll=.25) monitor=Monitor(job/'accounting-errors.jsonl') monitor.retry('guard_receipt',lambda:save(job/'guard-result.json',dict(worker=r,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]), original_index=c['original_indices'][index],labelled_infrastructure_recovery=c['original_indices'][index] in c['infra_aborted_indices'])),.25) return 0 if r['status']=='PASS' else 1 def run(root): with (root/'batch.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root) def run_locked(root): c=contract(root);assert len(c['cases'])==len(c['original_indices'])==134 assert not (root/'batch-started.json').exists(),'One recovery batch only' assert c['projected_cumulative_worker_cpu_hours']<=60 and c['projected_cumulative_gib']<=100 assert c['limits']['worker_cpu_hours']==60 and c['limits']['disk_bytes']==100*2**30 monitor=Monitor(root/'monitor-errors.jsonl');start=time.monotonic() save(root/'batch-started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json'))) active={};done=[];next_index=0;reason=None;live_cpu=live_rss=used=peak_rss=0;fresh=False env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='') def interrupted(*_): nonlocal reason reason='RECOVERY_COORDINATOR_INTERRUPTED' signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted) def read_account(path): r=json.loads(path.read_text());w=r['worker'] assert math.isfinite(w['cpu_seconds']) and w['cpu_seconds']>=0 assert math.isfinite(r['supervisor_cpu_seconds']) and r['supervisor_cpu_seconds']>=0 assert isinstance(w['returncode'],int) and isinstance(w['status'],str) assert 'stop_reason' in w return r try: while active or (next_index60*3600:reason='GLOBAL_CPU_LIMIT_AFTER_EXIT' if used>100*2**30:reason='GLOBAL_DISK_LIMIT_AFTER_EXIT' result=dict(status='ATTEMPTS_COMPLETE' if len(done)==134 and reason is None else 'STOPPED',stop_reason=reason,jobs=done, not_run_indices=list(range(next_index,134)),prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],worker_cpu_seconds=spent, cumulative_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done), coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-start, peak_aggregate_rss_bytes=peak_rss,disk_bytes=used,failed=sum(r['worker']['status']!='PASS' for r in done), monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,retry=False,replacement=False,test_labels_read=False) monitor.retry('batch_receipt',lambda:save(root/'batch-complete.json',result),.5) print(json.dumps({k:v for k,v in result.items() if k!='jobs'}),flush=True) if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);a=p.parse_args() sys.exit(guard(a.root,a.guard) if a.guard is not None else run(a.root))