Files
MeltMind/tools/run_s2_recovery_preparation.py

121 lines
7.4 KiB
Python

"""One authorized 134-part recovery; independent pins and cumulative accounting."""
import argparse
import fcntl
import json
import math
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
from freeze_g1_components import sha
from s2_recovery_supervisor import supervise,save,tree_usage
from s2_recovery_monitoring import disk_usage,Monitor,budget_reason
from run_s2_pilot_preparation import terminate
def contract(root):
c=json.loads((root/'contract.json').read_text())
for p,h in c['code_sha256'].items():assert sha(p)==h,p
assert sha(c['incident_ledger'])==c['incident_ledger_sha256']
assert sha(c['original_snapshot'])==c['original_snapshot_sha256']
return c
def guard(root,index):
c=contract(root);job=root/'jobs'/f'job-{index:04d}';job.mkdir(parents=True,exist_ok=True)
r=supervise([sys.executable,'tools/s2_pilot_prepare.py',str(root),str(index)],job,'worker',
dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=1800,tree_rss_bytes=1342177280,
disk_bytes=100*2**30,max_processes=1),poll=.25)
monitor=Monitor(job/'accounting-errors.jsonl')
monitor.retry('guard_receipt',lambda:save(job/'guard-result.json',dict(worker=r,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
original_index=c['original_indices'][index],labelled_infrastructure_recovery=c['original_indices'][index] in c['infra_aborted_indices'])),.25)
return 0 if r['status']=='PASS' else 1
def run(root):
with (root/'batch.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root)
def run_locked(root):
c=contract(root);assert len(c['cases'])==len(c['original_indices'])==134
assert not (root/'batch-started.json').exists(),'One recovery batch only'
assert c['projected_cumulative_worker_cpu_hours']<=60 and c['projected_cumulative_gib']<=100
assert c['limits']['worker_cpu_hours']==60 and c['limits']['disk_bytes']==100*2**30
monitor=Monitor(root/'monitor-errors.jsonl');start=time.monotonic()
save(root/'batch-started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json')))
active={};done=[];next_index=0;reason=None;live_cpu=live_rss=used=peak_rss=0;fresh=False
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
def interrupted(*_):
nonlocal reason
reason='RECOVERY_COORDINATOR_INTERRUPTED'
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
def read_account(path):
r=json.loads(path.read_text());w=r['worker']
assert math.isfinite(w['cpu_seconds']) and w['cpu_seconds']>=0
assert math.isfinite(r['supervisor_cpu_seconds']) and r['supervisor_cpu_seconds']>=0
assert isinstance(w['returncode'],int) and isinstance(w['status'],str)
assert 'stop_reason' in w
return r
try:
while active or (next_index<len(c['cases']) and reason is None):
cycle_ok=True
for i,(p,log) in list(active.items()):
code,ok=monitor.sample(f'poll_{i}',p.poll,None);cycle_ok &= ok
if ok and code is not None:
path=root/'jobs'/f'job-{i:04d}'/'guard-result.json'
r,ok=monitor.sample(f'account_{i}',lambda path=path:read_account(path),None);cycle_ok &= ok
if not ok:continue
done.append(dict(index=i,case=c['cases'][i]['case'],returncode=code,**r));del active[i];log.close()
if reason is None and r['worker']['returncode']<0 and r['worker']['stop_reason'] is None:reason='UNEXPLAINED_RECOVERY_WORKER_SIGNAL'
print(json.dumps(dict(event='FINISHED',index=i,original_index=c['original_indices'][i],completed=len(done),status=r['worker']['status'])),flush=True)
def sample_trees():
values=[tree_usage(p.pid) for p,_ in active.values()]
return sum(v[0] for v in values),sum(v[1] for v in values)
(live_cpu,live_rss),ok=monitor.sample('aggregate_live_usage',sample_trees,(live_cpu,live_rss));cycle_ok &= ok
peak_rss=max(peak_rss,live_rss)
used,ok=monitor.sample('aggregate_disk',lambda:sum(disk_usage(p) for p in c['budget_roots']),used);cycle_ok &= ok
spent=sum(r['worker']['cpu_seconds'] for r in done)
over=budget_reason(c['prior_worker_cpu_seconds'],spent,live_cpu,used)
if over:reason=over
if reason:terminate(active)
progress=dict(completed=len(done),active=list(active),next_index=next_index,stop_reason=reason,
prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],recovery_worker_cpu_seconds=spent,
cumulative_settled_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,
live_guard_and_worker_cpu_seconds=live_cpu,active_tree_rss_bytes=live_rss,disk_bytes=used,
elapsed_seconds=time.monotonic()-start,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,
fresh_accounting=cycle_ok,no_new_first50_barrier=True)
_,ok=monitor.sample('progress_write',lambda:save(root/'progress.json',progress),None);cycle_ok &= ok
# Do not dispatch new work on a stale accounting cycle. Existing
# workers keep their independent wall/CPU/RSS guards while re-polling.
while cycle_ok and reason is None and len(active)<16 and next_index<len(c['cases']):
i=next_index;job=root/'jobs'/f'job-{i:04d}';job.mkdir(parents=True,exist_ok=True)
log=(job/'guard.log').open('x')
p=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(i)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
active[i]=(p,log);next_index+=1
if active or (not cycle_ok and reason is None):time.sleep(.5)
finally:
terminate(active)
for p,log in active.values():
p.wait();log.close()
used=monitor.retry('final_disk',lambda:sum(disk_usage(p) for p in c['budget_roots']),.5)
spent=sum(r['worker']['cpu_seconds'] for r in done)
if c['prior_worker_cpu_seconds']+spent>60*3600:reason='GLOBAL_CPU_LIMIT_AFTER_EXIT'
if used>100*2**30:reason='GLOBAL_DISK_LIMIT_AFTER_EXIT'
result=dict(status='ATTEMPTS_COMPLETE' if len(done)==134 and reason is None else 'STOPPED',stop_reason=reason,jobs=done,
not_run_indices=list(range(next_index,134)),prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],worker_cpu_seconds=spent,
cumulative_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done),
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-start,
peak_aggregate_rss_bytes=peak_rss,disk_bytes=used,failed=sum(r['worker']['status']!='PASS' for r in done),
monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,retry=False,replacement=False,test_labels_read=False)
monitor.retry('batch_receipt',lambda:save(root/'batch-complete.json',result),.5)
print(json.dumps({k:v for k,v in result.items() if k!='jobs'}),flush=True)
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);a=p.parse_args()
sys.exit(guard(a.root,a.guard) if a.guard is not None else run(a.root))