121 lines
7.4 KiB
Python
121 lines
7.4 KiB
Python
"""One authorized 134-part recovery; independent pins and cumulative accounting."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from freeze_g1_components import sha
|
|
from s2_recovery_supervisor import supervise,save,tree_usage
|
|
from s2_recovery_monitoring import disk_usage,Monitor,budget_reason
|
|
from run_s2_pilot_preparation import terminate
|
|
|
|
|
|
def contract(root):
|
|
c=json.loads((root/'contract.json').read_text())
|
|
for p,h in c['code_sha256'].items():assert sha(p)==h,p
|
|
assert sha(c['incident_ledger'])==c['incident_ledger_sha256']
|
|
assert sha(c['original_snapshot'])==c['original_snapshot_sha256']
|
|
return c
|
|
|
|
|
|
def guard(root,index):
|
|
c=contract(root);job=root/'jobs'/f'job-{index:04d}';job.mkdir(parents=True,exist_ok=True)
|
|
r=supervise([sys.executable,'tools/s2_pilot_prepare.py',str(root),str(index)],job,'worker',
|
|
dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=1800,tree_rss_bytes=1342177280,
|
|
disk_bytes=100*2**30,max_processes=1),poll=.25)
|
|
monitor=Monitor(job/'accounting-errors.jsonl')
|
|
monitor.retry('guard_receipt',lambda:save(job/'guard-result.json',dict(worker=r,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
|
original_index=c['original_indices'][index],labelled_infrastructure_recovery=c['original_indices'][index] in c['infra_aborted_indices'])),.25)
|
|
return 0 if r['status']=='PASS' else 1
|
|
|
|
|
|
def run(root):
|
|
with (root/'batch.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root)
|
|
|
|
|
|
def run_locked(root):
|
|
c=contract(root);assert len(c['cases'])==len(c['original_indices'])==134
|
|
assert not (root/'batch-started.json').exists(),'One recovery batch only'
|
|
assert c['projected_cumulative_worker_cpu_hours']<=60 and c['projected_cumulative_gib']<=100
|
|
assert c['limits']['worker_cpu_hours']==60 and c['limits']['disk_bytes']==100*2**30
|
|
monitor=Monitor(root/'monitor-errors.jsonl');start=time.monotonic()
|
|
save(root/'batch-started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json')))
|
|
active={};done=[];next_index=0;reason=None;live_cpu=live_rss=used=peak_rss=0;fresh=False
|
|
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
|
def interrupted(*_):
|
|
nonlocal reason
|
|
reason='RECOVERY_COORDINATOR_INTERRUPTED'
|
|
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
|
|
def read_account(path):
|
|
r=json.loads(path.read_text());w=r['worker']
|
|
assert math.isfinite(w['cpu_seconds']) and w['cpu_seconds']>=0
|
|
assert math.isfinite(r['supervisor_cpu_seconds']) and r['supervisor_cpu_seconds']>=0
|
|
assert isinstance(w['returncode'],int) and isinstance(w['status'],str)
|
|
assert 'stop_reason' in w
|
|
return r
|
|
try:
|
|
while active or (next_index<len(c['cases']) and reason is None):
|
|
cycle_ok=True
|
|
for i,(p,log) in list(active.items()):
|
|
code,ok=monitor.sample(f'poll_{i}',p.poll,None);cycle_ok &= ok
|
|
if ok and code is not None:
|
|
path=root/'jobs'/f'job-{i:04d}'/'guard-result.json'
|
|
r,ok=monitor.sample(f'account_{i}',lambda path=path:read_account(path),None);cycle_ok &= ok
|
|
if not ok:continue
|
|
done.append(dict(index=i,case=c['cases'][i]['case'],returncode=code,**r));del active[i];log.close()
|
|
if reason is None and r['worker']['returncode']<0 and r['worker']['stop_reason'] is None:reason='UNEXPLAINED_RECOVERY_WORKER_SIGNAL'
|
|
print(json.dumps(dict(event='FINISHED',index=i,original_index=c['original_indices'][i],completed=len(done),status=r['worker']['status'])),flush=True)
|
|
def sample_trees():
|
|
values=[tree_usage(p.pid) for p,_ in active.values()]
|
|
return sum(v[0] for v in values),sum(v[1] for v in values)
|
|
(live_cpu,live_rss),ok=monitor.sample('aggregate_live_usage',sample_trees,(live_cpu,live_rss));cycle_ok &= ok
|
|
peak_rss=max(peak_rss,live_rss)
|
|
used,ok=monitor.sample('aggregate_disk',lambda:sum(disk_usage(p) for p in c['budget_roots']),used);cycle_ok &= ok
|
|
spent=sum(r['worker']['cpu_seconds'] for r in done)
|
|
over=budget_reason(c['prior_worker_cpu_seconds'],spent,live_cpu,used)
|
|
if over:reason=over
|
|
if reason:terminate(active)
|
|
progress=dict(completed=len(done),active=list(active),next_index=next_index,stop_reason=reason,
|
|
prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],recovery_worker_cpu_seconds=spent,
|
|
cumulative_settled_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,
|
|
live_guard_and_worker_cpu_seconds=live_cpu,active_tree_rss_bytes=live_rss,disk_bytes=used,
|
|
elapsed_seconds=time.monotonic()-start,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,
|
|
fresh_accounting=cycle_ok,no_new_first50_barrier=True)
|
|
_,ok=monitor.sample('progress_write',lambda:save(root/'progress.json',progress),None);cycle_ok &= ok
|
|
# Do not dispatch new work on a stale accounting cycle. Existing
|
|
# workers keep their independent wall/CPU/RSS guards while re-polling.
|
|
while cycle_ok and reason is None and len(active)<16 and next_index<len(c['cases']):
|
|
i=next_index;job=root/'jobs'/f'job-{i:04d}';job.mkdir(parents=True,exist_ok=True)
|
|
log=(job/'guard.log').open('x')
|
|
p=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(i)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
|
active[i]=(p,log);next_index+=1
|
|
if active or (not cycle_ok and reason is None):time.sleep(.5)
|
|
finally:
|
|
terminate(active)
|
|
for p,log in active.values():
|
|
p.wait();log.close()
|
|
used=monitor.retry('final_disk',lambda:sum(disk_usage(p) for p in c['budget_roots']),.5)
|
|
spent=sum(r['worker']['cpu_seconds'] for r in done)
|
|
if c['prior_worker_cpu_seconds']+spent>60*3600:reason='GLOBAL_CPU_LIMIT_AFTER_EXIT'
|
|
if used>100*2**30:reason='GLOBAL_DISK_LIMIT_AFTER_EXIT'
|
|
result=dict(status='ATTEMPTS_COMPLETE' if len(done)==134 and reason is None else 'STOPPED',stop_reason=reason,jobs=done,
|
|
not_run_indices=list(range(next_index,134)),prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],worker_cpu_seconds=spent,
|
|
cumulative_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done),
|
|
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-start,
|
|
peak_aggregate_rss_bytes=peak_rss,disk_bytes=used,failed=sum(r['worker']['status']!='PASS' for r in done),
|
|
monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,retry=False,replacement=False,test_labels_read=False)
|
|
monitor.retry('batch_receipt',lambda:save(root/'batch-complete.json',result),.5)
|
|
print(json.dumps({k:v for k,v in result.items() if k!='jobs'}),flush=True)
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);a=p.parse_args()
|
|
sys.exit(guard(a.root,a.guard) if a.guard is not None else run(a.root))
|