142 lines
8.8 KiB
Python
142 lines
8.8 KiB
Python
"""Approved 131 geometry-only attempts; independent stage CPU and cumulative disk."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from freeze_g1_components import sha
|
|
from s2_recovery_supervisor import supervise,save,tree_usage
|
|
from s2_recovery_monitoring import disk_usage,Monitor,budget_reason
|
|
from run_s2_pilot_preparation import terminate
|
|
|
|
|
|
def contract(root):
|
|
c=json.loads((root/'contract.json').read_text())
|
|
for p,h in c['code_sha256'].items():assert sha(p)==h,p
|
|
for p,h in c['source_sha256'].items():assert sha(p)==h,p
|
|
assert len(c['cases'])==131 and len({s['case'] for s in c['cases']})==131
|
|
for spec in c['cases']:
|
|
a=spec['attempt']
|
|
assert a['decode_labels'] is False and a['retries']==0
|
|
expected=3600 if a['kind']=='labelled_timeout_repreparation' else 1800
|
|
assert a['wall_limit_seconds']==expected and a['tree_rss_bytes']==1342177280
|
|
roots=[Path(p).resolve() for p in c['budget_roots']]
|
|
assert all(a!=b and a not in b.parents and b not in a.parents for i,a in enumerate(roots) for b in roots[i+1:])
|
|
return c
|
|
|
|
|
|
def guard(root,index):
|
|
c=contract(root);job=root/'jobs'/f'job-{index:04d}';job.mkdir(parents=True,exist_ok=True)
|
|
wall=c['cases'][index]['attempt']['wall_limit_seconds']
|
|
r=supervise([sys.executable,'tools/s2_main_prepare.py',str(root),str(index)],job,'worker',
|
|
dict(total_cpu_seconds=wall+1,job_cpu_seconds=wall,job_wall_seconds=wall,tree_rss_bytes=1342177280,
|
|
disk_bytes=100*2**30,max_processes=1),poll=.25)
|
|
monitor=Monitor(job/'accounting-errors.jsonl')
|
|
monitor.retry('guard_receipt',lambda:save(job/'guard-result.json',dict(worker=r,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
|
attempt=c['cases'][index]['attempt'],label_arrays_decoded=[])),.25)
|
|
return 0 if r['status']=='PASS' else 1
|
|
|
|
|
|
def run(root):
|
|
with (root/'batch.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root)
|
|
|
|
|
|
def run_locked(root):
|
|
c=contract(root)
|
|
assert not (root/'batch-started.json').exists(),'One main preparation batch only; never restart'
|
|
assert c['limits']['workers']==16
|
|
assert c['limits']['worker_cpu_hours']==20 and c['limits']['disk_bytes']==100*2**30
|
|
monitor=Monitor(root/'monitor-errors.jsonl');start=time.monotonic()
|
|
save(root/'batch-started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json'),cpu_affinity=sorted(os.sched_getaffinity(0)),
|
|
limits=c['limits'],labels_decoded=False))
|
|
active={};done=[];next_index=0;reason=None;live_cpu=live_rss=used=peak_rss=0;fresh=False
|
|
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
|
def interrupted(*_):
|
|
nonlocal reason
|
|
reason='MAIN_PREPARATION_COORDINATOR_INTERRUPTED'
|
|
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
|
|
def read_account(path):
|
|
r=json.loads(path.read_text());w=r['worker']
|
|
assert math.isfinite(w['cpu_seconds']) and w['cpu_seconds']>=0
|
|
assert math.isfinite(r['supervisor_cpu_seconds']) and r['supervisor_cpu_seconds']>=0
|
|
assert isinstance(w['returncode'],int) and isinstance(w['status'],str)
|
|
assert 'stop_reason' in w
|
|
return r
|
|
try:
|
|
while active or (next_index<len(c['cases']) and reason is None):
|
|
cycle_ok=True
|
|
for i,(p,log) in list(active.items()):
|
|
code,ok=monitor.sample(f'poll_{i}',p.poll,None);cycle_ok &= ok
|
|
if ok and code is not None:
|
|
path=root/'jobs'/f'job-{i:04d}'/'guard-result.json'
|
|
r,ok=monitor.sample(f'account_{i}',lambda path=path:read_account(path),None);cycle_ok &= ok
|
|
if not ok:
|
|
reason='GUARD_ACCOUNTING_UNAVAILABLE'
|
|
terminate(active)
|
|
save(root/'infrastructure-stop.json',dict(status=reason,index=i,guard_returncode=code,
|
|
active_indices=list(active),not_run_indices=list(range(next_index,len(c['cases']))),
|
|
cpu_accounting_complete=False,prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],
|
|
settled_worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done),
|
|
contract_sha256=sha(root/'contract.json'),retry=False))
|
|
raise RuntimeError(reason)
|
|
done.append(dict(index=i,case=c['cases'][i]['case'],returncode=code,**r));del active[i];log.close()
|
|
if reason is None and r['worker']['returncode']<0 and r['worker']['stop_reason'] is None:reason='UNEXPLAINED_MAIN_PREPARATION_WORKER_SIGNAL'
|
|
print(json.dumps(dict(event='FINISHED',index=i,completed=len(done),status=r['worker']['status'])),flush=True)
|
|
def sample_trees():
|
|
values=[tree_usage(p.pid) for p,_ in active.values()]
|
|
return sum(v[0] for v in values),sum(v[1] for v in values)
|
|
(live_cpu,live_rss),ok=monitor.sample('aggregate_live_usage',sample_trees,(live_cpu,live_rss));cycle_ok &= ok
|
|
peak_rss=max(peak_rss,live_rss)
|
|
used,ok=monitor.sample('aggregate_disk',lambda:sum(disk_usage(p) for p in c['budget_roots']),used);cycle_ok &= ok
|
|
spent=sum(r['worker']['cpu_seconds'] for r in done)
|
|
over=budget_reason(0,spent,live_cpu,used,cpu_limit=c['limits']['worker_cpu_hours']*3600,disk_limit=c['limits']['disk_bytes'])
|
|
if over:reason=over
|
|
if reason:terminate(active)
|
|
progress=dict(completed=len(done),active=list(active),next_index=next_index,stop_reason=reason,
|
|
prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],stage_worker_cpu_seconds=spent,
|
|
cumulative_settled_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,
|
|
live_guard_and_worker_cpu_seconds=live_cpu,active_tree_rss_bytes=live_rss,disk_bytes=used,
|
|
elapsed_seconds=time.monotonic()-start,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,
|
|
fresh_accounting=cycle_ok,no_new_first50_barrier=True)
|
|
_,ok=monitor.sample('progress_write',lambda:save(root/'progress.json',progress),None);cycle_ok &= ok
|
|
# Do not dispatch new work on a stale accounting cycle. Existing
|
|
# workers keep their independent wall/CPU/RSS guards while re-polling.
|
|
while cycle_ok and reason is None and len(active)<c['limits']['workers'] and next_index<len(c['cases']):
|
|
i=next_index;job=root/'jobs'/f'job-{i:04d}';job.mkdir(parents=True,exist_ok=True)
|
|
log=(job/'guard.log').open('x')
|
|
p=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(i)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
|
active[i]=(p,log);next_index+=1
|
|
if active or (not cycle_ok and reason is None):time.sleep(.5)
|
|
finally:
|
|
terminate(active)
|
|
for p,log in active.values():
|
|
p.wait();log.close()
|
|
used=monitor.retry('final_disk',lambda:sum(disk_usage(p) for p in c['budget_roots']),.5)
|
|
spent=sum(r['worker']['cpu_seconds'] for r in done)
|
|
if spent>c['limits']['worker_cpu_hours']*3600:reason='GLOBAL_CPU_LIMIT_AFTER_EXIT'
|
|
if used>100*2**30:reason='GLOBAL_DISK_LIMIT_AFTER_EXIT'
|
|
result=dict(status='ATTEMPTS_COMPLETE' if len(done)==len(c['cases']) and reason is None else 'STOPPED',stop_reason=reason,jobs=done,
|
|
not_run_indices=list(range(next_index,len(c['cases']))),prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],worker_cpu_seconds=spent,
|
|
cumulative_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done),
|
|
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-start,
|
|
peak_aggregate_rss_bytes=peak_rss,disk_bytes=used,failed=sum(r['worker']['status']!='PASS' for r in done),
|
|
monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,retry=False,replacement=False,test_labels_read=False)
|
|
monitor.retry('batch_receipt',lambda:save(root/'batch-complete.json',result),.5)
|
|
print(json.dumps({k:v for k,v in result.items() if k!='jobs'}),flush=True)
|
|
if result['status']=='ATTEMPTS_COMPLETE':
|
|
from freeze_s2_main_roster import freeze
|
|
freeze(root)
|
|
return 0 if result['status']=='ATTEMPTS_COMPLETE' else 1
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);a=p.parse_args()
|
|
sys.exit(guard(a.root,a.guard) if a.guard is not None else run(a.root))
|