Files
MeltMind/tools/run_s2_main_preparation.py
T

142 lines
8.8 KiB
Python

"""Approved 131 geometry-only attempts; independent stage CPU and cumulative disk."""
import argparse
import fcntl
import json
import math
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
from freeze_g1_components import sha
from s2_recovery_supervisor import supervise,save,tree_usage
from s2_recovery_monitoring import disk_usage,Monitor,budget_reason
from run_s2_pilot_preparation import terminate
def contract(root):
c=json.loads((root/'contract.json').read_text())
for p,h in c['code_sha256'].items():assert sha(p)==h,p
for p,h in c['source_sha256'].items():assert sha(p)==h,p
assert len(c['cases'])==131 and len({s['case'] for s in c['cases']})==131
for spec in c['cases']:
a=spec['attempt']
assert a['decode_labels'] is False and a['retries']==0
expected=3600 if a['kind']=='labelled_timeout_repreparation' else 1800
assert a['wall_limit_seconds']==expected and a['tree_rss_bytes']==1342177280
roots=[Path(p).resolve() for p in c['budget_roots']]
assert all(a!=b and a not in b.parents and b not in a.parents for i,a in enumerate(roots) for b in roots[i+1:])
return c
def guard(root,index):
c=contract(root);job=root/'jobs'/f'job-{index:04d}';job.mkdir(parents=True,exist_ok=True)
wall=c['cases'][index]['attempt']['wall_limit_seconds']
r=supervise([sys.executable,'tools/s2_main_prepare.py',str(root),str(index)],job,'worker',
dict(total_cpu_seconds=wall+1,job_cpu_seconds=wall,job_wall_seconds=wall,tree_rss_bytes=1342177280,
disk_bytes=100*2**30,max_processes=1),poll=.25)
monitor=Monitor(job/'accounting-errors.jsonl')
monitor.retry('guard_receipt',lambda:save(job/'guard-result.json',dict(worker=r,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
attempt=c['cases'][index]['attempt'],label_arrays_decoded=[])),.25)
return 0 if r['status']=='PASS' else 1
def run(root):
with (root/'batch.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root)
def run_locked(root):
c=contract(root)
assert not (root/'batch-started.json').exists(),'One main preparation batch only; never restart'
assert c['limits']['workers']==16
assert c['limits']['worker_cpu_hours']==20 and c['limits']['disk_bytes']==100*2**30
monitor=Monitor(root/'monitor-errors.jsonl');start=time.monotonic()
save(root/'batch-started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json'),cpu_affinity=sorted(os.sched_getaffinity(0)),
limits=c['limits'],labels_decoded=False))
active={};done=[];next_index=0;reason=None;live_cpu=live_rss=used=peak_rss=0;fresh=False
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
def interrupted(*_):
nonlocal reason
reason='MAIN_PREPARATION_COORDINATOR_INTERRUPTED'
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
def read_account(path):
r=json.loads(path.read_text());w=r['worker']
assert math.isfinite(w['cpu_seconds']) and w['cpu_seconds']>=0
assert math.isfinite(r['supervisor_cpu_seconds']) and r['supervisor_cpu_seconds']>=0
assert isinstance(w['returncode'],int) and isinstance(w['status'],str)
assert 'stop_reason' in w
return r
try:
while active or (next_index<len(c['cases']) and reason is None):
cycle_ok=True
for i,(p,log) in list(active.items()):
code,ok=monitor.sample(f'poll_{i}',p.poll,None);cycle_ok &= ok
if ok and code is not None:
path=root/'jobs'/f'job-{i:04d}'/'guard-result.json'
r,ok=monitor.sample(f'account_{i}',lambda path=path:read_account(path),None);cycle_ok &= ok
if not ok:
reason='GUARD_ACCOUNTING_UNAVAILABLE'
terminate(active)
save(root/'infrastructure-stop.json',dict(status=reason,index=i,guard_returncode=code,
active_indices=list(active),not_run_indices=list(range(next_index,len(c['cases']))),
cpu_accounting_complete=False,prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],
settled_worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done),
contract_sha256=sha(root/'contract.json'),retry=False))
raise RuntimeError(reason)
done.append(dict(index=i,case=c['cases'][i]['case'],returncode=code,**r));del active[i];log.close()
if reason is None and r['worker']['returncode']<0 and r['worker']['stop_reason'] is None:reason='UNEXPLAINED_MAIN_PREPARATION_WORKER_SIGNAL'
print(json.dumps(dict(event='FINISHED',index=i,completed=len(done),status=r['worker']['status'])),flush=True)
def sample_trees():
values=[tree_usage(p.pid) for p,_ in active.values()]
return sum(v[0] for v in values),sum(v[1] for v in values)
(live_cpu,live_rss),ok=monitor.sample('aggregate_live_usage',sample_trees,(live_cpu,live_rss));cycle_ok &= ok
peak_rss=max(peak_rss,live_rss)
used,ok=monitor.sample('aggregate_disk',lambda:sum(disk_usage(p) for p in c['budget_roots']),used);cycle_ok &= ok
spent=sum(r['worker']['cpu_seconds'] for r in done)
over=budget_reason(0,spent,live_cpu,used,cpu_limit=c['limits']['worker_cpu_hours']*3600,disk_limit=c['limits']['disk_bytes'])
if over:reason=over
if reason:terminate(active)
progress=dict(completed=len(done),active=list(active),next_index=next_index,stop_reason=reason,
prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],stage_worker_cpu_seconds=spent,
cumulative_settled_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,
live_guard_and_worker_cpu_seconds=live_cpu,active_tree_rss_bytes=live_rss,disk_bytes=used,
elapsed_seconds=time.monotonic()-start,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,
fresh_accounting=cycle_ok,no_new_first50_barrier=True)
_,ok=monitor.sample('progress_write',lambda:save(root/'progress.json',progress),None);cycle_ok &= ok
# Do not dispatch new work on a stale accounting cycle. Existing
# workers keep their independent wall/CPU/RSS guards while re-polling.
while cycle_ok and reason is None and len(active)<c['limits']['workers'] and next_index<len(c['cases']):
i=next_index;job=root/'jobs'/f'job-{i:04d}';job.mkdir(parents=True,exist_ok=True)
log=(job/'guard.log').open('x')
p=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(i)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
active[i]=(p,log);next_index+=1
if active or (not cycle_ok and reason is None):time.sleep(.5)
finally:
terminate(active)
for p,log in active.values():
p.wait();log.close()
used=monitor.retry('final_disk',lambda:sum(disk_usage(p) for p in c['budget_roots']),.5)
spent=sum(r['worker']['cpu_seconds'] for r in done)
if spent>c['limits']['worker_cpu_hours']*3600:reason='GLOBAL_CPU_LIMIT_AFTER_EXIT'
if used>100*2**30:reason='GLOBAL_DISK_LIMIT_AFTER_EXIT'
result=dict(status='ATTEMPTS_COMPLETE' if len(done)==len(c['cases']) and reason is None else 'STOPPED',stop_reason=reason,jobs=done,
not_run_indices=list(range(next_index,len(c['cases']))),prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],worker_cpu_seconds=spent,
cumulative_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done),
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-start,
peak_aggregate_rss_bytes=peak_rss,disk_bytes=used,failed=sum(r['worker']['status']!='PASS' for r in done),
monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,retry=False,replacement=False,test_labels_read=False)
monitor.retry('batch_receipt',lambda:save(root/'batch-complete.json',result),.5)
print(json.dumps({k:v for k,v in result.items() if k!='jobs'}),flush=True)
if result['status']=='ATTEMPTS_COMPLETE':
from freeze_s2_main_roster import freeze
freeze(root)
return 0 if result['status']=='ATTEMPTS_COMPLETE' else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);a=p.parse_args()
sys.exit(guard(a.root,a.guard) if a.guard is not None else run(a.root))