Files
MeltMind/tools/run_s2_r2.py

167 lines
11 KiB
Python

"""Independent four-worker CPU HPO/final queue; first-fit barrier, no neural mutations."""
import argparse
import fcntl
import json
import math
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
from freeze_g1_components import sha
from s2_recovery_supervisor import save,supervise,tree_usage
from s2_recovery_monitoring import Monitor,disk_usage,budget_reason
from run_s2_pilot_preparation import terminate
def contract(root):
c=json.loads((root/'contract.json').read_text())
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():assert sha(p)==h,p
assert len(c['jobs'])==195 and len(c['cells'])==15 and c['limits']['worker_cpu_seconds']==360000
assert c['limits']['per_fit_cpu_seconds']==1800 and c['limits']['workers']==4
assert set(c['worker_cpus']).isdisjoint(range(8))
assert c['final_refit']=='same_learning_pool_only' and c['selection_policy']=='best_finished_config_fixed_grid_tie_order'
return c
def select_finished(cell,trials):
assert len(trials)==12 and sorted(t['config_index'] for t in trials)==list(range(12))
finished=[t for t in trials if t['status']=='PASS' and t.get('score') is not None and math.isfinite(t['score'])]
if not finished:return dict(status='NA_ALL_HPO_FAILED',cell=cell,finished_configurations=0,trials=trials,config_index=None)
chosen=min(finished,key=lambda t:(t['score'],t['config_index']))
return dict(status='SELECTED',cell=cell,config_index=chosen['config_index'],validation_score=chosen['score'],
finished_configurations=len(finished),failed_configurations=12-len(finished),trials=trials,
final_refit_set='same_learning_pool_only',ruling='1794707190.141609')
def cost_check(c,first):
cap=c['limits']['per_fit_cpu_seconds'];hard=len(c['jobs'])*cap
if first['worker']['status']!='PASS':
return dict(continue_within_cap=hard<=c['limits']['worker_cpu_seconds'],method='No successful empirical fit; capped195attempts upper-bound only, not a completion forecast',
projected_worker_cpu_seconds=None,capped_attempt_upper_bound_seconds=hard,first_status=first['worker']['status'])
path=Path(first['result_path']);r=json.loads(path.read_text());initial=c['jobs'][0]['cost_units']
remaining=sum(j['cost_units'] for j in c['jobs'][1:]);rate=r['fit_cpu_seconds']/initial
# Non-fit work scales by query rows, final config conservatively at63leaves/1000iterations.
overhead=max(0.,first['worker']['cpu_seconds']-r['fit_cpu_seconds'])
initial_rows=c['jobs'][0]['cost_queries']
forecast=first['worker']['cpu_seconds']+1.25*(rate*remaining+overhead*sum(j['cost_queries']/initial_rows for j in c['jobs'][1:]))
return dict(continue_within_cap=forecast<=c['limits']['worker_cpu_seconds'],projected_worker_cpu_seconds=forecast,
capped_attempt_upper_bound_seconds=hard,first_fit_cpu_seconds=r['fit_cpu_seconds'],first_worker_cpu_seconds=first['worker']['cpu_seconds'],
method='Observed fit CPU/(learning eligible queries*iterations*leaves), query-scaled nonfit CPU, final worst grid config,25percent reserve; heuristic not accuracy guarantee')
def guard(root,index,slot):
c=contract(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]])
dest=root/'jobs'/f'job-{index:03d}';dest.mkdir(parents=True,exist_ok=True)
w=supervise([sys.executable,'tools/run_s2_r2_fit.py',str(root),str(index)],dest,'worker',
dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=7200,tree_rss_bytes=16*2**30,
disk_bytes=5*2**30,max_processes=1),poll=.2)
path=dest/'output/result.json';result=None
if path.exists():result=json.loads(path.read_text())
if w['status']=='PASS' and (not result or result['status']!='COMPLETE'):
w.update(status='STOPPED_OR_FAILED',stop_reason='MISSING_COMPLETE_RESULT')
save(dest/'guard-result.json',dict(worker=w,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
cpu_affinity=sorted(os.sched_getaffinity(0)),result_path=str(path) if path.exists() else None,
result_sha256=sha(path) if path.exists() else None,job=c['jobs'][index]))
return 0 if w['status']=='PASS' else 1
def finalize_selections(root,c,done):
by_index={r['index']:r for r in done};directory=root/'selections';directory.mkdir(exist_ok=False)
selected={}
for cell in c['cells']:
trials=[]
for i in cell['hpo_indices']:
row=by_index[i];path=root/'jobs'/f'job-{i:03d}'/'guard-result.json'
result=json.loads(Path(row['result_path']).read_text()) if row.get('result_path') else None
trials.append(dict(index=i,config_index=c['jobs'][i]['config_index'],status=row['worker']['status'],
reason=row['worker']['stop_reason'] or (None if row['worker']['status']=='PASS' else 'WORKER_EXIT_'+str(row['worker']['returncode'])),
score=result.get('validation_component_equal_relative_l2') if result else None,
receipt_path=str(path),receipt_sha256=sha(path)))
decision=select_finished(cell['index'],trials);save(directory/f"cell-{cell['index']:02d}.json",decision);selected[cell['index']]=decision
return selected
def run(root):
with (root/'batch.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root)
def run_locked(root):
c=contract(root);assert not (root/'batch-started.json').exists(),'No repeat R2 queue'
os.sched_setaffinity(0,c['coordinator_cpus'])
start=time.monotonic();save(root/'batch-started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),cpu_affinity=sorted(os.sched_getaffinity(0))))
active={};done=[];next_index=0;reason=None;first_checked=False;selections=None;live_cpu=live_rss=disk=0
monitor=Monitor(root/'monitor-errors.jsonl')
env=dict(os.environ,OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
def interrupted(*_):
nonlocal reason
reason='R2_COORDINATOR_INTERRUPTED'
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
def terminate_active():terminate({k:(v['process'],v['log']) for k,v in active.items()})
try:
while active or (next_index<195 and reason is None):
for slot,item in list(active.items()):
code=item['process'].poll()
if code is None:continue
path=root/'jobs'/f"job-{item['index']:03d}"/'guard-result.json'
if not path.exists():
save(root/'infrastructure-stop.json',dict(reason='MISSING_GUARD_RECEIPT',index=item['index'],returncode=code,accounting_complete=False));raise RuntimeError('Missing R2 guard accounting')
row=json.loads(path.read_text());assert math.isfinite(row['worker']['cpu_seconds']) and row['worker']['cpu_seconds']>=0
done.append(dict(index=item['index'],**row));item['log'].close();del active[slot]
print(json.dumps(dict(event='R2_TERMINAL',index=item['index'],kind=c['jobs'][item['index']]['kind'],status=row['worker']['status'],completed=len(done))),flush=True)
def usage():
values=[tree_usage(v['process'].pid) for v in active.values()]
return sum(x[0] for x in values),sum(x[1] for x in values)
(live_cpu,live_rss),fresh=monitor.sample('live_usage',usage,(live_cpu,live_rss))
disk,ok=monitor.sample('disk',lambda:disk_usage(root),disk);fresh &= ok
spent=sum(x['worker']['cpu_seconds'] for x in done)
over=budget_reason(0,spent,live_cpu,disk,cpu_limit=c['limits']['worker_cpu_seconds'],disk_limit=c['limits']['disk_bytes'])
if over:reason=over
if not first_checked and len(done)==1 and not active and reason is None:
check=cost_check(c,done[0]);save(root/'first-fit-cost-check.json',check);first_checked=True
if not check['continue_within_cap']:reason='FIRST_FIT_COST_PROJECTION_OVER_CAP'
if next_index==180 and not active and len(done)==180 and selections is None and reason is None:
selections=finalize_selections(root,c,done)
if reason:terminate_active()
ceiling=1 if not first_checked else (180 if selections is None else 195)
while fresh and reason is None and len(active)<4 and next_index<ceiling:
i=next_index;job=c['jobs'][i]
if job['kind']=='FINAL' and selections[job['cell']]['status']!='SELECTED':
done.append(dict(index=i,job=job,worker=dict(status='NA',cpu_seconds=0.,returncode=0,stop_reason='ALL12_HPO_FAILED_NO_FINAL_FIT'),
supervisor_cpu_seconds=0.,attempted=False,result_path=None,result_sha256=None));next_index+=1;continue
slot=next(k for k in range(4) if k not in active);dest=root/'jobs'/f'job-{i:03d}';dest.mkdir(parents=True,exist_ok=True)
log=(dest/'guard.log').open('x')
proc=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(i),'--slot',str(slot)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
active[slot]=dict(process=proc,log=log,index=i);next_index+=1
save(root/'progress.json',dict(completed=len(done),active={str(k):v['index'] for k,v in active.items()},next_index=next_index,
worker_cpu_seconds=spent,live_guard_and_worker_cpu_seconds=live_cpu,active_tree_rss_bytes=live_rss,disk_bytes=disk,
first_fit_checked=first_checked,selections_frozen=selections is not None,stop_reason=reason,fresh_accounting=fresh,
monitor_errors=monitor.errors,elapsed_seconds=time.monotonic()-start))
if active or not fresh:time.sleep(.5)
finally:
terminate_active()
for item in active.values():item['process'].wait();item['log'].close()
attempted={x['index']:x for x in done};finals=[]
for job in c['jobs'][180:]:
row=attempted.get(job['index']);passed=row is not None and row['worker']['status']=='PASS'
result=json.loads(Path(row['result_path']).read_text()) if passed else None
finals.append(dict(cell=job['cell'],fold=job['fold'],pool=job['pool'],status='PASS' if passed else 'NA',
reason=None if passed else (row['worker']['stop_reason'] if row else 'NOT_RUN_'+str(reason)),
finished_configurations=selections[job['cell']]['finished_configurations'] if selections else None,
model_path=str(Path(row['result_path']).parent/'model.joblib') if passed else None,
model_sha256=result['model_sha256'] if result else None))
save(root/'batch-complete.json',dict(status='ALL_R2_TERMINAL',stop_reason=reason,final_fits=finals,jobs=done,
not_run_indices=[i for i in range(195) if i not in attempted],worker_cpu_seconds=sum(x['worker']['cpu_seconds'] for x in done),
guard_cpu_seconds=sum(x['supervisor_cpu_seconds'] for x in done),coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
disk_bytes=disk_usage(root),wall_seconds=time.monotonic()-start,monitor_errors=monitor.errors,roster_sha256=c['roster_sha256'],
test_predictions=False,test_scoring=False,retry=False))
return 0 if reason is None else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);p.add_argument('--slot',type=int);a=p.parse_args()
raise SystemExit(guard(a.root,a.guard,a.slot) if a.guard is not None else run(a.root))