167 lines
11 KiB
Python
167 lines
11 KiB
Python
"""Independent four-worker CPU HPO/final queue; first-fit barrier, no neural mutations."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from freeze_g1_components import sha
|
|
from s2_recovery_supervisor import save,supervise,tree_usage
|
|
from s2_recovery_monitoring import Monitor,disk_usage,budget_reason
|
|
from run_s2_pilot_preparation import terminate
|
|
|
|
|
|
def contract(root):
|
|
c=json.loads((root/'contract.json').read_text())
|
|
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():assert sha(p)==h,p
|
|
assert len(c['jobs'])==195 and len(c['cells'])==15 and c['limits']['worker_cpu_seconds']==360000
|
|
assert c['limits']['per_fit_cpu_seconds']==1800 and c['limits']['workers']==4
|
|
assert set(c['worker_cpus']).isdisjoint(range(8))
|
|
assert c['final_refit']=='same_learning_pool_only' and c['selection_policy']=='best_finished_config_fixed_grid_tie_order'
|
|
return c
|
|
|
|
|
|
def select_finished(cell,trials):
|
|
assert len(trials)==12 and sorted(t['config_index'] for t in trials)==list(range(12))
|
|
finished=[t for t in trials if t['status']=='PASS' and t.get('score') is not None and math.isfinite(t['score'])]
|
|
if not finished:return dict(status='NA_ALL_HPO_FAILED',cell=cell,finished_configurations=0,trials=trials,config_index=None)
|
|
chosen=min(finished,key=lambda t:(t['score'],t['config_index']))
|
|
return dict(status='SELECTED',cell=cell,config_index=chosen['config_index'],validation_score=chosen['score'],
|
|
finished_configurations=len(finished),failed_configurations=12-len(finished),trials=trials,
|
|
final_refit_set='same_learning_pool_only',ruling='1794707190.141609')
|
|
|
|
|
|
def cost_check(c,first):
|
|
cap=c['limits']['per_fit_cpu_seconds'];hard=len(c['jobs'])*cap
|
|
if first['worker']['status']!='PASS':
|
|
return dict(continue_within_cap=hard<=c['limits']['worker_cpu_seconds'],method='No successful empirical fit; capped195attempts upper-bound only, not a completion forecast',
|
|
projected_worker_cpu_seconds=None,capped_attempt_upper_bound_seconds=hard,first_status=first['worker']['status'])
|
|
path=Path(first['result_path']);r=json.loads(path.read_text());initial=c['jobs'][0]['cost_units']
|
|
remaining=sum(j['cost_units'] for j in c['jobs'][1:]);rate=r['fit_cpu_seconds']/initial
|
|
# Non-fit work scales by query rows, final config conservatively at63leaves/1000iterations.
|
|
overhead=max(0.,first['worker']['cpu_seconds']-r['fit_cpu_seconds'])
|
|
initial_rows=c['jobs'][0]['cost_queries']
|
|
forecast=first['worker']['cpu_seconds']+1.25*(rate*remaining+overhead*sum(j['cost_queries']/initial_rows for j in c['jobs'][1:]))
|
|
return dict(continue_within_cap=forecast<=c['limits']['worker_cpu_seconds'],projected_worker_cpu_seconds=forecast,
|
|
capped_attempt_upper_bound_seconds=hard,first_fit_cpu_seconds=r['fit_cpu_seconds'],first_worker_cpu_seconds=first['worker']['cpu_seconds'],
|
|
method='Observed fit CPU/(learning eligible queries*iterations*leaves), query-scaled nonfit CPU, final worst grid config,25percent reserve; heuristic not accuracy guarantee')
|
|
|
|
|
|
def guard(root,index,slot):
|
|
c=contract(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]])
|
|
dest=root/'jobs'/f'job-{index:03d}';dest.mkdir(parents=True,exist_ok=True)
|
|
w=supervise([sys.executable,'tools/run_s2_r2_fit.py',str(root),str(index)],dest,'worker',
|
|
dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=7200,tree_rss_bytes=16*2**30,
|
|
disk_bytes=5*2**30,max_processes=1),poll=.2)
|
|
path=dest/'output/result.json';result=None
|
|
if path.exists():result=json.loads(path.read_text())
|
|
if w['status']=='PASS' and (not result or result['status']!='COMPLETE'):
|
|
w.update(status='STOPPED_OR_FAILED',stop_reason='MISSING_COMPLETE_RESULT')
|
|
save(dest/'guard-result.json',dict(worker=w,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
|
cpu_affinity=sorted(os.sched_getaffinity(0)),result_path=str(path) if path.exists() else None,
|
|
result_sha256=sha(path) if path.exists() else None,job=c['jobs'][index]))
|
|
return 0 if w['status']=='PASS' else 1
|
|
|
|
|
|
def finalize_selections(root,c,done):
|
|
by_index={r['index']:r for r in done};directory=root/'selections';directory.mkdir(exist_ok=False)
|
|
selected={}
|
|
for cell in c['cells']:
|
|
trials=[]
|
|
for i in cell['hpo_indices']:
|
|
row=by_index[i];path=root/'jobs'/f'job-{i:03d}'/'guard-result.json'
|
|
result=json.loads(Path(row['result_path']).read_text()) if row.get('result_path') else None
|
|
trials.append(dict(index=i,config_index=c['jobs'][i]['config_index'],status=row['worker']['status'],
|
|
reason=row['worker']['stop_reason'] or (None if row['worker']['status']=='PASS' else 'WORKER_EXIT_'+str(row['worker']['returncode'])),
|
|
score=result.get('validation_component_equal_relative_l2') if result else None,
|
|
receipt_path=str(path),receipt_sha256=sha(path)))
|
|
decision=select_finished(cell['index'],trials);save(directory/f"cell-{cell['index']:02d}.json",decision);selected[cell['index']]=decision
|
|
return selected
|
|
|
|
|
|
def run(root):
|
|
with (root/'batch.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root)
|
|
|
|
|
|
def run_locked(root):
|
|
c=contract(root);assert not (root/'batch-started.json').exists(),'No repeat R2 queue'
|
|
os.sched_setaffinity(0,c['coordinator_cpus'])
|
|
start=time.monotonic();save(root/'batch-started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),cpu_affinity=sorted(os.sched_getaffinity(0))))
|
|
active={};done=[];next_index=0;reason=None;first_checked=False;selections=None;live_cpu=live_rss=disk=0
|
|
monitor=Monitor(root/'monitor-errors.jsonl')
|
|
env=dict(os.environ,OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
|
def interrupted(*_):
|
|
nonlocal reason
|
|
reason='R2_COORDINATOR_INTERRUPTED'
|
|
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
|
|
def terminate_active():terminate({k:(v['process'],v['log']) for k,v in active.items()})
|
|
try:
|
|
while active or (next_index<195 and reason is None):
|
|
for slot,item in list(active.items()):
|
|
code=item['process'].poll()
|
|
if code is None:continue
|
|
path=root/'jobs'/f"job-{item['index']:03d}"/'guard-result.json'
|
|
if not path.exists():
|
|
save(root/'infrastructure-stop.json',dict(reason='MISSING_GUARD_RECEIPT',index=item['index'],returncode=code,accounting_complete=False));raise RuntimeError('Missing R2 guard accounting')
|
|
row=json.loads(path.read_text());assert math.isfinite(row['worker']['cpu_seconds']) and row['worker']['cpu_seconds']>=0
|
|
done.append(dict(index=item['index'],**row));item['log'].close();del active[slot]
|
|
print(json.dumps(dict(event='R2_TERMINAL',index=item['index'],kind=c['jobs'][item['index']]['kind'],status=row['worker']['status'],completed=len(done))),flush=True)
|
|
def usage():
|
|
values=[tree_usage(v['process'].pid) for v in active.values()]
|
|
return sum(x[0] for x in values),sum(x[1] for x in values)
|
|
(live_cpu,live_rss),fresh=monitor.sample('live_usage',usage,(live_cpu,live_rss))
|
|
disk,ok=monitor.sample('disk',lambda:disk_usage(root),disk);fresh &= ok
|
|
spent=sum(x['worker']['cpu_seconds'] for x in done)
|
|
over=budget_reason(0,spent,live_cpu,disk,cpu_limit=c['limits']['worker_cpu_seconds'],disk_limit=c['limits']['disk_bytes'])
|
|
if over:reason=over
|
|
if not first_checked and len(done)==1 and not active and reason is None:
|
|
check=cost_check(c,done[0]);save(root/'first-fit-cost-check.json',check);first_checked=True
|
|
if not check['continue_within_cap']:reason='FIRST_FIT_COST_PROJECTION_OVER_CAP'
|
|
if next_index==180 and not active and len(done)==180 and selections is None and reason is None:
|
|
selections=finalize_selections(root,c,done)
|
|
if reason:terminate_active()
|
|
ceiling=1 if not first_checked else (180 if selections is None else 195)
|
|
while fresh and reason is None and len(active)<4 and next_index<ceiling:
|
|
i=next_index;job=c['jobs'][i]
|
|
if job['kind']=='FINAL' and selections[job['cell']]['status']!='SELECTED':
|
|
done.append(dict(index=i,job=job,worker=dict(status='NA',cpu_seconds=0.,returncode=0,stop_reason='ALL12_HPO_FAILED_NO_FINAL_FIT'),
|
|
supervisor_cpu_seconds=0.,attempted=False,result_path=None,result_sha256=None));next_index+=1;continue
|
|
slot=next(k for k in range(4) if k not in active);dest=root/'jobs'/f'job-{i:03d}';dest.mkdir(parents=True,exist_ok=True)
|
|
log=(dest/'guard.log').open('x')
|
|
proc=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(i),'--slot',str(slot)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
|
active[slot]=dict(process=proc,log=log,index=i);next_index+=1
|
|
save(root/'progress.json',dict(completed=len(done),active={str(k):v['index'] for k,v in active.items()},next_index=next_index,
|
|
worker_cpu_seconds=spent,live_guard_and_worker_cpu_seconds=live_cpu,active_tree_rss_bytes=live_rss,disk_bytes=disk,
|
|
first_fit_checked=first_checked,selections_frozen=selections is not None,stop_reason=reason,fresh_accounting=fresh,
|
|
monitor_errors=monitor.errors,elapsed_seconds=time.monotonic()-start))
|
|
if active or not fresh:time.sleep(.5)
|
|
finally:
|
|
terminate_active()
|
|
for item in active.values():item['process'].wait();item['log'].close()
|
|
attempted={x['index']:x for x in done};finals=[]
|
|
for job in c['jobs'][180:]:
|
|
row=attempted.get(job['index']);passed=row is not None and row['worker']['status']=='PASS'
|
|
result=json.loads(Path(row['result_path']).read_text()) if passed else None
|
|
finals.append(dict(cell=job['cell'],fold=job['fold'],pool=job['pool'],status='PASS' if passed else 'NA',
|
|
reason=None if passed else (row['worker']['stop_reason'] if row else 'NOT_RUN_'+str(reason)),
|
|
finished_configurations=selections[job['cell']]['finished_configurations'] if selections else None,
|
|
model_path=str(Path(row['result_path']).parent/'model.joblib') if passed else None,
|
|
model_sha256=result['model_sha256'] if result else None))
|
|
save(root/'batch-complete.json',dict(status='ALL_R2_TERMINAL',stop_reason=reason,final_fits=finals,jobs=done,
|
|
not_run_indices=[i for i in range(195) if i not in attempted],worker_cpu_seconds=sum(x['worker']['cpu_seconds'] for x in done),
|
|
guard_cpu_seconds=sum(x['supervisor_cpu_seconds'] for x in done),coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
|
disk_bytes=disk_usage(root),wall_seconds=time.monotonic()-start,monitor_errors=monitor.errors,roster_sha256=c['roster_sha256'],
|
|
test_predictions=False,test_scoring=False,retry=False))
|
|
return 0 if reason is None else 1
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);p.add_argument('--slot',type=int);a=p.parse_args()
|
|
raise SystemExit(guard(a.root,a.guard,a.slot) if a.guard is not None else run(a.root))
|