"""Independent four-worker CPU HPO/final queue; first-fit barrier, no neural mutations.""" import argparse import fcntl import json import math import os from pathlib import Path import resource import signal import subprocess import sys import time from freeze_g1_components import sha from s2_recovery_supervisor import save,supervise,tree_usage from s2_recovery_monitoring import Monitor,disk_usage,budget_reason from run_s2_pilot_preparation import terminate def contract(root): c=json.loads((root/'contract.json').read_text()) for p,h in {**c['code_sha256'],**c['source_sha256']}.items():assert sha(p)==h,p assert len(c['jobs'])==195 and len(c['cells'])==15 and c['limits']['worker_cpu_seconds']==360000 assert c['limits']['per_fit_cpu_seconds']==1800 and c['limits']['workers']==4 assert set(c['worker_cpus']).isdisjoint(range(8)) assert c['final_refit']=='same_learning_pool_only' and c['selection_policy']=='best_finished_config_fixed_grid_tie_order' return c def select_finished(cell,trials): assert len(trials)==12 and sorted(t['config_index'] for t in trials)==list(range(12)) finished=[t for t in trials if t['status']=='PASS' and t.get('score') is not None and math.isfinite(t['score'])] if not finished:return dict(status='NA_ALL_HPO_FAILED',cell=cell,finished_configurations=0,trials=trials,config_index=None) chosen=min(finished,key=lambda t:(t['score'],t['config_index'])) return dict(status='SELECTED',cell=cell,config_index=chosen['config_index'],validation_score=chosen['score'], finished_configurations=len(finished),failed_configurations=12-len(finished),trials=trials, final_refit_set='same_learning_pool_only',ruling='1794707190.141609') def cost_check(c,first): cap=c['limits']['per_fit_cpu_seconds'];hard=len(c['jobs'])*cap if first['worker']['status']!='PASS': return dict(continue_within_cap=hard<=c['limits']['worker_cpu_seconds'],method='No successful empirical fit; capped195attempts upper-bound only, not a completion forecast', projected_worker_cpu_seconds=None,capped_attempt_upper_bound_seconds=hard,first_status=first['worker']['status']) path=Path(first['result_path']);r=json.loads(path.read_text());initial=c['jobs'][0]['cost_units'] remaining=sum(j['cost_units'] for j in c['jobs'][1:]);rate=r['fit_cpu_seconds']/initial # Non-fit work scales by query rows, final config conservatively at63leaves/1000iterations. overhead=max(0.,first['worker']['cpu_seconds']-r['fit_cpu_seconds']) initial_rows=c['jobs'][0]['cost_queries'] forecast=first['worker']['cpu_seconds']+1.25*(rate*remaining+overhead*sum(j['cost_queries']/initial_rows for j in c['jobs'][1:])) return dict(continue_within_cap=forecast<=c['limits']['worker_cpu_seconds'],projected_worker_cpu_seconds=forecast, capped_attempt_upper_bound_seconds=hard,first_fit_cpu_seconds=r['fit_cpu_seconds'],first_worker_cpu_seconds=first['worker']['cpu_seconds'], method='Observed fit CPU/(learning eligible queries*iterations*leaves), query-scaled nonfit CPU, final worst grid config,25percent reserve; heuristic not accuracy guarantee') def guard(root,index,slot): c=contract(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]]) dest=root/'jobs'/f'job-{index:03d}';dest.mkdir(parents=True,exist_ok=True) w=supervise([sys.executable,'tools/run_s2_r2_fit.py',str(root),str(index)],dest,'worker', dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=7200,tree_rss_bytes=16*2**30, disk_bytes=5*2**30,max_processes=1),poll=.2) path=dest/'output/result.json';result=None if path.exists():result=json.loads(path.read_text()) if w['status']=='PASS' and (not result or result['status']!='COMPLETE'): w.update(status='STOPPED_OR_FAILED',stop_reason='MISSING_COMPLETE_RESULT') save(dest/'guard-result.json',dict(worker=w,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]), cpu_affinity=sorted(os.sched_getaffinity(0)),result_path=str(path) if path.exists() else None, result_sha256=sha(path) if path.exists() else None,job=c['jobs'][index])) return 0 if w['status']=='PASS' else 1 def finalize_selections(root,c,done): by_index={r['index']:r for r in done};directory=root/'selections';directory.mkdir(exist_ok=False) selected={} for cell in c['cells']: trials=[] for i in cell['hpo_indices']: row=by_index[i];path=root/'jobs'/f'job-{i:03d}'/'guard-result.json' result=json.loads(Path(row['result_path']).read_text()) if row.get('result_path') else None trials.append(dict(index=i,config_index=c['jobs'][i]['config_index'],status=row['worker']['status'], reason=row['worker']['stop_reason'] or (None if row['worker']['status']=='PASS' else 'WORKER_EXIT_'+str(row['worker']['returncode'])), score=result.get('validation_component_equal_relative_l2') if result else None, receipt_path=str(path),receipt_sha256=sha(path))) decision=select_finished(cell['index'],trials);save(directory/f"cell-{cell['index']:02d}.json",decision);selected[cell['index']]=decision return selected def run(root): with (root/'batch.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root) def run_locked(root): c=contract(root);assert not (root/'batch-started.json').exists(),'No repeat R2 queue' os.sched_setaffinity(0,c['coordinator_cpus']) start=time.monotonic();save(root/'batch-started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),cpu_affinity=sorted(os.sched_getaffinity(0)))) active={};done=[];next_index=0;reason=None;first_checked=False;selections=None;live_cpu=live_rss=disk=0 monitor=Monitor(root/'monitor-errors.jsonl') env=dict(os.environ,OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='') def interrupted(*_): nonlocal reason reason='R2_COORDINATOR_INTERRUPTED' signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted) def terminate_active():terminate({k:(v['process'],v['log']) for k,v in active.items()}) try: while active or (next_index<195 and reason is None): for slot,item in list(active.items()): code=item['process'].poll() if code is None:continue path=root/'jobs'/f"job-{item['index']:03d}"/'guard-result.json' if not path.exists(): save(root/'infrastructure-stop.json',dict(reason='MISSING_GUARD_RECEIPT',index=item['index'],returncode=code,accounting_complete=False));raise RuntimeError('Missing R2 guard accounting') row=json.loads(path.read_text());assert math.isfinite(row['worker']['cpu_seconds']) and row['worker']['cpu_seconds']>=0 done.append(dict(index=item['index'],**row));item['log'].close();del active[slot] print(json.dumps(dict(event='R2_TERMINAL',index=item['index'],kind=c['jobs'][item['index']]['kind'],status=row['worker']['status'],completed=len(done))),flush=True) def usage(): values=[tree_usage(v['process'].pid) for v in active.values()] return sum(x[0] for x in values),sum(x[1] for x in values) (live_cpu,live_rss),fresh=monitor.sample('live_usage',usage,(live_cpu,live_rss)) disk,ok=monitor.sample('disk',lambda:disk_usage(root),disk);fresh &= ok spent=sum(x['worker']['cpu_seconds'] for x in done) over=budget_reason(0,spent,live_cpu,disk,cpu_limit=c['limits']['worker_cpu_seconds'],disk_limit=c['limits']['disk_bytes']) if over:reason=over if not first_checked and len(done)==1 and not active and reason is None: check=cost_check(c,done[0]);save(root/'first-fit-cost-check.json',check);first_checked=True if not check['continue_within_cap']:reason='FIRST_FIT_COST_PROJECTION_OVER_CAP' if next_index==180 and not active and len(done)==180 and selections is None and reason is None: selections=finalize_selections(root,c,done) if reason:terminate_active() ceiling=1 if not first_checked else (180 if selections is None else 195) while fresh and reason is None and len(active)<4 and next_index