"""One detached R4bc fit queue. Pool barriers, reserved budgets and immutable attempts.""" import argparse import fcntl import math import os from pathlib import Path import signal import subprocess import sys import time import psutil from r4bc_common import (ARMS,HIST,POOLS,contract,read,sha,checked,save,selection,selection_path, job_dir,can_reserve,role_rows,histgbrt_grid) from s2_recovery_supervisor import tree_usage from s2_recovery_monitoring import disk_usage def tracks(c): return {cell:[j['index'] for j in c['jobs'] if j['cell']==cell] for cell in range(15)} def eligible_cells(c, cursors, *, checkpoint_passed, active_cells=()): by=tracks(c) if not checkpoint_passed:return [c['first_cell']] if cursors[c['first_cell']]c['limits']['setup_cpu_seconds']:raise ValueError('Setup budget already exceeded') accounting_complete=True;monitor_errors=[];waiting=None env=dict(os.environ,OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='') try: while True: for slot,item in list(active.items()): code=item['process'].poll() if code is None:continue p=job_dir(root,item['index'])/'guard-result.json' if not p.exists(): accounting_complete=False;reason='MISSING_GUARD_RECEIPT' save(root/'missing-guard.json',dict(index=item['index'],returncode=code,accounting_complete=False)) else: r=read(p) if not math.isfinite(r['worker']['cpu_seconds']) or r['worker']['cpu_seconds']<0: accounting_complete=False;reason='INVALID_ACCOUNTING' records.append(dict(**r,attempted=True)) if r['global_failure']:reason='WORKER_GLOBAL_FAILURE' item['log'].close();del active[slot] print('TERMINAL',item['index'],code,len(records),flush=True) spent=sum(r['worker']['cpu_seconds'] for r in records) if signal_request[0]:reason=reason or signal_request[0] if (root/'global-stop.json').exists():reason=reason or read(root/'global-stop.json')['reason'] try: usage=[tree_usage(v['process'].pid) for v in active.values()] live_cpu=sum(u[0] for u in usage);rss=sum(u[1] for u in usage)+psutil.Process().memory_info().rss disk=disk_usage(root);available=psutil.virtual_memory().available if rss>c['limits']['total_rss_bytes']:reason='GLOBAL_RSS_LIMIT' if disk>=c['limits']['disk_bytes']:reason='GLOBAL_DISK_LIMIT' # Live process-tree CPU includes guards: conservative, never an undercount. if spent+live_cpu>c['limits']['fit_cpu_seconds']:reason='GLOBAL_FIT_CPU_LIMIT' except Exception as exc: monitor_errors.append(dict(type=type(exc).__name__,message=str(exc)));reason='GLOBAL_MONITOR_ERROR' live_cpu=rss=disk=available=0 if reason: if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time())) # Signal the guard, not the worker, to preserve final CPU/peak accounting. for item in active.values(): if not item.get('stop_sent'): try:item['process'].send_signal(signal.SIGTERM) except ProcessLookupError:pass item['stop_sent']=True if not checkpoint and not active and cursors[c['first_cell']]==len(by[c['first_cell']]) and not reason: check=cost_check(c,records);save(root/'first-cell-cost-check.json',check) checkpoint=check['continue_within_cap'] if not checkpoint:cut='FIRST_CELL_COST_OR_FAILURE_STOP' if not reason and not cut: cells=eligible_cells(c,cursors,checkpoint_passed=checkpoint,active_cells={v['cell'] for v in active.values()}) busy={v['cell'] for v in active.values()};waiting=None for cell in cells: if cell in busy or len(active)>=c['limits']['workers']:continue if not checkpoint and active:break i=by[cell][cursors[cell]];job=c['jobs'][i] if job['kind']=='FINAL' and job['arm'] in HIST: candidates=[r for r in records if r['job']['cell']==cell and r['job']['arm']==job['arm'] and r['job']['kind']=='HPO'] trials=[] for r in candidates: value=read(checked(r['result_path'],r['result_sha256'])).get('validation_score') if r['worker']['status']=='PASS' else None path=job_dir(root,r['index'])/'guard-result.json' trials.append(dict(config_index=r['job']['config_index'],status=r['worker']['status'],score=value, receipt_path=str(path),receipt_sha256=sha(path))) decision=selection(trials);p=selection_path(root,job);p.parent.mkdir(exist_ok=True) if not p.exists():save(p,decision) elif read(p)!=decision:raise ValueError('Selection changed') if decision['status']!='SELECTED': records.append(dict(index=i,job=job,attempted=False,result_path=None,result_sha256=None, guard_cpu_seconds=0,worker=dict(status='NA',cpu_seconds=0.,stop_reason='ALL_HPO_FAILED',monitor_errors=[]))) cursors[cell]+=1;continue if not can_reserve(spent,len(active),c['limits']['fit_cpu_seconds']): if not active:cut='NOT_RUN_BUDGET' else:waiting='WAIT_ACTIVE_SETTLEMENT_FOR_BUDGET' break if available<10*2**30: waiting='WAIT_RESOURCES_V4_PRIORITY';break contract(root) slot=next(k for k in range(4) if k not in active);dest=job_dir(root,i);dest.mkdir(parents=True,exist_ok=True) if (dest/'worker.started.json').exists():raise ValueError('Attempt already started') log=(dest/'guard.log').open('x') process=subprocess.Popen([sys.executable,'tools/r4bc_guard.py',str(root),str(i),str(slot)], env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True) active[slot]=dict(process=process,log=log,index=i,cell=cell);cursors[cell]+=1 available-=8*2**30 save(root/'progress.json',dict(terminal_attempts=sum(r.get('attempted',True) for r in records),logical_terminal=len(records), active={str(k):dict(job=v['index'],cell=v['cell'],guard_pid=v['process'].pid) for k,v in active.items()}, fit_cpu_seconds=spent,setup_cpu_seconds=initial_setup,live_guard_and_worker_cpu_seconds=live_cpu, reserved_active_cpu_seconds=len(active)*1800,total_rss_bytes=rss,disk_bytes=disk, checkpoint_passed=checkpoint,cut=cut,global_stop=reason,waiting=waiting, monitor_errors=monitor_errors,wall_seconds=time.monotonic()-start)) if not active and (reason or cut or (checkpoint and not eligible_cells(c,cursors,checkpoint_passed=True))):break time.sleep(.5) except Exception as exc: reason='COORDINATOR_EXCEPTION';save(root/'coordinator-failure.json',dict(type=type(exc).__name__,message=str(exc))) save(root/'global-stop.json',dict(reason=reason)) finally: for item in active.values(): try:item['process'].send_signal(signal.SIGTERM) except ProcessLookupError:pass for item in active.values(): item['process'].wait();item['log'].close();p=job_dir(root,item['index'])/'guard-result.json' if p.exists():records.append(dict(**read(p),attempted=True)) else:accounting_complete=False seen={r['index'] for r in records};models=freeze_checkpoints(root,c,records,cut,reason) save(root/'batch-complete.json',dict(status='FIT_QUEUE_TERMINAL' if not reason else 'GLOBAL_STOP', global_stop=reason,cut=cut,accounting_complete=accounting_complete,records=records, never_attempted=[dict(index=j['index'],cell=j['cell'],arm=j['arm'],pool=j['pool'], reason='not run: budget' if cut=='NOT_RUN_BUDGET' else (reason or cut)) for j in c['jobs'] if j['index'] not in seen], fit_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in records),setup_cpu_seconds=initial_setup, guard_cpu_seconds=sum(r.get('guard_cpu_seconds',0) for r in records),coordinator_cpu_seconds=time.process_time(), model_manifest_sha256=sha(root/'models-terminal.json'),monitor_errors=monitor_errors, postfit_state='READY_FOR_GIT_CHECKPOINT_FREEZE' if not reason and accounting_complete else 'FORBIDDEN_GLOBAL_STOP', new_decodes=0,test_predictions=False,test_scoring=False,retries=0)) return 0 if not reason and accounting_complete else 1 if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();raise SystemExit(run(a.root))