228 lines
14 KiB
Python
228 lines
14 KiB
Python
"""One detached R4bc fit queue. Pool barriers, reserved budgets and immutable attempts."""
|
|
import argparse
|
|
import fcntl
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import psutil
|
|
|
|
from r4bc_common import (ARMS,HIST,POOLS,contract,read,sha,checked,save,selection,selection_path,
|
|
job_dir,can_reserve,role_rows,histgbrt_grid)
|
|
from s2_recovery_supervisor import tree_usage
|
|
from s2_recovery_monitoring import disk_usage
|
|
|
|
|
|
def tracks(c):
|
|
return {cell:[j['index'] for j in c['jobs'] if j['cell']==cell] for cell in range(15)}
|
|
|
|
|
|
def eligible_cells(c, cursors, *, checkpoint_passed, active_cells=()):
|
|
by=tracks(c)
|
|
if not checkpoint_passed:return [c['first_cell']] if cursors[c['first_cell']]<len(by[c['first_cell']]) else []
|
|
for pool in POOLS:
|
|
cells=[f*3+(25,50,100).index(pool) for f in range(5)]
|
|
pending=[cell for cell in cells if cursors[cell]<len(by[cell]) or cell in active_cells]
|
|
if pending:return pending
|
|
return []
|
|
|
|
|
|
def cost_check(c, records):
|
|
first=c['first_cell']; done={r['index']:r for r in records}
|
|
initial=[j for j in c['jobs'] if j['cell']==first]
|
|
finals=[done.get(j['index']) for j in initial if j['kind']=='FINAL']
|
|
if any(r is None or r['worker']['status']!='PASS' for r in finals):
|
|
return dict(continue_within_cap=False,reason='FIRST_CELL_FINAL_FAILED_OR_INCOMPLETE')
|
|
reference={}
|
|
for j in initial:
|
|
r=done.get(j['index'])
|
|
if r is None or not r.get('attempted',True):return dict(continue_within_cap=False,reason='FIRST_CELL_INCOMPLETE')
|
|
reference[j['arm'],j['kind'],j['config_index']]=r['worker']['cpu_seconds']
|
|
sizes={}
|
|
for job in c['jobs']:
|
|
key=job['cell'],job['arm']
|
|
if key in sizes:continue
|
|
rows=role_rows(c,job['fold'],job['pool'],'learning')
|
|
sizes[key]=sum(r['nodes'] if job['arm'] in ('H4','A') else r['preparation']['eligible_unique'] for r in rows)
|
|
remaining_100=remaining_all=0.
|
|
for j in c['jobs']:
|
|
if j['cell']==first:continue
|
|
sample=reference[j['arm'],j['kind'],j['config_index']]
|
|
scale=sizes[j['cell'],j['arm']]/sizes[first,j['arm']]
|
|
# The final selected configuration is not known. Reserve the max grid
|
|
# iteration ratio to the observed first-cell selection for final fits.
|
|
if j['arm'] in HIST and j['kind']=='FINAL':
|
|
fj=next(x for x in initial if x['arm']==j['arm'] and x['kind']=='FINAL')
|
|
result=read(done[fj['index']]['result_path'])
|
|
scale*=1000/histgbrt_grid()[result['config_index']]['max_iter']
|
|
projected=sample*scale
|
|
remaining_all+=projected
|
|
if j['pool']==100:remaining_100+=projected
|
|
spent=sum(r['worker']['cpu_seconds'] for r in records)
|
|
return dict(continue_within_cap=remaining_100<=c['limits']['fit_cpu_seconds']-spent,
|
|
first_cell_cpu_seconds=spent,remaining_100_projection=remaining_100,
|
|
all_three_pools_projection=spent+remaining_all,full_100_projection=spent+remaining_100,
|
|
method='Per arm/config measured guard CPU scaled by learning rows; final worst iteration ratio. Heuristic, hard reservation remains authoritative.',
|
|
all_three_over_cap_does_not_stop_100=True)
|
|
|
|
|
|
def freeze_checkpoints(root,c,records,cut,global_reason):
|
|
by={r['index']:r for r in records};models=[]
|
|
for j in c['jobs']:
|
|
if j['kind']!='FINAL':continue
|
|
record=by.get(j['index']);result=None
|
|
if record and record['worker']['status']=='PASS':
|
|
result=read(checked(record['result_path'],record['result_sha256']))
|
|
checked(result['model_path'],result['model_sha256'])
|
|
models.append(dict(cell=j['cell'],fold=j['fold'],pool=j['pool'],arm=j['arm'],
|
|
status='PASS' if result else 'NA',model_path=result['model_path'] if result else None,
|
|
model_sha256=result['model_sha256'] if result else None,
|
|
reason=None if result else (record['worker'].get('stop_reason') if record else (global_reason or cut or 'NOT_ATTEMPTED'))))
|
|
save(root/'models-terminal.json',dict(contract_sha256=sha(root/'contract.json'),models=models,
|
|
fit_records={str(r['index']):sha(job_dir(root,r['index'])/'guard-result.json')
|
|
for r in records if r.get('attempted',True)}))
|
|
return models
|
|
|
|
|
|
def run(root):
|
|
root=Path(root)
|
|
with (root/'batch.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
return locked(root)
|
|
|
|
|
|
def locked(root):
|
|
c=contract(root); launch=read(root/'launch-freeze.json')
|
|
checked(root/'contract.json',launch['contract_sha256']);checked(root/'input-freeze.json',launch['input_freeze_sha256'])
|
|
checked(root/'job-list.json',c['job_list_sha256'])
|
|
if read(root/'preflight-guard/worker.receipt.json')['status']!='PASS':raise ValueError('Preflight did not pass')
|
|
if (root/'batch-started.json').exists():raise FileExistsError('No queue restart')
|
|
os.sched_setaffinity(0,[c['coordinator_cpu']])
|
|
start=time.monotonic();save(root/'batch-started.json',dict(pid=os.getpid(),unix=time.time(),
|
|
contract_sha256=sha(root/'contract.json'),implementation_commit=launch['implementation_commit'],
|
|
input_freeze_sha256=launch['input_freeze_sha256'],cpu_affinity=sorted(os.sched_getaffinity(0))))
|
|
by=tracks(c);cursors={cell:0 for cell in by};active={};records=[];cut=None;reason=None;checkpoint=False
|
|
signal_request=[None]
|
|
def interrupted(sig,frame):signal_request[0]='COORDINATOR_SIGNAL_'+str(sig)
|
|
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
|
|
initial_setup=c['initial_setup_cpu_seconds']+read(root/'preflight-guard/worker.receipt.json')['cpu_seconds']
|
|
if initial_setup>c['limits']['setup_cpu_seconds']:raise ValueError('Setup budget already exceeded')
|
|
accounting_complete=True;monitor_errors=[];waiting=None
|
|
env=dict(os.environ,OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
|
try:
|
|
while True:
|
|
for slot,item in list(active.items()):
|
|
code=item['process'].poll()
|
|
if code is None:continue
|
|
p=job_dir(root,item['index'])/'guard-result.json'
|
|
if not p.exists():
|
|
accounting_complete=False;reason='MISSING_GUARD_RECEIPT'
|
|
save(root/'missing-guard.json',dict(index=item['index'],returncode=code,accounting_complete=False))
|
|
else:
|
|
r=read(p)
|
|
if not math.isfinite(r['worker']['cpu_seconds']) or r['worker']['cpu_seconds']<0:
|
|
accounting_complete=False;reason='INVALID_ACCOUNTING'
|
|
records.append(dict(**r,attempted=True))
|
|
if r['global_failure']:reason='WORKER_GLOBAL_FAILURE'
|
|
item['log'].close();del active[slot]
|
|
print('TERMINAL',item['index'],code,len(records),flush=True)
|
|
spent=sum(r['worker']['cpu_seconds'] for r in records)
|
|
if signal_request[0]:reason=reason or signal_request[0]
|
|
if (root/'global-stop.json').exists():reason=reason or read(root/'global-stop.json')['reason']
|
|
try:
|
|
usage=[tree_usage(v['process'].pid) for v in active.values()]
|
|
live_cpu=sum(u[0] for u in usage);rss=sum(u[1] for u in usage)+psutil.Process().memory_info().rss
|
|
disk=disk_usage(root);available=psutil.virtual_memory().available
|
|
if rss>c['limits']['total_rss_bytes']:reason='GLOBAL_RSS_LIMIT'
|
|
if disk>=c['limits']['disk_bytes']:reason='GLOBAL_DISK_LIMIT'
|
|
# Live process-tree CPU includes guards: conservative, never an undercount.
|
|
if spent+live_cpu>c['limits']['fit_cpu_seconds']:reason='GLOBAL_FIT_CPU_LIMIT'
|
|
except Exception as exc:
|
|
monitor_errors.append(dict(type=type(exc).__name__,message=str(exc)));reason='GLOBAL_MONITOR_ERROR'
|
|
live_cpu=rss=disk=available=0
|
|
if reason:
|
|
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time()))
|
|
# Signal the guard, not the worker, to preserve final CPU/peak accounting.
|
|
for item in active.values():
|
|
if not item.get('stop_sent'):
|
|
try:item['process'].send_signal(signal.SIGTERM)
|
|
except ProcessLookupError:pass
|
|
item['stop_sent']=True
|
|
if not checkpoint and not active and cursors[c['first_cell']]==len(by[c['first_cell']]) and not reason:
|
|
check=cost_check(c,records);save(root/'first-cell-cost-check.json',check)
|
|
checkpoint=check['continue_within_cap']
|
|
if not checkpoint:cut='FIRST_CELL_COST_OR_FAILURE_STOP'
|
|
if not reason and not cut:
|
|
cells=eligible_cells(c,cursors,checkpoint_passed=checkpoint,active_cells={v['cell'] for v in active.values()})
|
|
busy={v['cell'] for v in active.values()};waiting=None
|
|
for cell in cells:
|
|
if cell in busy or len(active)>=c['limits']['workers']:continue
|
|
if not checkpoint and active:break
|
|
i=by[cell][cursors[cell]];job=c['jobs'][i]
|
|
if job['kind']=='FINAL' and job['arm'] in HIST:
|
|
candidates=[r for r in records if r['job']['cell']==cell and r['job']['arm']==job['arm'] and r['job']['kind']=='HPO']
|
|
trials=[]
|
|
for r in candidates:
|
|
value=read(checked(r['result_path'],r['result_sha256'])).get('validation_score') if r['worker']['status']=='PASS' else None
|
|
path=job_dir(root,r['index'])/'guard-result.json'
|
|
trials.append(dict(config_index=r['job']['config_index'],status=r['worker']['status'],score=value,
|
|
receipt_path=str(path),receipt_sha256=sha(path)))
|
|
decision=selection(trials);p=selection_path(root,job);p.parent.mkdir(exist_ok=True)
|
|
if not p.exists():save(p,decision)
|
|
elif read(p)!=decision:raise ValueError('Selection changed')
|
|
if decision['status']!='SELECTED':
|
|
records.append(dict(index=i,job=job,attempted=False,result_path=None,result_sha256=None,
|
|
guard_cpu_seconds=0,worker=dict(status='NA',cpu_seconds=0.,stop_reason='ALL_HPO_FAILED',monitor_errors=[])))
|
|
cursors[cell]+=1;continue
|
|
if not can_reserve(spent,len(active),c['limits']['fit_cpu_seconds']):
|
|
if not active:cut='NOT_RUN_BUDGET'
|
|
else:waiting='WAIT_ACTIVE_SETTLEMENT_FOR_BUDGET'
|
|
break
|
|
if available<10*2**30:
|
|
waiting='WAIT_RESOURCES_V4_PRIORITY';break
|
|
contract(root)
|
|
slot=next(k for k in range(4) if k not in active);dest=job_dir(root,i);dest.mkdir(parents=True,exist_ok=True)
|
|
if (dest/'worker.started.json').exists():raise ValueError('Attempt already started')
|
|
log=(dest/'guard.log').open('x')
|
|
process=subprocess.Popen([sys.executable,'tools/r4bc_guard.py',str(root),str(i),str(slot)],
|
|
env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
|
active[slot]=dict(process=process,log=log,index=i,cell=cell);cursors[cell]+=1
|
|
available-=8*2**30
|
|
save(root/'progress.json',dict(terminal_attempts=sum(r.get('attempted',True) for r in records),logical_terminal=len(records),
|
|
active={str(k):dict(job=v['index'],cell=v['cell'],guard_pid=v['process'].pid) for k,v in active.items()},
|
|
fit_cpu_seconds=spent,setup_cpu_seconds=initial_setup,live_guard_and_worker_cpu_seconds=live_cpu,
|
|
reserved_active_cpu_seconds=len(active)*1800,total_rss_bytes=rss,disk_bytes=disk,
|
|
checkpoint_passed=checkpoint,cut=cut,global_stop=reason,waiting=waiting,
|
|
monitor_errors=monitor_errors,wall_seconds=time.monotonic()-start))
|
|
if not active and (reason or cut or (checkpoint and not eligible_cells(c,cursors,checkpoint_passed=True))):break
|
|
time.sleep(.5)
|
|
except Exception as exc:
|
|
reason='COORDINATOR_EXCEPTION';save(root/'coordinator-failure.json',dict(type=type(exc).__name__,message=str(exc)))
|
|
save(root/'global-stop.json',dict(reason=reason))
|
|
finally:
|
|
for item in active.values():
|
|
try:item['process'].send_signal(signal.SIGTERM)
|
|
except ProcessLookupError:pass
|
|
for item in active.values():
|
|
item['process'].wait();item['log'].close();p=job_dir(root,item['index'])/'guard-result.json'
|
|
if p.exists():records.append(dict(**read(p),attempted=True))
|
|
else:accounting_complete=False
|
|
seen={r['index'] for r in records};models=freeze_checkpoints(root,c,records,cut,reason)
|
|
save(root/'batch-complete.json',dict(status='FIT_QUEUE_TERMINAL' if not reason else 'GLOBAL_STOP',
|
|
global_stop=reason,cut=cut,accounting_complete=accounting_complete,records=records,
|
|
never_attempted=[dict(index=j['index'],cell=j['cell'],arm=j['arm'],pool=j['pool'],
|
|
reason='not run: budget' if cut=='NOT_RUN_BUDGET' else (reason or cut)) for j in c['jobs'] if j['index'] not in seen],
|
|
fit_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in records),setup_cpu_seconds=initial_setup,
|
|
guard_cpu_seconds=sum(r.get('guard_cpu_seconds',0) for r in records),coordinator_cpu_seconds=time.process_time(),
|
|
model_manifest_sha256=sha(root/'models-terminal.json'),monitor_errors=monitor_errors,
|
|
postfit_state='READY_FOR_GIT_CHECKPOINT_FREEZE' if not reason and accounting_complete else 'FORBIDDEN_GLOBAL_STOP',
|
|
new_decodes=0,test_predictions=False,test_scoring=False,retries=0))
|
|
return 0 if not reason and accounting_complete else 1
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();raise SystemExit(run(a.root))
|