Files
MeltMind/tools/run_r4bc.py
T

228 lines
14 KiB
Python

"""One detached R4bc fit queue. Pool barriers, reserved budgets and immutable attempts."""
import argparse
import fcntl
import math
import os
from pathlib import Path
import signal
import subprocess
import sys
import time
import psutil
from r4bc_common import (ARMS,HIST,POOLS,contract,read,sha,checked,save,selection,selection_path,
job_dir,can_reserve,role_rows,histgbrt_grid)
from s2_recovery_supervisor import tree_usage
from s2_recovery_monitoring import disk_usage
def tracks(c):
return {cell:[j['index'] for j in c['jobs'] if j['cell']==cell] for cell in range(15)}
def eligible_cells(c, cursors, *, checkpoint_passed, active_cells=()):
by=tracks(c)
if not checkpoint_passed:return [c['first_cell']] if cursors[c['first_cell']]<len(by[c['first_cell']]) else []
for pool in POOLS:
cells=[f*3+(25,50,100).index(pool) for f in range(5)]
pending=[cell for cell in cells if cursors[cell]<len(by[cell]) or cell in active_cells]
if pending:return pending
return []
def cost_check(c, records):
first=c['first_cell']; done={r['index']:r for r in records}
initial=[j for j in c['jobs'] if j['cell']==first]
finals=[done.get(j['index']) for j in initial if j['kind']=='FINAL']
if any(r is None or r['worker']['status']!='PASS' for r in finals):
return dict(continue_within_cap=False,reason='FIRST_CELL_FINAL_FAILED_OR_INCOMPLETE')
reference={}
for j in initial:
r=done.get(j['index'])
if r is None or not r.get('attempted',True):return dict(continue_within_cap=False,reason='FIRST_CELL_INCOMPLETE')
reference[j['arm'],j['kind'],j['config_index']]=r['worker']['cpu_seconds']
sizes={}
for job in c['jobs']:
key=job['cell'],job['arm']
if key in sizes:continue
rows=role_rows(c,job['fold'],job['pool'],'learning')
sizes[key]=sum(r['nodes'] if job['arm'] in ('H4','A') else r['preparation']['eligible_unique'] for r in rows)
remaining_100=remaining_all=0.
for j in c['jobs']:
if j['cell']==first:continue
sample=reference[j['arm'],j['kind'],j['config_index']]
scale=sizes[j['cell'],j['arm']]/sizes[first,j['arm']]
# The final selected configuration is not known. Reserve the max grid
# iteration ratio to the observed first-cell selection for final fits.
if j['arm'] in HIST and j['kind']=='FINAL':
fj=next(x for x in initial if x['arm']==j['arm'] and x['kind']=='FINAL')
result=read(done[fj['index']]['result_path'])
scale*=1000/histgbrt_grid()[result['config_index']]['max_iter']
projected=sample*scale
remaining_all+=projected
if j['pool']==100:remaining_100+=projected
spent=sum(r['worker']['cpu_seconds'] for r in records)
return dict(continue_within_cap=remaining_100<=c['limits']['fit_cpu_seconds']-spent,
first_cell_cpu_seconds=spent,remaining_100_projection=remaining_100,
all_three_pools_projection=spent+remaining_all,full_100_projection=spent+remaining_100,
method='Per arm/config measured guard CPU scaled by learning rows; final worst iteration ratio. Heuristic, hard reservation remains authoritative.',
all_three_over_cap_does_not_stop_100=True)
def freeze_checkpoints(root,c,records,cut,global_reason):
by={r['index']:r for r in records};models=[]
for j in c['jobs']:
if j['kind']!='FINAL':continue
record=by.get(j['index']);result=None
if record and record['worker']['status']=='PASS':
result=read(checked(record['result_path'],record['result_sha256']))
checked(result['model_path'],result['model_sha256'])
models.append(dict(cell=j['cell'],fold=j['fold'],pool=j['pool'],arm=j['arm'],
status='PASS' if result else 'NA',model_path=result['model_path'] if result else None,
model_sha256=result['model_sha256'] if result else None,
reason=None if result else (record['worker'].get('stop_reason') if record else (global_reason or cut or 'NOT_ATTEMPTED'))))
save(root/'models-terminal.json',dict(contract_sha256=sha(root/'contract.json'),models=models,
fit_records={str(r['index']):sha(job_dir(root,r['index'])/'guard-result.json')
for r in records if r.get('attempted',True)}))
return models
def run(root):
root=Path(root)
with (root/'batch.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
return locked(root)
def locked(root):
c=contract(root); launch=read(root/'launch-freeze.json')
checked(root/'contract.json',launch['contract_sha256']);checked(root/'input-freeze.json',launch['input_freeze_sha256'])
checked(root/'job-list.json',c['job_list_sha256'])
if read(root/'preflight-guard/worker.receipt.json')['status']!='PASS':raise ValueError('Preflight did not pass')
if (root/'batch-started.json').exists():raise FileExistsError('No queue restart')
os.sched_setaffinity(0,[c['coordinator_cpu']])
start=time.monotonic();save(root/'batch-started.json',dict(pid=os.getpid(),unix=time.time(),
contract_sha256=sha(root/'contract.json'),implementation_commit=launch['implementation_commit'],
input_freeze_sha256=launch['input_freeze_sha256'],cpu_affinity=sorted(os.sched_getaffinity(0))))
by=tracks(c);cursors={cell:0 for cell in by};active={};records=[];cut=None;reason=None;checkpoint=False
signal_request=[None]
def interrupted(sig,frame):signal_request[0]='COORDINATOR_SIGNAL_'+str(sig)
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
initial_setup=c['initial_setup_cpu_seconds']+read(root/'preflight-guard/worker.receipt.json')['cpu_seconds']
if initial_setup>c['limits']['setup_cpu_seconds']:raise ValueError('Setup budget already exceeded')
accounting_complete=True;monitor_errors=[];waiting=None
env=dict(os.environ,OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
try:
while True:
for slot,item in list(active.items()):
code=item['process'].poll()
if code is None:continue
p=job_dir(root,item['index'])/'guard-result.json'
if not p.exists():
accounting_complete=False;reason='MISSING_GUARD_RECEIPT'
save(root/'missing-guard.json',dict(index=item['index'],returncode=code,accounting_complete=False))
else:
r=read(p)
if not math.isfinite(r['worker']['cpu_seconds']) or r['worker']['cpu_seconds']<0:
accounting_complete=False;reason='INVALID_ACCOUNTING'
records.append(dict(**r,attempted=True))
if r['global_failure']:reason='WORKER_GLOBAL_FAILURE'
item['log'].close();del active[slot]
print('TERMINAL',item['index'],code,len(records),flush=True)
spent=sum(r['worker']['cpu_seconds'] for r in records)
if signal_request[0]:reason=reason or signal_request[0]
if (root/'global-stop.json').exists():reason=reason or read(root/'global-stop.json')['reason']
try:
usage=[tree_usage(v['process'].pid) for v in active.values()]
live_cpu=sum(u[0] for u in usage);rss=sum(u[1] for u in usage)+psutil.Process().memory_info().rss
disk=disk_usage(root);available=psutil.virtual_memory().available
if rss>c['limits']['total_rss_bytes']:reason='GLOBAL_RSS_LIMIT'
if disk>=c['limits']['disk_bytes']:reason='GLOBAL_DISK_LIMIT'
# Live process-tree CPU includes guards: conservative, never an undercount.
if spent+live_cpu>c['limits']['fit_cpu_seconds']:reason='GLOBAL_FIT_CPU_LIMIT'
except Exception as exc:
monitor_errors.append(dict(type=type(exc).__name__,message=str(exc)));reason='GLOBAL_MONITOR_ERROR'
live_cpu=rss=disk=available=0
if reason:
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time()))
# Signal the guard, not the worker, to preserve final CPU/peak accounting.
for item in active.values():
if not item.get('stop_sent'):
try:item['process'].send_signal(signal.SIGTERM)
except ProcessLookupError:pass
item['stop_sent']=True
if not checkpoint and not active and cursors[c['first_cell']]==len(by[c['first_cell']]) and not reason:
check=cost_check(c,records);save(root/'first-cell-cost-check.json',check)
checkpoint=check['continue_within_cap']
if not checkpoint:cut='FIRST_CELL_COST_OR_FAILURE_STOP'
if not reason and not cut:
cells=eligible_cells(c,cursors,checkpoint_passed=checkpoint,active_cells={v['cell'] for v in active.values()})
busy={v['cell'] for v in active.values()};waiting=None
for cell in cells:
if cell in busy or len(active)>=c['limits']['workers']:continue
if not checkpoint and active:break
i=by[cell][cursors[cell]];job=c['jobs'][i]
if job['kind']=='FINAL' and job['arm'] in HIST:
candidates=[r for r in records if r['job']['cell']==cell and r['job']['arm']==job['arm'] and r['job']['kind']=='HPO']
trials=[]
for r in candidates:
value=read(checked(r['result_path'],r['result_sha256'])).get('validation_score') if r['worker']['status']=='PASS' else None
path=job_dir(root,r['index'])/'guard-result.json'
trials.append(dict(config_index=r['job']['config_index'],status=r['worker']['status'],score=value,
receipt_path=str(path),receipt_sha256=sha(path)))
decision=selection(trials);p=selection_path(root,job);p.parent.mkdir(exist_ok=True)
if not p.exists():save(p,decision)
elif read(p)!=decision:raise ValueError('Selection changed')
if decision['status']!='SELECTED':
records.append(dict(index=i,job=job,attempted=False,result_path=None,result_sha256=None,
guard_cpu_seconds=0,worker=dict(status='NA',cpu_seconds=0.,stop_reason='ALL_HPO_FAILED',monitor_errors=[])))
cursors[cell]+=1;continue
if not can_reserve(spent,len(active),c['limits']['fit_cpu_seconds']):
if not active:cut='NOT_RUN_BUDGET'
else:waiting='WAIT_ACTIVE_SETTLEMENT_FOR_BUDGET'
break
if available<10*2**30:
waiting='WAIT_RESOURCES_V4_PRIORITY';break
contract(root)
slot=next(k for k in range(4) if k not in active);dest=job_dir(root,i);dest.mkdir(parents=True,exist_ok=True)
if (dest/'worker.started.json').exists():raise ValueError('Attempt already started')
log=(dest/'guard.log').open('x')
process=subprocess.Popen([sys.executable,'tools/r4bc_guard.py',str(root),str(i),str(slot)],
env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
active[slot]=dict(process=process,log=log,index=i,cell=cell);cursors[cell]+=1
available-=8*2**30
save(root/'progress.json',dict(terminal_attempts=sum(r.get('attempted',True) for r in records),logical_terminal=len(records),
active={str(k):dict(job=v['index'],cell=v['cell'],guard_pid=v['process'].pid) for k,v in active.items()},
fit_cpu_seconds=spent,setup_cpu_seconds=initial_setup,live_guard_and_worker_cpu_seconds=live_cpu,
reserved_active_cpu_seconds=len(active)*1800,total_rss_bytes=rss,disk_bytes=disk,
checkpoint_passed=checkpoint,cut=cut,global_stop=reason,waiting=waiting,
monitor_errors=monitor_errors,wall_seconds=time.monotonic()-start))
if not active and (reason or cut or (checkpoint and not eligible_cells(c,cursors,checkpoint_passed=True))):break
time.sleep(.5)
except Exception as exc:
reason='COORDINATOR_EXCEPTION';save(root/'coordinator-failure.json',dict(type=type(exc).__name__,message=str(exc)))
save(root/'global-stop.json',dict(reason=reason))
finally:
for item in active.values():
try:item['process'].send_signal(signal.SIGTERM)
except ProcessLookupError:pass
for item in active.values():
item['process'].wait();item['log'].close();p=job_dir(root,item['index'])/'guard-result.json'
if p.exists():records.append(dict(**read(p),attempted=True))
else:accounting_complete=False
seen={r['index'] for r in records};models=freeze_checkpoints(root,c,records,cut,reason)
save(root/'batch-complete.json',dict(status='FIT_QUEUE_TERMINAL' if not reason else 'GLOBAL_STOP',
global_stop=reason,cut=cut,accounting_complete=accounting_complete,records=records,
never_attempted=[dict(index=j['index'],cell=j['cell'],arm=j['arm'],pool=j['pool'],
reason='not run: budget' if cut=='NOT_RUN_BUDGET' else (reason or cut)) for j in c['jobs'] if j['index'] not in seen],
fit_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in records),setup_cpu_seconds=initial_setup,
guard_cpu_seconds=sum(r.get('guard_cpu_seconds',0) for r in records),coordinator_cpu_seconds=time.process_time(),
model_manifest_sha256=sha(root/'models-terminal.json'),monitor_errors=monitor_errors,
postfit_state='READY_FOR_GIT_CHECKPOINT_FREEZE' if not reason and accounting_complete else 'FORBIDDEN_GLOBAL_STOP',
new_decodes=0,test_predictions=False,test_scoring=False,retries=0))
return 0 if not reason and accounting_complete else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();raise SystemExit(run(a.root))