Files
MeltMind/tools/track_a_recovery_queue.py
T

138 lines
11 KiB
Python

"""Only folds3/4 fresh; failed folds isolated, all old costs retained, never postfit."""
import fcntl,os,resource,signal,subprocess,sys,time
from pathlib import Path
import psutil
from phase2_h1_worker_v2 import save,sha,checked
from phase2_home_freeze_io import freeze
from phase2_gate_outcome_review import validate_review
from track_a_recovery_common import config,read,reuse_identity,execution_command,accepted_guard,project_cell,project_total,canonical_models,OLD
from track_a_recovery_monitor import Monitor,device_usage
from track_a_recovery_supervisor import tree_usage,disk_usage
def run(root):
started_clock=time.monotonic();cpu=time.process_time();children0=resource.getrusage(resource.RUSAGE_CHILDREN)
c,_=config(root);validate_review(root);models=reuse_identity(c)
lock=(root/'queue.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('One labelled recovery only')
if any(p.pid!=os.getpid() and execution_command(p.info['cmdline'] or []) for p in psutil.process_iter(['pid','cmdline'])):raise RuntimeError('Existing Track A queue')
os.sched_setaffinity(0,c['coordinator_cpus']);os.environ['PYTORCH_CUDA_ALLOC_CONF']='expandable_segments:True'
active={};terminal={};estimates={};last_epoch={};pending=list(c['recovery_folds']);requests=[];idle={}
for i in pending:
(root/'jobs'/str(i)).mkdir(parents=True,exist_ok=False)
idle[i]=Monitor(root/'jobs'/str(i)/'dispatch-monitor-errors.jsonl')
def age():return time.monotonic()-started_clock
def live():return {i:60+age() for i in pending+list(active)}
def charged():return sum(e['gpu_seconds_charged'] for e in terminal.values())+sum(live().values())
def global_stop(why):
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=why,unix=time.time(),scope='cumulative_resource_integrity_or_operator_only'))
def own_stop(i,why):
p=root/'jobs'/str(i)/'stop.json'
if not p.exists():save(p,dict(id=i,reason=why,unix=time.time(),other_fold_must_continue=True))
def terminate(v):
if v.get('stop_sent') is None:
if v['process'].poll() is None:v['process'].send_signal(signal.SIGTERM)
v['stop_sent']=time.monotonic()
elif time.monotonic()-v['stop_sent']>30:
try:
p=psutil.Process(v['process'].pid)
for q in reversed(p.children(recursive=True)):
try:q.kill()
except psutil.NoSuchProcess:pass
p.kill()
except psutil.NoSuchProcess:pass
def ledger():
save(root/'terminal-ledger.json',dict(entries=list(terminal.values()),old_gpu_seconds_once=OLD,new_settled_gpu_seconds=sum(e['gpu_seconds_charged'] for e in terminal.values()),new_charged_gpu_seconds=charged(),cumulative_training_gpu_seconds=OLD+charged()))
def settle_unlaunched(i,reason):
terminal[i]=dict(id=i,status='STOPPED_BEFORE_DISPATCH',returncode=None,gpu_seconds_charged=60+age(),reason=reason,monitor_policy=idle[i].policy(),no_retry=True)
pending.remove(i);own_stop(i,reason);ledger()
def collect(i,v):
rc=v['process'].poll()
if rc is None:return False
p=root/'jobs'/str(i)/'complete.json';g=None;error=None
try:g=read(p) if p.exists() else None;ok=accepted_guard(rc,g)
except Exception as exc:ok=False;error=str(exc)
cost=max(60+age(),v['wait']+(g.get('gpu_seconds_charged',0) if isinstance(g,dict) else 0))
terminal[i]=dict(id=i,status='PASS' if ok else 'STOPPED_OR_FAILED',returncode=rc,gpu_seconds_charged=cost,guard_complete_sha256=sha(p) if p.exists() else None,pre_dispatch_wait_seconds=v['wait'],receipt_error=error)
v['log'].close();del active[i];ledger()
# Deliberately no global-stop on failed/malformed job receipt.
return True
def dispatch():
for i in list(pending):
j=c['jobs'][i]
if 60+age()>=j['gpu_seconds']-120:settle_unlaunched(i,'IDLE_WAIT_EXHAUSTED_OWN_CAP');continue
begin=time.time();usage,ok=idle[i].sample('gpu_usage',lambda:device_usage(j['gpu_uuid']))
save(root/'jobs'/str(i)/'dispatch-idle.json',dict(start_unix=begin,end_unix=time.time(),valid=ok,memory_bytes=usage[0] if ok else None,utilization=usage[1] if ok else None,monitor_policy=idle[i].policy(),charged_gpu_seconds=60+age()))
if idle[i].failed:settle_unlaunched(i,'THREE_FAILED_DISPATCH_GPU_SAMPLES');continue
if not ok or usage[0]>128*2**20 or usage[1]!=0:continue
wait=age()
if wait>=j['gpu_seconds']-120:settle_unlaunched(i,'IDLE_WAIT_EXHAUSTED_OWN_CAP');continue
p=root/'jobs'/str(i);save(p/'assignment.json',dict(id=i,gpu_uuid=j['gpu_uuid'],contract_sha256=sha(root/'contract.json'),pre_dispatch_wait_seconds=wait,unix=time.time()))
log=(p/'entry.log').open('x')
try:proc=subprocess.Popen([sys.executable,'tools/track_a_recovery_guard.py',str(root),str(i)],stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
except Exception as exc:log.close();settle_unlaunched(i,'SPAWN_FAILED: '+str(exc));continue
active[i]=dict(process=proc,log=log,wait=wait,stop_sent=None);pending.remove(i)
save(p/'dispatch.json',dict(pid=proc.pid,gpu_uuid=j['gpu_uuid'],unix=time.time(),pre_dispatch_wait_seconds=wait))
handlers={s:signal.signal(s,lambda n,f:requests.append('OPERATOR_SIGNAL_'+str(n))) for s in [signal.SIGINT,signal.SIGTERM]}
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),new_fits=2,reused_folds=[0,1,2],fresh_folds=[3,4],old_gpu_seconds_once=OLD,new_train_cap=c['train_gpu_cap'],cumulative_gpu_cap=162000,test_targets_read=False,postfit_started=False))
try:
save(root/'reused-models.json',dict(models=models,old_gpu_seconds_once=OLD))
freeze(root,'reuse',[root/'contract.json',root/'reused-models.json',*[Path(m['path']) for m in models],*[Path(m['standardization']) for m in models]])
while pending or active:
for i,v in list(active.items()):collect(i,v)
for reason in requests:global_stop(reason)
for p,h in c['code_sha256'].items():checked(p,h)
if charged()>=c['train_gpu_cap']-80:global_stop('CUMULATIVE_TRAIN_BOUNDARY')
_,rss,_=tree_usage(os.getpid())
if rss>c['aggregate_rss_bytes'] or disk_usage(root)>c['aggregate_disk_bytes']:global_stop('AGGREGATE_RESOURCE_LIMIT')
for i in list(active):
rows=[read(p) for p in sorted((root/'jobs'/str(i)/'worker/history').glob('*.json'))]
if rows and rows[-1]['epoch']>last_epoch.get(i,0):
epoch=rows[-1]['epoch'];cell=project_cell(c['jobs'][i],dict(epoch=epoch,max_epoch_seconds=max(r['train_epoch_seconds'] for r in rows),charged=60+age()))
estimates[i]=cell
# Isolate the rejected cell before budgeting future work.
# Its raw estimate is retained in estimates/result unchanged.
if not cell['within_cap']:own_stop(i,'OWN_PROJECTED_CAP')
stopping={j for j in active if (root/'jobs'/str(j)/'stop.json').exists()}
total=project_total(c,estimates,terminal,live(),stopping=stopping)
result=dict(status='PASS' if cell['within_cap'] and total['status']=='PASS' else 'STOP_OWN_COST_PROJECTION',cell=cell,total=total,contract_sha256=sha(root/'contract.json'),unix=time.time())
name='first-epoch-projection.json' if i not in last_epoch else 'latest-cost-projection.json'
save(root/'jobs'/str(i)/name,result);save(root/'latest-cost-projection.json',total);last_epoch[i]=epoch
if total['status']!='PASS':global_stop('CUMULATIVE_PROJECTED_CAP')
if not (root/'global-stop.json').exists():dispatch()
else:
for i in list(pending):settle_unlaunched(i,'GLOBAL_STOP_BEFORE_DISPATCH')
for i,v in active.items():
if (root/'global-stop.json').exists() or (root/'jobs'/str(i)/'stop.json').exists():terminate(v)
save(root/'live-progress.json',dict(unix=time.time(),active=[dict(id=i,guard_pid=v['process'].pid) for i,v in active.items()],pending=list(pending),terminal=list(terminal.values()),old_gpu_seconds_once=OLD,new_training_gpu_seconds=charged(),cumulative_training_gpu_seconds=OLD+charged(),postfit_started=False))
if active or pending:time.sleep(1)
config(root);reuse_identity(c);new=[]
for i in c['recovery_folds']:
if terminal[i]['status']!='PASS':continue
w=root/'jobs'/str(i)/'worker';r=read(w/'result.json');stats=read(checked(w/'standardization.json',r['standardization_sha256']));j=c['jobs'][i]
if r['completed_epochs']!=200 or r['optimizer_steps']!=j['target_steps'] or stats['parts']!=j['learning'] or stats['nodes']!=j['learning_nodes'] or r['test_targets_read']:raise ValueError('Recovery final identity')
checked(r['checkpoint'],r['checkpoint_sha256'])
new.append(dict(job=i,path=r['checkpoint'],sha256=r['checkpoint_sha256'],standardization=str(w/'standardization.json'),standardization_sha256=r['standardization_sha256'],origin='RECOVERY_FRESH_SEED0',training_contract_sha256=sha(root/'contract.json'),guard_complete_sha256=terminal[i]['guard_complete_sha256']))
complete=len(new)==2 and not (root/'global-stop.json').exists() and charged()<=c['train_gpu_cap']
available=canonical_models(c,new) if complete else sorted(models+new,key=lambda m:m['job'])
name='models-terminal.json' if complete else 'available-models-incomplete.json'
save(root/name,dict(models=available,new_fits=2,reused_folds=[0,1,2],all_five_complete=complete,postfit_started=False))
freeze(root,'models' if complete else 'incomplete-models',[root/'contract.json',root/name,root/'terminal-ledger.json',*[Path(m['path']) for m in available],*[Path(m['standardization']) for m in available]])
save(root/'complete.json',dict(status='TRAIN_COMPLETE_POSTFIT_REBIND_REQUIRED' if complete else 'RECOVERY_INCOMPLETE_NO_FURTHER_RECOVERY',entries=list(terminal.values()),old_gpu_seconds_once=OLD,new_training_gpu_seconds=charged(),cumulative_training_gpu_seconds=OLD+charged(),available_folds=[m['job'] for m in available],coordinator_cpu_seconds=time.process_time()-cpu,postfit_started=False,test_targets_read=False,no_further_recovery=True,original_postfit_exact_binding_compatible=False))
freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/name,root/'terminal-ledger.json'])
return 0 if complete else 1
except BaseException as exc:
global_stop(type(exc).__name__+': '+str(exc))
for i in list(pending):settle_unlaunched(i,'COORDINATOR_ABORT')
while active:
for i,v in list(active.items()):
if not collect(i,v):terminate(v)
if active:time.sleep(.5)
save(root/'complete.json',dict(status='RECOVERY_COORDINATOR_STOPPED',reason=str(exc),entries=list(terminal.values()),old_gpu_seconds_once=OLD,new_training_gpu_seconds=charged(),cumulative_training_gpu_seconds=OLD+charged(),coordinator_cpu_seconds=time.process_time()-cpu,postfit_started=False,no_further_recovery=True))
freeze(root,'stopped',[root/'contract.json',root/'complete.json',root/'global-stop.json',root/'terminal-ledger.json'])
raise
finally:
for s,h in handlers.items():signal.signal(s,h)
if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1])))