"""Only folds3/4 fresh; failed folds isolated, all old costs retained, never postfit.""" import fcntl,os,resource,signal,subprocess,sys,time from pathlib import Path import psutil from phase2_h1_worker_v2 import save,sha,checked from phase2_home_freeze_io import freeze from phase2_gate_outcome_review import validate_review from track_a_recovery_common import config,read,reuse_identity,execution_command,accepted_guard,project_cell,project_total,canonical_models,OLD from track_a_recovery_monitor import Monitor,device_usage from track_a_recovery_supervisor import tree_usage,disk_usage def run(root): started_clock=time.monotonic();cpu=time.process_time();children0=resource.getrusage(resource.RUSAGE_CHILDREN) c,_=config(root);validate_review(root);models=reuse_identity(c) lock=(root/'queue.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if (root/'started.json').exists():raise FileExistsError('One labelled recovery only') if any(p.pid!=os.getpid() and execution_command(p.info['cmdline'] or []) for p in psutil.process_iter(['pid','cmdline'])):raise RuntimeError('Existing Track A queue') os.sched_setaffinity(0,c['coordinator_cpus']);os.environ['PYTORCH_CUDA_ALLOC_CONF']='expandable_segments:True' active={};terminal={};estimates={};last_epoch={};pending=list(c['recovery_folds']);requests=[];idle={} for i in pending: (root/'jobs'/str(i)).mkdir(parents=True,exist_ok=False) idle[i]=Monitor(root/'jobs'/str(i)/'dispatch-monitor-errors.jsonl') def age():return time.monotonic()-started_clock def live():return {i:60+age() for i in pending+list(active)} def charged():return sum(e['gpu_seconds_charged'] for e in terminal.values())+sum(live().values()) def global_stop(why): if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=why,unix=time.time(),scope='cumulative_resource_integrity_or_operator_only')) def own_stop(i,why): p=root/'jobs'/str(i)/'stop.json' if not p.exists():save(p,dict(id=i,reason=why,unix=time.time(),other_fold_must_continue=True)) def terminate(v): if v.get('stop_sent') is None: if v['process'].poll() is None:v['process'].send_signal(signal.SIGTERM) v['stop_sent']=time.monotonic() elif time.monotonic()-v['stop_sent']>30: try: p=psutil.Process(v['process'].pid) for q in reversed(p.children(recursive=True)): try:q.kill() except psutil.NoSuchProcess:pass p.kill() except psutil.NoSuchProcess:pass def ledger(): save(root/'terminal-ledger.json',dict(entries=list(terminal.values()),old_gpu_seconds_once=OLD,new_settled_gpu_seconds=sum(e['gpu_seconds_charged'] for e in terminal.values()),new_charged_gpu_seconds=charged(),cumulative_training_gpu_seconds=OLD+charged())) def settle_unlaunched(i,reason): terminal[i]=dict(id=i,status='STOPPED_BEFORE_DISPATCH',returncode=None,gpu_seconds_charged=60+age(),reason=reason,monitor_policy=idle[i].policy(),no_retry=True) pending.remove(i);own_stop(i,reason);ledger() def collect(i,v): rc=v['process'].poll() if rc is None:return False p=root/'jobs'/str(i)/'complete.json';g=None;error=None try:g=read(p) if p.exists() else None;ok=accepted_guard(rc,g) except Exception as exc:ok=False;error=str(exc) cost=max(60+age(),v['wait']+(g.get('gpu_seconds_charged',0) if isinstance(g,dict) else 0)) terminal[i]=dict(id=i,status='PASS' if ok else 'STOPPED_OR_FAILED',returncode=rc,gpu_seconds_charged=cost,guard_complete_sha256=sha(p) if p.exists() else None,pre_dispatch_wait_seconds=v['wait'],receipt_error=error) v['log'].close();del active[i];ledger() # Deliberately no global-stop on failed/malformed job receipt. return True def dispatch(): for i in list(pending): j=c['jobs'][i] if 60+age()>=j['gpu_seconds']-120:settle_unlaunched(i,'IDLE_WAIT_EXHAUSTED_OWN_CAP');continue begin=time.time();usage,ok=idle[i].sample('gpu_usage',lambda:device_usage(j['gpu_uuid'])) save(root/'jobs'/str(i)/'dispatch-idle.json',dict(start_unix=begin,end_unix=time.time(),valid=ok,memory_bytes=usage[0] if ok else None,utilization=usage[1] if ok else None,monitor_policy=idle[i].policy(),charged_gpu_seconds=60+age())) if idle[i].failed:settle_unlaunched(i,'THREE_FAILED_DISPATCH_GPU_SAMPLES');continue if not ok or usage[0]>128*2**20 or usage[1]!=0:continue wait=age() if wait>=j['gpu_seconds']-120:settle_unlaunched(i,'IDLE_WAIT_EXHAUSTED_OWN_CAP');continue p=root/'jobs'/str(i);save(p/'assignment.json',dict(id=i,gpu_uuid=j['gpu_uuid'],contract_sha256=sha(root/'contract.json'),pre_dispatch_wait_seconds=wait,unix=time.time())) log=(p/'entry.log').open('x') try:proc=subprocess.Popen([sys.executable,'tools/track_a_recovery_guard.py',str(root),str(i)],stdout=log,stderr=subprocess.STDOUT,start_new_session=True) except Exception as exc:log.close();settle_unlaunched(i,'SPAWN_FAILED: '+str(exc));continue active[i]=dict(process=proc,log=log,wait=wait,stop_sent=None);pending.remove(i) save(p/'dispatch.json',dict(pid=proc.pid,gpu_uuid=j['gpu_uuid'],unix=time.time(),pre_dispatch_wait_seconds=wait)) handlers={s:signal.signal(s,lambda n,f:requests.append('OPERATOR_SIGNAL_'+str(n))) for s in [signal.SIGINT,signal.SIGTERM]} save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),new_fits=2,reused_folds=[0,1,2],fresh_folds=[3,4],old_gpu_seconds_once=OLD,new_train_cap=c['train_gpu_cap'],cumulative_gpu_cap=162000,test_targets_read=False,postfit_started=False)) try: save(root/'reused-models.json',dict(models=models,old_gpu_seconds_once=OLD)) freeze(root,'reuse',[root/'contract.json',root/'reused-models.json',*[Path(m['path']) for m in models],*[Path(m['standardization']) for m in models]]) while pending or active: for i,v in list(active.items()):collect(i,v) for reason in requests:global_stop(reason) for p,h in c['code_sha256'].items():checked(p,h) if charged()>=c['train_gpu_cap']-80:global_stop('CUMULATIVE_TRAIN_BOUNDARY') _,rss,_=tree_usage(os.getpid()) if rss>c['aggregate_rss_bytes'] or disk_usage(root)>c['aggregate_disk_bytes']:global_stop('AGGREGATE_RESOURCE_LIMIT') for i in list(active): rows=[read(p) for p in sorted((root/'jobs'/str(i)/'worker/history').glob('*.json'))] if rows and rows[-1]['epoch']>last_epoch.get(i,0): epoch=rows[-1]['epoch'];cell=project_cell(c['jobs'][i],dict(epoch=epoch,max_epoch_seconds=max(r['train_epoch_seconds'] for r in rows),charged=60+age())) estimates[i]=cell # Isolate the rejected cell before budgeting future work. # Its raw estimate is retained in estimates/result unchanged. if not cell['within_cap']:own_stop(i,'OWN_PROJECTED_CAP') stopping={j for j in active if (root/'jobs'/str(j)/'stop.json').exists()} total=project_total(c,estimates,terminal,live(),stopping=stopping) result=dict(status='PASS' if cell['within_cap'] and total['status']=='PASS' else 'STOP_OWN_COST_PROJECTION',cell=cell,total=total,contract_sha256=sha(root/'contract.json'),unix=time.time()) name='first-epoch-projection.json' if i not in last_epoch else 'latest-cost-projection.json' save(root/'jobs'/str(i)/name,result);save(root/'latest-cost-projection.json',total);last_epoch[i]=epoch if total['status']!='PASS':global_stop('CUMULATIVE_PROJECTED_CAP') if not (root/'global-stop.json').exists():dispatch() else: for i in list(pending):settle_unlaunched(i,'GLOBAL_STOP_BEFORE_DISPATCH') for i,v in active.items(): if (root/'global-stop.json').exists() or (root/'jobs'/str(i)/'stop.json').exists():terminate(v) save(root/'live-progress.json',dict(unix=time.time(),active=[dict(id=i,guard_pid=v['process'].pid) for i,v in active.items()],pending=list(pending),terminal=list(terminal.values()),old_gpu_seconds_once=OLD,new_training_gpu_seconds=charged(),cumulative_training_gpu_seconds=OLD+charged(),postfit_started=False)) if active or pending:time.sleep(1) config(root);reuse_identity(c);new=[] for i in c['recovery_folds']: if terminal[i]['status']!='PASS':continue w=root/'jobs'/str(i)/'worker';r=read(w/'result.json');stats=read(checked(w/'standardization.json',r['standardization_sha256']));j=c['jobs'][i] if r['completed_epochs']!=200 or r['optimizer_steps']!=j['target_steps'] or stats['parts']!=j['learning'] or stats['nodes']!=j['learning_nodes'] or r['test_targets_read']:raise ValueError('Recovery final identity') checked(r['checkpoint'],r['checkpoint_sha256']) new.append(dict(job=i,path=r['checkpoint'],sha256=r['checkpoint_sha256'],standardization=str(w/'standardization.json'),standardization_sha256=r['standardization_sha256'],origin='RECOVERY_FRESH_SEED0',training_contract_sha256=sha(root/'contract.json'),guard_complete_sha256=terminal[i]['guard_complete_sha256'])) complete=len(new)==2 and not (root/'global-stop.json').exists() and charged()<=c['train_gpu_cap'] available=canonical_models(c,new) if complete else sorted(models+new,key=lambda m:m['job']) name='models-terminal.json' if complete else 'available-models-incomplete.json' save(root/name,dict(models=available,new_fits=2,reused_folds=[0,1,2],all_five_complete=complete,postfit_started=False)) freeze(root,'models' if complete else 'incomplete-models',[root/'contract.json',root/name,root/'terminal-ledger.json',*[Path(m['path']) for m in available],*[Path(m['standardization']) for m in available]]) save(root/'complete.json',dict(status='TRAIN_COMPLETE_POSTFIT_REBIND_REQUIRED' if complete else 'RECOVERY_INCOMPLETE_NO_FURTHER_RECOVERY',entries=list(terminal.values()),old_gpu_seconds_once=OLD,new_training_gpu_seconds=charged(),cumulative_training_gpu_seconds=OLD+charged(),available_folds=[m['job'] for m in available],coordinator_cpu_seconds=time.process_time()-cpu,postfit_started=False,test_targets_read=False,no_further_recovery=True,original_postfit_exact_binding_compatible=False)) freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/name,root/'terminal-ledger.json']) return 0 if complete else 1 except BaseException as exc: global_stop(type(exc).__name__+': '+str(exc)) for i in list(pending):settle_unlaunched(i,'COORDINATOR_ABORT') while active: for i,v in list(active.items()): if not collect(i,v):terminate(v) if active:time.sleep(.5) save(root/'complete.json',dict(status='RECOVERY_COORDINATOR_STOPPED',reason=str(exc),entries=list(terminal.values()),old_gpu_seconds_once=OLD,new_training_gpu_seconds=charged(),cumulative_training_gpu_seconds=OLD+charged(),coordinator_cpu_seconds=time.process_time()-cpu,postfit_started=False,no_further_recovery=True)) freeze(root,'stopped',[root/'contract.json',root/'complete.json',root/'global-stop.json',root/'terminal-ledger.json']) raise finally: for s,h in handlers.items():signal.signal(s,h) if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1])))