"""Single six-cell queue, at most four GPU workers; first-epoch and periodic cost gates; checkpoint-only terminal.""" import fcntl,os,signal,subprocess,sys,time from pathlib import Path import psutil from phase3_data_sufficiency_v2_common import config,read,project_cell,project_total,execution_command,dispatchable from phase2_h1_worker_v2 import save,sha,checked from phase2_home_freeze_io import freeze from phase2_gate_outcome_review import validate_review from s2_recovery_gpu_supervisor import tree_usage,disk_usage def run(root): c,pilot=config(root);validate_review(root);lock=(root/'queue.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if (root/'started.json').exists():raise FileExistsError('Single item1 attempt; no retry') for p in psutil.process_iter(['pid','cmdline']): if p.pid!=os.getpid() and execution_command(p.info['cmdline'] or []):raise RuntimeError('Existing item1 execution') os.sched_setaffinity(0,c['coordinator_cpus']);os.environ['PYTORCH_CUDA_ALLOC_CONF']='expandable_segments:True';start=time.monotonic();cpu=time.process_time();active={};terminal={};requested=[];last_projection={};estimates={};pending=list(c['jobs']) def stop(reason): if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time(),no_retry=True)) def charged():return sum(v['gpu_seconds_charged'] for v in terminal.values())+sum(60+time.monotonic()-v['start'] for v in active.values()) def terminate(v): if v.get('stop_sent') is None:v['process'].send_signal(signal.SIGTERM);v['stop_sent']=time.monotonic() elif time.monotonic()-v['stop_sent']>20: try: parent=psutil.Process(v['process'].pid) for p in reversed(parent.children(recursive=True)): try:p.kill() except psutil.NoSuchProcess:pass parent.kill() except psutil.NoSuchProcess:pass def collect(i,v): rc=v['process'].poll() if rc is None:return False p=root/'jobs'/str(i)/'complete.json';r=read(p) if p.exists() else None;observed=60+time.monotonic()-v['start'];ok=rc==0 and r is not None and r['status']=='PASS' terminal[i]=dict(id=i,status='PASS' if ok else 'STOPPED_OR_FAILED',returncode=rc,gpu_seconds_charged=max(observed,r['gpu_seconds_charged'] if r else 0),guard_complete_sha256=sha(p) if r else None);v['log'].close();del active[i] save(root/'terminal-ledger.json',dict(entries=list(terminal.values()),settled_gpu_seconds=sum(x['gpu_seconds_charged'] for x in terminal.values()))) if not ok:stop('CELL_'+str(i)+'_FAILED') return True def observe(j): i=j['id'];rows=[read(p) for p in sorted((root/'jobs'/str(i)/'worker/history').glob('*.json'))];full=[r for r in rows if r['full_epoch']] if not full:return None return dict(steps=rows[-1]['optimizer_steps'],charged=terminal[i]['gpu_seconds_charged'] if i in terminal else 60+time.monotonic()-active[i]['start'],max_epoch_seconds=max(r['train_epoch_seconds'] for r in full),max_validation_seconds=max(r['validation_seconds'] for r in rows)) def dispatch(): for j in dispatchable(pending,[c['jobs'][i] for i in active]): i=j['id'];d=root/'jobs'/str(i);d.mkdir(parents=True,exist_ok=False);log=(d/'entry.log').open('x');begin=time.monotonic();cmd=[sys.executable,'tools/phase3_data_sufficiency_v2_guard.py',str(root),str(i)];p=subprocess.Popen(cmd,stdout=log,stderr=subprocess.STDOUT,start_new_session=True);active[i]=dict(process=p,log=log,start=begin,stop_sent=None);save(d/'dispatch.json',dict(pid=p.pid,command=cmd,unix=time.time(),gpu_uuid=j['gpu_uuid']));pending.remove(j) handlers={s:signal.signal(s,lambda n,f:requested.append('SIGNAL_'+str(n))) for s in [signal.SIGTERM,signal.SIGINT]} save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),review_sha256=sha(root/'review.json'),new_fits=6,S1_retrained=False,item_gpu_cap=120000,postfit_started=False)) try: while active or pending: if not (root/'global-stop.json').exists():dispatch() for i,v in list(active.items()):collect(i,v) for x in requested:stop(x) amount=charged();_,rss,_=tree_usage(os.getpid()) if amount>=c['train_gpu_cap']-80:stop('CUMULATIVE_TRAIN_GPU_RESERVE_BOUNDARY') if rss>64*2**30 or disk_usage(root)>80*2**30:stop('AGGREGATE_MEMORY_DISK_LIMIT') if not (root/'global-stop.json').exists(): for j in c['jobs']: i=j['id'] if i not in active:continue obs=observe(j) if obs is None:continue stage=int(obs['steps']/j['target_steps']*20) if stage>last_projection.get(i,-1): cell=project_cell(j,obs);estimates[i]=cell;total=project_total(c,estimates,terminal);p=dict(**total,cell=cell,contract_sha256=sha(root/'contract.json'),unix=time.time()) if not cell['within_cap']:p['status']='STOP_BUDGET_PROJECTION' save(root/'jobs'/str(i)/('first-epoch-projection.json' if i not in last_projection else 'latest-cost-projection.json'),p);save(root/'latest-cost-projection.json',p);last_projection[i]=stage if p['status']!='PASS':stop('PROJECTED_ITEM_GPU_BUDGET') if (root/'global-stop.json').exists(): for v in active.values():terminate(v) save(root/'live-progress.json',dict(stage='STOPPING' if (root/'global-stop.json').exists() else 'ITEM1_TRAINING',unix=time.time(),active=[dict(id=i,guard_pid=v['process'].pid) for i,v in active.items()],terminal=list(terminal.values()),pending=[j['id'] for j in pending],charged_training_gpu_seconds=amount,postfit_gpu_reserve=c['postfit_gpu_reserve'],item_cap=120000,phase3_cap=612000)) if (root/'global-stop.json').exists() and not active:break if active:time.sleep(1) if (root/'global-stop.json').exists() or len(terminal)!=6 or any(x['status']!='PASS' for x in terminal.values()):raise RuntimeError('Item1 train incomplete') config(root);models=[] for j in c['jobs']: w=root/'jobs'/str(j['id'])/'worker';r=read(w/'result.json');stats=read(checked(w/'standardization.json',r['standardization_sha256'])) if r['optimizer_steps']!=j['target_steps'] or stats['parts']!=c['pools'][str(j['pool'])]['learning'] or r['test_targets_read']:raise ValueError('Terminal cell identity') for m in r['checkpoints']:checked(m['path'],m['sha256']);models.append(dict(job=j['id'],pool=j['pool'],mode=j['mode'],**m)) if len(models)!=6 or charged()>c['train_gpu_cap']:raise ValueError('Checkpoint or cumulative cost') save(root/'models-terminal.json',dict(models=models,reused_S1=c['S1_reuse'],new_fits=6,new_checkpoints=6,test_prediction_started=False)) freeze(root,'models',[root/'contract.json',root/'models-terminal.json',root/'terminal-ledger.json',*[Path(m['path']) for m in models],*[Path(m['standardization']) for m in models]]) save(root/'complete.json',dict(status='TRAIN_COMPLETE_POSTFIT_PENDING',entries=list(terminal.values()),training_gpu_seconds=charged(),postfit_gpu_reserve=c['postfit_gpu_reserve'],item_gpu_cap=120000,coordinator_cpu_seconds=time.process_time()-cpu,wall_seconds=time.monotonic()-start,no_retry=True,test_targets_read=False));freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'models-terminal.json']);return 0 except BaseException as exc: stop(type(exc).__name__+': '+str(exc)) while active: for i,v in list(active.items()): try: if not collect(i,v):terminate(v) except Exception: terminate(v) if v['process'].poll() is not None:terminal[i]=dict(id=i,status='MALFORMED_TERMINAL',gpu_seconds_charged=60+time.monotonic()-v['start']);v['log'].close();del active[i] if active:time.sleep(.5) save(root/'complete.json',dict(status='STOPPED',reason=str(exc),entries=list(terminal.values()),never_attempted=[j['id'] for j in pending],training_gpu_seconds=charged(),postfit_started=False,no_retry=True));return 1 finally: for s,h in handlers.items():signal.signal(s,h) if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1])))