"""Review-gated five-fold b training. Stops at frozen models, never postfit.""" import fcntl import json import os from pathlib import Path import signal import subprocess import sys import time import psutil from phase2_h1_worker_v2 import save,sha from phase2_home_freeze_io import freeze from phase2_gate_outcome_review import validate_review from phase3_item_b_shared_common import ROOT,config,read,project from phase3_item_b_shared_capacity import register_entry,acquire,transfer,release,process_identity,alive from track_a_recovery_monitor import Monitor def run(root): c,_=config(root);validate_review(root) with (root/'queue.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if (root/'started.json').exists():raise FileExistsError('Sole b attempt, no resume') if os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True': raise ValueError('Explicit allocator required') if psutil.disk_usage(root).free<100*2**30:raise RuntimeError('Less than100GiB free') register_entry(root,sha(root/'contract.json'),c) start=time.monotonic();identity=process_identity(os.getpid()) save(root/'started.json',dict(unix=time.time(),identity=identity,contract_sha256=sha(root/'contract.json'))) (root/'jobs').mkdir(exist_ok=False) active={};terminal={};pending=[0,1,2,3,4];histories={};waits={};stops=[];failure=None monitor=Monitor(root/'admission-errors.jsonl') for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,lambda n,f:stops.append('SIGNAL_'+str(n))) def charge(i): return terminal[i]['gpu_seconds_charged'] if i in terminal else 60+time.monotonic()-start def waiting_charges(charges): return {i: waits.get(i, max(0, charged - 60)) for i, charged in charges.items()} def projection_at_current_time(): charges = {i: charge(i) for i in range(5)} return project(c, charges, waiting_charges(charges), histories, terminal) def stop(reason): if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time())) try: while active or pending: if stops:raise RuntimeError(stops[0]) if (root/'global-stop.json').exists():raise RuntimeError('EXTERNAL_GLOBAL_STOP') # Collect exits before granting any further admission. for i,(p,lease,log,owner) in list(active.items()): if p.poll() is None:continue log.close();jr=root/'jobs'/str(i);cp=jr/'complete.json' result=read(cp) if cp.exists() else dict(status='MISSING_GUARD_RECEIPT') worker_identity=jr/'worker/worker-identity.json' if worker_identity.exists() and alive(read(worker_identity)): raise RuntimeError('WORKER_OUTLIVED_GUARD_LEASE_HELD') terminal[i]=dict(id=i,status=result['status'],returncode=p.returncode, gpu_seconds_charged=charge(i),pre_dispatch_wait_seconds=waits[i],guard_sha256=sha(cp) if cp.exists() else None) release(lease['token'],owner);del active[i] if result['status']!='PASS' or p.returncode!=0:raise RuntimeError('FOLD_FAILED_'+str(i)) for i in active: folder=root/'jobs'/str(i)/'worker/history' histories[i]=[read(p) for p in sorted(folder.glob('*.json'))] projection=projection_at_current_time() save(root/'latest-cost-projection.json',projection) if projection['status']!='PASS':raise RuntimeError('TOTAL_PROJECTION_OR_ACTUAL_CAP') for i in active: gate=root/'jobs'/str(i)/'first-epoch-projection.json' if histories.get(i) and not gate.exists(): save(gate,dict(status='PASS',contract_sha256=sha(root/'contract.json'),projection=projection)) save(root/'live-progress.json',dict(unix=time.time(),stage='ITEM_B_TRAINING',active=list(active),pending=pending, terminal=list(terminal.values()),charged_training_gpu_seconds=projection['actual_training_and_preparation_gpu_seconds'],postfit_started=False)) if pending: i=pending[0] decision,ok=monitor.sample('admission',lambda:acquire(root,i,sha(root/'contract.json'))) if monitor.failed:raise RuntimeError('THREE_ADMISSION_FAILURES') if not ok: time.sleep(1) continue lease,observation=decision save(root/'latest-admission.json',observation) # Acquisition includes bounded GPU queries; recheck cost before Popen. fresh=projection_at_current_time() if fresh['status']!='PASS': if lease:release(lease['token']) raise RuntimeError('CAP_DURING_ADMISSION') if lease: jr=root/'jobs'/str(i);jr.mkdir(exist_ok=False) waits[i]=time.monotonic()-start save(jr/'assignment.json',dict(id=i,contract_sha256=sha(root/'contract.json'), gpu_uuid=lease['gpu_uuid'],worker_cpus=lease['worker_cpus'],token=lease['token'], queue_identity=identity,pre_dispatch_wait_seconds=waits[i])) log=(jr/'guard.log').open('x') env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1') p=subprocess.Popen([sys.executable,'tools/phase3_item_b_shared_guard.py',str(root),str(i),sha(jr/'assignment.json')], cwd=str(Path(__file__).resolve().parents[1]),env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True) owner=process_identity(p.pid) active[i]=(p,lease,log,owner);pending.remove(i) transfer(lease['token'],owner) save(jr/'lease-ready.json',dict(token=lease['token'],owner=owner)) time.sleep(1) except BaseException as exc: failure=type(exc).__name__+': '+str(exc);stop(failure) finally: if active: stop(failure or 'QUEUE_CLEANUP') for p,lease,log,owner in active.values(): if p.poll() is None:p.terminate() deadline=time.monotonic()+45 while time.monotonic()c['gpu_cap']:failure=failure or 'FINAL_CUMULATIVE_CAP' complete=not failure and len(terminal)==5 and all(r['status']=='PASS' and r['returncode']==0 for r in terminal.values()) files=[root/'contract.json'] if complete: models=[] for i in range(5): r=read(root/'jobs'/str(i)/'worker/result.json') models.append(dict(r));files.append(Path(r['checkpoint'])) files.extend([root/'jobs'/str(i)/'complete.json',root/'jobs'/str(i)/'worker/result.json',root/'jobs'/str(i)/'worker/standardization.json']) save(root/'models.json',models);files.append(root/'models.json') save(root/'complete.json',dict(status='TRAIN_COMPLETE_POSTFIT_PENDING' if complete else 'STOPPED_INCOMPLETE', failure=failure,entries=list(terminal.values()),never_attempted=pending, never_attempted_wait_charges={str(i):final_charges[i] for i in pending}, cumulative_gpu_seconds=final_cost,actual_waiting_gpu_seconds=sum(waiting_charges(final_charges).values()), postfit_gpu_reserve=c['postfit_gpu_reserve'], no_retry=True,test_targets_read=False,prediction_started=False,scoring_started=False)) files.append(root/'complete.json') freeze_start=time.monotonic() freeze(root,'training-terminal' if complete else 'training-stop',files) final_cost+=time.monotonic()-freeze_start within=final_cost+c['postfit_gpu_reserve']<=c['gpu_cap'] save(root/'post-terminal-cost.json',dict(cumulative_gpu_seconds=final_cost,within_cap=within, freeze_tail_charged_once=True,postfit_gpu_reserve=c['postfit_gpu_reserve'],postfit_started=False, training_complete_sha256=sha(root/'complete.json'),required_before_any_postfit=True)) return 0 if complete and within else 1 if __name__=='__main__':raise SystemExit(run(ROOT))