"""Review-gated five-fold 1b training. Stops at frozen models, never postfit.""" import fcntl import json import os from pathlib import Path import signal import subprocess import sys import time import psutil from phase2_h1_worker_v2 import save,sha from phase2_home_freeze_io import freeze from phase2_gate_outcome_review import validate_review from phase3_item1b_common import ROOT,config,read,project from phase3_item1b_capacity import register_entry,acquire,transfer,release,process_identity,alive from track_a_recovery_monitor import Monitor def run(root): c,_=config(root);validate_review(root) with (root/'queue.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if (root/'started.json').exists():raise FileExistsError('Sole 1b attempt, no resume') if os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True': raise ValueError('Explicit allocator required') if psutil.disk_usage(root).free<100*2**30:raise RuntimeError('Less than100GiB free') register_entry(root,sha(root/'contract.json')) start=time.monotonic();identity=process_identity(os.getpid()) save(root/'started.json',dict(unix=time.time(),identity=identity,contract_sha256=sha(root/'contract.json'))) (root/'jobs').mkdir(exist_ok=False) active={};terminal={};pending=[1,0,2,3,4];histories={};stops=[];failure=None monitor=Monitor(root/'admission-errors.jsonl') for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,lambda n,f:stops.append('SIGNAL_'+str(n))) def charge(i): return terminal[i]['gpu_seconds_charged'] if i in terminal else 60+time.monotonic()-start def stop(reason): if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time())) try: while active or pending: if stops:raise RuntimeError(stops[0]) if (root/'global-stop.json').exists():raise RuntimeError('EXTERNAL_GLOBAL_STOP') # Collect exits before granting any further admission. for i,(p,lease,log,owner) in list(active.items()): if p.poll() is None:continue log.close();jr=root/'jobs'/str(i);cp=jr/'complete.json' result=read(cp) if cp.exists() else dict(status='MISSING_GUARD_RECEIPT') worker_identity=jr/'worker/worker-identity.json' if worker_identity.exists() and alive(read(worker_identity)): raise RuntimeError('WORKER_OUTLIVED_GUARD_LEASE_HELD') terminal[i]=dict(id=i,status=result['status'],returncode=p.returncode, gpu_seconds_charged=charge(i),guard_sha256=sha(cp) if cp.exists() else None) release(lease['token'],owner);del active[i] if result['status']!='PASS' or p.returncode!=0:raise RuntimeError('FOLD_FAILED_'+str(i)) for i in active: folder=root/'jobs'/str(i)/'worker/history' histories[i]=[read(p) for p in sorted(folder.glob('*.json'))] charges={i:charge(i) for i in range(5)} projection=project(c,charges,histories,terminal) save(root/'latest-cost-projection.json',projection) if projection['status']!='PASS':raise RuntimeError('TOTAL_PROJECTION_OR_ACTUAL_CAP') for i in active: gate=root/'jobs'/str(i)/'first-epoch-projection.json' if histories.get(i) and not gate.exists(): save(gate,dict(status='PASS',contract_sha256=sha(root/'contract.json'),projection=projection)) save(root/'live-progress.json',dict(unix=time.time(),stage='ITEM_1B_TRAINING',active=list(active),pending=pending, terminal=list(terminal.values()),charged_training_gpu_seconds=projection['actual_training_and_preparation_gpu_seconds'],postfit_started=False)) if pending: i=pending[0] decision,ok=monitor.sample('admission',lambda:acquire(root,i,sha(root/'contract.json'))) if monitor.failed:raise RuntimeError('THREE_ADMISSION_FAILURES') if not ok: time.sleep(1) continue lease,observation=decision save(root/'latest-admission.json',observation) # Acquisition includes bounded GPU queries; recheck cost before Popen. fresh=project(c,{j:charge(j) for j in range(5)},histories,terminal) if fresh['status']!='PASS': if lease:release(lease['token']) raise RuntimeError('CAP_DURING_ADMISSION') if lease: jr=root/'jobs'/str(i);jr.mkdir(exist_ok=False) save(jr/'assignment.json',dict(id=i,contract_sha256=sha(root/'contract.json'), gpu_uuid=lease['gpu_uuid'],worker_cpus=lease['worker_cpus'],token=lease['token'], queue_identity=identity,pre_dispatch_wait_seconds=time.monotonic()-start)) log=(jr/'guard.log').open('x') env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1') p=subprocess.Popen([sys.executable,'tools/phase3_item1b_guard.py',str(root),str(i),sha(jr/'assignment.json')], cwd=str(Path(__file__).resolve().parents[1]),env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True) owner=process_identity(p.pid) active[i]=(p,lease,log,owner);pending.remove(i) transfer(lease['token'],owner) save(jr/'lease-ready.json',dict(token=lease['token'],owner=owner)) time.sleep(1) except BaseException as exc: failure=type(exc).__name__+': '+str(exc);stop(failure) finally: if active: stop(failure or 'QUEUE_CLEANUP') for p,lease,log,owner in active.values(): if p.poll() is None:p.terminate() deadline=time.monotonic()+45 while time.monotonic()c['gpu_cap']:failure=failure or 'FINAL_CUMULATIVE_CAP' complete=not failure and len(terminal)==5 and all(r['status']=='PASS' and r['returncode']==0 for r in terminal.values()) files=[root/'contract.json'] if complete: models=[] for i in range(5): r=read(root/'jobs'/str(i)/'worker/result.json') models.append(dict(r));files.append(Path(r['checkpoint'])) files.extend([root/'jobs'/str(i)/'complete.json',root/'jobs'/str(i)/'worker/result.json',root/'jobs'/str(i)/'worker/standardization.json']) save(root/'models.json',models);files.append(root/'models.json') save(root/'complete.json',dict(status='TRAIN_COMPLETE_VALIDATION_SCREEN_PENDING' if complete else 'STOPPED_INCOMPLETE', failure=failure,entries=list(terminal.values()),never_attempted=pending, never_attempted_wait_charges={str(i):final_charges[i] for i in pending}, cumulative_gpu_seconds=final_cost,postfit_gpu_reserve=c['postfit_gpu_reserve'], no_retry=True,test_targets_read=False,prediction_started=False,scoring_started=False)) files.append(root/'complete.json') freeze_start=time.monotonic() freeze(root,'training-terminal' if complete else 'training-stop',files) final_cost+=time.monotonic()-freeze_start within=final_cost+c['postfit_gpu_reserve']<=c['gpu_cap'] save(root/'post-terminal-cost.json',dict(cumulative_gpu_seconds=final_cost,within_cap=within, freeze_tail_charged_once=True,postfit_gpu_reserve=c['postfit_gpu_reserve'],postfit_started=False, training_complete_sha256=sha(root/'complete.json'),required_before_any_postfit=True)) return 0 if complete and within else 1 if __name__=='__main__':raise SystemExit(run(ROOT))