Files
MeltMind/tools/phase3_item1b_queue.py
T

145 lines
9.2 KiB
Python

"""Review-gated five-fold 1b training. Stops at frozen models, never postfit."""
import fcntl
import json
import os
from pathlib import Path
import signal
import subprocess
import sys
import time
import psutil
from phase2_h1_worker_v2 import save,sha
from phase2_home_freeze_io import freeze
from phase2_gate_outcome_review import validate_review
from phase3_item1b_common import ROOT,config,read,project
from phase3_item1b_capacity import register_entry,acquire,transfer,release,process_identity,alive
from track_a_recovery_monitor import Monitor
def run(root):
c,_=config(root);validate_review(root)
with (root/'queue.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('Sole 1b attempt, no resume')
if os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':
raise ValueError('Explicit allocator required')
if psutil.disk_usage(root).free<100*2**30:raise RuntimeError('Less than100GiB free')
register_entry(root,sha(root/'contract.json'))
start=time.monotonic();identity=process_identity(os.getpid())
save(root/'started.json',dict(unix=time.time(),identity=identity,contract_sha256=sha(root/'contract.json')))
(root/'jobs').mkdir(exist_ok=False)
active={};terminal={};pending=[1,0,2,3,4];histories={};stops=[];failure=None
monitor=Monitor(root/'admission-errors.jsonl')
for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,lambda n,f:stops.append('SIGNAL_'+str(n)))
def charge(i):
return terminal[i]['gpu_seconds_charged'] if i in terminal else 60+time.monotonic()-start
def stop(reason):
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time()))
try:
while active or pending:
if stops:raise RuntimeError(stops[0])
if (root/'global-stop.json').exists():raise RuntimeError('EXTERNAL_GLOBAL_STOP')
# Collect exits before granting any further admission.
for i,(p,lease,log,owner) in list(active.items()):
if p.poll() is None:continue
log.close();jr=root/'jobs'/str(i);cp=jr/'complete.json'
result=read(cp) if cp.exists() else dict(status='MISSING_GUARD_RECEIPT')
worker_identity=jr/'worker/worker-identity.json'
if worker_identity.exists() and alive(read(worker_identity)):
raise RuntimeError('WORKER_OUTLIVED_GUARD_LEASE_HELD')
terminal[i]=dict(id=i,status=result['status'],returncode=p.returncode,
gpu_seconds_charged=charge(i),guard_sha256=sha(cp) if cp.exists() else None)
release(lease['token'],owner);del active[i]
if result['status']!='PASS' or p.returncode!=0:raise RuntimeError('FOLD_FAILED_'+str(i))
for i in active:
folder=root/'jobs'/str(i)/'worker/history'
histories[i]=[read(p) for p in sorted(folder.glob('*.json'))]
charges={i:charge(i) for i in range(5)}
projection=project(c,charges,histories,terminal)
save(root/'latest-cost-projection.json',projection)
if projection['status']!='PASS':raise RuntimeError('TOTAL_PROJECTION_OR_ACTUAL_CAP')
for i in active:
gate=root/'jobs'/str(i)/'first-epoch-projection.json'
if histories.get(i) and not gate.exists():
save(gate,dict(status='PASS',contract_sha256=sha(root/'contract.json'),projection=projection))
save(root/'live-progress.json',dict(unix=time.time(),stage='ITEM_1B_TRAINING',active=list(active),pending=pending,
terminal=list(terminal.values()),charged_training_gpu_seconds=projection['actual_training_and_preparation_gpu_seconds'],postfit_started=False))
if pending:
i=pending[0]
decision,ok=monitor.sample('admission',lambda:acquire(root,i,sha(root/'contract.json')))
if monitor.failed:raise RuntimeError('THREE_ADMISSION_FAILURES')
if not ok:
time.sleep(1)
continue
lease,observation=decision
save(root/'latest-admission.json',observation)
# Acquisition includes bounded GPU queries; recheck cost before Popen.
fresh=project(c,{j:charge(j) for j in range(5)},histories,terminal)
if fresh['status']!='PASS':
if lease:release(lease['token'])
raise RuntimeError('CAP_DURING_ADMISSION')
if lease:
jr=root/'jobs'/str(i);jr.mkdir(exist_ok=False)
save(jr/'assignment.json',dict(id=i,contract_sha256=sha(root/'contract.json'),
gpu_uuid=lease['gpu_uuid'],worker_cpus=lease['worker_cpus'],token=lease['token'],
queue_identity=identity,pre_dispatch_wait_seconds=time.monotonic()-start))
log=(jr/'guard.log').open('x')
env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1')
p=subprocess.Popen([sys.executable,'tools/phase3_item1b_guard.py',str(root),str(i),sha(jr/'assignment.json')],
cwd=str(Path(__file__).resolve().parents[1]),env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
owner=process_identity(p.pid)
active[i]=(p,lease,log,owner);pending.remove(i)
transfer(lease['token'],owner)
save(jr/'lease-ready.json',dict(token=lease['token'],owner=owner))
time.sleep(1)
except BaseException as exc:
failure=type(exc).__name__+': '+str(exc);stop(failure)
finally:
if active:
stop(failure or 'QUEUE_CLEANUP')
for p,lease,log,owner in active.values():
if p.poll() is None:p.terminate()
deadline=time.monotonic()+45
while time.monotonic()<deadline and any(p.poll() is None for p,_,_,_ in active.values()):time.sleep(.2)
for i,(p,lease,log,owner) in active.items():
if p.poll() is None:p.kill()
p.wait();log.close();jr=root/'jobs'/str(i);cp=jr/'complete.json'
result=read(cp) if cp.exists() else dict(status='MISSING_GUARD_RECEIPT')
terminal[i]=dict(id=i,status=result['status'],returncode=p.returncode,gpu_seconds_charged=charge(i),
guard_sha256=sha(cp) if cp.exists() else None)
wp=jr/'worker/worker-identity.json'
try:
if wp.exists() and alive(read(wp)):raise RuntimeError('Live worker; lease retained')
release(lease['token'],owner)
except Exception as exc:
terminal[i]['lease_release_failure']=str(exc);failure=failure or str(exc)
final_charges={i:charge(i) for i in range(5)}
final_cost=c['preparation_charge_gpu_seconds']+sum(final_charges.values())
if final_cost+c['postfit_gpu_reserve']>c['gpu_cap']:failure=failure or 'FINAL_CUMULATIVE_CAP'
complete=not failure and len(terminal)==5 and all(r['status']=='PASS' and r['returncode']==0 for r in terminal.values())
files=[root/'contract.json']
if complete:
models=[]
for i in range(5):
r=read(root/'jobs'/str(i)/'worker/result.json')
models.append(dict(r));files.append(Path(r['checkpoint']))
files.extend([root/'jobs'/str(i)/'complete.json',root/'jobs'/str(i)/'worker/result.json',root/'jobs'/str(i)/'worker/standardization.json'])
save(root/'models.json',models);files.append(root/'models.json')
save(root/'complete.json',dict(status='TRAIN_COMPLETE_VALIDATION_SCREEN_PENDING' if complete else 'STOPPED_INCOMPLETE',
failure=failure,entries=list(terminal.values()),never_attempted=pending,
never_attempted_wait_charges={str(i):final_charges[i] for i in pending},
cumulative_gpu_seconds=final_cost,postfit_gpu_reserve=c['postfit_gpu_reserve'],
no_retry=True,test_targets_read=False,prediction_started=False,scoring_started=False))
files.append(root/'complete.json')
freeze_start=time.monotonic()
freeze(root,'training-terminal' if complete else 'training-stop',files)
final_cost+=time.monotonic()-freeze_start
within=final_cost+c['postfit_gpu_reserve']<=c['gpu_cap']
save(root/'post-terminal-cost.json',dict(cumulative_gpu_seconds=final_cost,within_cap=within,
freeze_tail_charged_once=True,postfit_gpu_reserve=c['postfit_gpu_reserve'],postfit_started=False,
training_complete_sha256=sha(root/'complete.json'),required_before_any_postfit=True))
return 0 if complete and within else 1
if __name__=='__main__':raise SystemExit(run(ROOT))