Publish item b shared-mode budget and wait-accounting delta for review
This commit is contained in:
1 parent
482c490afb
commit
5bfef6a61a
16 files changed
+16580
No files matched your search
@@ -0,0 +1,152 @@
|
||||
"""Review-gated five-fold b training. Stops at frozen models, never postfit."""
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import psutil
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from phase2_home_freeze_io import freeze
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase3_item_b_shared_common import ROOT,config,read,project
|
||||
from phase3_item_b_shared_capacity import register_entry,acquire,transfer,release,process_identity,alive
|
||||
from track_a_recovery_monitor import Monitor
|
||||
|
||||
|
||||
def run(root):
|
||||
c,_=config(root);validate_review(root)
|
||||
with (root/'queue.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():raise FileExistsError('Sole b attempt, no resume')
|
||||
if os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':
|
||||
raise ValueError('Explicit allocator required')
|
||||
if psutil.disk_usage(root).free<100*2**30:raise RuntimeError('Less than100GiB free')
|
||||
register_entry(root,sha(root/'contract.json'),c)
|
||||
start=time.monotonic();identity=process_identity(os.getpid())
|
||||
save(root/'started.json',dict(unix=time.time(),identity=identity,contract_sha256=sha(root/'contract.json')))
|
||||
(root/'jobs').mkdir(exist_ok=False)
|
||||
active={};terminal={};pending=[0,1,2,3,4];histories={};waits={};stops=[];failure=None
|
||||
monitor=Monitor(root/'admission-errors.jsonl')
|
||||
for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,lambda n,f:stops.append('SIGNAL_'+str(n)))
|
||||
def charge(i):
|
||||
return terminal[i]['gpu_seconds_charged'] if i in terminal else 60+time.monotonic()-start
|
||||
def waiting_charges(charges):
|
||||
return {i: waits.get(i, max(0, charged - 60)) for i, charged in charges.items()}
|
||||
|
||||
def projection_at_current_time():
|
||||
charges = {i: charge(i) for i in range(5)}
|
||||
return project(c, charges, waiting_charges(charges), histories, terminal)
|
||||
|
||||
def stop(reason):
|
||||
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time()))
|
||||
try:
|
||||
while active or pending:
|
||||
if stops:raise RuntimeError(stops[0])
|
||||
if (root/'global-stop.json').exists():raise RuntimeError('EXTERNAL_GLOBAL_STOP')
|
||||
# Collect exits before granting any further admission.
|
||||
for i,(p,lease,log,owner) in list(active.items()):
|
||||
if p.poll() is None:continue
|
||||
log.close();jr=root/'jobs'/str(i);cp=jr/'complete.json'
|
||||
result=read(cp) if cp.exists() else dict(status='MISSING_GUARD_RECEIPT')
|
||||
worker_identity=jr/'worker/worker-identity.json'
|
||||
if worker_identity.exists() and alive(read(worker_identity)):
|
||||
raise RuntimeError('WORKER_OUTLIVED_GUARD_LEASE_HELD')
|
||||
terminal[i]=dict(id=i,status=result['status'],returncode=p.returncode,
|
||||
gpu_seconds_charged=charge(i),pre_dispatch_wait_seconds=waits[i],guard_sha256=sha(cp) if cp.exists() else None)
|
||||
release(lease['token'],owner);del active[i]
|
||||
if result['status']!='PASS' or p.returncode!=0:raise RuntimeError('FOLD_FAILED_'+str(i))
|
||||
for i in active:
|
||||
folder=root/'jobs'/str(i)/'worker/history'
|
||||
histories[i]=[read(p) for p in sorted(folder.glob('*.json'))]
|
||||
projection=projection_at_current_time()
|
||||
save(root/'latest-cost-projection.json',projection)
|
||||
if projection['status']!='PASS':raise RuntimeError('TOTAL_PROJECTION_OR_ACTUAL_CAP')
|
||||
for i in active:
|
||||
gate=root/'jobs'/str(i)/'first-epoch-projection.json'
|
||||
if histories.get(i) and not gate.exists():
|
||||
save(gate,dict(status='PASS',contract_sha256=sha(root/'contract.json'),projection=projection))
|
||||
save(root/'live-progress.json',dict(unix=time.time(),stage='ITEM_B_TRAINING',active=list(active),pending=pending,
|
||||
terminal=list(terminal.values()),charged_training_gpu_seconds=projection['actual_training_and_preparation_gpu_seconds'],postfit_started=False))
|
||||
if pending:
|
||||
i=pending[0]
|
||||
decision,ok=monitor.sample('admission',lambda:acquire(root,i,sha(root/'contract.json')))
|
||||
if monitor.failed:raise RuntimeError('THREE_ADMISSION_FAILURES')
|
||||
if not ok:
|
||||
time.sleep(1)
|
||||
continue
|
||||
lease,observation=decision
|
||||
save(root/'latest-admission.json',observation)
|
||||
# Acquisition includes bounded GPU queries; recheck cost before Popen.
|
||||
fresh=projection_at_current_time()
|
||||
if fresh['status']!='PASS':
|
||||
if lease:release(lease['token'])
|
||||
raise RuntimeError('CAP_DURING_ADMISSION')
|
||||
if lease:
|
||||
jr=root/'jobs'/str(i);jr.mkdir(exist_ok=False)
|
||||
waits[i]=time.monotonic()-start
|
||||
save(jr/'assignment.json',dict(id=i,contract_sha256=sha(root/'contract.json'),
|
||||
gpu_uuid=lease['gpu_uuid'],worker_cpus=lease['worker_cpus'],token=lease['token'],
|
||||
queue_identity=identity,pre_dispatch_wait_seconds=waits[i]))
|
||||
log=(jr/'guard.log').open('x')
|
||||
env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1')
|
||||
p=subprocess.Popen([sys.executable,'tools/phase3_item_b_shared_guard.py',str(root),str(i),sha(jr/'assignment.json')],
|
||||
cwd=str(Path(__file__).resolve().parents[1]),env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
||||
owner=process_identity(p.pid)
|
||||
active[i]=(p,lease,log,owner);pending.remove(i)
|
||||
transfer(lease['token'],owner)
|
||||
save(jr/'lease-ready.json',dict(token=lease['token'],owner=owner))
|
||||
time.sleep(1)
|
||||
except BaseException as exc:
|
||||
failure=type(exc).__name__+': '+str(exc);stop(failure)
|
||||
finally:
|
||||
if active:
|
||||
stop(failure or 'QUEUE_CLEANUP')
|
||||
for p,lease,log,owner in active.values():
|
||||
if p.poll() is None:p.terminate()
|
||||
deadline=time.monotonic()+45
|
||||
while time.monotonic()<deadline and any(p.poll() is None for p,_,_,_ in active.values()):time.sleep(.2)
|
||||
for i,(p,lease,log,owner) in active.items():
|
||||
if p.poll() is None:p.kill()
|
||||
p.wait();log.close();jr=root/'jobs'/str(i);cp=jr/'complete.json'
|
||||
result=read(cp) if cp.exists() else dict(status='MISSING_GUARD_RECEIPT')
|
||||
terminal[i]=dict(id=i,status=result['status'],returncode=p.returncode,gpu_seconds_charged=charge(i),pre_dispatch_wait_seconds=waits[i],
|
||||
guard_sha256=sha(cp) if cp.exists() else None)
|
||||
wp=jr/'worker/worker-identity.json'
|
||||
try:
|
||||
if wp.exists() and alive(read(wp)):raise RuntimeError('Live worker; lease retained')
|
||||
release(lease['token'],owner)
|
||||
except Exception as exc:
|
||||
terminal[i]['lease_release_failure']=str(exc);failure=failure or str(exc)
|
||||
final_charges={i:charge(i) for i in range(5)}
|
||||
final_cost=c['preparation_charge_gpu_seconds']+sum(final_charges.values())
|
||||
if final_cost+c['postfit_gpu_reserve']>c['gpu_cap']:failure=failure or 'FINAL_CUMULATIVE_CAP'
|
||||
complete=not failure and len(terminal)==5 and all(r['status']=='PASS' and r['returncode']==0 for r in terminal.values())
|
||||
files=[root/'contract.json']
|
||||
if complete:
|
||||
models=[]
|
||||
for i in range(5):
|
||||
r=read(root/'jobs'/str(i)/'worker/result.json')
|
||||
models.append(dict(r));files.append(Path(r['checkpoint']))
|
||||
files.extend([root/'jobs'/str(i)/'complete.json',root/'jobs'/str(i)/'worker/result.json',root/'jobs'/str(i)/'worker/standardization.json'])
|
||||
save(root/'models.json',models);files.append(root/'models.json')
|
||||
save(root/'complete.json',dict(status='TRAIN_COMPLETE_POSTFIT_PENDING' if complete else 'STOPPED_INCOMPLETE',
|
||||
failure=failure,entries=list(terminal.values()),never_attempted=pending,
|
||||
never_attempted_wait_charges={str(i):final_charges[i] for i in pending},
|
||||
cumulative_gpu_seconds=final_cost,actual_waiting_gpu_seconds=sum(waiting_charges(final_charges).values()),
|
||||
postfit_gpu_reserve=c['postfit_gpu_reserve'],
|
||||
no_retry=True,test_targets_read=False,prediction_started=False,scoring_started=False))
|
||||
files.append(root/'complete.json')
|
||||
freeze_start=time.monotonic()
|
||||
freeze(root,'training-terminal' if complete else 'training-stop',files)
|
||||
final_cost+=time.monotonic()-freeze_start
|
||||
within=final_cost+c['postfit_gpu_reserve']<=c['gpu_cap']
|
||||
save(root/'post-terminal-cost.json',dict(cumulative_gpu_seconds=final_cost,within_cap=within,
|
||||
freeze_tail_charged_once=True,postfit_gpu_reserve=c['postfit_gpu_reserve'],postfit_started=False,
|
||||
training_complete_sha256=sha(root/'complete.json'),required_before_any_postfit=True))
|
||||
return 0 if complete and within else 1
|
||||
|
||||
|
||||
if __name__=='__main__':raise SystemExit(run(ROOT))
|
||||
Reference in new issue
Block a user