Files
MeltMind/tools/phase3_data_sufficiency_queue.py
T

92 lines
7.0 KiB
Python

"""Single four-cell queue; first-epoch and periodic cost gates; checkpoint-only terminal."""
import fcntl,os,signal,subprocess,sys,time
from pathlib import Path
import psutil
from phase3_data_sufficiency_common import config,read,project,execution_command
from phase2_h1_worker_v2 import save,sha,checked
from phase2_home_freeze_io import freeze
from phase2_gate_outcome_review import validate_review
from s2_recovery_gpu_supervisor import tree_usage,disk_usage
def run(root):
c,pilot=config(root);validate_review(root);lock=(root/'queue.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('Single item1 attempt; no retry')
for p in psutil.process_iter(['pid','cmdline']):
if p.pid!=os.getpid() and execution_command(p.info['cmdline'] or []):raise RuntimeError('Existing item1 execution')
os.sched_setaffinity(0,c['coordinator_cpus']);os.environ['PYTORCH_CUDA_ALLOC_CONF']='expandable_segments:True';start=time.monotonic();cpu=time.process_time();active={};terminal={};requested=[];last_projection=-1
def stop(reason):
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time(),no_retry=True))
def charged():return sum(v['gpu_seconds_charged'] for v in terminal.values())+sum(60+time.monotonic()-v['start'] for v in active.values())
def terminate(v):
if v.get('stop_sent') is None:v['process'].send_signal(signal.SIGTERM);v['stop_sent']=time.monotonic()
elif time.monotonic()-v['stop_sent']>20:
try:
parent=psutil.Process(v['process'].pid)
for p in reversed(parent.children(recursive=True)):
try:p.kill()
except psutil.NoSuchProcess:pass
parent.kill()
except psutil.NoSuchProcess:pass
def collect(i,v):
rc=v['process'].poll()
if rc is None:return False
p=root/'jobs'/str(i)/'complete.json';r=read(p) if p.exists() else None;observed=60+time.monotonic()-v['start'];ok=rc==0 and r is not None and r['status']=='PASS'
terminal[i]=dict(id=i,status='PASS' if ok else 'STOPPED_OR_FAILED',returncode=rc,gpu_seconds_charged=max(observed,r['gpu_seconds_charged'] if r else 0),guard_complete_sha256=sha(p) if r else None);v['log'].close();del active[i]
save(root/'terminal-ledger.json',dict(entries=list(terminal.values()),settled_gpu_seconds=sum(x['gpu_seconds_charged'] for x in terminal.values())))
if not ok:stop('CELL_'+str(i)+'_FAILED')
return True
def observations():
result={}
for j in c['jobs']:
i=j['id'];rows=[read(p) for p in sorted((root/'jobs'/str(i)/'worker/history').glob('*.json'))]
full=[r for r in rows if r['full_epoch']]
if not full:return None
result[i]=dict(steps=rows[-1]['optimizer_steps'],charged=terminal[i]['gpu_seconds_charged'] if i in terminal else 60+time.monotonic()-active[i]['start'],max_epoch_seconds=max(r['train_epoch_seconds'] for r in full),max_validation_seconds=max(r['validation_seconds'] for r in rows))
return result
handlers={s:signal.signal(s,lambda n,f:requested.append('SIGNAL_'+str(n))) for s in [signal.SIGTERM,signal.SIGINT]}
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),review_sha256=sha(root/'review.json'),new_fits=4,S1_retrained=False,item_gpu_cap=86400,postfit_started=False))
try:
for j in c['jobs']:
i=j['id'];d=root/'jobs'/str(i);d.mkdir(parents=True,exist_ok=False);log=(d/'entry.log').open('x');begin=time.monotonic();cmd=[sys.executable,'tools/phase3_data_sufficiency_guard.py',str(root),str(i)];p=subprocess.Popen(cmd,stdout=log,stderr=subprocess.STDOUT,start_new_session=True);active[i]=dict(process=p,log=log,start=begin,stop_sent=None);save(d/'dispatch.json',dict(pid=p.pid,command=cmd,unix=time.time(),gpu_uuid=j['gpu_uuid']))
while active:
for i,v in list(active.items()):collect(i,v)
for x in requested:stop(x)
amount=charged();_,rss,_=tree_usage(os.getpid())
if amount>=c['train_gpu_cap']-80:stop('CUMULATIVE_TRAIN_GPU_RESERVE_BOUNDARY')
if rss>64*2**30 or disk_usage(root)>80*2**30:stop('AGGREGATE_MEMORY_DISK_LIMIT')
if not (root/'global-stop.json').exists():
obs=observations()
if obs is not None:
stage=min(int(obs[j['id']]['steps']/j['target_steps']*20) for j in c['jobs'])
if stage>last_projection:
p=project(c,obs);p.update(contract_sha256=sha(root/'contract.json'),unix=time.time());save(root/('first-epoch-projection.json' if last_projection<0 else 'latest-cost-projection.json'),p);last_projection=stage
if p['status']!='PASS':stop('PROJECTED_ITEM_GPU_BUDGET')
if (root/'global-stop.json').exists():
for v in active.values():terminate(v)
save(root/'live-progress.json',dict(stage='STOPPING' if (root/'global-stop.json').exists() else 'ITEM1_TRAINING',unix=time.time(),active=[dict(id=i,guard_pid=v['process'].pid) for i,v in active.items()],terminal=list(terminal.values()),charged_training_gpu_seconds=amount,postfit_gpu_reserve=c['postfit_gpu_reserve'],item_cap=86400,phase3_cap=612000))
if active:time.sleep(1)
if (root/'global-stop.json').exists() or len(terminal)!=4 or any(x['status']!='PASS' for x in terminal.values()):raise RuntimeError('Item1 train incomplete')
config(root);models=[]
for j in c['jobs']:
w=root/'jobs'/str(j['id'])/'worker';r=read(w/'result.json');stats=read(checked(w/'standardization.json',r['standardization_sha256']))
if r['optimizer_steps']!=j['target_steps'] or stats['parts']!=c['pools'][str(j['pool'])]['learning'] or r['test_targets_read']:raise ValueError('Terminal cell identity')
for m in r['checkpoints']:checked(m['path'],m['sha256']);models.append(dict(job=j['id'],pool=j['pool'],mode=j['mode'],**m))
if len(models)!=6 or charged()>c['train_gpu_cap']:raise ValueError('Checkpoint or cumulative cost')
save(root/'models-terminal.json',dict(models=models,reused_S1=c['S1_reuse'],new_fits=4,new_checkpoints=6,test_prediction_started=False))
freeze(root,'models',[root/'contract.json',root/'models-terminal.json',root/'terminal-ledger.json',*[Path(m['path']) for m in models],*[Path(m['standardization']) for m in models]])
save(root/'complete.json',dict(status='TRAIN_COMPLETE_POSTFIT_PENDING',entries=list(terminal.values()),training_gpu_seconds=charged(),postfit_gpu_reserve=c['postfit_gpu_reserve'],item_gpu_cap=86400,coordinator_cpu_seconds=time.process_time()-cpu,wall_seconds=time.monotonic()-start,no_retry=True,test_targets_read=False));freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'models-terminal.json']);return 0
except BaseException as exc:
stop(type(exc).__name__+': '+str(exc))
while active:
for i,v in list(active.items()):
try:
if not collect(i,v):terminate(v)
except Exception:
terminate(v)
if v['process'].poll() is not None:terminal[i]=dict(id=i,status='MALFORMED_TERMINAL',gpu_seconds_charged=60+time.monotonic()-v['start']);v['log'].close();del active[i]
if active:time.sleep(.5)
save(root/'complete.json',dict(status='STOPPED',reason=str(exc),entries=list(terminal.values()),training_gpu_seconds=charged(),postfit_started=False,no_retry=True));return 1
finally:
for s,h in handlers.items():signal.signal(s,h)
if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1])))