Files
MeltMind/tools/phase2_h2_s2_queue.py
T

101 lines
8.6 KiB
Python

"""One four-fold S2 queue, global GPU budget, first-epoch gate; stops before postfit."""
import fcntl,json,os,signal,subprocess,sys,time
from pathlib import Path
import psutil
from phase2_h2_s2_common import config,read,projection,execution_command
from phase2_h1_worker_v2 import save,sha,checked
from phase2_home_freeze_io import freeze,verify
from s2_recovery_gpu_supervisor import tree_usage,disk_usage
def run(root):
c,pilot=config(root);lock=(root/'queue.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('Single S2 queue, no retry')
for p in psutil.process_iter(['pid','cmdline']):
if p.pid!=os.getpid() and execution_command(p.info['cmdline'] or []):raise RuntimeError('Existing H2 execution process')
os.sched_setaffinity(0,c['coordinator_cpus']);os.environ['PYTORCH_CUDA_ALLOC_CONF']='expandable_segments:True';start=time.monotonic();cpu=time.process_time();active={};terminal={};requested=[];errors=[];timings={};projection_round=None
def stop(reason):
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time(),no_retry=True))
def charged():
return sum(v['gpu_seconds_charged'] for v in terminal.values())+sum(60+time.monotonic()-v['start'] for v in active.values())
def terminate(v):
if v.get('stop_sent') is None:
v['process'].send_signal(signal.SIGTERM);v['stop_sent']=time.monotonic()
elif time.monotonic()-v['stop_sent']>20:
try:
parent=psutil.Process(v['process'].pid);children=parent.children(recursive=True)
for p in reversed(children):
try:p.kill()
except psutil.NoSuchProcess:pass
parent.kill()
except psutil.NoSuchProcess:pass
def collect(fold,v):
p=v['process'];rc=p.poll()
if rc is None:return False
elapsed=60+time.monotonic()-v['start'];receipt_path=root/'folds'/str(fold)/'complete.json';d=read(receipt_path) if receipt_path.exists() else None
cost=max(elapsed,d['gpu_seconds_charged'] if d else 0);ok=rc==0 and d is not None and d['status']=='PASS'
row=dict(fold=fold,status='PASS' if ok else 'STOPPED_OR_FAILED',returncode=rc,gpu_seconds_charged=cost,coordinator_observed_seconds=elapsed,guard_gpu_seconds=d['gpu_seconds_charged'] if d else None,complete_sha256=sha(receipt_path) if d else None,receipt_present=d is not None)
terminal[fold]=row;v['log'].close();del active[fold];save(root/'terminal-ledger.json',dict(entries=list(terminal.values()),settled_gpu_seconds=sum(x['gpu_seconds_charged'] for x in terminal.values()),prior_gpu_once=c['prior_gpu_seconds']))
if not ok:stop('FOLD_'+str(fold)+'_FAILED_OR_MISSING_TERMINAL')
return True
def observe(fold):
history=root/'folds'/str(fold)/'worker/history';rows=[]
for p in sorted(history.glob('*.json')):
d=read(p)
if d['epoch_parts']!=c['jobs'][fold-1]['learning_parts'] or d['epoch_nodes']!=c['jobs'][fold-1]['learning_nodes'] or d['optimizer_steps']!=d['epoch']*c['jobs'][fold-1]['learning_parts']:raise ValueError('Incomplete timing epoch evidence')
rows.append(d)
if not rows:return None
cost=terminal[fold]['gpu_seconds_charged'] if fold in terminal else 60+time.monotonic()-active[fold]['start']
return dict(epoch=rows[-1]['epoch'],charged_gpu_seconds=cost,max_train_epoch_seconds=max(r['train_epoch_seconds'] for r in rows),max_validation_seconds=max(r.get('validation_seconds',0) for r in rows))
old={s:signal.signal(s,lambda n,f:requested.append('COORDINATOR_SIGNAL_'+str(n))) for s in [signal.SIGTERM,signal.SIGINT]}
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),folds=[1,2,3,4],prior_gpu_once=c['prior_gpu_seconds'],prediction_gpu_reserve=c['prediction_gpu_reserve'],postfit_started=False))
try:
for job in c['jobs']:
fold=job['fold'];d=root/'folds'/str(fold);d.mkdir(parents=True,exist_ok=False);log=(d/'entry.log').open('x');begin=time.monotonic()
command=[sys.executable,'tools/phase2_h2_s2_guard.py',str(root),str(fold)];p=subprocess.Popen(command,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
active[fold]=dict(process=p,log=log,start=begin,stop_sent=None);save(d/'dispatch.json',dict(fold=fold,guard_pid=p.pid,command=command,unix=time.time(),gpu_uuid=job['gpu_uuid'],setup_gpu_seconds=60))
while active:
for fold,v in list(active.items()):collect(fold,v)
for reason in requested:stop(reason)
amount=charged();_,rss,_=tree_usage(os.getpid());used=disk_usage(root)
if amount>=c['train_gpu_cap']-80:stop('CUMULATIVE_GPU_RESERVE_BOUNDARY')
if rss>c['aggregate_rss_bytes']:stop('AGGREGATE_RSS_LIMIT')
if used>c['disk_bytes']:stop('TOTAL_OUTPUT_DISK_LIMIT')
if not (root/'global-stop.json').exists():
timings={f:observe(f) for f in range(1,5)}
if all(v is not None for v in timings.values()):
current_round=min(v['epoch'] for v in timings.values())//10
if projection_round is None or current_round>projection_round:
d=projection(c,timings);d.update(contract_sha256=sha(root/'contract.json'),unix=time.time());save(root/('first-epoch-projection.json' if projection_round is None else 'latest-cost-projection.json'),d)
projection_round=current_round
if d['status']!='PASS':stop('PROJECTED_GPU_BUDGET_EXCEEDED')
if (root/'global-stop.json').exists():
for v in active.values():terminate(v)
save(root/'live-progress.json',dict(stage='STOPPING' if (root/'global-stop.json').exists() else 'S2_TRAINING',unix=time.time(),active=[dict(fold=f,guard_pid=v['process'].pid) for f,v in active.items()],terminal=list(terminal.values()),new_gpu_seconds_charged=amount,cumulative_h2_gpu_seconds=c['prior_gpu_seconds']+amount,prediction_gpu_reserve=c['prediction_gpu_reserve'],train_gpu_cap=c['train_gpu_cap'],epoch_counts={str(f):(x['epoch'] if x else 0) for f,x in timings.items()},rss_bytes=rss,disk_bytes=used))
if active:time.sleep(1)
if (root/'global-stop.json').exists() or len(terminal)!=4 or any(r['status']!='PASS' for r in terminal.values()):raise RuntimeError('S2 training incomplete; originals retained, no retries')
config(root);models=[]
for j in c['jobs']:
fold=j['fold'];w=root/'folds'/str(fold)/'worker';r=read(w/'result.json');checkpoint=checked(r['checkpoint'],r['checkpoint_sha256']);stats=read(checked(w/'standardization.json',r['standardization_sha256']))
if r['fold']!=fold or r['completed_epochs']!=200 or r['optimizer_steps']!=200*j['learning_parts'] or r['test_targets_read'] is not False or stats['parts']!=pilot['folds'][fold]['learning']:raise ValueError('Final checkpoint/normalization role identity')
models.append(dict(fold=fold,checkpoint=str(checkpoint),checkpoint_sha256=r['checkpoint_sha256'],standardization=str(w/'standardization.json'),standardization_sha256=r['standardization_sha256'],worker_result_sha256=sha(w/'result.json')))
amount=charged()
if amount>c['train_gpu_cap'] or c['prior_gpu_seconds']+amount+c['prediction_gpu_reserve']>144000:raise RuntimeError('Final aggregate GPU limit')
save(root/'models-terminal.json',dict(models=models,fold0_reused_by_hash=c['fold0_reuse'],new_fits=4,test_targets_read=False,postfit_started=False));freeze(root,'models',[root/'contract.json',root/'models-terminal.json',root/'terminal-ledger.json',*[Path(m['checkpoint']) for m in models],*[Path(m['standardization']) for m in models]])
save(root/'complete.json',dict(status='TRAIN_COMPLETE_POSTFIT_PENDING',terminal=list(terminal.values()),new_gpu_seconds_charged=amount,prior_gpu_once=c['prior_gpu_seconds'],cumulative_h2_gpu_seconds=c['prior_gpu_seconds']+amount,prediction_gpu_reserve=c['prediction_gpu_reserve'],coordinator_cpu_seconds=time.process_time()-cpu,wall_seconds=time.monotonic()-start,new_fits=4,postfit_started=False,no_retry=True));freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'models-terminal.json'])
except BaseException as exc:
stop(type(exc).__name__+': '+str(exc));errors.append(str(exc))
while active:
for f,v in list(active.items()):
try:
if not collect(f,v):terminate(v)
except Exception as e:
errors.append(str(e));terminate(v)
if v['process'].poll() is not None:
terminal[f]=dict(fold=f,status='MALFORMED_TERMINAL',gpu_seconds_charged=60+time.monotonic()-v['start'],returncode=v['process'].returncode);v['log'].close();del active[f]
if active:time.sleep(.5)
amount=charged();save(root/'complete.json',dict(status='STOPPED',reason=str(exc),errors=errors,terminal=list(terminal.values()),new_gpu_seconds_charged=amount,prior_gpu_once=c['prior_gpu_seconds'],cumulative_h2_gpu_seconds=c['prior_gpu_seconds']+amount,coordinator_cpu_seconds=time.process_time()-cpu,postfit_started=False,no_retry=True));return 1
finally:
for s,h in old.items():signal.signal(s,h)
return 0
if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1])))