Files
MeltMind/tools/phase3_seed1_postfit.py
T

42 lines
5.9 KiB
Python

"""Seed1 postfit only after five frozen training terminals; exact review before test access."""
import fcntl,json,os,resource,sys,time
from pathlib import Path
import psutil
from phase3_seed1_postfit_common import config,read,terminal,prediction_identity,budget,OLD,PHASE3_PRIOR
from phase2_gate_outcome_review import validate_review
from phase2_home_freeze_io import freeze,verify
from phase2_h1_worker_v2 import save,sha,checked
from r4bc_guard import supervise
def run(root):
gpu_clock=time.monotonic();start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);c,tc,pilot=config(root);validate_review(root);adopted=terminal(root,c,tc);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('One Seed1 postfit attempt, no retry')
for p in psutil.process_iter(['pid','cmdline']):
if p.pid!=os.getpid() and any(Path(x).name in {'phase3_seed1_postfit.py','phase3_seed1_postfit_worker.py'} for x in p.info['cmdline'] or []):raise RuntimeError('Existing postfit process')
gpu_lock=Path('/tmp/meltmind-seed1-postfit-'+c['gpu_uuid']+'.lock').open('a');fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
os.sched_setaffinity(0,c['coordinator_cpus']);os.environ['PYTORCH_CUDA_ALLOC_CONF']='expandable_segments:True';gpu=0.;charged=0.;records=[];prior=adopted['new_training_gpu_seconds_once'];prediction_start=None
def cpu():
u=resource.getrusage(resource.RUSAGE_CHILDREN);return 180+time.process_time()-start+max(charged,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),new_training_gpu_seconds_once=prior,old_seed1_gpu_seconds_once=OLD,phase3_prior_gpu_once=PHASE3_PRIOR,postfit_gpu_cap=7200,seed1_gpu_cap=167200,phase3_gpu_cap=612000,review_sha256=sha(root/'review.json'),cpu_start_counters=dict(self_process_time=start,children_user=u0.ru_utime,children_system=u0.ru_stime),cpu_setup_allowance=180,new_fits=0,new_decodes=0))
try:
budget(prior,gpu,cpu(),5);save(root/'models-adopted.json',adopted);train=Path(c['training_root']);freeze(root,'models',[root/'contract.json',root/'models-adopted.json',train/'complete.json',train/'models-terminal.json',train/'models-git-freeze.json',train/'terminal-git-freeze.json',*[Path(m['path']) for m in adopted['models']],*[Path(m['standardization']) for m in adopted['models']]])
entries=[]
from phase3_seed1_postfit_guard import run_prediction
for index in range(5):
gpu=max(gpu,60+time.monotonic()-gpu_clock);budget(prior,gpu,cpu(),5-index);(root/'jobs'/str(index)).mkdir(parents=True,exist_ok=False);prediction_start=time.monotonic();g=run_prediction(root,c,tc,index,OLD+prior+gpu,300);observed=60+time.monotonic()-prediction_start;prediction_start=None;cost=max(observed,g['gpu_seconds_charged']);gpu+=cost;charged+=g['worker']['cpu_seconds'] if g['worker'] else 0.;records.append(dict(mode='predict',index=index,charged_gpu_seconds=cost,guard=g));save(root/'resource-progress.json',dict(records=records,prediction_gpu_seconds=gpu,postfit_cpu_seconds=cpu(),cumulative_seed1_gpu_seconds=OLD+prior+gpu,cumulative_phase3_gpu_seconds=PHASE3_PRIOR+prior+gpu))
if g['status']!='PASS' or cost>900 or gpu>7200 or OLD+prior+gpu>167200 or PHASE3_PRIOR+prior+gpu>612000:raise RuntimeError('Prediction/resource guard failure')
m=read(root/'jobs'/str(index)/'prediction-records/manifest.json');entries.extend(m['entries'])
prediction_identity(root,c,tc,entries);save(root/'predictions-manifest.json',dict(entries=entries,seed0_predictions_reused=552,new_fits=0,targets_read=False));freeze(root,'predictions',[root/'contract.json',root/'models-adopted.json',root/'predictions-manifest.json',root/'resource-progress.json',*[Path(p) for p in c['code_sha256']]])
gpu=max(gpu,60+time.monotonic()-gpu_clock);budget(prior,gpu,cpu(),0);validate_review(root);verify(root,'models');verify(root,'predictions')
g=supervise([sys.executable,'tools/phase3_seed1_postfit_worker.py','score',str(root)],root/'score-guard',dict(cpu_seconds=900,wall_seconds=1200,rss_bytes=16*2**30,disk_bytes=20*2**30));charged+=g['cpu_seconds'];records.append(dict(mode='score',guard=g))
if g['status']!='PASS' or g['returncode']!=0 or g['monitor_errors'] or cpu()+120>3600:raise RuntimeError('Score/resource failure')
save(root/'complete.json',dict(status='COMPLETE',records=records,prediction_gpu_seconds=gpu,postfit_cpu_seconds=cpu(),new_training_gpu_seconds_once=prior,old_seed1_gpu_seconds_once=OLD,phase3_prior_gpu_once=PHASE3_PRIOR,cumulative_seed1_gpu_seconds=OLD+prior+gpu,cumulative_phase3_gpu_seconds=PHASE3_PRIOR+prior+gpu,seed1_gpu_cap=167200,phase3_gpu_cap=612000,new_fits=0,new_decodes=0,future_training_started=False,no_retry=True));freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'score/result.json',root/'review.json'])
final=cpu();save(root/'post-terminal-cost.json',dict(postfit_cpu_seconds=final,prediction_gpu_seconds=gpu,cumulative_seed1_gpu_seconds=OLD+prior+gpu,cumulative_phase3_gpu_seconds=PHASE3_PRIOR+prior+gpu,within_cap=final<=3600 and gpu<=7200 and OLD+prior+gpu<=167200 and PHASE3_PRIOR+prior+gpu<=612000,cpu_start_counters=read(root/'started.json')['cpu_start_counters']))
if final>3600:raise RuntimeError('Final CPU cap after freeze')
except BaseException as exc:
if prediction_start is not None:gpu+=60+time.monotonic()-prediction_start
gpu=max(gpu,60+time.monotonic()-gpu_clock) if not any(r['mode']=='score' for r in records) else gpu
if (root/'complete.json').exists():save(root/'pre-failure-complete.json',read(root/'complete.json'))
save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),records=records,postfit_cpu_seconds=cpu(),prediction_gpu_seconds=gpu,new_training_gpu_seconds_once=prior,old_seed1_gpu_seconds_once=OLD,phase3_prior_gpu_once=PHASE3_PRIOR,cumulative_seed1_gpu_seconds=OLD+prior+gpu,cumulative_phase3_gpu_seconds=PHASE3_PRIOR+prior+gpu,new_fits=0,no_retry=True));raise
if __name__=='__main__':run(Path(sys.argv[1]))