"""Four-split Track A postfit after exact mixed-source review and training terminal verification.""" import fcntl,json,os,resource,sys,time from pathlib import Path import psutil from track_a_postfit_common import prediction_identity,budget from track_a_postfit_rebind_common import config,read,terminal,selected_training,SPLITS,RECOVERY,ENTRYPOINTS from phase2_gate_outcome_review import validate_review from phase2_home_freeze_io import freeze,verify from phase2_h1_worker_v2 import save,sha,checked from r4bc_guard import supervise def run(root): gpu_clock=time.monotonic();start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);c,tc,entry=config(root);validate_review(root);adopted=terminal(tc);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if adopted!=c['training_evidence']:raise ValueError('Training terminal changed since rebind') if (root/'started.json').exists():raise FileExistsError('One Track A postfit attempt, no retry') for p in psutil.process_iter(['pid','cmdline']): if p.pid!=os.getpid() and any(Path(x).name in ENTRYPOINTS for x in p.info['cmdline'] or []):raise RuntimeError('Existing postfit process') gpu_lock=Path('/tmp/meltmind-track-a-postfit-'+c['gpu_uuid']+'.lock').open('a');fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB) os.sched_setaffinity(0,c['coordinator_cpus']);os.environ['PYTORCH_CUDA_ALLOC_CONF']='expandable_segments:True';gpu=0.;charged=0.;records=[];prior=adopted['training_gpu_seconds_once'];prediction_start=None def cpu(): u=resource.getrusage(resource.RUSAGE_CHILDREN);return 180+time.process_time()-start+max(charged,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime) save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),training_gpu_seconds_once=prior,postfit_gpu_cap=7200,track_a_gpu_cap=162000,review_sha256=sha(root/'review.json'),cpu_start_counters=dict(self_process_time=start,children_user=u0.ru_utime,children_system=u0.ru_stime),cpu_setup_allowance=180,new_fits=0,new_decodes=0)) try: budget(prior,gpu,cpu(),len(SPLITS));save(root/'models-adopted.json',adopted);train=Path(c['training_root']);freeze(root,'models',[root/'contract.json',root/'models-adopted.json',train/'complete.json',RECOVERY/'complete.json',RECOVERY/'available-models-incomplete.json',RECOVERY/'incomplete-models-git-freeze.json',RECOVERY/'terminal-git-freeze.json',*[Path(m['path']) for m in adopted['models']],*[Path(m['standardization']) for m in adopted['models']]]) entries=[] from track_a_postfit_rebind_guard import run_prediction for position,index in enumerate(SPLITS): gpu=max(gpu,60+time.monotonic()-gpu_clock);budget(prior,gpu,cpu(),len(SPLITS)-position);(root/'jobs'/str(index)).mkdir(parents=True,exist_ok=False);prediction_start=time.monotonic();g=run_prediction(root,c,tc,index,prior+gpu,300);observed=60+time.monotonic()-prediction_start;prediction_start=None;cost=max(observed,g['gpu_seconds_charged']);gpu+=cost;charged+=g['worker']['cpu_seconds'] if g['worker'] else 0.;records.append(dict(mode='predict',index=index,charged_gpu_seconds=cost,guard=g));save(root/'resource-progress.json',dict(records=records,prediction_gpu_seconds=gpu,postfit_cpu_seconds=cpu(),cumulative_track_a_gpu_seconds=prior+gpu)) if g['status']!='PASS' or cost>900 or gpu>7200 or prior+gpu>162000:raise RuntimeError('Prediction/resource guard failure') m=read(root/'jobs'/str(index)/'prediction-records/manifest.json');entries.extend(m['entries']) prediction_identity(root,c,selected_training(tc),entries);save(root/'predictions-manifest.json',dict(entries=entries,A_predictions_reused=c['expected_pair_visits'],evaluated_splits=list(SPLITS),excluded_split=c['excluded_split'],new_fits=0,targets_read=False));freeze(root,'predictions',[root/'contract.json',root/'models-adopted.json',root/'predictions-manifest.json',root/'resource-progress.json',*[Path(p) for p in c['code_sha256']]]) gpu=max(gpu,60+time.monotonic()-gpu_clock);budget(prior,gpu,cpu(),0);validate_review(root);verify(root,'models');verify(root,'predictions') g=supervise([sys.executable,'tools/track_a_postfit_rebind_worker.py','score',str(root)],root/'score-guard',dict(cpu_seconds=900,wall_seconds=1200,rss_bytes=16*2**30,disk_bytes=20*2**30));charged+=g['cpu_seconds'];records.append(dict(mode='score',guard=g)) if g['status']!='PASS' or cpu()+120>3600:raise RuntimeError('Score/resource failure') save(root/'complete.json',dict(status='COMPLETE_FOUR_OF_FIVE',evaluated_splits=list(SPLITS),excluded_split=c['excluded_split'],records=records,prediction_gpu_seconds=gpu,postfit_cpu_seconds=cpu(),training_gpu_seconds_once=prior,cumulative_track_a_gpu_seconds=prior+gpu,track_a_gpu_cap=162000,new_fits=0,new_decodes=0,future_training_started=False,no_retry=True));freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'score/result.json',root/'review.json']) final=cpu();save(root/'post-terminal-cost.json',dict(postfit_cpu_seconds=final,prediction_gpu_seconds=gpu,cumulative_track_a_gpu_seconds=prior+gpu,within_cap=final<=3600 and gpu<=7200 and prior+gpu<=162000,cpu_start_counters=read(root/'started.json')['cpu_start_counters'])) if final>3600:raise RuntimeError('Final CPU cap after freeze') except BaseException as exc: if prediction_start is not None:gpu+=60+time.monotonic()-prediction_start if (root/'complete.json').exists():save(root/'pre-failure-complete.json',read(root/'complete.json')) save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),records=records,postfit_cpu_seconds=cpu(),prediction_gpu_seconds=gpu,training_gpu_seconds_once=prior,cumulative_track_a_gpu_seconds=prior+gpu,new_fits=0,no_retry=True));raise if __name__=='__main__':run(Path(sys.argv[1]))