"""Run one exact-reviewed candidate postfit within the original cumulative budgets.""" import fcntl import os from pathlib import Path import resource import sys import time from phase2_h1_worker_v2 import save, sha from phase2_home_freeze_io import freeze from phase3_followup_postfit_common import ( CLEANUP_CPU, FOLD_CPU, FOLD_GPU, POSTFIT_CPU, POSTFIT_GPU, SCORE_CPU, SETUP_CPU, TRAINING, budget, config, prediction_identity, read, training_terminal, ) from phase3_followup_postfit_guard import run_prediction from r4bc_guard import supervise def run(root): started_wall = time.monotonic() started_cpu = time.process_time() child_start = resource.getrusage(resource.RUSAGE_CHILDREN) c, tc, _, by = config(root) item = c['item'] evidence = training_terminal(item, tc) if evidence != c['training'] or c['item_gpu_cap'] != tc['gpu_cap']: raise ValueError('Original training-to-postfit evidence changed') lock = (root/'run.lock').open('a') fcntl.flock(lock, fcntl.LOCK_EX|fcntl.LOCK_NB) gpu_lock = Path(f"/tmp/meltmind-candidate-postfit-{c['gpu_uuid']}.lock").open('a') fcntl.flock(gpu_lock, fcntl.LOCK_EX|fcntl.LOCK_NB) if (root/'started.json').exists(): raise FileExistsError('One candidate postfit attempt') os.sched_setaffinity(0,c['coordinator_cpus']) os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True' counters = dict(self_process_time=started_cpu, children_user=child_start.ru_utime, children_system=child_start.ru_stime) train = TRAINING[item][0] prior = evidence['training_gpu_seconds'] accrued_gpu = 0. accrued_child = 0. in_flight = None records = [] def cpu(): child = resource.getrusage(resource.RUSAGE_CHILDREN) reaped = child.ru_utime + child.ru_stime - child_start.ru_utime - child_start.ru_stime return SETUP_CPU + time.process_time() - started_cpu + max(accrued_child,reaped) def ledger(): return dict(records=records,prediction_gpu_seconds=accrued_gpu, postfit_cpu_seconds=cpu(),training_gpu_seconds_once=prior, cumulative_item_gpu_seconds=prior+accrued_gpu,new_fits=0,new_decodes=0,no_retry=True) budget(c,0,cpu(),5) save(root/'started.json',dict(item=item,pid=os.getpid(),unix=time.time(),training_gpu_seconds_once=prior, postfit_gpu_cap=POSTFIT_GPU,postfit_cpu_cap=POSTFIT_CPU,cpu_start_counters=counters, review_sha256=sha(root/'review.json'),new_fits=0,new_decodes=0)) try: save(root/'models-adopted.json',evidence) freeze(root,'models',[root/'contract.json',root/'models-adopted.json',train/'complete.json', train/'training-terminal-git-freeze.json',train/'post-terminal-cost.json', *[Path(m[k]) for m in evidence['models'] for k in ('path','standardization')]]) predictions = [] for fold in range(5): accrued_gpu = max(accrued_gpu,60+time.monotonic()-started_wall) budget(c,accrued_gpu,cpu(),5-fold) (root/'jobs'/str(fold)).mkdir(parents=True,exist_ok=False) in_flight = time.monotonic() guard = run_prediction(root,c,fold,prior+accrued_gpu,FOLD_CPU) elapsed = max(60+time.monotonic()-in_flight,guard['gpu_seconds_charged']) in_flight = None accrued_gpu += elapsed accrued_child += guard['worker']['cpu_seconds'] if guard['worker'] else 0. records.append(dict(fold=fold,charged_gpu_seconds=elapsed,guard=guard)) save(root/'resource-progress.json',ledger()) if guard['status'] != 'PASS' or guard['monitor_errors'] or elapsed > FOLD_GPU: raise RuntimeError(f'Fold {fold} prediction stop') p = root/'jobs'/str(fold)/'prediction-records/manifest.json' predictions.extend(read(p)['entries']) prediction_identity(root,c,tc,by,predictions) path = root/'predictions-manifest.json' save(path,dict(entries=predictions,test_targets_read=False,new_fits=0)) freeze(root,'predictions',[root/'contract.json',root/'models-adopted.json',path, root/'resource-progress.json',*map(Path,c['code_sha256'])]) accrued_gpu = max(accrued_gpu,60+time.monotonic()-started_wall) budget(c,accrued_gpu,cpu(),0) g = supervise([sys.executable,'tools/phase3_followup_postfit_worker.py','score',str(root)], root/'score-guard',dict(cpu_seconds=SCORE_CPU,wall_seconds=1200, rss_bytes=16*2**30,disk_bytes=20*2**30)) accrued_child += g['cpu_seconds'] records.append(dict(mode='score',guard=g)) if (g['status']!='PASS' or g['returncode']!=0 or g['monitor_errors'] or cpu()+CLEANUP_CPU>POSTFIT_CPU): raise RuntimeError('Score or CPU limit') save(root/'complete.json',dict(status='COMPLETE',**ledger())) freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'score/result.json', root/'score/target-access.jsonl',root/'review.json']) final = cpu() save(root/'post-terminal-cost.json',dict(**(ledger() | {'postfit_cpu_seconds': final}), within_cap=final<=POSTFIT_CPU,training_complete_sha256=evidence['training_complete_sha256'], cpu_start_counters=counters)) if final>POSTFIT_CPU: raise RuntimeError('CPU cap reached during terminal freeze') except BaseException as exc: if in_flight is not None: accrued_gpu += 60+time.monotonic()-in_flight if not any(r.get('mode')=='score' for r in records): accrued_gpu = max(accrued_gpu,60+time.monotonic()-started_wall) if (root/'complete.json').exists(): save(root/'pre-failure-complete.json',read(root/'complete.json')) save(root/'complete.json',dict(status='STOPPED',reason=f'{type(exc).__name__}: {exc}',**ledger())) raise if __name__=='__main__': run(Path(sys.argv[1]))