"""Single labelled resume: reuse0, predict1..6, freeze all, unchanged score.""" import fcntl import os from pathlib import Path import resource import shutil import sys import time import psutil from phase2_gate_outcome_review import validate_review from phase2_h1_worker_v2 import checked, save, sha from phase2_home_freeze_io import freeze, verify from phase3_item1_postfit_common import read, manifest_identity, require_budget from phase3_item1_resume_common import config, adoption, reuse_entries, cumulative_cost from phase3_item1_resume_guard import run_prediction from r4bc_guard import supervise def install_reuse(root, entries): result = [] for e in entries: original = checked(e['path'], e['sha256']) target = root / 'predictions' / e['arm'] / (e['case'] + '.npz') target.parent.mkdir(parents=True, exist_ok=True) if target.exists(): raise FileExistsError('No duplicate prediction adoption') shutil.copyfile(original, target) checked(target, e['sha256']) result.append(dict(e, path=str(target), reuse_original_path=str(original), reused_without_prediction=True)) out = root / 'jobs/0/prediction-records' out.mkdir(parents=True, exist_ok=False) save(out / 'reuse.json', dict(status='REUSED_BY_HASH_NO_WORKER', entries=result, targets_read=False)) return result def run(root): gpu_clock = time.monotonic() start = time.process_time() u0 = resource.getrusage(resource.RUSAGE_CHILDREN) c, tc, pilot = config(root) validate_review(root) adopted = adoption(c, tc) lock = (root / 'run.lock').open('a') fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) if (root / 'started.json').exists(): raise FileExistsError('One labelled recovery, no repeat launch') for p in psutil.process_iter(['pid', 'cmdline']): if p.pid != os.getpid() and any(Path(a).name in {'phase3_item1_postfit.py', 'phase3_item1_resume.py', 'phase3_item1_postfit_worker.py'} for a in p.info['cmdline'] or []): raise RuntimeError('Existing original or resumed postfit') gpu_lock = Path('/tmp/meltmind-item1-resume-' + c['gpu_uuid'] + '.lock').open('a') fcntl.flock(gpu_lock, fcntl.LOCK_EX | fcntl.LOCK_NB) os.sched_setaffinity(0, c['coordinator_cpus']) os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True' new_gpu = 0. charged_children = 0. records = [] prediction_start = None def costs(): u = resource.getrusage(resource.RUSAGE_CHILDREN) children = u.ru_utime + u.ru_stime - u0.ru_utime - u0.ru_stime new_cpu = 180 + time.process_time() - start + max(charged_children, children) return cumulative_cost(c, new_gpu, new_cpu) def budget(remaining): d = costs() require_budget(c['training_gpu_once'], d['prediction_gpu_seconds'], d['postfit_cpu_seconds'], remaining) return d save(root / 'started.json', dict(pid=os.getpid(), unix=time.time(), recovery_attempt=1, original_root=c['original_root'], prior_postfit_gpu=c['prior_postfit_gpu'], prior_postfit_cpu=c['prior_postfit_cpu'], training_gpu_seconds_once=c['training_gpu_once'], cpu_setup_allowance=180, review_sha256=sha(root / 'review.json'), indices=c['resume_indices'], gpu_uuid=c['gpu_uuid'], new_fits=0, new_decodes=0)) try: budget(6) entries = install_reuse(root, reuse_entries(c, tc)) save(root / 'models-adopted.json', adopted) freeze(root, 'models', [root / 'contract.json', root / 'models-adopted.json', Path(c['original_root']) / 'models-adopted.json', *[Path(m['checkpoint']) for m in adopted['models']], *[Path(m['standardization']) for m in adopted['models']]]) for index in c['resume_indices']: new_gpu = max(new_gpu, 60 + time.monotonic() - gpu_clock) budget(7 - index) (root / 'jobs' / str(index)).mkdir(parents=True, exist_ok=False) prediction_start = time.monotonic() g = run_prediction(root, c, pilot, index, c['training_gpu_once'] + costs()['prediction_gpu_seconds'], 300) cost = max(60 + time.monotonic() - prediction_start, g['gpu_seconds_charged']) prediction_start = None new_gpu += cost charged_children += g['worker']['cpu_seconds'] if g['worker'] else 0. records.append(dict(mode='predict', index=index, charged_gpu_seconds=cost, guard=g)) d = costs() save(root / 'resource-progress.json', dict(records=records, **d, original_complete_sha256=c['original_complete_sha256'])) if g['status'] != 'PASS' or cost > 900 or d['prediction_gpu_seconds'] > 7200 or d['cumulative_item_gpu_seconds'] > 120000: raise RuntimeError('Recovery prediction/resource guard failure') entries.extend(read(root / 'jobs' / str(index) / 'prediction-records/manifest.json')['entries']) manifest_identity(entries, c, tc, pilot) save(root / 'predictions-manifest.json', dict(entries=entries, predict0_reused_by_hash=686, S1_test_predictions_reused=121, new_fits=0, targets_read=False)) freeze(root, 'predictions', [root / 'contract.json', root / 'models-adopted.json', root / 'predictions-manifest.json', root / 'resource-progress.json', root / 'jobs/0/prediction-records/reuse.json', *[Path(p) for p in c['code_sha256']]]) new_gpu = max(new_gpu, 60 + time.monotonic() - gpu_clock) budget(0) validate_review(root) verify(root, 'models') verify(root, 'predictions') g = supervise([sys.executable, 'tools/phase3_item1_postfit_worker.py', 'score', str(root)], root / 'score-guard', dict(cpu_seconds=900, wall_seconds=1200, rss_bytes=16*2**30, disk_bytes=20*2**30)) charged_children += g['cpu_seconds'] records.append(dict(mode='score', guard=g)) if g['status'] != 'PASS' or costs()['postfit_cpu_seconds'] + 120 > 3600: raise RuntimeError('Score/resource failure') save(root / 'complete.json', dict(status='COMPLETE', records=records, **costs(), original_complete_sha256=c['original_complete_sha256'], predict0_reused=True, resumed_indices=c['resume_indices'], item_gpu_cap=120000, new_fits=0, new_decodes=0, future_phase3_started=False, no_retry=True)) freeze(root, 'terminal', [root / 'contract.json', root / 'complete.json', root / 'score/result.json', root / 'review.json']) final = costs() within = final['postfit_cpu_seconds'] <= 3600 and final['prediction_gpu_seconds'] <= 7200 and final['cumulative_item_gpu_seconds'] <= 120000 save(root / 'post-terminal-cost.json', dict(**final, within_cap=within)) if not within: raise RuntimeError('Final cumulative resource cap') except BaseException as exc: if prediction_start is not None: new_gpu += 60 + time.monotonic() - prediction_start if (root / 'complete.json').exists(): save(root / 'pre-failure-complete.json', read(root / 'complete.json')) save(root / 'complete.json', dict(status='STOPPED', reason=type(exc).__name__ + ': ' + str(exc), records=records, **costs(), original_complete_sha256=c['original_complete_sha256'], new_fits=0, no_retry=True, recovery_attempt=1)) raise if __name__ == '__main__': run(Path(sys.argv[1]))