"""Serial, shared-slot prediction followed by the original paired scoring barrier.""" import fcntl import os from pathlib import Path import resource import signal import subprocess import sys import time from phase2_gate_outcome_review import validate_review from phase2_h1_worker_v2 import save, sha from phase2_home_freeze_io import freeze, verify from phase3_shared_capacity import alive, process_identity, release, transfer from r4bc_guard import supervise from track_a_postfit_common import prediction_identity from track_a_postfit_rebind_common import RECOVERY, SPLITS, selected_training, terminal from track_a_postfit_shared_capacity import acquire, register from track_a_postfit_shared_common import ROOT, SETUP, budget, config, read def stop_guard(guard): if guard.poll() is not None: return guard.terminate() try: guard.wait(timeout=45) except subprocess.TimeoutExpired: guard.kill() guard.wait() def predict_fold(root, job, reservation, queue_identity, wait_seconds, resource_check): folder = root/'jobs'/str(job['id']) token = reservation['token'] guard = guard_id = None transferred = False try: folder.mkdir(exist_ok=False) path = folder/'assignment.json' save(path, dict(id=job['id'], contract_sha256=sha(root/'contract.json'), token=token, gpu_uuid=reservation['gpu_uuid'], worker_cpus=reservation['worker_cpus'], queue_identity=queue_identity, pre_dispatch_wait_seconds=wait_seconds)) with (folder/'guard.log').open('x') as log: guard = subprocess.Popen([sys.executable, 'tools/track_a_postfit_shared_guard.py', str(root), str(job['id']), sha(path)], stdout=log, stderr=subprocess.STDOUT, start_new_session=True) guard_id = process_identity(guard.pid) transfer(token, guard_id) transferred = True save(folder/'lease-ready.json', dict(token=token, owner=guard_id)) while guard.poll() is None: resource_check() time.sleep(0.5) receipt = read(folder/'complete.json') if (guard.returncode != 0 or receipt['status'] != 'PASS' or receipt['gpu_seconds_charged'] > job['gpu_seconds'] or receipt['monitor_policy']['failed']): raise RuntimeError(f'Shared prediction guard rejected fold {job["id"]}') return receipt finally: if guard is not None: stop_guard(guard) worker = folder/'prediction-records/worker-identity.json' if worker.exists() and alive(read(worker)): raise RuntimeError('Prediction worker outlived guard; retained own lease') release(token, guard_id if transferred else None) def run(root): gpu_clock = time.monotonic() started_cpu = time.process_time() initial_children = resource.getrusage(resource.RUSAGE_CHILDREN) c, tc, _ = config(root) validate_review(root) adopted = terminal(tc) if adopted != c['training_evidence']: raise ValueError('Mixed-source training evidence changed') prior = adopted['training_gpu_seconds_once'] charged_cpu = 0. frozen_gpu = None attempted = 0 records = [] stop_requests = [] original_affinity = os.sched_getaffinity(0) def gpu(): return (SETUP + attempted*SETUP + time.monotonic()-gpu_clock if frozen_gpu is None else frozen_gpu) def cpu(): children = resource.getrusage(resource.RUSAGE_CHILDREN) child_time = children.ru_utime + children.ru_stime - initial_children.ru_utime - initial_children.ru_stime return 180 + time.process_time() - started_cpu + max(charged_cpu, child_time) def resource_check(pending): if stop_requests: raise RuntimeError(stop_requests[0]) budget(c, gpu(), cpu(), pending) with (root/'run.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) if (root/'started.json').exists(): raise FileExistsError('One Track A shared postfit, zero retries') identity = process_identity(os.getpid()) digest = sha(root/'contract.json') register(root, digest, c) os.sched_setaffinity(0, c['coordinator_cpus']) os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True' for sig in (signal.SIGINT, signal.SIGTERM): signal.signal(sig, lambda n, f: stop_requests.append(f'COORDINATOR_SIGNAL_{n}')) cpu_counters = dict(self_process_time=started_cpu, children_user=initial_children.ru_utime, children_system=initial_children.ru_stime) save(root/'started.json', dict(identity=identity, unix=time.time(), contract_sha256=digest, review_sha256=sha(root/'review.json'), training_gpu_seconds_once=prior, shared_slot_limit=1, postfit_gpu_cap=c['postfit_gpu_cap'], track_a_gpu_cap=c['gpu_cap'], cpu_start_counters=cpu_counters, cpu_setup_allowance=180, new_fits=0, new_decodes=0)) try: save(root/'models-adopted.json', adopted) train = Path(c['training_root']) freeze(root, 'models', [root/'contract.json', root/'models-adopted.json', train/'complete.json', RECOVERY/'complete.json', RECOVERY/'available-models-incomplete.json', RECOVERY/'incomplete-models-git-freeze.json', RECOVERY/'terminal-git-freeze.json', *[Path(m['path']) for m in adopted['models']], *[Path(m['standardization']) for m in adopted['models']]]) entries = [] (root/'jobs').mkdir(exist_ok=False) jobs = c['predictions'] for pos, job in enumerate(jobs): wait_started = time.monotonic() while True: resource_check(jobs[pos:]) lease, observation = acquire(root, job['id'], digest, original_affinity) if lease is not None: break save(root/'latest-admission.json', observation) time.sleep(1) attempted += 1 receipt = predict_fold(root, job, lease, identity, time.monotonic()-wait_started, lambda: resource_check(jobs[pos+1:])) charged_cpu += receipt['worker']['cpu_seconds'] records.append(dict(mode='predict', split=job['id'], gpu_uuid=lease['gpu_uuid'], guard=receipt)) save(root/'resource-progress.json', dict(records=records, prediction_gpu_seconds=gpu(), postfit_cpu_seconds=cpu(), cumulative_track_a_gpu_seconds=prior+gpu())) entries.extend(read(root/'jobs'/str(job['id'])/'prediction-records/manifest.json')['entries']) prediction_identity(root, c, selected_training(tc), entries) save(root/'predictions-manifest.json', dict(entries=entries, A_predictions_reused=c['expected_pair_visits'], evaluated_splits=list(SPLITS), excluded_split=c['excluded_split'], new_fits=0, targets_read=False)) freeze(root, 'predictions', [root/'contract.json', root/'models-adopted.json', root/'predictions-manifest.json', root/'resource-progress.json', *map(Path,c['code_sha256'])]) resource_check([]) frozen_gpu = gpu() validate_review(root) verify(root, 'models') verify(root, 'predictions') score = supervise([sys.executable, 'tools/track_a_postfit_shared_worker.py', 'score', str(root)], root/'score-guard', dict(cpu_seconds=900, wall_seconds=1200, rss_bytes=16*2**30, disk_bytes=20*2**30)) charged_cpu += score['cpu_seconds'] records.append(dict(mode='score', guard=score)) if score['status'] != 'PASS' or cpu()+120 > c['postfit_cpu_cap']: raise RuntimeError('Original Track A scoring cap or guard') save(root/'complete.json', dict(status='COMPLETE_FOUR_OF_FIVE', evaluated_splits=list(SPLITS), excluded_split=c['excluded_split'], records=records, prediction_gpu_seconds=gpu(), postfit_cpu_seconds=cpu(), training_gpu_seconds_once=prior, cumulative_track_a_gpu_seconds=prior+gpu(), track_a_gpu_cap=c['gpu_cap'], new_fits=0, new_decodes=0, no_retry=True)) freeze(root, 'terminal', [root/'contract.json', root/'complete.json', root/'score/result.json', root/'review.json']) within = cpu()<=c['postfit_cpu_cap'] and gpu()<=c['postfit_gpu_cap'] and prior+gpu()<=c['gpu_cap'] save(root/'post-terminal-cost.json', dict(postfit_cpu_seconds=cpu(), prediction_gpu_seconds=gpu(), cumulative_track_a_gpu_seconds=prior+gpu(), within_cap=within, cpu_start_counters=cpu_counters)) if not within: raise RuntimeError('Shared Track A final resource cap') except BaseException as exc: if (root/'complete.json').exists(): save(root/'pre-failure-complete.json', read(root/'complete.json')) save(root/'complete.json', dict(status='STOPPED', reason=f'{type(exc).__name__}: {exc}', records=records, postfit_cpu_seconds=cpu(), prediction_gpu_seconds=gpu(), training_gpu_seconds_once=prior, cumulative_track_a_gpu_seconds=prior+gpu(), new_fits=0, no_retry=True)) raise if __name__ == '__main__': run(ROOT)