"""Explicit 4+155 infrastructure continuation. Original workers and scientific contract unchanged.""" import argparse import fcntl import json import math import os from pathlib import Path import resource import signal import subprocess import sys import time import traceback from freeze_g1_components import sha from s2_custody_common import checked, read_json from s2_recovery_supervisor import save, supervise, tree_usage from s2_recovery_monitoring import Monitor, disk_usage, budget_reason from run_s2_pilot_preparation import terminate from run_s2_r2 import finalize_selections from s3a_budget import first_fit_projection, can_dispatch from s3a_runtime import contract from s3a_recovery_common import state, canonical_accounting, selection_ready, manifest, ORDER FIRST = [0, 12] def dispatch_order(): return list(ORDER) def guard(root, index, slot): if index not in ORDER: raise ValueError('Original successful jobs may not be dispatched again') c = contract(root) os.sched_setaffinity(0, [c['worker_cpus'][slot]]) limits = c['limits'] dest = root/'jobs'/f'job-{index:03d}' dest.mkdir(parents=True, exist_ok=True) w = supervise([sys.executable, 'tools/run_s3a_fit.py', str(root), str(index)], dest, 'worker', dict(total_cpu_seconds=limits['per_fit_cpu_seconds']+1, job_cpu_seconds=limits['per_fit_cpu_seconds'], job_wall_seconds=limits['per_fit_wall_seconds'], tree_rss_bytes=limits['per_fit_tree_rss_bytes'], disk_bytes=limits['per_fit_disk_bytes'], max_processes=1), poll=.2) path = dest/'output/result.json' result = read_json(path) if path.exists() else None if w['status'] == 'PASS' and (not result or result['status'] != 'COMPLETE'): w.update(status='STOPPED_OR_FAILED', stop_reason='MISSING_COMPLETE_RESULT') if w['status'] == 'PASS': checked(dest/'output/model.joblib', result['model_sha256']) save(dest/'guard-result.json', dict(worker=w, supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]), cpu_affinity=sorted(os.sched_getaffinity(0)), result_path=str(path) if path.exists() else None, result_sha256=sha(path) if path.exists() else None, job=c['jobs'][index])) return 0 if w['status'] == 'PASS' else 1 def first_check(c, done): samples = {} for i in FIRST: item = next(r for r in done if r['index'] == i) result = read_json(checked(item['result_path'], item['result_sha256'])) if item.get('result_path') else {} samples[c['jobs'][i]['arm']] = dict(status=item['worker']['status'], worker_cpu_seconds=item['worker']['cpu_seconds'], fit_cpu_seconds=result.get('fit_cpu_seconds', 0.)) return first_fit_projection(c, samples) def freeze_terminal(root, c, done, selections, reason, started, monitor, accounting_complete): _, scope, old, _ = state(root) merged = manifest(root, scope, old, done) save(root/'two-source-manifest.json', merged) attempts = {r['index']: r for r in done} finals = [] for job in c['jobs'][360:]: row = attempts.get(job['index']) passed = row is not None and row['worker']['status'] == 'PASS' result = read_json(checked(row['result_path'], row['result_sha256'])) if passed else None model_path = str(Path(row['result_path']).parent/'model.joblib') if passed else None if passed: checked(model_path, result['model_sha256']) finals.append(dict(cell=job['cell'], arm=job['arm'], fold=job['fold'], pool=job['pool'], status='PASS' if passed else 'NA', reason=None if passed else ((row['worker']['stop_reason'] or 'WORKER_EXIT_'+str(row['worker']['returncode'])) if row else 'NOT_RUN_'+str(reason)), finished_configurations=selections[job['cell']]['finished_configurations'] if selections else None, model_path=model_path, model_sha256=result['model_sha256'] if result else None)) complete = reason is None and len(attempts) == 390 and accounting_complete result = dict(status='ALL_S3A_FITS_TERMINAL' if complete else 'S3A_STOPPED_INCOMPLETE', contract_sha256=sha(root/'contract.json'), stop_reason=reason, accounting_complete=accounting_complete, final_fits=finals, jobs=done, not_run_indices=[i for i in range(390) if i not in attempts], worker_cpu_seconds=canonical_accounting(old, done), original_worker_cpu_seconds=old['worker_cpu_seconds'], original_aborted_jobs_retained=[230,232,233,234], two_source_manifest_sha256=sha(root/'two-source-manifest.json'), guard_cpu_seconds=old['guard_cpu_seconds']+sum(r['supervisor_cpu_seconds'] for r in done if r['index'] in ORDER), original_coordinator_cpu_seconds=old['coordinator_cpu_seconds'], coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]), disk_bytes=disk_usage(root)+disk_usage(Path(scope['original_root'])), wall_seconds=time.monotonic()-started, monitor_errors=monitor.errors, roster_sha256=c['roster_sha256'], test_predictions=False, test_scoring=False, retry=False, baseline_refits=0) save(root/'batch-complete.json', result) if complete: save(root/'terminal-freeze.json', dict(status='ALL_S3A_FITS_TERMINAL', contract_sha256=sha(root/'contract.json'), hpo_terminal_indices=list(range(360)), final_fits=finals, batch_receipt=str(root/'batch-complete.json'), batch_receipt_sha256=sha(root/'batch-complete.json'), test_predictions_started=False)) return 0 if complete else 1 def run(root): with (root/'batch.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) return run_locked(root) def run_locked(root): c, scope, old, original_passed = state(root) if (root/'batch-started.json').exists(): raise FileExistsError('No restart/retry of an attempted S3-A queue') os.sched_setaffinity(0, c['coordinator_cpus']) start = time.monotonic() save(root/'batch-started.json', dict(pid=os.getpid(), unix=time.time(), contract_sha256=sha(root/'contract.json'), implementation_freeze=read_json(root/'implementation-freeze.json'), cpu_affinity=sorted(os.sched_getaffinity(0)))) active, done = {}, list(original_passed) order = dispatch_order() cursor = 0 reason = None accounting_complete = True first_checked = True selections = None live_cpu = live_rss = disk = 0 monitor = Monitor(root/'monitor-errors.jsonl') env = dict(os.environ, OMP_NUM_THREADS='1', MKL_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='') def interrupted(*_): nonlocal reason reason = 'S3A_RECOVERY_COORDINATOR_INTERRUPTED' save(root/'signal-received.json', dict(signal_number=_[0], unix=time.time(), pid=os.getpid())) signal.signal(signal.SIGTERM, interrupted) signal.signal(signal.SIGINT, interrupted) def terminate_active(): terminate({k: (v['process'], v['log']) for k, v in active.items()}) def collect(): nonlocal reason, accounting_complete for slot, item in list(active.items()): code = item['process'].poll() if code is None: continue path = root/'jobs'/f"job-{item['index']:03d}"/'guard-result.json' if not path.exists(): reason = 'MISSING_GUARD_RECEIPT' accounting_complete = False save(root/'infrastructure-stop.json', dict(reason=reason, index=item['index'], returncode=code, last_observed_guard_tree_CPU_s=item.get('cpu_peak', 0.), accounting_complete=False)) else: row = read_json(path) if row['job'] != c['jobs'][item['index']] or not math.isfinite(row['worker']['cpu_seconds']) or row['worker']['cpu_seconds'] < 0: raise ValueError('Invalid job/accounting receipt') if row.get('result_path'): checked(row['result_path'], row['result_sha256']) done.append(dict(index=item['index'], **row)) print(json.dumps(dict(event='S3A_TERMINAL', index=item['index'], arm=row['job']['arm'], status=row['worker']['status'], completed=len(done))), flush=True) item['log'].close() del active[slot] try: while active or (cursor < len(order) and reason is None): collect() def usage(): values = [] for v in active.values(): value = tree_usage(v['process'].pid) v['cpu_peak'] = max(v.get('cpu_peak', 0.), value[0]) values.append((v['cpu_peak'], value[1])) return sum(v[0] for v in values), sum(v[1] for v in values) (live_cpu, live_rss), fresh = monitor.sample('live_usage', usage, (live_cpu, live_rss)) disk, ok = monitor.sample('disk', lambda: disk_usage(root)+disk_usage(Path(scope['original_root'])), disk) fresh &= ok spent = canonical_accounting(old, done) over = budget_reason(0, spent, live_cpu, disk, cpu_limit=c['limits']['fitting_worker_cpu_seconds'], disk_limit=c['limits']['disk_bytes']) if over: reason = over if not first_checked and not active and done and any(r['worker']['status'] != 'PASS' for r in done): reason = 'FIRST_TRIAL_FAILED_NO_RETRY' if not first_checked and len(done) == 2 and not active and reason is None: check = first_check(c, done) save(root/'first-fit-cost-check.json', check) first_checked = True if not check['continue_within_cap']: reason = 'FIRST_TRIAL_COST_PROJECTION_OVER_CAP' if selection_ready(done, active) and selections is None and reason is None: selections = finalize_selections(root, c, done) if reason: terminate_active() ceiling = len(order)-30 if selections is None else len(order) parallel = 1 if not first_checked else 4 while fresh and reason is None and len(active) < parallel and cursor < ceiling: i = order[cursor] job = c['jobs'][i] if job['kind'] == 'FINAL' and selections[job['cell']]['status'] != 'SELECTED': done.append(dict(index=i, job=job, worker=dict(status='NA', cpu_seconds=0., returncode=0, stop_reason='ALL12_HPO_FAILED_NO_FINAL_FIT'), supervisor_cpu_seconds=0., attempted=False, result_path=None, result_sha256=None)) cursor += 1 continue # Conservatively reserve full caps of active workers plus the next task. cap = c['limits']['per_fit_cpu_seconds'] if not can_dispatch(c, spent, live_cpu, [cap for _ in active], cap): if not active: reason = 'INSUFFICIENT_REMAINING_FIT_CAP_NO_LAUNCH' break slot = next(k for k in range(4) if k not in active) dest = root/'jobs'/f'job-{i:03d}' dest.mkdir(parents=True, exist_ok=True) log = (dest/'guard.log').open('x') proc = subprocess.Popen([sys.executable, __file__, str(root), '--guard', str(i), '--slot', str(slot)], env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) active[slot] = dict(process=proc, log=log, index=i, cpu_peak=0.) cursor += 1 save(root/'progress.json', dict(completed=len(done), active={str(k): v['index'] for k,v in active.items()}, dispatch_cursor=cursor, worker_cpu_seconds=spent, live_guard_and_worker_cpu_seconds=live_cpu, active_tree_rss_bytes=live_rss, disk_bytes=disk, first_fit_checked=first_checked, selections_frozen=selections is not None, stop_reason=reason, fresh_accounting=fresh, monitor_errors=monitor.errors, elapsed_seconds=time.monotonic()-start)) if active or not fresh: time.sleep(.5) except Exception as exc: reason = 'COORDINATOR_EXCEPTION' save(root/'coordinator-failure.json', dict(type=type(exc).__name__, message=str(exc), traceback=traceback.format_exc())) finally: terminate_active() for item in active.values(): item['process'].wait() collect() return freeze_terminal(root, c, done, selections, reason, start, monitor, accounting_complete) if __name__ == '__main__': p = argparse.ArgumentParser() p.add_argument('root', type=Path) p.add_argument('--guard', type=int) p.add_argument('--slot', type=int) a = p.parse_args() raise SystemExit(guard(a.root, a.guard, a.slot) if a.guard is not None else run(a.root))