"""S3-A single-attempt CPU queue: two sequential witnesses then bounded four-way HPO.""" import argparse import fcntl import json import math import os from pathlib import Path import resource import signal import subprocess import sys import time import traceback from freeze_g1_components import sha from s2_custody_common import checked, read_json from s2_recovery_supervisor import save, supervise, tree_usage from s2_recovery_monitoring import Monitor, disk_usage, budget_reason from run_s2_pilot_preparation import terminate from run_s2_r2 import finalize_selections from s3a_budget import first_fit_projection, can_dispatch from s3a_runtime import contract FIRST = [0, 12] def dispatch_order(): return FIRST + [i for i in range(390) if i not in FIRST] def guard(root, index, slot): c = contract(root) os.sched_setaffinity(0, [c['worker_cpus'][slot]]) limits = c['limits'] dest = root/'jobs'/f'job-{index:03d}' dest.mkdir(parents=True, exist_ok=True) w = supervise([sys.executable, 'tools/run_s3a_fit.py', str(root), str(index)], dest, 'worker', dict(total_cpu_seconds=limits['per_fit_cpu_seconds']+1, job_cpu_seconds=limits['per_fit_cpu_seconds'], job_wall_seconds=limits['per_fit_wall_seconds'], tree_rss_bytes=limits['per_fit_tree_rss_bytes'], disk_bytes=limits['per_fit_disk_bytes'], max_processes=1), poll=.2) path = dest/'output/result.json' result = read_json(path) if path.exists() else None if w['status'] == 'PASS' and (not result or result['status'] != 'COMPLETE'): w.update(status='STOPPED_OR_FAILED', stop_reason='MISSING_COMPLETE_RESULT') if w['status'] == 'PASS': checked(dest/'output/model.joblib', result['model_sha256']) save(dest/'guard-result.json', dict(worker=w, supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]), cpu_affinity=sorted(os.sched_getaffinity(0)), result_path=str(path) if path.exists() else None, result_sha256=sha(path) if path.exists() else None, job=c['jobs'][index])) return 0 if w['status'] == 'PASS' else 1 def first_check(c, done): samples = {} for i in FIRST: item = next(r for r in done if r['index'] == i) result = read_json(checked(item['result_path'], item['result_sha256'])) if item.get('result_path') else {} samples[c['jobs'][i]['arm']] = dict(status=item['worker']['status'], worker_cpu_seconds=item['worker']['cpu_seconds'], fit_cpu_seconds=result.get('fit_cpu_seconds', 0.)) return first_fit_projection(c, samples) def freeze_terminal(root, c, done, selections, reason, started, monitor, accounting_complete): attempts = {r['index']: r for r in done} finals = [] for job in c['jobs'][360:]: row = attempts.get(job['index']) passed = row is not None and row['worker']['status'] == 'PASS' result = read_json(checked(row['result_path'], row['result_sha256'])) if passed else None model_path = str(Path(row['result_path']).parent/'model.joblib') if passed else None if passed: checked(model_path, result['model_sha256']) finals.append(dict(cell=job['cell'], arm=job['arm'], fold=job['fold'], pool=job['pool'], status='PASS' if passed else 'NA', reason=None if passed else ((row['worker']['stop_reason'] or 'WORKER_EXIT_'+str(row['worker']['returncode'])) if row else 'NOT_RUN_'+str(reason)), finished_configurations=selections[job['cell']]['finished_configurations'] if selections else None, model_path=model_path, model_sha256=result['model_sha256'] if result else None)) complete = reason is None and len(attempts) == 390 and accounting_complete result = dict(status='ALL_S3A_FITS_TERMINAL' if complete else 'S3A_STOPPED_INCOMPLETE', contract_sha256=sha(root/'contract.json'), stop_reason=reason, accounting_complete=accounting_complete, final_fits=finals, jobs=done, not_run_indices=[i for i in range(390) if i not in attempts], worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done), guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done), coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]), disk_bytes=disk_usage(root), wall_seconds=time.monotonic()-started, monitor_errors=monitor.errors, roster_sha256=c['roster_sha256'], test_predictions=False, test_scoring=False, retry=False, baseline_refits=0) save(root/'batch-complete.json', result) if complete: save(root/'terminal-freeze.json', dict(status='ALL_S3A_FITS_TERMINAL', contract_sha256=sha(root/'contract.json'), hpo_terminal_indices=list(range(360)), final_fits=finals, batch_receipt=str(root/'batch-complete.json'), batch_receipt_sha256=sha(root/'batch-complete.json'), test_predictions_started=False)) return 0 if complete else 1 def run(root): with (root/'batch.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) return run_locked(root) def run_locked(root): c = contract(root) if (root/'batch-started.json').exists(): raise FileExistsError('No restart/retry of an attempted S3-A queue') os.sched_setaffinity(0, c['coordinator_cpus']) start = time.monotonic() save(root/'batch-started.json', dict(pid=os.getpid(), unix=time.time(), contract_sha256=sha(root/'contract.json'), implementation_freeze=read_json(root/'implementation-freeze.json'), cpu_affinity=sorted(os.sched_getaffinity(0)))) active, done = {}, [] order = dispatch_order() cursor = 0 reason = None accounting_complete = True first_checked = False selections = None live_cpu = live_rss = disk = 0 monitor = Monitor(root/'monitor-errors.jsonl') env = dict(os.environ, OMP_NUM_THREADS='1', MKL_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='') def interrupted(*_): nonlocal reason reason = 'S3A_COORDINATOR_INTERRUPTED' signal.signal(signal.SIGTERM, interrupted) signal.signal(signal.SIGINT, interrupted) def terminate_active(): terminate({k: (v['process'], v['log']) for k, v in active.items()}) def collect(): nonlocal reason, accounting_complete for slot, item in list(active.items()): code = item['process'].poll() if code is None: continue path = root/'jobs'/f"job-{item['index']:03d}"/'guard-result.json' if not path.exists(): reason = 'MISSING_GUARD_RECEIPT' accounting_complete = False save(root/'infrastructure-stop.json', dict(reason=reason, index=item['index'], returncode=code, last_observed_guard_tree_CPU_s=item.get('cpu_peak', 0.), accounting_complete=False)) else: row = read_json(path) if row['job'] != c['jobs'][item['index']] or not math.isfinite(row['worker']['cpu_seconds']) or row['worker']['cpu_seconds'] < 0: raise ValueError('Invalid job/accounting receipt') if row.get('result_path'): checked(row['result_path'], row['result_sha256']) done.append(dict(index=item['index'], **row)) print(json.dumps(dict(event='S3A_TERMINAL', index=item['index'], arm=row['job']['arm'], status=row['worker']['status'], completed=len(done))), flush=True) item['log'].close() del active[slot] try: while active or (cursor < 390 and reason is None): collect() def usage(): values = [] for v in active.values(): value = tree_usage(v['process'].pid) v['cpu_peak'] = max(v.get('cpu_peak', 0.), value[0]) values.append((v['cpu_peak'], value[1])) return sum(v[0] for v in values), sum(v[1] for v in values) (live_cpu, live_rss), fresh = monitor.sample('live_usage', usage, (live_cpu, live_rss)) disk, ok = monitor.sample('disk', lambda: disk_usage(root), disk) fresh &= ok spent = sum(r['worker']['cpu_seconds'] for r in done) over = budget_reason(0, spent, live_cpu, disk, cpu_limit=c['limits']['fitting_worker_cpu_seconds'], disk_limit=c['limits']['disk_bytes']) if over: reason = over if not first_checked and not active and done and any(r['worker']['status'] != 'PASS' for r in done): reason = 'FIRST_TRIAL_FAILED_NO_RETRY' if not first_checked and len(done) == 2 and not active and reason is None: check = first_check(c, done) save(root/'first-fit-cost-check.json', check) first_checked = True if not check['continue_within_cap']: reason = 'FIRST_TRIAL_COST_PROJECTION_OVER_CAP' if cursor == 360 and not active and len(done) == 360 and selections is None and reason is None: selections = finalize_selections(root, c, done) if reason: terminate_active() ceiling = (len(done)+1 if len(done) < 2 else 2) if not first_checked else (360 if selections is None else 390) parallel = 1 if not first_checked else 4 while fresh and reason is None and len(active) < parallel and cursor < ceiling: i = order[cursor] job = c['jobs'][i] if job['kind'] == 'FINAL' and selections[job['cell']]['status'] != 'SELECTED': done.append(dict(index=i, job=job, worker=dict(status='NA', cpu_seconds=0., returncode=0, stop_reason='ALL12_HPO_FAILED_NO_FINAL_FIT'), supervisor_cpu_seconds=0., attempted=False, result_path=None, result_sha256=None)) cursor += 1 continue # Conservatively reserve full caps of active workers plus the next task. cap = c['limits']['per_fit_cpu_seconds'] if not can_dispatch(c, spent, live_cpu, [cap for _ in active], cap): if not active: reason = 'INSUFFICIENT_REMAINING_FIT_CAP_NO_LAUNCH' break slot = next(k for k in range(4) if k not in active) dest = root/'jobs'/f'job-{i:03d}' dest.mkdir(parents=True, exist_ok=True) log = (dest/'guard.log').open('x') proc = subprocess.Popen([sys.executable, __file__, str(root), '--guard', str(i), '--slot', str(slot)], env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) active[slot] = dict(process=proc, log=log, index=i, cpu_peak=0.) cursor += 1 save(root/'progress.json', dict(completed=len(done), active={str(k): v['index'] for k,v in active.items()}, dispatch_cursor=cursor, worker_cpu_seconds=spent, live_guard_and_worker_cpu_seconds=live_cpu, active_tree_rss_bytes=live_rss, disk_bytes=disk, first_fit_checked=first_checked, selections_frozen=selections is not None, stop_reason=reason, fresh_accounting=fresh, monitor_errors=monitor.errors, elapsed_seconds=time.monotonic()-start)) if active or not fresh: time.sleep(.5) except Exception as exc: reason = 'COORDINATOR_EXCEPTION' save(root/'coordinator-failure.json', dict(type=type(exc).__name__, message=str(exc), traceback=traceback.format_exc())) finally: terminate_active() for item in active.values(): item['process'].wait() collect() return freeze_terminal(root, c, done, selections, reason, start, monitor, accounting_complete) if __name__ == '__main__': p = argparse.ArgumentParser() p.add_argument('root', type=Path) p.add_argument('--guard', type=int) p.add_argument('--slot', type=int) a = p.parse_args() raise SystemExit(guard(a.root, a.guard, a.slot) if a.guard is not None else run(a.root))