"""Detached continuation of never-attempted parts; original science stays pinned.""" import argparse import fcntl import json import os from pathlib import Path import resource import signal import subprocess import sys import time import traceback sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from gre24_prepare_v2 import checked, sha from run_gre24_protocol_preparation import guard from s2_recovery_supervisor import save, stop_group def partition(order, old_records, remaining): old_ids = [r['index'] for r in old_records] if len(set(old_ids)) != len(old_ids) or len(set(remaining)) != len(remaining): raise ValueError('Duplicate attempt identity') expected = [i for i in order if i not in set(old_ids)] if remaining != expected or set(old_ids) | set(remaining) != set(order): raise ValueError('Continuation must contain exactly never-attempted indices') return expected def load_scope(root): c = checked(root) m = json.loads((root / 'continuation-manifest.json').read_text()) for path, digest in m['source_sha256'].items(): if sha(path) != digest: raise ValueError('Continuation source changed: ' + path) for name in ['contract.json', 'splits.json']: if sha(root / name) != m['input_sha256'][name]: raise ValueError('Original input changed: ' + name) if sha(m['original_audit']) != m['original_audit_sha256']: raise ValueError('Original interruption audit changed') old = [] for witness in m['original_records']: path = Path(witness['path']) if sha(path) != witness['sha256']: raise ValueError('Original guard receipt changed') r = json.loads(path.read_text()) if r['case'] != c['parts'][r['index']]['case']: raise ValueError('Original case/index mismatch') if r['worker']['status'] == 'PASS' and sha(r['result_path']) != r['result_sha256']: raise ValueError('Original successful result changed') old.append(r) partition(c['dispatch_order'], old, m['remaining_indices']) actual = sum(r['worker']['cpu_seconds'] for r in old) if abs(actual - m['original_worker_cpu_seconds']) > 1e-6: raise ValueError('Original CPU ledger mismatch') return c, m, old def terminal(c, m, old, new, attempted, reason, elapsed): combined = old + new ids = [r['index'] for r in combined] if len(set(ids)) != len(ids): raise ValueError('Duplicate canonical terminal') missing = sorted(set(attempted) - {r['index'] for r in new}) complete = not reason and not missing and len(combined) == len(c['parts']) return dict(status='COMPLETE' if complete else 'STOPPED', stop_reason=reason, records=sorted(combined, key=lambda r: r['index']), never_attempted=[i for i in m['remaining_indices'] if i not in attempted], missing_attempt_receipts=missing, accounting_complete=not missing, worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in combined), initial_cpu_reserve_seconds=c['limits']['preparation_initial_cpu_reserve_seconds'] + m['engineering_test_cpu_seconds'], old_worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in old), new_worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in new), coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]), wall_seconds=elapsed, provenance='Two-source canonical continuation; original complete remains absent', original_audit_sha256=m['original_audit_sha256']) def run(root): with (root / 'queue.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) return run_locked(root) def run_locked(root): c, m, old = load_scope(root) if (root / 'queue-started.json').exists(): raise FileExistsError('Never restart continuation') os.sched_setaffinity(0, c['coordinator_cpus']) save(root / 'queue-started.json', dict(pid=os.getpid(), unix=time.time(), contract_sha256=sha(root / 'contract.json'), manifest_sha256=sha(root / 'continuation-manifest.json'))) active = {}; done = []; attempted = []; remaining = m['remaining_indices']; reason = None start = time.monotonic() initial = c['limits']['preparation_initial_cpu_reserve_seconds'] + m['engineering_test_cpu_seconds'] cap = c['limits']['total_worker_cpu_seconds']; per = c['limits']['per_part_cpu_seconds'] def interrupted(*_): nonlocal reason reason = 'COORDINATOR_INTERRUPTED' signal.signal(signal.SIGTERM, interrupted); signal.signal(signal.SIGINT, interrupted) env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='') try: while active or (len(attempted) < len(remaining) and reason is None): for slot, item in list(active.items()): if item['process'].poll() is None: continue path = root / 'parts' / c['parts'][item['index']]['case'] / 'guard-result.json' if not path.exists(): raise RuntimeError('MISSING_GUARD_RECEIPT') record = json.loads(path.read_text()) if record['index'] != item['index'] or record['case'] != c['parts'][item['index']]['case']: raise ValueError('New guard identity mismatch') done.append(record); item['log'].close(); del active[slot] spent = initial + sum(r['worker']['cpu_seconds'] for r in old + done) while reason is None and len(active) < c['limits']['workers'] and len(attempted) < len(remaining): if spent + (len(active) + 1) * (per + 2) > cap: if not active: reason = 'REMAINING_CPU_INSUFFICIENT_FOR_ANOTHER_CAPPED_ATTEMPT' break slot = next(s for s in range(c['limits']['workers']) if s not in active) index = remaining[len(attempted)] dest = root / 'parts' / c['parts'][index]['case']; dest.mkdir(parents=True, exist_ok=False) log = (dest / 'guard.log').open('x') process = subprocess.Popen([sys.executable, __file__, str(root), '--guard', str(index), '--slot', str(slot)], stdout=log, stderr=subprocess.STDOUT, env=env, start_new_session=True) active[slot] = dict(process=process, log=log, index=index); attempted.append(index) save(root / 'progress.json', dict(completed=len(old) + len(done), old_terminal=len(old), new_terminal=len(done), new_attempted=len(attempted), attempted=len(old) + len(attempted), total=len(c['parts']), active={str(k): v['index'] for k, v in active.items()}, settled_worker_cpu_seconds=spent, old_worker_cpu_seconds=m['original_worker_cpu_seconds'], new_worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done), reserved_active_worker_cpu_seconds=len(active) * (per + 2), stop_reason=reason, elapsed_seconds=time.monotonic() - start, first_checkpoint_reused=True)) if active: time.sleep(.5) except Exception as exc: reason = 'CONTINUATION_INFRASTRUCTURE_EXCEPTION' save(root / 'coordinator-failure.json', dict(type=type(exc).__name__, message=str(exc), traceback=traceback.format_exc())) finally: for item in active.values(): stop_group(item['process'].pid); item['process'].wait(); item['log'].close() known = {r['index'] for r in done} for index in attempted: path = root / 'parts' / c['parts'][index]['case'] / 'guard-result.json' if index not in known and path.exists(): done.append(json.loads(path.read_text())) result = terminal(c, m, old, done, attempted, reason, time.monotonic() - start) save(root / 'complete.json', result) return 0 if result['status'] == 'COMPLETE' else 1 if __name__ == '__main__': p = argparse.ArgumentParser(); p.add_argument('root', type=Path) p.add_argument('--guard', type=int); p.add_argument('--slot', type=int, default=0) a = p.parse_args() if a.guard is not None: _, manifest, _ = load_scope(a.root) if a.guard not in manifest['remaining_indices']: raise ValueError('Cannot rerun any original attempt') raise SystemExit(run(a.root) if a.guard is None else guard(a.root, a.guard, a.slot))