Files
MeltMind/tools/gre24_continuation/run_gre24_protocol_preparation.py

162 lines
8.4 KiB
Python

"""Detached continuation of never-attempted parts; original science stays pinned."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
import traceback
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from gre24_prepare_v2 import checked, sha
from run_gre24_protocol_preparation import guard
from s2_recovery_supervisor import save, stop_group
def partition(order, old_records, remaining):
old_ids = [r['index'] for r in old_records]
if len(set(old_ids)) != len(old_ids) or len(set(remaining)) != len(remaining):
raise ValueError('Duplicate attempt identity')
expected = [i for i in order if i not in set(old_ids)]
if remaining != expected or set(old_ids) | set(remaining) != set(order):
raise ValueError('Continuation must contain exactly never-attempted indices')
return expected
def load_scope(root):
c = checked(root)
m = json.loads((root / 'continuation-manifest.json').read_text())
for path, digest in m['source_sha256'].items():
if sha(path) != digest:
raise ValueError('Continuation source changed: ' + path)
for name in ['contract.json', 'splits.json']:
if sha(root / name) != m['input_sha256'][name]:
raise ValueError('Original input changed: ' + name)
if sha(m['original_audit']) != m['original_audit_sha256']:
raise ValueError('Original interruption audit changed')
old = []
for witness in m['original_records']:
path = Path(witness['path'])
if sha(path) != witness['sha256']:
raise ValueError('Original guard receipt changed')
r = json.loads(path.read_text())
if r['case'] != c['parts'][r['index']]['case']:
raise ValueError('Original case/index mismatch')
if r['worker']['status'] == 'PASS' and sha(r['result_path']) != r['result_sha256']:
raise ValueError('Original successful result changed')
old.append(r)
partition(c['dispatch_order'], old, m['remaining_indices'])
actual = sum(r['worker']['cpu_seconds'] for r in old)
if abs(actual - m['original_worker_cpu_seconds']) > 1e-6:
raise ValueError('Original CPU ledger mismatch')
return c, m, old
def terminal(c, m, old, new, attempted, reason, elapsed):
combined = old + new
ids = [r['index'] for r in combined]
if len(set(ids)) != len(ids):
raise ValueError('Duplicate canonical terminal')
missing = sorted(set(attempted) - {r['index'] for r in new})
complete = not reason and not missing and len(combined) == len(c['parts'])
return dict(status='COMPLETE' if complete else 'STOPPED', stop_reason=reason,
records=sorted(combined, key=lambda r: r['index']),
never_attempted=[i for i in m['remaining_indices'] if i not in attempted],
missing_attempt_receipts=missing, accounting_complete=not missing,
worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in combined),
initial_cpu_reserve_seconds=c['limits']['preparation_initial_cpu_reserve_seconds'] + m['engineering_test_cpu_seconds'],
old_worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in old),
new_worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in new),
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
wall_seconds=elapsed, provenance='Two-source canonical continuation; original complete remains absent',
original_audit_sha256=m['original_audit_sha256'])
def run(root):
with (root / 'queue.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
return run_locked(root)
def run_locked(root):
c, m, old = load_scope(root)
if (root / 'queue-started.json').exists():
raise FileExistsError('Never restart continuation')
os.sched_setaffinity(0, c['coordinator_cpus'])
save(root / 'queue-started.json', dict(pid=os.getpid(), unix=time.time(),
contract_sha256=sha(root / 'contract.json'), manifest_sha256=sha(root / 'continuation-manifest.json')))
active = {}; done = []; attempted = []; remaining = m['remaining_indices']; reason = None
start = time.monotonic()
initial = c['limits']['preparation_initial_cpu_reserve_seconds'] + m['engineering_test_cpu_seconds']
cap = c['limits']['total_worker_cpu_seconds']; per = c['limits']['per_part_cpu_seconds']
def interrupted(*_):
nonlocal reason
reason = 'COORDINATOR_INTERRUPTED'
signal.signal(signal.SIGTERM, interrupted); signal.signal(signal.SIGINT, interrupted)
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', MKL_NUM_THREADS='1',
NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
try:
while active or (len(attempted) < len(remaining) and reason is None):
for slot, item in list(active.items()):
if item['process'].poll() is None:
continue
path = root / 'parts' / c['parts'][item['index']]['case'] / 'guard-result.json'
if not path.exists():
raise RuntimeError('MISSING_GUARD_RECEIPT')
record = json.loads(path.read_text())
if record['index'] != item['index'] or record['case'] != c['parts'][item['index']]['case']:
raise ValueError('New guard identity mismatch')
done.append(record); item['log'].close(); del active[slot]
spent = initial + sum(r['worker']['cpu_seconds'] for r in old + done)
while reason is None and len(active) < c['limits']['workers'] and len(attempted) < len(remaining):
if spent + (len(active) + 1) * (per + 2) > cap:
if not active:
reason = 'REMAINING_CPU_INSUFFICIENT_FOR_ANOTHER_CAPPED_ATTEMPT'
break
slot = next(s for s in range(c['limits']['workers']) if s not in active)
index = remaining[len(attempted)]
dest = root / 'parts' / c['parts'][index]['case']; dest.mkdir(parents=True, exist_ok=False)
log = (dest / 'guard.log').open('x')
process = subprocess.Popen([sys.executable, __file__, str(root), '--guard', str(index), '--slot', str(slot)],
stdout=log, stderr=subprocess.STDOUT, env=env, start_new_session=True)
active[slot] = dict(process=process, log=log, index=index); attempted.append(index)
save(root / 'progress.json', dict(completed=len(old) + len(done),
old_terminal=len(old), new_terminal=len(done), new_attempted=len(attempted),
attempted=len(old) + len(attempted), total=len(c['parts']),
active={str(k): v['index'] for k, v in active.items()},
settled_worker_cpu_seconds=spent, old_worker_cpu_seconds=m['original_worker_cpu_seconds'],
new_worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done),
reserved_active_worker_cpu_seconds=len(active) * (per + 2), stop_reason=reason,
elapsed_seconds=time.monotonic() - start, first_checkpoint_reused=True))
if active:
time.sleep(.5)
except Exception as exc:
reason = 'CONTINUATION_INFRASTRUCTURE_EXCEPTION'
save(root / 'coordinator-failure.json', dict(type=type(exc).__name__, message=str(exc), traceback=traceback.format_exc()))
finally:
for item in active.values():
stop_group(item['process'].pid); item['process'].wait(); item['log'].close()
known = {r['index'] for r in done}
for index in attempted:
path = root / 'parts' / c['parts'][index]['case'] / 'guard-result.json'
if index not in known and path.exists():
done.append(json.loads(path.read_text()))
result = terminal(c, m, old, done, attempted, reason, time.monotonic() - start)
save(root / 'complete.json', result)
return 0 if result['status'] == 'COMPLETE' else 1
if __name__ == '__main__':
p = argparse.ArgumentParser(); p.add_argument('root', type=Path)
p.add_argument('--guard', type=int); p.add_argument('--slot', type=int, default=0)
a = p.parse_args()
if a.guard is not None:
_, manifest, _ = load_scope(a.root)
if a.guard not in manifest['remaining_indices']:
raise ValueError('Cannot rerun any original attempt')
raise SystemExit(run(a.root) if a.guard is None else guard(a.root, a.guard, a.slot))