162 lines
8.4 KiB
Python
162 lines
8.4 KiB
Python
"""Detached continuation of never-attempted parts; original science stays pinned."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from gre24_prepare_v2 import checked, sha
|
|
from run_gre24_protocol_preparation import guard
|
|
from s2_recovery_supervisor import save, stop_group
|
|
|
|
|
|
def partition(order, old_records, remaining):
|
|
old_ids = [r['index'] for r in old_records]
|
|
if len(set(old_ids)) != len(old_ids) or len(set(remaining)) != len(remaining):
|
|
raise ValueError('Duplicate attempt identity')
|
|
expected = [i for i in order if i not in set(old_ids)]
|
|
if remaining != expected or set(old_ids) | set(remaining) != set(order):
|
|
raise ValueError('Continuation must contain exactly never-attempted indices')
|
|
return expected
|
|
|
|
|
|
def load_scope(root):
|
|
c = checked(root)
|
|
m = json.loads((root / 'continuation-manifest.json').read_text())
|
|
for path, digest in m['source_sha256'].items():
|
|
if sha(path) != digest:
|
|
raise ValueError('Continuation source changed: ' + path)
|
|
for name in ['contract.json', 'splits.json']:
|
|
if sha(root / name) != m['input_sha256'][name]:
|
|
raise ValueError('Original input changed: ' + name)
|
|
if sha(m['original_audit']) != m['original_audit_sha256']:
|
|
raise ValueError('Original interruption audit changed')
|
|
old = []
|
|
for witness in m['original_records']:
|
|
path = Path(witness['path'])
|
|
if sha(path) != witness['sha256']:
|
|
raise ValueError('Original guard receipt changed')
|
|
r = json.loads(path.read_text())
|
|
if r['case'] != c['parts'][r['index']]['case']:
|
|
raise ValueError('Original case/index mismatch')
|
|
if r['worker']['status'] == 'PASS' and sha(r['result_path']) != r['result_sha256']:
|
|
raise ValueError('Original successful result changed')
|
|
old.append(r)
|
|
partition(c['dispatch_order'], old, m['remaining_indices'])
|
|
actual = sum(r['worker']['cpu_seconds'] for r in old)
|
|
if abs(actual - m['original_worker_cpu_seconds']) > 1e-6:
|
|
raise ValueError('Original CPU ledger mismatch')
|
|
return c, m, old
|
|
|
|
|
|
def terminal(c, m, old, new, attempted, reason, elapsed):
|
|
combined = old + new
|
|
ids = [r['index'] for r in combined]
|
|
if len(set(ids)) != len(ids):
|
|
raise ValueError('Duplicate canonical terminal')
|
|
missing = sorted(set(attempted) - {r['index'] for r in new})
|
|
complete = not reason and not missing and len(combined) == len(c['parts'])
|
|
return dict(status='COMPLETE' if complete else 'STOPPED', stop_reason=reason,
|
|
records=sorted(combined, key=lambda r: r['index']),
|
|
never_attempted=[i for i in m['remaining_indices'] if i not in attempted],
|
|
missing_attempt_receipts=missing, accounting_complete=not missing,
|
|
worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in combined),
|
|
initial_cpu_reserve_seconds=c['limits']['preparation_initial_cpu_reserve_seconds'] + m['engineering_test_cpu_seconds'],
|
|
old_worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in old),
|
|
new_worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in new),
|
|
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
|
wall_seconds=elapsed, provenance='Two-source canonical continuation; original complete remains absent',
|
|
original_audit_sha256=m['original_audit_sha256'])
|
|
|
|
|
|
def run(root):
|
|
with (root / 'queue.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
return run_locked(root)
|
|
|
|
|
|
def run_locked(root):
|
|
c, m, old = load_scope(root)
|
|
if (root / 'queue-started.json').exists():
|
|
raise FileExistsError('Never restart continuation')
|
|
os.sched_setaffinity(0, c['coordinator_cpus'])
|
|
save(root / 'queue-started.json', dict(pid=os.getpid(), unix=time.time(),
|
|
contract_sha256=sha(root / 'contract.json'), manifest_sha256=sha(root / 'continuation-manifest.json')))
|
|
active = {}; done = []; attempted = []; remaining = m['remaining_indices']; reason = None
|
|
start = time.monotonic()
|
|
initial = c['limits']['preparation_initial_cpu_reserve_seconds'] + m['engineering_test_cpu_seconds']
|
|
cap = c['limits']['total_worker_cpu_seconds']; per = c['limits']['per_part_cpu_seconds']
|
|
def interrupted(*_):
|
|
nonlocal reason
|
|
reason = 'COORDINATOR_INTERRUPTED'
|
|
signal.signal(signal.SIGTERM, interrupted); signal.signal(signal.SIGINT, interrupted)
|
|
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', MKL_NUM_THREADS='1',
|
|
NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
|
|
try:
|
|
while active or (len(attempted) < len(remaining) and reason is None):
|
|
for slot, item in list(active.items()):
|
|
if item['process'].poll() is None:
|
|
continue
|
|
path = root / 'parts' / c['parts'][item['index']]['case'] / 'guard-result.json'
|
|
if not path.exists():
|
|
raise RuntimeError('MISSING_GUARD_RECEIPT')
|
|
record = json.loads(path.read_text())
|
|
if record['index'] != item['index'] or record['case'] != c['parts'][item['index']]['case']:
|
|
raise ValueError('New guard identity mismatch')
|
|
done.append(record); item['log'].close(); del active[slot]
|
|
spent = initial + sum(r['worker']['cpu_seconds'] for r in old + done)
|
|
while reason is None and len(active) < c['limits']['workers'] and len(attempted) < len(remaining):
|
|
if spent + (len(active) + 1) * (per + 2) > cap:
|
|
if not active:
|
|
reason = 'REMAINING_CPU_INSUFFICIENT_FOR_ANOTHER_CAPPED_ATTEMPT'
|
|
break
|
|
slot = next(s for s in range(c['limits']['workers']) if s not in active)
|
|
index = remaining[len(attempted)]
|
|
dest = root / 'parts' / c['parts'][index]['case']; dest.mkdir(parents=True, exist_ok=False)
|
|
log = (dest / 'guard.log').open('x')
|
|
process = subprocess.Popen([sys.executable, __file__, str(root), '--guard', str(index), '--slot', str(slot)],
|
|
stdout=log, stderr=subprocess.STDOUT, env=env, start_new_session=True)
|
|
active[slot] = dict(process=process, log=log, index=index); attempted.append(index)
|
|
save(root / 'progress.json', dict(completed=len(old) + len(done),
|
|
old_terminal=len(old), new_terminal=len(done), new_attempted=len(attempted),
|
|
attempted=len(old) + len(attempted), total=len(c['parts']),
|
|
active={str(k): v['index'] for k, v in active.items()},
|
|
settled_worker_cpu_seconds=spent, old_worker_cpu_seconds=m['original_worker_cpu_seconds'],
|
|
new_worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done),
|
|
reserved_active_worker_cpu_seconds=len(active) * (per + 2), stop_reason=reason,
|
|
elapsed_seconds=time.monotonic() - start, first_checkpoint_reused=True))
|
|
if active:
|
|
time.sleep(.5)
|
|
except Exception as exc:
|
|
reason = 'CONTINUATION_INFRASTRUCTURE_EXCEPTION'
|
|
save(root / 'coordinator-failure.json', dict(type=type(exc).__name__, message=str(exc), traceback=traceback.format_exc()))
|
|
finally:
|
|
for item in active.values():
|
|
stop_group(item['process'].pid); item['process'].wait(); item['log'].close()
|
|
known = {r['index'] for r in done}
|
|
for index in attempted:
|
|
path = root / 'parts' / c['parts'][index]['case'] / 'guard-result.json'
|
|
if index not in known and path.exists():
|
|
done.append(json.loads(path.read_text()))
|
|
result = terminal(c, m, old, done, attempted, reason, time.monotonic() - start)
|
|
save(root / 'complete.json', result)
|
|
return 0 if result['status'] == 'COMPLETE' else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
p = argparse.ArgumentParser(); p.add_argument('root', type=Path)
|
|
p.add_argument('--guard', type=int); p.add_argument('--slot', type=int, default=0)
|
|
a = p.parse_args()
|
|
if a.guard is not None:
|
|
_, manifest, _ = load_scope(a.root)
|
|
if a.guard not in manifest['remaining_indices']:
|
|
raise ValueError('Cannot rerun any original attempt')
|
|
raise SystemExit(run(a.root) if a.guard is None else guard(a.root, a.guard, a.slot))
|