Freeze detached Gre24 continuation for 385 never-attempted parts

This commit is contained in:
Frank14f committed 2026-11-16 02:06:57 +08:00
1 parent 006ca542e2
commit 41b3d07770
8 files changed
+1747

No files matched your search

@@ -0,0 +1,161 @@
"""Detached continuation of never-attempted parts; original science stays pinned."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
import traceback
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from gre24_prepare_v2 import checked, sha
from run_gre24_protocol_preparation import guard
from s2_recovery_supervisor import save, stop_group
def partition(order, old_records, remaining):
old_ids = [r['index'] for r in old_records]
if len(set(old_ids)) != len(old_ids) or len(set(remaining)) != len(remaining):
raise ValueError('Duplicate attempt identity')
expected = [i for i in order if i not in set(old_ids)]
if remaining != expected or set(old_ids) | set(remaining) != set(order):
raise ValueError('Continuation must contain exactly never-attempted indices')
return expected
def load_scope(root):
c = checked(root)
m = json.loads((root / 'continuation-manifest.json').read_text())
for path, digest in m['source_sha256'].items():
if sha(path) != digest:
raise ValueError('Continuation source changed: ' + path)
for name in ['contract.json', 'splits.json']:
if sha(root / name) != m['input_sha256'][name]:
raise ValueError('Original input changed: ' + name)
if sha(m['original_audit']) != m['original_audit_sha256']:
raise ValueError('Original interruption audit changed')
old = []
for witness in m['original_records']:
path = Path(witness['path'])
if sha(path) != witness['sha256']:
raise ValueError('Original guard receipt changed')
r = json.loads(path.read_text())
if r['case'] != c['parts'][r['index']]['case']:
raise ValueError('Original case/index mismatch')
if r['worker']['status'] == 'PASS' and sha(r['result_path']) != r['result_sha256']:
raise ValueError('Original successful result changed')
old.append(r)
partition(c['dispatch_order'], old, m['remaining_indices'])
actual = sum(r['worker']['cpu_seconds'] for r in old)
if abs(actual - m['original_worker_cpu_seconds']) > 1e-6:
raise ValueError('Original CPU ledger mismatch')
return c, m, old
def terminal(c, m, old, new, attempted, reason, elapsed):
combined = old + new
ids = [r['index'] for r in combined]
if len(set(ids)) != len(ids):
raise ValueError('Duplicate canonical terminal')
missing = sorted(set(attempted) - {r['index'] for r in new})
complete = not reason and not missing and len(combined) == len(c['parts'])
return dict(status='COMPLETE' if complete else 'STOPPED', stop_reason=reason,
records=sorted(combined, key=lambda r: r['index']),
never_attempted=[i for i in m['remaining_indices'] if i not in attempted],
missing_attempt_receipts=missing, accounting_complete=not missing,
worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in combined),
initial_cpu_reserve_seconds=c['limits']['preparation_initial_cpu_reserve_seconds'] + m['engineering_test_cpu_seconds'],
old_worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in old),
new_worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in new),
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
wall_seconds=elapsed, provenance='Two-source canonical continuation; original complete remains absent',
original_audit_sha256=m['original_audit_sha256'])
def run(root):
with (root / 'queue.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
return run_locked(root)
def run_locked(root):
c, m, old = load_scope(root)
if (root / 'queue-started.json').exists():
raise FileExistsError('Never restart continuation')
os.sched_setaffinity(0, c['coordinator_cpus'])
save(root / 'queue-started.json', dict(pid=os.getpid(), unix=time.time(),
contract_sha256=sha(root / 'contract.json'), manifest_sha256=sha(root / 'continuation-manifest.json')))
active = {}; done = []; attempted = []; remaining = m['remaining_indices']; reason = None
start = time.monotonic()
initial = c['limits']['preparation_initial_cpu_reserve_seconds'] + m['engineering_test_cpu_seconds']
cap = c['limits']['total_worker_cpu_seconds']; per = c['limits']['per_part_cpu_seconds']
def interrupted(*_):
nonlocal reason
reason = 'COORDINATOR_INTERRUPTED'
signal.signal(signal.SIGTERM, interrupted); signal.signal(signal.SIGINT, interrupted)
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', MKL_NUM_THREADS='1',
NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
try:
while active or (len(attempted) < len(remaining) and reason is None):
for slot, item in list(active.items()):
if item['process'].poll() is None:
continue
path = root / 'parts' / c['parts'][item['index']]['case'] / 'guard-result.json'
if not path.exists():
raise RuntimeError('MISSING_GUARD_RECEIPT')
record = json.loads(path.read_text())
if record['index'] != item['index'] or record['case'] != c['parts'][item['index']]['case']:
raise ValueError('New guard identity mismatch')
done.append(record); item['log'].close(); del active[slot]
spent = initial + sum(r['worker']['cpu_seconds'] for r in old + done)
while reason is None and len(active) < c['limits']['workers'] and len(attempted) < len(remaining):
if spent + (len(active) + 1) * (per + 2) > cap:
if not active:
reason = 'REMAINING_CPU_INSUFFICIENT_FOR_ANOTHER_CAPPED_ATTEMPT'
break
slot = next(s for s in range(c['limits']['workers']) if s not in active)
index = remaining[len(attempted)]
dest = root / 'parts' / c['parts'][index]['case']; dest.mkdir(parents=True, exist_ok=False)
log = (dest / 'guard.log').open('x')
process = subprocess.Popen([sys.executable, __file__, str(root), '--guard', str(index), '--slot', str(slot)],
stdout=log, stderr=subprocess.STDOUT, env=env, start_new_session=True)
active[slot] = dict(process=process, log=log, index=index); attempted.append(index)
save(root / 'progress.json', dict(completed=len(old) + len(done),
old_terminal=len(old), new_terminal=len(done), new_attempted=len(attempted),
attempted=len(old) + len(attempted), total=len(c['parts']),
active={str(k): v['index'] for k, v in active.items()},
settled_worker_cpu_seconds=spent, old_worker_cpu_seconds=m['original_worker_cpu_seconds'],
new_worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done),
reserved_active_worker_cpu_seconds=len(active) * (per + 2), stop_reason=reason,
elapsed_seconds=time.monotonic() - start, first_checkpoint_reused=True))
if active:
time.sleep(.5)
except Exception as exc:
reason = 'CONTINUATION_INFRASTRUCTURE_EXCEPTION'
save(root / 'coordinator-failure.json', dict(type=type(exc).__name__, message=str(exc), traceback=traceback.format_exc()))
finally:
for item in active.values():
stop_group(item['process'].pid); item['process'].wait(); item['log'].close()
known = {r['index'] for r in done}
for index in attempted:
path = root / 'parts' / c['parts'][index]['case'] / 'guard-result.json'
if index not in known and path.exists():
done.append(json.loads(path.read_text()))
result = terminal(c, m, old, done, attempted, reason, time.monotonic() - start)
save(root / 'complete.json', result)
return 0 if result['status'] == 'COMPLETE' else 1
if __name__ == '__main__':
p = argparse.ArgumentParser(); p.add_argument('root', type=Path)
p.add_argument('--guard', type=int); p.add_argument('--slot', type=int, default=0)
a = p.parse_args()
if a.guard is not None:
_, manifest, _ = load_scope(a.root)
if a.guard not in manifest['remaining_indices']:
raise ValueError('Cannot rerun any original attempt')
raise SystemExit(run(a.root) if a.guard is None else guard(a.root, a.guard, a.slot))
+100
View File
@@ -0,0 +1,100 @@
"""Synthetic scope, accounting and stop paths of the continuation coordinator."""
import importlib.util
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
spec = importlib.util.spec_from_file_location('gre24_continuation', Path(__file__).parent / 'gre24_continuation/run_gre24_protocol_preparation.py')
module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module)
def record(index, status='PASS', cpu=2.):
return dict(index=index, case=str(index), worker=dict(status=status, cpu_seconds=cpu,
stop_reason=None if status == 'PASS' else 'wall_limit'))
class ContinuationTests(unittest.TestCase):
def test_original_failures_are_not_retried(self):
self.assertEqual(module.partition([0, 1, 2], [record(0), record(1, 'STOPPED_OR_FAILED')], [2]), [2])
with self.assertRaises(ValueError):
module.partition([0, 1, 2], [record(0), record(1, 'STOPPED_OR_FAILED')], [1, 2])
def test_duplicate_reuse_rejected(self):
with self.assertRaises(ValueError):
module.partition([0, 1], [record(0), record(0)], [1])
def test_order_and_coverage_enforced(self):
for remaining in [[2, 1], [1], [1, 1, 2]]:
with self.assertRaises(ValueError):
module.partition([0, 1, 2], [record(0)], remaining)
def test_canonical_accounting_and_failure_preservation(self):
c = dict(parts=[0, 1, 2], limits={'preparation_initial_cpu_reserve_seconds': 10})
m = dict(remaining_indices=[2], engineering_test_cpu_seconds=.5, original_audit_sha256='audit')
result = module.terminal(c, m, [record(0), record(1, 'STOPPED_OR_FAILED')], [record(2, cpu=3)], [2], None, 1)
self.assertEqual(result['status'], 'COMPLETE')
self.assertEqual(result['worker_cpu_seconds'] + result['initial_cpu_reserve_seconds'], 17.5)
self.assertEqual(result['records'][1]['worker']['stop_reason'], 'wall_limit')
self.assertEqual(result['old_worker_cpu_seconds'], 4)
self.assertEqual(result['new_worker_cpu_seconds'], 3)
def test_missing_attempt_cannot_claim_complete(self):
c = dict(parts=[0, 1], limits={'preparation_initial_cpu_reserve_seconds': 0})
m = dict(remaining_indices=[1], engineering_test_cpu_seconds=0, original_audit_sha256='audit')
result = module.terminal(c, m, [record(0)], [], [1], 'INTERRUPTED', 1)
self.assertEqual(result['status'], 'STOPPED')
self.assertFalse(result['accounting_complete'])
self.assertEqual(result['missing_attempt_receipts'], [1])
self.assertEqual(result['never_attempted'], [])
def exercise(self, cap, missing_receipt=False):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
for name in ['contract.json', 'continuation-manifest.json']:
(root / name).write_text('{}')
c = dict(parts=[{'case':str(i)} for i in range(3)], coordinator_cpus=[0],
limits=dict(preparation_initial_cpu_reserve_seconds=1, total_worker_cpu_seconds=cap,
per_part_cpu_seconds=10, workers=2))
m = dict(remaining_indices=[1, 2], engineering_test_cpu_seconds=.5,
original_worker_cpu_seconds=2, original_audit_sha256='audit')
launched = []
class FakeProcess:
pid = 99999999
def poll(self): return 0
def wait(self): return 0
def fake_launch(command, **_):
index = int(command[command.index('--guard') + 1]); launched.append(index)
if not missing_receipt:
(root/'parts'/str(index)/'guard-result.json').write_text(json.dumps(record(index)))
return FakeProcess()
with patch.object(module, 'load_scope', return_value=(c, m, [record(0)])), \
patch.object(module.os, 'sched_setaffinity'), \
patch.object(module.signal, 'signal'), \
patch.object(module.subprocess, 'Popen', side_effect=fake_launch), \
patch.object(module.time, 'sleep'), patch.object(module, 'stop_group'):
code = module.run_locked(root)
result = json.loads((root/'complete.json').read_text())
with self.assertRaises(FileExistsError): module.run_locked(root)
return code, launched, result
def test_actual_coordinator_only_dispatches_remaining(self):
code, launched, result = self.exercise(100)
self.assertEqual((code, launched, result['status']), (0, [1, 2], 'COMPLETE'))
self.assertEqual(result['worker_cpu_seconds'] + result['initial_cpu_reserve_seconds'], 7.5)
def test_budget_reserves_old_cost_before_any_launch(self):
code, launched, result = self.exercise(15)
self.assertEqual((code, launched), (1, []))
self.assertEqual(result['stop_reason'], 'REMAINING_CPU_INSUFFICIENT_FOR_ANOTHER_CAPPED_ATTEMPT')
def test_missing_guard_stops_without_retry(self):
code, launched, result = self.exercise(100, missing_receipt=True)
self.assertEqual((code, launched), (1, [1, 2]))
self.assertFalse(result['accounting_complete'])
self.assertEqual(result['stop_reason'], 'CONTINUATION_INFRASTRUCTURE_EXCEPTION')
if __name__ == '__main__':
unittest.main()