"""Exact review, custody and resource binding for formal three-seed GINO.""" import argparse from datetime import datetime import fcntl import json import math import os from pathlib import Path import signal import time from zoneinfo import ZoneInfo from phase2_gate_outcome_review import validate_review from track_b_gino_formal_guard import guard from track_b_gino_import_checked_entry import accepted_environment from track_b_gino_ordinal_recovery import ROOT as MEASUREMENT from track_b_gino_shapenet_entry import BASE, checked, read, save, sha ROOT = MEASUREMENT.with_name('track-b-gino-shapenet611-formal-v3-2026-11-19') SOURCES = Path(__file__).resolve().parent OLD_FILES = { 'contract.json': 'ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a', 'complete.json': 'db3605159c5ade3deda5313ab409e2f1c8bf630511a617f527177da06d6c9a2d', 'recovery-settlement.json': '851eaedf716f01c1d80ce637f7de3edd0664c734bcbfed5622623d99f572fcec', 'worker-records/progress.json': '8f8fb1902c44e2f9c3198608cbecb3a74967ed00fdbcaf1e990d146c894b0daf', } DEVICES = { '0': 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76', '1': 'GPU-8b71b032-1b75-2891-935c-ce106a53034a', '2': 'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c', } ORDINALS = {'0': 2, '1': 3, '2': 0} INVENTORY = BASE / 'artifacts/track-b-gino-gpu2-ordinal-preparation-2026-11-19/physical-inventory.json' INVENTORY_SHA = 'da2a7b1bf168c7cc4d45b6eaa2e007ae90cb3640eeadde94c10fd1d32fa30dd8' TZ = ZoneInfo('Asia/Hong_Kong') RESERVATION_START = datetime(2026, 11, 19, tzinfo=TZ) RESERVATION_END = datetime(2026, 11, 20, tzinfo=TZ) NEW_SOURCES = ('track_b_gino_formal.py', 'track_b_gino_formal_guard.py', 'track_b_gino_formal_worker.py') def expected_contract(): old = {name: read(checked(MEASUREMENT / name, digest)) for name, digest in OLD_FILES.items()} c = old['contract.json'] stop, epoch = old['complete.json'], old['worker-records/progress.json'] account = old['recovery-settlement.json'] if (stop['status'], stop['trigger'], stop['result_sha256']) != ('STOPPED', 'FOREIGN_GPU_OCCUPANT', None): raise ValueError('Historical guard STOP differs') if (epoch['stage'], epoch['seed'], epoch['epoch'], epoch['examples']) != ('EPOCH_COMPLETE', 0, 1, 500): raise ValueError('Only the completed 500-example epoch can size this reproduction') seconds = epoch['seconds'] inventory = read(checked(INVENTORY, INVENTORY_SHA))['inventory'] physical = {(int(index), uuid) for index, uuid, _, name in inventory if 'V100' in name} if not {(ORDINALS[seed], uuid) for seed, uuid in DEVICES.items()} <= physical: raise ValueError('Reviewed physical V100 inventory differs') budgets = {seed: {'fit': 13033 + (seed == '0'), 'score': 300} for seed in DEVICES} fit_reference = 100 * seconds * 1.25 if (not math.isfinite(fit_reference) or fit_reference <= 0 or min(b['fit'] for b in budgets.values()) <= fit_reference or sum(b['fit'] + b['score'] for b in budgets.values()) != 40000): raise ValueError('Formal booking must cover the sizing reference and score reserve') c['mode'] = 'formal-gino-reproduction' for key in ('gpu_uuid', 'measurement_allowed_heldout_labels', 'gpu_ruling', 'ordinal_recovery'): del c[key] c['code_sha256'].update({str(SOURCES / name): sha(SOURCES / name) for name in NEW_SOURCES}) c['formal'] = dict( handoff='h-4e4e680b', ruling_slack_ts='1795074651.222599', devices=DEVICES, lifecycle_review_slack_ts='1795075951.091959', device_pci_ordinals=ORDINALS, reservation=dict(handoff='h-a429935e', timezone=TZ.key, start_inclusive=RESERVATION_START.isoformat(), end_exclusive=RESERVATION_END.isoformat(), gpu3_single_task=True, historical_inventory_path=str(INVENTORY), historical_inventory_sha256=INVENTORY_SHA, runtime_idle_check_required=True, applies_to_new_dispatch_only=True, already_running_guards_unchanged=True), budget_per_seed_stage_gpu_seconds=budgets, total_gpu_wall_seconds=40000, timing=dict(record_paths={str(MEASUREMENT / k): v for k, v in OLD_FILES.items()}, status='CONTENTION_AFFECTED_NOT_GUARD_CERTIFIED', sizing_only=True, epoch_seconds=seconds, train_sizing_gpu_seconds=300 * seconds * 1.25, train_sizing_multiplier=1.25, score_reserved_gpu_seconds=900, prior_measurement_gpu_seconds=account['cumulative_measurement_gpu_wall_seconds']), heldout_policy=dict(preparation_reads=0, training_reads=0, score_reads_per_seed=111, all_epoch100_checkpoints_required=True, retry=False), ) del c['limits']['gpu_wall_seconds'] c['limits']['cleanup_seconds'] = 20 del c['limits']['cpu_seconds'] c['limits']['cpu_seconds_per_gpu_second'] = 8 return c def contract(): c = read(ROOT / 'contract.json') if c != expected_contract(): raise PermissionError('The sole formal contract or its implementation changed') for path, digest in c['code_sha256'].items(): checked(path, digest) for name in ('paper_supplement', 'split_receipt', 'mapping_audit'): pin = c[name] checked(pin['path'], pin['sha256']) # Archive members are opened only in the scoped workers, which verify each # member before using it. Preparation does not hash the archive payload. comparison = read(checked(c['comparison_contract'], c['comparison_contract_sha256'])) split = read(c['split_receipt']['path']) train, test = split['train'], split['test'] if (c['protocol'] != comparison['protocol'] or len(train) != 500 or len(test) != 111 or len(set(train) | set(test)) != 611 or any(c['split'][k] != split[k] for k in ('train', 'test', 'raw_global_bounds')) or c['protocol']['epochs_per_seed'] != 100 or c['protocol']['seeds'] != [0, 1, 2] or c['protocol']['acceptance_percent'] != [6.62, 7.62] or c['paper_compatibility']['identity'] != 'supplement code + 2-keyword patch' or not Path(c['paper_data']['path']).is_file()): raise ValueError('Original GINO comparison, split or pinned data path differs') accepted_environment(c) return c def authorize(c): validate_review(ROOT) authorization = read(ROOT / 'stage-authorization.json') required = dict(contract_sha256=sha(ROOT / 'contract.json'), implementation_proof_sha256=sha(ROOT / 'implementation-git-freeze.json'), decision='APPROVED', mode=c['mode'], reserved_devices=DEVICES, reservation_end_exclusive=RESERVATION_END.isoformat()) if any(authorization.get(k) != v for k, v in required.items()): raise PermissionError('Exact execution and card-reservation authorization required') def check_reservation(unix): if not RESERVATION_START.timestamp() <= unix < RESERVATION_END.timestamp(): raise PermissionError('New dispatch must begin in the November 19 Hong Kong reservation') def run_root(seed, phase): return ROOT / f'seed-{seed}' / phase def epoch100(root, seed, c): progress = read(root / 'worker-records/progress.json') if (progress['stage'], progress['seed'], progress['epoch'], progress['examples']) != ( 'EPOCH_COMPLETE', seed, c['protocol']['epochs_per_seed'], 500): raise ValueError('No completed 100th full training epoch for this seed') return progress def checkpoint(seed, result): path = run_root(seed, 'fit') / 'epoch-100.pt' if result['checkpoint']['path'] != str(path): raise ValueError('Checkpoint path differs from this seed') return checked(path, result['checkpoint']['sha256']) def score_mean(rows, test_ids): if [r['case'] for r in rows] != test_ids: raise ValueError('Scoring requires the original 111-case order') values = [r['relative_l2'] for r in rows] if any(type(v) not in (int, float) or not math.isfinite(v) or v < 0 for v in values): raise ValueError('Incomplete or invalid relative L2') return 100 * (math.fsum(values) / len(values)) def validate_result(c, seed, phase): root = run_root(seed, phase) path = root / 'worker-records/result.json' result = read(path) spawned = read(root / 'actual-worker-spawn.json') device = DEVICES[str(seed)] ordinal = str(ORDINALS[str(seed)]) if (spawned['physical_uuid'], spawned['ordinal']) != (device, ordinal) or ( read(root / 'worker-device-verified.json') != dict( process=spawned['process'], physical_uuid=device, ordinal=ordinal, observed_uuids=[device])): raise ValueError('Worker has no matching physical-device witness') if (result['phase'], result['seed'], result['status']) != (phase, seed, 'COMPLETE'): raise ValueError('Worker result identity differs') if phase == 'fit': epoch100(root, seed, c) checkpoint(seed, result) if (result['target_reads'], result['epochs'], result['training_examples']) != (0, 100, 500): raise ValueError('Fit must complete all 100 epochs with zero held-out reads') else: fit_terminal = read(run_root(seed, 'fit') / 'complete.json') fit_path = run_root(seed, 'fit') / 'worker-records/result.json' original = read(checked(fit_path, fit_terminal['result_sha256'])) access = root / 'heldout-target-access.jsonl' mapping = {row['case']: row['pressure_sha256'] for row in read(c['mapping_audit']['path'])['rows']} actual = [json.loads(line) for line in access.read_text().splitlines()] expected = [dict(case=i, role='heldout', sha256=mapping[i]) for i in c['split']['test']] if (actual != expected or result['target_reads'] != 111 or result['target_access_sha256'] != sha(access) or result['checkpoints_manifest_sha256'] != sha(root / 'checkpoints-frozen.json') or read(root / 'checkpoints-frozen.json') != all_checkpoints(c) or result['checkpoint_sha256'] != original['checkpoint']['sha256'] or score_mean(result['cases'], c['split']['test']) != result['mean_percent']): raise ValueError('Score or held-out target audit differs') return sha(path) def accepted_terminal(c, seed, phase): root = run_root(seed, phase) record = read(root / 'complete.json') if (record['status'], record['seed'], record['phase'], record['contract_sha256'], record['physical_uuid']) != ( 'COMPLETE', seed, phase, sha(ROOT / 'contract.json'), DEVICES[str(seed)]): raise PermissionError(f'Seed {seed} {phase} has no accepted terminal result') booked = c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase] if (record['booked_gpu_seconds'] != booked or record['charged_gpu_seconds'] > booked or record['result_sha256'] != validate_result(c, seed, phase)): raise ValueError('Terminal result bytes or resource accounting differ') return record def all_checkpoints(c): checkpoints = {} for seed in c['protocol']['seeds']: accepted_terminal(c, seed, 'fit') root = run_root(seed, 'fit') checkpoints[str(seed)] = dict( result_sha256=sha(root / 'worker-records/result.json'), terminal_sha256=sha(root / 'complete.json'), checkpoint=read(root / 'worker-records/result.json')['checkpoint']) return checkpoints def dispatch(c, seed, phase): began = time.monotonic() unix = time.time() authorize(c) budget = c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase] check_reservation(unix) seed_root = run_root(seed, phase).parent seed_root.mkdir(exist_ok=True) with (seed_root / 'execution.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) roster = all_checkpoints(c) if phase == 'score' else None root = run_root(seed, phase) root.mkdir(exist_ok=False) if roster is not None: save(root / 'checkpoints-frozen.json', roster) ticket = dict(seed=seed, phase=phase, ordinal=str(ORDINALS[str(seed)]), booked_gpu_seconds=budget, unix=unix, dispatch_before_unix=RESERVATION_END.timestamp(), contract_sha256=sha(ROOT / 'contract.json'), guard_started_monotonic=began, booked_deadline_monotonic=began + budget) save(root / 'started.json', ticket) command = [c['environment']['python'], str(SOURCES / 'track_b_gino_formal_worker.py'), str(seed), phase] def request_stop(signum, _frame): raise KeyboardInterrupt(signal.Signals(signum).name) previous = signal.signal(signal.SIGTERM, request_stop) try: limits = dict(c['limits'], cpu_seconds=c['limits']['cpu_seconds_per_gpu_second'] * budget) result = guard(root, command, uuid=DEVICES[str(seed)], limits=limits, deadline=began + budget, ticket=ticket, validate_result=lambda: validate_result(c, seed, phase)) finally: signal.signal(signal.SIGTERM, previous) return 0 if result['status'] == 'COMPLETE' else 1 def pool(means, limits): if len(means) != 3 or any(v is None for v in means): return None, None mean = math.fsum(means) / 3 return mean, limits[0] <= mean <= limits[1] def collect(c): seeds = {} accounted = unsettled = 0 for seed in c['protocol']['seeds']: rows = {} for phase in ('fit', 'score'): root = run_root(seed, phase) terminal = root / 'complete.json' if terminal.exists(): record = read(terminal) if record['status'] == 'COMPLETE': accepted_terminal(c, seed, phase) accounted += record['charged_gpu_seconds'] rows[phase] = dict(status=record['status'], trigger=record['trigger'], terminal_sha256=sha(terminal)) if phase == 'score' and record['status'] == 'COMPLETE': rows['mean_percent'] = read(root / 'worker-records/result.json')['mean_percent'] elif (root / 'started.json').exists(): rows[phase] = dict(status='NO_TERMINAL_RECEIPT') unsettled += c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase] else: rows[phase] = dict(status='NOT_STARTED') seeds[str(seed)] = rows mean, verdict = pool([row.get('mean_percent') for row in seeds.values()], c['protocol']['acceptance_percent']) prior = c['formal']['timing']['prior_measurement_gpu_seconds'] return dict(seeds=seeds, mean_percent=mean, within_registered_range=verdict, reproduction_identity=c['paper_compatibility']['identity'], prior_measurement_gpu_seconds=prior, settled_formal_gpu_seconds=accounted, settled_combined_gpu_seconds=accounted + prior, unsettled_formal_bookings_gpu_seconds=unsettled, formal_gpu_cap_seconds=40000) if __name__ == '__main__': parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('action', choices=('fit', 'score', 'collect')) parser.add_argument('--seed', type=int, choices=(0, 1, 2)) args = parser.parse_args() c = contract() if args.action == 'collect': if args.seed is not None: parser.error('collect reports all three seeds') print(json.dumps(collect(c), indent=2, allow_nan=False)) else: if args.seed is None: parser.error('--seed is required for fit or score') raise SystemExit(dispatch(c, args.seed, args.action))