315 lines
15 KiB
Python
315 lines
15 KiB
Python
"""Exact review, custody and resource binding for formal three-seed GINO."""
|
|
|
|
import argparse
|
|
from datetime import datetime
|
|
import fcntl
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import time
|
|
from zoneinfo import ZoneInfo
|
|
|
|
from phase2_gate_outcome_review import validate_review
|
|
from track_b_gino_formal_guard import guard
|
|
from track_b_gino_import_checked_entry import accepted_environment
|
|
from track_b_gino_ordinal_recovery import ROOT as MEASUREMENT
|
|
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
|
|
|
|
|
|
ROOT = MEASUREMENT.with_name('track-b-gino-shapenet611-formal-2026-11-19')
|
|
SOURCES = Path(__file__).resolve().parent
|
|
OLD_FILES = {
|
|
'contract.json': 'ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a',
|
|
'complete.json': 'db3605159c5ade3deda5313ab409e2f1c8bf630511a617f527177da06d6c9a2d',
|
|
'recovery-settlement.json': '851eaedf716f01c1d80ce637f7de3edd0664c734bcbfed5622623d99f572fcec',
|
|
'worker-records/progress.json': '8f8fb1902c44e2f9c3198608cbecb3a74967ed00fdbcaf1e990d146c894b0daf',
|
|
}
|
|
DEVICES = {
|
|
'0': 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76',
|
|
'1': 'GPU-8b71b032-1b75-2891-935c-ce106a53034a',
|
|
'2': 'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c',
|
|
}
|
|
ORDINALS = {'0': 2, '1': 3, '2': 0}
|
|
INVENTORY = BASE / 'artifacts/track-b-gino-gpu2-ordinal-preparation-2026-11-19/physical-inventory.json'
|
|
INVENTORY_SHA = 'da2a7b1bf168c7cc4d45b6eaa2e007ae90cb3640eeadde94c10fd1d32fa30dd8'
|
|
TZ = ZoneInfo('Asia/Hong_Kong')
|
|
RESERVATION_START = datetime(2026, 11, 19, tzinfo=TZ)
|
|
RESERVATION_END = datetime(2026, 11, 20, tzinfo=TZ)
|
|
NEW_SOURCES = ('track_b_gino_formal.py', 'track_b_gino_formal_guard.py',
|
|
'track_b_gino_formal_worker.py')
|
|
|
|
|
|
def expected_contract():
|
|
old = {name: read(checked(MEASUREMENT / name, digest)) for name, digest in OLD_FILES.items()}
|
|
c = old['contract.json']
|
|
stop, epoch = old['complete.json'], old['worker-records/progress.json']
|
|
account = old['recovery-settlement.json']
|
|
if (stop['status'], stop['trigger'], stop['result_sha256']) != ('STOPPED', 'FOREIGN_GPU_OCCUPANT', None):
|
|
raise ValueError('Historical guard STOP differs')
|
|
if (epoch['stage'], epoch['seed'], epoch['epoch'], epoch['examples']) != ('EPOCH_COMPLETE', 0, 1, 500):
|
|
raise ValueError('Only the completed 500-example epoch can size this reproduction')
|
|
seconds = epoch['seconds']
|
|
inventory = read(checked(INVENTORY, INVENTORY_SHA))['inventory']
|
|
physical = {(int(index), uuid) for index, uuid, _, name in inventory if 'V100' in name}
|
|
if not {(ORDINALS[seed], uuid) for seed, uuid in DEVICES.items()} <= physical:
|
|
raise ValueError('Reviewed physical V100 inventory differs')
|
|
budgets = {seed: {'fit': 13033 + (seed == '0'), 'score': 300} for seed in DEVICES}
|
|
fit_reference = 100 * seconds * 1.25
|
|
if (not math.isfinite(fit_reference) or fit_reference <= 0
|
|
or min(b['fit'] for b in budgets.values()) <= fit_reference
|
|
or sum(b['fit'] + b['score'] for b in budgets.values()) != 40000):
|
|
raise ValueError('Formal booking must cover the sizing reference and score reserve')
|
|
c['mode'] = 'formal-gino-reproduction'
|
|
for key in ('gpu_uuid', 'measurement_allowed_heldout_labels', 'gpu_ruling', 'ordinal_recovery'):
|
|
del c[key]
|
|
c['code_sha256'].update({str(SOURCES / name): sha(SOURCES / name) for name in NEW_SOURCES})
|
|
c['formal'] = dict(
|
|
handoff='h-4e4e680b', ruling_slack_ts='1795074651.222599', devices=DEVICES,
|
|
device_pci_ordinals=ORDINALS,
|
|
reservation=dict(handoff='h-a429935e', timezone=TZ.key,
|
|
start_inclusive=RESERVATION_START.isoformat(),
|
|
end_exclusive=RESERVATION_END.isoformat(), gpu3_single_task=True,
|
|
historical_inventory_path=str(INVENTORY), historical_inventory_sha256=INVENTORY_SHA,
|
|
runtime_idle_check_required=True),
|
|
budget_per_seed_stage_gpu_seconds=budgets, total_gpu_wall_seconds=40000,
|
|
timing=dict(record_paths={str(MEASUREMENT / k): v for k, v in OLD_FILES.items()},
|
|
status='CONTENTION_AFFECTED_NOT_GUARD_CERTIFIED', sizing_only=True,
|
|
epoch_seconds=seconds, train_sizing_gpu_seconds=300 * seconds * 1.25,
|
|
train_sizing_multiplier=1.25, score_reserved_gpu_seconds=900,
|
|
prior_measurement_gpu_seconds=account['cumulative_measurement_gpu_wall_seconds']),
|
|
heldout_policy=dict(preparation_reads=0, training_reads=0, score_reads_per_seed=111,
|
|
all_epoch100_checkpoints_required=True, retry=False),
|
|
)
|
|
del c['limits']['gpu_wall_seconds']
|
|
c['limits']['cleanup_seconds'] = 20
|
|
del c['limits']['cpu_seconds']
|
|
c['limits']['cpu_seconds_per_gpu_second'] = 8
|
|
return c
|
|
|
|
|
|
def contract():
|
|
c = read(ROOT / 'contract.json')
|
|
if c != expected_contract():
|
|
raise PermissionError('The sole formal contract or its implementation changed')
|
|
for path, digest in c['code_sha256'].items():
|
|
checked(path, digest)
|
|
for name in ('paper_supplement', 'split_receipt', 'mapping_audit'):
|
|
pin = c[name]
|
|
checked(pin['path'], pin['sha256'])
|
|
# Archive members are opened only in the scoped workers, which verify each
|
|
# member before using it. Preparation does not hash the archive payload.
|
|
comparison = read(checked(c['comparison_contract'], c['comparison_contract_sha256']))
|
|
split = read(c['split_receipt']['path'])
|
|
train, test = split['train'], split['test']
|
|
if (c['protocol'] != comparison['protocol'] or len(train) != 500 or len(test) != 111
|
|
or len(set(train) | set(test)) != 611
|
|
or any(c['split'][k] != split[k] for k in ('train', 'test', 'raw_global_bounds'))
|
|
or c['protocol']['epochs_per_seed'] != 100 or c['protocol']['seeds'] != [0, 1, 2]
|
|
or c['protocol']['acceptance_percent'] != [6.62, 7.62]
|
|
or c['paper_compatibility']['identity'] != 'supplement code + 2-keyword patch'
|
|
or not Path(c['paper_data']['path']).is_file()):
|
|
raise ValueError('Original GINO comparison, split or pinned data path differs')
|
|
accepted_environment(c)
|
|
return c
|
|
|
|
|
|
def authorize(c):
|
|
validate_review(ROOT)
|
|
authorization = read(ROOT / 'stage-authorization.json')
|
|
required = dict(contract_sha256=sha(ROOT / 'contract.json'),
|
|
implementation_proof_sha256=sha(ROOT / 'implementation-git-freeze.json'),
|
|
decision='APPROVED', mode=c['mode'], reserved_devices=DEVICES,
|
|
reservation_end_exclusive=RESERVATION_END.isoformat())
|
|
if any(authorization.get(k) != v for k, v in required.items()):
|
|
raise PermissionError('Exact execution and card-reservation authorization required')
|
|
|
|
|
|
def check_reservation(unix, duration):
|
|
if (unix < RESERVATION_START.timestamp()
|
|
or unix + duration >= RESERVATION_END.timestamp()):
|
|
raise PermissionError('The entire booking must fit the November 19 Hong Kong reservation')
|
|
|
|
|
|
def run_root(seed, phase):
|
|
return ROOT / f'seed-{seed}' / phase
|
|
|
|
|
|
def epoch100(root, seed, c):
|
|
progress = read(root / 'worker-records/progress.json')
|
|
if (progress['stage'], progress['seed'], progress['epoch'], progress['examples']) != (
|
|
'EPOCH_COMPLETE', seed, c['protocol']['epochs_per_seed'], 500):
|
|
raise ValueError('No completed 100th full training epoch for this seed')
|
|
return progress
|
|
|
|
|
|
def checkpoint(seed, result):
|
|
path = run_root(seed, 'fit') / 'epoch-100.pt'
|
|
if result['checkpoint']['path'] != str(path):
|
|
raise ValueError('Checkpoint path differs from this seed')
|
|
return checked(path, result['checkpoint']['sha256'])
|
|
|
|
|
|
def score_mean(rows, test_ids):
|
|
if [r['case'] for r in rows] != test_ids:
|
|
raise ValueError('Scoring requires the original 111-case order')
|
|
values = [r['relative_l2'] for r in rows]
|
|
if any(type(v) not in (int, float) or not math.isfinite(v) or v < 0 for v in values):
|
|
raise ValueError('Incomplete or invalid relative L2')
|
|
return 100 * (math.fsum(values) / len(values))
|
|
|
|
|
|
def validate_result(c, seed, phase):
|
|
root = run_root(seed, phase)
|
|
path = root / 'worker-records/result.json'
|
|
result = read(path)
|
|
spawned = read(root / 'actual-worker-spawn.json')
|
|
device = DEVICES[str(seed)]
|
|
ordinal = str(ORDINALS[str(seed)])
|
|
if (spawned['physical_uuid'], spawned['ordinal']) != (device, ordinal) or (
|
|
read(root / 'worker-device-verified.json') != dict(
|
|
process=spawned['process'], physical_uuid=device, ordinal=ordinal, observed_uuids=[device])):
|
|
raise ValueError('Worker has no matching physical-device witness')
|
|
if (result['phase'], result['seed'], result['status']) != (phase, seed, 'COMPLETE'):
|
|
raise ValueError('Worker result identity differs')
|
|
if phase == 'fit':
|
|
epoch100(root, seed, c)
|
|
checkpoint(seed, result)
|
|
if (result['target_reads'], result['epochs'], result['training_examples']) != (0, 100, 500):
|
|
raise ValueError('Fit must complete all 100 epochs with zero held-out reads')
|
|
else:
|
|
fit_terminal = read(run_root(seed, 'fit') / 'complete.json')
|
|
fit_path = run_root(seed, 'fit') / 'worker-records/result.json'
|
|
original = read(checked(fit_path, fit_terminal['result_sha256']))
|
|
access = root / 'heldout-target-access.jsonl'
|
|
mapping = {row['case']: row['pressure_sha256'] for row in read(c['mapping_audit']['path'])['rows']}
|
|
actual = [json.loads(line) for line in access.read_text().splitlines()]
|
|
expected = [dict(case=i, role='heldout', sha256=mapping[i]) for i in c['split']['test']]
|
|
if (actual != expected or result['target_reads'] != 111
|
|
or result['target_access_sha256'] != sha(access)
|
|
or result['checkpoints_manifest_sha256'] != sha(root / 'checkpoints-frozen.json')
|
|
or read(root / 'checkpoints-frozen.json') != all_checkpoints(c)
|
|
or result['checkpoint_sha256'] != original['checkpoint']['sha256']
|
|
or score_mean(result['cases'], c['split']['test']) != result['mean_percent']):
|
|
raise ValueError('Score or held-out target audit differs')
|
|
return sha(path)
|
|
|
|
|
|
def accepted_terminal(c, seed, phase):
|
|
root = run_root(seed, phase)
|
|
record = read(root / 'complete.json')
|
|
if (record['status'], record['seed'], record['phase'], record['contract_sha256'], record['physical_uuid']) != (
|
|
'COMPLETE', seed, phase, sha(ROOT / 'contract.json'), DEVICES[str(seed)]):
|
|
raise PermissionError(f'Seed {seed} {phase} has no accepted terminal result')
|
|
booked = c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase]
|
|
if (record['booked_gpu_seconds'] != booked or record['charged_gpu_seconds'] > booked
|
|
or record['result_sha256'] != validate_result(c, seed, phase)):
|
|
raise ValueError('Terminal result bytes or resource accounting differ')
|
|
return record
|
|
|
|
|
|
def all_checkpoints(c):
|
|
checkpoints = {}
|
|
for seed in c['protocol']['seeds']:
|
|
accepted_terminal(c, seed, 'fit')
|
|
root = run_root(seed, 'fit')
|
|
checkpoints[str(seed)] = dict(
|
|
result_sha256=sha(root / 'worker-records/result.json'),
|
|
terminal_sha256=sha(root / 'complete.json'),
|
|
checkpoint=read(root / 'worker-records/result.json')['checkpoint'])
|
|
return checkpoints
|
|
|
|
|
|
def dispatch(c, seed, phase):
|
|
began = time.monotonic()
|
|
unix = time.time()
|
|
authorize(c)
|
|
budget = c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase]
|
|
check_reservation(unix, budget)
|
|
seed_root = run_root(seed, phase).parent
|
|
seed_root.mkdir(exist_ok=True)
|
|
with (seed_root / 'execution.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
roster = all_checkpoints(c) if phase == 'score' else None
|
|
root = run_root(seed, phase)
|
|
root.mkdir(exist_ok=False)
|
|
if roster is not None:
|
|
save(root / 'checkpoints-frozen.json', roster)
|
|
ticket = dict(seed=seed, phase=phase, ordinal=str(ORDINALS[str(seed)]),
|
|
booked_gpu_seconds=budget, unix=unix,
|
|
contract_sha256=sha(ROOT / 'contract.json'),
|
|
guard_started_monotonic=began, booked_deadline_monotonic=began + budget)
|
|
save(root / 'started.json', ticket)
|
|
command = [c['environment']['python'], str(SOURCES / 'track_b_gino_formal_worker.py'),
|
|
str(seed), phase]
|
|
|
|
def request_stop(signum, _frame):
|
|
raise KeyboardInterrupt(signal.Signals(signum).name)
|
|
|
|
previous = signal.signal(signal.SIGTERM, request_stop)
|
|
try:
|
|
limits = dict(c['limits'], cpu_seconds=c['limits']['cpu_seconds_per_gpu_second'] * budget)
|
|
result = guard(root, command, uuid=DEVICES[str(seed)], limits=limits,
|
|
deadline=began + budget, ticket=ticket,
|
|
validate_result=lambda: validate_result(c, seed, phase))
|
|
finally:
|
|
signal.signal(signal.SIGTERM, previous)
|
|
return 0 if result['status'] == 'COMPLETE' else 1
|
|
|
|
|
|
def pool(means, limits):
|
|
if len(means) != 3 or any(v is None for v in means):
|
|
return None, None
|
|
mean = math.fsum(means) / 3
|
|
return mean, limits[0] <= mean <= limits[1]
|
|
|
|
|
|
def collect(c):
|
|
seeds = {}
|
|
accounted = unsettled = 0
|
|
for seed in c['protocol']['seeds']:
|
|
rows = {}
|
|
for phase in ('fit', 'score'):
|
|
root = run_root(seed, phase)
|
|
terminal = root / 'complete.json'
|
|
if terminal.exists():
|
|
record = read(terminal)
|
|
if record['status'] == 'COMPLETE':
|
|
accepted_terminal(c, seed, phase)
|
|
accounted += record['charged_gpu_seconds']
|
|
rows[phase] = dict(status=record['status'], trigger=record['trigger'],
|
|
terminal_sha256=sha(terminal))
|
|
if phase == 'score' and record['status'] == 'COMPLETE':
|
|
rows['mean_percent'] = read(root / 'worker-records/result.json')['mean_percent']
|
|
elif (root / 'started.json').exists():
|
|
rows[phase] = dict(status='NO_TERMINAL_RECEIPT')
|
|
unsettled += c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase]
|
|
else:
|
|
rows[phase] = dict(status='NOT_STARTED')
|
|
seeds[str(seed)] = rows
|
|
mean, verdict = pool([row.get('mean_percent') for row in seeds.values()],
|
|
c['protocol']['acceptance_percent'])
|
|
prior = c['formal']['timing']['prior_measurement_gpu_seconds']
|
|
return dict(seeds=seeds, mean_percent=mean, within_registered_range=verdict,
|
|
reproduction_identity=c['paper_compatibility']['identity'],
|
|
prior_measurement_gpu_seconds=prior, settled_formal_gpu_seconds=accounted,
|
|
settled_combined_gpu_seconds=accounted + prior,
|
|
unsettled_formal_bookings_gpu_seconds=unsettled, formal_gpu_cap_seconds=40000)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('action', choices=('fit', 'score', 'collect'))
|
|
parser.add_argument('--seed', type=int, choices=(0, 1, 2))
|
|
args = parser.parse_args()
|
|
c = contract()
|
|
if args.action == 'collect':
|
|
if args.seed is not None:
|
|
parser.error('collect reports all three seeds')
|
|
print(json.dumps(collect(c), indent=2, allow_nan=False))
|
|
else:
|
|
if args.seed is None:
|
|
parser.error('--seed is required for fit or score')
|
|
raise SystemExit(dispatch(c, args.seed, args.action))
|