Files
MeltMind/tools/track_b_gino_formal.py
T

315 lines
15 KiB
Python

"""Exact review, custody and resource binding for formal three-seed GINO."""
import argparse
from datetime import datetime
import fcntl
import json
import math
import os
from pathlib import Path
import signal
import time
from zoneinfo import ZoneInfo
from phase2_gate_outcome_review import validate_review
from track_b_gino_formal_guard import guard
from track_b_gino_import_checked_entry import accepted_environment
from track_b_gino_ordinal_recovery import ROOT as MEASUREMENT
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
ROOT = MEASUREMENT.with_name('track-b-gino-shapenet611-formal-2026-11-19')
SOURCES = Path(__file__).resolve().parent
OLD_FILES = {
'contract.json': 'ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a',
'complete.json': 'db3605159c5ade3deda5313ab409e2f1c8bf630511a617f527177da06d6c9a2d',
'recovery-settlement.json': '851eaedf716f01c1d80ce637f7de3edd0664c734bcbfed5622623d99f572fcec',
'worker-records/progress.json': '8f8fb1902c44e2f9c3198608cbecb3a74967ed00fdbcaf1e990d146c894b0daf',
}
DEVICES = {
'0': 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76',
'1': 'GPU-8b71b032-1b75-2891-935c-ce106a53034a',
'2': 'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c',
}
ORDINALS = {'0': 2, '1': 3, '2': 0}
INVENTORY = BASE / 'artifacts/track-b-gino-gpu2-ordinal-preparation-2026-11-19/physical-inventory.json'
INVENTORY_SHA = 'da2a7b1bf168c7cc4d45b6eaa2e007ae90cb3640eeadde94c10fd1d32fa30dd8'
TZ = ZoneInfo('Asia/Hong_Kong')
RESERVATION_START = datetime(2026, 11, 19, tzinfo=TZ)
RESERVATION_END = datetime(2026, 11, 20, tzinfo=TZ)
NEW_SOURCES = ('track_b_gino_formal.py', 'track_b_gino_formal_guard.py',
'track_b_gino_formal_worker.py')
def expected_contract():
old = {name: read(checked(MEASUREMENT / name, digest)) for name, digest in OLD_FILES.items()}
c = old['contract.json']
stop, epoch = old['complete.json'], old['worker-records/progress.json']
account = old['recovery-settlement.json']
if (stop['status'], stop['trigger'], stop['result_sha256']) != ('STOPPED', 'FOREIGN_GPU_OCCUPANT', None):
raise ValueError('Historical guard STOP differs')
if (epoch['stage'], epoch['seed'], epoch['epoch'], epoch['examples']) != ('EPOCH_COMPLETE', 0, 1, 500):
raise ValueError('Only the completed 500-example epoch can size this reproduction')
seconds = epoch['seconds']
inventory = read(checked(INVENTORY, INVENTORY_SHA))['inventory']
physical = {(int(index), uuid) for index, uuid, _, name in inventory if 'V100' in name}
if not {(ORDINALS[seed], uuid) for seed, uuid in DEVICES.items()} <= physical:
raise ValueError('Reviewed physical V100 inventory differs')
budgets = {seed: {'fit': 13033 + (seed == '0'), 'score': 300} for seed in DEVICES}
fit_reference = 100 * seconds * 1.25
if (not math.isfinite(fit_reference) or fit_reference <= 0
or min(b['fit'] for b in budgets.values()) <= fit_reference
or sum(b['fit'] + b['score'] for b in budgets.values()) != 40000):
raise ValueError('Formal booking must cover the sizing reference and score reserve')
c['mode'] = 'formal-gino-reproduction'
for key in ('gpu_uuid', 'measurement_allowed_heldout_labels', 'gpu_ruling', 'ordinal_recovery'):
del c[key]
c['code_sha256'].update({str(SOURCES / name): sha(SOURCES / name) for name in NEW_SOURCES})
c['formal'] = dict(
handoff='h-4e4e680b', ruling_slack_ts='1795074651.222599', devices=DEVICES,
device_pci_ordinals=ORDINALS,
reservation=dict(handoff='h-a429935e', timezone=TZ.key,
start_inclusive=RESERVATION_START.isoformat(),
end_exclusive=RESERVATION_END.isoformat(), gpu3_single_task=True,
historical_inventory_path=str(INVENTORY), historical_inventory_sha256=INVENTORY_SHA,
runtime_idle_check_required=True),
budget_per_seed_stage_gpu_seconds=budgets, total_gpu_wall_seconds=40000,
timing=dict(record_paths={str(MEASUREMENT / k): v for k, v in OLD_FILES.items()},
status='CONTENTION_AFFECTED_NOT_GUARD_CERTIFIED', sizing_only=True,
epoch_seconds=seconds, train_sizing_gpu_seconds=300 * seconds * 1.25,
train_sizing_multiplier=1.25, score_reserved_gpu_seconds=900,
prior_measurement_gpu_seconds=account['cumulative_measurement_gpu_wall_seconds']),
heldout_policy=dict(preparation_reads=0, training_reads=0, score_reads_per_seed=111,
all_epoch100_checkpoints_required=True, retry=False),
)
del c['limits']['gpu_wall_seconds']
c['limits']['cleanup_seconds'] = 20
del c['limits']['cpu_seconds']
c['limits']['cpu_seconds_per_gpu_second'] = 8
return c
def contract():
c = read(ROOT / 'contract.json')
if c != expected_contract():
raise PermissionError('The sole formal contract or its implementation changed')
for path, digest in c['code_sha256'].items():
checked(path, digest)
for name in ('paper_supplement', 'split_receipt', 'mapping_audit'):
pin = c[name]
checked(pin['path'], pin['sha256'])
# Archive members are opened only in the scoped workers, which verify each
# member before using it. Preparation does not hash the archive payload.
comparison = read(checked(c['comparison_contract'], c['comparison_contract_sha256']))
split = read(c['split_receipt']['path'])
train, test = split['train'], split['test']
if (c['protocol'] != comparison['protocol'] or len(train) != 500 or len(test) != 111
or len(set(train) | set(test)) != 611
or any(c['split'][k] != split[k] for k in ('train', 'test', 'raw_global_bounds'))
or c['protocol']['epochs_per_seed'] != 100 or c['protocol']['seeds'] != [0, 1, 2]
or c['protocol']['acceptance_percent'] != [6.62, 7.62]
or c['paper_compatibility']['identity'] != 'supplement code + 2-keyword patch'
or not Path(c['paper_data']['path']).is_file()):
raise ValueError('Original GINO comparison, split or pinned data path differs')
accepted_environment(c)
return c
def authorize(c):
validate_review(ROOT)
authorization = read(ROOT / 'stage-authorization.json')
required = dict(contract_sha256=sha(ROOT / 'contract.json'),
implementation_proof_sha256=sha(ROOT / 'implementation-git-freeze.json'),
decision='APPROVED', mode=c['mode'], reserved_devices=DEVICES,
reservation_end_exclusive=RESERVATION_END.isoformat())
if any(authorization.get(k) != v for k, v in required.items()):
raise PermissionError('Exact execution and card-reservation authorization required')
def check_reservation(unix, duration):
if (unix < RESERVATION_START.timestamp()
or unix + duration >= RESERVATION_END.timestamp()):
raise PermissionError('The entire booking must fit the November 19 Hong Kong reservation')
def run_root(seed, phase):
return ROOT / f'seed-{seed}' / phase
def epoch100(root, seed, c):
progress = read(root / 'worker-records/progress.json')
if (progress['stage'], progress['seed'], progress['epoch'], progress['examples']) != (
'EPOCH_COMPLETE', seed, c['protocol']['epochs_per_seed'], 500):
raise ValueError('No completed 100th full training epoch for this seed')
return progress
def checkpoint(seed, result):
path = run_root(seed, 'fit') / 'epoch-100.pt'
if result['checkpoint']['path'] != str(path):
raise ValueError('Checkpoint path differs from this seed')
return checked(path, result['checkpoint']['sha256'])
def score_mean(rows, test_ids):
if [r['case'] for r in rows] != test_ids:
raise ValueError('Scoring requires the original 111-case order')
values = [r['relative_l2'] for r in rows]
if any(type(v) not in (int, float) or not math.isfinite(v) or v < 0 for v in values):
raise ValueError('Incomplete or invalid relative L2')
return 100 * (math.fsum(values) / len(values))
def validate_result(c, seed, phase):
root = run_root(seed, phase)
path = root / 'worker-records/result.json'
result = read(path)
spawned = read(root / 'actual-worker-spawn.json')
device = DEVICES[str(seed)]
ordinal = str(ORDINALS[str(seed)])
if (spawned['physical_uuid'], spawned['ordinal']) != (device, ordinal) or (
read(root / 'worker-device-verified.json') != dict(
process=spawned['process'], physical_uuid=device, ordinal=ordinal, observed_uuids=[device])):
raise ValueError('Worker has no matching physical-device witness')
if (result['phase'], result['seed'], result['status']) != (phase, seed, 'COMPLETE'):
raise ValueError('Worker result identity differs')
if phase == 'fit':
epoch100(root, seed, c)
checkpoint(seed, result)
if (result['target_reads'], result['epochs'], result['training_examples']) != (0, 100, 500):
raise ValueError('Fit must complete all 100 epochs with zero held-out reads')
else:
fit_terminal = read(run_root(seed, 'fit') / 'complete.json')
fit_path = run_root(seed, 'fit') / 'worker-records/result.json'
original = read(checked(fit_path, fit_terminal['result_sha256']))
access = root / 'heldout-target-access.jsonl'
mapping = {row['case']: row['pressure_sha256'] for row in read(c['mapping_audit']['path'])['rows']}
actual = [json.loads(line) for line in access.read_text().splitlines()]
expected = [dict(case=i, role='heldout', sha256=mapping[i]) for i in c['split']['test']]
if (actual != expected or result['target_reads'] != 111
or result['target_access_sha256'] != sha(access)
or result['checkpoints_manifest_sha256'] != sha(root / 'checkpoints-frozen.json')
or read(root / 'checkpoints-frozen.json') != all_checkpoints(c)
or result['checkpoint_sha256'] != original['checkpoint']['sha256']
or score_mean(result['cases'], c['split']['test']) != result['mean_percent']):
raise ValueError('Score or held-out target audit differs')
return sha(path)
def accepted_terminal(c, seed, phase):
root = run_root(seed, phase)
record = read(root / 'complete.json')
if (record['status'], record['seed'], record['phase'], record['contract_sha256'], record['physical_uuid']) != (
'COMPLETE', seed, phase, sha(ROOT / 'contract.json'), DEVICES[str(seed)]):
raise PermissionError(f'Seed {seed} {phase} has no accepted terminal result')
booked = c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase]
if (record['booked_gpu_seconds'] != booked or record['charged_gpu_seconds'] > booked
or record['result_sha256'] != validate_result(c, seed, phase)):
raise ValueError('Terminal result bytes or resource accounting differ')
return record
def all_checkpoints(c):
checkpoints = {}
for seed in c['protocol']['seeds']:
accepted_terminal(c, seed, 'fit')
root = run_root(seed, 'fit')
checkpoints[str(seed)] = dict(
result_sha256=sha(root / 'worker-records/result.json'),
terminal_sha256=sha(root / 'complete.json'),
checkpoint=read(root / 'worker-records/result.json')['checkpoint'])
return checkpoints
def dispatch(c, seed, phase):
began = time.monotonic()
unix = time.time()
authorize(c)
budget = c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase]
check_reservation(unix, budget)
seed_root = run_root(seed, phase).parent
seed_root.mkdir(exist_ok=True)
with (seed_root / 'execution.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
roster = all_checkpoints(c) if phase == 'score' else None
root = run_root(seed, phase)
root.mkdir(exist_ok=False)
if roster is not None:
save(root / 'checkpoints-frozen.json', roster)
ticket = dict(seed=seed, phase=phase, ordinal=str(ORDINALS[str(seed)]),
booked_gpu_seconds=budget, unix=unix,
contract_sha256=sha(ROOT / 'contract.json'),
guard_started_monotonic=began, booked_deadline_monotonic=began + budget)
save(root / 'started.json', ticket)
command = [c['environment']['python'], str(SOURCES / 'track_b_gino_formal_worker.py'),
str(seed), phase]
def request_stop(signum, _frame):
raise KeyboardInterrupt(signal.Signals(signum).name)
previous = signal.signal(signal.SIGTERM, request_stop)
try:
limits = dict(c['limits'], cpu_seconds=c['limits']['cpu_seconds_per_gpu_second'] * budget)
result = guard(root, command, uuid=DEVICES[str(seed)], limits=limits,
deadline=began + budget, ticket=ticket,
validate_result=lambda: validate_result(c, seed, phase))
finally:
signal.signal(signal.SIGTERM, previous)
return 0 if result['status'] == 'COMPLETE' else 1
def pool(means, limits):
if len(means) != 3 or any(v is None for v in means):
return None, None
mean = math.fsum(means) / 3
return mean, limits[0] <= mean <= limits[1]
def collect(c):
seeds = {}
accounted = unsettled = 0
for seed in c['protocol']['seeds']:
rows = {}
for phase in ('fit', 'score'):
root = run_root(seed, phase)
terminal = root / 'complete.json'
if terminal.exists():
record = read(terminal)
if record['status'] == 'COMPLETE':
accepted_terminal(c, seed, phase)
accounted += record['charged_gpu_seconds']
rows[phase] = dict(status=record['status'], trigger=record['trigger'],
terminal_sha256=sha(terminal))
if phase == 'score' and record['status'] == 'COMPLETE':
rows['mean_percent'] = read(root / 'worker-records/result.json')['mean_percent']
elif (root / 'started.json').exists():
rows[phase] = dict(status='NO_TERMINAL_RECEIPT')
unsettled += c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase]
else:
rows[phase] = dict(status='NOT_STARTED')
seeds[str(seed)] = rows
mean, verdict = pool([row.get('mean_percent') for row in seeds.values()],
c['protocol']['acceptance_percent'])
prior = c['formal']['timing']['prior_measurement_gpu_seconds']
return dict(seeds=seeds, mean_percent=mean, within_registered_range=verdict,
reproduction_identity=c['paper_compatibility']['identity'],
prior_measurement_gpu_seconds=prior, settled_formal_gpu_seconds=accounted,
settled_combined_gpu_seconds=accounted + prior,
unsettled_formal_bookings_gpu_seconds=unsettled, formal_gpu_cap_seconds=40000)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('fit', 'score', 'collect'))
parser.add_argument('--seed', type=int, choices=(0, 1, 2))
args = parser.parse_args()
c = contract()
if args.action == 'collect':
if args.seed is not None:
parser.error('collect reports all three seeds')
print(json.dumps(collect(c), indent=2, allow_nan=False))
else:
if args.seed is None:
parser.error('--seed is required for fit or score')
raise SystemExit(dispatch(c, args.seed, args.action))