Files
MeltMind/tools/track_b_h2_pressure_entry.py

347 lines
18 KiB
Python

"""One reviewed, cost-gated H2-P pipeline with frozen three-seed custody."""
import argparse
from collections import defaultdict
import fcntl
import json
import math
from pathlib import Path
import signal
import time
from phase2_gate_outcome_review import validate_review
from s2_gpu_idle_gate import snapshot
import track_b_gino_formal as reference
from track_b_gino_formal_guard import guard
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
ROOT = BASE / 'artifacts/track-b-h2-pressure-combined-2026-11-19'
REFERENCE_SHA = 'c3c5c5f0fab049c757032c343673104eb0f99cbaead5e0f78247f2a3aa6173c3'
DEVICES = {'0': reference.DEVICES['2'], '1': 'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
'2': reference.DEVICES['0']}
SEEDS = (0, 1, 2)
CAP = 40000
SCORE_RESERVE = 300
BOOTSTRAP_REPS = 2000
BOOTSTRAP_SEED = 20261119
SOURCES = Path(__file__).resolve().parent
def geometry_components(c):
mapping = read(c['mapping_audit']['path'])['rows']
by_case = {row['case']: row['mesh_sha256'] for row in mapping}
if set(by_case) != set(c['split']['train'] + c['split']['test']):
raise ValueError('Mesh identity roster differs')
if {by_case[i] for i in c['split']['train']} & {by_case[i] for i in c['split']['test']}:
raise ValueError('Identical raw mesh would cross the fixed learning and held-out split')
groups = defaultdict(list)
for case in c['split']['test']:
groups[by_case[case]].append(case)
return list(groups.values())
def expected_comparison(c):
return dict(handoff='h-b5985d03', identity='H2-P', seeds=list(SEEDS),
train_cases=c['split']['train'], heldout_cases=c['split']['test'],
pressure_selection='original raw rows [0:16] and [112:] in original vertex order',
metric=c['protocol']['metric'], summary='equal cases, then equal matched seeds',
component_definition='exact source triangle-mesh SHA-256, geometry-only; no known family identifiers',
heldout_components=geometry_components(c),
bootstrap=dict(replicates=BOOTSTRAP_REPS, seed=BOOTSTRAP_SEED, confidence=.95,
method='paired component resampling with replacement; case weighted; all three paired seeds retained'),
paired_delta='H2-P minus GINO in percentage points',
gate='all three exact seed pairs valid, GINO reproduction mean in [6.62,7.62], paired CI high < 0',
no_test_selection=True)
def expected_contract():
base = reference.contract()
base_path = checked(reference.ROOT / 'contract.json', REFERENCE_SHA)
ref = dict(path=str(base_path), sha256=REFERENCE_SHA,
comparison_sha256=base['comparison_contract_sha256'])
geometry = ROOT / 'geometry-audit.json'
audit = read(geometry)
if (audit['status'] != 'PASS' or audit['mesh_members_opened'] != 611 or audit['target_members_opened'] != 0
or [r['case'] for r in audit['rows']] != base['split']['train'] + base['split']['test']):
raise ValueError('Complete geometry-only coverage is required')
comparison = ROOT / 'comparison-contract.json'
if read(comparison) != expected_comparison(base):
raise ValueError('Pre-registered comparison differs')
c = {key: base[key] for key in (
'paper_data', 'paper_supplement', 'paper_code_zip_sha256', 'paper_compatibility',
'split', 'split_receipt', 'mapping_audit', 'protocol', 'environment', 'environment_ruling',
'environment_acceptance', 'limits', 'code_sha256')}
c['code_sha256'].update({str(SOURCES / name): sha(SOURCES / name) for name in (
'track_b_h2_pressure_entry.py', 'track_b_h2_pressure_worker.py',
'track_b_h2_pressure_prepare.py', 'phase2_h2_model.py')})
c.update(mode='h2-pressure-combined', identity='H2-P', handoff='h-b5985d03', reference=ref,
geometry_audit=dict(path=str(geometry), sha256=sha(geometry)),
comparison_contract=str(comparison), comparison_contract_sha256=sha(comparison),
devices=DEVICES, first_epoch_seed=0, total_gpu_seconds=CAP, scoring_seconds_per_seed=SCORE_RESERVE,
timing_margin=1.25, future_fit_setup_floor_seconds=120, gpu3_eligible=False,
model=dict(inputs=['x','y','z'], edges='original triangle edges both directions',
mass='one-third incident area', message_components='surface edge connectivity',
architecture='shared four 64-wide H2 blocks and direct pressure head',
optimizer_and_case_loss='same Adam, StepLR and LpLoss as fixed GINO training body'),
custody=dict(train_reads_per_seed=500, heldout_reads_per_seed=111,
require_all_frozen_checkpoints=True, retries=0))
return c
def contract():
c = read(ROOT / 'contract.json')
if c != expected_contract():
raise PermissionError('H2-P contract or source binding changed')
for path, digest in c['code_sha256'].items():
checked(path, digest)
return c
def authorize(c):
review = validate_review(ROOT)
auth = read(ROOT / 'stage-authorization.json')
approved = auth.get('eligible_ordinals')
expires = auth.get('dispatch_before_unix')
if ((auth['decision'], auth['mode'], auth['contract_sha256'], auth['review_sha256']) !=
('APPROVED', c['mode'], sha(ROOT / 'contract.json'), sha(ROOT / 'review.json'))
or review['verdict'] != 'PASS'
or not isinstance(approved, list) or not approved or len(set(approved)) != len(approved)
or not set(approved) <= set(DEVICES)
or type(expires) not in (float, int) or not math.isfinite(expires)):
raise PermissionError('Exact independent PASS and stage/card authorization required')
return auth
def first_epoch_projection(ticket, progress, now):
if tuple(progress[k] for k in ('stage', 'seed', 'epoch', 'examples')) != ('EPOCH_COMPLETE', 0, 1, 500):
raise ValueError('Only seed 0 first complete 500-case epoch measures cost')
epoch = progress['seconds']
spent = now - ticket['guard_started_monotonic']
setup = spent - epoch
if not math.isfinite(epoch) or not math.isfinite(setup) or epoch <= 0 or setup < 0:
raise ValueError('First full epoch and setup clock differ')
future_setup = max(120, math.ceil(1.25 * setup))
remaining = math.ceil(299 * epoch * 1.25) + 2 * future_setup + 3 * SCORE_RESERVE + 60
projected = spent + remaining
return dict(seed=0, epoch=1, examples=500, epoch_seconds=epoch, booked_seconds_so_far=spent,
measured_setup_seconds=setup, future_fit_setup_reserve_seconds=2 * future_setup,
remaining_train_epochs=299, safety_factor=1.25, scoring_reserve_seconds=3 * SCORE_RESERVE,
fit_cleanup_reserve_seconds=60, projected_total_gpu_seconds=projected,
formal_cap_gpu_seconds=CAP, within_cap=projected <= CAP)
def run_root(seed, phase):
return ROOT / f'seed-{seed}' / phase
score_mean = reference.score_mean
def validate_result(c, seed, phase):
root = run_root(seed, phase)
path = root / 'worker-records/result.json'
result = read(path)
ticket = read(root / 'started.json')
ordinal, uuid = ticket['ordinal'], ticket['physical_uuid']
spawn = read(root / 'actual-worker-spawn.json')
if (DEVICES[ordinal] != uuid or spawn['physical_uuid'] != uuid or spawn['ordinal'] != ordinal
or read(root / 'worker-device-verified.json') !=
dict(process=spawn['process'], physical_uuid=uuid, ordinal=ordinal, observed_uuids=[uuid])):
raise ValueError('Actual H2-P worker/device witness differs')
if (result['status'], result['seed'], result['phase']) != ('COMPLETE', seed, phase):
raise ValueError('Worker result identity differs')
if phase == 'fit':
reference.epoch100(root, seed, c)
checkpoint = result['checkpoint']
if (checkpoint['path'] != str(root / 'epoch-100.pt')
or (result['epochs'], result['training_examples'], result['target_reads']) != (100, 500, 0)
or (root / 'heldout-target-access.jsonl').exists()):
raise ValueError('Fit checkpoint or held-out seal differs')
checked(checkpoint['path'], checkpoint['sha256'])
if seed == 0 and read(root / 'first-epoch-projection.json')['within_cap'] is not True:
raise ValueError('Missing first epoch cost PASS')
else:
manifest = root / 'checkpoints-frozen.json'
roster = all_checkpoints(c)
access = root / 'heldout-target-access.jsonl'
mapping = {row['case']: row['pressure_sha256'] for row in read(c['mapping_audit']['path'])['rows']}
rows = [json.loads(line) for line in access.read_text().splitlines()]
expected = [dict(case=i, role='heldout', sha256=mapping[i]) for i in c['split']['test']]
if (read(manifest) != roster or result['checkpoints_manifest_sha256'] != sha(manifest)
or result['checkpoint_sha256'] != roster[str(seed)]['checkpoint']['sha256']
or result['target_reads'] != 111 or result['target_access_sha256'] != sha(access)
or rows != expected or score_mean(result['cases'], c['split']['test']) != result['mean_percent']):
raise ValueError('Held-out custody or score differs')
return sha(path)
def accepted_terminal(c, seed, phase):
root = run_root(seed, phase)
terminal = read(root / 'complete.json')
if (terminal['status'], terminal['seed'], terminal['phase'], terminal['contract_sha256']) != (
'COMPLETE', seed, phase, sha(ROOT / 'contract.json')):
raise PermissionError('No exact accepted stage terminal')
if (terminal['booked_gpu_cap_exceeded'] or terminal['charged_gpu_seconds'] > terminal['booked_gpu_seconds']
or terminal['result_sha256'] != validate_result(c, seed, phase)):
raise ValueError('H2-P terminal budget or result differs')
return terminal
def all_checkpoints(c):
roster = {}
for seed in SEEDS:
terminal = accepted_terminal(c, seed, 'fit')
root = run_root(seed, 'fit')
result = read(root / 'worker-records/result.json')
roster[str(seed)] = dict(checkpoint=result['checkpoint'], terminal_sha256=sha(root / 'complete.json'),
result_sha256=terminal['result_sha256'])
return roster
def run_stage(c, seed, phase, ordinal, auth, began, deadline, roster=None):
root = run_root(seed, phase)
root.mkdir(parents=True, exist_ok=False)
if phase == 'score':
save(root / 'checkpoints-frozen.json', roster)
uuid = DEVICES[ordinal]
ticket = dict(seed=seed, phase=phase, ordinal=ordinal, physical_uuid=uuid, unix=time.time(),
contract_sha256=sha(ROOT / 'contract.json'), dispatch_before_unix=auth['dispatch_before_unix'],
guard_started_monotonic=began, booked_deadline_monotonic=deadline,
booked_gpu_seconds=deadline - began)
save(root / 'started.json', ticket)
command = [c['environment']['python'], '-B', str(SOURCES / 'track_b_h2_pressure_worker.py'), str(seed), phase]
limits = dict(c['limits'], cpu_seconds=8 * (deadline - began))
return guard(root, command, uuid=uuid, limits=limits, deadline=deadline, ticket=ticket,
validate_result=lambda: validate_result(c, seed, phase))
def run(c):
auth = authorize(c)
with (ROOT / 'combined.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (ROOT / 'started.json').exists() or any(ROOT.glob('seed-*')):
raise FileExistsError('H2-P combined attempt already exists')
if time.time() >= auth['dispatch_before_unix']:
raise PermissionError('Card reservation for new dispatch has expired')
ordinal = next((i for i, uuid in DEVICES.items()
if i in auth['eligible_ordinals'] and snapshot(uuid)['idle']), None)
if ordinal is None:
raise RuntimeError('NO_IDLE_APPROVED_GPU0_TO_2')
began = time.monotonic()
deadline = began + CAP
save(ROOT / 'started.json', dict(ordinal=ordinal, physical_uuid=DEVICES[ordinal],
started_monotonic=began, contract_sha256=sha(ROOT / 'contract.json'), phase_budget=CAP))
def request_stop(number, _frame):
raise KeyboardInterrupt(signal.Signals(number).name)
old = signal.signal(signal.SIGTERM, request_stop)
status, trigger = 'STOPPED', None
try:
fit_deadline = deadline - len(SEEDS) * SCORE_RESERVE
for seed in SEEDS:
epoch_start = began if seed == 0 else time.monotonic()
if epoch_start >= fit_deadline:
raise RuntimeError('INSUFFICIENT_FIT_BUDGET')
receipt = run_stage(c, seed, 'fit', ordinal, auth, epoch_start, fit_deadline)
if receipt['status'] != 'COMPLETE':
cost = run_root(0, 'fit') / 'first-epoch-projection.json'
if seed == 0 and cost.exists() and read(cost)['within_cap'] is False:
raise RuntimeError('STOP_PROJECTED_CAP')
raise RuntimeError(f"Seed {seed} fit HOLD: {receipt['trigger']}")
roster = all_checkpoints(c)
save(ROOT / 'checkpoints-frozen.json', roster)
for seed in SEEDS:
score_start = time.monotonic()
score_deadline = min(deadline, score_start + SCORE_RESERVE)
if score_deadline - score_start <= c['limits']['cleanup_seconds']:
raise RuntimeError('INSUFFICIENT_SCORE_BUDGET')
receipt = run_stage(c, seed, 'score', ordinal, auth, score_start, score_deadline, roster)
if receipt['status'] != 'COMPLETE':
raise RuntimeError(f"Seed {seed} score HOLD: {receipt['trigger']}")
status = 'COMPLETE'
except (Exception, KeyboardInterrupt) as exc:
trigger = str(exc) or type(exc).__name__
finally:
signal.signal(signal.SIGTERM, old)
charged = time.monotonic() - began
if charged > CAP and status == 'COMPLETE':
status, trigger = 'STOPPED', 'GPU_BUDGET_OVERAGE'
save(ROOT / 'complete.json', dict(status=status, trigger=trigger, charged_gpu_seconds=charged,
cap_gpu_seconds=CAP, no_retry=True))
return 0 if status == 'COMPLETE' else 1
def paired_bootstrap(comparison, h2, gino):
import numpy as np
ids = comparison['heldout_cases']
groups = comparison['heldout_components']
if sorted(case for group in groups for case in group) != sorted(ids) or min(map(len, groups)) < 1:
raise ValueError('Fixed paired test component support differs')
if len(h2) != 3 or len(gino) != 3:
raise ValueError('All three exact matched seeds are required')
h2_means = [score_mean(rows, ids) for rows in h2]
gino_means = [score_mean(rows, ids) for rows in gino]
deltas = np.array([[a['relative_l2'] - b['relative_l2'] for a, b in zip(hr, gr)]
for hr, gr in zip(h2, gino)], dtype=np.float64).mean(axis=0)
per_case = dict(zip(ids, deltas))
masses = np.array([len(group) for group in groups])
sums = np.array([sum(per_case[case] for case in group) for group in groups])
drawn = np.random.default_rng(BOOTSTRAP_SEED).integers(len(groups), size=(BOOTSTRAP_REPS, len(groups)))
sampled = 100 * sums[drawn].sum(axis=1) / masses[drawn].sum(axis=1)
low, high = map(float, np.quantile(sampled, [.025, .975]))
reference_mean, within = reference.pool(gino_means, [6.62, 7.62])
return dict(h2_percent_by_seed=dict(zip(SEEDS, h2_means)), gino_percent_by_seed=dict(zip(SEEDS, gino_means)),
gino_mean_percent=reference_mean, gino_reproduction_pass=within,
paired_mean_difference_pp=float(100 * deltas.mean()), paired_ci95_pp=[low, high],
components=len(groups), decision=('GINO_REPRODUCTION_GATE_NOT_MET' if not within else
'H2_P_IMPROVEMENT_PASS' if high < 0 else 'H2_P_IMPROVEMENT_NOT_ESTABLISHED'))
def collect(c):
seeds, scores = {}, []
for seed in SEEDS:
rows = {}
for phase in ('fit', 'score'):
root = run_root(seed, phase)
if (root / 'complete.json').exists():
terminal = read(root / 'complete.json')
rows[phase] = dict(status=terminal['status'], trigger=terminal['trigger'])
if phase == 'score' and terminal['status'] == 'COMPLETE':
accepted_terminal(c, seed, phase)
result = read(root / 'worker-records/result.json')
rows['mean_percent'] = result['mean_percent']
scores.append(result['cases'])
else:
rows[phase] = dict(status='UNSETTLED' if (root / 'started.json').exists() else 'NOT_STARTED')
seeds[str(seed)] = rows
report = dict(identity='H2-P', seeds=seeds, paired_status='PENDING_ALL_ACCEPTED_SEEDS')
if len(scores) != 3 or not (ROOT / 'complete.json').exists() or read(ROOT / 'complete.json')['status'] != 'COMPLETE':
return report
gino = reference.contract()
validate_review(reference.ROOT)
refs = []
for seed in SEEDS:
path = reference.run_root(seed, 'score')
terminal = path / 'complete.json'
if not terminal.exists():
return report
if read(terminal)['status'] != 'COMPLETE':
return dict(report, paired_status='GINO_SEED_SCORE_UNAVAILABLE')
reference.accepted_terminal(gino, seed, 'score')
refs.append(read(path / 'worker-records/result.json')['cases'])
stats = paired_bootstrap(read(c['comparison_contract']), scores, refs)
return dict(report, paired_status=stats['decision'], comparison=stats)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('run', 'collect'))
args = parser.parse_args()
c = contract()
if args.action == 'run':
raise SystemExit(run(c))
print(json.dumps(collect(c), indent=2, allow_nan=False))