347 lines
18 KiB
Python
347 lines
18 KiB
Python
"""One reviewed, cost-gated H2-P pipeline with frozen three-seed custody."""
|
|
|
|
import argparse
|
|
from collections import defaultdict
|
|
import fcntl
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import signal
|
|
import time
|
|
|
|
from phase2_gate_outcome_review import validate_review
|
|
from s2_gpu_idle_gate import snapshot
|
|
import track_b_gino_formal as reference
|
|
from track_b_gino_formal_guard import guard
|
|
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
|
|
|
|
|
|
ROOT = BASE / 'artifacts/track-b-h2-pressure-combined-2026-11-19'
|
|
REFERENCE_SHA = 'c3c5c5f0fab049c757032c343673104eb0f99cbaead5e0f78247f2a3aa6173c3'
|
|
DEVICES = {'0': reference.DEVICES['2'], '1': 'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
|
|
'2': reference.DEVICES['0']}
|
|
SEEDS = (0, 1, 2)
|
|
CAP = 40000
|
|
SCORE_RESERVE = 300
|
|
BOOTSTRAP_REPS = 2000
|
|
BOOTSTRAP_SEED = 20261119
|
|
SOURCES = Path(__file__).resolve().parent
|
|
|
|
|
|
def geometry_components(c):
|
|
mapping = read(c['mapping_audit']['path'])['rows']
|
|
by_case = {row['case']: row['mesh_sha256'] for row in mapping}
|
|
if set(by_case) != set(c['split']['train'] + c['split']['test']):
|
|
raise ValueError('Mesh identity roster differs')
|
|
if {by_case[i] for i in c['split']['train']} & {by_case[i] for i in c['split']['test']}:
|
|
raise ValueError('Identical raw mesh would cross the fixed learning and held-out split')
|
|
groups = defaultdict(list)
|
|
for case in c['split']['test']:
|
|
groups[by_case[case]].append(case)
|
|
return list(groups.values())
|
|
|
|
|
|
def expected_comparison(c):
|
|
return dict(handoff='h-b5985d03', identity='H2-P', seeds=list(SEEDS),
|
|
train_cases=c['split']['train'], heldout_cases=c['split']['test'],
|
|
pressure_selection='original raw rows [0:16] and [112:] in original vertex order',
|
|
metric=c['protocol']['metric'], summary='equal cases, then equal matched seeds',
|
|
component_definition='exact source triangle-mesh SHA-256, geometry-only; no known family identifiers',
|
|
heldout_components=geometry_components(c),
|
|
bootstrap=dict(replicates=BOOTSTRAP_REPS, seed=BOOTSTRAP_SEED, confidence=.95,
|
|
method='paired component resampling with replacement; case weighted; all three paired seeds retained'),
|
|
paired_delta='H2-P minus GINO in percentage points',
|
|
gate='all three exact seed pairs valid, GINO reproduction mean in [6.62,7.62], paired CI high < 0',
|
|
no_test_selection=True)
|
|
|
|
|
|
def expected_contract():
|
|
base = reference.contract()
|
|
base_path = checked(reference.ROOT / 'contract.json', REFERENCE_SHA)
|
|
ref = dict(path=str(base_path), sha256=REFERENCE_SHA,
|
|
comparison_sha256=base['comparison_contract_sha256'])
|
|
geometry = ROOT / 'geometry-audit.json'
|
|
audit = read(geometry)
|
|
if (audit['status'] != 'PASS' or audit['mesh_members_opened'] != 611 or audit['target_members_opened'] != 0
|
|
or [r['case'] for r in audit['rows']] != base['split']['train'] + base['split']['test']):
|
|
raise ValueError('Complete geometry-only coverage is required')
|
|
comparison = ROOT / 'comparison-contract.json'
|
|
if read(comparison) != expected_comparison(base):
|
|
raise ValueError('Pre-registered comparison differs')
|
|
c = {key: base[key] for key in (
|
|
'paper_data', 'paper_supplement', 'paper_code_zip_sha256', 'paper_compatibility',
|
|
'split', 'split_receipt', 'mapping_audit', 'protocol', 'environment', 'environment_ruling',
|
|
'environment_acceptance', 'limits', 'code_sha256')}
|
|
c['code_sha256'].update({str(SOURCES / name): sha(SOURCES / name) for name in (
|
|
'track_b_h2_pressure_entry.py', 'track_b_h2_pressure_worker.py',
|
|
'track_b_h2_pressure_prepare.py', 'phase2_h2_model.py')})
|
|
c.update(mode='h2-pressure-combined', identity='H2-P', handoff='h-b5985d03', reference=ref,
|
|
geometry_audit=dict(path=str(geometry), sha256=sha(geometry)),
|
|
comparison_contract=str(comparison), comparison_contract_sha256=sha(comparison),
|
|
devices=DEVICES, first_epoch_seed=0, total_gpu_seconds=CAP, scoring_seconds_per_seed=SCORE_RESERVE,
|
|
timing_margin=1.25, future_fit_setup_floor_seconds=120, gpu3_eligible=False,
|
|
model=dict(inputs=['x','y','z'], edges='original triangle edges both directions',
|
|
mass='one-third incident area', message_components='surface edge connectivity',
|
|
architecture='shared four 64-wide H2 blocks and direct pressure head',
|
|
optimizer_and_case_loss='same Adam, StepLR and LpLoss as fixed GINO training body'),
|
|
custody=dict(train_reads_per_seed=500, heldout_reads_per_seed=111,
|
|
require_all_frozen_checkpoints=True, retries=0))
|
|
return c
|
|
|
|
|
|
def contract():
|
|
c = read(ROOT / 'contract.json')
|
|
if c != expected_contract():
|
|
raise PermissionError('H2-P contract or source binding changed')
|
|
for path, digest in c['code_sha256'].items():
|
|
checked(path, digest)
|
|
return c
|
|
|
|
|
|
def authorize(c):
|
|
review = validate_review(ROOT)
|
|
auth = read(ROOT / 'stage-authorization.json')
|
|
approved = auth.get('eligible_ordinals')
|
|
expires = auth.get('dispatch_before_unix')
|
|
if ((auth['decision'], auth['mode'], auth['contract_sha256'], auth['review_sha256']) !=
|
|
('APPROVED', c['mode'], sha(ROOT / 'contract.json'), sha(ROOT / 'review.json'))
|
|
or review['verdict'] != 'PASS'
|
|
or not isinstance(approved, list) or not approved or len(set(approved)) != len(approved)
|
|
or not set(approved) <= set(DEVICES)
|
|
or type(expires) not in (float, int) or not math.isfinite(expires)):
|
|
raise PermissionError('Exact independent PASS and stage/card authorization required')
|
|
return auth
|
|
|
|
|
|
def first_epoch_projection(ticket, progress, now):
|
|
if tuple(progress[k] for k in ('stage', 'seed', 'epoch', 'examples')) != ('EPOCH_COMPLETE', 0, 1, 500):
|
|
raise ValueError('Only seed 0 first complete 500-case epoch measures cost')
|
|
epoch = progress['seconds']
|
|
spent = now - ticket['guard_started_monotonic']
|
|
setup = spent - epoch
|
|
if not math.isfinite(epoch) or not math.isfinite(setup) or epoch <= 0 or setup < 0:
|
|
raise ValueError('First full epoch and setup clock differ')
|
|
future_setup = max(120, math.ceil(1.25 * setup))
|
|
remaining = math.ceil(299 * epoch * 1.25) + 2 * future_setup + 3 * SCORE_RESERVE + 60
|
|
projected = spent + remaining
|
|
return dict(seed=0, epoch=1, examples=500, epoch_seconds=epoch, booked_seconds_so_far=spent,
|
|
measured_setup_seconds=setup, future_fit_setup_reserve_seconds=2 * future_setup,
|
|
remaining_train_epochs=299, safety_factor=1.25, scoring_reserve_seconds=3 * SCORE_RESERVE,
|
|
fit_cleanup_reserve_seconds=60, projected_total_gpu_seconds=projected,
|
|
formal_cap_gpu_seconds=CAP, within_cap=projected <= CAP)
|
|
|
|
|
|
def run_root(seed, phase):
|
|
return ROOT / f'seed-{seed}' / phase
|
|
|
|
|
|
score_mean = reference.score_mean
|
|
|
|
|
|
def validate_result(c, seed, phase):
|
|
root = run_root(seed, phase)
|
|
path = root / 'worker-records/result.json'
|
|
result = read(path)
|
|
ticket = read(root / 'started.json')
|
|
ordinal, uuid = ticket['ordinal'], ticket['physical_uuid']
|
|
spawn = read(root / 'actual-worker-spawn.json')
|
|
if (DEVICES[ordinal] != uuid or spawn['physical_uuid'] != uuid or spawn['ordinal'] != ordinal
|
|
or read(root / 'worker-device-verified.json') !=
|
|
dict(process=spawn['process'], physical_uuid=uuid, ordinal=ordinal, observed_uuids=[uuid])):
|
|
raise ValueError('Actual H2-P worker/device witness differs')
|
|
if (result['status'], result['seed'], result['phase']) != ('COMPLETE', seed, phase):
|
|
raise ValueError('Worker result identity differs')
|
|
if phase == 'fit':
|
|
reference.epoch100(root, seed, c)
|
|
checkpoint = result['checkpoint']
|
|
if (checkpoint['path'] != str(root / 'epoch-100.pt')
|
|
or (result['epochs'], result['training_examples'], result['target_reads']) != (100, 500, 0)
|
|
or (root / 'heldout-target-access.jsonl').exists()):
|
|
raise ValueError('Fit checkpoint or held-out seal differs')
|
|
checked(checkpoint['path'], checkpoint['sha256'])
|
|
if seed == 0 and read(root / 'first-epoch-projection.json')['within_cap'] is not True:
|
|
raise ValueError('Missing first epoch cost PASS')
|
|
else:
|
|
manifest = root / 'checkpoints-frozen.json'
|
|
roster = all_checkpoints(c)
|
|
access = root / 'heldout-target-access.jsonl'
|
|
mapping = {row['case']: row['pressure_sha256'] for row in read(c['mapping_audit']['path'])['rows']}
|
|
rows = [json.loads(line) for line in access.read_text().splitlines()]
|
|
expected = [dict(case=i, role='heldout', sha256=mapping[i]) for i in c['split']['test']]
|
|
if (read(manifest) != roster or result['checkpoints_manifest_sha256'] != sha(manifest)
|
|
or result['checkpoint_sha256'] != roster[str(seed)]['checkpoint']['sha256']
|
|
or result['target_reads'] != 111 or result['target_access_sha256'] != sha(access)
|
|
or rows != expected or score_mean(result['cases'], c['split']['test']) != result['mean_percent']):
|
|
raise ValueError('Held-out custody or score differs')
|
|
return sha(path)
|
|
|
|
|
|
def accepted_terminal(c, seed, phase):
|
|
root = run_root(seed, phase)
|
|
terminal = read(root / 'complete.json')
|
|
if (terminal['status'], terminal['seed'], terminal['phase'], terminal['contract_sha256']) != (
|
|
'COMPLETE', seed, phase, sha(ROOT / 'contract.json')):
|
|
raise PermissionError('No exact accepted stage terminal')
|
|
if (terminal['booked_gpu_cap_exceeded'] or terminal['charged_gpu_seconds'] > terminal['booked_gpu_seconds']
|
|
or terminal['result_sha256'] != validate_result(c, seed, phase)):
|
|
raise ValueError('H2-P terminal budget or result differs')
|
|
return terminal
|
|
|
|
|
|
def all_checkpoints(c):
|
|
roster = {}
|
|
for seed in SEEDS:
|
|
terminal = accepted_terminal(c, seed, 'fit')
|
|
root = run_root(seed, 'fit')
|
|
result = read(root / 'worker-records/result.json')
|
|
roster[str(seed)] = dict(checkpoint=result['checkpoint'], terminal_sha256=sha(root / 'complete.json'),
|
|
result_sha256=terminal['result_sha256'])
|
|
return roster
|
|
|
|
|
|
def run_stage(c, seed, phase, ordinal, auth, began, deadline, roster=None):
|
|
root = run_root(seed, phase)
|
|
root.mkdir(parents=True, exist_ok=False)
|
|
if phase == 'score':
|
|
save(root / 'checkpoints-frozen.json', roster)
|
|
uuid = DEVICES[ordinal]
|
|
ticket = dict(seed=seed, phase=phase, ordinal=ordinal, physical_uuid=uuid, unix=time.time(),
|
|
contract_sha256=sha(ROOT / 'contract.json'), dispatch_before_unix=auth['dispatch_before_unix'],
|
|
guard_started_monotonic=began, booked_deadline_monotonic=deadline,
|
|
booked_gpu_seconds=deadline - began)
|
|
save(root / 'started.json', ticket)
|
|
command = [c['environment']['python'], '-B', str(SOURCES / 'track_b_h2_pressure_worker.py'), str(seed), phase]
|
|
limits = dict(c['limits'], cpu_seconds=8 * (deadline - began))
|
|
return guard(root, command, uuid=uuid, limits=limits, deadline=deadline, ticket=ticket,
|
|
validate_result=lambda: validate_result(c, seed, phase))
|
|
|
|
|
|
def run(c):
|
|
auth = authorize(c)
|
|
with (ROOT / 'combined.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
if (ROOT / 'started.json').exists() or any(ROOT.glob('seed-*')):
|
|
raise FileExistsError('H2-P combined attempt already exists')
|
|
if time.time() >= auth['dispatch_before_unix']:
|
|
raise PermissionError('Card reservation for new dispatch has expired')
|
|
ordinal = next((i for i, uuid in DEVICES.items()
|
|
if i in auth['eligible_ordinals'] and snapshot(uuid)['idle']), None)
|
|
if ordinal is None:
|
|
raise RuntimeError('NO_IDLE_APPROVED_GPU0_TO_2')
|
|
began = time.monotonic()
|
|
deadline = began + CAP
|
|
save(ROOT / 'started.json', dict(ordinal=ordinal, physical_uuid=DEVICES[ordinal],
|
|
started_monotonic=began, contract_sha256=sha(ROOT / 'contract.json'), phase_budget=CAP))
|
|
|
|
def request_stop(number, _frame):
|
|
raise KeyboardInterrupt(signal.Signals(number).name)
|
|
|
|
old = signal.signal(signal.SIGTERM, request_stop)
|
|
status, trigger = 'STOPPED', None
|
|
try:
|
|
fit_deadline = deadline - len(SEEDS) * SCORE_RESERVE
|
|
for seed in SEEDS:
|
|
epoch_start = began if seed == 0 else time.monotonic()
|
|
if epoch_start >= fit_deadline:
|
|
raise RuntimeError('INSUFFICIENT_FIT_BUDGET')
|
|
receipt = run_stage(c, seed, 'fit', ordinal, auth, epoch_start, fit_deadline)
|
|
if receipt['status'] != 'COMPLETE':
|
|
cost = run_root(0, 'fit') / 'first-epoch-projection.json'
|
|
if seed == 0 and cost.exists() and read(cost)['within_cap'] is False:
|
|
raise RuntimeError('STOP_PROJECTED_CAP')
|
|
raise RuntimeError(f"Seed {seed} fit HOLD: {receipt['trigger']}")
|
|
roster = all_checkpoints(c)
|
|
save(ROOT / 'checkpoints-frozen.json', roster)
|
|
for seed in SEEDS:
|
|
score_start = time.monotonic()
|
|
score_deadline = min(deadline, score_start + SCORE_RESERVE)
|
|
if score_deadline - score_start <= c['limits']['cleanup_seconds']:
|
|
raise RuntimeError('INSUFFICIENT_SCORE_BUDGET')
|
|
receipt = run_stage(c, seed, 'score', ordinal, auth, score_start, score_deadline, roster)
|
|
if receipt['status'] != 'COMPLETE':
|
|
raise RuntimeError(f"Seed {seed} score HOLD: {receipt['trigger']}")
|
|
status = 'COMPLETE'
|
|
except (Exception, KeyboardInterrupt) as exc:
|
|
trigger = str(exc) or type(exc).__name__
|
|
finally:
|
|
signal.signal(signal.SIGTERM, old)
|
|
charged = time.monotonic() - began
|
|
if charged > CAP and status == 'COMPLETE':
|
|
status, trigger = 'STOPPED', 'GPU_BUDGET_OVERAGE'
|
|
save(ROOT / 'complete.json', dict(status=status, trigger=trigger, charged_gpu_seconds=charged,
|
|
cap_gpu_seconds=CAP, no_retry=True))
|
|
return 0 if status == 'COMPLETE' else 1
|
|
|
|
|
|
def paired_bootstrap(comparison, h2, gino):
|
|
import numpy as np
|
|
|
|
ids = comparison['heldout_cases']
|
|
groups = comparison['heldout_components']
|
|
if sorted(case for group in groups for case in group) != sorted(ids) or min(map(len, groups)) < 1:
|
|
raise ValueError('Fixed paired test component support differs')
|
|
if len(h2) != 3 or len(gino) != 3:
|
|
raise ValueError('All three exact matched seeds are required')
|
|
h2_means = [score_mean(rows, ids) for rows in h2]
|
|
gino_means = [score_mean(rows, ids) for rows in gino]
|
|
deltas = np.array([[a['relative_l2'] - b['relative_l2'] for a, b in zip(hr, gr)]
|
|
for hr, gr in zip(h2, gino)], dtype=np.float64).mean(axis=0)
|
|
per_case = dict(zip(ids, deltas))
|
|
masses = np.array([len(group) for group in groups])
|
|
sums = np.array([sum(per_case[case] for case in group) for group in groups])
|
|
drawn = np.random.default_rng(BOOTSTRAP_SEED).integers(len(groups), size=(BOOTSTRAP_REPS, len(groups)))
|
|
sampled = 100 * sums[drawn].sum(axis=1) / masses[drawn].sum(axis=1)
|
|
low, high = map(float, np.quantile(sampled, [.025, .975]))
|
|
reference_mean, within = reference.pool(gino_means, [6.62, 7.62])
|
|
return dict(h2_percent_by_seed=dict(zip(SEEDS, h2_means)), gino_percent_by_seed=dict(zip(SEEDS, gino_means)),
|
|
gino_mean_percent=reference_mean, gino_reproduction_pass=within,
|
|
paired_mean_difference_pp=float(100 * deltas.mean()), paired_ci95_pp=[low, high],
|
|
components=len(groups), decision=('GINO_REPRODUCTION_GATE_NOT_MET' if not within else
|
|
'H2_P_IMPROVEMENT_PASS' if high < 0 else 'H2_P_IMPROVEMENT_NOT_ESTABLISHED'))
|
|
|
|
|
|
def collect(c):
|
|
seeds, scores = {}, []
|
|
for seed in SEEDS:
|
|
rows = {}
|
|
for phase in ('fit', 'score'):
|
|
root = run_root(seed, phase)
|
|
if (root / 'complete.json').exists():
|
|
terminal = read(root / 'complete.json')
|
|
rows[phase] = dict(status=terminal['status'], trigger=terminal['trigger'])
|
|
if phase == 'score' and terminal['status'] == 'COMPLETE':
|
|
accepted_terminal(c, seed, phase)
|
|
result = read(root / 'worker-records/result.json')
|
|
rows['mean_percent'] = result['mean_percent']
|
|
scores.append(result['cases'])
|
|
else:
|
|
rows[phase] = dict(status='UNSETTLED' if (root / 'started.json').exists() else 'NOT_STARTED')
|
|
seeds[str(seed)] = rows
|
|
report = dict(identity='H2-P', seeds=seeds, paired_status='PENDING_ALL_ACCEPTED_SEEDS')
|
|
if len(scores) != 3 or not (ROOT / 'complete.json').exists() or read(ROOT / 'complete.json')['status'] != 'COMPLETE':
|
|
return report
|
|
gino = reference.contract()
|
|
validate_review(reference.ROOT)
|
|
refs = []
|
|
for seed in SEEDS:
|
|
path = reference.run_root(seed, 'score')
|
|
terminal = path / 'complete.json'
|
|
if not terminal.exists():
|
|
return report
|
|
if read(terminal)['status'] != 'COMPLETE':
|
|
return dict(report, paired_status='GINO_SEED_SCORE_UNAVAILABLE')
|
|
reference.accepted_terminal(gino, seed, 'score')
|
|
refs.append(read(path / 'worker-records/result.json')['cases'])
|
|
stats = paired_bootstrap(read(c['comparison_contract']), scores, refs)
|
|
return dict(report, paired_status=stats['decision'], comparison=stats)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('action', choices=('run', 'collect'))
|
|
args = parser.parse_args()
|
|
c = contract()
|
|
if args.action == 'run':
|
|
raise SystemExit(run(c))
|
|
print(json.dumps(collect(c), indent=2, allow_nan=False))
|