Files
MeltMind/tools/track_b_gino_greif_prepare.py

210 lines
13 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Prepare a grouped two-fold GINO-Greif entry from frozen H2 metadata."""
import argparse
import json
import math
from pathlib import Path
import track_b_gino_formal as reference
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
ROOT = BASE / 'artifacts/track-b-gino-greif-preparation-2026-11-19'
REVIEW_ROOT = ROOT / 'adapter-review'
STATUS = 'CPU_ADAPTER_READY_PENDING_EXECUTION_BINDING_AND_MEASURED_COST'
PILOT = BASE / 'artifacts/phase2-h2-pilot-2026-11-17/contract.json'
QUERY = Path('/home/frank14f/MeltMind-runs/phase2-h2-s2-postfit-2026-11-17/contract.json')
H2_REFERENCE = (QUERY.parent, QUERY.parent.with_name('phase3-seed1-postfit-2026-11-18'))
H2_PRESSURE_AUTH = BASE / 'artifacts/track-b-h2-pressure-combined-2026-11-19/stage-authorization.json'
H2_PRESSURE_AUTH_SHA = '42ef5913159cb0b2320f1ff1fb4fe5c2460d98fa51b240974ed662c4b7c84d0a'
PHASE_TERMINAL = BASE / 'artifacts/phase3-item1b-fold0-labelled-recovery-2026-11-19/complete.json'
PINS = {
str(PILOT): '9f616974fb01f8f5cab6233273d541b3e53f5975f3632db45ab4676f911fba7b',
str(QUERY): '0bd03f42c3b2db5bef26708b2ef873edfe674f724b59bc4b4de8641980d8d330',
str(PHASE_TERMINAL): '631a4c679d94466af71ff79e72cb3066c073eaa7d68030805419ff28a74b476e',
}
ROLES = ('learning', 'validation', 'test')
CAP = 200 * 3600
EPOCHS = 200
def grouped_folds(pilot, query):
included = {row['case']: row for row in query['rows'] if row['included']}
fold_records = {}
for fold in (0, 1):
data = pilot['folds'][fold]
if data['fold'] != fold or any(pilot['geometry'][case]['protocol'] != 'grouped' for case in included):
raise ValueError('Frozen grouped Greif protocol required')
owners = {}
sizes = {}
for role in ROLES:
cases = data[role]
if len(set(cases)) != len(cases):
raise ValueError('Duplicate grouped role member')
for case in cases:
row = included[case]
family = row['component_id']
assigned = [item['role'] for item in row['assignments'] if item['fold'] == fold]
if assigned != [role] or owners.setdefault(family, role) != role:
raise ValueError(f'Grouped family or role crossed a boundary in fold {fold}')
sizes[role] = len(cases)
if set().union(*(set(data[role]) for role in ROLES)) != set(included):
raise ValueError(f'Incomplete frozen support for fold {fold}')
fold_records[str(fold)] = dict(counts=sizes, clock_coefficient=data['c'],
clock_sha256=data['clock_sha256'],
learning_nodes=data['learning_nodes'],
family_counts={role: sum(r == role for r in owners.values()) for role in ROLES},
parts={role: data[role] for role in ROLES})
return fold_records
def project(phase_charged, approvals, epoch_seconds_by_fold, epochs, overhead_gpu_seconds):
if set(approvals) != {'gino_shapenet', 'h2_pressure', 'other_unsettled'}:
raise ValueError('List all three other phase-3 reservations exactly once')
if set(epoch_seconds_by_fold) != {'0', '1'} or type(epochs) is not int or epochs <= 0:
raise ValueError('Each frozen fold needs its own measured full epoch')
basis = list(epoch_seconds_by_fold.values())
amounts = [phase_charged, overhead_gpu_seconds, *approvals.values()]
if (any(not math.isfinite(v) or v < 0 for v in amounts)
or any(not math.isfinite(v) or v <= 0 for v in basis)):
raise ValueError('Missing or invalid budget terms')
greif = math.ceil(1.25 * sum(basis) * 2 * epochs + overhead_gpu_seconds)
phase_total = phase_charged + sum(approvals.values()) + greif
return dict(greif_projection_gpu_seconds=greif, phase_projection_gpu_seconds=phase_total,
phase_cap_gpu_seconds=CAP, within_phase_cap=phase_total <= CAP)
def paired_support(pilot, query, folds):
"""Check the already frozen H2 reference roster without opening prediction or target arrays."""
family = {r['case']: r['component_id'] for r in query['rows'] if r['included']}
support, manifests, pins = {}, [], {}
for root in H2_REFERENCE:
manifest_path = root / 'predictions-manifest.json'
terminal_path = root / 'complete.json'
if read(terminal_path)['status'] != 'COMPLETE':
raise ValueError('Both H2 reference postfits must be complete')
rows = read(manifest_path)['entries']
row_map = {(str(r['fold']), r['case']): r for r in rows}
if len(row_map) != len(rows):
raise ValueError('Duplicate H2 reference predictions')
manifests.append(row_map)
pins.update({str(p): sha(p) for p in (manifest_path, terminal_path)})
for fold, data in folds.items():
for case in data['parts']['test']:
r0, r1 = (r[fold, case] for r in manifests)
expected = (family[case], pilot['parts'][case]['nodes'])
if (any((r['component_id'], r['nodes']) != expected for r in (r0, r1))
or r0['query_nodes'] != r1['query_nodes'] or r1['seed0_sha256'] != r0['sha256']
or r1['raw_ids_sha256'] != pilot['parts'][case]['output_sha256']['raw_ids.npy']):
raise ValueError('H2 paired fold/seed full node or query support differs')
support[fold] = dict(cases=len(data['parts']['test']), families=data['family_counts']['test'],
references=[0, 1], raw_prediction_or_target_arrays_opened=0)
return dict(status='REFERENCE_MANIFESTS_ALIGNED', folds=support, sha256=pins)
def phase_ledger(prior, timing, gino_cap):
auth = read(checked(H2_PRESSURE_AUTH, H2_PRESSURE_AUTH_SHA))
h2 = auth['phase3_booked']
if (h2['anchor_gpu_seconds'] != prior or h2['gino_formal_cap_gpu_seconds'] != gino_cap
or h2['gino_historical_measurement_gpu_seconds_once'] != timing['prior_measurement_gpu_seconds']
or h2['h2_pressure_formal_cap_gpu_seconds'] != 40000):
raise ValueError('Previously adopted phase-3 bookings differ')
historic = timing['prior_measurement_gpu_seconds']
total = prior + historic + gino_cap + h2['h2_pressure_formal_cap_gpu_seconds']
if total != h2['total_gpu_seconds'] or total >= CAP:
raise ValueError('Phase-3 published accounting differs')
return dict(settled_phase3_through_1b_gpu_seconds=prior,
gino_measurement_once_gpu_seconds=historic, gino_shapenet_formal_reserve_gpu_seconds=gino_cap,
h2_pressure_unsettled_max_gpu_seconds=40000, accounted_gpu_seconds=total,
phase_cap_gpu_seconds=CAP, upper_remaining_before_other_bookings_gpu_seconds=CAP-total,
h2_stage_authorization_sha256=H2_PRESSURE_AUTH_SHA,
other_unsettled_bookings_gpu_seconds=None, measured_greif_epoch_seconds=None,
complete_projection=None, status='HOLD_PENDING_DISPATCH_LEDGER_AND_TWO_MEASURED_EPOCHS')
def first_epoch_gate(candidate, receipts, *, other_unsettled, approved_greif_cap, remaining_overhead):
"""Charge the two initial full epochs via their bookings once, then project only remaining work."""
p = candidate['projection']
amounts = (other_unsettled, approved_greif_cap, remaining_overhead)
if any(type(v) not in (int, float) or not math.isfinite(v) or v < 0 for v in amounts) or approved_greif_cap == 0:
raise ValueError('Explicit phase reservations, approved Greif cap and overhead needed')
cap = min(approved_greif_cap, p['upper_remaining_before_other_bookings_gpu_seconds'] - other_unsettled)
if set(receipts) != set(candidate['folds']):
return dict(status='HOLD_MISSING_BOTH_EPOCH_RECEIPTS', release_epoch2=False)
spent = work = 0.
for fold, expected in candidate['folds'].items():
r = receipts[fold]
if (r['fold'], r['seed'], r['epoch'], r['learning_cases'], r['learning_nodes'], r['heldout_target_reads']) != (
int(fold), 0, 1, len(expected['parts']['learning']), expected['learning_nodes'], 0):
raise ValueError('First epoch receipt is not this fold’s full learning scope')
t, charge = r['epoch_seconds'], r['booked_gpu_seconds']
if not all(type(v) in (float,int) and math.isfinite(v) for v in (t,charge)) or not 0 < t <= charge:
raise ValueError('Missing measured full epoch or booked setup time')
spent += charge
work += (2 * candidate['epochs'] - 1) * t
projected = spent + math.ceil(1.25 * work) + remaining_overhead
within = projected <= cap
return dict(status='COST_WITHIN_CAP' if within else 'STOP_PROJECTED_CAP',
charged_first_epochs_and_setup_gpu_seconds=spent, remaining_train_gpu_seconds=work,
remaining_overhead_gpu_seconds=remaining_overhead, safety_factor=1.25,
projected_greif_gpu_seconds=projected, phase_projection_gpu_seconds=p['accounted_gpu_seconds'] +
other_unsettled + projected, effective_greif_cap_gpu_seconds=cap,
release_epoch2=within)
def binding():
from track_b_gino_greif_adapter import COLUMNS, GRID, MODES, WIDTH
pilot, query, terminal = (read(checked(path, PINS[str(path)])) for path in (PILOT, QUERY, PHASE_TERMINAL))
folds = grouped_folds(pilot, query)
reference_formal = reference.contract()
timing = reference_formal['formal']['timing']
prior = terminal['cumulative_phase3_gpu_seconds']
return dict(status=STATUS, handoff='h-1789ca7e', working_entry='preparation_only',
folds=folds, seeds=[0, 1], epochs=EPOCHS, h2_reference_support=paired_support(pilot, query, folds),
jobs=[dict(fold=fold, seed=seed) for fold in (0, 1) for seed in (0, 1)],
sources=PINS,
reuse_h2=dict(features='original 19 standardized on each fold learning only', geometry='full retained tet nodes',
mass='original positive nodal mass', input_clock='original FP64 Vf and frozen learning c',
train_target='same normalized residual as H2 candidate', readout='same cF and seconds decode',
case_query='original fixed PPS/query multiplicity and raw IDs', target_reads_in_preparation=0),
adapter_candidate=dict(identity='GINO-Greif fixed trilinear GNO / Tucker FNO / trilinear GNO',
resolution=[GRID]*3, width=WIDTH, modes=list(MODES), columns=list(COLUMNS),
first_grid_input='nineteen mass-weighted trilinear means and one occupancy bit',
coordinates='isotropic part bounding cube, one grid cell margin',
readout='tri-linear gather at original nodes plus same original 19 column skip',
pressure_or_sdf_channels_added=False,
model_environment=reference_formal['environment']),
training_candidate=dict(schedule='same H2 AdamW, 10 warmup, 190 cosine, 200 full epochs',
epoch_shuffle='PCG64 SeedSequence([seed, epoch])', loss='same H2 node-equal per-part weighted FP32 MSE and gradient clip 1',
no_validation_selection=True, match='exact fold × seed, matching H2 features and residual target',
model_choice_before_launch=True),
scoring_custody=dict(review_gate=True, require_all_four_final_checkpoints=True,
predictions_all_full_nodes_frozen_before_target_access=True,
planned_single_read_of_each_original_test_case=True,
paper_gate_applies_to_shape_pressure_only=True, use_frozen_pps_multiplicity=True,
report=['relative L2','last_location/Rg','Kendall tau-b'],
topology_weld_line_claims=False),
projection=phase_ledger(prior, timing, reference_formal['formal']['total_gpu_wall_seconds']),
admission=dict(preferred_ordinals=[0,1,2], last_ordinal=3, gpu3_single_task=True,
require_actual_stage_approval=True, max_two_initial_seed0_full_epochs_before_projection=True),
production_guard_limits=reference_formal['limits'],
launch_authorized=False,
code_sha256={str(path): sha(path) for path in (
Path(__file__).resolve(), reference.SOURCES/'phase2_h2_data.py',
reference.SOURCES/'phase2_residual_math.py', reference.SOURCES/'track_b_gino_formal_guard.py',
reference.SOURCES/'s2_recovery_gpu_supervisor.py', reference.SOURCES/'track_b_gino_greif_adapter.py',
reference.SOURCES/'gre24_features_v2.py', reference.SOURCES/'s2_custody_statistics.py',
reference.SOURCES/'phase2_h2_model.py')})
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--write-binding', action='store_true')
args = parser.parse_args()
c = binding()
if args.write_binding:
REVIEW_ROOT.mkdir(exist_ok=False)
save(REVIEW_ROOT/'preparation-binding.json', c)
print(json.dumps({key: c[key] for key in ('status', 'jobs', 'projection', 'launch_authorized')}, indent=2))