210 lines
13 KiB
Python
210 lines
13 KiB
Python
"""Prepare a grouped two-fold GINO-Greif entry from frozen H2 metadata."""
|
||
|
||
import argparse
|
||
import json
|
||
import math
|
||
from pathlib import Path
|
||
|
||
import track_b_gino_formal as reference
|
||
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
|
||
|
||
|
||
ROOT = BASE / 'artifacts/track-b-gino-greif-preparation-2026-11-19'
|
||
REVIEW_ROOT = ROOT / 'adapter-review'
|
||
STATUS = 'CPU_ADAPTER_READY_PENDING_EXECUTION_BINDING_AND_MEASURED_COST'
|
||
PILOT = BASE / 'artifacts/phase2-h2-pilot-2026-11-17/contract.json'
|
||
QUERY = Path('/home/frank14f/MeltMind-runs/phase2-h2-s2-postfit-2026-11-17/contract.json')
|
||
H2_REFERENCE = (QUERY.parent, QUERY.parent.with_name('phase3-seed1-postfit-2026-11-18'))
|
||
H2_PRESSURE_AUTH = BASE / 'artifacts/track-b-h2-pressure-combined-2026-11-19/stage-authorization.json'
|
||
H2_PRESSURE_AUTH_SHA = '42ef5913159cb0b2320f1ff1fb4fe5c2460d98fa51b240974ed662c4b7c84d0a'
|
||
PHASE_TERMINAL = BASE / 'artifacts/phase3-item1b-fold0-labelled-recovery-2026-11-19/complete.json'
|
||
PINS = {
|
||
str(PILOT): '9f616974fb01f8f5cab6233273d541b3e53f5975f3632db45ab4676f911fba7b',
|
||
str(QUERY): '0bd03f42c3b2db5bef26708b2ef873edfe674f724b59bc4b4de8641980d8d330',
|
||
str(PHASE_TERMINAL): '631a4c679d94466af71ff79e72cb3066c073eaa7d68030805419ff28a74b476e',
|
||
}
|
||
ROLES = ('learning', 'validation', 'test')
|
||
CAP = 200 * 3600
|
||
EPOCHS = 200
|
||
|
||
|
||
def grouped_folds(pilot, query):
|
||
included = {row['case']: row for row in query['rows'] if row['included']}
|
||
fold_records = {}
|
||
for fold in (0, 1):
|
||
data = pilot['folds'][fold]
|
||
if data['fold'] != fold or any(pilot['geometry'][case]['protocol'] != 'grouped' for case in included):
|
||
raise ValueError('Frozen grouped Greif protocol required')
|
||
owners = {}
|
||
sizes = {}
|
||
for role in ROLES:
|
||
cases = data[role]
|
||
if len(set(cases)) != len(cases):
|
||
raise ValueError('Duplicate grouped role member')
|
||
for case in cases:
|
||
row = included[case]
|
||
family = row['component_id']
|
||
assigned = [item['role'] for item in row['assignments'] if item['fold'] == fold]
|
||
if assigned != [role] or owners.setdefault(family, role) != role:
|
||
raise ValueError(f'Grouped family or role crossed a boundary in fold {fold}')
|
||
sizes[role] = len(cases)
|
||
if set().union(*(set(data[role]) for role in ROLES)) != set(included):
|
||
raise ValueError(f'Incomplete frozen support for fold {fold}')
|
||
fold_records[str(fold)] = dict(counts=sizes, clock_coefficient=data['c'],
|
||
clock_sha256=data['clock_sha256'],
|
||
learning_nodes=data['learning_nodes'],
|
||
family_counts={role: sum(r == role for r in owners.values()) for role in ROLES},
|
||
parts={role: data[role] for role in ROLES})
|
||
return fold_records
|
||
|
||
|
||
def project(phase_charged, approvals, epoch_seconds_by_fold, epochs, overhead_gpu_seconds):
|
||
if set(approvals) != {'gino_shapenet', 'h2_pressure', 'other_unsettled'}:
|
||
raise ValueError('List all three other phase-3 reservations exactly once')
|
||
if set(epoch_seconds_by_fold) != {'0', '1'} or type(epochs) is not int or epochs <= 0:
|
||
raise ValueError('Each frozen fold needs its own measured full epoch')
|
||
basis = list(epoch_seconds_by_fold.values())
|
||
amounts = [phase_charged, overhead_gpu_seconds, *approvals.values()]
|
||
if (any(not math.isfinite(v) or v < 0 for v in amounts)
|
||
or any(not math.isfinite(v) or v <= 0 for v in basis)):
|
||
raise ValueError('Missing or invalid budget terms')
|
||
greif = math.ceil(1.25 * sum(basis) * 2 * epochs + overhead_gpu_seconds)
|
||
phase_total = phase_charged + sum(approvals.values()) + greif
|
||
return dict(greif_projection_gpu_seconds=greif, phase_projection_gpu_seconds=phase_total,
|
||
phase_cap_gpu_seconds=CAP, within_phase_cap=phase_total <= CAP)
|
||
|
||
|
||
def paired_support(pilot, query, folds):
|
||
"""Check the already frozen H2 reference roster without opening prediction or target arrays."""
|
||
family = {r['case']: r['component_id'] for r in query['rows'] if r['included']}
|
||
support, manifests, pins = {}, [], {}
|
||
for root in H2_REFERENCE:
|
||
manifest_path = root / 'predictions-manifest.json'
|
||
terminal_path = root / 'complete.json'
|
||
if read(terminal_path)['status'] != 'COMPLETE':
|
||
raise ValueError('Both H2 reference postfits must be complete')
|
||
rows = read(manifest_path)['entries']
|
||
row_map = {(str(r['fold']), r['case']): r for r in rows}
|
||
if len(row_map) != len(rows):
|
||
raise ValueError('Duplicate H2 reference predictions')
|
||
manifests.append(row_map)
|
||
pins.update({str(p): sha(p) for p in (manifest_path, terminal_path)})
|
||
for fold, data in folds.items():
|
||
for case in data['parts']['test']:
|
||
r0, r1 = (r[fold, case] for r in manifests)
|
||
expected = (family[case], pilot['parts'][case]['nodes'])
|
||
if (any((r['component_id'], r['nodes']) != expected for r in (r0, r1))
|
||
or r0['query_nodes'] != r1['query_nodes'] or r1['seed0_sha256'] != r0['sha256']
|
||
or r1['raw_ids_sha256'] != pilot['parts'][case]['output_sha256']['raw_ids.npy']):
|
||
raise ValueError('H2 paired fold/seed full node or query support differs')
|
||
support[fold] = dict(cases=len(data['parts']['test']), families=data['family_counts']['test'],
|
||
references=[0, 1], raw_prediction_or_target_arrays_opened=0)
|
||
return dict(status='REFERENCE_MANIFESTS_ALIGNED', folds=support, sha256=pins)
|
||
|
||
|
||
def phase_ledger(prior, timing, gino_cap):
|
||
auth = read(checked(H2_PRESSURE_AUTH, H2_PRESSURE_AUTH_SHA))
|
||
h2 = auth['phase3_booked']
|
||
if (h2['anchor_gpu_seconds'] != prior or h2['gino_formal_cap_gpu_seconds'] != gino_cap
|
||
or h2['gino_historical_measurement_gpu_seconds_once'] != timing['prior_measurement_gpu_seconds']
|
||
or h2['h2_pressure_formal_cap_gpu_seconds'] != 40000):
|
||
raise ValueError('Previously adopted phase-3 bookings differ')
|
||
historic = timing['prior_measurement_gpu_seconds']
|
||
total = prior + historic + gino_cap + h2['h2_pressure_formal_cap_gpu_seconds']
|
||
if total != h2['total_gpu_seconds'] or total >= CAP:
|
||
raise ValueError('Phase-3 published accounting differs')
|
||
return dict(settled_phase3_through_1b_gpu_seconds=prior,
|
||
gino_measurement_once_gpu_seconds=historic, gino_shapenet_formal_reserve_gpu_seconds=gino_cap,
|
||
h2_pressure_unsettled_max_gpu_seconds=40000, accounted_gpu_seconds=total,
|
||
phase_cap_gpu_seconds=CAP, upper_remaining_before_other_bookings_gpu_seconds=CAP-total,
|
||
h2_stage_authorization_sha256=H2_PRESSURE_AUTH_SHA,
|
||
other_unsettled_bookings_gpu_seconds=None, measured_greif_epoch_seconds=None,
|
||
complete_projection=None, status='HOLD_PENDING_DISPATCH_LEDGER_AND_TWO_MEASURED_EPOCHS')
|
||
|
||
|
||
def first_epoch_gate(candidate, receipts, *, other_unsettled, approved_greif_cap, remaining_overhead):
|
||
"""Charge the two initial full epochs via their bookings once, then project only remaining work."""
|
||
p = candidate['projection']
|
||
amounts = (other_unsettled, approved_greif_cap, remaining_overhead)
|
||
if any(type(v) not in (int, float) or not math.isfinite(v) or v < 0 for v in amounts) or approved_greif_cap == 0:
|
||
raise ValueError('Explicit phase reservations, approved Greif cap and overhead needed')
|
||
cap = min(approved_greif_cap, p['upper_remaining_before_other_bookings_gpu_seconds'] - other_unsettled)
|
||
if set(receipts) != set(candidate['folds']):
|
||
return dict(status='HOLD_MISSING_BOTH_EPOCH_RECEIPTS', release_epoch2=False)
|
||
spent = work = 0.
|
||
for fold, expected in candidate['folds'].items():
|
||
r = receipts[fold]
|
||
if (r['fold'], r['seed'], r['epoch'], r['learning_cases'], r['learning_nodes'], r['heldout_target_reads']) != (
|
||
int(fold), 0, 1, len(expected['parts']['learning']), expected['learning_nodes'], 0):
|
||
raise ValueError('First epoch receipt is not this fold’s full learning scope')
|
||
t, charge = r['epoch_seconds'], r['booked_gpu_seconds']
|
||
if not all(type(v) in (float,int) and math.isfinite(v) for v in (t,charge)) or not 0 < t <= charge:
|
||
raise ValueError('Missing measured full epoch or booked setup time')
|
||
spent += charge
|
||
work += (2 * candidate['epochs'] - 1) * t
|
||
projected = spent + math.ceil(1.25 * work) + remaining_overhead
|
||
within = projected <= cap
|
||
return dict(status='COST_WITHIN_CAP' if within else 'STOP_PROJECTED_CAP',
|
||
charged_first_epochs_and_setup_gpu_seconds=spent, remaining_train_gpu_seconds=work,
|
||
remaining_overhead_gpu_seconds=remaining_overhead, safety_factor=1.25,
|
||
projected_greif_gpu_seconds=projected, phase_projection_gpu_seconds=p['accounted_gpu_seconds'] +
|
||
other_unsettled + projected, effective_greif_cap_gpu_seconds=cap,
|
||
release_epoch2=within)
|
||
|
||
|
||
def binding():
|
||
from track_b_gino_greif_adapter import COLUMNS, GRID, MODES, WIDTH
|
||
|
||
pilot, query, terminal = (read(checked(path, PINS[str(path)])) for path in (PILOT, QUERY, PHASE_TERMINAL))
|
||
folds = grouped_folds(pilot, query)
|
||
reference_formal = reference.contract()
|
||
timing = reference_formal['formal']['timing']
|
||
prior = terminal['cumulative_phase3_gpu_seconds']
|
||
return dict(status=STATUS, handoff='h-1789ca7e', working_entry='preparation_only',
|
||
folds=folds, seeds=[0, 1], epochs=EPOCHS, h2_reference_support=paired_support(pilot, query, folds),
|
||
jobs=[dict(fold=fold, seed=seed) for fold in (0, 1) for seed in (0, 1)],
|
||
sources=PINS,
|
||
reuse_h2=dict(features='original 19 standardized on each fold learning only', geometry='full retained tet nodes',
|
||
mass='original positive nodal mass', input_clock='original FP64 Vf and frozen learning c',
|
||
train_target='same normalized residual as H2 candidate', readout='same cF and seconds decode',
|
||
case_query='original fixed PPS/query multiplicity and raw IDs', target_reads_in_preparation=0),
|
||
adapter_candidate=dict(identity='GINO-Greif fixed trilinear GNO / Tucker FNO / trilinear GNO',
|
||
resolution=[GRID]*3, width=WIDTH, modes=list(MODES), columns=list(COLUMNS),
|
||
first_grid_input='nineteen mass-weighted trilinear means and one occupancy bit',
|
||
coordinates='isotropic part bounding cube, one grid cell margin',
|
||
readout='tri-linear gather at original nodes plus same original 19 column skip',
|
||
pressure_or_sdf_channels_added=False,
|
||
model_environment=reference_formal['environment']),
|
||
training_candidate=dict(schedule='same H2 AdamW, 10 warmup, 190 cosine, 200 full epochs',
|
||
epoch_shuffle='PCG64 SeedSequence([seed, epoch])', loss='same H2 node-equal per-part weighted FP32 MSE and gradient clip 1',
|
||
no_validation_selection=True, match='exact fold × seed, matching H2 features and residual target',
|
||
model_choice_before_launch=True),
|
||
scoring_custody=dict(review_gate=True, require_all_four_final_checkpoints=True,
|
||
predictions_all_full_nodes_frozen_before_target_access=True,
|
||
planned_single_read_of_each_original_test_case=True,
|
||
paper_gate_applies_to_shape_pressure_only=True, use_frozen_pps_multiplicity=True,
|
||
report=['relative L2','last_location/Rg','Kendall tau-b'],
|
||
topology_weld_line_claims=False),
|
||
projection=phase_ledger(prior, timing, reference_formal['formal']['total_gpu_wall_seconds']),
|
||
admission=dict(preferred_ordinals=[0,1,2], last_ordinal=3, gpu3_single_task=True,
|
||
require_actual_stage_approval=True, max_two_initial_seed0_full_epochs_before_projection=True),
|
||
production_guard_limits=reference_formal['limits'],
|
||
launch_authorized=False,
|
||
code_sha256={str(path): sha(path) for path in (
|
||
Path(__file__).resolve(), reference.SOURCES/'phase2_h2_data.py',
|
||
reference.SOURCES/'phase2_residual_math.py', reference.SOURCES/'track_b_gino_formal_guard.py',
|
||
reference.SOURCES/'s2_recovery_gpu_supervisor.py', reference.SOURCES/'track_b_gino_greif_adapter.py',
|
||
reference.SOURCES/'gre24_features_v2.py', reference.SOURCES/'s2_custody_statistics.py',
|
||
reference.SOURCES/'phase2_h2_model.py')})
|
||
|
||
|
||
if __name__ == '__main__':
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument('--write-binding', action='store_true')
|
||
args = parser.parse_args()
|
||
c = binding()
|
||
if args.write_binding:
|
||
REVIEW_ROOT.mkdir(exist_ok=False)
|
||
save(REVIEW_ROOT/'preparation-binding.json', c)
|
||
print(json.dumps({key: c[key] for key in ('status', 'jobs', 'projection', 'launch_authorized')}, indent=2))
|