Files
MeltMind/tools/freeze_g1_components.py

315 lines
18 KiB
Python

"""Metadata-only custody join, conservative components and deterministic S2 splits."""
import argparse
import collections
import csv
import hashlib
import json
import math
from pathlib import Path
import platform
import shutil
import zipfile
import numpy as np
BASE = Path('artifacts')
OUT = BASE / 's2-g1-components-2026-11-15'
REG = BASE / 's2-g1-registration-2026-11-14'
BAND = BASE / 's2-g1-band25-2026-11-15'
CMP = BASE / 's2-g1-band25-review-comparison-2026-11-15'
CONTRACT = Path('docs/s2-g1-component-split-contract-2026-11-15.md')
MERGE = 'MERGE_SAME_SIDE_LEAK_PROTECTION'
NOEDGE = 'NO_ADDITIONAL_EDGE_BOTH_UNRELATED'
EXPECTED = {
str(REG/'batch-complete.json'): 'b0ee7469c1ab755e809f8ff85ebc6ce940e9838a23eb091a4a7a0e91e77d0956',
str(REG/'contract.json'): 'c4271622e1775d430324dd8e8ad4524006184122079d2fc3bbb8fd0ba6a349d8',
str(REG/'custodian/plan.json'): '61f4695b7fdf16d747bdebafc1d0c6d362087e250e9d85a36ccce3b3caa71af6',
str(BAND/'custodian/mapping.json'): '7531566edf060cdae522b7cf32d0164d0cc6bd9d5b26be6a8d1c3a53bbf61142',
str(CMP/'comparison.csv'): 'e7603e06ac5529608c23b889475632b5d75cfbeea7686528460a6c81dcd10bfd',
str(CMP/'sources/mold/frozen-review.csv'): '6479eebdfc09629c7188f18f888494c5d9bea8dd160d92763813b15ef813ce18',
str(CMP/'sources/claude/band25-review.csv'): '93096734186c1247e46e98f26c7c9b0f7a4987c8d8d588e8147110ae3880be2e',
}
def sha(path):
h = hashlib.sha256()
with Path(path).open('rb') as f:
for block in iter(lambda: f.read(1048576), b''):
h.update(block)
return h.hexdigest()
def read(path):
return json.loads(Path(path).read_text())
def save(path, data):
with Path(path).open('x') as f:
json.dump(data, f, ensure_ascii=False, indent=2)
f.write('\n')
class DSU:
def __init__(self, members):
self.parent = {x: x for x in members}
def root(self, x):
while self.parent[x] != x:
self.parent[x] = self.parent[self.parent[x]]
x = self.parent[x]
return x
def union(self, a, b):
a, b = sorted((self.root(a), self.root(b)))
self.parent[b] = a
def action(a, b):
allowed = {'unrelated', 'UNRESOLVED', 'rigid / same part'}
assert a in allowed and b in allowed
return NOEDGE if a == b == 'unrelated' else MERGE
def order(ids, stage, fold):
ids = sorted(ids)
rng = np.random.Generator(np.random.PCG64(np.random.SeedSequence([2026111409, stage, fold])))
return [ids[int(i)] for i in rng.permutation(len(ids))]
def split_components(components):
by_id = {c['component_id']: c for c in components}
ids = sorted(by_id)
rank = {x: i for i, x in enumerate(order(ids, 0, 0))}
placement = sorted(ids, key=lambda x: (-by_id[x]['parts'], rank[x]))
folds = [[] for _ in range(5)]
parts = [0] * 5
for x in placement:
k = min(range(5), key=lambda k: (parts[k], len(folds[k]), k))
folds[k].append(x)
parts[k] += by_id[x]['parts']
rows = []
for k, test in enumerate(folds):
remaining = set(ids) - set(test)
val_order = order(remaining, 1, k)
val = val_order[:math.ceil(len(remaining)/5)]
learning = order(remaining-set(val), 2, k)
pools = {str(p): learning[:n] for p, n in [(25, math.ceil(len(learning)/4)), (50, math.ceil(len(learning)/2)), (100, len(learning))]}
reasons = []
if len(ids) < 5: reasons.append('FEWER_THAN_FIVE_COMPONENTS')
if not test: reasons.append('EMPTY_TEST_FOLD')
if not val: reasons.append('EMPTY_VALIDATION_POOL')
if not learning: reasons.append('EMPTY_LEARNING_POOL')
if len({len(v) for v in pools.values()}) < 3: reasons.append('COINCIDENT_LEARNING_TIERS')
assert not(set(test)&set(val) or set(test)&set(learning) or set(val)&set(learning))
assert set(test)|set(val)|set(learning) == set(ids)
assert pools['25'] == pools['50'][:len(pools['25'])]
assert pools['50'] == pools['100'][:len(pools['50'])]
rows.append(dict(fold=k, test=test, validation=val, validation_order=val_order,
learning_order=learning, pools=pools, noncomparability_reasons=reasons))
assert collections.Counter(x for f in folds for x in f) == collections.Counter(ids)
return dict(order_version='g1-pcg64-seedsequence-v1', seed=2026111409,
numpy_version=np.__version__, placement_order=placement, folds=rows)
def components_stage():
assert not OUT.exists(), 'Stage already exists: inspect rather than overwrite'
for p, digest in EXPECTED.items(): assert sha(p) == digest, p
for p, digest in read(CMP/'receipt.json')['source_sha256'].items():
assert sha(CMP/p) == digest, p
for p, digest in read(REG/'contract.json')['input_sha256'].items(): assert sha(p) == digest, p
terminal = read(REG/'batch-complete.json')
assert terminal['status'] == 'ALL_REGISTRATIONS_COMPLETE' and terminal['failed_jobs'] == 0
plan = read(REG/'custodian/plan.json')
assert len(terminal['jobs']) == len(plan['jobs']) == 2012
assert {j['index'] for j in terminal['jobs']} == set(range(2012))
assert all(j['returncode'] == 0 and j['worker']['status'] == 'PASS' and not j['worker']['retry'] for j in terminal['jobs'])
inventory_path = BASE/'s2-g1-inventory-2026-11-14/custodian/parts.json'
candidate_plan = BASE/'s2-g1-candidates-2026-11-14/custodian/plan.json'
old_join = BASE/'s2-g1-custody-handoff-2026-11-14/custodian/joined-frozen-review.json'
inventory = read(inventory_path)
by_case = {r['case']: r for r in inventory}
assert len(inventory) == len(by_case) == 624
assert set(plan['geometry_cache']) == set(by_case)
dev = set(read(candidate_plan)['development_cases'])
old = read(old_join)['pairs']
old_reviewed = {r[k] for r in old for k in ['a', 'b']}
assert all(by_case[r['a']]['source_key'] == by_case[r['b']]['source_key'] for r in old)
mapping = read(BAND/'custodian/mapping.json')
rows = list(csv.DictReader((CMP/'comparison.csv').open()))
original_fields = list(rows[0])
assert len(rows) == 25 and len({r['anonymous_pair_id'] for r in rows}) == 25
ids = {r['anonymous_pair_id'] for r in rows}
assert ids == {p['anonymous_pair_id'] for p in mapping['pairs']}
map_by_id = {p['anonymous_pair_id']: p for p in mapping['pairs']}
for r in rows: r['final_action'] = action(r['mold_relation'], r['claude_relation'])
assert {r['anonymous_pair_id'][:8] for r in rows if r['final_action'] == MERGE} == {'3058d0a6', '731fb76c', '1ead5efb', 'a7496efb'}
OUT.mkdir()
private = OUT/'custodian'; private.mkdir(mode=0o700)
public = OUT/'public'; public.mkdir()
shutil.copy2(CMP/'comparison.csv', public/'comparison-before-ruling.csv')
with (public/'comparison.csv').open('x') as f:
w = csv.DictWriter(f, original_fields+['final_action'], lineterminator='\n'); w.writeheader(); w.writerows(rows)
reread = list(csv.DictReader((public/'comparison.csv').open()))
assert [{k:r[k] for k in original_fields} for r in reread] == list(csv.DictReader((CMP/'comparison.csv').open()))
shutil.copy2(CONTRACT, public/'execution-contract.md')
input_sha = dict(EXPECTED)
for p in [inventory_path, candidate_plan, old_join, CONTRACT, Path(__file__)]: input_sha[str(p)] = sha(p)
source_groups = collections.defaultdict(list)
dsu = DSU(by_case)
edges = []
for case, r in by_case.items(): source_groups[r['source_key']].append(case)
assert len(source_groups) == 249
for key, members in sorted(source_groups.items()):
members.sort()
for b in members[1:]:
edges.append(dict(a=members[0], b=b, reason='SAME_SOURCE_KEY'))
dsu.union(members[0], b)
results = {}
counts = collections.Counter()
for i, pair in enumerate(plan['jobs']):
path = REG/f'custodian/results/pair-{i:04d}.json'
digest = sha(path); input_sha[str(path)] = digest
r = read(path)
assert r['index'] == i and r['pair'] == pair and r['simulation_fields_read'] is False
status = r['engineering_rule']['engineering_status']; counts[status] += 1
results[i] = r
if status == 'LINK_UNDER_FIXED_RULE':
edges.append(dict(a=pair['a'], b=pair['b'], reason=status, registration_index=i, result_sha256=digest))
dsu.union(pair['a'], pair['b'])
assert dict(counts) == {'OUTSIDE_FIXED_RULE':1897, 'BAND_REVIEW_REQUIRED':25, 'LINK_UNDER_FIXED_RULE':90}
assert {p['registration_index'] for p in mapping['pairs']} == {i for i,r in results.items() if r['engineering_rule']['engineering_status']=='BAND_REVIEW_REQUIRED'}
for path, digest in mapping['source_result_sha256'].items(): assert sha(path) == digest, path
joined = []
band_reviewed = set()
for r in rows:
m = map_by_id[r['anonymous_pair_id']]; i = m['registration_index']; pair = plan['jobs'][i]
endpoints = {s['case'] for s in m['sides'].values()}
assert endpoints == {pair['a'], pair['b']}
band_reviewed.update(endpoints)
joined.append(dict(**r, registration_index=i, a=pair['a'], b=pair['b'],
engineering_rule=results[i]['engineering_rule'], result_sha256=input_sha[str(REG/f'custodian/results/pair-{i:04d}.json')]))
if r['final_action'] == MERGE:
edges.append(dict(a=pair['a'], b=pair['b'], reason=MERGE, anonymous_pair_id=r['anonymous_pair_id'], registration_index=i))
dsu.union(pair['a'], pair['b'])
nodes = {}; node_sources = {}; exposure = {}
for case in sorted(by_case):
cache = plan['geometry_cache'][case]; path = Path(cache['path'])
assert sha(path) == cache['sha256'], case
with zipfile.ZipFile(path) as z:
info = z.getinfo('points.npy')
with z.open(info) as f:
version = np.lib.format.read_magic(f)
assert version == (1,0)
shape, fortran, dtype = np.lib.format.read_array_header_1_0(f)
header_bytes = f.tell()
assert len(shape) == 2 and shape[1] == 3 and shape[0] > 0 and not dtype.hasobject
assert info.file_size == header_bytes + math.prod(shape)*dtype.itemsize
nodes[case] = int(shape[0])
node_sources[case] = dict(**cache, points_shape=shape, points_dtype=str(dtype), header_bytes=header_bytes,
points_crc32=info.CRC, points_uncompressed_bytes=info.file_size, payload_deserialized=False)
inv = by_case[case]
exposure[case] = dict(inventory_state=inv['exposure_state'], old_30_pair_endpoint=inv['old_30_pair_endpoint'],
individual_diagnostic_stages=inv['individual_diagnostic_stages'], calibration_development=case in dev,
blind120_geometry_review=case in old_reviewed, band25_geometry_review=case in band_reviewed,
all_corpus_historical_screening=inv['all_corpus_historical_screening'], unexposed_claim=False,
development_exposed=inv['exposure_state']=='KNOWN_DEVELOPMENT_EXPOSED' or case in dev)
groups = collections.defaultdict(list)
for case in sorted(by_case): groups[dsu.root(case)].append(case)
components = []; private_components = []; case_component = {}
for members in groups.values():
cid = 'component-'+hashlib.sha256('\n'.join(members).encode()).hexdigest()[:16]
record = dict(component_id=cid, parts=len(members), nodes=sum(nodes[c] for c in members),
source_keys=len({by_case[c]['source_key'] for c in members}),
development_exposed_parts=sum(exposure[c]['development_exposed'] for c in members),
blind120_geometry_review_parts=sum(c in old_reviewed for c in members),
band25_geometry_review_parts=sum(c in band_reviewed for c in members), unexposed_claim=False)
components.append(record); private_components.append(dict(**record, members=members))
case_component.update({c:cid for c in members})
components.sort(key=lambda c:c['component_id'])
assert len({c['component_id'] for c in components}) == len(components)
assert all(case_component[e['a']] == case_component[e['b']] for e in edges)
key_public = []; key_private = []
for i,(key, members) in enumerate(sorted(source_groups.items())):
kid = f'key-{i:03d}'
row = dict(key_id=kid, parts=len(members), nodes=sum(nodes[c] for c in members), component_id=case_component[members[0]])
key_public.append(row); key_private.append(dict(**row, source_key=key, members=members))
total_nodes = sum(nodes.values())
largest = max(components, key=lambda c:(c['parts'], c['nodes'], c['component_id']))
largest_nodes = max(components, key=lambda c:(c['nodes'], c['parts'], c['component_id']))
def shares(c): return dict(**c, part_share=c['parts']/624, node_share=c['nodes']/total_nodes)
flags = [shares(c) for c in components if c['parts']*5 > 624 or c['nodes']*5 > total_nodes]
stats = dict(status='COMPONENTS_FROZEN_SPLITS_NOT_YET_RUN', parts=624, source_keys=249, nodes=total_nodes,
components=len(components), component_size_distribution=dict(sorted(collections.Counter(c['parts'] for c in components).items())),
source_key_size_distribution=dict(sorted(collections.Counter(len(m) for m in source_groups.values()).items())),
largest_by_parts=shares(largest), largest_by_nodes=shares(largest_nodes), oversized_report_flags=flags,
edge_counts=dict(collections.Counter(e['reason'] for e in edges)), registration_status_counts=dict(counts),
band_final_action_counts=dict(collections.Counter(r['final_action'] for r in rows)),
development_exposed_parts=sum(x['development_exposed'] for x in exposure.values()),
development_exposed_components=sum(c['development_exposed_parts']>0 for c in components),
labels_read=False, geometry_recomputed=False, equivalence_truth=False, cross_key_recall_certified=False,
ruling_message_ts='1794678062.264009', node_definition='Unique nodes used by tetrahedra; existing points.npy shape',
node_cache_sha256_all_verified=True, original_comparison_columns_preserved=True)
save(private/'joined-band-review.json', joined)
save(private/'edges.json', edges)
save(private/'components.json', private_components)
save(private/'source-keys.json', key_private)
save(private/'parts.json', [dict(case=c, source_key=by_case[c]['source_key'], component_id=case_component[c], nodes=nodes[c], exposure=exposure[c]) for c in sorted(by_case)])
save(private/'node-metadata-provenance.json', node_sources)
save(private/'input-sha256.json', input_sha)
save(public/'components.json', components)
save(public/'parts-per-key.json', key_public)
save(public/'component-statistics.json', stats)
receipt = dict(status='COMPONENTS_FROZEN', ruling_message_ts='1794678062.264009', python=platform.python_version(), numpy=np.__version__,
source_sha256=EXPECTED, contract_sha256=sha(CONTRACT), code_sha256=sha(__file__),
custody_artifact_sha256={p.name:sha(p) for p in sorted(private.iterdir())},
public_artifact_sha256={p.name:sha(p) for p in sorted(public.iterdir())},
reviewer_mapping_returned=False, original_source_files_unchanged=all(sha(p)==h for p,h in EXPECTED.items()))
save(public/'component-receipt.json', receipt)
for p in private.iterdir(): p.chmod(0o600)
print(json.dumps(stats, ensure_ascii=False))
def split_stage(report_ts):
public = OUT/'public'; private = OUT/'custodian'
assert not (public/'splits.json').exists(), 'Splits already exist'
stats = read(public/'component-statistics.json')
assert not stats['oversized_report_flags'] or report_ts, 'Report oversized components before splitting'
receipt = read(public/'component-receipt.json')
assert sha(CONTRACT) == receipt['contract_sha256'] and sha(__file__) == receipt['code_sha256']
for name, digest in receipt['custody_artifact_sha256'].items(): assert sha(private/name) == digest
for name, digest in receipt['public_artifact_sha256'].items(): assert sha(public/name) == digest
components = read(public/'components.json'); by_id = {c['component_id']:c for c in components}
splits = split_components(components)
def summarize(ids):
return dict(components=len(ids), parts=sum(by_id[c]['parts'] for c in ids), nodes=sum(by_id[c]['nodes'] for c in ids),
development_exposed_parts=sum(by_id[c]['development_exposed_parts'] for c in ids),
development_exposed_components=sum(by_id[c]['development_exposed_parts']>0 for c in ids), unexposed_claim=False)
for fold in splits['folds']:
fold['counts'] = dict(test=summarize(fold['test']), validation=summarize(fold['validation']),
pools={p:summarize(ids) for p,ids in fold['pools'].items()})
save(public/'splits.json', splits)
private_rows = []
for part in read(private/'parts.json'):
cid = part['component_id']; assignments = []
for f in splits['folds']:
role = 'test' if cid in f['test'] else 'validation' if cid in f['validation'] else 'learning'
assignments.append(dict(fold=f['fold'], role=role, learning_pools=[p for p,ids in f['pools'].items() if cid in ids]))
private_rows.append(dict(**part, assignments=assignments))
save(private/'part-splits.json', private_rows)
(private/'part-splits.json').chmod(0o600)
result = dict(status='COMPONENTS_AND_FIVE_FOLD_NESTED_POOLS_FROZEN_AWAITING_REVIEW',
contract_sha256=sha(CONTRACT), code_sha256=sha(__file__), components_receipt_sha256=sha(public/'component-receipt.json'),
splits_sha256=sha(public/'splits.json'), custody_part_splits_sha256=sha(private/'part-splits.json'),
oversized_components_report_message_ts=report_ts, no_component_cut=True,
all_required_edges_same_fold=True, component_test_partition_verified=True, nested_pools_verified=True,
no_geometry_registration_review_rerun=True, training_pilot_gpu=False,
fold_counts=[dict(fold=f['fold'], **f['counts'], noncomparability_reasons=f['noncomparability_reasons']) for f in splits['folds']])
save(public/'split-receipt.json', result)
print(json.dumps(result, ensure_ascii=False))
if __name__ == '__main__':
p = argparse.ArgumentParser(); p.add_argument('stage', choices=['components','splits']); p.add_argument('--oversized-report-ts')
args = p.parse_args()
if args.stage == 'components': components_stage()
else: split_stage(args.oversized_report_ts)