"""Metadata-only custody join, conservative components and deterministic S2 splits.""" import argparse import collections import csv import hashlib import json import math from pathlib import Path import platform import shutil import zipfile import numpy as np BASE = Path('artifacts') OUT = BASE / 's2-g1-components-2026-11-15' REG = BASE / 's2-g1-registration-2026-11-14' BAND = BASE / 's2-g1-band25-2026-11-15' CMP = BASE / 's2-g1-band25-review-comparison-2026-11-15' CONTRACT = Path('docs/s2-g1-component-split-contract-2026-11-15.md') MERGE = 'MERGE_SAME_SIDE_LEAK_PROTECTION' NOEDGE = 'NO_ADDITIONAL_EDGE_BOTH_UNRELATED' EXPECTED = { str(REG/'batch-complete.json'): 'b0ee7469c1ab755e809f8ff85ebc6ce940e9838a23eb091a4a7a0e91e77d0956', str(REG/'contract.json'): 'c4271622e1775d430324dd8e8ad4524006184122079d2fc3bbb8fd0ba6a349d8', str(REG/'custodian/plan.json'): '61f4695b7fdf16d747bdebafc1d0c6d362087e250e9d85a36ccce3b3caa71af6', str(BAND/'custodian/mapping.json'): '7531566edf060cdae522b7cf32d0164d0cc6bd9d5b26be6a8d1c3a53bbf61142', str(CMP/'comparison.csv'): 'e7603e06ac5529608c23b889475632b5d75cfbeea7686528460a6c81dcd10bfd', str(CMP/'sources/mold/frozen-review.csv'): '6479eebdfc09629c7188f18f888494c5d9bea8dd160d92763813b15ef813ce18', str(CMP/'sources/claude/band25-review.csv'): '93096734186c1247e46e98f26c7c9b0f7a4987c8d8d588e8147110ae3880be2e', } def sha(path): h = hashlib.sha256() with Path(path).open('rb') as f: for block in iter(lambda: f.read(1048576), b''): h.update(block) return h.hexdigest() def read(path): return json.loads(Path(path).read_text()) def save(path, data): with Path(path).open('x') as f: json.dump(data, f, ensure_ascii=False, indent=2) f.write('\n') class DSU: def __init__(self, members): self.parent = {x: x for x in members} def root(self, x): while self.parent[x] != x: self.parent[x] = self.parent[self.parent[x]] x = self.parent[x] return x def union(self, a, b): a, b = sorted((self.root(a), self.root(b))) self.parent[b] = a def action(a, b): allowed = {'unrelated', 'UNRESOLVED', 'rigid / same part'} assert a in allowed and b in allowed return NOEDGE if a == b == 'unrelated' else MERGE def order(ids, stage, fold): ids = sorted(ids) rng = np.random.Generator(np.random.PCG64(np.random.SeedSequence([2026111409, stage, fold]))) return [ids[int(i)] for i in rng.permutation(len(ids))] def split_components(components): by_id = {c['component_id']: c for c in components} ids = sorted(by_id) rank = {x: i for i, x in enumerate(order(ids, 0, 0))} placement = sorted(ids, key=lambda x: (-by_id[x]['parts'], rank[x])) folds = [[] for _ in range(5)] parts = [0] * 5 for x in placement: k = min(range(5), key=lambda k: (parts[k], len(folds[k]), k)) folds[k].append(x) parts[k] += by_id[x]['parts'] rows = [] for k, test in enumerate(folds): remaining = set(ids) - set(test) val_order = order(remaining, 1, k) val = val_order[:math.ceil(len(remaining)/5)] learning = order(remaining-set(val), 2, k) pools = {str(p): learning[:n] for p, n in [(25, math.ceil(len(learning)/4)), (50, math.ceil(len(learning)/2)), (100, len(learning))]} reasons = [] if len(ids) < 5: reasons.append('FEWER_THAN_FIVE_COMPONENTS') if not test: reasons.append('EMPTY_TEST_FOLD') if not val: reasons.append('EMPTY_VALIDATION_POOL') if not learning: reasons.append('EMPTY_LEARNING_POOL') if len({len(v) for v in pools.values()}) < 3: reasons.append('COINCIDENT_LEARNING_TIERS') assert not(set(test)&set(val) or set(test)&set(learning) or set(val)&set(learning)) assert set(test)|set(val)|set(learning) == set(ids) assert pools['25'] == pools['50'][:len(pools['25'])] assert pools['50'] == pools['100'][:len(pools['50'])] rows.append(dict(fold=k, test=test, validation=val, validation_order=val_order, learning_order=learning, pools=pools, noncomparability_reasons=reasons)) assert collections.Counter(x for f in folds for x in f) == collections.Counter(ids) return dict(order_version='g1-pcg64-seedsequence-v1', seed=2026111409, numpy_version=np.__version__, placement_order=placement, folds=rows) def components_stage(): assert not OUT.exists(), 'Stage already exists: inspect rather than overwrite' for p, digest in EXPECTED.items(): assert sha(p) == digest, p for p, digest in read(CMP/'receipt.json')['source_sha256'].items(): assert sha(CMP/p) == digest, p for p, digest in read(REG/'contract.json')['input_sha256'].items(): assert sha(p) == digest, p terminal = read(REG/'batch-complete.json') assert terminal['status'] == 'ALL_REGISTRATIONS_COMPLETE' and terminal['failed_jobs'] == 0 plan = read(REG/'custodian/plan.json') assert len(terminal['jobs']) == len(plan['jobs']) == 2012 assert {j['index'] for j in terminal['jobs']} == set(range(2012)) assert all(j['returncode'] == 0 and j['worker']['status'] == 'PASS' and not j['worker']['retry'] for j in terminal['jobs']) inventory_path = BASE/'s2-g1-inventory-2026-11-14/custodian/parts.json' candidate_plan = BASE/'s2-g1-candidates-2026-11-14/custodian/plan.json' old_join = BASE/'s2-g1-custody-handoff-2026-11-14/custodian/joined-frozen-review.json' inventory = read(inventory_path) by_case = {r['case']: r for r in inventory} assert len(inventory) == len(by_case) == 624 assert set(plan['geometry_cache']) == set(by_case) dev = set(read(candidate_plan)['development_cases']) old = read(old_join)['pairs'] old_reviewed = {r[k] for r in old for k in ['a', 'b']} assert all(by_case[r['a']]['source_key'] == by_case[r['b']]['source_key'] for r in old) mapping = read(BAND/'custodian/mapping.json') rows = list(csv.DictReader((CMP/'comparison.csv').open())) original_fields = list(rows[0]) assert len(rows) == 25 and len({r['anonymous_pair_id'] for r in rows}) == 25 ids = {r['anonymous_pair_id'] for r in rows} assert ids == {p['anonymous_pair_id'] for p in mapping['pairs']} map_by_id = {p['anonymous_pair_id']: p for p in mapping['pairs']} for r in rows: r['final_action'] = action(r['mold_relation'], r['claude_relation']) assert {r['anonymous_pair_id'][:8] for r in rows if r['final_action'] == MERGE} == {'3058d0a6', '731fb76c', '1ead5efb', 'a7496efb'} OUT.mkdir() private = OUT/'custodian'; private.mkdir(mode=0o700) public = OUT/'public'; public.mkdir() shutil.copy2(CMP/'comparison.csv', public/'comparison-before-ruling.csv') with (public/'comparison.csv').open('x') as f: w = csv.DictWriter(f, original_fields+['final_action'], lineterminator='\n'); w.writeheader(); w.writerows(rows) reread = list(csv.DictReader((public/'comparison.csv').open())) assert [{k:r[k] for k in original_fields} for r in reread] == list(csv.DictReader((CMP/'comparison.csv').open())) shutil.copy2(CONTRACT, public/'execution-contract.md') input_sha = dict(EXPECTED) for p in [inventory_path, candidate_plan, old_join, CONTRACT, Path(__file__)]: input_sha[str(p)] = sha(p) source_groups = collections.defaultdict(list) dsu = DSU(by_case) edges = [] for case, r in by_case.items(): source_groups[r['source_key']].append(case) assert len(source_groups) == 249 for key, members in sorted(source_groups.items()): members.sort() for b in members[1:]: edges.append(dict(a=members[0], b=b, reason='SAME_SOURCE_KEY')) dsu.union(members[0], b) results = {} counts = collections.Counter() for i, pair in enumerate(plan['jobs']): path = REG/f'custodian/results/pair-{i:04d}.json' digest = sha(path); input_sha[str(path)] = digest r = read(path) assert r['index'] == i and r['pair'] == pair and r['simulation_fields_read'] is False status = r['engineering_rule']['engineering_status']; counts[status] += 1 results[i] = r if status == 'LINK_UNDER_FIXED_RULE': edges.append(dict(a=pair['a'], b=pair['b'], reason=status, registration_index=i, result_sha256=digest)) dsu.union(pair['a'], pair['b']) assert dict(counts) == {'OUTSIDE_FIXED_RULE':1897, 'BAND_REVIEW_REQUIRED':25, 'LINK_UNDER_FIXED_RULE':90} assert {p['registration_index'] for p in mapping['pairs']} == {i for i,r in results.items() if r['engineering_rule']['engineering_status']=='BAND_REVIEW_REQUIRED'} for path, digest in mapping['source_result_sha256'].items(): assert sha(path) == digest, path joined = [] band_reviewed = set() for r in rows: m = map_by_id[r['anonymous_pair_id']]; i = m['registration_index']; pair = plan['jobs'][i] endpoints = {s['case'] for s in m['sides'].values()} assert endpoints == {pair['a'], pair['b']} band_reviewed.update(endpoints) joined.append(dict(**r, registration_index=i, a=pair['a'], b=pair['b'], engineering_rule=results[i]['engineering_rule'], result_sha256=input_sha[str(REG/f'custodian/results/pair-{i:04d}.json')])) if r['final_action'] == MERGE: edges.append(dict(a=pair['a'], b=pair['b'], reason=MERGE, anonymous_pair_id=r['anonymous_pair_id'], registration_index=i)) dsu.union(pair['a'], pair['b']) nodes = {}; node_sources = {}; exposure = {} for case in sorted(by_case): cache = plan['geometry_cache'][case]; path = Path(cache['path']) assert sha(path) == cache['sha256'], case with zipfile.ZipFile(path) as z: info = z.getinfo('points.npy') with z.open(info) as f: version = np.lib.format.read_magic(f) assert version == (1,0) shape, fortran, dtype = np.lib.format.read_array_header_1_0(f) header_bytes = f.tell() assert len(shape) == 2 and shape[1] == 3 and shape[0] > 0 and not dtype.hasobject assert info.file_size == header_bytes + math.prod(shape)*dtype.itemsize nodes[case] = int(shape[0]) node_sources[case] = dict(**cache, points_shape=shape, points_dtype=str(dtype), header_bytes=header_bytes, points_crc32=info.CRC, points_uncompressed_bytes=info.file_size, payload_deserialized=False) inv = by_case[case] exposure[case] = dict(inventory_state=inv['exposure_state'], old_30_pair_endpoint=inv['old_30_pair_endpoint'], individual_diagnostic_stages=inv['individual_diagnostic_stages'], calibration_development=case in dev, blind120_geometry_review=case in old_reviewed, band25_geometry_review=case in band_reviewed, all_corpus_historical_screening=inv['all_corpus_historical_screening'], unexposed_claim=False, development_exposed=inv['exposure_state']=='KNOWN_DEVELOPMENT_EXPOSED' or case in dev) groups = collections.defaultdict(list) for case in sorted(by_case): groups[dsu.root(case)].append(case) components = []; private_components = []; case_component = {} for members in groups.values(): cid = 'component-'+hashlib.sha256('\n'.join(members).encode()).hexdigest()[:16] record = dict(component_id=cid, parts=len(members), nodes=sum(nodes[c] for c in members), source_keys=len({by_case[c]['source_key'] for c in members}), development_exposed_parts=sum(exposure[c]['development_exposed'] for c in members), blind120_geometry_review_parts=sum(c in old_reviewed for c in members), band25_geometry_review_parts=sum(c in band_reviewed for c in members), unexposed_claim=False) components.append(record); private_components.append(dict(**record, members=members)) case_component.update({c:cid for c in members}) components.sort(key=lambda c:c['component_id']) assert len({c['component_id'] for c in components}) == len(components) assert all(case_component[e['a']] == case_component[e['b']] for e in edges) key_public = []; key_private = [] for i,(key, members) in enumerate(sorted(source_groups.items())): kid = f'key-{i:03d}' row = dict(key_id=kid, parts=len(members), nodes=sum(nodes[c] for c in members), component_id=case_component[members[0]]) key_public.append(row); key_private.append(dict(**row, source_key=key, members=members)) total_nodes = sum(nodes.values()) largest = max(components, key=lambda c:(c['parts'], c['nodes'], c['component_id'])) largest_nodes = max(components, key=lambda c:(c['nodes'], c['parts'], c['component_id'])) def shares(c): return dict(**c, part_share=c['parts']/624, node_share=c['nodes']/total_nodes) flags = [shares(c) for c in components if c['parts']*5 > 624 or c['nodes']*5 > total_nodes] stats = dict(status='COMPONENTS_FROZEN_SPLITS_NOT_YET_RUN', parts=624, source_keys=249, nodes=total_nodes, components=len(components), component_size_distribution=dict(sorted(collections.Counter(c['parts'] for c in components).items())), source_key_size_distribution=dict(sorted(collections.Counter(len(m) for m in source_groups.values()).items())), largest_by_parts=shares(largest), largest_by_nodes=shares(largest_nodes), oversized_report_flags=flags, edge_counts=dict(collections.Counter(e['reason'] for e in edges)), registration_status_counts=dict(counts), band_final_action_counts=dict(collections.Counter(r['final_action'] for r in rows)), development_exposed_parts=sum(x['development_exposed'] for x in exposure.values()), development_exposed_components=sum(c['development_exposed_parts']>0 for c in components), labels_read=False, geometry_recomputed=False, equivalence_truth=False, cross_key_recall_certified=False, ruling_message_ts='1794678062.264009', node_definition='Unique nodes used by tetrahedra; existing points.npy shape', node_cache_sha256_all_verified=True, original_comparison_columns_preserved=True) save(private/'joined-band-review.json', joined) save(private/'edges.json', edges) save(private/'components.json', private_components) save(private/'source-keys.json', key_private) save(private/'parts.json', [dict(case=c, source_key=by_case[c]['source_key'], component_id=case_component[c], nodes=nodes[c], exposure=exposure[c]) for c in sorted(by_case)]) save(private/'node-metadata-provenance.json', node_sources) save(private/'input-sha256.json', input_sha) save(public/'components.json', components) save(public/'parts-per-key.json', key_public) save(public/'component-statistics.json', stats) receipt = dict(status='COMPONENTS_FROZEN', ruling_message_ts='1794678062.264009', python=platform.python_version(), numpy=np.__version__, source_sha256=EXPECTED, contract_sha256=sha(CONTRACT), code_sha256=sha(__file__), custody_artifact_sha256={p.name:sha(p) for p in sorted(private.iterdir())}, public_artifact_sha256={p.name:sha(p) for p in sorted(public.iterdir())}, reviewer_mapping_returned=False, original_source_files_unchanged=all(sha(p)==h for p,h in EXPECTED.items())) save(public/'component-receipt.json', receipt) for p in private.iterdir(): p.chmod(0o600) print(json.dumps(stats, ensure_ascii=False)) def split_stage(report_ts): public = OUT/'public'; private = OUT/'custodian' assert not (public/'splits.json').exists(), 'Splits already exist' stats = read(public/'component-statistics.json') assert not stats['oversized_report_flags'] or report_ts, 'Report oversized components before splitting' receipt = read(public/'component-receipt.json') assert sha(CONTRACT) == receipt['contract_sha256'] and sha(__file__) == receipt['code_sha256'] for name, digest in receipt['custody_artifact_sha256'].items(): assert sha(private/name) == digest for name, digest in receipt['public_artifact_sha256'].items(): assert sha(public/name) == digest components = read(public/'components.json'); by_id = {c['component_id']:c for c in components} splits = split_components(components) def summarize(ids): return dict(components=len(ids), parts=sum(by_id[c]['parts'] for c in ids), nodes=sum(by_id[c]['nodes'] for c in ids), development_exposed_parts=sum(by_id[c]['development_exposed_parts'] for c in ids), development_exposed_components=sum(by_id[c]['development_exposed_parts']>0 for c in ids), unexposed_claim=False) for fold in splits['folds']: fold['counts'] = dict(test=summarize(fold['test']), validation=summarize(fold['validation']), pools={p:summarize(ids) for p,ids in fold['pools'].items()}) save(public/'splits.json', splits) private_rows = [] for part in read(private/'parts.json'): cid = part['component_id']; assignments = [] for f in splits['folds']: role = 'test' if cid in f['test'] else 'validation' if cid in f['validation'] else 'learning' assignments.append(dict(fold=f['fold'], role=role, learning_pools=[p for p,ids in f['pools'].items() if cid in ids])) private_rows.append(dict(**part, assignments=assignments)) save(private/'part-splits.json', private_rows) (private/'part-splits.json').chmod(0o600) result = dict(status='COMPONENTS_AND_FIVE_FOLD_NESTED_POOLS_FROZEN_AWAITING_REVIEW', contract_sha256=sha(CONTRACT), code_sha256=sha(__file__), components_receipt_sha256=sha(public/'component-receipt.json'), splits_sha256=sha(public/'splits.json'), custody_part_splits_sha256=sha(private/'part-splits.json'), oversized_components_report_message_ts=report_ts, no_component_cut=True, all_required_edges_same_fold=True, component_test_partition_verified=True, nested_pools_verified=True, no_geometry_registration_review_rerun=True, training_pilot_gpu=False, fold_counts=[dict(fold=f['fold'], **f['counts'], noncomparability_reasons=f['noncomparability_reasons']) for f in splits['folds']]) save(public/'split-receipt.json', result) print(json.dumps(result, ensure_ascii=False)) if __name__ == '__main__': p = argparse.ArgumentParser(); p.add_argument('stage', choices=['components','splits']); p.add_argument('--oversized-report-ts') args = p.parse_args() if args.stage == 'components': components_stage() else: split_stage(args.oversized_report_ts)