315 lines
18 KiB
Python
315 lines
18 KiB
Python
"""Metadata-only custody join, conservative components and deterministic S2 splits."""
|
|
import argparse
|
|
import collections
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import platform
|
|
import shutil
|
|
import zipfile
|
|
import numpy as np
|
|
|
|
BASE = Path('artifacts')
|
|
OUT = BASE / 's2-g1-components-2026-11-15'
|
|
REG = BASE / 's2-g1-registration-2026-11-14'
|
|
BAND = BASE / 's2-g1-band25-2026-11-15'
|
|
CMP = BASE / 's2-g1-band25-review-comparison-2026-11-15'
|
|
CONTRACT = Path('docs/s2-g1-component-split-contract-2026-11-15.md')
|
|
MERGE = 'MERGE_SAME_SIDE_LEAK_PROTECTION'
|
|
NOEDGE = 'NO_ADDITIONAL_EDGE_BOTH_UNRELATED'
|
|
EXPECTED = {
|
|
str(REG/'batch-complete.json'): 'b0ee7469c1ab755e809f8ff85ebc6ce940e9838a23eb091a4a7a0e91e77d0956',
|
|
str(REG/'contract.json'): 'c4271622e1775d430324dd8e8ad4524006184122079d2fc3bbb8fd0ba6a349d8',
|
|
str(REG/'custodian/plan.json'): '61f4695b7fdf16d747bdebafc1d0c6d362087e250e9d85a36ccce3b3caa71af6',
|
|
str(BAND/'custodian/mapping.json'): '7531566edf060cdae522b7cf32d0164d0cc6bd9d5b26be6a8d1c3a53bbf61142',
|
|
str(CMP/'comparison.csv'): 'e7603e06ac5529608c23b889475632b5d75cfbeea7686528460a6c81dcd10bfd',
|
|
str(CMP/'sources/mold/frozen-review.csv'): '6479eebdfc09629c7188f18f888494c5d9bea8dd160d92763813b15ef813ce18',
|
|
str(CMP/'sources/claude/band25-review.csv'): '93096734186c1247e46e98f26c7c9b0f7a4987c8d8d588e8147110ae3880be2e',
|
|
}
|
|
|
|
|
|
def sha(path):
|
|
h = hashlib.sha256()
|
|
with Path(path).open('rb') as f:
|
|
for block in iter(lambda: f.read(1048576), b''):
|
|
h.update(block)
|
|
return h.hexdigest()
|
|
|
|
|
|
def read(path):
|
|
return json.loads(Path(path).read_text())
|
|
|
|
|
|
def save(path, data):
|
|
with Path(path).open('x') as f:
|
|
json.dump(data, f, ensure_ascii=False, indent=2)
|
|
f.write('\n')
|
|
|
|
|
|
class DSU:
|
|
def __init__(self, members):
|
|
self.parent = {x: x for x in members}
|
|
|
|
def root(self, x):
|
|
while self.parent[x] != x:
|
|
self.parent[x] = self.parent[self.parent[x]]
|
|
x = self.parent[x]
|
|
return x
|
|
|
|
def union(self, a, b):
|
|
a, b = sorted((self.root(a), self.root(b)))
|
|
self.parent[b] = a
|
|
|
|
|
|
def action(a, b):
|
|
allowed = {'unrelated', 'UNRESOLVED', 'rigid / same part'}
|
|
assert a in allowed and b in allowed
|
|
return NOEDGE if a == b == 'unrelated' else MERGE
|
|
|
|
|
|
def order(ids, stage, fold):
|
|
ids = sorted(ids)
|
|
rng = np.random.Generator(np.random.PCG64(np.random.SeedSequence([2026111409, stage, fold])))
|
|
return [ids[int(i)] for i in rng.permutation(len(ids))]
|
|
|
|
|
|
def split_components(components):
|
|
by_id = {c['component_id']: c for c in components}
|
|
ids = sorted(by_id)
|
|
rank = {x: i for i, x in enumerate(order(ids, 0, 0))}
|
|
placement = sorted(ids, key=lambda x: (-by_id[x]['parts'], rank[x]))
|
|
folds = [[] for _ in range(5)]
|
|
parts = [0] * 5
|
|
for x in placement:
|
|
k = min(range(5), key=lambda k: (parts[k], len(folds[k]), k))
|
|
folds[k].append(x)
|
|
parts[k] += by_id[x]['parts']
|
|
rows = []
|
|
for k, test in enumerate(folds):
|
|
remaining = set(ids) - set(test)
|
|
val_order = order(remaining, 1, k)
|
|
val = val_order[:math.ceil(len(remaining)/5)]
|
|
learning = order(remaining-set(val), 2, k)
|
|
pools = {str(p): learning[:n] for p, n in [(25, math.ceil(len(learning)/4)), (50, math.ceil(len(learning)/2)), (100, len(learning))]}
|
|
reasons = []
|
|
if len(ids) < 5: reasons.append('FEWER_THAN_FIVE_COMPONENTS')
|
|
if not test: reasons.append('EMPTY_TEST_FOLD')
|
|
if not val: reasons.append('EMPTY_VALIDATION_POOL')
|
|
if not learning: reasons.append('EMPTY_LEARNING_POOL')
|
|
if len({len(v) for v in pools.values()}) < 3: reasons.append('COINCIDENT_LEARNING_TIERS')
|
|
assert not(set(test)&set(val) or set(test)&set(learning) or set(val)&set(learning))
|
|
assert set(test)|set(val)|set(learning) == set(ids)
|
|
assert pools['25'] == pools['50'][:len(pools['25'])]
|
|
assert pools['50'] == pools['100'][:len(pools['50'])]
|
|
rows.append(dict(fold=k, test=test, validation=val, validation_order=val_order,
|
|
learning_order=learning, pools=pools, noncomparability_reasons=reasons))
|
|
assert collections.Counter(x for f in folds for x in f) == collections.Counter(ids)
|
|
return dict(order_version='g1-pcg64-seedsequence-v1', seed=2026111409,
|
|
numpy_version=np.__version__, placement_order=placement, folds=rows)
|
|
|
|
|
|
def components_stage():
|
|
assert not OUT.exists(), 'Stage already exists: inspect rather than overwrite'
|
|
for p, digest in EXPECTED.items(): assert sha(p) == digest, p
|
|
for p, digest in read(CMP/'receipt.json')['source_sha256'].items():
|
|
assert sha(CMP/p) == digest, p
|
|
for p, digest in read(REG/'contract.json')['input_sha256'].items(): assert sha(p) == digest, p
|
|
terminal = read(REG/'batch-complete.json')
|
|
assert terminal['status'] == 'ALL_REGISTRATIONS_COMPLETE' and terminal['failed_jobs'] == 0
|
|
plan = read(REG/'custodian/plan.json')
|
|
assert len(terminal['jobs']) == len(plan['jobs']) == 2012
|
|
assert {j['index'] for j in terminal['jobs']} == set(range(2012))
|
|
assert all(j['returncode'] == 0 and j['worker']['status'] == 'PASS' and not j['worker']['retry'] for j in terminal['jobs'])
|
|
inventory_path = BASE/'s2-g1-inventory-2026-11-14/custodian/parts.json'
|
|
candidate_plan = BASE/'s2-g1-candidates-2026-11-14/custodian/plan.json'
|
|
old_join = BASE/'s2-g1-custody-handoff-2026-11-14/custodian/joined-frozen-review.json'
|
|
inventory = read(inventory_path)
|
|
by_case = {r['case']: r for r in inventory}
|
|
assert len(inventory) == len(by_case) == 624
|
|
assert set(plan['geometry_cache']) == set(by_case)
|
|
dev = set(read(candidate_plan)['development_cases'])
|
|
old = read(old_join)['pairs']
|
|
old_reviewed = {r[k] for r in old for k in ['a', 'b']}
|
|
assert all(by_case[r['a']]['source_key'] == by_case[r['b']]['source_key'] for r in old)
|
|
mapping = read(BAND/'custodian/mapping.json')
|
|
rows = list(csv.DictReader((CMP/'comparison.csv').open()))
|
|
original_fields = list(rows[0])
|
|
assert len(rows) == 25 and len({r['anonymous_pair_id'] for r in rows}) == 25
|
|
ids = {r['anonymous_pair_id'] for r in rows}
|
|
assert ids == {p['anonymous_pair_id'] for p in mapping['pairs']}
|
|
map_by_id = {p['anonymous_pair_id']: p for p in mapping['pairs']}
|
|
for r in rows: r['final_action'] = action(r['mold_relation'], r['claude_relation'])
|
|
assert {r['anonymous_pair_id'][:8] for r in rows if r['final_action'] == MERGE} == {'3058d0a6', '731fb76c', '1ead5efb', 'a7496efb'}
|
|
OUT.mkdir()
|
|
private = OUT/'custodian'; private.mkdir(mode=0o700)
|
|
public = OUT/'public'; public.mkdir()
|
|
shutil.copy2(CMP/'comparison.csv', public/'comparison-before-ruling.csv')
|
|
with (public/'comparison.csv').open('x') as f:
|
|
w = csv.DictWriter(f, original_fields+['final_action'], lineterminator='\n'); w.writeheader(); w.writerows(rows)
|
|
reread = list(csv.DictReader((public/'comparison.csv').open()))
|
|
assert [{k:r[k] for k in original_fields} for r in reread] == list(csv.DictReader((CMP/'comparison.csv').open()))
|
|
shutil.copy2(CONTRACT, public/'execution-contract.md')
|
|
input_sha = dict(EXPECTED)
|
|
for p in [inventory_path, candidate_plan, old_join, CONTRACT, Path(__file__)]: input_sha[str(p)] = sha(p)
|
|
source_groups = collections.defaultdict(list)
|
|
dsu = DSU(by_case)
|
|
edges = []
|
|
for case, r in by_case.items(): source_groups[r['source_key']].append(case)
|
|
assert len(source_groups) == 249
|
|
for key, members in sorted(source_groups.items()):
|
|
members.sort()
|
|
for b in members[1:]:
|
|
edges.append(dict(a=members[0], b=b, reason='SAME_SOURCE_KEY'))
|
|
dsu.union(members[0], b)
|
|
results = {}
|
|
counts = collections.Counter()
|
|
for i, pair in enumerate(plan['jobs']):
|
|
path = REG/f'custodian/results/pair-{i:04d}.json'
|
|
digest = sha(path); input_sha[str(path)] = digest
|
|
r = read(path)
|
|
assert r['index'] == i and r['pair'] == pair and r['simulation_fields_read'] is False
|
|
status = r['engineering_rule']['engineering_status']; counts[status] += 1
|
|
results[i] = r
|
|
if status == 'LINK_UNDER_FIXED_RULE':
|
|
edges.append(dict(a=pair['a'], b=pair['b'], reason=status, registration_index=i, result_sha256=digest))
|
|
dsu.union(pair['a'], pair['b'])
|
|
assert dict(counts) == {'OUTSIDE_FIXED_RULE':1897, 'BAND_REVIEW_REQUIRED':25, 'LINK_UNDER_FIXED_RULE':90}
|
|
assert {p['registration_index'] for p in mapping['pairs']} == {i for i,r in results.items() if r['engineering_rule']['engineering_status']=='BAND_REVIEW_REQUIRED'}
|
|
for path, digest in mapping['source_result_sha256'].items(): assert sha(path) == digest, path
|
|
joined = []
|
|
band_reviewed = set()
|
|
for r in rows:
|
|
m = map_by_id[r['anonymous_pair_id']]; i = m['registration_index']; pair = plan['jobs'][i]
|
|
endpoints = {s['case'] for s in m['sides'].values()}
|
|
assert endpoints == {pair['a'], pair['b']}
|
|
band_reviewed.update(endpoints)
|
|
joined.append(dict(**r, registration_index=i, a=pair['a'], b=pair['b'],
|
|
engineering_rule=results[i]['engineering_rule'], result_sha256=input_sha[str(REG/f'custodian/results/pair-{i:04d}.json')]))
|
|
if r['final_action'] == MERGE:
|
|
edges.append(dict(a=pair['a'], b=pair['b'], reason=MERGE, anonymous_pair_id=r['anonymous_pair_id'], registration_index=i))
|
|
dsu.union(pair['a'], pair['b'])
|
|
nodes = {}; node_sources = {}; exposure = {}
|
|
for case in sorted(by_case):
|
|
cache = plan['geometry_cache'][case]; path = Path(cache['path'])
|
|
assert sha(path) == cache['sha256'], case
|
|
with zipfile.ZipFile(path) as z:
|
|
info = z.getinfo('points.npy')
|
|
with z.open(info) as f:
|
|
version = np.lib.format.read_magic(f)
|
|
assert version == (1,0)
|
|
shape, fortran, dtype = np.lib.format.read_array_header_1_0(f)
|
|
header_bytes = f.tell()
|
|
assert len(shape) == 2 and shape[1] == 3 and shape[0] > 0 and not dtype.hasobject
|
|
assert info.file_size == header_bytes + math.prod(shape)*dtype.itemsize
|
|
nodes[case] = int(shape[0])
|
|
node_sources[case] = dict(**cache, points_shape=shape, points_dtype=str(dtype), header_bytes=header_bytes,
|
|
points_crc32=info.CRC, points_uncompressed_bytes=info.file_size, payload_deserialized=False)
|
|
inv = by_case[case]
|
|
exposure[case] = dict(inventory_state=inv['exposure_state'], old_30_pair_endpoint=inv['old_30_pair_endpoint'],
|
|
individual_diagnostic_stages=inv['individual_diagnostic_stages'], calibration_development=case in dev,
|
|
blind120_geometry_review=case in old_reviewed, band25_geometry_review=case in band_reviewed,
|
|
all_corpus_historical_screening=inv['all_corpus_historical_screening'], unexposed_claim=False,
|
|
development_exposed=inv['exposure_state']=='KNOWN_DEVELOPMENT_EXPOSED' or case in dev)
|
|
groups = collections.defaultdict(list)
|
|
for case in sorted(by_case): groups[dsu.root(case)].append(case)
|
|
components = []; private_components = []; case_component = {}
|
|
for members in groups.values():
|
|
cid = 'component-'+hashlib.sha256('\n'.join(members).encode()).hexdigest()[:16]
|
|
record = dict(component_id=cid, parts=len(members), nodes=sum(nodes[c] for c in members),
|
|
source_keys=len({by_case[c]['source_key'] for c in members}),
|
|
development_exposed_parts=sum(exposure[c]['development_exposed'] for c in members),
|
|
blind120_geometry_review_parts=sum(c in old_reviewed for c in members),
|
|
band25_geometry_review_parts=sum(c in band_reviewed for c in members), unexposed_claim=False)
|
|
components.append(record); private_components.append(dict(**record, members=members))
|
|
case_component.update({c:cid for c in members})
|
|
components.sort(key=lambda c:c['component_id'])
|
|
assert len({c['component_id'] for c in components}) == len(components)
|
|
assert all(case_component[e['a']] == case_component[e['b']] for e in edges)
|
|
key_public = []; key_private = []
|
|
for i,(key, members) in enumerate(sorted(source_groups.items())):
|
|
kid = f'key-{i:03d}'
|
|
row = dict(key_id=kid, parts=len(members), nodes=sum(nodes[c] for c in members), component_id=case_component[members[0]])
|
|
key_public.append(row); key_private.append(dict(**row, source_key=key, members=members))
|
|
total_nodes = sum(nodes.values())
|
|
largest = max(components, key=lambda c:(c['parts'], c['nodes'], c['component_id']))
|
|
largest_nodes = max(components, key=lambda c:(c['nodes'], c['parts'], c['component_id']))
|
|
def shares(c): return dict(**c, part_share=c['parts']/624, node_share=c['nodes']/total_nodes)
|
|
flags = [shares(c) for c in components if c['parts']*5 > 624 or c['nodes']*5 > total_nodes]
|
|
stats = dict(status='COMPONENTS_FROZEN_SPLITS_NOT_YET_RUN', parts=624, source_keys=249, nodes=total_nodes,
|
|
components=len(components), component_size_distribution=dict(sorted(collections.Counter(c['parts'] for c in components).items())),
|
|
source_key_size_distribution=dict(sorted(collections.Counter(len(m) for m in source_groups.values()).items())),
|
|
largest_by_parts=shares(largest), largest_by_nodes=shares(largest_nodes), oversized_report_flags=flags,
|
|
edge_counts=dict(collections.Counter(e['reason'] for e in edges)), registration_status_counts=dict(counts),
|
|
band_final_action_counts=dict(collections.Counter(r['final_action'] for r in rows)),
|
|
development_exposed_parts=sum(x['development_exposed'] for x in exposure.values()),
|
|
development_exposed_components=sum(c['development_exposed_parts']>0 for c in components),
|
|
labels_read=False, geometry_recomputed=False, equivalence_truth=False, cross_key_recall_certified=False,
|
|
ruling_message_ts='1794678062.264009', node_definition='Unique nodes used by tetrahedra; existing points.npy shape',
|
|
node_cache_sha256_all_verified=True, original_comparison_columns_preserved=True)
|
|
save(private/'joined-band-review.json', joined)
|
|
save(private/'edges.json', edges)
|
|
save(private/'components.json', private_components)
|
|
save(private/'source-keys.json', key_private)
|
|
save(private/'parts.json', [dict(case=c, source_key=by_case[c]['source_key'], component_id=case_component[c], nodes=nodes[c], exposure=exposure[c]) for c in sorted(by_case)])
|
|
save(private/'node-metadata-provenance.json', node_sources)
|
|
save(private/'input-sha256.json', input_sha)
|
|
save(public/'components.json', components)
|
|
save(public/'parts-per-key.json', key_public)
|
|
save(public/'component-statistics.json', stats)
|
|
receipt = dict(status='COMPONENTS_FROZEN', ruling_message_ts='1794678062.264009', python=platform.python_version(), numpy=np.__version__,
|
|
source_sha256=EXPECTED, contract_sha256=sha(CONTRACT), code_sha256=sha(__file__),
|
|
custody_artifact_sha256={p.name:sha(p) for p in sorted(private.iterdir())},
|
|
public_artifact_sha256={p.name:sha(p) for p in sorted(public.iterdir())},
|
|
reviewer_mapping_returned=False, original_source_files_unchanged=all(sha(p)==h for p,h in EXPECTED.items()))
|
|
save(public/'component-receipt.json', receipt)
|
|
for p in private.iterdir(): p.chmod(0o600)
|
|
print(json.dumps(stats, ensure_ascii=False))
|
|
|
|
|
|
def split_stage(report_ts):
|
|
public = OUT/'public'; private = OUT/'custodian'
|
|
assert not (public/'splits.json').exists(), 'Splits already exist'
|
|
stats = read(public/'component-statistics.json')
|
|
assert not stats['oversized_report_flags'] or report_ts, 'Report oversized components before splitting'
|
|
receipt = read(public/'component-receipt.json')
|
|
assert sha(CONTRACT) == receipt['contract_sha256'] and sha(__file__) == receipt['code_sha256']
|
|
for name, digest in receipt['custody_artifact_sha256'].items(): assert sha(private/name) == digest
|
|
for name, digest in receipt['public_artifact_sha256'].items(): assert sha(public/name) == digest
|
|
components = read(public/'components.json'); by_id = {c['component_id']:c for c in components}
|
|
splits = split_components(components)
|
|
def summarize(ids):
|
|
return dict(components=len(ids), parts=sum(by_id[c]['parts'] for c in ids), nodes=sum(by_id[c]['nodes'] for c in ids),
|
|
development_exposed_parts=sum(by_id[c]['development_exposed_parts'] for c in ids),
|
|
development_exposed_components=sum(by_id[c]['development_exposed_parts']>0 for c in ids), unexposed_claim=False)
|
|
for fold in splits['folds']:
|
|
fold['counts'] = dict(test=summarize(fold['test']), validation=summarize(fold['validation']),
|
|
pools={p:summarize(ids) for p,ids in fold['pools'].items()})
|
|
save(public/'splits.json', splits)
|
|
private_rows = []
|
|
for part in read(private/'parts.json'):
|
|
cid = part['component_id']; assignments = []
|
|
for f in splits['folds']:
|
|
role = 'test' if cid in f['test'] else 'validation' if cid in f['validation'] else 'learning'
|
|
assignments.append(dict(fold=f['fold'], role=role, learning_pools=[p for p,ids in f['pools'].items() if cid in ids]))
|
|
private_rows.append(dict(**part, assignments=assignments))
|
|
save(private/'part-splits.json', private_rows)
|
|
(private/'part-splits.json').chmod(0o600)
|
|
result = dict(status='COMPONENTS_AND_FIVE_FOLD_NESTED_POOLS_FROZEN_AWAITING_REVIEW',
|
|
contract_sha256=sha(CONTRACT), code_sha256=sha(__file__), components_receipt_sha256=sha(public/'component-receipt.json'),
|
|
splits_sha256=sha(public/'splits.json'), custody_part_splits_sha256=sha(private/'part-splits.json'),
|
|
oversized_components_report_message_ts=report_ts, no_component_cut=True,
|
|
all_required_edges_same_fold=True, component_test_partition_verified=True, nested_pools_verified=True,
|
|
no_geometry_registration_review_rerun=True, training_pilot_gpu=False,
|
|
fold_counts=[dict(fold=f['fold'], **f['counts'], noncomparability_reasons=f['noncomparability_reasons']) for f in splits['folds']])
|
|
save(public/'split-receipt.json', result)
|
|
print(json.dumps(result, ensure_ascii=False))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
p = argparse.ArgumentParser(); p.add_argument('stage', choices=['components','splits']); p.add_argument('--oversized-report-ts')
|
|
args = p.parse_args()
|
|
if args.stage == 'components': components_stage()
|
|
else: split_stage(args.oversized_report_ts)
|