Files
MeltMind/tools/prepare_g1_band_review.py

221 lines
14 KiB
Python

"""Completed registrations -> sealed 25-pair geometry-only blind package.
No registration or original-dataset reads. Reuses validated geometry/pose/render
helpers; a separate existing supervisor bounds the presentation-only batch.
"""
from pathlib import Path
import argparse
import collections
import csv
import datetime
import hashlib
import json
import os
import resource
import secrets
import subprocess
import sys
import time
import zipfile
import numpy as np
import psutil
from g1_blind_geometry import surface, ply_bytes, UNITS, validate_pair
from g1_blind_pose import pose
from render_g1_blind_surface import render_surface
from s2_e1_supervisor import save, supervise
SOURCE = Path('artifacts/s2-g1-registration-2026-11-14')
ROOT = Path('artifacts/s2-g1-band25-2026-11-15')
def sha(path):
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
def prepare():
final = json.loads((SOURCE/'batch-complete.json').read_text())
assert final['status'] == 'ALL_REGISTRATIONS_COMPLETE' and final['failed_jobs'] == 0
assert len(final['jobs']) == 2012 and {j['index'] for j in final['jobs']} == set(range(2012))
assert all(j['worker']['status'] == 'PASS' and j['returncode'] == 0 for j in final['jobs'])
contract = json.loads((SOURCE/'contract.json').read_text())
for path, digest in {**contract['code_sha256'], **contract['input_sha256']}.items():
assert sha(path) == digest, path
plan = json.loads((SOURCE/'custodian/plan.json').read_text())
results = []
result_hashes = {}
for i in range(2012):
path = SOURCE/'custodian/results'/f'pair-{i:04d}.json'
row = json.loads(path.read_text())
assert row['index'] == i and row['pair'] == plan['jobs'][i]
assert row['simulation_fields_read'] is False
results.append(row)
result_hashes[str(path)] = sha(path)
band = [r for r in results if r['engineering_rule']['engineering_status'] == 'BAND_REVIEW_REQUIRED']
assert len(band) == 25 and all(1 < r['engineering_rule']['combined_score'] <= 2 for r in band)
ROOT.mkdir(exist_ok=False)
(ROOT/'custodian').mkdir(mode=0o700)
(ROOT/'public').mkdir()
(ROOT/'blind-package').mkdir()
# Entropy and order stay under custody. Every pair is included exactly once.
entropy = secrets.token_hex(32)
rng = np.random.default_rng(np.random.SeedSequence(int(entropy, 16)))
exports = []
for rank in rng.permutation(len(band)):
row = band[int(rank)]
pid = hashlib.sha256(bytes.fromhex(entropy)+str(row['index']).encode()).hexdigest()[:32]
names = [row['pair']['a'], row['pair']['b']]
if rng.integers(2): names.reverse()
record = dict(anonymous_pair_id=pid, registration_index=row['index'], sides={})
for side, case in zip(['left', 'right'], names):
rotation, _ = np.linalg.qr(rng.normal(size=(3, 3)))
if np.linalg.det(rotation) < 0: rotation[:, 0] *= -1
record['sides'][side] = dict(case=case, cache=plan['geometry_cache'][case],
rotation=rotation.tolist(), translation=rng.uniform(-2, 2, 3).tolist())
exports.append(record)
mapping = dict(entropy_hex=entropy, pairs=exports, source_result_sha256=result_hashes,
inclusion='all and only fixed-rule score in (1,2]; no sampling or rank truncation')
save(ROOT/'custodian/mapping.json', mapping)
(ROOT/'custodian/mapping.json').chmod(0o600)
sources = [__file__, 'tools/g1_blind_geometry.py', 'tools/g1_blind_pose.py',
'tools/render_g1_blind_surface.py', 'tools/s2_e1_supervisor.py']
save(ROOT/'contract.json', dict(mapping_sha256=sha(ROOT/'custodian/mapping.json'),
source_sha256={str(p):sha(p) for p in sources}, pairs=25, exports=50, workers=4,
total_cpu_seconds=3600, wall_seconds=1800, tree_rss_bytes=5*1024**3,
per_worker_rss_bytes=1342177280, disk_bytes=5*1024**3,
scope='existing geometry cache only; no new scientific registration or original-data reads',
authorizing_ruling='1794661944.308269; delegated completion/band handoff'))
counts = collections.Counter(r['engineering_rule']['engineering_status'] for r in results)
summary = {k:v for k,v in final.items() if k != 'jobs'}
summary.update(pairs=2012, engineering_counts=dict(counts), band_unique_parts=len({r['pair'][k] for r in band for k in ['a','b']}),
registration_contract_sha256=sha(SOURCE/'contract.json'),
complete_receipt_sha256=sha(SOURCE/'batch-complete.json'),
result_manifest_sha256=hashlib.sha256(json.dumps(result_hashes, sort_keys=True).encode()).hexdigest(),
custody_mapping_sha256=sha(ROOT/'custodian/mapping.json'),
completion_observed_at=datetime.datetime.now(datetime.timezone.utc).isoformat(),
relationship_truth='UNREVIEWED', components_frozen=False, process_exit_code=0)
save(ROOT/'public/registration-final.json', summary)
# Terminal worker receipts contain job indices, resource use and no case IDs.
save(ROOT/'public/registration-terminal-receipts.json', final)
print(json.dumps(summary), flush=True)
def worker(index):
c = json.loads((ROOT/'contract.json').read_text())
assert sha(ROOT/'custodian/mapping.json') == c['mapping_sha256']
for path, digest in c['source_sha256'].items(): assert sha(path) == digest
row = json.loads((ROOT/'custodian/mapping.json').read_text())['pairs'][index]
pid = row['anonymous_pair_id']
pair = ROOT/'blind-package/pairs'/pid
views = ROOT/'blind-package/views'/pid
pair.mkdir(parents=True, exist_ok=False)
views.mkdir(parents=True, exist_ok=False)
transformations = {}
for side in ['left', 'right']:
item = row['sides'][side]
assert sha(item['cache']['path']) == item['cache']['sha256']
with np.load(item['cache']['path'], allow_pickle=False) as z:
assert set(z.files) == {'points','boundary','center','rg'}
x, faces = surface(z['points'], z['boundary'])
y, transform = pose(x, faces, item['rotation'], item['translation'])
(pair/f'{side}.ply').write_bytes(ply_bytes(y, faces))
temp = ROOT/'custodian/render'/pid/side
temp.mkdir(parents=True, exist_ok=False)
render_surface(y, faces, temp)
for name in ['overview', 'detail-grid']:
(temp/f'{name}.png').replace(views/f'{side}-{name}.png')
transformations[side] = transform
(pair/'units.txt').write_text(UNITS)
save(pair/'manifest.json', dict(format='blind-surface-pair-v1', anonymous_pair_id=pid,
files=[dict(name=p.name, bytes=p.stat().st_size, sha256=sha(p)) for p in sorted(pair.iterdir())]))
validate_pair(pair)
save(ROOT/'custodian'/f'worker-{index:02d}.json', dict(index=index, anonymous_pair_id=pid,
status='PASS', transforms=transformations, cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
same_posed_coordinates_for_ply_and_png=True, original_archive_reads=0, registration=False))
def run():
c = json.loads((ROOT/'contract.json').read_text())
active = {}
next_index = 0
completed = 0
try:
while active or next_index < c['pairs']:
while len(active) < c['workers'] and next_index < c['pairs']:
i = next_index; next_index += 1
log = (ROOT/'custodian'/f'worker-{i:02d}.log').open('x')
p = subprocess.Popen([sys.executable, __file__, 'worker', str(i)], stdout=log, stderr=subprocess.STDOUT)
active[i] = (p, log)
for i, (p, log) in list(active.items()):
if p.poll() is not None:
log.close()
if p.returncode != 0: raise RuntimeError(f'presentation worker {i} failed; no retry')
del active[i]; completed += 1
print(json.dumps(dict(completed=completed, pairs=c['pairs'])), flush=True)
else:
try:
process = psutil.Process(p.pid)
rss = sum(q.memory_info().rss for q in [process]+process.children(recursive=True))
if rss > c['per_worker_rss_bytes']: raise RuntimeError(f'worker {i} RSS limit')
except psutil.NoSuchProcess: pass
if active: time.sleep(.5)
finally:
for p, log in active.values():
if p.poll() is None: p.kill()
p.wait(); log.close()
def package():
receipt = json.loads((ROOT/'presentation.receipt.json').read_text())
assert receipt['status'] == 'PASS'
mapping = json.loads((ROOT/'custodian/mapping.json').read_text())
c = json.loads((ROOT/'contract.json').read_text())
assert sha(ROOT/'custodian/mapping.json') == c['mapping_sha256']
ids = [r['anonymous_pair_id'] for r in mapping['pairs']]
bundle = ROOT/'blind-package'
pages = bundle/'review-pairs'; pages.mkdir(exist_ok=False)
for i,pid in enumerate(ids):
record = json.loads((ROOT/'custodian'/f'worker-{i:02d}.json').read_text())
assert record['status']=='PASS' and record['anonymous_pair_id']==pid
validate_pair(bundle/'pairs'/pid)
for tr in record['transforms'].values():
assert abs(tr['determinant']-1)<1e-12 and max(tr['edge_error_over_bbox_diagonal'],tr['double_area_error_over_bbox_diagonal_squared'])<=1e-10
(pages/f'{pid}.md').write_text(f'# Anonymous pair {pid}\n\nRelation unassigned; use original numeric axes, not pixel size.\n\n'+
'\n\n'.join(f'![{side} {view}](../views/{pid}/{side}-{view}.png)' for view in ['overview','detail-grid'] for side in ['left','right'])+
f'\n\n[Left full surface](../pairs/{pid}/left.ply) · [Right full surface](../pairs/{pid}/right.ply)\n')
(bundle/'review-index.md').write_text('# Anonymous geometry review\n\nRead README.md first. All labels are blank.\n\n'+ '\n'.join(f'- [Pair {i+1}: {pid}](review-pairs/{pid}.md)' for i,pid in enumerate(ids))+'\n')
with (bundle/'blank-review.csv').open('w') as stream:
w=csv.writer(stream,lineterminator='\n');w.writerow(['anonymous_pair_id','relation','local_evidence','resolvable','unresolved_reason','reviewer','review_version'])
for pid in ids:w.writerow([pid,'','','','','',''])
(bundle/'README.md').write_text('''# Geometry-only blind review\n\n25 anonymous pairs, all relations unassigned. Use rigid / template / variant / unrelated / unresolved and give local geometric evidence. Inclusion is not a positive label. If parent provenance or a local hole/bridge cannot be resolved from geometry, record unresolved.\n\nOpen review-index.md for four-view overviews and fixed oblique quadrant crops. Original PLY surfaces are authoritative; numerical coordinate units are preserved, not asserted to be mm. Each export has its own independent proper rotation and translation, with no rescaling, reflection, fitted registration or simplification. Each view fits its own extent: compare numeric axes, not pixel dimensions. Triangle winding need not point outward; use double-sided display. Raster marks are not proof of topology.\n\nNo identities, source keys, original rank, residuals, process fields or labels are supplied. Counts and shape remain visible; posing mitigates recognition but does not guarantee anonymity. Do not consult census, source archive, custody mapping or registration outputs. Request any necessary detail by anonymous ID. Freeze completed review CSV and attestation before the custody join. UNRESOLVED is conservatively merged under the existing grouping rule; it is not evidence of equivalence. This is the new review batch, not a repeat of the prior 120 pairs.\n''')
(bundle/'reviewer-attestation.md').write_text('# Reviewer attestation\n\nI reviewed only the anonymous materials and did not consult census, source archive, identity/rank/residual mappings or registration results.\n\nReviewer:\nDate:\nExplicit confirmation:\nFrozen label commit/version:\n')
files=[dict(path=str(p.relative_to(bundle)),bytes=p.stat().st_size,sha256=sha(p)) for p in sorted(bundle.rglob('*')) if p.is_file()]
save(bundle/'manifest.json', dict(scope='anonymous geometry and blank labels only',pairs=25,files=files))
archive=ROOT/'blind-package.zip'
with zipfile.ZipFile(archive,'x',compression=zipfile.ZIP_DEFLATED,compresslevel=6) as z:
for p in sorted(bundle.rglob('*')):
if p.is_file():
info=zipfile.ZipInfo(str(p.relative_to(bundle)),date_time=(1980,1,1,0,0,0));info.external_attr=0o100644<<16;info.compress_type=zipfile.ZIP_DEFLATED;z.writestr(info,p.read_bytes())
with zipfile.ZipFile(archive) as z:
for entry in z.infolist():
assert entry.date_time==(1980,1,1,0,0,0) and z.read(entry.filename)==(bundle/entry.filename).read_bytes()
save(ROOT/'public/package-result.json',dict(status='25_BLIND_PAIRS_READY_NOT_REVIEWED',pairs=25,exports=50,pngs=100,
zip_sha256=sha(archive),zip_bytes=archive.stat().st_size,manifest_sha256=sha(bundle/'manifest.json'),
custody_mapping_sha256=c['mapping_sha256'],all_pose_and_pair_schema_checks_pass=True,zip_full_byte_check_pass=True,
presentation_receipt=receipt,components_frozen=False))
print((ROOT/'public/package-result.json').read_text())
if __name__ == '__main__':
parser=argparse.ArgumentParser();parser.add_argument('action',choices=['prepare','worker','run','supervise','package']);parser.add_argument('index',nargs='?',type=int);args=parser.parse_args()
if args.action=='prepare':prepare()
elif args.action=='worker':worker(args.index)
elif args.action=='run':run()
elif args.action=='package':package()
else:
c=json.loads((ROOT/'contract.json').read_text())
limits=dict(total_cpu_seconds=c['total_cpu_seconds'],job_cpu_seconds=c['total_cpu_seconds'],job_wall_seconds=c['wall_seconds'],tree_rss_bytes=c['tree_rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=5)
result=supervise([sys.executable,__file__,'run'],ROOT,'presentation',limits,poll=.5)
print(json.dumps(result));sys.exit(0 if result['status']=='PASS' else 1)