"""One O1 preparation attempt: immutable geometry/features only, no labels.""" import argparse import json import os from pathlib import Path import time import traceback from xml.etree import ElementTree as ET from zipfile import ZipFile import numpy as np from scipy.spatial import QhullError from audit_samples import decode_array from gre24_features import COLUMNS from o1_partition import (Metis, InputViolation, GeometryFailure, checked, sha, canonical_mesh, graph, partition_metrics, match, type_fields, shuffle_type, objective, readout, interface_difference) from s2_recovery_supervisor import save def read(path): return json.loads(Path(path).read_text()) def contract(root): c = read(Path(root)/'contract.json') for p, h in c['source_sha256'].items(): checked(p, h) for p, h in c['code_sha256'].items(): checked(p, h) stat = Path(c['archive']).stat(); old = c['archive_stat'] if (stat.st_size, stat.st_mtime_ns, stat.st_ino) != (old['bytes'], old['mtime_ns'], old['inode']): raise InputViolation('Frozen source archive stat changed') return c def source_geometry(archive, case): """Stop before field arrays; keep actual mixed-VTU cell IDs.""" stack = []; cells = {}; points = None; decoded = [] with ZipFile(archive) as z, z.open(case+'_sim.vtu') as stream: for event, element in ET.iterparse(stream, events=('start', 'end')): if event == 'start': stack.append(element.tag) if element.tag == 'VTKFile': if element.get('byte_order') != 'LittleEndian' or element.get('compressor') != 'vtkZLibDataCompressor': raise InputViolation('Unsupported source encoding') header = element.get('header_type', 'UInt32') if element.tag in ('PointData', 'CellData'): raise InputViolation('Refuse source field section') else: if element.tag == 'DataArray': parent = stack[-2] if parent == 'Points': points = decode_array(element, header).copy(); decoded.append('Points') elif parent == 'Cells': name = element.get('Name') if name not in ('connectivity', 'offsets', 'types'): raise InputViolation('Unexpected Cells array') cells[name] = decode_array(element, header).copy(); decoded.append(name) element.clear() if points is not None and all(k in cells for k in ('connectivity', 'offsets', 'types')): break stack.pop() if points is None or set(cells) != {'connectivity', 'offsets', 'types'}: raise InputViolation('Missing geometry before field section') offsets = np.r_[0, cells['offsets']] ids = np.flatnonzero(cells['types'] == 10) if not len(ids) or np.any(np.diff(offsets)[ids] != 4): raise GeometryFailure('INVALID_SOURCE_TET') raw_tets = cells['connectivity'][offsets[ids, None]+np.arange(4)] raw_nodes = np.unique(raw_tets) t = np.searchsorted(raw_nodes, raw_tets) return canonical_mesh(points[raw_nodes], t, raw_nodes, ids), decoded def load_inputs(c, row): v, p = row['v5_cache'], row['preparation'] vr, pr = Path(v['output']), Path(p['preparation_root']) files = {} def check(path, digest): checked(path, digest); files[str(path)] = digest; return path old = read(check(pr/'result.json', p['result_sha256'])) new = read(check(vr/'result.json', v['result_sha256'])) if old['case'] != row['case'] or new['case'] != row['case'] or new['status'] != 'PASS': raise InputViolation('Preparation identity/status') if new['features']['columns'] != list(COLUMNS) or new['t_scale_seconds'] != p['t_set'] or v['t_scale_seconds'] != p['t_set']: raise InputViolation('Frozen feature columns/clock') mesh, decoded = source_geometry(c['archive'], row['case']) with np.load(check(pr/'geometry.npz', p['output_sha256']['geometry.npz']), allow_pickle=False) as z: if not np.array_equal(z['raw_node_ids'], mesh['raw_nodes']) or not np.array_equal(z['points_raw'], mesh['points']): raise InputViolation('Source/cache raw node coordinates') t = np.sort(z['tetra'], axis=1); expected = mesh['tets'] order = lambda a: np.lexsort(tuple(a[:, i] for i in (3, 2, 1, 0))) if not np.array_equal(t[order(t)], expected[order(expected)]): raise InputViolation('Source/cache tetrahedra differ') raw = np.load(check(vr/'raw_ids.npy', v['output_sha256']['raw_ids.npy']), allow_pickle=False) if not np.array_equal(raw, mesh['raw_nodes']): raise InputViolation('Feature raw IDs differ') features = np.load(check(vr/'features.npy', v['output_sha256']['features.npy']), allow_pickle=False) if features.shape != (len(raw), 19) or not np.isfinite(features).all(): raise InputViolation('Invalid feature coverage/values') with np.load(check(pr/'queries.npz', p['output_sha256']['queries.npz']), allow_pickle=False) as z: keep = z['eligible_s2']; queries = z['raw_node_ids'][keep]; counts = z['multiplicity'][keep] qi = np.searchsorted(raw, queries) if (not len(qi) or np.any(qi >= len(raw)) or np.any(np.diff(queries) <= 0) or np.any(counts <= 0) or not np.array_equal(raw[qi], queries)): raise InputViolation('Query identity/multiplicity') return mesh, features, dict(indices=qi, raw_ids=queries, counts=counts), dict(files=files, decoded_geometry_arrays=decoded, labels_decoded=[], label_cache_opened=False, raw_cells_are_original_VTU_indices=True) def difference(a, b): d = np.abs(a-b) return [dict(median=float(np.median(d[:, j])), p95=float(np.quantile(d[:, j], .95)), nonzero_fraction=float(np.count_nonzero(d[:, j])/len(d))) for j in range(2)] def prepare(root, index): root = Path(root); c = contract(root); row = c['parts'][index] dest = root/'parts'/row['case']/'output'; dest.mkdir(parents=True, exist_ok=False) start = time.process_time(); wall = time.monotonic(); stage = 'INPUTS'; candidates = [] def progress(**kw): save(dest/'progress.json', dict(stage=stage, case=row['case'], pid=os.getpid(), cpu_seconds=time.process_time()-start, wall_seconds=time.monotonic()-wall, **kw)) try: progress(); mesh, features, queries, provenance = load_inputs(c, row) save(dest/'input-provenance.json', provenance) stage = 'GEOMETRY_LIBRARY'; progress(tets=len(mesh['tets'])) g = graph(mesh); engine = Metis() np.savez_compressed(dest/'graph.npz', **{k:g[k] for k in ('xadj','adjncy','vwgt','adjwgt','edges','faces_raw','area')}, raw_nodes=mesh['raw_nodes'], raw_cells=mesh['raw_cells'], volume=mesh['volume']) save(dest/'graph.json', dict(nodes=len(mesh['points']), tets=len(mesh['tets']), faces=len(g['area']), quantization=g['quantization'], graph_sha256=sha(dest/'graph.npz'))) reference = None; seen = {}; legal = [] for seed in range(32): before = time.process_time(); rec = dict(seed=seed) progress(seed=seed, tets=len(mesh['tets']), candidates_terminal=len(candidates)) try: labels, call = engine.partition(g, mesh['raw_cells'], seed); rec.update(call) path = dest/f'partition-{seed:02d}.npz' np.savez_compressed(path, labels=labels) rec.update(path=str(path), sha256=sha(path)) digest = __import__('hashlib').sha256(labels.tobytes()).hexdigest() rec['canonical_label_sha256'] = digest if digest in seen: rec.update(status='DUPLICATE', earliest_seed=seen[digest]) else: seen[digest] = seed rec['physical_fractions'] = (np.bincount(labels, weights=mesh['volume'], minlength=8)/mesh['volume'].sum()).tolist() metrics = partition_metrics(mesh, g, labels); rec['metrics'] = metrics if seed == 0: reference = metrics rec['matching_G'] = match(metrics, reference) rec['status'] = 'LEGAL' if rec['matching_G']['passed'] else 'MATCH_FAIL' if rec['status'] == 'LEGAL': legal.append(rec) except (GeometryFailure, QhullError) as error: rec.update(status='CANDIDATE_FAILED', cause=type(error).__name__, message=str(error)) rec['cpu_seconds'] = time.process_time()-before candidates.append(rec); save(dest/'candidate-records.json', candidates) if seed == 0 and rec['status'] != 'LEGAL': raise GeometryFailure('G_SEED0_INVALID') stage = 'TYPE_SELECTION'; progress(tets=len(mesh['tets']), candidates_terminal=32) z, b = type_fields(features, mesh['tets']) shuffled, permutation = shuffle_type(z, mesh['volume'], mesh['raw_cells'], row['numeric_case']) np.save(dest/'shuffle-permutation.npy', permutation) for rec in legal: with np.load(rec['path'], allow_pickle=False) as source: labels = source['labels'] rec['J_true'] = objective(z, mesh['volume'], labels) rec['J_shuffled'] = objective(shuffled, mesh['volume'], labels) selected = dict(G=legal[0], T=min(legal, key=lambda r:(r['J_true'],r['seed'])), C=min(legal, key=lambda r:(r['J_shuffled'],r['seed']))) mutual = match(selected['T']['metrics'], selected['C']['metrics'], mutual=True) save(dest/'selection.json', dict(seeds={a:r['seed'] for a,r in selected.items()}, mutual=mutual, legal_candidates=legal, shuffle_sha256=sha(dest/'shuffle-permutation.npy'), z_min=z.min(axis=0).tolist(), z_max=z.max(axis=0).tolist())) if not mutual['passed']: raise GeometryFailure('T_C_MUTUAL_MATCH_FAIL') stage = 'READOUT'; progress(tets=len(mesh['tets']), candidates_terminal=32) fields = {}; labels_by_arm = {} for arm, rec in selected.items(): with np.load(rec['path'], allow_pickle=False) as source: labels = source['labels'] labels_by_arm[arm] = labels fields[arm] = readout(mesh['tets'], mesh['volume'], b, labels, len(features)) if not np.isfinite(fields[arm]).all(): raise GeometryFailure('NONFINITE_READOUT') np.save(dest/(arm+'-extra.npy'), fields[arm]) qi = queries['indices']; diagnostics = {} for arm in ('G', 'C'): diagnostics['T/'+arm] = dict(interface=interface_difference(labels_by_arm['T'], labels_by_arm[arm], g), float64=difference(fields['T'][qi], fields[arm][qi]), float32=difference(fields['T'][qi].astype(np.float32), fields[arm][qi].astype(np.float32))) np.savez_compressed(dest/'queries.npz', **queries) save(dest/'result.json', dict(status='PASS',case=row['case'],nodes=len(features),tets=len(mesh['tets']), faces=len(g['area']),candidate_calls=32,unique_candidates=len(seen),legal_candidates=len(legal), seeds={a:r['seed'] for a,r in selected.items()},diagnostics=diagnostics, output_sha256={p.name:sha(p) for p in dest.iterdir() if p.suffix in ('.npz','.npy')}, input_provenance_sha256=sha(dest/'input-provenance.json'), selection_sha256=sha(dest/'selection.json'), cpu_seconds=time.process_time()-start,wall_seconds=time.monotonic()-wall, original_features_unchanged=True,labels_decoded=[],label_cache_opened=False,model_fit=False)) stage = 'COMPLETE'; progress(tets=len(mesh['tets']), candidates_terminal=32); return 0 except GeometryFailure as error: save(dest/'result.json', dict(status='INELIGIBLE',case=row['case'],reason=str(error),stage=stage, candidate_calls=len(candidates),tets=row['tets'],cpu_seconds=time.process_time()-start, labels_decoded=[],label_cache_opened=False,model_fit=False)) return 0 # completed eligibility measurement, not a valid part except Exception as error: save(dest/'failure.json', dict(case=row['case'],type=type(error).__name__,message=str(error), stage=stage,global_stop=True,traceback=traceback.format_exc(),labels_decoded=[])) return 1 if __name__ == '__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int) a=p.parse_args();raise SystemExit(prepare(a.root,a.index))