Files
MeltMind/tools/o1_prepare.py

210 lines
12 KiB
Python

"""One O1 preparation attempt: immutable geometry/features only, no labels."""
import argparse
import json
import os
from pathlib import Path
import time
import traceback
from xml.etree import ElementTree as ET
from zipfile import ZipFile
import numpy as np
from scipy.spatial import QhullError
from audit_samples import decode_array
from gre24_features import COLUMNS
from o1_partition import (Metis, InputViolation, GeometryFailure, checked, sha,
canonical_mesh, graph, partition_metrics, match, type_fields, shuffle_type,
objective, readout, interface_difference)
from s2_recovery_supervisor import save
def read(path):
return json.loads(Path(path).read_text())
def contract(root):
c = read(Path(root)/'contract.json')
for p, h in c['source_sha256'].items(): checked(p, h)
for p, h in c['code_sha256'].items(): checked(p, h)
stat = Path(c['archive']).stat(); old = c['archive_stat']
if (stat.st_size, stat.st_mtime_ns, stat.st_ino) != (old['bytes'], old['mtime_ns'], old['inode']):
raise InputViolation('Frozen source archive stat changed')
return c
def source_geometry(archive, case):
"""Stop before field arrays; keep actual mixed-VTU cell IDs."""
stack = []; cells = {}; points = None; decoded = []
with ZipFile(archive) as z, z.open(case+'_sim.vtu') as stream:
for event, element in ET.iterparse(stream, events=('start', 'end')):
if event == 'start':
stack.append(element.tag)
if element.tag == 'VTKFile':
if element.get('byte_order') != 'LittleEndian' or element.get('compressor') != 'vtkZLibDataCompressor':
raise InputViolation('Unsupported source encoding')
header = element.get('header_type', 'UInt32')
if element.tag in ('PointData', 'CellData'):
raise InputViolation('Refuse source field section')
else:
if element.tag == 'DataArray':
parent = stack[-2]
if parent == 'Points':
points = decode_array(element, header).copy(); decoded.append('Points')
elif parent == 'Cells':
name = element.get('Name')
if name not in ('connectivity', 'offsets', 'types'):
raise InputViolation('Unexpected Cells array')
cells[name] = decode_array(element, header).copy(); decoded.append(name)
element.clear()
if points is not None and all(k in cells for k in ('connectivity', 'offsets', 'types')):
break
stack.pop()
if points is None or set(cells) != {'connectivity', 'offsets', 'types'}:
raise InputViolation('Missing geometry before field section')
offsets = np.r_[0, cells['offsets']]
ids = np.flatnonzero(cells['types'] == 10)
if not len(ids) or np.any(np.diff(offsets)[ids] != 4):
raise GeometryFailure('INVALID_SOURCE_TET')
raw_tets = cells['connectivity'][offsets[ids, None]+np.arange(4)]
raw_nodes = np.unique(raw_tets)
t = np.searchsorted(raw_nodes, raw_tets)
return canonical_mesh(points[raw_nodes], t, raw_nodes, ids), decoded
def load_inputs(c, row):
v, p = row['v5_cache'], row['preparation']
vr, pr = Path(v['output']), Path(p['preparation_root'])
files = {}
def check(path, digest):
checked(path, digest); files[str(path)] = digest; return path
old = read(check(pr/'result.json', p['result_sha256']))
new = read(check(vr/'result.json', v['result_sha256']))
if old['case'] != row['case'] or new['case'] != row['case'] or new['status'] != 'PASS':
raise InputViolation('Preparation identity/status')
if new['features']['columns'] != list(COLUMNS) or new['t_scale_seconds'] != p['t_set'] or v['t_scale_seconds'] != p['t_set']:
raise InputViolation('Frozen feature columns/clock')
mesh, decoded = source_geometry(c['archive'], row['case'])
with np.load(check(pr/'geometry.npz', p['output_sha256']['geometry.npz']), allow_pickle=False) as z:
if not np.array_equal(z['raw_node_ids'], mesh['raw_nodes']) or not np.array_equal(z['points_raw'], mesh['points']):
raise InputViolation('Source/cache raw node coordinates')
t = np.sort(z['tetra'], axis=1); expected = mesh['tets']
order = lambda a: np.lexsort(tuple(a[:, i] for i in (3, 2, 1, 0)))
if not np.array_equal(t[order(t)], expected[order(expected)]):
raise InputViolation('Source/cache tetrahedra differ')
raw = np.load(check(vr/'raw_ids.npy', v['output_sha256']['raw_ids.npy']), allow_pickle=False)
if not np.array_equal(raw, mesh['raw_nodes']): raise InputViolation('Feature raw IDs differ')
features = np.load(check(vr/'features.npy', v['output_sha256']['features.npy']), allow_pickle=False)
if features.shape != (len(raw), 19) or not np.isfinite(features).all():
raise InputViolation('Invalid feature coverage/values')
with np.load(check(pr/'queries.npz', p['output_sha256']['queries.npz']), allow_pickle=False) as z:
keep = z['eligible_s2']; queries = z['raw_node_ids'][keep]; counts = z['multiplicity'][keep]
qi = np.searchsorted(raw, queries)
if (not len(qi) or np.any(qi >= len(raw)) or np.any(np.diff(queries) <= 0) or
np.any(counts <= 0) or not np.array_equal(raw[qi], queries)):
raise InputViolation('Query identity/multiplicity')
return mesh, features, dict(indices=qi, raw_ids=queries, counts=counts), dict(files=files, decoded_geometry_arrays=decoded,
labels_decoded=[], label_cache_opened=False, raw_cells_are_original_VTU_indices=True)
def difference(a, b):
d = np.abs(a-b)
return [dict(median=float(np.median(d[:, j])), p95=float(np.quantile(d[:, j], .95)),
nonzero_fraction=float(np.count_nonzero(d[:, j])/len(d))) for j in range(2)]
def prepare(root, index):
root = Path(root); c = contract(root); row = c['parts'][index]
dest = root/'parts'/row['case']/'output'; dest.mkdir(parents=True, exist_ok=False)
start = time.process_time(); wall = time.monotonic(); stage = 'INPUTS'; candidates = []
def progress(**kw):
save(dest/'progress.json', dict(stage=stage, case=row['case'], pid=os.getpid(),
cpu_seconds=time.process_time()-start, wall_seconds=time.monotonic()-wall, **kw))
try:
progress(); mesh, features, queries, provenance = load_inputs(c, row)
save(dest/'input-provenance.json', provenance)
stage = 'GEOMETRY_LIBRARY'; progress(tets=len(mesh['tets']))
g = graph(mesh); engine = Metis()
np.savez_compressed(dest/'graph.npz', **{k:g[k] for k in ('xadj','adjncy','vwgt','adjwgt','edges','faces_raw','area')},
raw_nodes=mesh['raw_nodes'], raw_cells=mesh['raw_cells'], volume=mesh['volume'])
save(dest/'graph.json', dict(nodes=len(mesh['points']), tets=len(mesh['tets']), faces=len(g['area']),
quantization=g['quantization'], graph_sha256=sha(dest/'graph.npz')))
reference = None; seen = {}; legal = []
for seed in range(32):
before = time.process_time(); rec = dict(seed=seed)
progress(seed=seed, tets=len(mesh['tets']), candidates_terminal=len(candidates))
try:
labels, call = engine.partition(g, mesh['raw_cells'], seed); rec.update(call)
path = dest/f'partition-{seed:02d}.npz'
np.savez_compressed(path, labels=labels)
rec.update(path=str(path), sha256=sha(path))
digest = __import__('hashlib').sha256(labels.tobytes()).hexdigest()
rec['canonical_label_sha256'] = digest
if digest in seen:
rec.update(status='DUPLICATE', earliest_seed=seen[digest])
else:
seen[digest] = seed
rec['physical_fractions'] = (np.bincount(labels, weights=mesh['volume'], minlength=8)/mesh['volume'].sum()).tolist()
metrics = partition_metrics(mesh, g, labels); rec['metrics'] = metrics
if seed == 0: reference = metrics
rec['matching_G'] = match(metrics, reference)
rec['status'] = 'LEGAL' if rec['matching_G']['passed'] else 'MATCH_FAIL'
if rec['status'] == 'LEGAL': legal.append(rec)
except (GeometryFailure, QhullError) as error:
rec.update(status='CANDIDATE_FAILED', cause=type(error).__name__, message=str(error))
rec['cpu_seconds'] = time.process_time()-before
candidates.append(rec); save(dest/'candidate-records.json', candidates)
if seed == 0 and rec['status'] != 'LEGAL':
raise GeometryFailure('G_SEED0_INVALID')
stage = 'TYPE_SELECTION'; progress(tets=len(mesh['tets']), candidates_terminal=32)
z, b = type_fields(features, mesh['tets'])
shuffled, permutation = shuffle_type(z, mesh['volume'], mesh['raw_cells'], row['numeric_case'])
np.save(dest/'shuffle-permutation.npy', permutation)
for rec in legal:
with np.load(rec['path'], allow_pickle=False) as source: labels = source['labels']
rec['J_true'] = objective(z, mesh['volume'], labels)
rec['J_shuffled'] = objective(shuffled, mesh['volume'], labels)
selected = dict(G=legal[0], T=min(legal, key=lambda r:(r['J_true'],r['seed'])),
C=min(legal, key=lambda r:(r['J_shuffled'],r['seed'])))
mutual = match(selected['T']['metrics'], selected['C']['metrics'], mutual=True)
save(dest/'selection.json', dict(seeds={a:r['seed'] for a,r in selected.items()}, mutual=mutual,
legal_candidates=legal, shuffle_sha256=sha(dest/'shuffle-permutation.npy'),
z_min=z.min(axis=0).tolist(), z_max=z.max(axis=0).tolist()))
if not mutual['passed']: raise GeometryFailure('T_C_MUTUAL_MATCH_FAIL')
stage = 'READOUT'; progress(tets=len(mesh['tets']), candidates_terminal=32)
fields = {}; labels_by_arm = {}
for arm, rec in selected.items():
with np.load(rec['path'], allow_pickle=False) as source: labels = source['labels']
labels_by_arm[arm] = labels
fields[arm] = readout(mesh['tets'], mesh['volume'], b, labels, len(features))
if not np.isfinite(fields[arm]).all(): raise GeometryFailure('NONFINITE_READOUT')
np.save(dest/(arm+'-extra.npy'), fields[arm])
qi = queries['indices']; diagnostics = {}
for arm in ('G', 'C'):
diagnostics['T/'+arm] = dict(interface=interface_difference(labels_by_arm['T'], labels_by_arm[arm], g),
float64=difference(fields['T'][qi], fields[arm][qi]),
float32=difference(fields['T'][qi].astype(np.float32), fields[arm][qi].astype(np.float32)))
np.savez_compressed(dest/'queries.npz', **queries)
save(dest/'result.json', dict(status='PASS',case=row['case'],nodes=len(features),tets=len(mesh['tets']),
faces=len(g['area']),candidate_calls=32,unique_candidates=len(seen),legal_candidates=len(legal),
seeds={a:r['seed'] for a,r in selected.items()},diagnostics=diagnostics,
output_sha256={p.name:sha(p) for p in dest.iterdir() if p.suffix in ('.npz','.npy')},
input_provenance_sha256=sha(dest/'input-provenance.json'), selection_sha256=sha(dest/'selection.json'),
cpu_seconds=time.process_time()-start,wall_seconds=time.monotonic()-wall,
original_features_unchanged=True,labels_decoded=[],label_cache_opened=False,model_fit=False))
stage = 'COMPLETE'; progress(tets=len(mesh['tets']), candidates_terminal=32); return 0
except GeometryFailure as error:
save(dest/'result.json', dict(status='INELIGIBLE',case=row['case'],reason=str(error),stage=stage,
candidate_calls=len(candidates),tets=row['tets'],cpu_seconds=time.process_time()-start,
labels_decoded=[],label_cache_opened=False,model_fit=False))
return 0 # completed eligibility measurement, not a valid part
except Exception as error:
save(dest/'failure.json', dict(case=row['case'],type=type(error).__name__,message=str(error),
stage=stage,global_stop=True,traceback=traceback.format_exc(),labels_decoded=[]))
return 1
if __name__ == '__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int)
a=p.parse_args();raise SystemExit(prepare(a.root,a.index))