210 lines
12 KiB
Python
210 lines
12 KiB
Python
"""One O1 preparation attempt: immutable geometry/features only, no labels."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import time
|
|
import traceback
|
|
from xml.etree import ElementTree as ET
|
|
from zipfile import ZipFile
|
|
|
|
import numpy as np
|
|
from scipy.spatial import QhullError
|
|
|
|
from audit_samples import decode_array
|
|
from gre24_features import COLUMNS
|
|
from o1_partition import (Metis, InputViolation, GeometryFailure, checked, sha,
|
|
canonical_mesh, graph, partition_metrics, match, type_fields, shuffle_type,
|
|
objective, readout, interface_difference)
|
|
from s2_recovery_supervisor import save
|
|
|
|
|
|
def read(path):
|
|
return json.loads(Path(path).read_text())
|
|
|
|
|
|
def contract(root):
|
|
c = read(Path(root)/'contract.json')
|
|
for p, h in c['source_sha256'].items(): checked(p, h)
|
|
for p, h in c['code_sha256'].items(): checked(p, h)
|
|
stat = Path(c['archive']).stat(); old = c['archive_stat']
|
|
if (stat.st_size, stat.st_mtime_ns, stat.st_ino) != (old['bytes'], old['mtime_ns'], old['inode']):
|
|
raise InputViolation('Frozen source archive stat changed')
|
|
return c
|
|
|
|
|
|
def source_geometry(archive, case):
|
|
"""Stop before field arrays; keep actual mixed-VTU cell IDs."""
|
|
stack = []; cells = {}; points = None; decoded = []
|
|
with ZipFile(archive) as z, z.open(case+'_sim.vtu') as stream:
|
|
for event, element in ET.iterparse(stream, events=('start', 'end')):
|
|
if event == 'start':
|
|
stack.append(element.tag)
|
|
if element.tag == 'VTKFile':
|
|
if element.get('byte_order') != 'LittleEndian' or element.get('compressor') != 'vtkZLibDataCompressor':
|
|
raise InputViolation('Unsupported source encoding')
|
|
header = element.get('header_type', 'UInt32')
|
|
if element.tag in ('PointData', 'CellData'):
|
|
raise InputViolation('Refuse source field section')
|
|
else:
|
|
if element.tag == 'DataArray':
|
|
parent = stack[-2]
|
|
if parent == 'Points':
|
|
points = decode_array(element, header).copy(); decoded.append('Points')
|
|
elif parent == 'Cells':
|
|
name = element.get('Name')
|
|
if name not in ('connectivity', 'offsets', 'types'):
|
|
raise InputViolation('Unexpected Cells array')
|
|
cells[name] = decode_array(element, header).copy(); decoded.append(name)
|
|
element.clear()
|
|
if points is not None and all(k in cells for k in ('connectivity', 'offsets', 'types')):
|
|
break
|
|
stack.pop()
|
|
if points is None or set(cells) != {'connectivity', 'offsets', 'types'}:
|
|
raise InputViolation('Missing geometry before field section')
|
|
offsets = np.r_[0, cells['offsets']]
|
|
ids = np.flatnonzero(cells['types'] == 10)
|
|
if not len(ids) or np.any(np.diff(offsets)[ids] != 4):
|
|
raise GeometryFailure('INVALID_SOURCE_TET')
|
|
raw_tets = cells['connectivity'][offsets[ids, None]+np.arange(4)]
|
|
raw_nodes = np.unique(raw_tets)
|
|
t = np.searchsorted(raw_nodes, raw_tets)
|
|
return canonical_mesh(points[raw_nodes], t, raw_nodes, ids), decoded
|
|
|
|
|
|
def load_inputs(c, row):
|
|
v, p = row['v5_cache'], row['preparation']
|
|
vr, pr = Path(v['output']), Path(p['preparation_root'])
|
|
files = {}
|
|
def check(path, digest):
|
|
checked(path, digest); files[str(path)] = digest; return path
|
|
old = read(check(pr/'result.json', p['result_sha256']))
|
|
new = read(check(vr/'result.json', v['result_sha256']))
|
|
if old['case'] != row['case'] or new['case'] != row['case'] or new['status'] != 'PASS':
|
|
raise InputViolation('Preparation identity/status')
|
|
if new['features']['columns'] != list(COLUMNS) or new['t_scale_seconds'] != p['t_set'] or v['t_scale_seconds'] != p['t_set']:
|
|
raise InputViolation('Frozen feature columns/clock')
|
|
mesh, decoded = source_geometry(c['archive'], row['case'])
|
|
with np.load(check(pr/'geometry.npz', p['output_sha256']['geometry.npz']), allow_pickle=False) as z:
|
|
if not np.array_equal(z['raw_node_ids'], mesh['raw_nodes']) or not np.array_equal(z['points_raw'], mesh['points']):
|
|
raise InputViolation('Source/cache raw node coordinates')
|
|
t = np.sort(z['tetra'], axis=1); expected = mesh['tets']
|
|
order = lambda a: np.lexsort(tuple(a[:, i] for i in (3, 2, 1, 0)))
|
|
if not np.array_equal(t[order(t)], expected[order(expected)]):
|
|
raise InputViolation('Source/cache tetrahedra differ')
|
|
raw = np.load(check(vr/'raw_ids.npy', v['output_sha256']['raw_ids.npy']), allow_pickle=False)
|
|
if not np.array_equal(raw, mesh['raw_nodes']): raise InputViolation('Feature raw IDs differ')
|
|
features = np.load(check(vr/'features.npy', v['output_sha256']['features.npy']), allow_pickle=False)
|
|
if features.shape != (len(raw), 19) or not np.isfinite(features).all():
|
|
raise InputViolation('Invalid feature coverage/values')
|
|
with np.load(check(pr/'queries.npz', p['output_sha256']['queries.npz']), allow_pickle=False) as z:
|
|
keep = z['eligible_s2']; queries = z['raw_node_ids'][keep]; counts = z['multiplicity'][keep]
|
|
qi = np.searchsorted(raw, queries)
|
|
if (not len(qi) or np.any(qi >= len(raw)) or np.any(np.diff(queries) <= 0) or
|
|
np.any(counts <= 0) or not np.array_equal(raw[qi], queries)):
|
|
raise InputViolation('Query identity/multiplicity')
|
|
return mesh, features, dict(indices=qi, raw_ids=queries, counts=counts), dict(files=files, decoded_geometry_arrays=decoded,
|
|
labels_decoded=[], label_cache_opened=False, raw_cells_are_original_VTU_indices=True)
|
|
|
|
|
|
def difference(a, b):
|
|
d = np.abs(a-b)
|
|
return [dict(median=float(np.median(d[:, j])), p95=float(np.quantile(d[:, j], .95)),
|
|
nonzero_fraction=float(np.count_nonzero(d[:, j])/len(d))) for j in range(2)]
|
|
|
|
|
|
def prepare(root, index):
|
|
root = Path(root); c = contract(root); row = c['parts'][index]
|
|
dest = root/'parts'/row['case']/'output'; dest.mkdir(parents=True, exist_ok=False)
|
|
start = time.process_time(); wall = time.monotonic(); stage = 'INPUTS'; candidates = []
|
|
def progress(**kw):
|
|
save(dest/'progress.json', dict(stage=stage, case=row['case'], pid=os.getpid(),
|
|
cpu_seconds=time.process_time()-start, wall_seconds=time.monotonic()-wall, **kw))
|
|
try:
|
|
progress(); mesh, features, queries, provenance = load_inputs(c, row)
|
|
save(dest/'input-provenance.json', provenance)
|
|
stage = 'GEOMETRY_LIBRARY'; progress(tets=len(mesh['tets']))
|
|
g = graph(mesh); engine = Metis()
|
|
np.savez_compressed(dest/'graph.npz', **{k:g[k] for k in ('xadj','adjncy','vwgt','adjwgt','edges','faces_raw','area')},
|
|
raw_nodes=mesh['raw_nodes'], raw_cells=mesh['raw_cells'], volume=mesh['volume'])
|
|
save(dest/'graph.json', dict(nodes=len(mesh['points']), tets=len(mesh['tets']), faces=len(g['area']),
|
|
quantization=g['quantization'], graph_sha256=sha(dest/'graph.npz')))
|
|
reference = None; seen = {}; legal = []
|
|
for seed in range(32):
|
|
before = time.process_time(); rec = dict(seed=seed)
|
|
progress(seed=seed, tets=len(mesh['tets']), candidates_terminal=len(candidates))
|
|
try:
|
|
labels, call = engine.partition(g, mesh['raw_cells'], seed); rec.update(call)
|
|
path = dest/f'partition-{seed:02d}.npz'
|
|
np.savez_compressed(path, labels=labels)
|
|
rec.update(path=str(path), sha256=sha(path))
|
|
digest = __import__('hashlib').sha256(labels.tobytes()).hexdigest()
|
|
rec['canonical_label_sha256'] = digest
|
|
if digest in seen:
|
|
rec.update(status='DUPLICATE', earliest_seed=seen[digest])
|
|
else:
|
|
seen[digest] = seed
|
|
rec['physical_fractions'] = (np.bincount(labels, weights=mesh['volume'], minlength=8)/mesh['volume'].sum()).tolist()
|
|
metrics = partition_metrics(mesh, g, labels); rec['metrics'] = metrics
|
|
if seed == 0: reference = metrics
|
|
rec['matching_G'] = match(metrics, reference)
|
|
rec['status'] = 'LEGAL' if rec['matching_G']['passed'] else 'MATCH_FAIL'
|
|
if rec['status'] == 'LEGAL': legal.append(rec)
|
|
except (GeometryFailure, QhullError) as error:
|
|
rec.update(status='CANDIDATE_FAILED', cause=type(error).__name__, message=str(error))
|
|
rec['cpu_seconds'] = time.process_time()-before
|
|
candidates.append(rec); save(dest/'candidate-records.json', candidates)
|
|
if seed == 0 and rec['status'] != 'LEGAL':
|
|
raise GeometryFailure('G_SEED0_INVALID')
|
|
stage = 'TYPE_SELECTION'; progress(tets=len(mesh['tets']), candidates_terminal=32)
|
|
z, b = type_fields(features, mesh['tets'])
|
|
shuffled, permutation = shuffle_type(z, mesh['volume'], mesh['raw_cells'], row['numeric_case'])
|
|
np.save(dest/'shuffle-permutation.npy', permutation)
|
|
for rec in legal:
|
|
with np.load(rec['path'], allow_pickle=False) as source: labels = source['labels']
|
|
rec['J_true'] = objective(z, mesh['volume'], labels)
|
|
rec['J_shuffled'] = objective(shuffled, mesh['volume'], labels)
|
|
selected = dict(G=legal[0], T=min(legal, key=lambda r:(r['J_true'],r['seed'])),
|
|
C=min(legal, key=lambda r:(r['J_shuffled'],r['seed'])))
|
|
mutual = match(selected['T']['metrics'], selected['C']['metrics'], mutual=True)
|
|
save(dest/'selection.json', dict(seeds={a:r['seed'] for a,r in selected.items()}, mutual=mutual,
|
|
legal_candidates=legal, shuffle_sha256=sha(dest/'shuffle-permutation.npy'),
|
|
z_min=z.min(axis=0).tolist(), z_max=z.max(axis=0).tolist()))
|
|
if not mutual['passed']: raise GeometryFailure('T_C_MUTUAL_MATCH_FAIL')
|
|
stage = 'READOUT'; progress(tets=len(mesh['tets']), candidates_terminal=32)
|
|
fields = {}; labels_by_arm = {}
|
|
for arm, rec in selected.items():
|
|
with np.load(rec['path'], allow_pickle=False) as source: labels = source['labels']
|
|
labels_by_arm[arm] = labels
|
|
fields[arm] = readout(mesh['tets'], mesh['volume'], b, labels, len(features))
|
|
if not np.isfinite(fields[arm]).all(): raise GeometryFailure('NONFINITE_READOUT')
|
|
np.save(dest/(arm+'-extra.npy'), fields[arm])
|
|
qi = queries['indices']; diagnostics = {}
|
|
for arm in ('G', 'C'):
|
|
diagnostics['T/'+arm] = dict(interface=interface_difference(labels_by_arm['T'], labels_by_arm[arm], g),
|
|
float64=difference(fields['T'][qi], fields[arm][qi]),
|
|
float32=difference(fields['T'][qi].astype(np.float32), fields[arm][qi].astype(np.float32)))
|
|
np.savez_compressed(dest/'queries.npz', **queries)
|
|
save(dest/'result.json', dict(status='PASS',case=row['case'],nodes=len(features),tets=len(mesh['tets']),
|
|
faces=len(g['area']),candidate_calls=32,unique_candidates=len(seen),legal_candidates=len(legal),
|
|
seeds={a:r['seed'] for a,r in selected.items()},diagnostics=diagnostics,
|
|
output_sha256={p.name:sha(p) for p in dest.iterdir() if p.suffix in ('.npz','.npy')},
|
|
input_provenance_sha256=sha(dest/'input-provenance.json'), selection_sha256=sha(dest/'selection.json'),
|
|
cpu_seconds=time.process_time()-start,wall_seconds=time.monotonic()-wall,
|
|
original_features_unchanged=True,labels_decoded=[],label_cache_opened=False,model_fit=False))
|
|
stage = 'COMPLETE'; progress(tets=len(mesh['tets']), candidates_terminal=32); return 0
|
|
except GeometryFailure as error:
|
|
save(dest/'result.json', dict(status='INELIGIBLE',case=row['case'],reason=str(error),stage=stage,
|
|
candidate_calls=len(candidates),tets=row['tets'],cpu_seconds=time.process_time()-start,
|
|
labels_decoded=[],label_cache_opened=False,model_fit=False))
|
|
return 0 # completed eligibility measurement, not a valid part
|
|
except Exception as error:
|
|
save(dest/'failure.json', dict(case=row['case'],type=type(error).__name__,message=str(error),
|
|
stage=stage,global_stop=True,traceback=traceback.format_exc(),labels_decoded=[]))
|
|
return 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int)
|
|
a=p.parse_args();raise SystemExit(prepare(a.root,a.index))
|