Files
MeltMind/artifacts/phase2-ml-planning-2026-11-17/summarize_b9.py

39 lines
2.5 KiB
Python

"""Aggregate schema metadata, never numeric field values."""
from pathlib import Path
from collections import Counter,defaultdict
import json,re,hashlib,gzip,time
ROOT=Path(__file__).parent;start=time.process_time()
end=json.loads((ROOT/'b9-corpus-complete.json').read_text())
assert end['status']=='COMPLETE' and end['counts']=={'v4':629,'v5':624}
groups={v:defaultdict(lambda:dict(parts=0,counts=[],types=set(),components=set(),associations=set(),unit_attributes=set(),time_min=[],time_max=[])) for v in ['v4','v5']}
for line in (ROOT/'b9-corpus-headers.jsonl').open():
r=json.loads(line);families=defaultdict(list)
for f in r['fields']:
if f['association'] not in ('PointData','CellData'):continue
key=re.sub(r'_Block:.*','',f.get('Name',''));families[key].append(f)
for key,fs in families.items():
d=groups[r['version']][key];d['parts']+=1;d['counts'].append(len(fs))
for f in fs:
d['types'].add(f.get('type'));d['components'].add(int(f.get('NumberOfComponents',1)));d['associations'].add(f['association'])
d['unit_attributes'].update((k,str(v)) for k,v in f.items() if 'unit' in k.lower())
times=[float(m.group(1)) for f in fs if (m:=re.search(r'_Time:([0-9.eE+\-]+)s',f.get('Name','')))]
if times:d['time_min'].append(min(times));d['time_max'].append(max(times))
out={}
for v,items in groups.items():
out[v]={}
for k,d in items.items():
out[v][k]=dict(parts=d['parts'],frames_min=min(d['counts']),frames_max=max(d['counts']),
types=sorted(d['types']),components=sorted(d['components']),associations=sorted(d['associations']),
unit_attributes=sorted(d['unit_attributes']),
earliest_named_time=min(d['time_min']) if d['time_min'] else None,
latest_named_time=max(d['time_max']) if d['time_max'] else None)
raw=ROOT/'b9-corpus-headers.jsonl';digest=hashlib.sha256()
with raw.open('rb') as src,(ROOT/'b9-corpus-headers.jsonl.gz').open('xb') as dst,gzip.GzipFile(fileobj=dst,mode='wb',mtime=0,filename='') as gz:
for b in iter(lambda:src.read(1024**2),b''):digest.update(b);gz.write(b)
summary=dict(schema_inventory=out,raw_jsonl_sha256=digest.hexdigest(),raw_bytes=raw.stat().st_size,
raw_file_retained=True,all_header_attributes_preserved_in_gzip=True,
summarization_cpu_seconds=time.process_time()-start,
scope='Schema presence only; no new field-value validation')
(ROOT/'b9-corpus-summary.json').write_text(json.dumps(summary,indent=2)+'\n')
print(json.dumps(summary,indent=2))