39 lines
2.5 KiB
Python
39 lines
2.5 KiB
Python
"""Aggregate schema metadata, never numeric field values."""
|
|
from pathlib import Path
|
|
from collections import Counter,defaultdict
|
|
import json,re,hashlib,gzip,time
|
|
ROOT=Path(__file__).parent;start=time.process_time()
|
|
end=json.loads((ROOT/'b9-corpus-complete.json').read_text())
|
|
assert end['status']=='COMPLETE' and end['counts']=={'v4':629,'v5':624}
|
|
groups={v:defaultdict(lambda:dict(parts=0,counts=[],types=set(),components=set(),associations=set(),unit_attributes=set(),time_min=[],time_max=[])) for v in ['v4','v5']}
|
|
for line in (ROOT/'b9-corpus-headers.jsonl').open():
|
|
r=json.loads(line);families=defaultdict(list)
|
|
for f in r['fields']:
|
|
if f['association'] not in ('PointData','CellData'):continue
|
|
key=re.sub(r'_Block:.*','',f.get('Name',''));families[key].append(f)
|
|
for key,fs in families.items():
|
|
d=groups[r['version']][key];d['parts']+=1;d['counts'].append(len(fs))
|
|
for f in fs:
|
|
d['types'].add(f.get('type'));d['components'].add(int(f.get('NumberOfComponents',1)));d['associations'].add(f['association'])
|
|
d['unit_attributes'].update((k,str(v)) for k,v in f.items() if 'unit' in k.lower())
|
|
times=[float(m.group(1)) for f in fs if (m:=re.search(r'_Time:([0-9.eE+\-]+)s',f.get('Name','')))]
|
|
if times:d['time_min'].append(min(times));d['time_max'].append(max(times))
|
|
out={}
|
|
for v,items in groups.items():
|
|
out[v]={}
|
|
for k,d in items.items():
|
|
out[v][k]=dict(parts=d['parts'],frames_min=min(d['counts']),frames_max=max(d['counts']),
|
|
types=sorted(d['types']),components=sorted(d['components']),associations=sorted(d['associations']),
|
|
unit_attributes=sorted(d['unit_attributes']),
|
|
earliest_named_time=min(d['time_min']) if d['time_min'] else None,
|
|
latest_named_time=max(d['time_max']) if d['time_max'] else None)
|
|
raw=ROOT/'b9-corpus-headers.jsonl';digest=hashlib.sha256()
|
|
with raw.open('rb') as src,(ROOT/'b9-corpus-headers.jsonl.gz').open('xb') as dst,gzip.GzipFile(fileobj=dst,mode='wb',mtime=0,filename='') as gz:
|
|
for b in iter(lambda:src.read(1024**2),b''):digest.update(b);gz.write(b)
|
|
summary=dict(schema_inventory=out,raw_jsonl_sha256=digest.hexdigest(),raw_bytes=raw.stat().st_size,
|
|
raw_file_retained=True,all_header_attributes_preserved_in_gzip=True,
|
|
summarization_cpu_seconds=time.process_time()-start,
|
|
scope='Schema presence only; no new field-value validation')
|
|
(ROOT/'b9-corpus-summary.json').write_text(json.dumps(summary,indent=2)+'\n')
|
|
print(json.dumps(summary,indent=2))
|