"""Aggregate schema metadata, never numeric field values.""" from pathlib import Path from collections import Counter,defaultdict import json,re,hashlib,gzip,time ROOT=Path(__file__).parent;start=time.process_time() end=json.loads((ROOT/'b9-corpus-complete.json').read_text()) assert end['status']=='COMPLETE' and end['counts']=={'v4':629,'v5':624} groups={v:defaultdict(lambda:dict(parts=0,counts=[],types=set(),components=set(),associations=set(),unit_attributes=set(),time_min=[],time_max=[])) for v in ['v4','v5']} for line in (ROOT/'b9-corpus-headers.jsonl').open(): r=json.loads(line);families=defaultdict(list) for f in r['fields']: if f['association'] not in ('PointData','CellData'):continue key=re.sub(r'_Block:.*','',f.get('Name',''));families[key].append(f) for key,fs in families.items(): d=groups[r['version']][key];d['parts']+=1;d['counts'].append(len(fs)) for f in fs: d['types'].add(f.get('type'));d['components'].add(int(f.get('NumberOfComponents',1)));d['associations'].add(f['association']) d['unit_attributes'].update((k,str(v)) for k,v in f.items() if 'unit' in k.lower()) times=[float(m.group(1)) for f in fs if (m:=re.search(r'_Time:([0-9.eE+\-]+)s',f.get('Name','')))] if times:d['time_min'].append(min(times));d['time_max'].append(max(times)) out={} for v,items in groups.items(): out[v]={} for k,d in items.items(): out[v][k]=dict(parts=d['parts'],frames_min=min(d['counts']),frames_max=max(d['counts']), types=sorted(d['types']),components=sorted(d['components']),associations=sorted(d['associations']), unit_attributes=sorted(d['unit_attributes']), earliest_named_time=min(d['time_min']) if d['time_min'] else None, latest_named_time=max(d['time_max']) if d['time_max'] else None) raw=ROOT/'b9-corpus-headers.jsonl';digest=hashlib.sha256() with raw.open('rb') as src,(ROOT/'b9-corpus-headers.jsonl.gz').open('xb') as dst,gzip.GzipFile(fileobj=dst,mode='wb',mtime=0,filename='') as gz: for b in iter(lambda:src.read(1024**2),b''):digest.update(b);gz.write(b) summary=dict(schema_inventory=out,raw_jsonl_sha256=digest.hexdigest(),raw_bytes=raw.stat().st_size, raw_file_retained=True,all_header_attributes_preserved_in_gzip=True, summarization_cpu_seconds=time.process_time()-start, scope='Schema presence only; no new field-value validation') (ROOT/'b9-corpus-summary.json').write_text(json.dumps(summary,indent=2)+'\n') print(json.dumps(summary,indent=2))