Files
MeltMind/artifacts/phase2-ml-planning-2026-11-17/b9_corpus_headers.py

50 lines
3.2 KiB
Python

"""Read-only B9 header inventory, bounded; skip all numeric/base64 array decoding."""
import hashlib, json, resource, time
from pathlib import Path
from zipfile import ZipFile
from xml.parsers import expat
ROOT=Path(__file__).parent;BASE=ROOT.parent.parent
resource.setrlimit(resource.RLIMIT_CPU,(600,605))
resource.setrlimit(resource.RLIMIT_AS,(1024**3,1024**3))
started=time.monotonic();cpu=time.process_time();counts={};failures=[];cpu_sample=1.6381158869999999
def save(name,data):
p=ROOT/name;tmp=p.with_suffix('.tmp');tmp.write_text(json.dumps(data,indent=2)+'\n');tmp.replace(p)
save('b9-corpus-started.json',dict(pid=__import__('os').getpid(),scope='DataArray attributes only, no numeric decoding',
cpu_limit_seconds=600,wall_limit_seconds=900,prior_sample_cpu_seconds=cpu_sample,
source_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest()))
try:
with (ROOT/'b9-corpus-headers.jsonl').open('x') as output:
for version,path in [('v4',BASE/'artifacts/gre24-v4-reproduction-2026-11-16/data/data.zip'),('v5',BASE/'data.zip')]:
counts[version]=0
with ZipFile(path) as z:
names=sorted(n for n in z.namelist() if n.endswith('.vtu'))
for name in names:
fields=[];stack=[];meta={};parser=expat.ParserCreate()
def start(tag,attrs):
if tag in ('VTKFile','Piece'):meta[tag]=dict(attrs)
if tag=='DataArray':fields.append(dict(association=stack[-1],**attrs))
stack.append(tag)
def end(tag):stack.pop()
parser.StartElementHandler=start;parser.EndElementHandler=end
try:
with z.open(name) as f:
while True:
if time.monotonic()-started>900:raise TimeoutError('900s wall cap')
b=f.read(1024**2);parser.Parse(b,not b)
if not b:break
row=dict(version=version,member=name,status='SCHEMA_ONLY_CRC_PASS',fields=fields,metadata=meta)
except Exception as e:
row=dict(version=version,member=name,status='FAILED',error=repr(e));failures.append(row)
if isinstance(e,TimeoutError):raise
output.write(json.dumps(row)+'\n');output.flush();counts[version]+=1
if counts[version]%25==0:
save('b9-corpus-progress.json',dict(counts=counts,cpu_seconds=time.process_time()-cpu,failures=len(failures)))
print(version,counts[version],round(time.process_time()-cpu,2),flush=True)
status='COMPLETE' if not failures else 'COMPLETE_WITH_FAILURES'
except Exception as e:status='STOPPED';failures.append(dict(error=repr(e)))
save('b9-corpus-complete.json',dict(status=status,counts=counts,failures=failures,
cpu_seconds=time.process_time()-cpu,wall_seconds=time.monotonic()-started,
cumulative_inventory_cpu_seconds=time.process_time()-cpu+cpu_sample,
new_numeric_array_decodes=0,peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss))
print(status,counts,flush=True)