50 lines
3.2 KiB
Python
50 lines
3.2 KiB
Python
"""Read-only B9 header inventory, bounded; skip all numeric/base64 array decoding."""
|
|
import hashlib, json, resource, time
|
|
from pathlib import Path
|
|
from zipfile import ZipFile
|
|
from xml.parsers import expat
|
|
ROOT=Path(__file__).parent;BASE=ROOT.parent.parent
|
|
resource.setrlimit(resource.RLIMIT_CPU,(600,605))
|
|
resource.setrlimit(resource.RLIMIT_AS,(1024**3,1024**3))
|
|
started=time.monotonic();cpu=time.process_time();counts={};failures=[];cpu_sample=1.6381158869999999
|
|
def save(name,data):
|
|
p=ROOT/name;tmp=p.with_suffix('.tmp');tmp.write_text(json.dumps(data,indent=2)+'\n');tmp.replace(p)
|
|
save('b9-corpus-started.json',dict(pid=__import__('os').getpid(),scope='DataArray attributes only, no numeric decoding',
|
|
cpu_limit_seconds=600,wall_limit_seconds=900,prior_sample_cpu_seconds=cpu_sample,
|
|
source_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest()))
|
|
try:
|
|
with (ROOT/'b9-corpus-headers.jsonl').open('x') as output:
|
|
for version,path in [('v4',BASE/'artifacts/gre24-v4-reproduction-2026-11-16/data/data.zip'),('v5',BASE/'data.zip')]:
|
|
counts[version]=0
|
|
with ZipFile(path) as z:
|
|
names=sorted(n for n in z.namelist() if n.endswith('.vtu'))
|
|
for name in names:
|
|
fields=[];stack=[];meta={};parser=expat.ParserCreate()
|
|
def start(tag,attrs):
|
|
if tag in ('VTKFile','Piece'):meta[tag]=dict(attrs)
|
|
if tag=='DataArray':fields.append(dict(association=stack[-1],**attrs))
|
|
stack.append(tag)
|
|
def end(tag):stack.pop()
|
|
parser.StartElementHandler=start;parser.EndElementHandler=end
|
|
try:
|
|
with z.open(name) as f:
|
|
while True:
|
|
if time.monotonic()-started>900:raise TimeoutError('900s wall cap')
|
|
b=f.read(1024**2);parser.Parse(b,not b)
|
|
if not b:break
|
|
row=dict(version=version,member=name,status='SCHEMA_ONLY_CRC_PASS',fields=fields,metadata=meta)
|
|
except Exception as e:
|
|
row=dict(version=version,member=name,status='FAILED',error=repr(e));failures.append(row)
|
|
if isinstance(e,TimeoutError):raise
|
|
output.write(json.dumps(row)+'\n');output.flush();counts[version]+=1
|
|
if counts[version]%25==0:
|
|
save('b9-corpus-progress.json',dict(counts=counts,cpu_seconds=time.process_time()-cpu,failures=len(failures)))
|
|
print(version,counts[version],round(time.process_time()-cpu,2),flush=True)
|
|
status='COMPLETE' if not failures else 'COMPLETE_WITH_FAILURES'
|
|
except Exception as e:status='STOPPED';failures.append(dict(error=repr(e)))
|
|
save('b9-corpus-complete.json',dict(status=status,counts=counts,failures=failures,
|
|
cpu_seconds=time.process_time()-cpu,wall_seconds=time.monotonic()-started,
|
|
cumulative_inventory_cpu_seconds=time.process_time()-cpu+cpu_sample,
|
|
new_numeric_array_decodes=0,peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss))
|
|
print(status,counts,flush=True)
|