163 lines
11 KiB
Python
163 lines
11 KiB
Python
"""Decode once after prediction/code freeze, then execute immutable statistics."""
|
|
import argparse
|
|
from collections import Counter
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import threading
|
|
from xml.etree import ElementTree as ET
|
|
from zipfile import ZipFile
|
|
import numpy as np
|
|
from audit_samples import decode_array
|
|
from freeze_g1_components import sha
|
|
from s2_custody_common import contract,read_json,checked,verify_prediction_index,test_rows,expected_models,prediction_path,prediction_inputs
|
|
from s2_custody_statistics import part_metrics,component_values,contrast
|
|
from s2_recovery_supervisor import save,supervise,stop_group
|
|
from s2_recovery_monitoring import disk_usage
|
|
|
|
|
|
def decode_custody_label(archive,case,raw_ids):
|
|
"""Called only inside gated custody; never reuse another fold's target cache."""
|
|
ids=np.asarray(raw_ids);stack=[];count=None;header=None
|
|
if ids.ndim!=1 or ids.dtype.kind not in 'iu' or not len(ids) or np.any(np.diff(ids)<=0):raise ValueError('Ascending original IDs required')
|
|
with ZipFile(archive) as z,z.open(case+'_sim.vtu') as stream:
|
|
for event,e in ET.iterparse(stream,events=('start','end')):
|
|
if event=='start':
|
|
stack.append(e.tag)
|
|
if e.tag=='VTKFile':
|
|
if e.get('byte_order')!='LittleEndian' or e.get('compressor')!='vtkZLibDataCompressor':raise ValueError('Unsupported encoding')
|
|
header=e.get('header_type','UInt32')
|
|
elif e.tag=='Piece':
|
|
if count is not None:raise ValueError('Multiple pieces')
|
|
count=int(e.get('NumberOfPoints'))
|
|
else:
|
|
if e.tag=='DataArray':
|
|
if stack[-2]=='PointData' and e.get('Name')=='Fill_time':
|
|
values=decode_array(e,header)
|
|
if values.shape!=(count,) or ids[0]<0 or ids[-1]>=count:raise ValueError('Label/ID mismatch')
|
|
values=values[ids].copy()
|
|
if not np.isfinite(values).all() or np.any(values<0):raise ValueError('Invalid eligible Fill_time')
|
|
return values
|
|
e.clear()
|
|
stack.pop()
|
|
raise ValueError('Fill_time missing')
|
|
|
|
|
|
def decode_gate(root,c):
|
|
proof,index=verify_prediction_index(root,c)
|
|
gate=read_json(root/'scoring-freeze.json')
|
|
if not gate['approved'] or not gate['git_commit'] or gate['contract_sha256']!=sha(root/'contract.json'):
|
|
raise ValueError('Scoring code/config not approved and Git-frozen')
|
|
if gate['prediction_complete_sha256']!=sha(root/'predictions-complete.json'):raise ValueError('Prediction freeze mismatch')
|
|
return proof,index
|
|
|
|
|
|
def reports(rows,roster,c):
|
|
tables=[];contrasts=[];component_tables=[]
|
|
members={p for f in read_json(checked(c['exposure_membership'],c['exposure_membership_sha256'])) for p in f['parts']}
|
|
for subset in ['all','not_known_exposed']:
|
|
for pool in [25,50,100]:
|
|
full={}
|
|
for metric in ['relative_l2','near_wall','kendall_tau_b','last_location']:
|
|
for arm in ['N','E','R2','B0']:
|
|
vals,excluded,details=component_values(rows,roster,arm,pool,metric,subset!='all',members)
|
|
folds={f:[v for k,v in vals.items() if details[k]['fold']==f] for f in range(5)}
|
|
worst=max(vals,key=vals.get) if vals and metric=='relative_l2' else None
|
|
tables.append(dict(subset=subset,pool_component_fraction=pool/100,arm=arm,metric=metric,
|
|
mean=float(np.mean(list(vals.values()))) if vals else None,components=len(vals),
|
|
included_parts=sum(details[k]['parts'] for k in vals),included_nodes=sum(details[k]['nodes'] for k in vals),
|
|
excluded_components=excluded,fold_means={f:float(np.mean(v)) if v else None for f,v in folds.items()},
|
|
worst_component=dict(component_id=worst,value=vals[worst],**details[worst]) if worst else None))
|
|
component_tables.extend(dict(subset=subset,pool=pool,arm=arm,metric=metric,component_id=k,value=v,**details[k]) for k,v in vals.items())
|
|
if metric=='relative_l2':full[arm]=(vals,excluded,details)
|
|
for ref,arm in [('N','E'),('R2','N'),('R2','E'),('B0','N'),('B0','E'),('B0','R2')]:
|
|
a,_,ad=full[ref];b,_,bd=full[arm];pair=set(a)&set(b)
|
|
result=contrast(a,b,c['statistics']['replicates'],c['statistics']['seed'])
|
|
all_ids=set(ad)|set(bd);excluded=sorted(all_ids-pair)
|
|
contrasts.append(dict(subset=subset,pool=pool,reference=ref,candidate=arm,
|
|
primary=subset=='all' and pool==100 and (ref,arm)==('N','E'),
|
|
included_folds=sorted({ad[k]['fold'] for k in pair}),excluded_components=excluded,
|
|
excluded_parts=sum(ad[k]['parts'] for k in excluded),excluded_nodes=sum(ad[k]['nodes'] for k in excluded),**result))
|
|
return tables,contrasts,component_tables
|
|
|
|
|
|
def run(root):
|
|
c=contract(root);proof,index=decode_gate(root,c)
|
|
dest=root/'custody';dest.mkdir(exist_ok=False)
|
|
save(dest/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),
|
|
scoring_freeze_sha256=sha(root/'scoring-freeze.json'),prediction_complete_sha256=sha(root/'predictions-complete.json')))
|
|
start=time.monotonic();cpu=time.process_time();decoded=[];scores=[];coverage=[]
|
|
try:
|
|
st=Path(c['archive']).stat();v=c['archive_stat']
|
|
assert (st.st_size,st.st_mtime_ns,st.st_ino)==(v['bytes'],v['mtime_ns'],v['inode'])
|
|
for fold in range(5):
|
|
for row in test_rows(c,fold):
|
|
d=prediction_inputs(row);truth=decode_custody_label(c['archive'],row['case'],d['raw_ids'])
|
|
target=dest/'labels'/(row['case']+'.npz');target.parent.mkdir(exist_ok=True)
|
|
if target.exists():raise FileExistsError('No repeated custody decode')
|
|
np.savez_compressed(target,seconds=truth,raw_ids=d['raw_ids'],counts=d['counts'])
|
|
decoded.append(dict(case=row['case'],fold=fold,path=str(target),sha256=sha(target),decoded_arrays=['Fill_time']))
|
|
for model in expected_models(c,fold):
|
|
p=prediction_path(root,model,row['case']);checked(p,index[str(p)])
|
|
with np.load(p,allow_pickle=False) as z:
|
|
if not np.array_equal(z['raw_ids'],d['raw_ids']) or not np.array_equal(z['counts'],d['counts']):raise ValueError('Prediction/query order mismatch')
|
|
metrics=part_metrics(z['seconds'],truth,d['counts'],d['near_wall'],d['raw_ids'],d['coordinates_over_Rg'],row['case'].split('_')[0])
|
|
scores.append(dict(case=row['case'],component_id=row['component_id'],fold=fold,pool=model['pool'],arm=model['arm'],seed=model['seed'],**metrics))
|
|
prep=row['preparation'];cp=Path(prep['preparation_root'])/'result.json';checked(cp,prep['result_sha256'])
|
|
pr=read_json(cp)
|
|
coverage.append(dict(case=row['case'],fold=fold,component_id=row['component_id'],nodes=row['nodes'],
|
|
**{k:pr[k] for k in ['draw_slots','unique_drawn','eligible_unique','eligible_slots','conditional_coverage','full_lumped_coverage','boundary_mass_fraction','eligible_near_wall_slots','rejection_counts_overlapping']},
|
|
positive_mass_fraction=pr['scales']['positive_mass_fraction'],exposure=row['exposure']))
|
|
save(dest/'progress.json',dict(decoded_parts=len(decoded),scored_rows=len(scores),cpu_seconds=time.process_time()-cpu,wall_seconds=time.monotonic()-start))
|
|
save(dest/'label-index.json',decoded);save(dest/'part-scores.json',scores);save(dest/'coverage.json',coverage)
|
|
roster=read_json(c['roster']);tables,contrasts,components=reports(scores,roster['parts'],c)
|
|
save(dest/'tables.json',tables);save(dest/'contrasts.json',contrasts);save(dest/'component-scores.json',components)
|
|
save(dest/'learning-pool-counts.json',roster['fold_pool_counts'])
|
|
excluded=[r for r in roster['parts'] if not r['included']]
|
|
result=dict(status='COMPLETE',decoded_parts=len(decoded),scored_rows=len(scores),primary=next(r for r in contrasts if r['primary']),
|
|
neural_useful_vs_R2=(next(r for r in contrasts if r['subset']=='all' and r['pool']==100 and r['reference']=='R2' and r['candidate']=='N').get('low') or 0)>0,
|
|
excluded_parts=excluded,excluded_nodes=sum(r['nodes'] for r in excluded),all_nodes=sum(r['nodes'] for r in roster['parts']),
|
|
neural_NA=c['NA_neural'],cpu_seconds=time.process_time()-cpu,wall_seconds=time.monotonic()-start,
|
|
limitations=['query common eligibility only, not whole-part result','fixed folds and trained seeds; no retraining variance',
|
|
'not-known-exposed is part-filtered descriptive, not untouched','historical all-corpus geometry screening and pilot fitting disclosed',
|
|
'width128 feasibility, width effect not tested','AMP skipped case not saved','R2 job118 child identity unknown',
|
|
'S2 tests E/N increment and R2 baseline, no conclusion about O1/geometry organisation'],
|
|
output_sha256={p.name:sha(p) for p in dest.glob('*.json') if p.name not in ['progress.json','started.json']})
|
|
save(dest/'result.json',result);return 0
|
|
except Exception as exc:
|
|
save(dest/'failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),decoded_parts=len(decoded),scored_rows=len(scores)))
|
|
raise
|
|
|
|
|
|
def guarded(root):
|
|
c=contract(root);decode_gate(root,c);l=c['limits']
|
|
import psutil
|
|
done=threading.Event();reasons=[];command=[sys.executable,__file__,str(root),'--worker']
|
|
def watch():
|
|
while not done.is_set():
|
|
try:
|
|
if disk_usage(root)>l['disk_bytes']:reasons.append('TOTAL_DISK_LIMIT')
|
|
except Exception as exc:reasons.append('DISK_MONITOR_ERROR:'+str(exc))
|
|
if reasons:
|
|
for child in psutil.Process().children(recursive=True):
|
|
try:
|
|
if child.cmdline()==command:stop_group(child.pid)
|
|
except psutil.NoSuchProcess:pass
|
|
done.wait(1)
|
|
t=threading.Thread(target=watch,daemon=True);t.start()
|
|
try:
|
|
r=supervise(command,root/'custody-guard','worker',dict(total_cpu_seconds=l['custody_cpu_seconds'],
|
|
job_cpu_seconds=l['custody_cpu_seconds'],job_wall_seconds=l['custody_wall_seconds'],tree_rss_bytes=l['host_bytes'],disk_bytes=l['disk_bytes'],max_processes=2),poll=.5)
|
|
finally:done.set();t.join(timeout=3)
|
|
if reasons:r.update(status='STOPPED_OR_FAILED',stop_reason=reasons[0])
|
|
save(root/'custody-guard-result.json',r)
|
|
return 0 if r['status']=='PASS' and (root/'custody/result.json').exists() else 1
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--worker',action='store_true');a=p.parse_args()
|
|
raise SystemExit(run(a.root) if a.worker else guarded(a.root))
|