"""Decode once after prediction/code freeze, then execute immutable statistics.""" import argparse from collections import Counter import json import os from pathlib import Path import sys import time import traceback import threading from xml.etree import ElementTree as ET from zipfile import ZipFile import numpy as np from audit_samples import decode_array from freeze_g1_components import sha from s2_custody_common import contract,read_json,checked,verify_prediction_index,test_rows,expected_models,prediction_path,prediction_inputs from s2_custody_statistics import part_metrics,component_values,contrast from s2_recovery_supervisor import save,supervise,stop_group from s2_recovery_monitoring import disk_usage def decode_custody_label(archive,case,raw_ids): """Called only inside gated custody; never reuse another fold's target cache.""" ids=np.asarray(raw_ids);stack=[];count=None;header=None if ids.ndim!=1 or ids.dtype.kind not in 'iu' or not len(ids) or np.any(np.diff(ids)<=0):raise ValueError('Ascending original IDs required') with ZipFile(archive) as z,z.open(case+'_sim.vtu') as stream: for event,e in ET.iterparse(stream,events=('start','end')): if event=='start': stack.append(e.tag) if e.tag=='VTKFile': if e.get('byte_order')!='LittleEndian' or e.get('compressor')!='vtkZLibDataCompressor':raise ValueError('Unsupported encoding') header=e.get('header_type','UInt32') elif e.tag=='Piece': if count is not None:raise ValueError('Multiple pieces') count=int(e.get('NumberOfPoints')) else: if e.tag=='DataArray': if stack[-2]=='PointData' and e.get('Name')=='Fill_time': values=decode_array(e,header) if values.shape!=(count,) or ids[0]<0 or ids[-1]>=count:raise ValueError('Label/ID mismatch') values=values[ids].copy() if not np.isfinite(values).all() or np.any(values<0):raise ValueError('Invalid eligible Fill_time') return values e.clear() stack.pop() raise ValueError('Fill_time missing') def decode_gate(root,c): proof,index=verify_prediction_index(root,c) gate=read_json(root/'scoring-freeze.json') if not gate['approved'] or not gate['git_commit'] or gate['contract_sha256']!=sha(root/'contract.json'): raise ValueError('Scoring code/config not approved and Git-frozen') if gate['prediction_complete_sha256']!=sha(root/'predictions-complete.json'):raise ValueError('Prediction freeze mismatch') return proof,index def reports(rows,roster,c): tables=[];contrasts=[];component_tables=[] members={p for f in read_json(checked(c['exposure_membership'],c['exposure_membership_sha256'])) for p in f['parts']} for subset in ['all','not_known_exposed']: for pool in [25,50,100]: full={} for metric in ['relative_l2','near_wall','kendall_tau_b','last_location']: for arm in ['N','E','R2','B0']: vals,excluded,details=component_values(rows,roster,arm,pool,metric,subset!='all',members) folds={f:[v for k,v in vals.items() if details[k]['fold']==f] for f in range(5)} worst=max(vals,key=vals.get) if vals and metric=='relative_l2' else None tables.append(dict(subset=subset,pool_component_fraction=pool/100,arm=arm,metric=metric, mean=float(np.mean(list(vals.values()))) if vals else None,components=len(vals), included_parts=sum(details[k]['parts'] for k in vals),included_nodes=sum(details[k]['nodes'] for k in vals), excluded_components=excluded,fold_means={f:float(np.mean(v)) if v else None for f,v in folds.items()}, worst_component=dict(component_id=worst,value=vals[worst],**details[worst]) if worst else None)) component_tables.extend(dict(subset=subset,pool=pool,arm=arm,metric=metric,component_id=k,value=v,**details[k]) for k,v in vals.items()) if metric=='relative_l2':full[arm]=(vals,excluded,details) for ref,arm in [('N','E'),('R2','N'),('R2','E'),('B0','N'),('B0','E'),('B0','R2')]: a,_,ad=full[ref];b,_,bd=full[arm];pair=set(a)&set(b) result=contrast(a,b,c['statistics']['replicates'],c['statistics']['seed']) all_ids=set(ad)|set(bd);excluded=sorted(all_ids-pair) contrasts.append(dict(subset=subset,pool=pool,reference=ref,candidate=arm, primary=subset=='all' and pool==100 and (ref,arm)==('N','E'), included_folds=sorted({ad[k]['fold'] for k in pair}),excluded_components=excluded, excluded_parts=sum(ad[k]['parts'] for k in excluded),excluded_nodes=sum(ad[k]['nodes'] for k in excluded),**result)) return tables,contrasts,component_tables def run(root): c=contract(root);proof,index=decode_gate(root,c) dest=root/'custody';dest.mkdir(exist_ok=False) save(dest/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'), scoring_freeze_sha256=sha(root/'scoring-freeze.json'),prediction_complete_sha256=sha(root/'predictions-complete.json'))) start=time.monotonic();cpu=time.process_time();decoded=[];scores=[];coverage=[] try: st=Path(c['archive']).stat();v=c['archive_stat'] assert (st.st_size,st.st_mtime_ns,st.st_ino)==(v['bytes'],v['mtime_ns'],v['inode']) for fold in range(5): for row in test_rows(c,fold): d=prediction_inputs(row);truth=decode_custody_label(c['archive'],row['case'],d['raw_ids']) target=dest/'labels'/(row['case']+'.npz');target.parent.mkdir(exist_ok=True) if target.exists():raise FileExistsError('No repeated custody decode') np.savez_compressed(target,seconds=truth,raw_ids=d['raw_ids'],counts=d['counts']) decoded.append(dict(case=row['case'],fold=fold,path=str(target),sha256=sha(target),decoded_arrays=['Fill_time'])) for model in expected_models(c,fold): p=prediction_path(root,model,row['case']);checked(p,index[str(p)]) with np.load(p,allow_pickle=False) as z: if not np.array_equal(z['raw_ids'],d['raw_ids']) or not np.array_equal(z['counts'],d['counts']):raise ValueError('Prediction/query order mismatch') metrics=part_metrics(z['seconds'],truth,d['counts'],d['near_wall'],d['raw_ids'],d['coordinates_over_Rg'],row['case'].split('_')[0]) scores.append(dict(case=row['case'],component_id=row['component_id'],fold=fold,pool=model['pool'],arm=model['arm'],seed=model['seed'],**metrics)) prep=row['preparation'];cp=Path(prep['preparation_root'])/'result.json';checked(cp,prep['result_sha256']) pr=read_json(cp) coverage.append(dict(case=row['case'],fold=fold,component_id=row['component_id'],nodes=row['nodes'], **{k:pr[k] for k in ['draw_slots','unique_drawn','eligible_unique','eligible_slots','conditional_coverage','full_lumped_coverage','boundary_mass_fraction','eligible_near_wall_slots','rejection_counts_overlapping']}, positive_mass_fraction=pr['scales']['positive_mass_fraction'],exposure=row['exposure'])) save(dest/'progress.json',dict(decoded_parts=len(decoded),scored_rows=len(scores),cpu_seconds=time.process_time()-cpu,wall_seconds=time.monotonic()-start)) save(dest/'label-index.json',decoded);save(dest/'part-scores.json',scores);save(dest/'coverage.json',coverage) roster=read_json(c['roster']);tables,contrasts,components=reports(scores,roster['parts'],c) save(dest/'tables.json',tables);save(dest/'contrasts.json',contrasts);save(dest/'component-scores.json',components) save(dest/'learning-pool-counts.json',roster['fold_pool_counts']) excluded=[r for r in roster['parts'] if not r['included']] result=dict(status='COMPLETE',decoded_parts=len(decoded),scored_rows=len(scores),primary=next(r for r in contrasts if r['primary']), neural_useful_vs_R2=(next(r for r in contrasts if r['subset']=='all' and r['pool']==100 and r['reference']=='R2' and r['candidate']=='N').get('low') or 0)>0, excluded_parts=excluded,excluded_nodes=sum(r['nodes'] for r in excluded),all_nodes=sum(r['nodes'] for r in roster['parts']), neural_NA=c['NA_neural'],cpu_seconds=time.process_time()-cpu,wall_seconds=time.monotonic()-start, limitations=['query common eligibility only, not whole-part result','fixed folds and trained seeds; no retraining variance', 'not-known-exposed is part-filtered descriptive, not untouched','historical all-corpus geometry screening and pilot fitting disclosed', 'width128 feasibility, width effect not tested','AMP skipped case not saved','R2 job118 child identity unknown', 'S2 tests E/N increment and R2 baseline, no conclusion about O1/geometry organisation'], output_sha256={p.name:sha(p) for p in dest.glob('*.json') if p.name not in ['progress.json','started.json']}) save(dest/'result.json',result);return 0 except Exception as exc: save(dest/'failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),decoded_parts=len(decoded),scored_rows=len(scores))) raise def guarded(root): c=contract(root);decode_gate(root,c);l=c['limits'] import psutil done=threading.Event();reasons=[];command=[sys.executable,__file__,str(root),'--worker'] def watch(): while not done.is_set(): try: if disk_usage(root)>l['disk_bytes']:reasons.append('TOTAL_DISK_LIMIT') except Exception as exc:reasons.append('DISK_MONITOR_ERROR:'+str(exc)) if reasons: for child in psutil.Process().children(recursive=True): try: if child.cmdline()==command:stop_group(child.pid) except psutil.NoSuchProcess:pass done.wait(1) t=threading.Thread(target=watch,daemon=True);t.start() try: r=supervise(command,root/'custody-guard','worker',dict(total_cpu_seconds=l['custody_cpu_seconds'], job_cpu_seconds=l['custody_cpu_seconds'],job_wall_seconds=l['custody_wall_seconds'],tree_rss_bytes=l['host_bytes'],disk_bytes=l['disk_bytes'],max_processes=2),poll=.5) finally:done.set();t.join(timeout=3) if reasons:r.update(status='STOPPED_OR_FAILED',stop_reason=reasons[0]) save(root/'custody-guard-result.json',r) return 0 if r['status']=='PASS' and (root/'custody/result.json').exists() else 1 if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--worker',action='store_true');a=p.parse_args() raise SystemExit(run(a.root) if a.worker else guarded(a.root))