Files
MeltMind/tools/run_naive_ordering_analysis.py
T

263 lines
17 KiB
Python

"""Frozen clocks, mapped predictions and diagnostic scoring; no model fitting."""
import argparse
from collections import defaultdict
import json
import os
from pathlib import Path
import resource
import sys
import threading
import time
import numpy as np
import psutil
from freeze_g1_components import sha
from s2_custody_common import checked, read_json, prediction_inputs, expected_models, prediction_path
from s2_custody_statistics import part_metrics, component_values, contrast
from s2_recovery_supervisor import save, supervise, stop_group
from s2_recovery_monitoring import disk_usage
from naive_ordering_common import contract, rows
from naive_ordering_math import pps_cdf, training_clock
ROUTES = {'R2': 'R2', 'ORACLE_FULL_TRAIN': 'R2', 'ORACLE_FULL_END': 'R2',
'ORACLE_MASK_TRAIN': 'R2', 'B0_DW1_CLOCK': 'R2', 'B0_DW2_CLOCK': 'R2',
'REORDER_R2': 'R2', 'REORDER_N': 'N', 'REORDER_E': 'E', 'REORDER_B0': 'R2'}
def validate_manifest(c,index):
original=read_json(checked(c['S2_contract'],c['S2_contract_sha256']))
expected=set()
for row in rows(c):
fold=next(a['fold'] for a in row['assignments'] if a['role']=='test')
for pool in (25,50,100):
for arm in ('ORACLE_FULL_TRAIN','ORACLE_FULL_END','ORACLE_MASK_TRAIN','B0_DW1_CLOCK','B0_DW2_CLOCK'):
expected.add((row['case'],fold,pool,arm,None))
for model in expected_models(original,fold):
if model['arm']=='B0' or model['pool']==pool:
expected.add((row['case'],fold,pool,'REORDER_'+model['arm'],model['seed']))
actual=[(v['case'],v['fold'],v['pool'],v['arm'],v['seed']) for v in index.values()]
if len(actual)!=len(set(actual)) or set(actual)!=expected:
raise ValueError('Missing/extra/duplicate mapped predictions against frozen original model roster')
def prepared(root,c):
batch=read_json(root/'batch-complete.json')
if batch['status']!='ALL_PARTS_TERMINAL' or batch['passed']!=617 or batch['failed'] or not batch['accounting_complete']:
raise ValueError('All617 custody/geometry attempts must PASS; no silent calibration roster change')
if batch['contract_sha256']!=sha(root/'contract.json'):raise ValueError('Wrong preparation contract')
result={}
for job in batch['jobs']:
p=checked(job['result_path'],job['result_sha256']);r=read_json(p)
for name,digest in r['output_sha256'].items():checked(p.parent/name,digest)
result[r['case']] = (r,p.parent)
if set(result)!=set(c['cases']):raise ValueError('Missing/extra prepared part')
return result
def make_maps(root):
c=contract(root);parts=rows(c);prep=prepared(root,c)
if (root/'mapping-started.json').exists():raise FileExistsError('No repeat map phase')
save(root/'mapping-started.json',dict(unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json')))
start=time.process_time()
original=read_json(checked(c['S2_contract'],c['S2_contract_sha256']))
original_index=read_json(checked(c['prediction_index'],c['prediction_index_sha256']))
ends={case:r['full_VTU_T_end'] for case,(r,_) in prep.items()}
clocks=[]
for fold in range(5):
for pool in (25,50,100):
value,access=training_clock(parts,fold,pool,ends)
clocks.append(dict(fold=fold,pool=pool,c=value,learning_access=access,
definition='mean across components of within-component mean(full_VTU_T_end/t_set)',
validation_or_test_used_for_scale=False))
save(root/'training-clocks.json',clocks)
clock={(r['fold'],r['pool']):r['c'] for r in clocks}
save(root/'clock-freeze.json',dict(contract_sha256=sha(root/'contract.json'),
clocks_sha256=sha(root/'training-clocks.json'),batch_sha256=sha(root/'batch-complete.json'),
before_new_test_mapping=True))
index={};counter=0;curves=[]
for row in parts:
case=row['case'];fold=next(a['fold'] for a in row['assignments'] if a['role']=='test')
meta,directory=prep[case];d=prediction_inputs(row)
with np.load(directory/'full-domain-maps.npz',allow_pickle=False) as z:
if not np.array_equal(z['raw_ids'][d['ids']],d['raw_ids']):raise ValueError('Full/query map order')
full={key:z[key][d['ids']].copy() for key in z.files if key!='raw_ids'}
with np.load(directory/'full-node-labels.npz',allow_pickle=False) as z:
truth=z['seconds'][d['raw_ids']].copy()
masked_left,masked_right=pps_cdf(truth,d['counts'])
def output(route,pool,seed,left,right,seconds,source=None):
nonlocal counter
tag=route+('' if seed is None else f'-seed{seed}')
path=root/'predictions'/str(fold)/str(pool)/tag/(case+'.npz')
path.parent.mkdir(parents=True,exist_ok=True)
if path.exists():raise FileExistsError('Duplicate mapped prediction')
extra={}
if route.startswith('REORDER_') or route=='ORACLE_MASK_TRAIN':
# These are plug-in estimates, not certified whole-domain bounds.
alpha=meta['estimated_alpha']
extra=dict(estimated_domain_lower=alpha*left,estimated_domain_upper=alpha*right+1-alpha)
np.savez_compressed(path,seconds=seconds,raw_ids=d['raw_ids'],counts=d['counts'],F_left=left,F_right=right,**extra)
index[str(path)]=dict(sha256=sha(path),case=case,component_id=row['component_id'],fold=fold,pool=pool,
arm=route,seed=seed,source_prediction=source,oracle=route.startswith('ORACLE_'),
domain='conditional_PPS' if extra else 'full_tet_lumped_volume')
counter+=1
for pool in (25,50,100):
scale=d['t_set']*clock[fold,pool]
output('ORACLE_FULL_TRAIN',pool,None,full['truth_left'],full['truth_right'],scale*full['truth_left'])
output('ORACLE_FULL_END',pool,None,full['truth_left'],full['truth_right'],meta['full_VTU_T_end']*full['truth_left'])
output('ORACLE_MASK_TRAIN',pool,None,masked_left,masked_right,scale*masked_left)
output('B0_DW1_CLOCK',pool,None,full['df1_left'],full['df1_right'],scale*full['df1_left'])
output('B0_DW2_CLOCK',pool,None,full['df2_left'],full['df2_right'],scale*full['df2_left'])
for model in expected_models(original,fold):
if model['arm']!='B0' and model['pool']!=pool:continue
p=prediction_path(Path(c['S2_root']),model,case)
with np.load(checked(p,original_index[str(p)]),allow_pickle=False) as z:
if not np.array_equal(z['raw_ids'],d['raw_ids']) or not np.array_equal(z['counts'],d['counts']):raise ValueError('Source prediction domain')
score=z['seconds'].copy()
left,right=pps_cdf(score,d['counts']);mapped=scale*left
old_last=d['raw_ids'][score==score.max()].min();new_last=d['raw_ids'][mapped==mapped.max()].min()
if old_last!=new_last:raise ValueError('Order-preserving map changed final location')
if model['arm']=='B0' and not np.allclose(score,d['t_set']*full['df1_left'],rtol=1e-12,atol=1e-12):
raise ValueError('Unscaled Dw fallback differs from frozen B0')
output('REORDER_'+model['arm'],pool,model['seed'],left,right,mapped,str(p))
curves.append(dict(case=case,component_id=row['component_id'],**read_json(directory/'volume-clock-curve.json')))
save(root/'mapping-progress.json',dict(completed_parts=len(curves),prediction_files=counter,CPU_s=time.process_time()-start))
validate_manifest(c,index)
save(root/'prediction-index.json',index)
save(root/'volume-clock-curves.json',curves)
distribution={}
for field in ('seconds','over_t_set','over_full_end'):
values=np.array([r[field] for r in curves])
distribution[field]=dict(median=np.median(values,axis=0).tolist(),p10=np.percentile(values,10,axis=0,method='linear').tolist(),
p90=np.percentile(values,90,axis=0,method='linear').tolist())
save(root/'curve-summary.json',dict(part_count=len(curves),aggregation='part-equal descriptive median/p10/p90; no independent-sample inference',
volume_quantiles=curves[0]['volume_quantiles'],**distribution))
save(root/'predictions-complete.json',dict(status='ALL_NAIVE_MAPS_FROZEN',contract_sha256=sha(root/'contract.json'),
code_freeze=read_json(root/'code-freeze.json'),prediction_index_sha256=sha(root/'prediction-index.json'),
clock_freeze_sha256=sha(root/'clock-freeze.json'),prediction_files=counter,CPU_s=time.process_time()-start,
full_decode_count=617,original_predictions_unchanged=True,new_model_fits=0))
return 0
def reports(scores,roster,members):
tables=[];contrasts=[];components=[]
for subset in ('all','not_known_exposed'):
for pool in (25,50,100):
primary={}
for route,alias in ROUTES.items():
data=[dict(r,arm=alias) for r in scores if r['arm']==route]
for metric in ('relative_l2','near_wall','kendall_tau_b','last_location'):
values,excluded,details=component_values(data,roster,alias,pool,metric,subset!='all',members)
worst=max(values,key=values.get) if values and metric=='relative_l2' else None
tables.append(dict(subset=subset,pool=pool,arm=route,metric=metric,
mean=float(np.mean(list(values.values()))) if values else None,components=len(values),
parts=sum(details[k]['parts'] for k in values),nodes=sum(details[k]['nodes'] for k in values),
excluded_components=excluded,worst_component=dict(id=worst,value=values[worst]) if worst else None))
components.extend(dict(subset=subset,pool=pool,arm=route,metric=metric,component_id=k,value=v,**details[k]) for k,v in values.items())
if metric=='relative_l2':primary[route]=(values,details)
ref,details=primary['R2']
for route,(candidate,cd) in primary.items():
if route=='R2':continue
pair=set(ref)&set(candidate);excluded=sorted((set(details)|set(cd))-pair)
result=contrast(ref,candidate)
low,high=result.get('low'),result.get('high')
contrasts.append(dict(subset=subset,pool=pool,reference='R2',candidate=route,
oracle=route.startswith('ORACLE_'),exploratory=True,primary_confirmatory=False,
within_descriptive_10percent_band=low is not None and low>=-.1 and high<=.1,
included_folds=sorted({details[k]['fold'] for k in pair}),excluded_components=excluded,
excluded_parts=sum(details[k]['parts'] for k in excluded),excluded_nodes=sum(details[k]['nodes'] for k in excluded),**result))
return tables,contrasts,components
def score_maps(root):
c=contract(root);roster=read_json(c['roster'])['parts'];by_case={r['case']:r for r in roster}
prepared(root,c)
proof=read_json(root/'predictions-complete.json')
if proof['status']!='ALL_NAIVE_MAPS_FROZEN' or proof['contract_sha256']!=sha(root/'contract.json'):raise ValueError('Mapping not frozen')
clock_proof=read_json(checked(root/'clock-freeze.json',proof['clock_freeze_sha256']))
checked(root/'training-clocks.json',clock_proof['clocks_sha256'])
index=read_json(checked(root/'prediction-index.json',proof['prediction_index_sha256']))
if len(index)!=proof['prediction_files']:raise ValueError('Manifest size')
validate_manifest(c,index)
for path,row in index.items():checked(path,row['sha256'])
dest=root/'scores';dest.mkdir(exist_ok=False)
start=time.process_time();save(dest/'started.json',dict(unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json')))
originals=read_json(checked(c['original_part_scores'],c['original_part_scores_sha256']))
original_lookup={(r['case'],r['arm'],r['pool'],r['seed']):r for r in originals}
scores=[r for r in originals if r['arm']=='R2']
labels={r['case']:r for r in read_json(checked(c['label_index'],c['label_index_sha256']))}
grouped=defaultdict(list)
for path,item in index.items():grouped[item['case']].append((path,item))
unchanged_last=0
for case,items in sorted(grouped.items()):
d=prediction_inputs(by_case[case]);entry=labels[case]
with np.load(checked(entry['path'],entry['sha256']),allow_pickle=False) as z:
if not np.array_equal(z['raw_ids'],d['raw_ids']) or not np.array_equal(z['counts'],d['counts']):raise ValueError('Scoring cache domain')
truth=z['seconds'].copy()
for path,item in items:
with np.load(path,allow_pickle=False) as z:
if not np.array_equal(z['raw_ids'],d['raw_ids']) or not np.array_equal(z['counts'],d['counts']):raise ValueError('Mapped domain')
metrics=part_metrics(z['seconds'],truth,d['counts'],d['near_wall'],d['raw_ids'],d['coordinates_over_Rg'],case.split('_')[0])
if item['arm'].startswith('REORDER_'):
source=item['arm'].removeprefix('REORDER_');pool=None if source=='B0' else item['pool']
old=original_lookup[case,source,pool,item['seed']]
if metrics['last_location']!=old['last_location'] or metrics['kendall_tau_b']!=old['kendall_tau_b']:
raise ValueError('Reorder altered invariant last location or sampled tau')
unchanged_last+=1
scores.append({k:item[k] for k in ('case','component_id','fold','pool','arm','seed')}|metrics)
members={p for f in read_json(checked(c['exposure_membership'],c['exposure_membership_sha256'])) for p in f['parts']}
tables,contrasts,components=reports(scores,roster,members)
save(dest/'part-scores.json',scores);save(dest/'tables.json',tables);save(dest/'contrasts.json',contrasts);save(dest/'component-scores.json',components)
save(dest/'result.json',dict(status='NAIVE_ORDERING_COMPLETE',headline='Exploration after S2 results were known; not a fresh confirmatory test',
no_model_training=True,new_full_custody_decodes=617,reorder_invariant_rows=unchanged_last,
eikonal_vs_Dijkstra_tested=False,Dw_distance_reused=True,only_new_distance_exponent=2,
CPU_s=time.process_time()-start,prediction_rows=len(index),
limitations=['Full discrete tet lumped-volume measure, not exact continuous cavity integration',
'T_end_full is maximum over original VTU Fill_time; raw/tet/query maxima separately reported',
'PPS conditional CDF and estimated alpha do not certify unobserved domain',
'Oracle success does not establish usable learned ordering; poor masked oracle is not a standalone refutation of constant flow',
'Reordering preserves final location and ties; no gate-ranking validation',
'All comparisons exploratory; no new material/process generalization or O1 test'],
output_sha256={p.name:sha(p) for p in dest.glob('*.json')}))
return 0
def guarded(root,phase):
c=contract(root);lim=c['limits'];batch=read_json(root/'batch-complete.json')
prepared(root,c)
spent=batch['worker_CPU_s']
if phase=='score':
previous=read_json(root/'map-guard-result.json')
if previous['worker']['status']!='PASS':raise ValueError('Map phase did not PASS')
spent+=previous['worker']['cpu_seconds']
phase_budget=min(5400 if phase=='map' else 7200,lim['worker_cpu_seconds']-spent,
lim['post_cpu_seconds']-(spent-batch['worker_CPU_s']))
if phase_budget<=0:raise ValueError('No postprocess budget')
command=[sys.executable,__file__,str(root),phase,'--worker']
done=threading.Event();reasons=[]
def watch():
while not done.is_set():
try:
if disk_usage(root)>lim['disk_bytes']:reasons.append('GLOBAL_DISK_LIMIT')
except Exception as exc:reasons.append('DISK_MONITOR_ERROR:'+str(exc))
if reasons:
for child in psutil.Process().children(recursive=True):
try:
if child.cmdline()==command:stop_group(child.pid)
except psutil.NoSuchProcess:pass
done.wait(1)
os.sched_setaffinity(0,[c['worker_cpus'][0]])
watcher=threading.Thread(target=watch,daemon=True);watcher.start()
try:
w=supervise(command,root/(phase+'-guard'),'worker',dict(total_cpu_seconds=phase_budget,job_cpu_seconds=phase_budget,
job_wall_seconds=10800,tree_rss_bytes=lim['part_rss_bytes'],disk_bytes=lim['disk_bytes'],max_processes=1),poll=.2)
finally:done.set();watcher.join(timeout=3)
if reasons:w.update(status='STOPPED_OR_FAILED',stop_reason=reasons[0])
save(root/(phase+'-guard-result.json'),dict(worker=w,prior_worker_CPU_s=spent,cumulative_worker_CPU_s=spent+w['cpu_seconds'],
total_cap_CPU_s=lim['worker_cpu_seconds'],postphase_cap_CPU_s=lim['post_cpu_seconds'],
guard_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2])))
return 0 if w['status']=='PASS' else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['map','score']);p.add_argument('--worker',action='store_true');a=p.parse_args()
raise SystemExit((make_maps(a.root) if a.phase=='map' else score_maps(a.root)) if a.worker else guarded(a.root,a.phase))