263 lines
17 KiB
Python
263 lines
17 KiB
Python
"""Frozen clocks, mapped predictions and diagnostic scoring; no model fitting."""
|
|
import argparse
|
|
from collections import defaultdict
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import sys
|
|
import threading
|
|
import time
|
|
import numpy as np
|
|
import psutil
|
|
from freeze_g1_components import sha
|
|
from s2_custody_common import checked, read_json, prediction_inputs, expected_models, prediction_path
|
|
from s2_custody_statistics import part_metrics, component_values, contrast
|
|
from s2_recovery_supervisor import save, supervise, stop_group
|
|
from s2_recovery_monitoring import disk_usage
|
|
from naive_ordering_common import contract, rows
|
|
from naive_ordering_math import pps_cdf, training_clock
|
|
|
|
ROUTES = {'R2': 'R2', 'ORACLE_FULL_TRAIN': 'R2', 'ORACLE_FULL_END': 'R2',
|
|
'ORACLE_MASK_TRAIN': 'R2', 'B0_DW1_CLOCK': 'R2', 'B0_DW2_CLOCK': 'R2',
|
|
'REORDER_R2': 'R2', 'REORDER_N': 'N', 'REORDER_E': 'E', 'REORDER_B0': 'R2'}
|
|
|
|
|
|
def validate_manifest(c,index):
|
|
original=read_json(checked(c['S2_contract'],c['S2_contract_sha256']))
|
|
expected=set()
|
|
for row in rows(c):
|
|
fold=next(a['fold'] for a in row['assignments'] if a['role']=='test')
|
|
for pool in (25,50,100):
|
|
for arm in ('ORACLE_FULL_TRAIN','ORACLE_FULL_END','ORACLE_MASK_TRAIN','B0_DW1_CLOCK','B0_DW2_CLOCK'):
|
|
expected.add((row['case'],fold,pool,arm,None))
|
|
for model in expected_models(original,fold):
|
|
if model['arm']=='B0' or model['pool']==pool:
|
|
expected.add((row['case'],fold,pool,'REORDER_'+model['arm'],model['seed']))
|
|
actual=[(v['case'],v['fold'],v['pool'],v['arm'],v['seed']) for v in index.values()]
|
|
if len(actual)!=len(set(actual)) or set(actual)!=expected:
|
|
raise ValueError('Missing/extra/duplicate mapped predictions against frozen original model roster')
|
|
|
|
|
|
def prepared(root,c):
|
|
batch=read_json(root/'batch-complete.json')
|
|
if batch['status']!='ALL_PARTS_TERMINAL' or batch['passed']!=617 or batch['failed'] or not batch['accounting_complete']:
|
|
raise ValueError('All617 custody/geometry attempts must PASS; no silent calibration roster change')
|
|
if batch['contract_sha256']!=sha(root/'contract.json'):raise ValueError('Wrong preparation contract')
|
|
result={}
|
|
for job in batch['jobs']:
|
|
p=checked(job['result_path'],job['result_sha256']);r=read_json(p)
|
|
for name,digest in r['output_sha256'].items():checked(p.parent/name,digest)
|
|
result[r['case']] = (r,p.parent)
|
|
if set(result)!=set(c['cases']):raise ValueError('Missing/extra prepared part')
|
|
return result
|
|
|
|
|
|
def make_maps(root):
|
|
c=contract(root);parts=rows(c);prep=prepared(root,c)
|
|
if (root/'mapping-started.json').exists():raise FileExistsError('No repeat map phase')
|
|
save(root/'mapping-started.json',dict(unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json')))
|
|
start=time.process_time()
|
|
original=read_json(checked(c['S2_contract'],c['S2_contract_sha256']))
|
|
original_index=read_json(checked(c['prediction_index'],c['prediction_index_sha256']))
|
|
ends={case:r['full_VTU_T_end'] for case,(r,_) in prep.items()}
|
|
clocks=[]
|
|
for fold in range(5):
|
|
for pool in (25,50,100):
|
|
value,access=training_clock(parts,fold,pool,ends)
|
|
clocks.append(dict(fold=fold,pool=pool,c=value,learning_access=access,
|
|
definition='mean across components of within-component mean(full_VTU_T_end/t_set)',
|
|
validation_or_test_used_for_scale=False))
|
|
save(root/'training-clocks.json',clocks)
|
|
clock={(r['fold'],r['pool']):r['c'] for r in clocks}
|
|
save(root/'clock-freeze.json',dict(contract_sha256=sha(root/'contract.json'),
|
|
clocks_sha256=sha(root/'training-clocks.json'),batch_sha256=sha(root/'batch-complete.json'),
|
|
before_new_test_mapping=True))
|
|
index={};counter=0;curves=[]
|
|
for row in parts:
|
|
case=row['case'];fold=next(a['fold'] for a in row['assignments'] if a['role']=='test')
|
|
meta,directory=prep[case];d=prediction_inputs(row)
|
|
with np.load(directory/'full-domain-maps.npz',allow_pickle=False) as z:
|
|
if not np.array_equal(z['raw_ids'][d['ids']],d['raw_ids']):raise ValueError('Full/query map order')
|
|
full={key:z[key][d['ids']].copy() for key in z.files if key!='raw_ids'}
|
|
with np.load(directory/'full-node-labels.npz',allow_pickle=False) as z:
|
|
truth=z['seconds'][d['raw_ids']].copy()
|
|
masked_left,masked_right=pps_cdf(truth,d['counts'])
|
|
def output(route,pool,seed,left,right,seconds,source=None):
|
|
nonlocal counter
|
|
tag=route+('' if seed is None else f'-seed{seed}')
|
|
path=root/'predictions'/str(fold)/str(pool)/tag/(case+'.npz')
|
|
path.parent.mkdir(parents=True,exist_ok=True)
|
|
if path.exists():raise FileExistsError('Duplicate mapped prediction')
|
|
extra={}
|
|
if route.startswith('REORDER_') or route=='ORACLE_MASK_TRAIN':
|
|
# These are plug-in estimates, not certified whole-domain bounds.
|
|
alpha=meta['estimated_alpha']
|
|
extra=dict(estimated_domain_lower=alpha*left,estimated_domain_upper=alpha*right+1-alpha)
|
|
np.savez_compressed(path,seconds=seconds,raw_ids=d['raw_ids'],counts=d['counts'],F_left=left,F_right=right,**extra)
|
|
index[str(path)]=dict(sha256=sha(path),case=case,component_id=row['component_id'],fold=fold,pool=pool,
|
|
arm=route,seed=seed,source_prediction=source,oracle=route.startswith('ORACLE_'),
|
|
domain='conditional_PPS' if extra else 'full_tet_lumped_volume')
|
|
counter+=1
|
|
for pool in (25,50,100):
|
|
scale=d['t_set']*clock[fold,pool]
|
|
output('ORACLE_FULL_TRAIN',pool,None,full['truth_left'],full['truth_right'],scale*full['truth_left'])
|
|
output('ORACLE_FULL_END',pool,None,full['truth_left'],full['truth_right'],meta['full_VTU_T_end']*full['truth_left'])
|
|
output('ORACLE_MASK_TRAIN',pool,None,masked_left,masked_right,scale*masked_left)
|
|
output('B0_DW1_CLOCK',pool,None,full['df1_left'],full['df1_right'],scale*full['df1_left'])
|
|
output('B0_DW2_CLOCK',pool,None,full['df2_left'],full['df2_right'],scale*full['df2_left'])
|
|
for model in expected_models(original,fold):
|
|
if model['arm']!='B0' and model['pool']!=pool:continue
|
|
p=prediction_path(Path(c['S2_root']),model,case)
|
|
with np.load(checked(p,original_index[str(p)]),allow_pickle=False) as z:
|
|
if not np.array_equal(z['raw_ids'],d['raw_ids']) or not np.array_equal(z['counts'],d['counts']):raise ValueError('Source prediction domain')
|
|
score=z['seconds'].copy()
|
|
left,right=pps_cdf(score,d['counts']);mapped=scale*left
|
|
old_last=d['raw_ids'][score==score.max()].min();new_last=d['raw_ids'][mapped==mapped.max()].min()
|
|
if old_last!=new_last:raise ValueError('Order-preserving map changed final location')
|
|
if model['arm']=='B0' and not np.allclose(score,d['t_set']*full['df1_left'],rtol=1e-12,atol=1e-12):
|
|
raise ValueError('Unscaled Dw fallback differs from frozen B0')
|
|
output('REORDER_'+model['arm'],pool,model['seed'],left,right,mapped,str(p))
|
|
curves.append(dict(case=case,component_id=row['component_id'],**read_json(directory/'volume-clock-curve.json')))
|
|
save(root/'mapping-progress.json',dict(completed_parts=len(curves),prediction_files=counter,CPU_s=time.process_time()-start))
|
|
validate_manifest(c,index)
|
|
save(root/'prediction-index.json',index)
|
|
save(root/'volume-clock-curves.json',curves)
|
|
distribution={}
|
|
for field in ('seconds','over_t_set','over_full_end'):
|
|
values=np.array([r[field] for r in curves])
|
|
distribution[field]=dict(median=np.median(values,axis=0).tolist(),p10=np.percentile(values,10,axis=0,method='linear').tolist(),
|
|
p90=np.percentile(values,90,axis=0,method='linear').tolist())
|
|
save(root/'curve-summary.json',dict(part_count=len(curves),aggregation='part-equal descriptive median/p10/p90; no independent-sample inference',
|
|
volume_quantiles=curves[0]['volume_quantiles'],**distribution))
|
|
save(root/'predictions-complete.json',dict(status='ALL_NAIVE_MAPS_FROZEN',contract_sha256=sha(root/'contract.json'),
|
|
code_freeze=read_json(root/'code-freeze.json'),prediction_index_sha256=sha(root/'prediction-index.json'),
|
|
clock_freeze_sha256=sha(root/'clock-freeze.json'),prediction_files=counter,CPU_s=time.process_time()-start,
|
|
full_decode_count=617,original_predictions_unchanged=True,new_model_fits=0))
|
|
return 0
|
|
|
|
|
|
def reports(scores,roster,members):
|
|
tables=[];contrasts=[];components=[]
|
|
for subset in ('all','not_known_exposed'):
|
|
for pool in (25,50,100):
|
|
primary={}
|
|
for route,alias in ROUTES.items():
|
|
data=[dict(r,arm=alias) for r in scores if r['arm']==route]
|
|
for metric in ('relative_l2','near_wall','kendall_tau_b','last_location'):
|
|
values,excluded,details=component_values(data,roster,alias,pool,metric,subset!='all',members)
|
|
worst=max(values,key=values.get) if values and metric=='relative_l2' else None
|
|
tables.append(dict(subset=subset,pool=pool,arm=route,metric=metric,
|
|
mean=float(np.mean(list(values.values()))) if values else None,components=len(values),
|
|
parts=sum(details[k]['parts'] for k in values),nodes=sum(details[k]['nodes'] for k in values),
|
|
excluded_components=excluded,worst_component=dict(id=worst,value=values[worst]) if worst else None))
|
|
components.extend(dict(subset=subset,pool=pool,arm=route,metric=metric,component_id=k,value=v,**details[k]) for k,v in values.items())
|
|
if metric=='relative_l2':primary[route]=(values,details)
|
|
ref,details=primary['R2']
|
|
for route,(candidate,cd) in primary.items():
|
|
if route=='R2':continue
|
|
pair=set(ref)&set(candidate);excluded=sorted((set(details)|set(cd))-pair)
|
|
result=contrast(ref,candidate)
|
|
low,high=result.get('low'),result.get('high')
|
|
contrasts.append(dict(subset=subset,pool=pool,reference='R2',candidate=route,
|
|
oracle=route.startswith('ORACLE_'),exploratory=True,primary_confirmatory=False,
|
|
within_descriptive_10percent_band=low is not None and low>=-.1 and high<=.1,
|
|
included_folds=sorted({details[k]['fold'] for k in pair}),excluded_components=excluded,
|
|
excluded_parts=sum(details[k]['parts'] for k in excluded),excluded_nodes=sum(details[k]['nodes'] for k in excluded),**result))
|
|
return tables,contrasts,components
|
|
|
|
|
|
def score_maps(root):
|
|
c=contract(root);roster=read_json(c['roster'])['parts'];by_case={r['case']:r for r in roster}
|
|
prepared(root,c)
|
|
proof=read_json(root/'predictions-complete.json')
|
|
if proof['status']!='ALL_NAIVE_MAPS_FROZEN' or proof['contract_sha256']!=sha(root/'contract.json'):raise ValueError('Mapping not frozen')
|
|
clock_proof=read_json(checked(root/'clock-freeze.json',proof['clock_freeze_sha256']))
|
|
checked(root/'training-clocks.json',clock_proof['clocks_sha256'])
|
|
index=read_json(checked(root/'prediction-index.json',proof['prediction_index_sha256']))
|
|
if len(index)!=proof['prediction_files']:raise ValueError('Manifest size')
|
|
validate_manifest(c,index)
|
|
for path,row in index.items():checked(path,row['sha256'])
|
|
dest=root/'scores';dest.mkdir(exist_ok=False)
|
|
start=time.process_time();save(dest/'started.json',dict(unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json')))
|
|
originals=read_json(checked(c['original_part_scores'],c['original_part_scores_sha256']))
|
|
original_lookup={(r['case'],r['arm'],r['pool'],r['seed']):r for r in originals}
|
|
scores=[r for r in originals if r['arm']=='R2']
|
|
labels={r['case']:r for r in read_json(checked(c['label_index'],c['label_index_sha256']))}
|
|
grouped=defaultdict(list)
|
|
for path,item in index.items():grouped[item['case']].append((path,item))
|
|
unchanged_last=0
|
|
for case,items in sorted(grouped.items()):
|
|
d=prediction_inputs(by_case[case]);entry=labels[case]
|
|
with np.load(checked(entry['path'],entry['sha256']),allow_pickle=False) as z:
|
|
if not np.array_equal(z['raw_ids'],d['raw_ids']) or not np.array_equal(z['counts'],d['counts']):raise ValueError('Scoring cache domain')
|
|
truth=z['seconds'].copy()
|
|
for path,item in items:
|
|
with np.load(path,allow_pickle=False) as z:
|
|
if not np.array_equal(z['raw_ids'],d['raw_ids']) or not np.array_equal(z['counts'],d['counts']):raise ValueError('Mapped domain')
|
|
metrics=part_metrics(z['seconds'],truth,d['counts'],d['near_wall'],d['raw_ids'],d['coordinates_over_Rg'],case.split('_')[0])
|
|
if item['arm'].startswith('REORDER_'):
|
|
source=item['arm'].removeprefix('REORDER_');pool=None if source=='B0' else item['pool']
|
|
old=original_lookup[case,source,pool,item['seed']]
|
|
if metrics['last_location']!=old['last_location'] or metrics['kendall_tau_b']!=old['kendall_tau_b']:
|
|
raise ValueError('Reorder altered invariant last location or sampled tau')
|
|
unchanged_last+=1
|
|
scores.append({k:item[k] for k in ('case','component_id','fold','pool','arm','seed')}|metrics)
|
|
members={p for f in read_json(checked(c['exposure_membership'],c['exposure_membership_sha256'])) for p in f['parts']}
|
|
tables,contrasts,components=reports(scores,roster,members)
|
|
save(dest/'part-scores.json',scores);save(dest/'tables.json',tables);save(dest/'contrasts.json',contrasts);save(dest/'component-scores.json',components)
|
|
save(dest/'result.json',dict(status='NAIVE_ORDERING_COMPLETE',headline='Exploration after S2 results were known; not a fresh confirmatory test',
|
|
no_model_training=True,new_full_custody_decodes=617,reorder_invariant_rows=unchanged_last,
|
|
eikonal_vs_Dijkstra_tested=False,Dw_distance_reused=True,only_new_distance_exponent=2,
|
|
CPU_s=time.process_time()-start,prediction_rows=len(index),
|
|
limitations=['Full discrete tet lumped-volume measure, not exact continuous cavity integration',
|
|
'T_end_full is maximum over original VTU Fill_time; raw/tet/query maxima separately reported',
|
|
'PPS conditional CDF and estimated alpha do not certify unobserved domain',
|
|
'Oracle success does not establish usable learned ordering; poor masked oracle is not a standalone refutation of constant flow',
|
|
'Reordering preserves final location and ties; no gate-ranking validation',
|
|
'All comparisons exploratory; no new material/process generalization or O1 test'],
|
|
output_sha256={p.name:sha(p) for p in dest.glob('*.json')}))
|
|
return 0
|
|
|
|
|
|
def guarded(root,phase):
|
|
c=contract(root);lim=c['limits'];batch=read_json(root/'batch-complete.json')
|
|
prepared(root,c)
|
|
spent=batch['worker_CPU_s']
|
|
if phase=='score':
|
|
previous=read_json(root/'map-guard-result.json')
|
|
if previous['worker']['status']!='PASS':raise ValueError('Map phase did not PASS')
|
|
spent+=previous['worker']['cpu_seconds']
|
|
phase_budget=min(5400 if phase=='map' else 7200,lim['worker_cpu_seconds']-spent,
|
|
lim['post_cpu_seconds']-(spent-batch['worker_CPU_s']))
|
|
if phase_budget<=0:raise ValueError('No postprocess budget')
|
|
command=[sys.executable,__file__,str(root),phase,'--worker']
|
|
done=threading.Event();reasons=[]
|
|
def watch():
|
|
while not done.is_set():
|
|
try:
|
|
if disk_usage(root)>lim['disk_bytes']:reasons.append('GLOBAL_DISK_LIMIT')
|
|
except Exception as exc:reasons.append('DISK_MONITOR_ERROR:'+str(exc))
|
|
if reasons:
|
|
for child in psutil.Process().children(recursive=True):
|
|
try:
|
|
if child.cmdline()==command:stop_group(child.pid)
|
|
except psutil.NoSuchProcess:pass
|
|
done.wait(1)
|
|
os.sched_setaffinity(0,[c['worker_cpus'][0]])
|
|
watcher=threading.Thread(target=watch,daemon=True);watcher.start()
|
|
try:
|
|
w=supervise(command,root/(phase+'-guard'),'worker',dict(total_cpu_seconds=phase_budget,job_cpu_seconds=phase_budget,
|
|
job_wall_seconds=10800,tree_rss_bytes=lim['part_rss_bytes'],disk_bytes=lim['disk_bytes'],max_processes=1),poll=.2)
|
|
finally:done.set();watcher.join(timeout=3)
|
|
if reasons:w.update(status='STOPPED_OR_FAILED',stop_reason=reasons[0])
|
|
save(root/(phase+'-guard-result.json'),dict(worker=w,prior_worker_CPU_s=spent,cumulative_worker_CPU_s=spent+w['cpu_seconds'],
|
|
total_cap_CPU_s=lim['worker_cpu_seconds'],postphase_cap_CPU_s=lim['post_cpu_seconds'],
|
|
guard_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2])))
|
|
return 0 if w['status']=='PASS' else 1
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['map','score']);p.add_argument('--worker',action='store_true');a=p.parse_args()
|
|
raise SystemExit((make_maps(a.root) if a.phase=='map' else score_maps(a.root)) if a.worker else guarded(a.root,a.phase))
|