123 lines
10 KiB
Python
123 lines
10 KiB
Python
"""One reviewed descriptive CPU pass over existing S2 predictions; no inference."""
|
|
import hashlib,json,math,os,resource,sys,time
|
|
from pathlib import Path
|
|
import numpy as np
|
|
from s2_tail_diagnostic_math import weights,tail_membership,tail_error,peak_diagnostic
|
|
|
|
BASE=Path('/home/frank14f/optane/MeltMind')
|
|
ROOT=BASE/'artifacts/phase2-h2-s2-tail-diagnostic-2026-11-18'
|
|
|
|
def sha(path):
|
|
h=hashlib.sha256()
|
|
with Path(path).open('rb') as f:
|
|
for b in iter(lambda:f.read(2**20),b''):h.update(b)
|
|
return h.hexdigest()
|
|
def read(path):return json.loads(Path(path).read_text())
|
|
def save(path,data):
|
|
p=Path(path);p.parent.mkdir(parents=True,exist_ok=True);tmp=p.with_suffix(p.suffix+'.tmp');tmp.write_text(json.dumps(data,indent=2,allow_nan=False)+'\n');tmp.replace(p)
|
|
def checked(path,h):
|
|
p=Path(path)
|
|
if sha(p)!=h:raise ValueError('Source hash mismatch: '+str(p))
|
|
return p
|
|
|
|
def config(root,review=True):
|
|
c=read(root/'plan.json')
|
|
if c['approval']!='1794966078.809549' or c['cpu_cap']!=3600 or c['spec_sha256']!='05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521':raise PermissionError('Fixed descriptive scope')
|
|
for p,h in c['code_sha256'].items():checked(p,h)
|
|
checked(c['spec_path'],c['spec_sha256'])
|
|
if review:
|
|
a=read(root/'review-accepted.json')
|
|
if a['verdict']!='PASS' or a['plan_sha256']!=sha(root/'plan.json') or a['code_sha256']!=c['code_sha256']:raise PermissionError('Actual lightweight review required')
|
|
checked(a['source_receipt'],a['source_receipt_sha256'])
|
|
return c
|
|
|
|
def field(row,arm,name):
|
|
if name in ['E_tail','Miss','False_alarm']:return row['arms'][arm]['tail'][name]
|
|
if name=='original_query_last':return row['arms'][arm]['original_query_last']
|
|
domain,key=name.split('.',1);return row['arms'][arm][domain][key]
|
|
|
|
def summarize(rows):
|
|
metrics=['E_tail','Miss','False_alarm','original_query_last']+[d+'.'+k for d in ['query','full'] for k in ['peak_distance_over_Rg','d_peakset_over_Rg','epsilon','true_time_drop_at_predicted_peak']]+['full.d_tail_over_Rg']
|
|
groups={}
|
|
for row in rows:groups.setdefault(row['component_id'],[]).append(row)
|
|
result={};all_ids=sorted(groups)
|
|
for metric in metrics:
|
|
ids=[];values={'A':[],'H2':[]};excluded=[]
|
|
for cid in all_ids:
|
|
part={arm:[field(r,arm,metric) for r in groups[cid]] for arm in values}
|
|
if any(v is None or not math.isfinite(v) for vs in part.values() for v in vs):excluded.append(cid);continue
|
|
ids.append(cid)
|
|
for arm in values:values[arm].append(float(np.mean(part[arm])))
|
|
if not ids:
|
|
result[metric]=dict(A=None,H2=None,difference=None,low=None,high=None,components=0,excluded=excluded);continue
|
|
a,h=np.array(values['A']),np.array(values['H2']);delta=h-a
|
|
rng=np.random.Generator(np.random.PCG64(np.random.SeedSequence(2026111408)));draw=rng.integers(0,len(ids),size=(2000,len(ids)))
|
|
boot=delta[draw].mean(1);lo,hi=np.percentile(boot,[2.5,97.5],method='linear')
|
|
result[metric]=dict(A=float(a.mean()),H2=float(h.mean()),difference=float(delta.mean()),low=float(lo),high=float(hi),components=len(ids),paired_parts=sum(len(groups[i]) for i in ids),excluded=excluded,undefined_bootstrap=int(np.sum(~np.isfinite(boot))))
|
|
diagnostics={}
|
|
for domain in ['query','full']:
|
|
diagnostics[domain]={}
|
|
for arm in ['A','H2']:
|
|
rs=[r['arms'][arm][domain] for r in rows];categories=sorted({r['category'] for r in rs})
|
|
diagnostics[domain][arm]=dict(parts=len(rs),certified=sum(r['certified'] for r in rs),stable_certificate=sum(r['stable_certificate'] for r in rs),near_boundary=sum(r['near_boundary'] for r in rs),trivial_radius=sum(r['trivial_radius'] for r in rs),constant_truth=sum(r['constant_truth'] for r in rs),categories={k:dict(parts=sum(r['category']==k for r in rs),component_mean_fraction=float(np.mean([np.mean([r['arms'][arm][domain]['category']==k for r in gr]) for gr in groups.values()]))) for k in categories})
|
|
ties=dict(truth_threshold_tied_parts=sum(r['truth_tail']['plateau_nodes']>1 for r in rows),arms={a:dict(prediction_threshold_tied_parts=sum(r['arms'][a]['prediction_tail']['plateau_nodes']>1 for r in rows),max_threshold_mass_correction=max(abs(r['arms'][a]['prediction_tail']['mass_correction']) for r in rows)) for a in ['A','H2']})
|
|
return dict(descriptive_only=True,gate=None,S2_decision_unchanged=True,parts=len(rows),components=len(groups),metrics=result,diagnostics=diagnostics,bootstrap=dict(replicates=2000,seed=2026111408,joint_indices_when_same_component_support=True,unit='evaluation component, not training randomness'),constant_truth_parts=sum(r['truth_tail']['constant_field'] for r in rows),tie_coverage=ties)
|
|
|
|
def worker(root):
|
|
c=config(root);os.sched_setaffinity(0,[24])
|
|
if os.environ.get('CUDA_VISIBLE_DEVICES','')!='':raise PermissionError('CPU only')
|
|
out=root/'output';out.mkdir(exist_ok=False);cpu=time.process_time();rows=[]
|
|
# Existing single-point scores are retained, not replaced by the new endpoint.
|
|
old={r['case']:r for r in read(checked(c['S2_score']['path'],c['S2_score']['sha256']))['rows']}
|
|
save(out/'review-consumed.json',dict(plan_sha256=sha(root/'plan.json'),review_acceptance_sha256=sha(root/'review-accepted.json'),before_target_arrays=True,unix=time.time()))
|
|
for item in c['parts']:
|
|
case=item['case'];source=item['sources']
|
|
for rec in source.values():checked(rec['path'],rec['sha256'])
|
|
with np.load(source['H2']['path'],allow_pickle=False) as z:
|
|
raw=z['raw_ids'].copy();h2=z['seconds'].astype(np.float64);qid=z['query_raw_ids'].copy();query_a=z['A'].copy();query_h=z['pred'].copy()
|
|
with np.load(source['A']['path'],allow_pickle=False) as z:
|
|
if not np.array_equal(raw,z['raw_ids']):raise ValueError('Full A/H2 raw-ID identity')
|
|
a=z['A_seconds'].astype(np.float64)
|
|
with np.load(source['geometry']['path'],allow_pickle=False) as z:
|
|
if not np.array_equal(raw,z['raw_ids']):raise ValueError('Geometry prediction raw-ID identity')
|
|
points=z['points'].copy();mass=z['mass'].copy();rg=float(z['rg'])
|
|
cache_raw=np.load(source['raw_ids']['path'],allow_pickle=False);cache_mass=np.load(source['mass']['path'],allow_pickle=False)
|
|
if not np.array_equal(raw,cache_raw) or not np.array_equal(mass,cache_mass) or len(raw)!=item['nodes'] or np.any(np.diff(raw)<=0):raise ValueError('Complete retained-tet domain/mass')
|
|
ix=np.searchsorted(raw,qid)
|
|
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],qid) or len(np.unique(qid))!=len(qid):raise ValueError('Exact unique query identity')
|
|
if not np.array_equal(a[ix],query_a) or not np.array_equal(h2[ix],query_h):raise ValueError('Original S2 A/H2 query values differ from reused full fields')
|
|
if not math.isfinite(item['t_set']) or item['t_set']<=0 or not np.isfinite(a).all() or not np.isfinite(h2).all():raise ValueError('Finite prediction / time denominator')
|
|
with (out/'target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,path=source['truth']['path'],sha256=source['truth']['sha256'],unix=time.time()))+'\n')
|
|
truth=np.load(source['truth']['path'],allow_pickle=False).astype(np.float64)/item['t_set']
|
|
if truth.shape!=raw.shape or not np.isfinite(truth).all():raise ValueError('Full target domain finite values')
|
|
w=weights(mass);ta,td=tail_membership(truth,w);arms={}
|
|
for arm,pred in [('A',a/item['t_set']),('H2',h2/item['t_set'])]:
|
|
pa,pd=tail_membership(pred,w)
|
|
arms[arm]=dict(tail=tail_error(ta,pa,w),prediction_tail=pd,full=peak_diagnostic(truth,pred,points,raw,rg,w=w,true_tail=ta),query=peak_diagnostic(truth[ix],pred[ix],points[ix],raw[ix],rg),original_query_last=old[case][arm]['last_location']['value'],negative_full_predictions=int(np.sum(pred<0)))
|
|
row=dict(case=case,component_id=item['component_id'],fold=item['fold'],nodes=len(raw),query_nodes=len(ix),t_set=item['t_set'],Rg=rg,truth_tail=td,arms=arms)
|
|
save(out/'parts'/(case+'.json'),row);rows.append(row);save(out/'progress.json',dict(parts=len(rows),worker_cpu_seconds=time.process_time()-cpu,unix=time.time()))
|
|
if len(rows)!=552 or len({r['component_id'] for r in rows})!=204:raise ValueError('Original complete S2 denominator')
|
|
summary=summarize(rows);save(out/'result.json',dict(status='COMPLETE',summary=summary,rows=rows,new_fits=0,new_predictions=0,target_arrays_read=552,new_decodes=0,negative_clipping=False,worker_cpu_seconds=time.process_time()-cpu))
|
|
|
|
def run(root):
|
|
import fcntl,psutil
|
|
from s2_tail_diagnostic_guard import supervise
|
|
start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);c=config(root);os.sched_setaffinity(0,[25]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if (root/'started.json').exists():raise FileExistsError('One diagnostic attempt; no duplicate or retry')
|
|
for p in psutil.process_iter(['pid','cmdline']):
|
|
if p.pid!=os.getpid() and any(Path(x).name=='s2_tail_diagnostic.py' for x in p.info['cmdline'] or []):raise RuntimeError('Existing diagnostic')
|
|
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),cpu_cap=3600,setup_reserve=180,descriptive_only=True,review_sha256=sha(root/'review-accepted.json'),cpu_start=dict(self=start,child_user=u0.ru_utime,child_system=u0.ru_stime)))
|
|
guard=supervise([sys.executable,str(BASE/'tools/s2_tail_diagnostic.py'),'worker',str(root)],root/'guard',dict(cpu_seconds=3300,wall_seconds=7200,rss_bytes=4*2**30,disk_bytes=2**30),coordinator_cpu_start=start,coordinator_cpu_limit=90.)
|
|
u=resource.getrusage(resource.RUSAGE_CHILDREN);charge=180+time.process_time()-start+max(guard['cpu_seconds'],u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)
|
|
ok=guard['status']=='PASS' and not guard['monitor_errors'] and charge<=3600
|
|
if ok:
|
|
result=read(root/'output/result.json');ok=result['status']=='COMPLETE' and result['summary']['parts']==552 and result['summary']['components']==204
|
|
save(root/'complete.json',dict(status='COMPLETE' if ok else 'STOPPED',guard=guard,cumulative_cpu_seconds=charge,cpu_cap=3600,new_fits=0,new_predictions=0,descriptive_only=True,no_gate=True,S2_decision_unchanged=True,retry=False,output_sha256=sha(root/'output/result.json') if (root/'output/result.json').exists() else None))
|
|
if not ok:raise RuntimeError('Diagnostic guard/coverage failed; preserve receipts')
|
|
|
|
if __name__=='__main__':
|
|
root=Path(sys.argv[2]) if len(sys.argv)>2 else ROOT
|
|
if sys.argv[1]=='worker':worker(root)
|
|
elif sys.argv[1]=='run':run(root)
|
|
else:raise ValueError('worker or run')
|