"""One reviewed descriptive CPU pass over existing S2 predictions; no inference.""" import hashlib,json,math,os,resource,sys,time from pathlib import Path import numpy as np from s2_tail_diagnostic_math import weights,tail_membership,tail_error,peak_diagnostic BASE=Path('/home/frank14f/optane/MeltMind') ROOT=BASE/'artifacts/phase2-h2-s2-tail-diagnostic-2026-11-18' def sha(path): h=hashlib.sha256() with Path(path).open('rb') as f: for b in iter(lambda:f.read(2**20),b''):h.update(b) return h.hexdigest() def read(path):return json.loads(Path(path).read_text()) def save(path,data): p=Path(path);p.parent.mkdir(parents=True,exist_ok=True);tmp=p.with_suffix(p.suffix+'.tmp');tmp.write_text(json.dumps(data,indent=2,allow_nan=False)+'\n');tmp.replace(p) def checked(path,h): p=Path(path) if sha(p)!=h:raise ValueError('Source hash mismatch: '+str(p)) return p def config(root,review=True): c=read(root/'plan.json') if c['approval']!='1794966078.809549' or c['cpu_cap']!=3600 or c['spec_sha256']!='05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521':raise PermissionError('Fixed descriptive scope') for p,h in c['code_sha256'].items():checked(p,h) checked(c['spec_path'],c['spec_sha256']) if review: a=read(root/'review-accepted.json') if a['verdict']!='PASS' or a['plan_sha256']!=sha(root/'plan.json') or a['code_sha256']!=c['code_sha256']:raise PermissionError('Actual lightweight review required') checked(a['source_receipt'],a['source_receipt_sha256']) return c def field(row,arm,name): if name in ['E_tail','Miss','False_alarm']:return row['arms'][arm]['tail'][name] if name=='original_query_last':return row['arms'][arm]['original_query_last'] domain,key=name.split('.',1);return row['arms'][arm][domain][key] def summarize(rows): metrics=['E_tail','Miss','False_alarm','original_query_last']+[d+'.'+k for d in ['query','full'] for k in ['peak_distance_over_Rg','d_peakset_over_Rg','epsilon','true_time_drop_at_predicted_peak']]+['full.d_tail_over_Rg'] groups={} for row in rows:groups.setdefault(row['component_id'],[]).append(row) result={};all_ids=sorted(groups) for metric in metrics: ids=[];values={'A':[],'H2':[]};excluded=[] for cid in all_ids: part={arm:[field(r,arm,metric) for r in groups[cid]] for arm in values} if any(v is None or not math.isfinite(v) for vs in part.values() for v in vs):excluded.append(cid);continue ids.append(cid) for arm in values:values[arm].append(float(np.mean(part[arm]))) if not ids: result[metric]=dict(A=None,H2=None,difference=None,low=None,high=None,components=0,excluded=excluded);continue a,h=np.array(values['A']),np.array(values['H2']);delta=h-a rng=np.random.Generator(np.random.PCG64(np.random.SeedSequence(2026111408)));draw=rng.integers(0,len(ids),size=(2000,len(ids))) boot=delta[draw].mean(1);lo,hi=np.percentile(boot,[2.5,97.5],method='linear') result[metric]=dict(A=float(a.mean()),H2=float(h.mean()),difference=float(delta.mean()),low=float(lo),high=float(hi),components=len(ids),paired_parts=sum(len(groups[i]) for i in ids),excluded=excluded,undefined_bootstrap=int(np.sum(~np.isfinite(boot)))) diagnostics={} for domain in ['query','full']: diagnostics[domain]={} for arm in ['A','H2']: rs=[r['arms'][arm][domain] for r in rows];categories=sorted({r['category'] for r in rs}) diagnostics[domain][arm]=dict(parts=len(rs),certified=sum(r['certified'] for r in rs),stable_certificate=sum(r['stable_certificate'] for r in rs),near_boundary=sum(r['near_boundary'] for r in rs),trivial_radius=sum(r['trivial_radius'] for r in rs),constant_truth=sum(r['constant_truth'] for r in rs),categories={k:dict(parts=sum(r['category']==k for r in rs),component_mean_fraction=float(np.mean([np.mean([r['arms'][arm][domain]['category']==k for r in gr]) for gr in groups.values()]))) for k in categories}) ties=dict(truth_threshold_tied_parts=sum(r['truth_tail']['plateau_nodes']>1 for r in rows),arms={a:dict(prediction_threshold_tied_parts=sum(r['arms'][a]['prediction_tail']['plateau_nodes']>1 for r in rows),max_threshold_mass_correction=max(abs(r['arms'][a]['prediction_tail']['mass_correction']) for r in rows)) for a in ['A','H2']}) return dict(descriptive_only=True,gate=None,S2_decision_unchanged=True,parts=len(rows),components=len(groups),metrics=result,diagnostics=diagnostics,bootstrap=dict(replicates=2000,seed=2026111408,joint_indices_when_same_component_support=True,unit='evaluation component, not training randomness'),constant_truth_parts=sum(r['truth_tail']['constant_field'] for r in rows),tie_coverage=ties) def worker(root): c=config(root);os.sched_setaffinity(0,[24]) if os.environ.get('CUDA_VISIBLE_DEVICES','')!='':raise PermissionError('CPU only') out=root/'output';out.mkdir(exist_ok=False);cpu=time.process_time();rows=[] # Existing single-point scores are retained, not replaced by the new endpoint. old={r['case']:r for r in read(checked(c['S2_score']['path'],c['S2_score']['sha256']))['rows']} save(out/'review-consumed.json',dict(plan_sha256=sha(root/'plan.json'),review_acceptance_sha256=sha(root/'review-accepted.json'),before_target_arrays=True,unix=time.time())) for item in c['parts']: case=item['case'];source=item['sources'] for rec in source.values():checked(rec['path'],rec['sha256']) with np.load(source['H2']['path'],allow_pickle=False) as z: raw=z['raw_ids'].copy();h2=z['seconds'].astype(np.float64);qid=z['query_raw_ids'].copy();query_a=z['A'].copy();query_h=z['pred'].copy() with np.load(source['A']['path'],allow_pickle=False) as z: if not np.array_equal(raw,z['raw_ids']):raise ValueError('Full A/H2 raw-ID identity') a=z['A_seconds'].astype(np.float64) with np.load(source['geometry']['path'],allow_pickle=False) as z: if not np.array_equal(raw,z['raw_ids']):raise ValueError('Geometry prediction raw-ID identity') points=z['points'].copy();mass=z['mass'].copy();rg=float(z['rg']) cache_raw=np.load(source['raw_ids']['path'],allow_pickle=False);cache_mass=np.load(source['mass']['path'],allow_pickle=False) if not np.array_equal(raw,cache_raw) or not np.array_equal(mass,cache_mass) or len(raw)!=item['nodes'] or np.any(np.diff(raw)<=0):raise ValueError('Complete retained-tet domain/mass') ix=np.searchsorted(raw,qid) if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],qid) or len(np.unique(qid))!=len(qid):raise ValueError('Exact unique query identity') if not np.array_equal(a[ix],query_a) or not np.array_equal(h2[ix],query_h):raise ValueError('Original S2 A/H2 query values differ from reused full fields') if not math.isfinite(item['t_set']) or item['t_set']<=0 or not np.isfinite(a).all() or not np.isfinite(h2).all():raise ValueError('Finite prediction / time denominator') with (out/'target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,path=source['truth']['path'],sha256=source['truth']['sha256'],unix=time.time()))+'\n') truth=np.load(source['truth']['path'],allow_pickle=False).astype(np.float64)/item['t_set'] if truth.shape!=raw.shape or not np.isfinite(truth).all():raise ValueError('Full target domain finite values') w=weights(mass);ta,td=tail_membership(truth,w);arms={} for arm,pred in [('A',a/item['t_set']),('H2',h2/item['t_set'])]: pa,pd=tail_membership(pred,w) arms[arm]=dict(tail=tail_error(ta,pa,w),prediction_tail=pd,full=peak_diagnostic(truth,pred,points,raw,rg,w=w,true_tail=ta),query=peak_diagnostic(truth[ix],pred[ix],points[ix],raw[ix],rg),original_query_last=old[case][arm]['last_location']['value'],negative_full_predictions=int(np.sum(pred<0))) row=dict(case=case,component_id=item['component_id'],fold=item['fold'],nodes=len(raw),query_nodes=len(ix),t_set=item['t_set'],Rg=rg,truth_tail=td,arms=arms) save(out/'parts'/(case+'.json'),row);rows.append(row);save(out/'progress.json',dict(parts=len(rows),worker_cpu_seconds=time.process_time()-cpu,unix=time.time())) if len(rows)!=552 or len({r['component_id'] for r in rows})!=204:raise ValueError('Original complete S2 denominator') summary=summarize(rows);save(out/'result.json',dict(status='COMPLETE',summary=summary,rows=rows,new_fits=0,new_predictions=0,target_arrays_read=552,new_decodes=0,negative_clipping=False,worker_cpu_seconds=time.process_time()-cpu)) def run(root): import fcntl,psutil from s2_tail_diagnostic_guard import supervise start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);c=config(root);os.sched_setaffinity(0,[25]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if (root/'started.json').exists():raise FileExistsError('One diagnostic attempt; no duplicate or retry') for p in psutil.process_iter(['pid','cmdline']): if p.pid!=os.getpid() and any(Path(x).name=='s2_tail_diagnostic.py' for x in p.info['cmdline'] or []):raise RuntimeError('Existing diagnostic') save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),cpu_cap=3600,setup_reserve=180,descriptive_only=True,review_sha256=sha(root/'review-accepted.json'),cpu_start=dict(self=start,child_user=u0.ru_utime,child_system=u0.ru_stime))) guard=supervise([sys.executable,str(BASE/'tools/s2_tail_diagnostic.py'),'worker',str(root)],root/'guard',dict(cpu_seconds=3300,wall_seconds=7200,rss_bytes=4*2**30,disk_bytes=2**30),coordinator_cpu_start=start,coordinator_cpu_limit=90.) u=resource.getrusage(resource.RUSAGE_CHILDREN);charge=180+time.process_time()-start+max(guard['cpu_seconds'],u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime) ok=guard['status']=='PASS' and not guard['monitor_errors'] and charge<=3600 if ok: result=read(root/'output/result.json');ok=result['status']=='COMPLETE' and result['summary']['parts']==552 and result['summary']['components']==204 save(root/'complete.json',dict(status='COMPLETE' if ok else 'STOPPED',guard=guard,cumulative_cpu_seconds=charge,cpu_cap=3600,new_fits=0,new_predictions=0,descriptive_only=True,no_gate=True,S2_decision_unchanged=True,retry=False,output_sha256=sha(root/'output/result.json') if (root/'output/result.json').exists() else None)) if not ok:raise RuntimeError('Diagnostic guard/coverage failed; preserve receipts') if __name__=='__main__': root=Path(sys.argv[2]) if len(sys.argv)>2 else ROOT if sys.argv[1]=='worker':worker(root) elif sys.argv[1]=='run':run(root) else:raise ValueError('worker or run')