Files
MeltMind/tools/phase2_b11_gate.py
T

250 lines
17 KiB
Python

"""Single approved B11 chain: EIK, then frozen out-of-fold A/H1; no fitting."""
import argparse
import fcntl
import hashlib
import json
import os
from pathlib import Path
import resource
import shutil
import subprocess
import sys
import time
import numpy as np
from scipy.sparse import csr_matrix
from phase2_b11_math import reconstruct,gate_decision
from phase2_h1_worker_v2 import checked,sha,save
from phase2_h1_topology import events,match
from phase2_gates_34 import totals
from s2_custody_common import prediction_inputs
from s2_custody_statistics import part_metrics
from r4bc_guard import supervise
from s2_recovery_monitoring import disk_usage
BASE=Path('/home/frank14f/optane/MeltMind')
ROOT=BASE/'artifacts/phase2-b11-gate-2026-11-17'
H1=BASE/'artifacts/phase2-h1-execution-2026-11-17'
B3=BASE/'artifacts/phase2-b3-gate-2026-11-17'
WRITER=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8')
def read(p):return json.loads(Path(p).read_text())
def prepare(acceptance):
a=read(acceptance);assert a['status']=='PASS'
for p,h in a['code_sha256'].items():checked(BASE/p,h)
old=read(B3/'contract.json');sources={str(B3/'contract.json'):sha(B3/'contract.json'),str(H1/'contract.json'):sha(H1/'contract.json'),str(acceptance):sha(acceptance)}
for row in old['parts']:
for name in ['geometry.npz','truth-events.json','eik-events.json']:
p=B3/'parts'/row['case']/name;sources[str(p)]=sha(p)
spec=BASE/'references/2026-11-17/phase2-design/mold-b11-spec-364446c7.md'
checked(spec,'364446c724c434eea18ca646e1efa352bb9614249b8d6ac00340df4e83a45471');sources[str(spec)]=sha(spec)
names=['phase2_b11_math','phase2_b11_gate','phase2_h1_worker_v2','phase2_residual_math','phase2_h1_topology','phase2_gates_34','phase2_gate_math','s2_custody_common','s2_custody_statistics','r4bc_guard','s2_recovery_supervisor','s2_recovery_monitoring']
ROOT.mkdir(exist_ok=False)
save(ROOT/'contract.json',dict(version='B11-marker-reconstruction-v1',approval='1794891811.393839',parts=old['parts'],
source_sha256=sources,code_sha256={f'tools/{n}.py':sha(BASE/f'tools/{n}.py') for n in names},
threshold=.020,score_thresholds=[.010,.10,.05],gate=dict(precision_gain=.010,recall_loss=.020,last_increase=.010,tau_loss=.010),
cpu=22,total_cpu_seconds=7200,setup_charge_seconds=60,phase_cpu_max=1800,phase_wall_seconds=1800,rss_bytes=8*1024**3,disk_bytes=5*1024**3,
model_wait_wall_seconds=21600,arms=['EIK','A','H1'],query_metrics='unchanged R4bc query IDs/counts/ties and seeded5000-pair tau-b; 50 distinct components',
missing_policy='missing full50 or any undefined paired metric => PARK; no replacement or fitted learning values',new_models=0,new_decodes=0,gpu=False,retry=False))
def config(root):
c=read(root/'contract.json')
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():checked(p,h)
for r in c['parts']:
a=next(x for x in r['assignments'] if x['fold']==0)
if a['role']!='learning' or '100' not in a['learning_pools']:raise PermissionError('B11 label access outside panel')
return c
def freeze(root,name,files):
with Path('/tmp/meltmind-single-writer.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX)
def git(*args):return subprocess.check_output(['git','-C',str(WRITER),*args],text=True).strip()
if git('branch','--show-current')!='codex/s2-execution-diff-20261114' or git('status','--porcelain'):raise RuntimeError('Single writer not clean')
paths=[];hashes={}
for p in files:
p=Path(p);rel=str(p.resolve().relative_to(BASE));dest=WRITER/rel;dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(p,dest);paths.append(rel);hashes[rel]=sha(p)
subprocess.run(['git','-C',str(WRITER),'add','--',*paths],check=True)
if git('diff','--cached','--name-only'):subprocess.run(['git','-C',str(WRITER),'commit','-m','B11 '+name+' frozen predictions and scoring sources'],check=True)
commit=git('rev-parse','HEAD')
for p,h in hashes.items():
if hashlib.sha256(subprocess.check_output(['git','-C',str(WRITER),'show',commit+':'+p])).hexdigest()!=h:raise ValueError('B11 Git mismatch')
proof=dict(commit=commit,checkout=str(WRITER),sha256=hashes,remote_published=False)
save(root/(name+'-git-freeze.json'),proof)
return proof
def verify_freeze(path,check_git=True):
proof=read(path)
for rel,h in proof['sha256'].items():
checked(BASE/rel,h)
if check_git:
data=subprocess.check_output(['git','show',proof['commit']+':'+rel],cwd=proof['checkout'])
if hashlib.sha256(data).hexdigest()!=h:raise ValueError('Frozen Git bytes differ')
def model_entries(check_git=False):
proof=read(H1/'scoring-git-freeze.json')
for name in ['contract.json','predictions-freeze.json','geometry/manifest.json']:
checked(H1/name,proof['sha256'][name])
if check_git:
rel=str((H1/name).relative_to(BASE));data=subprocess.check_output(['git','show',proof['commit']+':'+rel],cwd=proof['checkout'])
if hashlib.sha256(data).hexdigest()!=proof['sha256'][name]:raise ValueError('B1 full-node freeze mismatch')
f=read(H1/'predictions-freeze.json')
if not f['all_prediction_attempts_terminal'] or not f['accounting_complete']:raise PermissionError('B1 prediction barrier incomplete')
return f['entries']
def select_entry(row,entries):
fold=next(a['fold'] for a in row['assignments'] if a['role']=='test')
found=[e for e in entries if e['protocol']=='grouped' and e['pool']==100 and e['fold']==fold and e['case']==row['case']]
if len(found)!=1:raise ValueError('Missing/duplicate full-node out-of-fold100 common A/H1 prediction')
return found[0]
def geometry(row,c):
p=B3/'parts'/row['case']/'geometry.npz'
with np.load(checked(p,c['source_sha256'][str(p)]),allow_pickle=False) as z:
x=z['points'].copy();raw=z['raw_ids'].copy();mass=z['mass'].copy();rg=float(z['rg'])
adjacency=csr_matrix((z['data'],z['indices'],z['indptr']),shape=(len(raw),len(raw)))
return x,raw,mass,rg,adjacency
def build(root,c,kind):
phase=root/'phases'/(kind+'-build');dest=phase/'parts';dest.mkdir(exist_ok=False)
if kind=='models':
authorization=read(root/'models-source-authorization.json')
checked(H1/'scoring-git-freeze.json',authorization['freeze_sha256']);checked(H1/'predictions-freeze.json',authorization['manifest_sha256'])
entries=model_entries() if kind=='models' else None
output=[];failures=[];start=time.process_time()
for row in c['parts']:
case=row['case'];out=dest/case;out.mkdir();x,raw,mass,rg,adjacency=geometry(row,c)
try:
if kind=='eik':
cache=row['v5_cache'];p=Path(cache['output'])/'Vf.npy'
fields=dict(EIK=np.load(checked(p,cache['output_sha256']['Vf.npy']),allow_pickle=False)*row['preparation']['t_set']);source=dict(path=str(p),sha256=sha(p),role='frozen input-only EIK')
else:
e=select_entry(row,entries)
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z:
if not np.array_equal(z['raw_ids'],raw):raise ValueError('B11 A/H1 original node domain differs')
fields=dict(A=z['A_seconds'].copy(),H1=z['seconds'].copy())
source=dict(**e,role='100pct original out-of-fold predictions on selected learning panel')
for arm,field in fields.items():
before=time.process_time();clean,bars=reconstruct(field,mass,raw,adjacency,c['threshold']);seconds=time.process_time()-before
p=out/(arm+'.npz');np.savez_compressed(p,raw=field,clean=clean,raw_ids=raw)
barpath=out/(arm+'-bars.json');save(barpath,bars)
top=lambda a:sorted(map(int,raw[a==a.max()]))
info=dict(case=case,component_id=row['component_id'],arm=arm,path=str(p),sha256=sha(p),bars_path=str(barpath),bars_sha256=sha(barpath),
source=source,reconstruction_cpu_seconds=seconds,raw_full_max_ids=top(field),clean_full_max_ids=top(clean),
changed_volume_fraction=float(mass[clean!=field].sum()/mass.sum()),target_values_read=False)
save(out/(arm+'-result.json'),info);output.append(info)
except Exception as exc:
failure=dict(case=case,type=type(exc).__name__,message=str(exc));save(out/'failure.json',failure);failures.append(failure)
if disk_usage(root)>=c['disk_bytes']:raise RuntimeError('Whole B11 disk cap')
save(phase/'progress.json',dict(cases=len({x['case'] for x in output}),fields=len(output),failures=failures,cpu_seconds=time.process_time()-start))
save(phase/'manifest.json',dict(entries=output,failures=failures,kind=kind,cpu_seconds=time.process_time()-start,contract_sha256=sha(root/'contract.json')))
def tie_fraction(a):
_,counts=np.unique(a,return_counts=True);n=len(a)
return float(np.sum(counts*(counts-1))/(n*(n-1))) if n>1 else None
def score(root,c,kind):
authorization=read(root/(kind+'-score-authorization.json'))
checked(root/(kind+'-git-freeze.json'),authorization['freeze_sha256'])
verify_freeze(root/(kind+'-git-freeze.json'),check_git=False)
built=read(root/'phases'/(kind+'-build')/'manifest.json');phase=root/'phases'/(kind+'-score');out=phase/'parts';out.mkdir(exist_ok=False)
rows=[];failures=list(built['failures']);start=time.process_time();by={r['case']:r for r in c['parts']}
for e in built['entries']:
case=e['case'];row=by[case];dest=out/(case+'-'+e['arm']);dest.mkdir()
try:
x,raw,mass,rg,adjacency=geometry(row,c)
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z:
if not np.array_equal(raw,z['raw_ids']):raise ValueError('B11 field IDs changed')
fields={k:z[k].copy() for k in ['raw','clean']}
for p,h in [(e['bars_path'],e['bars_sha256'])]:checked(p,h)
refpath=B3/'parts'/case/'truth-events.json';ref=read(checked(refpath,c['source_sha256'][str(refpath)]))
cache=row['v5_cache'];yp=Path(cache['output'])/'seconds.npy'
save(dest/'target-access.json',dict(role='B3_selected_learning_panel_only',path=str(yp),sha256=cache['output_sha256']['seconds.npy']))
truth=np.load(checked(yp,cache['output_sha256']['seconds.npy']),allow_pickle=False)
q=prediction_inputs(row);ix=np.searchsorted(raw,q['raw_ids'])
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):raise ValueError('R4bc query IDs differ')
result=dict(case=case,component_id=row['component_id'],arm=e['arm'],changed_volume_fraction=e['changed_volume_fraction'],query_ids=list(map(int,q['raw_ids'])),query_counts=list(map(int,q['counts'])))
for label,field in fields.items():
if e['arm']=='EIK' and label=='raw':
p=B3/'parts'/case/'eik-events.json';old=read(checked(p,c['source_sha256'][str(p)]));ev=old['events'];matching=old['match']
else:ev=events(field,mass,raw,x,adjacency);matching=match(ref,ev,adjacency,rg)
save(dest/(label+'-events.json'),dict(events=ev,match=matching));result[label]=matching
result[label+'_metrics']=part_metrics(field[ix],truth[ix],q['counts'],q['near_wall'],q['raw_ids'],q['coordinates_over_Rg'],int(case.split('_')[0]))
result[label+'_query_tie_fraction']=tie_fraction(field[ix])
qtop=q['raw_ids'][field[ix]==field[ix].max()];result[label+'_query_last_raw_id']=int(qtop.min());result[label+'_query_max_plateau_ids']=list(map(int,qtop))
save(dest/'result.json',result);rows.append(result)
except Exception as exc:
failure=dict(case=case,arm=e['arm'],type=type(exc).__name__,message=str(exc));save(dest/'failure.json',failure);failures.append(failure)
if disk_usage(root)>=c['disk_bytes']:raise RuntimeError('Whole B11 disk cap')
save(phase/'progress.json',dict(scored=len(rows),failures=failures,cpu_seconds=time.process_time()-start))
summaries={}
for arm in (['EIK'] if kind=='eik' else ['A','H1']):
selected=[r for r in rows if r['arm']==arm];rawtot=totals(selected,'raw');cleantot=totals(selected,'clean');paired={};undefined={}
for metric in ['last_location','kendall_tau_b','relative_l2']:
good=[(r['raw_metrics'][metric]['value'],r['clean_metrics'][metric]['value']) for r in selected if r['raw_metrics'][metric]['value'] is not None and r['clean_metrics'][metric]['value'] is not None]
paired[metric]=good;undefined[metric]=[r['case'] for r in selected if r['raw_metrics'][metric]['value'] is None or r['clean_metrics'][metric]['value'] is None]
summaries[arm]=dict(raw=rawtot,clean=cleantot,parts=len(selected),undefined=undefined,paired_denominators={k:len(v) for k,v in paired.items()},
field_means={k:dict(raw=float(np.mean([a for a,b in v])) if v else None,clean=float(np.mean([b for a,b in v])) if v else None) for k,v in paired.items()},
tree_gate=gate_decision(rawtot,cleantot,{k:paired[k] for k in ['last_location','kendall_tau_b']},len(selected),failures))
save(phase/'result.json',dict(status='COMPLETE',kind=kind,summaries=summaries,rows=rows,failures=failures,cpu_seconds=time.process_time()-start,
interpretation='learning-selected panel posthoc gate; no independent blind evaluation; no weld or remesh claim'))
def run(root):
c=config(root);os.sched_setaffinity(0,[c['cpu']]);start=time.process_time();charges=0.;records=[];completed=[]
usage0=resource.getrusage(resource.RUSAGE_CHILDREN);child_start=usage0.ru_utime+usage0.ru_stime
with (root/'run.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'actual-start.json').exists():raise FileExistsError('No B11 retry')
save(root/'actual-start.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json')))
def total():
usage=resource.getrusage(resource.RUSAGE_CHILDREN)
return c['setup_charge_seconds']+max(charges,usage.ru_utime+usage.ru_stime-child_start)+time.process_time()-start
def phase(kind,mode):
nonlocal charges
name=kind+'-'+mode;dest=root/'phases'/name;dest.mkdir(parents=True,exist_ok=False)
cap=min(c['phase_cpu_max'],c['total_cpu_seconds']-total()-120)
if cap<=0:raise RuntimeError('Cumulative B11 CPU exhausted')
save(root/'live-progress.json',dict(stage=name,completed=completed,cumulative_cpu_seconds=total(),unix=time.time()))
g=supervise([sys.executable,'tools/phase2_b11_gate.py',mode,str(root),kind],dest,
dict(cpu_seconds=cap,wall_seconds=c['phase_wall_seconds'],rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes']))
charges+=g['cpu_seconds'];records.append(dict(phase=name,guard=g));save(root/'phase-ledger.json',dict(records=records,cumulative_cpu_seconds=total()))
if g['status']!='PASS' or total()>c['total_cpu_seconds'] or disk_usage(root)>c['disk_bytes']:raise RuntimeError('B11 phase/resource stop: '+name)
try:
for kind in ['eik','models']:
if kind=='models':
begin=time.monotonic()
while not (H1/'scoring-git-freeze.json').exists():
done=read(H1/'complete.json') if (H1/'complete.json').exists() else None
if done or time.monotonic()-begin>=c['model_wait_wall_seconds']:raise RuntimeError('H1 full-node freeze unavailable; EIK retained')
save(root/'live-progress.json',dict(stage='WAIT_ORIGINAL_H1_PREDICTION_FREEZE',completed=completed,cumulative_cpu_seconds=total(),unix=time.time()))
if total()>=c['total_cpu_seconds']-120:raise RuntimeError('B11 waiting CPU cap')
time.sleep(15)
model_entries(check_git=True)
save(root/'models-source-authorization.json',dict(freeze_sha256=sha(H1/'scoring-git-freeze.json'),manifest_sha256=sha(H1/'predictions-freeze.json')))
phase(kind,'build')
manifest=root/'phases'/(kind+'-build')/'manifest.json'
freeze(root,kind,[root/'contract.json',manifest,*[BASE/p for p in c['code_sha256']]])
verify_freeze(root/(kind+'-git-freeze.json'))
save(root/(kind+'-score-authorization.json'),dict(freeze_sha256=sha(root/(kind+'-git-freeze.json'))))
phase(kind,'score');completed.append(kind)
save(root/(kind+'-complete.json'),dict(status='COMPLETE',result_sha256=sha(root/'phases'/(kind+'-score')/'result.json'),cumulative_cpu_seconds=total()))
save(root/'complete.json',dict(status='COMPLETE',completed=completed,records=records,cumulative_cpu_seconds=total(),retry=False))
except BaseException as exc:
save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),completed=completed,records=records,cumulative_cpu_seconds=total(),retry=False));raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('mode',choices=['prepare','run','build','score']);p.add_argument('path',type=Path);p.add_argument('kind',nargs='?');a=p.parse_args()
if a.mode=='prepare':prepare(a.path)
elif a.mode=='run':run(a.path)
else:(build if a.mode=='build' else score)(a.path,config(a.path),a.kind)