250 lines
17 KiB
Python
250 lines
17 KiB
Python
"""Single approved B11 chain: EIK, then frozen out-of-fold A/H1; no fitting."""
|
|
import argparse
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import numpy as np
|
|
from scipy.sparse import csr_matrix
|
|
from phase2_b11_math import reconstruct,gate_decision
|
|
from phase2_h1_worker_v2 import checked,sha,save
|
|
from phase2_h1_topology import events,match
|
|
from phase2_gates_34 import totals
|
|
from s2_custody_common import prediction_inputs
|
|
from s2_custody_statistics import part_metrics
|
|
from r4bc_guard import supervise
|
|
from s2_recovery_monitoring import disk_usage
|
|
|
|
BASE=Path('/home/frank14f/optane/MeltMind')
|
|
ROOT=BASE/'artifacts/phase2-b11-gate-2026-11-17'
|
|
H1=BASE/'artifacts/phase2-h1-execution-2026-11-17'
|
|
B3=BASE/'artifacts/phase2-b3-gate-2026-11-17'
|
|
WRITER=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8')
|
|
def read(p):return json.loads(Path(p).read_text())
|
|
|
|
|
|
def prepare(acceptance):
|
|
a=read(acceptance);assert a['status']=='PASS'
|
|
for p,h in a['code_sha256'].items():checked(BASE/p,h)
|
|
old=read(B3/'contract.json');sources={str(B3/'contract.json'):sha(B3/'contract.json'),str(H1/'contract.json'):sha(H1/'contract.json'),str(acceptance):sha(acceptance)}
|
|
for row in old['parts']:
|
|
for name in ['geometry.npz','truth-events.json','eik-events.json']:
|
|
p=B3/'parts'/row['case']/name;sources[str(p)]=sha(p)
|
|
spec=BASE/'references/2026-11-17/phase2-design/mold-b11-spec-364446c7.md'
|
|
checked(spec,'364446c724c434eea18ca646e1efa352bb9614249b8d6ac00340df4e83a45471');sources[str(spec)]=sha(spec)
|
|
names=['phase2_b11_math','phase2_b11_gate','phase2_h1_worker_v2','phase2_residual_math','phase2_h1_topology','phase2_gates_34','phase2_gate_math','s2_custody_common','s2_custody_statistics','r4bc_guard','s2_recovery_supervisor','s2_recovery_monitoring']
|
|
ROOT.mkdir(exist_ok=False)
|
|
save(ROOT/'contract.json',dict(version='B11-marker-reconstruction-v1',approval='1794891811.393839',parts=old['parts'],
|
|
source_sha256=sources,code_sha256={f'tools/{n}.py':sha(BASE/f'tools/{n}.py') for n in names},
|
|
threshold=.020,score_thresholds=[.010,.10,.05],gate=dict(precision_gain=.010,recall_loss=.020,last_increase=.010,tau_loss=.010),
|
|
cpu=22,total_cpu_seconds=7200,setup_charge_seconds=60,phase_cpu_max=1800,phase_wall_seconds=1800,rss_bytes=8*1024**3,disk_bytes=5*1024**3,
|
|
model_wait_wall_seconds=21600,arms=['EIK','A','H1'],query_metrics='unchanged R4bc query IDs/counts/ties and seeded5000-pair tau-b; 50 distinct components',
|
|
missing_policy='missing full50 or any undefined paired metric => PARK; no replacement or fitted learning values',new_models=0,new_decodes=0,gpu=False,retry=False))
|
|
|
|
|
|
def config(root):
|
|
c=read(root/'contract.json')
|
|
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():checked(p,h)
|
|
for r in c['parts']:
|
|
a=next(x for x in r['assignments'] if x['fold']==0)
|
|
if a['role']!='learning' or '100' not in a['learning_pools']:raise PermissionError('B11 label access outside panel')
|
|
return c
|
|
|
|
|
|
def freeze(root,name,files):
|
|
with Path('/tmp/meltmind-single-writer.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX)
|
|
def git(*args):return subprocess.check_output(['git','-C',str(WRITER),*args],text=True).strip()
|
|
if git('branch','--show-current')!='codex/s2-execution-diff-20261114' or git('status','--porcelain'):raise RuntimeError('Single writer not clean')
|
|
paths=[];hashes={}
|
|
for p in files:
|
|
p=Path(p);rel=str(p.resolve().relative_to(BASE));dest=WRITER/rel;dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(p,dest);paths.append(rel);hashes[rel]=sha(p)
|
|
subprocess.run(['git','-C',str(WRITER),'add','--',*paths],check=True)
|
|
if git('diff','--cached','--name-only'):subprocess.run(['git','-C',str(WRITER),'commit','-m','B11 '+name+' frozen predictions and scoring sources'],check=True)
|
|
commit=git('rev-parse','HEAD')
|
|
for p,h in hashes.items():
|
|
if hashlib.sha256(subprocess.check_output(['git','-C',str(WRITER),'show',commit+':'+p])).hexdigest()!=h:raise ValueError('B11 Git mismatch')
|
|
proof=dict(commit=commit,checkout=str(WRITER),sha256=hashes,remote_published=False)
|
|
save(root/(name+'-git-freeze.json'),proof)
|
|
return proof
|
|
|
|
|
|
def verify_freeze(path,check_git=True):
|
|
proof=read(path)
|
|
for rel,h in proof['sha256'].items():
|
|
checked(BASE/rel,h)
|
|
if check_git:
|
|
data=subprocess.check_output(['git','show',proof['commit']+':'+rel],cwd=proof['checkout'])
|
|
if hashlib.sha256(data).hexdigest()!=h:raise ValueError('Frozen Git bytes differ')
|
|
|
|
|
|
def model_entries(check_git=False):
|
|
proof=read(H1/'scoring-git-freeze.json')
|
|
for name in ['contract.json','predictions-freeze.json','geometry/manifest.json']:
|
|
checked(H1/name,proof['sha256'][name])
|
|
if check_git:
|
|
rel=str((H1/name).relative_to(BASE));data=subprocess.check_output(['git','show',proof['commit']+':'+rel],cwd=proof['checkout'])
|
|
if hashlib.sha256(data).hexdigest()!=proof['sha256'][name]:raise ValueError('B1 full-node freeze mismatch')
|
|
f=read(H1/'predictions-freeze.json')
|
|
if not f['all_prediction_attempts_terminal'] or not f['accounting_complete']:raise PermissionError('B1 prediction barrier incomplete')
|
|
return f['entries']
|
|
|
|
|
|
def select_entry(row,entries):
|
|
fold=next(a['fold'] for a in row['assignments'] if a['role']=='test')
|
|
found=[e for e in entries if e['protocol']=='grouped' and e['pool']==100 and e['fold']==fold and e['case']==row['case']]
|
|
if len(found)!=1:raise ValueError('Missing/duplicate full-node out-of-fold100 common A/H1 prediction')
|
|
return found[0]
|
|
|
|
|
|
def geometry(row,c):
|
|
p=B3/'parts'/row['case']/'geometry.npz'
|
|
with np.load(checked(p,c['source_sha256'][str(p)]),allow_pickle=False) as z:
|
|
x=z['points'].copy();raw=z['raw_ids'].copy();mass=z['mass'].copy();rg=float(z['rg'])
|
|
adjacency=csr_matrix((z['data'],z['indices'],z['indptr']),shape=(len(raw),len(raw)))
|
|
return x,raw,mass,rg,adjacency
|
|
|
|
|
|
def build(root,c,kind):
|
|
phase=root/'phases'/(kind+'-build');dest=phase/'parts';dest.mkdir(exist_ok=False)
|
|
if kind=='models':
|
|
authorization=read(root/'models-source-authorization.json')
|
|
checked(H1/'scoring-git-freeze.json',authorization['freeze_sha256']);checked(H1/'predictions-freeze.json',authorization['manifest_sha256'])
|
|
entries=model_entries() if kind=='models' else None
|
|
output=[];failures=[];start=time.process_time()
|
|
for row in c['parts']:
|
|
case=row['case'];out=dest/case;out.mkdir();x,raw,mass,rg,adjacency=geometry(row,c)
|
|
try:
|
|
if kind=='eik':
|
|
cache=row['v5_cache'];p=Path(cache['output'])/'Vf.npy'
|
|
fields=dict(EIK=np.load(checked(p,cache['output_sha256']['Vf.npy']),allow_pickle=False)*row['preparation']['t_set']);source=dict(path=str(p),sha256=sha(p),role='frozen input-only EIK')
|
|
else:
|
|
e=select_entry(row,entries)
|
|
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z:
|
|
if not np.array_equal(z['raw_ids'],raw):raise ValueError('B11 A/H1 original node domain differs')
|
|
fields=dict(A=z['A_seconds'].copy(),H1=z['seconds'].copy())
|
|
source=dict(**e,role='100pct original out-of-fold predictions on selected learning panel')
|
|
for arm,field in fields.items():
|
|
before=time.process_time();clean,bars=reconstruct(field,mass,raw,adjacency,c['threshold']);seconds=time.process_time()-before
|
|
p=out/(arm+'.npz');np.savez_compressed(p,raw=field,clean=clean,raw_ids=raw)
|
|
barpath=out/(arm+'-bars.json');save(barpath,bars)
|
|
top=lambda a:sorted(map(int,raw[a==a.max()]))
|
|
info=dict(case=case,component_id=row['component_id'],arm=arm,path=str(p),sha256=sha(p),bars_path=str(barpath),bars_sha256=sha(barpath),
|
|
source=source,reconstruction_cpu_seconds=seconds,raw_full_max_ids=top(field),clean_full_max_ids=top(clean),
|
|
changed_volume_fraction=float(mass[clean!=field].sum()/mass.sum()),target_values_read=False)
|
|
save(out/(arm+'-result.json'),info);output.append(info)
|
|
except Exception as exc:
|
|
failure=dict(case=case,type=type(exc).__name__,message=str(exc));save(out/'failure.json',failure);failures.append(failure)
|
|
if disk_usage(root)>=c['disk_bytes']:raise RuntimeError('Whole B11 disk cap')
|
|
save(phase/'progress.json',dict(cases=len({x['case'] for x in output}),fields=len(output),failures=failures,cpu_seconds=time.process_time()-start))
|
|
save(phase/'manifest.json',dict(entries=output,failures=failures,kind=kind,cpu_seconds=time.process_time()-start,contract_sha256=sha(root/'contract.json')))
|
|
|
|
|
|
def tie_fraction(a):
|
|
_,counts=np.unique(a,return_counts=True);n=len(a)
|
|
return float(np.sum(counts*(counts-1))/(n*(n-1))) if n>1 else None
|
|
|
|
|
|
def score(root,c,kind):
|
|
authorization=read(root/(kind+'-score-authorization.json'))
|
|
checked(root/(kind+'-git-freeze.json'),authorization['freeze_sha256'])
|
|
verify_freeze(root/(kind+'-git-freeze.json'),check_git=False)
|
|
built=read(root/'phases'/(kind+'-build')/'manifest.json');phase=root/'phases'/(kind+'-score');out=phase/'parts';out.mkdir(exist_ok=False)
|
|
rows=[];failures=list(built['failures']);start=time.process_time();by={r['case']:r for r in c['parts']}
|
|
for e in built['entries']:
|
|
case=e['case'];row=by[case];dest=out/(case+'-'+e['arm']);dest.mkdir()
|
|
try:
|
|
x,raw,mass,rg,adjacency=geometry(row,c)
|
|
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z:
|
|
if not np.array_equal(raw,z['raw_ids']):raise ValueError('B11 field IDs changed')
|
|
fields={k:z[k].copy() for k in ['raw','clean']}
|
|
for p,h in [(e['bars_path'],e['bars_sha256'])]:checked(p,h)
|
|
refpath=B3/'parts'/case/'truth-events.json';ref=read(checked(refpath,c['source_sha256'][str(refpath)]))
|
|
cache=row['v5_cache'];yp=Path(cache['output'])/'seconds.npy'
|
|
save(dest/'target-access.json',dict(role='B3_selected_learning_panel_only',path=str(yp),sha256=cache['output_sha256']['seconds.npy']))
|
|
truth=np.load(checked(yp,cache['output_sha256']['seconds.npy']),allow_pickle=False)
|
|
q=prediction_inputs(row);ix=np.searchsorted(raw,q['raw_ids'])
|
|
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):raise ValueError('R4bc query IDs differ')
|
|
result=dict(case=case,component_id=row['component_id'],arm=e['arm'],changed_volume_fraction=e['changed_volume_fraction'],query_ids=list(map(int,q['raw_ids'])),query_counts=list(map(int,q['counts'])))
|
|
for label,field in fields.items():
|
|
if e['arm']=='EIK' and label=='raw':
|
|
p=B3/'parts'/case/'eik-events.json';old=read(checked(p,c['source_sha256'][str(p)]));ev=old['events'];matching=old['match']
|
|
else:ev=events(field,mass,raw,x,adjacency);matching=match(ref,ev,adjacency,rg)
|
|
save(dest/(label+'-events.json'),dict(events=ev,match=matching));result[label]=matching
|
|
result[label+'_metrics']=part_metrics(field[ix],truth[ix],q['counts'],q['near_wall'],q['raw_ids'],q['coordinates_over_Rg'],int(case.split('_')[0]))
|
|
result[label+'_query_tie_fraction']=tie_fraction(field[ix])
|
|
qtop=q['raw_ids'][field[ix]==field[ix].max()];result[label+'_query_last_raw_id']=int(qtop.min());result[label+'_query_max_plateau_ids']=list(map(int,qtop))
|
|
save(dest/'result.json',result);rows.append(result)
|
|
except Exception as exc:
|
|
failure=dict(case=case,arm=e['arm'],type=type(exc).__name__,message=str(exc));save(dest/'failure.json',failure);failures.append(failure)
|
|
if disk_usage(root)>=c['disk_bytes']:raise RuntimeError('Whole B11 disk cap')
|
|
save(phase/'progress.json',dict(scored=len(rows),failures=failures,cpu_seconds=time.process_time()-start))
|
|
summaries={}
|
|
for arm in (['EIK'] if kind=='eik' else ['A','H1']):
|
|
selected=[r for r in rows if r['arm']==arm];rawtot=totals(selected,'raw');cleantot=totals(selected,'clean');paired={};undefined={}
|
|
for metric in ['last_location','kendall_tau_b','relative_l2']:
|
|
good=[(r['raw_metrics'][metric]['value'],r['clean_metrics'][metric]['value']) for r in selected if r['raw_metrics'][metric]['value'] is not None and r['clean_metrics'][metric]['value'] is not None]
|
|
paired[metric]=good;undefined[metric]=[r['case'] for r in selected if r['raw_metrics'][metric]['value'] is None or r['clean_metrics'][metric]['value'] is None]
|
|
summaries[arm]=dict(raw=rawtot,clean=cleantot,parts=len(selected),undefined=undefined,paired_denominators={k:len(v) for k,v in paired.items()},
|
|
field_means={k:dict(raw=float(np.mean([a for a,b in v])) if v else None,clean=float(np.mean([b for a,b in v])) if v else None) for k,v in paired.items()},
|
|
tree_gate=gate_decision(rawtot,cleantot,{k:paired[k] for k in ['last_location','kendall_tau_b']},len(selected),failures))
|
|
save(phase/'result.json',dict(status='COMPLETE',kind=kind,summaries=summaries,rows=rows,failures=failures,cpu_seconds=time.process_time()-start,
|
|
interpretation='learning-selected panel posthoc gate; no independent blind evaluation; no weld or remesh claim'))
|
|
|
|
|
|
def run(root):
|
|
c=config(root);os.sched_setaffinity(0,[c['cpu']]);start=time.process_time();charges=0.;records=[];completed=[]
|
|
usage0=resource.getrusage(resource.RUSAGE_CHILDREN);child_start=usage0.ru_utime+usage0.ru_stime
|
|
with (root/'run.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if (root/'actual-start.json').exists():raise FileExistsError('No B11 retry')
|
|
save(root/'actual-start.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json')))
|
|
def total():
|
|
usage=resource.getrusage(resource.RUSAGE_CHILDREN)
|
|
return c['setup_charge_seconds']+max(charges,usage.ru_utime+usage.ru_stime-child_start)+time.process_time()-start
|
|
def phase(kind,mode):
|
|
nonlocal charges
|
|
name=kind+'-'+mode;dest=root/'phases'/name;dest.mkdir(parents=True,exist_ok=False)
|
|
cap=min(c['phase_cpu_max'],c['total_cpu_seconds']-total()-120)
|
|
if cap<=0:raise RuntimeError('Cumulative B11 CPU exhausted')
|
|
save(root/'live-progress.json',dict(stage=name,completed=completed,cumulative_cpu_seconds=total(),unix=time.time()))
|
|
g=supervise([sys.executable,'tools/phase2_b11_gate.py',mode,str(root),kind],dest,
|
|
dict(cpu_seconds=cap,wall_seconds=c['phase_wall_seconds'],rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes']))
|
|
charges+=g['cpu_seconds'];records.append(dict(phase=name,guard=g));save(root/'phase-ledger.json',dict(records=records,cumulative_cpu_seconds=total()))
|
|
if g['status']!='PASS' or total()>c['total_cpu_seconds'] or disk_usage(root)>c['disk_bytes']:raise RuntimeError('B11 phase/resource stop: '+name)
|
|
try:
|
|
for kind in ['eik','models']:
|
|
if kind=='models':
|
|
begin=time.monotonic()
|
|
while not (H1/'scoring-git-freeze.json').exists():
|
|
done=read(H1/'complete.json') if (H1/'complete.json').exists() else None
|
|
if done or time.monotonic()-begin>=c['model_wait_wall_seconds']:raise RuntimeError('H1 full-node freeze unavailable; EIK retained')
|
|
save(root/'live-progress.json',dict(stage='WAIT_ORIGINAL_H1_PREDICTION_FREEZE',completed=completed,cumulative_cpu_seconds=total(),unix=time.time()))
|
|
if total()>=c['total_cpu_seconds']-120:raise RuntimeError('B11 waiting CPU cap')
|
|
time.sleep(15)
|
|
model_entries(check_git=True)
|
|
save(root/'models-source-authorization.json',dict(freeze_sha256=sha(H1/'scoring-git-freeze.json'),manifest_sha256=sha(H1/'predictions-freeze.json')))
|
|
phase(kind,'build')
|
|
manifest=root/'phases'/(kind+'-build')/'manifest.json'
|
|
freeze(root,kind,[root/'contract.json',manifest,*[BASE/p for p in c['code_sha256']]])
|
|
verify_freeze(root/(kind+'-git-freeze.json'))
|
|
save(root/(kind+'-score-authorization.json'),dict(freeze_sha256=sha(root/(kind+'-git-freeze.json'))))
|
|
phase(kind,'score');completed.append(kind)
|
|
save(root/(kind+'-complete.json'),dict(status='COMPLETE',result_sha256=sha(root/'phases'/(kind+'-score')/'result.json'),cumulative_cpu_seconds=total()))
|
|
save(root/'complete.json',dict(status='COMPLETE',completed=completed,records=records,cumulative_cpu_seconds=total(),retry=False))
|
|
except BaseException as exc:
|
|
save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),completed=completed,records=records,cumulative_cpu_seconds=total(),retry=False));raise
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('mode',choices=['prepare','run','build','score']);p.add_argument('path',type=Path);p.add_argument('kind',nargs='?');a=p.parse_args()
|
|
if a.mode=='prepare':prepare(a.path)
|
|
elif a.mode=='run':run(a.path)
|
|
else:(build if a.mode=='build' else score)(a.path,config(a.path),a.kind)
|