Files
MeltMind/tools/phase2_b1213_gate.py
T

226 lines
17 KiB
Python

"""Approved one-fold B12/B13 gates: prepare, freeze, fit, predict, freeze, score."""
import argparse
from collections import Counter
import fcntl
import json
import os
from pathlib import Path
import resource
import sys
import time
import traceback
import joblib
import numpy as np
import xgboost as xgb
from phase2_fixed_io import BASE,freeze,verify
from phase2_h1_worker_v2 import checked,sha,save
from phase2_b1213_features import b12,b13,cached,GeometryFailure,InputViolation
from gre24_models import xgb_model
from r4bc_guard import supervise
from r4bc_common import paired_difference
from s2_custody_common import prediction_inputs
from s2_custody_statistics import last_location
from s2_models.objectives import relative_l2
from s2_recovery_monitoring import disk_usage
def read(p):return json.loads(Path(p).read_text())
def decision(field,last,undefined=False):
if undefined or field['high'] is None or last['high'] is None:return 'PARK_UNDEFINED'
return 'EXPAND_THIS_FIXED_RECIPE' if field['high']<0 or (last['high']<0 and field['high']<.001) else 'PRUNE_THIS_FIXED_RECIPE'
def coverage(rows,results):
groups={}
for r in rows:groups.setdefault(r['component_id'],[]).append(r)
good=[g for g,parts in groups.items() if all(results[r['case']]['status']=='GEOMETRY_COMPLETE' for r in parts)]
return dict(components=len(groups),complete_components=len(good),fraction=len(good)/len(groups),complete_component_ids=good,
valid_parts=sum(results[r['case']]['status']=='GEOMETRY_COMPLETE' for r in rows),total_parts=len(rows))
def prepare(kind,acceptance):
a=read(acceptance)
if a['status']!='PASS':raise ValueError('Acceptance missing')
for p,h in a['code_sha256'].items():checked(BASE/p,h)
oldpath=BASE/'artifacts/phase2-b7-gate-2026-11-17/contract.json';old=read(oldpath)
o1path=BASE/'artifacts/o1-v2-preparation-2026-11-17/contract.json';o1=read(o1path)
root=BASE/f'artifacts/phase2-{kind.lower()}-gate-2026-11-17';root.mkdir(exist_ok=False)
parts=sorted(old['rows']['learning']+old['rows']['validation'],key=lambda r:r['case'])
if len({r['case'] for r in parts})!=431:raise ValueError('Frozen full431 roles differ')
cache={}
if kind=='B13':
for row in parts:
d=BASE/'artifacts/o1-v2-preparation-2026-11-17/parts'/row['case']/'output';rp=d/'result.json'
if rp.exists():
rec=read(rp)
if rec.get('arms',{}).get('G'):
cache[row['case']]=dict(result=str(rp),result_sha256=sha(rp),graph=str(d/'graph.npz'),partition=str(d/'G-partition.npz'))
names=['phase2_b1213_gate','phase2_b1213_features','phase2_b12_math','phase2_fixed_io','phase2_h1_worker_v2','phase2_residual_math','gre24_models','gre24_features_v2','gre24_features','boundary_distance','o1_prepare','o1_partition','o1_v2','audit_samples','r4bc_common','r4bc_guard','s2_custody_common','s2_custody_statistics','s2_recovery_supervisor','s2_recovery_monitoring','freeze_g1_components','s2_models/features','s2_models/objectives']
spec=BASE/('references/2026-11-17/phase2-design/mold-b12-spec-9ac2ed43.md' if kind=='B12' else 'references/2026-11-17/phase2-design/mold-b13-spec-b0af97e6.md')
expected='9ac2ed43ec494b597156fe1f0d2d853ae3e80659fbb309a354c4a707c60d7d94' if kind=='B12' else 'b0af97e6cf854b567c9d72d8a60b0f29f0d7af3ebddcb1ddb80fbf1c05598d5f';checked(spec,expected)
save(root/'contract.json',dict(version=kind+'-one-fold-v1',kind=kind,approval='1794896825.024679',parts=parts,rows=old['rows'],baseline=old['baseline'],G_cache=cache,
archive=o1['archive'],archive_stat=o1['archive_stat'],code_sha256={f'tools/{n}.py':sha(BASE/f'tools/{n}.py') for n in names},
source_sha256={str(oldpath):sha(oldpath),str(o1path):sha(o1path),str(spec):sha(spec),str(acceptance):sha(acceptance)},
extra_columns=3 if kind=='B12' else 2,cpu=24,total_cpu_seconds=7200,setup_seconds=60,geometry_cpu_cap=3600,part_cpu_seconds=30,part_wall_seconds=60,
fit_cpu_cap=3000,predict_cpu_cap=180,score_cpu_cap=180,rss_bytes=8*1024**3,disk_bytes=5*1024**3,cost_checkpoint=10,cost_projection_factor=1.25,
training='same frozen A learning320/all-tet/unit-node/FP32; directT/t_set; XGB100/depth8/eta.1/seed0/1thread/noHPO/noextraArefit',
gate='new-minus-A component-paired2000 bootstrap95 seed2026111408; EXPAND if fieldCI.high<0 OR (lastCI.high<0 AND fieldCI.high<.001); otherwise PRUNE fixed recipe',
missing='native NaN only for defined geometrical missing; engineer/source/resource failures stop, never fabricated completed missing',
B13_coverage='learning and validation each >=80pct components with all own-role members legalG; otherwise PARK coverage before fitting',
sources='frozen Df/Dw/geometry only; no recomputeDf; B13 reuse identical Gseed0 else one G call, noT/C',
new_models=1,retry=False,gpu=False,new_label_decodes=0))
def config(root):
c=read(root/'contract.json')
if xgb.__version__!='2.1.4' or np.__version__!='1.26.4':raise InputViolation('Pinned library version differs')
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():checked(p,h)
s=Path(c['archive']).stat();old=c['archive_stat']
if (s.st_size,s.st_mtime_ns,s.st_ino)!=(old['bytes'],old['mtime_ns'],old['inode']):raise InputViolation('Archive identity changed')
for role,rows in c['rows'].items():
for r in rows:
a=next(a for a in r['assignments'] if a['fold']==0)
if a['role']!=role or (role=='learning' and '100' not in a['learning_pools']):raise InputViolation('Role mismatch')
return c
def field(root,c,index):
row=c['parts'][index];dest=root/'parts'/f'{index:04d}';start=time.process_time()
try:
try:extra,report=(b12 if c['kind']=='B12' else b13)(row,c,dest)
except GeometryFailure as exc:
extra=np.full((row['nodes'],c['extra_columns']),np.nan);report=dict(status='SCIENTIFIC_MISSING',reason=str(exc),labels_read=False)
if extra.shape!=(row['nodes'],c['extra_columns']) or np.isinf(extra).any():raise InputViolation('Invalid extra feature shape/infinity')
np.save(dest/'extra.npy',extra,allow_pickle=False)
save(dest/'result.json',dict(case=row['case'],component_id=row['component_id'],index=index,nodes=row['nodes'],extra_sha256=sha(dest/'extra.npy'),
finite_nodes_per_column=np.isfinite(extra).sum(0).tolist(),**{k:v for k,v in report.items() if k!='finite_nodes_per_column'},cpu_seconds=time.process_time()-start))
except BaseException as exc:
save(dest/'failure.json',dict(type=type(exc).__name__,reason=str(exc),global_stop=True,traceback=traceback.format_exc(),cpu_seconds=time.process_time()-start));raise
def feature_manifest(root,c):
m=read(root/'features-manifest.json');by={e['case']:e for e in m['entries']}
if set(by)!={r['case'] for r in c['parts']}:raise InputViolation('Feature roster incomplete')
return by
def extra(root,row,entry):
d=root/'parts'/f"{entry['index']:04d}";r=read(checked(d/'result.json',entry['result_sha256']))
return np.load(checked(d/'extra.npy',r['extra_sha256']),allow_pickle=False),r
def train(root,c):
verify(root,'features');dest=root/'phases/train';entries=feature_manifest(root,c);n=sum(r['nodes'] for r in c['rows']['learning'])
X=np.empty((n,19+c['extra_columns']),np.float32);Y=np.empty(n,np.float32);offset=0;access=[]
for row in c['rows']['learning']:
f=cached(row,'features');e,report=extra(root,row,entries[row['case']]);raw=cached(row,'raw_ids')
if f.shape!=(row['nodes'],19) or not np.isfinite(f).all() or len(raw)!=row['nodes']:raise InputViolation('Learning feature identity')
access.append(dict(case=row['case'],role='fold0_learning',target_sha256=row['v5_cache']['output_sha256']['target.npy'],nodes=row['nodes']));save(dest/'target-access.json',access)
y=cached(row,'target');end=offset+len(y)
if y.shape!=(row['nodes'],) or not np.isfinite(y).all():raise InputViolation('Learning target shape/nonfinite')
X[offset:end,:19]=f;X[offset:end,19:]=e;Y[offset:end]=y;offset=end
save(dest/'progress.json',dict(stage='LOAD_LEARNING',parts=len(access),nodes=offset))
if offset!=n:raise InputViolation('Learning count changed')
model=xgb_model(0,1);save(dest/'progress.json',dict(stage='FIT',nodes=n,parts=len(access),columns=X.shape[1]))
model.fit(X,Y,sample_weight=np.ones(n,np.float64));model.save_model(dest/'model.ubj')
save(dest/'xgboost-config.json',json.loads(model.get_booster().save_config()))
save(dest/'result.json',dict(status='FIT_COMPLETE',checkpoint_sha256=sha(dest/'model.ubj'),nodes=n,parts=len(access),columns=X.shape[1],unit_node_weights=True,validation_labels_read=False))
def predict(root,c):
verify(root,'model');dest=root/'phases/predict';entries=feature_manifest(root,c);model=xgb.XGBRegressor();model.load_model(root/'phases/train/model.ubj')
baseline=joblib.load(checked(c['baseline']['model_path'],c['baseline']['model_sha256']));out=[]
for row in c['rows']['validation']:
q=prediction_inputs(row);raw=cached(row,'raw_ids');ix=np.searchsorted(raw,q['raw_ids'])
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):raise InputViolation('Validation query IDs')
f=cached(row,'features');e,report=extra(root,row,entries[row['case']]);X=np.column_stack([f[ix],e[ix]]).astype(np.float32)
pred=np.asarray(model.predict(X),np.float64);a=np.asarray(baseline.predict(f[ix].astype(np.float32)),np.float64)
if not np.isfinite(pred).all() or not np.isfinite(a).all():raise ValueError('Nonfinite prediction')
p=dest/(row['case']+'.npz');np.savez_compressed(p,pred=pred,A=a,raw_ids=q['raw_ids'],counts=q['counts'],coordinates_over_Rg=q['coordinates_over_Rg'])
out.append(dict(case=row['case'],component_id=row['component_id'],path=str(p),sha256=sha(p),geometry_valid=report['status']=='GEOMETRY_COMPLETE'))
save(dest/'progress.json',dict(parts=len(out),validation_labels_read=False))
save(dest/'manifest.json',dict(entries=out,validation_labels_read=False))
def summary(rows):
groups={}
for r in rows:groups.setdefault(r['component_id'],[]).append(r)
result={}
for metric in ['relative_l2','last_location']:
values={arm:{g:float(np.mean([r[arm][metric] for r in rs])) for g,rs in groups.items() if all(r[arm][metric] is not None for r in rs)} for arm in ['new','A']}
result[metric]=dict(means={a:float(np.mean(list(v.values()))) if v else None for a,v in values.items()},new_minus_A=paired_difference(values['new'],values['A']))
return dict(parts=len(rows),components=len(groups),metrics=result)
def score(root,c):
verify(root,'predictions');dest=root/'phases/score';m=read(root/'phases/predict/manifest.json');by={r['case']:r for r in c['rows']['validation']};rows=[];access=[];undefined=[]
if len(m['entries'])!=len(by) or {e['case'] for e in m['entries']}!=set(by):raise InputViolation('Prediction roster incomplete')
for e in m['entries']:
row=by[e['case']];access.append(dict(case=row['case'],role='fold0_validation',target_sha256=row['v5_cache']['output_sha256']['target.npy']));save(dest/'target-access.json',access)
y=cached(row,'target');raw=cached(row,'raw_ids')
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z:
ix=np.searchsorted(raw,z['raw_ids'])
if not np.array_equal(raw[ix],z['raw_ids']):raise InputViolation('Score query ID mismatch')
out=dict(case=row['case'],component_id=row['component_id'],geometry_valid=e['geometry_valid'])
for arm,key in [('new','pred'),('A','A')]:
l2=relative_l2(z[key],y[ix],z['counts'],np.ones(len(ix),bool))
out[arm]=dict(relative_l2=l2['value'],last_location=last_location(y[ix],z[key],z['raw_ids'],z['coordinates_over_Rg']))
if l2['value'] is None:undefined.append(dict(case=row['case'],arm=arm,reason=l2['reason']))
rows.append(out)
full=summary(rows);valid=summary([r for r in rows if r['geometry_valid']])
result=decision(full['metrics']['relative_l2']['new_minus_A'],full['metrics']['last_location']['new_minus_A'],bool(undefined))
save(dest/'result.json',dict(status='COMPLETE',kind=c['kind'],decision=result,full_roster=full,geometry_valid_subset_descriptive=valid,
invalid_parts=[r['case'] for r in rows if not r['geometry_valid']],undefined=undefined,rows=rows,validation_only=True,
interpretation='fixed one-fold recipe; includes missingness effect; subset never replaces full gate; B13 failure does not close type/O1/connectivity'))
def run(root):
c=config(root);os.sched_setaffinity(0,[24]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('Single launch; zero retries')
start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);charged=0.;records=[];entries=[];results={}
def total():
u=resource.getrusage(resource.RUSAGE_CHILDREN);return 60+max(charged,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)+time.process_time()-start
def guard(mode,dest,cap,wall,index=None):
nonlocal charged
remaining=c['disk_bytes']-disk_usage(root)
cap=min(cap,7200-total()-120)
if cap<=0 or remaining<=0:raise RuntimeError('Resource reserve exhausted')
command=[sys.executable,'tools/phase2_b1213_gate.py',mode,str(root)]+([] if index is None else [str(index)])
g=supervise(command,dest,dict(cpu_seconds=cap,wall_seconds=wall,rss_bytes=c['rss_bytes'],disk_bytes=remaining))
charged+=g['cpu_seconds'];records.append(dict(mode=mode,index=index,guard=g));save(root/'phase-ledger.json',dict(records=records,cumulative_cpu_seconds=total()))
if g['status']!='PASS' or total()>7200 or disk_usage(root)>c['disk_bytes']:raise RuntimeError('ENGINEERING_OR_RESOURCE_STOP')
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json')))
try:
for i,row in enumerate(c['parts']):
if total()-60+30>c['geometry_cpu_cap']:raise RuntimeError('PARK_RESOURCE_GEOMETRY_CAP')
dest=root/'parts'/f'{i:04d}';dest.mkdir(parents=True,exist_ok=False)
save(root/'live-progress.json',dict(stage='GEOMETRY_ONLY',index=i,case=row['case'],cumulative_cpu_seconds=total(),unix=time.time()))
guard('field',dest,c['part_cpu_seconds'],c['part_wall_seconds'],i)
result=read(dest/'result.json');results[row['case']]=result;entries.append(dict(case=row['case'],index=i,result_sha256=sha(dest/'result.json')))
if i+1==c['cost_checkpoint']:
projected=1.25*(total()-60)*sum(r['nodes'] for r in c['parts'])/sum(r['nodes'] for r in c['parts'][:i+1])
save(root/'first-ten-cost-check.json',dict(projected_geometry_cpu_seconds=projected,cap=c['geometry_cpu_cap'],passed=projected<=c['geometry_cpu_cap'],actual_cpu_seconds=total()))
if projected>c['geometry_cpu_cap']:raise RuntimeError('PARK_RESOURCE_FIRST_TEN_PROJECTION')
cov={role:coverage(rows,results) for role,rows in c['rows'].items()};save(root/'coverage.json',cov)
save(root/'features-manifest.json',dict(entries=entries,all431_terminal=True,labels_read=False))
freeze(root,'features',[root/'contract.json',root/'features-manifest.json',root/'coverage.json',*[BASE/p for p in c['code_sha256']]])
if c['kind']=='B13' and any(v['fraction']<.8 for v in cov.values()):
save(root/'complete.json',dict(status='COMPLETE',scientific_gate='PARK_G_COVERAGE',coverage=cov,cumulative_cpu_seconds=total(),records=records,models_fit=0,retry=False));return
for mode,cap in [('train',c['fit_cpu_cap']),('predict',c['predict_cpu_cap']),('score',c['score_cpu_cap'])]:
if mode=='predict':freeze(root,'model',[root/'contract.json',root/'phases/train/model.ubj',root/'phases/train/result.json',*[BASE/p for p in c['code_sha256']]])
if mode=='score':freeze(root,'predictions',[root/'contract.json',root/'phases/predict/manifest.json',*[BASE/p for p in c['code_sha256']]])
dest=root/'phases'/mode;dest.mkdir(parents=True,exist_ok=False);save(root/'live-progress.json',dict(stage=mode,cumulative_cpu_seconds=total(),unix=time.time()))
guard(mode,dest,cap,cap+60)
save(root/'complete.json',dict(status='COMPLETE',scientific_gate=read(root/'phases/score/result.json')['decision'],cumulative_cpu_seconds=total(),records=records,models_fit=1,retry=False))
except BaseException as exc:
save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),cumulative_cpu_seconds=total(),records=records,completed_geometry_parts=len(entries),retry=False));raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('mode',choices=['prepare','run','field','train','predict','score']);p.add_argument('path',type=Path);p.add_argument('index',nargs='?');a=p.parse_args()
if a.mode=='prepare':prepare(a.index,a.path)
elif a.mode=='run':run(a.path)
elif a.mode=='field':field(a.path,config(a.path),int(a.index))
else:dict(train=train,predict=predict,score=score)[a.mode](a.path,config(a.path))