docs: freeze B1 residual contract and exact learning clock weights for review

This commit is contained in:
Frank14f committed 2026-11-17 12:07:25 +08:00
1 parent 7cb1f1c146
commit e751e5b261
10 files changed
+39097 -12

No files matched your search

+139
View File
@@ -0,0 +1,139 @@
"""Freeze H1 clock membership/weights from metadata only; never load arrays."""
import csv
from fractions import Fraction
import hashlib
import json
from pathlib import Path
import time
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / 'artifacts/phase2-h1-planning-2026-11-17/clock-freeze-v1'
def main():
start = time.process_time()
sources = {}
def read(relative):
p = ROOT / relative
raw = p.read_bytes()
sources[str(relative)] = hashlib.sha256(raw).hexdigest()
return json.loads(raw)
grouped = read('artifacts/r4bc-execution-2026-11-16/contract.json')
oldmodels = read('artifacts/r4bc-execution-2026-11-16/models-terminal.json')
vr = Path('artifacts/gre24-v4-reproduction-2026-11-16/protocol')
splits = read(vr/'splits.json')
prep = read(vr/'preparation-freeze.json')
training = read(vr/'training-contract.json')
assert training['xgb_threads'] == 8
assert len(grouped['parts']) == 552
assert len({r['component_id'] for r in grouped['parts']}) == 204
assert len(prep['successful']) == 615
parts = {'grouped': {}, 'v4_random': {}}
cells = []; weights = []
def register(protocol, case, cache, group):
hashes = cache['output_sha256']
required = ('seconds.npy', 'target.npy', 'raw_ids.npy', 'Vf.npy', 'features.npy', 'mass.npy', 'Df.npy')
assert all(name in hashes for name in required)
parts[protocol][case] = dict(case=case, output=cache['output'], nodes=cache['nodes'],
t_set=cache['t_scale_seconds'], clock_group=group, output_sha256=hashes,
endpoint='max(seconds.npy on retained-tet raw_ids) / t_set')
for r in grouped['parts']:
assert r['nodes'] == r['v5_cache']['nodes']
assert r['preparation']['t_set'] == r['v5_cache']['t_scale_seconds']
register('grouped', r['case'], r['v5_cache'], r['component_id'])
for case, cache in prep['successful'].items():
# The frozen original random protocol has no component membership.
assert not any(k in cache for k in ('component_id', 'component_key', 'clock_group'))
register('v4_random', case, cache, 'part:'+case)
def cell(protocol, fold, pool, learning, test, seed, threads, baseline, original_access):
assert learning and len(learning) == len(set(learning))
assert len(test) == len(set(test)) and not set(learning).intersection(test)
assert [r['case'] for r in original_access] == learning
groups = {}
for case in learning:
group = parts[protocol][case]['clock_group']
groups.setdefault(group, []).append(case)
G = len(groups); total = Fraction(0)
key = f'{protocol}-fold{fold}-pool{pool}'
for group in sorted(groups):
n = len(groups[group]); weight = Fraction(1, G*n)
for case in groups[group]:
total += weight
weights.append(dict(cell=key, protocol=protocol, fold=fold, pool=pool,
case=case, clock_group=group, group_count=G, parts_in_group=n,
numerator=weight.numerator, denominator=weight.denominator,
weight_float=float(weight), original_role='learning' if protocol=='grouped' else 'train'))
assert total == 1
cells.append(dict(index=len(cells), cell=key, protocol=protocol, fold=fold, pool=pool,
seed=seed, threads=threads, learning=learning, test=test,
learning_parts=len(learning), test_parts=len(test), clock_groups=G,
learning_nodes=sum(parts[protocol][case]['nodes'] for case in learning),
baseline_model=baseline, clock_value=None, clock_status='NOT_EVALUATED_METADATA_ONLY',
exact_weight_sum='1/1', original_fit_access_order_match=True))
for pool in (100, 50, 25):
for fold in range(5):
roles = {r['case']: next(a for a in r['assignments'] if a['fold']==fold) for r in grouped['parts']}
learning = sorted(case for case,a in roles.items() if a['role']=='learning' and str(pool) in a['learning_pools'])
test = sorted(case for case,a in roles.items() if a['role']=='test')
# Clock groups cannot leak across held-out roles in this protocol.
seen = {}
for case,a in roles.items():
group = parts['grouped'][case]['clock_group']
seen.setdefault(group,set()).add(a['role'])
assert all(len(v)==1 for v in seen.values())
model = next(m for m in oldmodels['models'] if m['arm']=='A' and m['fold']==fold and m['pool']==pool)
assert model['status']=='PASS'
oldout = Path(model['model_path']).parent.relative_to(ROOT)
access = read(oldout/'access.json'); result = read(oldout/'result.json')
assert all(r['role']=='learning' and r['full'] for r in access)
assert result['learning_parts']==len(learning)
assert result['learning_nodes']==sum(parts['grouped'][case]['nodes'] for case in learning)
cell('grouped',fold,pool,learning,test,0,1,
dict(path=model['model_path'],sha256=model['model_sha256'],format='joblib'),access)
for split in splits:
fold=split['index']; learning=[x for x in split['train'] if x in prep['successful']]
test=[x for x in split['test'] if x in prep['successful']]
result=read(vr/f'fits/split-{fold}/GBRT/result.json')
access=read(vr/f'fits/split-{fold}/GBRT/training-target-access.json')
assert all(r['role']=='train' for r in access)
assert result['train_geometries']==len(learning)
assert result['train_nodes']==sum(parts['v4_random'][case]['nodes'] for case in learning)
assert set(split['fnn_validation']).issubset(split['train'])
assert result['parameters']['n_jobs']==8
cell('v4_random',fold,100,learning,test,split['seed'],8,
dict(path=result['checkpoint'],sha256=result['checkpoint_sha256'],format='ubj'),access)
cells[-1]['included_original_fnn_validation']=sorted(set(split['fnn_validation']) & set(learning))
cells[-1]['excluded_original_train_preparation_failures']=[x for x in split['train'] if x not in prep['successful']]
# No overwrite of an existing fixed review packet.
OUT.mkdir(parents=True, exist_ok=False)
fields=list(weights[0])
with (OUT/'clock-weights.csv').open('w',newline='') as stream:
writer=csv.DictWriter(stream,fieldnames=fields);writer.writeheader();writer.writerows(weights)
payload=dict(version='phase2-h1-clock-membership-v1', status='FROZEN_FOR_MAIN_REVIEW_NOT_EXECUTION_APPROVAL',
c_policy=dict(grouped='original component_id; 1/(learning component count * learning parts in component)',
v4_random='no component key in original protocol; part:<original case>, equal successful original GBRT train parts'),
cells=cells, parts=parts, source_sha256=sources,
target_arrays_read=False, feature_arrays_read=False, clock_values_computed=False,
fit_started=False, new_decodes=0)
(OUT/'membership-and-sources.json').write_text(json.dumps(payload,indent=2)+'\n')
receipt=dict(status='PASS_METADATA_INVARIANTS', cells=len(cells), weight_rows=len(weights),
grouped_parts=552, grouped_components=204, v4_successful_parts=615,
exact_weight_sums=True, original_access_order_match=True, learning_test_disjoint=True,
grouped_component_role_isolation=True, target_arrays_read=False, feature_arrays_read=False,
cpu_seconds=time.process_time()-start,
files={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in OUT.iterdir()})
(OUT/'receipt.json').write_text(json.dumps(receipt,indent=2)+'\n')
print(json.dumps(receipt))
if __name__=='__main__':
main()
+168
View File
@@ -0,0 +1,168 @@
"""H1-RES fixed-XGB worker. No queue or implicit execution approval.
The reviewed contract binds training/test roles, source metadata, precision,
clock weighting and source hashes. Test predictions never load target arrays.
"""
import argparse
import hashlib
import json
import os
from pathlib import Path
import subprocess
import time
import traceback
import numpy as np
from threadpoolctl import threadpool_limits
from gre24_models import xgb_model
from phase2_residual_math import learning_clock, residual_target, decode_seconds
def sha(path):
h = hashlib.sha256()
with Path(path).open('rb') as source:
for block in iter(lambda: source.read(4*1024**2), b''): h.update(block)
return h.hexdigest()
def save(path, value):
path = Path(path); tmp = path.with_suffix(path.suffix+'.tmp')
tmp.write_text(json.dumps(value, indent=2, allow_nan=False)+'\n'); tmp.replace(path)
def checked(path, digest):
if sha(path) != digest: raise ValueError('Pinned file mismatch: '+str(path))
return Path(path)
def contract(root):
import xgboost
if np.__version__ != '1.26.4' or xgboost.__version__ != '2.1.4':
raise ValueError('Frozen library version differs')
c = json.loads((root/'contract.json').read_text())
if c.get('version') != 'phase2-h1-res-v1' or not c.get('main_review'):
raise PermissionError('No fixed reviewed H1 contract')
for p, h in c['code_sha256'].items(): checked(p, h)
for p, h in c['source_sha256'].items(): checked(p, h)
return c
def role_rows(c, job, role):
if role not in ('learning', 'test'): raise ValueError('Unknown role')
if set(job['learning']) & set(job['test']): raise PermissionError('Training/test overlap')
if len(job[role]) != len(set(job[role])): raise ValueError('Repeated case')
# Contract builder must cross-check these frozen roles against original
# source contracts. Here a role cannot be expanded through row metadata.
return [c['parts'][job['protocol']][case] for case in job[role]]
def load_features(row):
source = Path(row['output']); hashes = row['output_sha256']
f = np.load(checked(source/'Vf.npy', hashes['Vf.npy']), allow_pickle=False)
x = np.load(checked(source/'features.npy', hashes['features.npy']), mmap_mode='r', allow_pickle=False)
ids = np.load(checked(source/'raw_ids.npy', hashes['raw_ids.npy']), allow_pickle=False)
if x.shape != (row['nodes'], 19) or f.shape != (row['nodes'],) or ids.shape != f.shape:
raise ValueError('Feature/node identity shape mismatch')
if f.dtype != np.dtype('float64') or not np.array_equal(x[:, 0], f):
raise ValueError('Original float64 F differs from frozen first column')
if np.any(np.diff(ids) <= 0) or not np.isfinite(x).all(): raise ValueError('Bad IDs/features')
return np.asarray(x, dtype=np.float32), f, ids
def fit(root, c, job):
out = root/'fits'/f"job-{job['index']:02d}"; out.mkdir(parents=True, exist_ok=False)
save(out/'started.json', {'unix': time.time(), 'job': job, 'contract_sha256': sha(root/'contract.json')})
cpu = time.process_time(); access = []; rows = role_rows(c, job, 'learning')
end_ratios, groups = [], []
# First pass: only learning-role retained-tet targets for this cell's clock.
for row in rows:
path = Path(row['output'])/'target.npy'
access.append({'case': row['case'], 'role': 'learning', 'purpose': 'clock', 'path': str(path)})
save(out/'target-access.json', access)
y = np.load(checked(path, row['output_sha256']['target.npy']), mmap_mode='r', allow_pickle=False)
if y.shape != (row['nodes'],) or not np.isfinite(y).all() or np.any(y < 0): raise ValueError('Invalid learning target')
end_ratios.append(float(y.max())); groups.append(row['clock_group'])
clock = learning_clock(end_ratios, groups)
save(out/'clock.json', {'c': clock, 'source': 'learning retained-tet max(T/t_set), component-equal then part-equal',
'parts': [row['case'] for row in rows], 'end_ratios': end_ratios, 'groups': groups})
n = sum(row['nodes'] for row in rows); x = np.empty((n, 19), np.float32); y = np.empty(n, np.float32); cursor = 0
for row in rows:
xx, f, ids = load_features(row)
path = checked(Path(row['output'])/'target.npy', row['output_sha256']['target.npy'])
target = np.load(path, mmap_mode='r', allow_pickle=False)
count = len(f); x[cursor:cursor+count] = xx
y[cursor:cursor+count] = residual_target(target, f, clock, fit_dtype='float32'); cursor += count
access.append({'case': row['case'], 'role': 'learning', 'purpose': 'residual_fit', 'path': str(path)})
save(out/'target-access.json', access)
with threadpool_limits(job['threads']):
model = xgb_model(job['seed'], job['threads'])
save(out/'progress.json', {'stage': 'FIT', 'nodes': n, 'clock': clock})
model.fit(x, y, verbose=False)
checkpoint = out/'model.ubj'; model.save_model(checkpoint)
(out/'xgboost-config.json').write_text(model.get_booster().save_config())
save(out/'result.json', {'status': 'FIT_COMPLETE_NO_TEST_SCORES', 'checkpoint': str(checkpoint),
'checkpoint_sha256': sha(checkpoint), 'clock_sha256': sha(out/'clock.json'), 'c': clock,
'learning_nodes': n, 'learning_parts': len(rows), 'cpu_seconds': time.process_time()-cpu,
'test_labels_read': False, 'new_decodes': 0, 'retry': False})
def predict(root, c, job):
import joblib
import xgboost
proof = json.loads((root/'models-git-freeze.json').read_text())
if not proof.get('commit') or not proof.get('checkout'):
raise PermissionError('All-terminal model Git freeze missing or mismatched')
for rel in ['contract.json', 'batch-complete.json', 'models-terminal.json']:
path = root/rel; expected = proof['sha256'][rel]; checked(path, expected)
repository_path = str(path.resolve().relative_to(Path.cwd()))
raw = subprocess.check_output(['git', 'show', proof['commit']+':'+repository_path], cwd=proof['checkout'])
if hashlib.sha256(raw).hexdigest() != expected: raise PermissionError('Git freeze differs from local files')
terminal = json.loads((root/'batch-complete.json').read_text())
if terminal['global_stop'] or not terminal['accounting_complete'] or terminal['attempted'] != len(c['jobs']):
raise PermissionError('Fit queue not complete')
freeze = json.loads((root/'models-terminal.json').read_text())
if len(freeze['models']) != len(c['jobs']): raise PermissionError('Incomplete model manifest')
entry = next(r for r in freeze['models'] if r['index'] == job['index'])
if entry['status'] != 'PASS': raise ValueError('No checkpoint for failed cell')
out = root/'predictions'/f"job-{job['index']:02d}"; out.mkdir(parents=True, exist_ok=False)
model = xgboost.XGBRegressor(); model.load_model(checked(entry['checkpoint'], entry['checkpoint_sha256']))
old = job['baseline_model']; checked(old['path'], old['sha256'])
if old['format'] == 'joblib':
baseline = joblib.load(old['path'])
elif old['format'] == 'ubj':
baseline = xgboost.XGBRegressor(); baseline.load_model(old['path'])
else:
raise ValueError('Unsupported frozen A checkpoint format')
clock = json.loads(checked(entry['clock_path'], entry['clock_sha256']).read_text())['c']
index = []; cpu = time.process_time()
with threadpool_limits(job['threads']):
for row in role_rows(c, job, 'test'):
x, f, ids = load_features(row) # No target.npy load on this path.
seconds = decode_seconds(model.predict(x), f, clock, row['t_set'])
old_seconds = np.asarray(baseline.predict(x), dtype=np.float64)*row['t_set']
if old_seconds.shape != seconds.shape or not np.isfinite(old_seconds).all(): raise ValueError('Invalid A prediction')
path = out/(row['case']+'.npz')
np.savez_compressed(path, seconds=seconds, A_seconds=old_seconds, cF_seconds=clock*f*row['t_set'],
EIK_seconds=f*row['t_set'], raw_ids=ids)
index.append({'case': row['case'], 'path': str(path), 'sha256': sha(path),
'negative_predictions': int(np.sum(seconds < 0)), 'nodes': len(ids)})
save(out/'progress.json', {'parts': len(index), 'cpu_seconds': time.process_time()-cpu})
save(out/'index.json', index)
save(out/'result.json', {'status': 'PREDICTIONS_COMPLETE_NO_SCORES', 'index_sha256': sha(out/'index.json'),
'cpu_seconds': time.process_time()-cpu, 'target_values_read': False})
if __name__ == '__main__':
parser = argparse.ArgumentParser(); parser.add_argument('root', type=Path)
parser.add_argument('mode', choices=['fit', 'predict']); parser.add_argument('index', type=int)
args = parser.parse_args(); start = time.process_time()
try:
config = contract(args.root); selected = config['jobs'][args.index]
if selected['index'] != args.index: raise ValueError('Job index mismatch')
(fit if args.mode == 'fit' else predict)(args.root, config, selected)
except Exception as exc:
dest = args.root/'failures'; dest.mkdir(exist_ok=True)
save(dest/f'{args.mode}-{args.index:02d}.json', {'type': type(exc).__name__, 'message': str(exc),
'traceback': traceback.format_exc(), 'pid': os.getpid(), 'cpu_seconds': time.process_time()-start,
'retry': False})
raise
+19
View File
@@ -6,6 +6,25 @@ in the reviewed contract. Passing F explicitly avoids a hidden extra feature.
import numpy as np
def learning_clock(end_ratios, component_ids):
"""Mean over components of mean retained-tet-end/t_set per learning part.
For the random v4 protocol each part can be its own unit, if the final
specification chooses part weighting. Caller enforces training roles.
"""
ratios = np.asarray(end_ratios, dtype=np.float64)
if ratios.ndim != 1 or not len(ratios) or len(component_ids) != len(ratios):
raise ValueError('Clock row count mismatch')
if not np.isfinite(ratios).all() or np.any(ratios <= 0):
raise ValueError('Invalid retained-tet endpoint ratio')
groups = {}
for group, ratio in zip(component_ids, ratios):
if not isinstance(group, str) or not group:
raise ValueError('Missing clock group identity')
groups.setdefault(group, []).append(float(ratio))
return float(np.mean([np.mean(v) for v in groups.values()]))
def prior(cdf, clock):
f = np.asarray(cdf)
if f.ndim != 1 or f.dtype.kind != 'f' or not len(f):
+7 -1
View File
@@ -1,10 +1,16 @@
"""Synthetic interface checks only; no model fit or project data access."""
import unittest
import numpy as np
from phase2_residual_math import prior, residual_target, decode_seconds
from phase2_residual_math import prior, residual_target, decode_seconds, learning_clock
class ResidualInterfaceTests(unittest.TestCase):
def test_component_equal_clock_not_node_or_part_weighted(self):
self.assertEqual(learning_clock([1., 3., 10.], ['a', 'a', 'b']), 6.)
self.assertAlmostEqual(learning_clock([1., 3., 10.], ['a', 'b', 'c']), 14./3)
with self.assertRaises(ValueError): learning_clock([], [])
with self.assertRaises(ValueError): learning_clock([0.], ['a'])
def test_negative_residual_and_exact_reconstruction(self):
f = np.array([0., .25, .75]); y = np.array([0., .125, .5])
r = residual_target(y, f, 2., fit_dtype='float64')