"""H1-RES fixed-XGB worker. No queue or implicit execution approval. The reviewed contract binds training/test roles, source metadata, precision, clock weighting and source hashes. Test predictions never load target arrays. """ import argparse import hashlib import json import os from pathlib import Path import subprocess import time import traceback import numpy as np from threadpoolctl import threadpool_limits from gre24_models import xgb_model from phase2_residual_math import learning_clock, residual_target, decode_seconds def sha(path): h = hashlib.sha256() with Path(path).open('rb') as source: for block in iter(lambda: source.read(4*1024**2), b''): h.update(block) return h.hexdigest() def save(path, value): path = Path(path); tmp = path.with_suffix(path.suffix+'.tmp') tmp.write_text(json.dumps(value, indent=2, allow_nan=False)+'\n'); tmp.replace(path) def checked(path, digest): if sha(path) != digest: raise ValueError('Pinned file mismatch: '+str(path)) return Path(path) def contract(root): import xgboost if np.__version__ != '1.26.4' or xgboost.__version__ != '2.1.4': raise ValueError('Frozen library version differs') c = json.loads((root/'contract.json').read_text()) if c.get('version') != 'phase2-h1-res-v1' or not c.get('main_review'): raise PermissionError('No fixed reviewed H1 contract') for p, h in c['code_sha256'].items(): checked(p, h) for p, h in c['source_sha256'].items(): checked(p, h) return c def role_rows(c, job, role): if role not in ('learning', 'test'): raise ValueError('Unknown role') if set(job['learning']) & set(job['test']): raise PermissionError('Training/test overlap') if len(job[role]) != len(set(job[role])): raise ValueError('Repeated case') # Contract builder must cross-check these frozen roles against original # source contracts. Here a role cannot be expanded through row metadata. return [c['parts'][job['protocol']][case] for case in job[role]] def load_features(row): source = Path(row['output']); hashes = row['output_sha256'] f = np.load(checked(source/'Vf.npy', hashes['Vf.npy']), allow_pickle=False) x = np.load(checked(source/'features.npy', hashes['features.npy']), mmap_mode='r', allow_pickle=False) ids = np.load(checked(source/'raw_ids.npy', hashes['raw_ids.npy']), allow_pickle=False) if x.shape != (row['nodes'], 19) or f.shape != (row['nodes'],) or ids.shape != f.shape: raise ValueError('Feature/node identity shape mismatch') if f.dtype != np.dtype('float64') or not np.array_equal(np.asarray(x[:, 0],dtype=np.float32), f.astype(np.float32)): raise ValueError('Original float64 F differs from frozen first column') if np.any(np.diff(ids) <= 0) or not np.isfinite(x).all(): raise ValueError('Bad IDs/features') return np.asarray(x, dtype=np.float32), f, ids def fit(root, c, job): out = root/'fits'/f"job-{job['index']:02d}"; out.mkdir(parents=True, exist_ok=False) save(out/'started.json', {'unix': time.time(), 'job': job, 'contract_sha256': sha(root/'contract.json')}) cpu = time.process_time(); access = []; rows = role_rows(c, job, 'learning') end_ratios, groups = [], [] # First pass: only learning-role retained-tet targets for this cell's clock. for row in rows: path = Path(row['output'])/'seconds.npy' access.append({'case': row['case'], 'role': 'learning', 'purpose': 'clock', 'path': str(path)}) save(out/'target-access.json', access) y = np.load(checked(path, row['output_sha256']['seconds.npy']), mmap_mode='r', allow_pickle=False) if y.shape != (row['nodes'],) or not np.isfinite(y).all() or np.any(y < 0): raise ValueError('Invalid learning target') if y.dtype != np.dtype('float64') or not np.isfinite(row['t_set']) or row['t_set']<=0: raise ValueError('Endpoint precision/scale mismatch') raw=np.load(checked(Path(row['output'])/'raw_ids.npy',row['output_sha256']['raw_ids.npy']),allow_pickle=False) if raw.shape!=y.shape or np.any(np.diff(raw)<=0): raise ValueError('Endpoint raw ID mismatch') end_ratios.append(float(y.max())/float(row['t_set'])); groups.append(row['clock_group']) clock = learning_clock(end_ratios, groups) save(out/'clock.json', {'c': clock, 'source': 'learning retained-tet max(T)/t_set, mean within component then mean across components', 'parts': [row['case'] for row in rows], 'end_ratios': end_ratios, 'groups': groups}) n = sum(row['nodes'] for row in rows); x = np.empty((n, 19), np.float32); y = np.empty(n, np.float32); cursor = 0 for row in rows: xx, f, ids = load_features(row) path = checked(Path(row['output'])/'target.npy', row['output_sha256']['target.npy']) target = np.load(path, mmap_mode='r', allow_pickle=False) physical=np.load(checked(Path(row['output'])/'seconds.npy',row['output_sha256']['seconds.npy']),mmap_mode='r',allow_pickle=False) if not np.array_equal(target,physical/row['t_set']): raise ValueError('Frozen normalized target differs from physical seconds/t_set') count = len(f); x[cursor:cursor+count] = xx y[cursor:cursor+count] = residual_target(target, f, clock, fit_dtype='float32'); cursor += count access.append({'case': row['case'], 'role': 'learning', 'purpose': 'residual_fit', 'path': str(path)}) save(out/'target-access.json', access) with threadpool_limits(job['threads']): model = xgb_model(job['seed'], job['threads']) save(out/'progress.json', {'stage': 'FIT', 'nodes': n, 'clock': clock}) model.fit(x, y, verbose=False) checkpoint = out/'model.ubj'; model.save_model(checkpoint) (out/'xgboost-config.json').write_text(model.get_booster().save_config()) save(out/'result.json', {'status': 'FIT_COMPLETE_NO_TEST_SCORES', 'checkpoint': str(checkpoint), 'checkpoint_sha256': sha(checkpoint), 'clock_sha256': sha(out/'clock.json'), 'c': clock, 'learning_nodes': n, 'learning_parts': len(rows), 'cpu_seconds': time.process_time()-cpu, 'test_labels_read': False, 'new_decodes': 0, 'retry': False}) def predict(root, c, job): import joblib import xgboost proof = json.loads((root/'models-git-freeze.json').read_text()) if not proof.get('commit') or not proof.get('checkout'): raise PermissionError('All-terminal model Git freeze missing or mismatched') for rel in ['contract.json', 'batch-complete.json', 'models-terminal.json']: path = root/rel; expected = proof['sha256'][rel]; checked(path, expected) repository_path = str(path.resolve().relative_to(Path.cwd())) raw = subprocess.check_output(['git', 'show', proof['commit']+':'+repository_path], cwd=proof['checkout']) if hashlib.sha256(raw).hexdigest() != expected: raise PermissionError('Git freeze differs from local files') terminal = json.loads((root/'batch-complete.json').read_text()) if terminal['global_stop'] or not terminal['accounting_complete'] or terminal['attempted'] != len(c['jobs']): raise PermissionError('Fit queue not complete') freeze = json.loads((root/'models-terminal.json').read_text()) if len(freeze['models']) != len(c['jobs']): raise PermissionError('Incomplete model manifest') entry = next(r for r in freeze['models'] if r['index'] == job['index']) if entry['status'] != 'PASS': raise ValueError('No checkpoint for failed cell') out = root/'predictions'/f"job-{job['index']:02d}"; out.mkdir(parents=True, exist_ok=False) model = xgboost.XGBRegressor(); model.load_model(checked(entry['checkpoint'], entry['checkpoint_sha256'])) old = job['baseline_model']; checked(old['path'], old['sha256']) if old['format'] == 'joblib': baseline = joblib.load(old['path']) elif old['format'] == 'ubj': baseline = xgboost.XGBRegressor(); baseline.load_model(old['path']) else: raise ValueError('Unsupported frozen A checkpoint format') clock = json.loads(checked(entry['clock_path'], entry['clock_sha256']).read_text())['c'] index = []; cpu = time.process_time() with threadpool_limits(job['threads']): for row in role_rows(c, job, 'test'): x, f, ids = load_features(row) # No target.npy load on this path. seconds = decode_seconds(model.predict(x), f, clock, row['t_set']) old_seconds = np.asarray(baseline.predict(x), dtype=np.float64)*row['t_set'] if old_seconds.shape != seconds.shape or not np.isfinite(old_seconds).all(): raise ValueError('Invalid A prediction') if job['protocol']=='grouped': prior=c['baseline_predictions'][str(job['index'])][row['case']] with np.load(checked(prior['path'],prior['sha256']),allow_pickle=False) as z: iq=np.searchsorted(ids,z['raw_ids']) if np.any(iq>=len(ids)) or not np.array_equal(ids[iq],z['raw_ids']):raise ValueError('Historical A query identity differs') if not np.array_equal(old_seconds[iq],z[prior['array']]):raise ValueError('Historical A query prediction differs') path = out/(row['case']+'.npz') np.savez_compressed(path, seconds=seconds, A_seconds=old_seconds, cF_seconds=clock*f*row['t_set'], EIK_seconds=f*row['t_set'], raw_ids=ids) index.append({'case': row['case'], 'path': str(path), 'sha256': sha(path), 'negative_predictions': int(np.sum(seconds < 0)), 'nodes': len(ids)}) save(out/'progress.json', {'parts': len(index), 'cpu_seconds': time.process_time()-cpu}) save(out/'index.json', index) save(out/'result.json', {'status': 'PREDICTIONS_COMPLETE_NO_SCORES', 'index_sha256': sha(out/'index.json'), 'cpu_seconds': time.process_time()-cpu, 'target_values_read': False}) if __name__ == '__main__': parser = argparse.ArgumentParser(); parser.add_argument('root', type=Path) parser.add_argument('mode', choices=['fit', 'predict']); parser.add_argument('index', type=int) args = parser.parse_args(); start = time.process_time() try: config = contract(args.root); selected = config['jobs'][args.index] if selected['index'] != args.index: raise ValueError('Job index mismatch') (fit if args.mode == 'fit' else predict)(args.root, config, selected) except Exception as exc: dest = args.root/'failures'; dest.mkdir(exist_ok=True) save(dest/f'{args.mode}-{args.index:02d}.json', {'type': type(exc).__name__, 'message': str(exc), 'traceback': traceback.format_exc(), 'pid': os.getpid(), 'cpu_seconds': time.process_time()-start, 'retry': False}) raise