"""Reviewed GINO-Greif training, predictions and Drive-backed custody entry.""" import argparse import fcntl import json import math from pathlib import Path import signal import time from phase2_gate_outcome_review import validate_review from s2_gpu_idle_gate import snapshot import track_b_gino_greif_prepare as prep from track_b_gino_formal_guard import guard from track_b_gino_shapenet_entry import BASE, checked, read, save, sha ROOT = prep.ROOT / 'combined-review-acl-v2' SOURCE = Path(__file__).resolve().parent ADAPTER = prep.REVIEW_ROOT / 'preparation-binding.json' ADAPTER_SHA = '01d6581c33a6c26d1965acb13248bdb741de892e5929fff08a7e4af431f00d0b' H2_SCORE = prep.H2_REFERENCE[1] / 'score/result.json' H2_SCORE_SHA = 'd259c156216f274f26ee8ba04fe56a763cb72e7750fe6d71df6e15ba7419f6d4' HARD = BASE / 'artifacts/phase3-h2-shared-miss-attributes-2026-11-19/by-part.jsonl' DEVICES = ('GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c','GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36', 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76','GPU-8b71b032-1b75-2891-935c-ce106a53034a') JOBS = ((0,0),(0,1),(1,0),(1,1)) CAP = 100000 PRED_SECONDS = 1800 FIT_SETUP_RESERVE = 1200 FIRST_STAGE_MAX = 3600 def comparison(candidate): rows = [json.loads(line) for line in HARD.read_text().splitlines()] hard = [r['case'] for r in rows if r['shared_miss']] if len(hard) != 34 or len(set(hard)) != 34: raise ValueError('The original named hard cohort is required') query = read(checked(prep.QUERY, prep.PINS[str(prep.QUERY)])) family = {r['case']:r['component_id'] for r in query['rows'] if r['included']} roster = [dict(fold=int(fold), case=case, family=family[case]) for fold,data in candidate['folds'].items() for case in data['parts']['test']] if len({r['case'] for r in roster}) != 212 or len({r['family'] for r in roster}) != 78: raise ValueError('The fixed 212-part / 78-family support is required') return dict(roster=roster, hard34=hard, frozen_hard_source=dict(path=str(HARD),sha256=sha(HARD)), h2_score=dict(path=str(H2_SCORE),sha256=H2_SCORE_SHA), method='Paired component bootstrap of GINO minus H2', metric='field relative L2; mean of seed0 and seed1 per arm, then case and component means', confidence=.95, replicates=2000, bootstrap_seed=2026111408, gates={'high_below_zero':'GINO_BETTER','contains_zero':'COMPARABLE','low_above_zero':'H2_BETTER'}, secondary=['last_location/Rg','Kendall tau-b','intersection of named hard34 and two-fold test'], custody='each of the 212 original targets once for both new predictions, after all four checkpoints and predictions frozen') def expected_contract(): c = read(checked(ADAPTER, ADAPTER_SHA)) for path,h in c['code_sha256'].items(): checked(path,h) base = prep.reference.contract() comparison_path = ROOT / 'comparison-contract.json' if read(comparison_path) != comparison(c): raise ValueError('Pre-registered Greif comparison differs') code=dict(base['code_sha256'],**c['code_sha256']) for name in ('track_b_gino_greif_entry.py','track_b_gino_greif_worker.py','track_b_gino_greif_custody.py', 'track_b_gino_greif_drive_acl.py', 'phase2_h2_model.py','phase2_h2_data.py','s2_custody_common.py','s2_custody_statistics.py'): code[str(SOURCE/name)] = sha(SOURCE/name) return dict(handoff='h-08c2a2a5',mode='gino-greif-combined',adapter_sha256=ADAPTER_SHA, comparison_contract=str(comparison_path),comparison_contract_sha256=sha(comparison_path),code_sha256=code, folds=c['folds'],jobs=[list(x) for x in JOBS],epochs=c['epochs'],environment=base['environment'], fixed_gino_reference=str(prep.reference.ROOT),phase3_anchor=c['projection'],limits=base['limits'], device_order=list(DEVICES),gpu3_single_task=True, budgets=dict(gpu_cap=CAP,phase3_cap=prep.CAP,max_first_stage=FIRST_STAGE_MAX, per_prediction=PRED_SECONDS,train_setup_reserve=FIT_SETUP_RESERVE,score_cpu_seconds=3600), run_order=['two first epochs as part of seed0 fits','reviewed cost gate','complete all four fits', 'freeze models','form four target-free prediction sets','freeze predictions', 'one controlled local custody pass; independent optional verified private Drive transfer'], drive={'purpose':'immutable model and prediction bundle','git':'small SHA indexes and receipts only', 'acl_revision':'h-c3d10e10 pre-upload parent ACL and post-upload file ACL', 'required_for_local_score':False}, no_retry=True) def contract(): c = read(ROOT/'contract.json') if c != expected_contract(): raise PermissionError('Exact reviewed Greif contract differs') return c def authorize(c): validate_review(ROOT) a=read(ROOT/'stage-authorization.json') if (a.get('decision'),a.get('mode'),a.get('contract_sha256'),a.get('review_sha256')) != ( 'APPROVED',c['mode'],sha(ROOT/'contract.json'),sha(ROOT/'review.json')): raise PermissionError('Exact review plus separate stage authorization required') if not a.get('yanqi_direct_reply_id') or not a.get('claude_shape_verdict_id'): raise PermissionError('Direct user launch and ShapeNet verdict are required') verdict=read(checked(a['shape_verdict']['path'],a['shape_verdict']['sha256'])) if (verdict['contract_sha256']!=prep.reference.sha(prep.reference.ROOT/'contract.json') or verdict['decision'] not in ('REPRODUCED','NOT_REPRODUCED') or verdict['claude_ruling_id']!=a['claude_shape_verdict_id']): raise PermissionError('Exact final ShapeNet verdict receipt required') latest=read(checked(a['phase3_ledger']['path'],a['phase3_ledger']['sha256'])) stamp=read(ROOT/'started.json')['unix'] if (ROOT/'started.json').exists() else time.time() # Charge each currently active program at its full un-settled reserve. keys={'earlier_phase3_settled','gino_old_measurement','gino_formal','h2_pressure','other'} figures=latest['gpu_seconds'] p=c['phase3_anchor'] if (latest['phase3_cap_gpu_seconds']!=prep.CAP or not 0<=stamp-latest['checked_unix']<=900 or set(figures)!=keys or figures['earlier_phase3_settled']
prep.CAP or type(a['dispatch_before_unix']) not in (float,int) or not math.isfinite(a['dispatch_before_unix'])): raise PermissionError('Live phase-3 ledger cannot book the declared Greif cap') return a, sum(figures.values()) def path(fold, seed, phase): return ROOT/f'fold-{fold}'/f'seed-{seed}'/phase def stage_terminal(fold, seed, phase): root = path(fold,seed,phase) s = read(root/'complete.json') if (s['status'],s['fold'],s['seed'],s['phase'],s['contract_sha256']) != ( 'COMPLETE',fold,seed,phase,sha(ROOT/'contract.json')): raise ValueError('Stage terminal differs') return read(checked(root/'worker-records/result.json',s['result_sha256'])) def files_for(result): entries=result['files'] for row in entries: checked(row['path'],row['sha256']) return entries def validate_result(c, fold, seed, phase): root=path(fold,seed,phase) result=root/'worker-records/result.json' obj=read(result) ticket=read(root/'started.json') spawn=read(root/'actual-worker-spawn.json') uuid=DEVICES[int(ticket['ordinal'])] if (spawn['physical_uuid']!=uuid or spawn['ordinal']!=ticket['ordinal'] or read(root/'worker-device-verified.json') != dict( process=spawn['process'],ordinal=ticket['ordinal'],physical_uuid=uuid,observed_uuids=[uuid]) or (obj['status'],obj['fold'],obj['seed'],obj['phase'],obj['test_targets_read']) != ('COMPLETE',fold,seed,phase,False)): raise ValueError('Worker, physical card or target scope differs') files_for(obj) model=c['folds'][str(fold)] if phase=='predict': if [r['case'] for r in obj['files']] != model['parts']['test']: raise ValueError('Test prediction roster differs') else: if (obj['epochs'],obj['learning_cases'],obj['learning_nodes']) != ( 1 if phase=='first' else c['epochs'],model['counts']['learning'],model['learning_nodes']): raise ValueError('Full fold epoch coverage required') return sha(result) def run_stage(c, auth, fold, seed, phase, deadline): if time.time()>=auth['dispatch_before_unix']: raise RuntimeError('APPROVED_DISPATCH_WINDOW_EXPIRED') ordinal=next((i for i,u in enumerate(DEVICES) if snapshot(u)['idle']),None) if ordinal is None: raise RuntimeError('NO_IDLE_APPROVED_CARD') started=time.monotonic() if deadline-started<=c['limits']['cleanup_seconds']: raise RuntimeError('INSUFFICIENT_REMAINING_BUDGET') root=path(fold,seed,phase) root.mkdir(parents=True,exist_ok=False) ticket=dict(fold=fold,seed=seed,phase=phase,ordinal=str(ordinal),contract_sha256=sha(ROOT/'contract.json'), booked_gpu_seconds=deadline-started,guard_started_monotonic=started, dispatch_before_unix=auth['dispatch_before_unix']) save(root/'started.json',ticket) command=[c['environment']['python'],'-B',str(SOURCE/'track_b_gino_greif_worker.py'),str(fold),str(seed),phase] receipt=guard(root,command,uuid=DEVICES[ordinal],limits=dict(c['limits'],cpu_seconds=8*(deadline-started)), deadline=deadline,ticket=ticket,validate_result=lambda:validate_result(c,fold,seed,phase)) if receipt['status']!='COMPLETE': raise RuntimeError(f"{fold}/{seed}/{phase} {receipt['trigger']}") return stage_terminal(fold,seed,phase), receipt['charged_gpu_seconds'] def cost_gate(c, receipts, phase_booked, elapsed): candidate=dict(folds=c['folds'],epochs=c['epochs'],projection=dict( accounted_gpu_seconds=phase_booked,upper_remaining_before_other_bookings_gpu_seconds=prep.CAP-phase_booked)) stages=sum(r['booked_gpu_seconds'] for r in receipts.values()) overhead=4*PRED_SECONDS+FIT_SETUP_RESERVE+max(0.,elapsed-stages) return prep.first_epoch_gate(candidate,receipts,other_unsettled=0.,approved_greif_cap=CAP,remaining_overhead=overhead) def run(c, auth, phase_booked): with (ROOT/'execution.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if (ROOT/'started.json').exists() or any(ROOT.glob('fold-*')): raise FileExistsError('The combined attempt already exists') began=time.monotonic() save(ROOT/'started.json',dict(started_monotonic=began,unix=time.time(),phase_booked=phase_booked, greif_reserved_gpu_seconds=CAP,stage_authorization_sha256=sha(ROOT/'stage-authorization.json'))) deadline=began+CAP fit_deadline=deadline-4*PRED_SECONDS status,trigger='STOPPED',None def stop(number,_frame): raise KeyboardInterrupt(signal.Signals(number).name) handler=signal.signal(signal.SIGTERM,stop) try: first={} for fold in (0,1): result,charge=run_stage(c,auth,fold,0,'first',min(fit_deadline,time.monotonic()+FIRST_STAGE_MAX)) first[str(fold)]=dict(fold=fold,seed=0,epoch=1,learning_cases=result['learning_cases'], learning_nodes=result['learning_nodes'],heldout_target_reads=0, epoch_seconds=result['first_epoch_seconds'],booked_gpu_seconds=charge) gate=cost_gate(c,first,phase_booked,time.monotonic()-began) save(ROOT/'first-epoch-cost-gate.json',dict(gate,measurements=first)) if not gate['release_epoch2']: raise RuntimeError(gate['status']) models=[] for fold,seed in JOBS: result,_=run_stage(c,auth,fold,seed,'fit',fit_deadline) models.extend(files_for(result)) save(ROOT/'models-frozen.json',models) predictions=[] for fold,seed in JOBS: result,_=run_stage(c,auth,fold,seed,'predict',min(deadline,time.monotonic()+PRED_SECONDS)) predictions.extend(files_for(result)) save(ROOT/'predictions-frozen.json',predictions) status='PREDICTIONS_FROZEN_NO_TARGETS' except (Exception,KeyboardInterrupt) as exc: trigger=str(exc) or type(exc).__name__ finally: signal.signal(signal.SIGTERM,handler) charged=time.monotonic()-began if charged>CAP and status!='STOPPED': status,trigger='STOPPED','TOTAL_GPU_CAP' save(ROOT/'complete.json',dict(status=status,trigger=trigger,charged_gpu_seconds=charged, contract_sha256=sha(ROOT/'contract.json'),test_targets_read=False,no_retry=True)) return 0 if status=='PREDICTIONS_FROZEN_NO_TARGETS' else 1 def main(): parser=argparse.ArgumentParser(description=__doc__) parser.add_argument('action',choices=('run','pack','score','drive-preflight','drive-readback')) args=parser.parse_args() c=contract() auth,booked=authorize(c) if args.action=='run': return run(c,auth,booked) from track_b_gino_greif_custody import drive_preflight, drive_readback, pack, score if args.action in ('drive-preflight','drive-readback'): with (ROOT/'drive-custody.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) return drive_preflight(c,auth) if args.action=='drive-preflight' else drive_readback(c,auth) return pack(c) if args.action=='pack' else score(c) if __name__=='__main__': raise SystemExit(main())