260 lines
13 KiB
Python
260 lines
13 KiB
Python
"""Reviewed GINO-Greif training, predictions and Drive-backed custody entry."""
|
|
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import signal
|
|
import time
|
|
|
|
from phase2_gate_outcome_review import validate_review
|
|
from s2_gpu_idle_gate import snapshot
|
|
import track_b_gino_greif_prepare as prep
|
|
from track_b_gino_formal_guard import guard
|
|
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
|
|
|
|
|
|
ROOT = prep.ROOT / 'combined-review'
|
|
SOURCE = Path(__file__).resolve().parent
|
|
ADAPTER = prep.REVIEW_ROOT / 'preparation-binding.json'
|
|
ADAPTER_SHA = '01d6581c33a6c26d1965acb13248bdb741de892e5929fff08a7e4af431f00d0b'
|
|
H2_SCORE = prep.H2_REFERENCE[1] / 'score/result.json'
|
|
H2_SCORE_SHA = 'd259c156216f274f26ee8ba04fe56a763cb72e7750fe6d71df6e15ba7419f6d4'
|
|
HARD = BASE / 'artifacts/phase3-h2-shared-miss-attributes-2026-11-19/by-part.jsonl'
|
|
DEVICES = ('GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c','GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
|
|
'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76','GPU-8b71b032-1b75-2891-935c-ce106a53034a')
|
|
JOBS = ((0,0),(0,1),(1,0),(1,1))
|
|
CAP = 100000
|
|
PRED_SECONDS = 1800
|
|
FIT_SETUP_RESERVE = 1200
|
|
FIRST_STAGE_MAX = 3600
|
|
|
|
|
|
def comparison(candidate):
|
|
rows = [json.loads(line) for line in HARD.read_text().splitlines()]
|
|
hard = [r['case'] for r in rows if r['shared_miss']]
|
|
if len(hard) != 34 or len(set(hard)) != 34:
|
|
raise ValueError('The original named hard cohort is required')
|
|
query = read(checked(prep.QUERY, prep.PINS[str(prep.QUERY)]))
|
|
family = {r['case']:r['component_id'] for r in query['rows'] if r['included']}
|
|
roster = [dict(fold=int(fold), case=case, family=family[case])
|
|
for fold,data in candidate['folds'].items() for case in data['parts']['test']]
|
|
if len({r['case'] for r in roster}) != 212 or len({r['family'] for r in roster}) != 78:
|
|
raise ValueError('The fixed 212-part / 78-family support is required')
|
|
return dict(roster=roster, hard34=hard, frozen_hard_source=dict(path=str(HARD),sha256=sha(HARD)),
|
|
h2_score=dict(path=str(H2_SCORE),sha256=H2_SCORE_SHA), method='Paired component bootstrap of GINO minus H2',
|
|
metric='field relative L2; mean of seed0 and seed1 per arm, then case and component means',
|
|
confidence=.95, replicates=2000, bootstrap_seed=2026111408,
|
|
gates={'high_below_zero':'GINO_BETTER','contains_zero':'COMPARABLE','low_above_zero':'H2_BETTER'},
|
|
secondary=['last_location/Rg','Kendall tau-b','intersection of named hard34 and two-fold test'],
|
|
custody='each of the 212 original targets once for both new predictions, after all four checkpoints and predictions frozen')
|
|
|
|
|
|
def expected_contract():
|
|
c = read(checked(ADAPTER, ADAPTER_SHA))
|
|
for path,h in c['code_sha256'].items():
|
|
checked(path,h)
|
|
base = prep.reference.contract()
|
|
comparison_path = ROOT / 'comparison-contract.json'
|
|
if read(comparison_path) != comparison(c):
|
|
raise ValueError('Pre-registered Greif comparison differs')
|
|
code=dict(base['code_sha256'],**c['code_sha256'])
|
|
for name in ('track_b_gino_greif_entry.py','track_b_gino_greif_worker.py','track_b_gino_greif_custody.py',
|
|
'phase2_h2_model.py','phase2_h2_data.py','s2_custody_common.py','s2_custody_statistics.py'):
|
|
code[str(SOURCE/name)] = sha(SOURCE/name)
|
|
return dict(handoff='h-08c2a2a5',mode='gino-greif-combined',adapter_sha256=ADAPTER_SHA,
|
|
comparison_contract=str(comparison_path),comparison_contract_sha256=sha(comparison_path),code_sha256=code,
|
|
folds=c['folds'],jobs=[list(x) for x in JOBS],epochs=c['epochs'],environment=base['environment'],
|
|
fixed_gino_reference=str(prep.reference.ROOT),phase3_anchor=c['projection'],limits=base['limits'],
|
|
device_order=list(DEVICES),gpu3_single_task=True,
|
|
budgets=dict(gpu_cap=CAP,phase3_cap=prep.CAP,max_first_stage=FIRST_STAGE_MAX,
|
|
per_prediction=PRED_SECONDS,train_setup_reserve=FIT_SETUP_RESERVE,score_cpu_seconds=3600),
|
|
run_order=['two first epochs as part of seed0 fits','reviewed cost gate','complete all four fits',
|
|
'freeze models','form four target-free prediction sets','freeze predictions',
|
|
'verified private Drive transfer','one controlled custody pass'],
|
|
drive={'purpose':'immutable model and prediction bundle','git':'small SHA indexes and receipts only'},
|
|
no_retry=True)
|
|
|
|
|
|
def contract():
|
|
c = read(ROOT/'contract.json')
|
|
if c != expected_contract():
|
|
raise PermissionError('Exact reviewed Greif contract differs')
|
|
return c
|
|
|
|
|
|
def authorize(c):
|
|
validate_review(ROOT)
|
|
a=read(ROOT/'stage-authorization.json')
|
|
if (a.get('decision'),a.get('mode'),a.get('contract_sha256'),a.get('review_sha256')) != (
|
|
'APPROVED',c['mode'],sha(ROOT/'contract.json'),sha(ROOT/'review.json')):
|
|
raise PermissionError('Exact review plus separate stage authorization required')
|
|
if (not a.get('yanqi_direct_reply_id') or not a.get('claude_shape_verdict_id')
|
|
or not a.get('private_drive_folder_id')):
|
|
raise PermissionError('Direct user launch and ShapeNet verdict are required')
|
|
verdict=read(checked(a['shape_verdict']['path'],a['shape_verdict']['sha256']))
|
|
if (verdict['contract_sha256']!=prep.reference.sha(prep.reference.ROOT/'contract.json')
|
|
or verdict['decision'] not in ('REPRODUCED','NOT_REPRODUCED')
|
|
or verdict['claude_ruling_id']!=a['claude_shape_verdict_id']):
|
|
raise PermissionError('Exact final ShapeNet verdict receipt required')
|
|
latest=read(checked(a['phase3_ledger']['path'],a['phase3_ledger']['sha256']))
|
|
stamp=read(ROOT/'started.json')['unix'] if (ROOT/'started.json').exists() else time.time()
|
|
# Charge each currently active program at its full un-settled reserve.
|
|
keys={'earlier_phase3_settled','gino_old_measurement','gino_formal','h2_pressure','other'}
|
|
figures=latest['gpu_seconds']
|
|
p=c['phase3_anchor']
|
|
if (latest['phase3_cap_gpu_seconds']!=prep.CAP or not 0<=stamp-latest['checked_unix']<=900
|
|
or set(figures)!=keys
|
|
or figures['earlier_phase3_settled']<p['settled_phase3_through_1b_gpu_seconds']
|
|
or figures['gino_old_measurement']!=p['gino_measurement_once_gpu_seconds']
|
|
or figures['gino_formal']<p['gino_shapenet_formal_reserve_gpu_seconds']
|
|
or figures['h2_pressure']<p['h2_pressure_unsettled_max_gpu_seconds']
|
|
or any(type(x) not in (int,float) or not math.isfinite(x) or x<0 for x in figures.values())
|
|
or sum(figures.values())+CAP>prep.CAP
|
|
or type(a['dispatch_before_unix']) not in (float,int) or not math.isfinite(a['dispatch_before_unix'])):
|
|
raise PermissionError('Live phase-3 ledger cannot book the declared Greif cap')
|
|
return a, sum(figures.values())
|
|
|
|
|
|
def path(fold, seed, phase):
|
|
return ROOT/f'fold-{fold}'/f'seed-{seed}'/phase
|
|
|
|
|
|
def stage_terminal(fold, seed, phase):
|
|
root = path(fold,seed,phase)
|
|
s = read(root/'complete.json')
|
|
if (s['status'],s['fold'],s['seed'],s['phase'],s['contract_sha256']) != (
|
|
'COMPLETE',fold,seed,phase,sha(ROOT/'contract.json')):
|
|
raise ValueError('Stage terminal differs')
|
|
return read(checked(root/'worker-records/result.json',s['result_sha256']))
|
|
|
|
|
|
def files_for(result):
|
|
entries=result['files']
|
|
for row in entries:
|
|
checked(row['path'],row['sha256'])
|
|
return entries
|
|
|
|
|
|
def validate_result(c, fold, seed, phase):
|
|
root=path(fold,seed,phase)
|
|
result=root/'worker-records/result.json'
|
|
obj=read(result)
|
|
ticket=read(root/'started.json')
|
|
spawn=read(root/'actual-worker-spawn.json')
|
|
uuid=DEVICES[int(ticket['ordinal'])]
|
|
if (spawn['physical_uuid']!=uuid or spawn['ordinal']!=ticket['ordinal']
|
|
or read(root/'worker-device-verified.json') != dict(
|
|
process=spawn['process'],ordinal=ticket['ordinal'],physical_uuid=uuid,observed_uuids=[uuid])
|
|
or (obj['status'],obj['fold'],obj['seed'],obj['phase'],obj['test_targets_read']) !=
|
|
('COMPLETE',fold,seed,phase,False)):
|
|
raise ValueError('Worker, physical card or target scope differs')
|
|
files_for(obj)
|
|
model=c['folds'][str(fold)]
|
|
if phase=='predict':
|
|
if [r['case'] for r in obj['files']] != model['parts']['test']:
|
|
raise ValueError('Test prediction roster differs')
|
|
else:
|
|
if (obj['epochs'],obj['learning_cases'],obj['learning_nodes']) != (
|
|
1 if phase=='first' else c['epochs'],model['counts']['learning'],model['learning_nodes']):
|
|
raise ValueError('Full fold epoch coverage required')
|
|
return sha(result)
|
|
|
|
|
|
def run_stage(c, auth, fold, seed, phase, deadline):
|
|
if time.time()>=auth['dispatch_before_unix']:
|
|
raise RuntimeError('APPROVED_DISPATCH_WINDOW_EXPIRED')
|
|
ordinal=next((i for i,u in enumerate(DEVICES) if snapshot(u)['idle']),None)
|
|
if ordinal is None:
|
|
raise RuntimeError('NO_IDLE_APPROVED_CARD')
|
|
started=time.monotonic()
|
|
if deadline-started<=c['limits']['cleanup_seconds']:
|
|
raise RuntimeError('INSUFFICIENT_REMAINING_BUDGET')
|
|
root=path(fold,seed,phase)
|
|
root.mkdir(parents=True,exist_ok=False)
|
|
ticket=dict(fold=fold,seed=seed,phase=phase,ordinal=str(ordinal),contract_sha256=sha(ROOT/'contract.json'),
|
|
booked_gpu_seconds=deadline-started,guard_started_monotonic=started,
|
|
dispatch_before_unix=auth['dispatch_before_unix'])
|
|
save(root/'started.json',ticket)
|
|
command=[c['environment']['python'],'-B',str(SOURCE/'track_b_gino_greif_worker.py'),str(fold),str(seed),phase]
|
|
receipt=guard(root,command,uuid=DEVICES[ordinal],limits=dict(c['limits'],cpu_seconds=8*(deadline-started)),
|
|
deadline=deadline,ticket=ticket,validate_result=lambda:validate_result(c,fold,seed,phase))
|
|
if receipt['status']!='COMPLETE':
|
|
raise RuntimeError(f"{fold}/{seed}/{phase} {receipt['trigger']}")
|
|
return stage_terminal(fold,seed,phase), receipt['charged_gpu_seconds']
|
|
|
|
|
|
def cost_gate(c, receipts, phase_booked, elapsed):
|
|
candidate=dict(folds=c['folds'],epochs=c['epochs'],projection=dict(
|
|
accounted_gpu_seconds=phase_booked,upper_remaining_before_other_bookings_gpu_seconds=prep.CAP-phase_booked))
|
|
stages=sum(r['booked_gpu_seconds'] for r in receipts.values())
|
|
overhead=4*PRED_SECONDS+FIT_SETUP_RESERVE+max(0.,elapsed-stages)
|
|
return prep.first_epoch_gate(candidate,receipts,other_unsettled=0.,approved_greif_cap=CAP,remaining_overhead=overhead)
|
|
|
|
|
|
def run(c, auth, phase_booked):
|
|
with (ROOT/'execution.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if (ROOT/'started.json').exists() or any(ROOT.glob('fold-*')):
|
|
raise FileExistsError('The combined attempt already exists')
|
|
began=time.monotonic()
|
|
save(ROOT/'started.json',dict(started_monotonic=began,unix=time.time(),phase_booked=phase_booked,
|
|
greif_reserved_gpu_seconds=CAP,stage_authorization_sha256=sha(ROOT/'stage-authorization.json')))
|
|
deadline=began+CAP
|
|
fit_deadline=deadline-4*PRED_SECONDS
|
|
status,trigger='STOPPED',None
|
|
|
|
def stop(number,_frame):
|
|
raise KeyboardInterrupt(signal.Signals(number).name)
|
|
|
|
handler=signal.signal(signal.SIGTERM,stop)
|
|
try:
|
|
first={}
|
|
for fold in (0,1):
|
|
result,charge=run_stage(c,auth,fold,0,'first',min(fit_deadline,time.monotonic()+FIRST_STAGE_MAX))
|
|
first[str(fold)]=dict(fold=fold,seed=0,epoch=1,learning_cases=result['learning_cases'],
|
|
learning_nodes=result['learning_nodes'],heldout_target_reads=0,
|
|
epoch_seconds=result['first_epoch_seconds'],booked_gpu_seconds=charge)
|
|
gate=cost_gate(c,first,phase_booked,time.monotonic()-began)
|
|
save(ROOT/'first-epoch-cost-gate.json',dict(gate,measurements=first))
|
|
if not gate['release_epoch2']:
|
|
raise RuntimeError(gate['status'])
|
|
models=[]
|
|
for fold,seed in JOBS:
|
|
result,_=run_stage(c,auth,fold,seed,'fit',fit_deadline)
|
|
models.extend(files_for(result))
|
|
save(ROOT/'models-frozen.json',models)
|
|
predictions=[]
|
|
for fold,seed in JOBS:
|
|
result,_=run_stage(c,auth,fold,seed,'predict',min(deadline,time.monotonic()+PRED_SECONDS))
|
|
predictions.extend(files_for(result))
|
|
save(ROOT/'predictions-frozen.json',predictions)
|
|
status='PREDICTIONS_FROZEN_NO_TARGETS'
|
|
except (Exception,KeyboardInterrupt) as exc:
|
|
trigger=str(exc) or type(exc).__name__
|
|
finally:
|
|
signal.signal(signal.SIGTERM,handler)
|
|
charged=time.monotonic()-began
|
|
if charged>CAP and status!='STOPPED':
|
|
status,trigger='STOPPED','TOTAL_GPU_CAP'
|
|
save(ROOT/'complete.json',dict(status=status,trigger=trigger,charged_gpu_seconds=charged,
|
|
contract_sha256=sha(ROOT/'contract.json'),test_targets_read=False,no_retry=True))
|
|
return 0 if status=='PREDICTIONS_FROZEN_NO_TARGETS' else 1
|
|
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('action',choices=('run','pack','score'))
|
|
args=parser.parse_args()
|
|
c=contract()
|
|
auth,booked=authorize(c)
|
|
if args.action=='run':
|
|
return run(c,auth,booked)
|
|
from track_b_gino_greif_custody import pack, score
|
|
return pack(c) if args.action=='pack' else score(c)
|
|
|
|
|
|
if __name__=='__main__':
|
|
raise SystemExit(main())
|