"""One labelled infrastructure recovery; original data/recipe and old GPU ledger fixed.""" import math from pathlib import Path from phase2_h1_worker_v2 import checked,sha from phase2_home_freeze_io import verify from track_a_h2_common import read,config as original_config,view_for,project_cell OLD=47943.94719455298 ORIGINAL=Path('/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18') ROOT=ORIGINAL.with_name('track-a-recovery-2026-11-18') RECOVERY=(3,4) def config(root): root=Path(root);c=read(root/'contract.json');verify(root,'implementation') if (root.resolve()!=ROOT or c['approval']!='1795000626.905989' or c['monitor_ruling']!='1795000465.748129' or c['recovery_folds']!=[3,4] or c['reused_folds']!=[0,1,2] or c['recovery_number']!=1 or not c['no_retry'] or not c['no_further_recovery'] or c['new_fits']!=2 or c['seed']!=0 or c['epochs']!=200 or c['old_gpu_seconds_once']!=OLD or c['gpu_cap']!=162000 or c['train_gpu_cap']!=154800-OLD or c['postfit_gpu_reserve']!=7200 or c['gpu_query_timeout_seconds']!=10 or c['consecutive_failed_samples']!=3 or not c['job_failure_isolated']):raise PermissionError('Exact recovery scope') for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h) old,_=original_config(ORIGINAL) if c['original_contract_sha256']!=sha(ORIGINAL/'contract.json'):raise ValueError('Old contract identity') for k in ['parts','geometry','jobs','model','versions','normalization','clock_policy','metric','original_failed','seed','epochs']: if c[k]!=old[k]:raise ValueError('Scientific recipe changed: '+k) d=read(ORIGINAL/'complete.json') if d['status']!='STOPPED' or d['training_gpu_seconds']!=OLD or sum(e['gpu_seconds_charged'] for e in d['entries'])!=OLD: raise ValueError('Old cost counted once') if c['gpu_cap']-OLD-c['postfit_gpu_reserve']!=c['train_gpu_cap']:raise ValueError('Remaining training ledger') return c,c def reuse_identity(c): d=read(ORIGINAL/'complete.json');stop=read(ORIGINAL/'global-stop.json') if d['status']!='STOPPED' or stop['reason']!='JOB_4_FAILED':raise ValueError('Original STOP must stay immutable') proof=verify(ORIGINAL,'infrastructure-stop');out=[] for i in [0,1,2]: e=next(e for e in d['entries'] if e['id']==i);w=ORIGINAL/'jobs'/str(i)/'worker' g=read(checked(w.parent/'complete.json',e['guard_complete_sha256']));receipt=read(w.parent/'guards/train.receipt.json') if e['status']!='PASS' or e['returncode']!=0 or g['status']!='PASS' or g['monitor_errors'] or g['reasons'] or g['worker']!=receipt or receipt['status']!='PASS' or receipt['returncode']!=0 or type(receipt['monitor_errors']) is not int or receipt['monitor_errors']!=0: raise ValueError('Only genuine old PASS may be reused') r=read(checked(w/'result.json',g['worker_result_sha256']));j=c['jobs'][i] if r['fold']!=i or r['job']!=i or r['completed_epochs']!=200 or r['optimizer_steps']!=j['target_steps'] or r['test_targets_read'] or r['test_prediction_started'] or r['validation_used_for_selection']: raise ValueError('Old final model roles') model=dict(job=i,path=r['checkpoint'],sha256=r['checkpoint_sha256'],standardization=str(w/'standardization.json'),standardization_sha256=r['standardization_sha256'],origin='ORIGINAL_HASH_REUSE',training_contract_sha256=c['original_contract_sha256'],guard_complete_sha256=e['guard_complete_sha256']) for p,h in [(model['path'],model['sha256']),(model['standardization'],model['standardization_sha256'])]: checked(p,h) if proof['files'].get(str(Path(p).resolve()))!=h:raise ValueError('Reuse omitted from immutable STOP freeze') stats=read(model['standardization']) if stats['parts']!=j['learning'] or stats['nodes']!=j['learning_nodes'] or len(stats['mean'])!=19 or len(stats['scale'])!=19:raise ValueError('Original normalization identity') out.append(model) if out!=c['reused_models']:raise ValueError('Exact reused model manifest') return out def execution_command(args): return any(Path(a).name in {'track_a_h2_queue.py','track_a_h2_guard.py','track_a_h2_worker.py','track_a_recovery_queue.py','track_a_recovery_guard.py','track_a_recovery_worker.py'} for a in args) def first_epoch_release(root,jobid): if jobid not in RECOVERY:raise PermissionError('No reused fold execution') if (root/'global-stop.json').exists() or (root/'jobs'/str(jobid)/'stop.json').exists():raise RuntimeError('This recovery fold stopped') p=root/'jobs'/str(jobid)/'first-epoch-projection.json' if not p.exists():return False d=read(p) if d['status']!='PASS' or d['contract_sha256']!=sha(root/'contract.json'):raise RuntimeError('Own cost gate rejected') return True def accepted_guard(rc,g): if rc!=0 or not g or g['status']!='PASS' or g['reasons'] or g['monitor_policy']['failed']:return False if g['monitor_policy']['errors'] != len(g['monitor_errors']):return False w=g.get('worker') return bool(w and w['status']=='PASS' and w['returncode']==0 and type(w['monitor_errors']) is int and not w['monitor_policy']['failed'] and w['monitor_policy']['errors']==w['monitor_errors']) def project_total(c,estimates,terminal,live,stopping=None): stopping=set(stopping or []) if (set(estimates)|set(terminal)|set(live)|stopping)-set(RECOVERY):raise ValueError('Unknown recovery fold') rows=[];new=c['postfit_gpu_reserve'] for i in RECOVERY: if i in terminal: value=terminal[i]['gpu_seconds_charged'];basis='terminal_actual' elif i in stopping: # No rejected future training remains scheduled. Retain the full # originally approved own cap for bounded cleanup until actual exit. value=max(live.get(i,0),c['jobs'][i]['gpu_seconds']);basis='locally_stopping_own_cap_or_actual' else: value=max(live.get(i,0),estimates.get(i,{}).get('projected_gpu_seconds',c['jobs'][i]['gpu_seconds']));basis='continuing_projection_or_pending_cap' if not math.isfinite(value) or value<0:raise ValueError('Invalid new charge') new+=value;rows.append(dict(id=i,projected_gpu_seconds=value,basis=basis)) # Per-cell excess is isolated by own stop.json; total exhaustion stays global. return dict(status='PASS' if OLD+new<=c['gpu_cap'] else 'STOP_CUMULATIVE_BUDGET',rows=rows,new_projected_gpu_seconds=new,old_gpu_seconds_once=OLD,cumulative_projected_gpu_seconds=OLD+new,postfit_reserve=c['postfit_gpu_reserve']) def canonical_models(c,new_models): ids=[m['job'] for m in new_models] if len(ids)!=2 or set(ids)!={3,4}:raise ValueError('No five-fold model adoption with missing recovery fold') result=c['reused_models']+new_models if len(result)!=5 or {m['job'] for m in result}!=set(range(5)):raise ValueError('Canonical five models') return sorted(result,key=lambda m:m['job'])