Files
MeltMind/tools/track_a_recovery_common.py
T

104 lines
6.8 KiB
Python

"""One labelled infrastructure recovery; original data/recipe and old GPU ledger fixed."""
import math
from pathlib import Path
from phase2_h1_worker_v2 import checked,sha
from phase2_home_freeze_io import verify
from track_a_h2_common import read,config as original_config,view_for,project_cell
OLD=47943.94719455298
ORIGINAL=Path('/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18')
ROOT=ORIGINAL.with_name('track-a-recovery-2026-11-18')
RECOVERY=(3,4)
def config(root):
root=Path(root);c=read(root/'contract.json');verify(root,'implementation')
if (root.resolve()!=ROOT or c['approval']!='1795000626.905989' or c['monitor_ruling']!='1795000465.748129'
or c['recovery_folds']!=[3,4] or c['reused_folds']!=[0,1,2] or c['recovery_number']!=1
or not c['no_retry'] or not c['no_further_recovery'] or c['new_fits']!=2 or c['seed']!=0 or c['epochs']!=200
or c['old_gpu_seconds_once']!=OLD or c['gpu_cap']!=162000 or c['train_gpu_cap']!=154800-OLD
or c['postfit_gpu_reserve']!=7200 or c['gpu_query_timeout_seconds']!=10
or c['consecutive_failed_samples']!=3 or not c['job_failure_isolated']):raise PermissionError('Exact recovery scope')
for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h)
old,_=original_config(ORIGINAL)
if c['original_contract_sha256']!=sha(ORIGINAL/'contract.json'):raise ValueError('Old contract identity')
for k in ['parts','geometry','jobs','model','versions','normalization','clock_policy','metric','original_failed','seed','epochs']:
if c[k]!=old[k]:raise ValueError('Scientific recipe changed: '+k)
d=read(ORIGINAL/'complete.json')
if d['status']!='STOPPED' or d['training_gpu_seconds']!=OLD or sum(e['gpu_seconds_charged'] for e in d['entries'])!=OLD:
raise ValueError('Old cost counted once')
if c['gpu_cap']-OLD-c['postfit_gpu_reserve']!=c['train_gpu_cap']:raise ValueError('Remaining training ledger')
return c,c
def reuse_identity(c):
d=read(ORIGINAL/'complete.json');stop=read(ORIGINAL/'global-stop.json')
if d['status']!='STOPPED' or stop['reason']!='JOB_4_FAILED':raise ValueError('Original STOP must stay immutable')
proof=verify(ORIGINAL,'infrastructure-stop');out=[]
for i in [0,1,2]:
e=next(e for e in d['entries'] if e['id']==i);w=ORIGINAL/'jobs'/str(i)/'worker'
g=read(checked(w.parent/'complete.json',e['guard_complete_sha256']));receipt=read(w.parent/'guards/train.receipt.json')
if e['status']!='PASS' or e['returncode']!=0 or g['status']!='PASS' or g['monitor_errors'] or g['reasons'] or g['worker']!=receipt or receipt['status']!='PASS' or receipt['returncode']!=0 or type(receipt['monitor_errors']) is not int or receipt['monitor_errors']!=0:
raise ValueError('Only genuine old PASS may be reused')
r=read(checked(w/'result.json',g['worker_result_sha256']));j=c['jobs'][i]
if r['fold']!=i or r['job']!=i or r['completed_epochs']!=200 or r['optimizer_steps']!=j['target_steps'] or r['test_targets_read'] or r['test_prediction_started'] or r['validation_used_for_selection']:
raise ValueError('Old final model roles')
model=dict(job=i,path=r['checkpoint'],sha256=r['checkpoint_sha256'],standardization=str(w/'standardization.json'),standardization_sha256=r['standardization_sha256'],origin='ORIGINAL_HASH_REUSE',training_contract_sha256=c['original_contract_sha256'],guard_complete_sha256=e['guard_complete_sha256'])
for p,h in [(model['path'],model['sha256']),(model['standardization'],model['standardization_sha256'])]:
checked(p,h)
if proof['files'].get(str(Path(p).resolve()))!=h:raise ValueError('Reuse omitted from immutable STOP freeze')
stats=read(model['standardization'])
if stats['parts']!=j['learning'] or stats['nodes']!=j['learning_nodes'] or len(stats['mean'])!=19 or len(stats['scale'])!=19:raise ValueError('Original normalization identity')
out.append(model)
if out!=c['reused_models']:raise ValueError('Exact reused model manifest')
return out
def execution_command(args):
return any(Path(a).name in {'track_a_h2_queue.py','track_a_h2_guard.py','track_a_h2_worker.py','track_a_recovery_queue.py','track_a_recovery_guard.py','track_a_recovery_worker.py'} for a in args)
def first_epoch_release(root,jobid):
if jobid not in RECOVERY:raise PermissionError('No reused fold execution')
if (root/'global-stop.json').exists() or (root/'jobs'/str(jobid)/'stop.json').exists():raise RuntimeError('This recovery fold stopped')
p=root/'jobs'/str(jobid)/'first-epoch-projection.json'
if not p.exists():return False
d=read(p)
if d['status']!='PASS' or d['contract_sha256']!=sha(root/'contract.json'):raise RuntimeError('Own cost gate rejected')
return True
def accepted_guard(rc,g):
if rc!=0 or not g or g['status']!='PASS' or g['reasons'] or g['monitor_policy']['failed']:return False
if g['monitor_policy']['errors'] != len(g['monitor_errors']):return False
w=g.get('worker')
return bool(w and w['status']=='PASS' and w['returncode']==0 and type(w['monitor_errors']) is int
and not w['monitor_policy']['failed'] and w['monitor_policy']['errors']==w['monitor_errors'])
def project_total(c,estimates,terminal,live,stopping=None):
stopping=set(stopping or [])
if (set(estimates)|set(terminal)|set(live)|stopping)-set(RECOVERY):raise ValueError('Unknown recovery fold')
rows=[];new=c['postfit_gpu_reserve']
for i in RECOVERY:
if i in terminal:
value=terminal[i]['gpu_seconds_charged'];basis='terminal_actual'
elif i in stopping:
# No rejected future training remains scheduled. Retain the full
# originally approved own cap for bounded cleanup until actual exit.
value=max(live.get(i,0),c['jobs'][i]['gpu_seconds']);basis='locally_stopping_own_cap_or_actual'
else:
value=max(live.get(i,0),estimates.get(i,{}).get('projected_gpu_seconds',c['jobs'][i]['gpu_seconds']));basis='continuing_projection_or_pending_cap'
if not math.isfinite(value) or value<0:raise ValueError('Invalid new charge')
new+=value;rows.append(dict(id=i,projected_gpu_seconds=value,basis=basis))
# Per-cell excess is isolated by own stop.json; total exhaustion stays global.
return dict(status='PASS' if OLD+new<=c['gpu_cap'] else 'STOP_CUMULATIVE_BUDGET',rows=rows,new_projected_gpu_seconds=new,old_gpu_seconds_once=OLD,cumulative_projected_gpu_seconds=OLD+new,postfit_reserve=c['postfit_gpu_reserve'])
def canonical_models(c,new_models):
ids=[m['job'] for m in new_models]
if len(ids)!=2 or set(ids)!={3,4}:raise ValueError('No five-fold model adoption with missing recovery fold')
result=c['reused_models']+new_models
if len(result)!=5 or {m['job'] for m in result}!=set(range(5)):raise ValueError('Canonical five models')
return sorted(result,key=lambda m:m['job'])