104 lines
6.8 KiB
Python
104 lines
6.8 KiB
Python
"""One labelled infrastructure recovery; original data/recipe and old GPU ledger fixed."""
|
|
import math
|
|
from pathlib import Path
|
|
from phase2_h1_worker_v2 import checked,sha
|
|
from phase2_home_freeze_io import verify
|
|
from track_a_h2_common import read,config as original_config,view_for,project_cell
|
|
|
|
OLD=47943.94719455298
|
|
ORIGINAL=Path('/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18')
|
|
ROOT=ORIGINAL.with_name('track-a-recovery-2026-11-18')
|
|
RECOVERY=(3,4)
|
|
|
|
|
|
def config(root):
|
|
root=Path(root);c=read(root/'contract.json');verify(root,'implementation')
|
|
if (root.resolve()!=ROOT or c['approval']!='1795000626.905989' or c['monitor_ruling']!='1795000465.748129'
|
|
or c['recovery_folds']!=[3,4] or c['reused_folds']!=[0,1,2] or c['recovery_number']!=1
|
|
or not c['no_retry'] or not c['no_further_recovery'] or c['new_fits']!=2 or c['seed']!=0 or c['epochs']!=200
|
|
or c['old_gpu_seconds_once']!=OLD or c['gpu_cap']!=162000 or c['train_gpu_cap']!=154800-OLD
|
|
or c['postfit_gpu_reserve']!=7200 or c['gpu_query_timeout_seconds']!=10
|
|
or c['consecutive_failed_samples']!=3 or not c['job_failure_isolated']):raise PermissionError('Exact recovery scope')
|
|
for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h)
|
|
old,_=original_config(ORIGINAL)
|
|
if c['original_contract_sha256']!=sha(ORIGINAL/'contract.json'):raise ValueError('Old contract identity')
|
|
for k in ['parts','geometry','jobs','model','versions','normalization','clock_policy','metric','original_failed','seed','epochs']:
|
|
if c[k]!=old[k]:raise ValueError('Scientific recipe changed: '+k)
|
|
d=read(ORIGINAL/'complete.json')
|
|
if d['status']!='STOPPED' or d['training_gpu_seconds']!=OLD or sum(e['gpu_seconds_charged'] for e in d['entries'])!=OLD:
|
|
raise ValueError('Old cost counted once')
|
|
if c['gpu_cap']-OLD-c['postfit_gpu_reserve']!=c['train_gpu_cap']:raise ValueError('Remaining training ledger')
|
|
return c,c
|
|
|
|
|
|
def reuse_identity(c):
|
|
d=read(ORIGINAL/'complete.json');stop=read(ORIGINAL/'global-stop.json')
|
|
if d['status']!='STOPPED' or stop['reason']!='JOB_4_FAILED':raise ValueError('Original STOP must stay immutable')
|
|
proof=verify(ORIGINAL,'infrastructure-stop');out=[]
|
|
for i in [0,1,2]:
|
|
e=next(e for e in d['entries'] if e['id']==i);w=ORIGINAL/'jobs'/str(i)/'worker'
|
|
g=read(checked(w.parent/'complete.json',e['guard_complete_sha256']));receipt=read(w.parent/'guards/train.receipt.json')
|
|
if e['status']!='PASS' or e['returncode']!=0 or g['status']!='PASS' or g['monitor_errors'] or g['reasons'] or g['worker']!=receipt or receipt['status']!='PASS' or receipt['returncode']!=0 or type(receipt['monitor_errors']) is not int or receipt['monitor_errors']!=0:
|
|
raise ValueError('Only genuine old PASS may be reused')
|
|
r=read(checked(w/'result.json',g['worker_result_sha256']));j=c['jobs'][i]
|
|
if r['fold']!=i or r['job']!=i or r['completed_epochs']!=200 or r['optimizer_steps']!=j['target_steps'] or r['test_targets_read'] or r['test_prediction_started'] or r['validation_used_for_selection']:
|
|
raise ValueError('Old final model roles')
|
|
model=dict(job=i,path=r['checkpoint'],sha256=r['checkpoint_sha256'],standardization=str(w/'standardization.json'),standardization_sha256=r['standardization_sha256'],origin='ORIGINAL_HASH_REUSE',training_contract_sha256=c['original_contract_sha256'],guard_complete_sha256=e['guard_complete_sha256'])
|
|
for p,h in [(model['path'],model['sha256']),(model['standardization'],model['standardization_sha256'])]:
|
|
checked(p,h)
|
|
if proof['files'].get(str(Path(p).resolve()))!=h:raise ValueError('Reuse omitted from immutable STOP freeze')
|
|
stats=read(model['standardization'])
|
|
if stats['parts']!=j['learning'] or stats['nodes']!=j['learning_nodes'] or len(stats['mean'])!=19 or len(stats['scale'])!=19:raise ValueError('Original normalization identity')
|
|
out.append(model)
|
|
if out!=c['reused_models']:raise ValueError('Exact reused model manifest')
|
|
return out
|
|
|
|
|
|
def execution_command(args):
|
|
return any(Path(a).name in {'track_a_h2_queue.py','track_a_h2_guard.py','track_a_h2_worker.py','track_a_recovery_queue.py','track_a_recovery_guard.py','track_a_recovery_worker.py'} for a in args)
|
|
|
|
|
|
def first_epoch_release(root,jobid):
|
|
if jobid not in RECOVERY:raise PermissionError('No reused fold execution')
|
|
if (root/'global-stop.json').exists() or (root/'jobs'/str(jobid)/'stop.json').exists():raise RuntimeError('This recovery fold stopped')
|
|
p=root/'jobs'/str(jobid)/'first-epoch-projection.json'
|
|
if not p.exists():return False
|
|
d=read(p)
|
|
if d['status']!='PASS' or d['contract_sha256']!=sha(root/'contract.json'):raise RuntimeError('Own cost gate rejected')
|
|
return True
|
|
|
|
|
|
def accepted_guard(rc,g):
|
|
if rc!=0 or not g or g['status']!='PASS' or g['reasons'] or g['monitor_policy']['failed']:return False
|
|
if g['monitor_policy']['errors'] != len(g['monitor_errors']):return False
|
|
w=g.get('worker')
|
|
return bool(w and w['status']=='PASS' and w['returncode']==0 and type(w['monitor_errors']) is int
|
|
and not w['monitor_policy']['failed'] and w['monitor_policy']['errors']==w['monitor_errors'])
|
|
|
|
|
|
def project_total(c,estimates,terminal,live,stopping=None):
|
|
stopping=set(stopping or [])
|
|
if (set(estimates)|set(terminal)|set(live)|stopping)-set(RECOVERY):raise ValueError('Unknown recovery fold')
|
|
rows=[];new=c['postfit_gpu_reserve']
|
|
for i in RECOVERY:
|
|
if i in terminal:
|
|
value=terminal[i]['gpu_seconds_charged'];basis='terminal_actual'
|
|
elif i in stopping:
|
|
# No rejected future training remains scheduled. Retain the full
|
|
# originally approved own cap for bounded cleanup until actual exit.
|
|
value=max(live.get(i,0),c['jobs'][i]['gpu_seconds']);basis='locally_stopping_own_cap_or_actual'
|
|
else:
|
|
value=max(live.get(i,0),estimates.get(i,{}).get('projected_gpu_seconds',c['jobs'][i]['gpu_seconds']));basis='continuing_projection_or_pending_cap'
|
|
if not math.isfinite(value) or value<0:raise ValueError('Invalid new charge')
|
|
new+=value;rows.append(dict(id=i,projected_gpu_seconds=value,basis=basis))
|
|
# Per-cell excess is isolated by own stop.json; total exhaustion stays global.
|
|
return dict(status='PASS' if OLD+new<=c['gpu_cap'] else 'STOP_CUMULATIVE_BUDGET',rows=rows,new_projected_gpu_seconds=new,old_gpu_seconds_once=OLD,cumulative_projected_gpu_seconds=OLD+new,postfit_reserve=c['postfit_gpu_reserve'])
|
|
|
|
|
|
def canonical_models(c,new_models):
|
|
ids=[m['job'] for m in new_models]
|
|
if len(ids)!=2 or set(ids)!={3,4}:raise ValueError('No five-fold model adoption with missing recovery fold')
|
|
result=c['reused_models']+new_models
|
|
if len(result)!=5 or {m['job'] for m in result}!=set(range(5)):raise ValueError('Canonical five models')
|
|
return sorted(result,key=lambda m:m['job'])
|