Files
MeltMind/tools/phase3_seed1_continuation_common.py
T

128 lines
8.2 KiB
Python

"""Five original grouped folds, new seed1 only; fixed scope and bounded parallel cost gates."""
import math
from pathlib import Path
from phase2_h1_worker_v2 import checked,sha
from phase2_home_freeze_io import verify
from track_a_h2_common import read
GPUS=['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c','GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36','GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76','GPU-8b71b032-1b75-2891-935c-ce106a53034a']
def execution_command(args):
return any(Path(x).name in {'phase3_seed1_queue.py','phase3_seed1_guard.py','phase3_seed1_worker.py','phase3_seed1_continuation_queue.py','phase3_seed1_continuation_guard.py','phase3_seed1_continuation_worker.py'} for x in args)
def fold_view(pilot,fold):
if type(fold) is not int or fold not in range(5):raise PermissionError('Exactly five original grouped folds')
f=pilot['folds'][fold]
if f['fold']!=fold:raise ValueError('Actual original fold identity')
roles=[set(f[k]) for k in ('learning','validation','test')]
if any(roles[i]&roles[j] for i in range(3) for j in range(i)):raise ValueError('Role overlap')
return dict(pilot,folds=[f])
def config(root):
c=read(root/'contract.json');verify(root,'implementation')
if c['seed']!=1 or c['epochs']!=200 or c['new_fits']!=5 or [j['id'] for j in c['jobs']]!=list(range(5)) or not c['no_retry']:raise PermissionError('Seed1 five-fold scope')
if c['dispatch_approval']!='1794983024.445549' or c['approval']!='1794984996.938059' or c['accounting_approval']!='1794985731.591689' or c['gpu_cap']!=167200-c['old_seed1_gpu_once'] or c['train_gpu_cap']!=160000-c['old_seed1_gpu_once'] or c['postfit_gpu_reserve']!=7200 or c['phase3_gpu_cap']!=612000:raise PermissionError('Frozen seed1 and phase3 ceilings')
for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h)
old=Path(c['original_seed1_root'])
if str(old)!='/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18' or root.resolve()!=old.with_name('phase3-seed1-continuation-2026-11-18'):raise PermissionError('Sole labelled continuation root')
if c['continuation_number']!=1 or c['fresh_restart_folds']!=[0,1] or c['previously_unattempted_folds']!=[2,3,4]:raise PermissionError('Fixed continuation scope')
d=read(old/'complete.json');oc=read(old/'contract.json')
if d['status']!='STOPPED' or d['never_attempted']!=[2,3,4] or [e['id'] for e in d['entries']]!=[0,1]:raise PermissionError('Immutable original stop')
spent={e['id']:e['gpu_seconds_charged'] for e in d['entries']}
if c['old_seed1_gpu_once']!=d['training_gpu_seconds'] or sum(spent.values())!=c['old_seed1_gpu_once']:raise ValueError('Old seed1 charged exactly once')
if c['phase3_prior_gpu_once']!=oc['phase3_prior_gpu_once']+c['old_seed1_gpu_once']:raise ValueError('Phase3 original item plus old seed1 once')
if c['cumulative_seed1_gpu_cap']!=167200 or c['cumulative_fold_gpu_cap']!=32000:raise ValueError('Final cumulative ceilings')
if any(j['gpu_seconds']!=32000-spent.get(j['id'],0) or j['cpu_seconds']!=j['gpu_seconds'] for j in c['jobs']):raise ValueError('Per-fold remaining caps')
if c['comparison_contract_sha256']!=oc['comparison_contract_sha256']:raise ValueError('Scientific comparison unchanged')
pilot=read(c['pilot_contract'])
import numpy as np,torch
if np.__version__!=pilot['versions']['numpy'] or torch.__version__!=pilot['versions']['torch']:raise ValueError('Original library versions')
if sum(j['gpu_seconds'] for j in c['jobs'])!=c['train_gpu_cap']:raise ValueError('Partitioned training budget')
if c['phase3_prior_gpu_once']+c['gpu_cap']>c['phase3_gpu_cap']:raise ValueError('Aggregate phase3 reserve')
tests=[]
for j in c['jobs']:
f=fold_view(pilot,j['id'])['folds'][0];tests.extend(f['test'])
for k in ['learning','validation','test','learning_nodes','validation_nodes','clock_path','clock_sha256','c']:
if j[k]!=f[k]:raise ValueError('Original roster/clock/geometry changed: '+k)
if j['target_steps']!=200*len(f['learning']):raise ValueError('Approved GPU assignment and all epochs')
if len(tests)!=552 or len(set(tests))!=552:raise ValueError('Original test partition')
return c,pilot
def project_cell(job,observation):
e=observation['epoch'];train=observation['max_epoch_seconds'];charged=observation['charged']
validation=max(job['validation_seconds_bound'],observation.get('max_validation_seconds',0))
if type(e) is not int or e not in range(1,201) or any(not math.isfinite(x) or x<0 for x in [train,charged,validation]) or train<=0:raise ValueError('Measured epoch timing')
future=1.25*((200-e)*train+(20-e//10)*validation)+120
value=charged+future
return dict(id=job['id'],projected_gpu_seconds=value,validation_seconds_bound=validation,within_cap=value<=job['gpu_seconds'])
def project_total(c,estimates,terminal,active_charges=None):
active_charges=active_charges or {};ids={j['id'] for j in c['jobs']}
if (set(estimates)|set(terminal)|set(active_charges))-ids:raise ValueError('Unknown job')
rows=[];total=c['postfit_gpu_reserve']
for j in c['jobs']:
i=j['id'];v=terminal[i]['gpu_seconds_charged'] if i in terminal else max(active_charges.get(i,0),estimates.get(i,{}).get('projected_gpu_seconds',j['gpu_seconds']))
if not math.isfinite(v) or v<0:raise ValueError('Invalid ledger')
total+=v;rows.append(dict(id=i,projected_gpu_seconds=v,within_cap=v<=j['gpu_seconds']))
ok=total<=c['gpu_cap'] and c['phase3_prior_gpu_once']+total<=c['phase3_gpu_cap'] and all(r['within_cap'] for r in rows)
return dict(status='PASS' if ok else 'STOP_BUDGET_PROJECTION',rows=rows,projected_gpu_seconds=total,phase3_prior_gpu_once=c['phase3_prior_gpu_once'],projected_phase3_gpu_seconds=c['phase3_prior_gpu_once']+total,postfit_reserve=c['postfit_gpu_reserve'])
def first_epoch_release(root,jobid):
if (root/'global-stop.json').exists():raise RuntimeError('Seed1 global stop')
p=root/'jobs'/str(jobid)/'first-epoch-projection.json'
if not p.exists():return False
r=read(p)
if r['status']!='PASS' or r['contract_sha256']!=sha(root/'contract.json'):raise RuntimeError('Cost gate rejected')
return True
def dispatchable(pending,active,available):
occupied={j['gpu_uuid'] for j in active}
if len(occupied)!=len(active) or len(active)>4 or not occupied<=set(GPUS) or not set(available)<=set(GPUS):raise ValueError('GPU collision or unknown GPU')
free=[u for u in GPUS if u in available and u not in occupied]
return [dict(j,gpu_uuid=u) for j,u in zip(pending,free)]
def idle_eligible_gpus(eligible,query=None):
if not set(eligible)<=set(GPUS):raise ValueError('Unknown eligible GPU')
if query is None:
from s2_precision_continuation_guard import device_usage
query=device_usage
idle=set();observations=[]
for u in GPUS:
if u not in eligible:continue
mem,util=query(u)
if not all(math.isfinite(x) and x>=0 for x in [mem,util]):raise ValueError('Invalid device occupancy')
ok=mem<=128*2**20 and util==0
observations.append(dict(gpu_uuid=u,memory_bytes=mem,utilization=util,idle=ok))
if ok:idle.add(u)
return idle,observations
def assigned_job(root,c,index,digest):
p=checked(root/'jobs'/str(index)/'assignment.json',digest);a=read(p)
if a['id']!=index or a['contract_sha256']!=sha(root/'contract.json') or a['gpu_uuid'] not in GPUS:raise ValueError('Actual assignment identity')
return dict(c['jobs'][index],gpu_uuid=a['gpu_uuid'])
def available_gpus(c):
import psutil
available={GPUS[0]}
item=Path(c['item1_recovery_root']);track=Path(c['track_a_training_root'])
names={Path(a).name for p in psutil.process_iter(['cmdline']) for a in p.info['cmdline'] or []}
if (item/'complete.json').exists() and not names & {'phase3_item1_postfit_worker.py','phase3_item1_resume.py'}:
available.add(GPUS[1])
if (track/'complete.json').exists() and not names & {'track_a_h2_queue.py','track_a_h2_guard.py','track_a_h2_worker.py'}:
# GPU2 stays reserved for the already-prioritized Track A postfit until its terminal.
post=Path(c['track_a_postfit_root'])
available.add(GPUS[3])
if (post/'complete.json').exists() and not names & {'track_a_postfit.py','track_a_postfit_worker.py'}:
available.add(GPUS[2])
return available