148 lines
9.9 KiB
Python
148 lines
9.9 KiB
Python
"""Five original grouped folds, new seed1 only; fixed scope and bounded parallel cost gates."""
|
|
import math
|
|
import statistics
|
|
from pathlib import Path
|
|
from phase2_h1_worker_v2 import checked,sha
|
|
from phase2_home_freeze_io import verify
|
|
from track_a_h2_common import read
|
|
|
|
GPUS=['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c','GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36','GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76','GPU-8b71b032-1b75-2891-935c-ce106a53034a']
|
|
|
|
|
|
def execution_command(args):
|
|
return any(Path(x).name in {'phase3_seed1_queue.py','phase3_seed1_guard.py','phase3_seed1_worker.py','phase3_seed1_continuation_queue.py','phase3_seed1_continuation_guard.py','phase3_seed1_continuation_worker.py','phase3_seed1_v2_queue.py','phase3_seed1_v2_guard.py','phase3_seed1_v2_worker.py'} for x in args)
|
|
|
|
|
|
def fold_view(pilot,fold):
|
|
if type(fold) is not int or fold not in range(5):raise PermissionError('Exactly five original grouped folds')
|
|
f=pilot['folds'][fold]
|
|
if f['fold']!=fold:raise ValueError('Actual original fold identity')
|
|
roles=[set(f[k]) for k in ('learning','validation','test')]
|
|
if any(roles[i]&roles[j] for i in range(3) for j in range(i)):raise ValueError('Role overlap')
|
|
return dict(pilot,folds=[f])
|
|
|
|
|
|
def config(root):
|
|
c=read(root/'contract.json');verify(root,'implementation')
|
|
if c['seed']!=1 or c['epochs']!=200 or c['new_fits']!=5 or [j['id'] for j in c['jobs']]!=list(range(5)) or not c['no_retry']:raise PermissionError('Seed1 five-fold scope')
|
|
if c['approval']!='1794990750.013949' or c['accounting_approval']!='1794990899.609759' or c['semantic_approval']!='1794990914.918719' or c['dispatch_approval']!='1794983024.445549':raise PermissionError('Exact v2 rulings')
|
|
for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h)
|
|
old=Path('/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18');prior=old.with_name('phase3-seed1-continuation-2026-11-18')
|
|
if root.resolve()!=old.with_name('phase3-seed1-v2-2026-11-18'):raise PermissionError('Sole new labelled v2 root')
|
|
if c['continuation_number']!=2 or not c['no_further_continuation'] or c['fresh_restart_folds']!=[0,1] or c['previously_unattempted_folds']!=[2,3,4]:raise PermissionError('Fixed v2 scope')
|
|
costs=[]
|
|
for p in [old,prior]:
|
|
d=read(p/'complete.json')
|
|
if d['status']!='STOPPED' or d['never_attempted']!=[2,3,4] or sorted(e['id'] for e in d['entries'])!=[0,1]:raise PermissionError('Both immutable old stops')
|
|
if sum(e['gpu_seconds_charged'] for e in d['entries'])!=d['training_gpu_seconds']:raise ValueError('Historical ledger mismatch')
|
|
costs.append(dict(root=str(p),gpu_seconds=d['training_gpu_seconds']))
|
|
old_spent=sum(d['gpu_seconds'] for d in costs);oc=read(old/'contract.json')
|
|
if c['old_attempts']!=costs or c['old_seed1_gpu_once']!=old_spent or c['phase3_prior_gpu_once']!=oc['phase3_prior_gpu_once']+old_spent:raise ValueError('Both old charges exactly once')
|
|
if c['gpu_cap']!=167200-old_spent or c['train_gpu_cap']!=160000-old_spent or c['postfit_gpu_reserve']!=7200 or c['phase3_gpu_cap']!=612000 or c['cumulative_seed1_gpu_cap']!=167200:raise ValueError('Unchanged cumulative ceiling')
|
|
if c['pending_rule']!='median_current_started_projections_else_train_remaining_div5' or c['fold_stop_rule']!='actual_charged_runaway_only' or c['runaway_gpu_seconds']!=40000:raise ValueError('Frozen total-projection rule')
|
|
if any(j['gpu_seconds']!=40000 or j['cpu_seconds']!=40000 for j in c['jobs']):raise ValueError('Actual per-fold runaway guard')
|
|
if c['comparison_contract_sha256']!=oc['comparison_contract_sha256']:raise ValueError('Scientific comparison unchanged')
|
|
pilot=read(c['pilot_contract'])
|
|
import numpy as np,torch
|
|
if np.__version__!=pilot['versions']['numpy'] or torch.__version__!=pilot['versions']['torch']:raise ValueError('Original library versions')
|
|
if c['phase3_prior_gpu_once']+c['gpu_cap']>c['phase3_gpu_cap']:raise ValueError('Aggregate phase3 reserve')
|
|
tests=[]
|
|
for j in c['jobs']:
|
|
f=fold_view(pilot,j['id'])['folds'][0];tests.extend(f['test'])
|
|
for k in ['learning','validation','test','learning_nodes','validation_nodes','clock_path','clock_sha256','c']:
|
|
if j[k]!=f[k]:raise ValueError('Original roster/clock/geometry changed: '+k)
|
|
if j['target_steps']!=200*len(f['learning']):raise ValueError('Approved all epochs')
|
|
if len(tests)!=552 or len(set(tests))!=552:raise ValueError('Original test partition')
|
|
return c,pilot
|
|
|
|
|
|
def project_cell(job,observation):
|
|
e=observation['epoch'];train=observation['max_epoch_seconds'];charged=observation['charged']
|
|
validation=max(job['validation_seconds_bound'],observation.get('max_validation_seconds',0))
|
|
if type(e) is not int or e not in range(1,201) or any(not math.isfinite(x) or x<0 for x in [train,charged,validation]) or train<=0:raise ValueError('Measured epoch timing')
|
|
future=1.25*((200-e)*train+(20-e//10)*validation)+120
|
|
value=charged+future
|
|
return dict(id=job['id'],projected_gpu_seconds=value,validation_seconds_bound=validation,actual_charged_gpu_seconds=charged,actual_within_runaway=charged<=job['gpu_seconds'],projected_above_runaway=value>job['gpu_seconds'])
|
|
|
|
|
|
def project_total(c,estimates,terminal,active_charges=None):
|
|
active_charges=active_charges or {};ids={j['id'] for j in c['jobs']}
|
|
if (set(estimates)|set(terminal)|set(active_charges))-ids:raise ValueError('Unknown job')
|
|
def valid(v):
|
|
if not math.isfinite(v) or v<0:raise ValueError('Invalid ledger')
|
|
return v
|
|
values={}
|
|
for i in ids:
|
|
if i in terminal:values[i]=valid(terminal[i]['gpu_seconds_charged'])
|
|
elif i in estimates:values[i]=max(valid(active_charges.get(i,estimates[i].get('actual_charged_gpu_seconds',0))),valid(estimates[i]['projected_gpu_seconds']))
|
|
# Measured started folds (including finished actuals) determine the median.
|
|
# An active fold before its first measured epoch is conservatively reserved at
|
|
# the larger of the initial equal share and this current measured median.
|
|
initial=c['train_gpu_cap']/len(c['jobs'])
|
|
median=statistics.median(values.values()) if values else initial
|
|
for i,charged in active_charges.items():
|
|
if i not in values:values[i]=max(valid(charged),initial,median)
|
|
pending_reserve=statistics.median(values.values()) if values else initial
|
|
rows=[];total=c['postfit_gpu_reserve'];actuals=[]
|
|
for j in c['jobs']:
|
|
i=j['id'];v=values.get(i,pending_reserve)
|
|
actual=valid(terminal[i]['gpu_seconds_charged'] if i in terminal else active_charges.get(i,estimates.get(i,{}).get('actual_charged_gpu_seconds',0)))
|
|
total+=v;actuals.append(actual)
|
|
rows.append(dict(id=i,projected_gpu_seconds=v,actual_charged_gpu_seconds=actual,actual_within_runaway=actual<=c['runaway_gpu_seconds'],basis='terminal_actual' if i in terminal else 'measured_projection' if i in estimates else 'active_unmeasured_reserve' if i in active_charges else 'pending_median_reserve'))
|
|
ok=total<=c['gpu_cap'] and c['phase3_prior_gpu_once']+total<=c['phase3_gpu_cap'] and all(r['actual_within_runaway'] for r in rows)
|
|
return dict(status='PASS' if ok else 'STOP_BUDGET_PROJECTION',rows=rows,projected_gpu_seconds=total,phase3_prior_gpu_once=c['phase3_prior_gpu_once'],projected_phase3_gpu_seconds=c['phase3_prior_gpu_once']+total,postfit_reserve=c['postfit_gpu_reserve'],pending_reserve_gpu_seconds=pending_reserve,actual_runaway_gpu_seconds=c['runaway_gpu_seconds'],total_projection_within_cap=total<=c['gpu_cap'])
|
|
|
|
|
|
def first_epoch_release(root,jobid):
|
|
if (root/'global-stop.json').exists():raise RuntimeError('Seed1 global stop')
|
|
p=root/'jobs'/str(jobid)/'first-epoch-projection.json'
|
|
if not p.exists():return False
|
|
r=read(p)
|
|
if r['status']!='PASS' or r['contract_sha256']!=sha(root/'contract.json'):raise RuntimeError('Cost gate rejected')
|
|
return True
|
|
|
|
|
|
def dispatchable(pending,active,available):
|
|
occupied={j['gpu_uuid'] for j in active}
|
|
if len(occupied)!=len(active) or len(active)>4 or not occupied<=set(GPUS) or not set(available)<=set(GPUS):raise ValueError('GPU collision or unknown GPU')
|
|
free=[u for u in GPUS if u in available and u not in occupied]
|
|
return [dict(j,gpu_uuid=u) for j,u in zip(pending,free)]
|
|
|
|
|
|
def idle_eligible_gpus(eligible,query=None):
|
|
if not set(eligible)<=set(GPUS):raise ValueError('Unknown eligible GPU')
|
|
if query is None:
|
|
from s2_precision_continuation_guard import device_usage
|
|
query=device_usage
|
|
idle=set();observations=[]
|
|
for u in GPUS:
|
|
if u not in eligible:continue
|
|
mem,util=query(u)
|
|
if not all(math.isfinite(x) and x>=0 for x in [mem,util]):raise ValueError('Invalid device occupancy')
|
|
ok=mem<=128*2**20 and util==0
|
|
observations.append(dict(gpu_uuid=u,memory_bytes=mem,utilization=util,idle=ok))
|
|
if ok:idle.add(u)
|
|
return idle,observations
|
|
|
|
|
|
def assigned_job(root,c,index,digest):
|
|
p=checked(root/'jobs'/str(index)/'assignment.json',digest);a=read(p)
|
|
if a['id']!=index or a['contract_sha256']!=sha(root/'contract.json') or a['gpu_uuid'] not in GPUS:raise ValueError('Actual assignment identity')
|
|
return dict(c['jobs'][index],gpu_uuid=a['gpu_uuid'])
|
|
|
|
|
|
def available_gpus(c):
|
|
import psutil
|
|
available={GPUS[0]}
|
|
item=Path(c['item1_recovery_root']);track=Path(c['track_a_training_root'])
|
|
names={Path(a).name for p in psutil.process_iter(['cmdline']) for a in p.info['cmdline'] or []}
|
|
if (item/'complete.json').exists() and not names & {'phase3_item1_postfit_worker.py','phase3_item1_resume.py'}:
|
|
available.add(GPUS[1])
|
|
if (track/'complete.json').exists() and not names & {'track_a_h2_queue.py','track_a_h2_guard.py','track_a_h2_worker.py'}:
|
|
# GPU2 stays reserved for the already-prioritized Track A postfit until its terminal.
|
|
post=Path(c['track_a_postfit_root'])
|
|
available.add(GPUS[3])
|
|
if (post/'complete.json').exists() and not names & {'track_a_postfit.py','track_a_postfit_worker.py'}:
|
|
available.add(GPUS[2])
|
|
return available
|