"""Five original grouped folds, new seed1 only; fixed scope and bounded parallel cost gates.""" import math import statistics from pathlib import Path from phase2_h1_worker_v2 import checked,sha from phase2_home_freeze_io import verify from track_a_h2_common import read GPUS=['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c','GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36','GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76','GPU-8b71b032-1b75-2891-935c-ce106a53034a'] def execution_command(args): return any(Path(x).name in {'phase3_seed1_queue.py','phase3_seed1_guard.py','phase3_seed1_worker.py','phase3_seed1_continuation_queue.py','phase3_seed1_continuation_guard.py','phase3_seed1_continuation_worker.py','phase3_seed1_v2_queue.py','phase3_seed1_v2_guard.py','phase3_seed1_v2_worker.py'} for x in args) def fold_view(pilot,fold): if type(fold) is not int or fold not in range(5):raise PermissionError('Exactly five original grouped folds') f=pilot['folds'][fold] if f['fold']!=fold:raise ValueError('Actual original fold identity') roles=[set(f[k]) for k in ('learning','validation','test')] if any(roles[i]&roles[j] for i in range(3) for j in range(i)):raise ValueError('Role overlap') return dict(pilot,folds=[f]) def config(root): c=read(root/'contract.json');verify(root,'implementation') if c['seed']!=1 or c['epochs']!=200 or c['new_fits']!=5 or [j['id'] for j in c['jobs']]!=list(range(5)) or not c['no_retry']:raise PermissionError('Seed1 five-fold scope') if c['approval']!='1794990750.013949' or c['accounting_approval']!='1794990899.609759' or c['semantic_approval']!='1794990914.918719' or c['dispatch_approval']!='1794983024.445549':raise PermissionError('Exact v2 rulings') for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h) old=Path('/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18');prior=old.with_name('phase3-seed1-continuation-2026-11-18') if root.resolve()!=old.with_name('phase3-seed1-v2-2026-11-18'):raise PermissionError('Sole new labelled v2 root') if c['continuation_number']!=2 or not c['no_further_continuation'] or c['fresh_restart_folds']!=[0,1] or c['previously_unattempted_folds']!=[2,3,4]:raise PermissionError('Fixed v2 scope') costs=[] for p in [old,prior]: d=read(p/'complete.json') if d['status']!='STOPPED' or d['never_attempted']!=[2,3,4] or sorted(e['id'] for e in d['entries'])!=[0,1]:raise PermissionError('Both immutable old stops') if sum(e['gpu_seconds_charged'] for e in d['entries'])!=d['training_gpu_seconds']:raise ValueError('Historical ledger mismatch') costs.append(dict(root=str(p),gpu_seconds=d['training_gpu_seconds'])) old_spent=sum(d['gpu_seconds'] for d in costs);oc=read(old/'contract.json') if c['old_attempts']!=costs or c['old_seed1_gpu_once']!=old_spent or c['phase3_prior_gpu_once']!=oc['phase3_prior_gpu_once']+old_spent:raise ValueError('Both old charges exactly once') if c['gpu_cap']!=167200-old_spent or c['train_gpu_cap']!=160000-old_spent or c['postfit_gpu_reserve']!=7200 or c['phase3_gpu_cap']!=612000 or c['cumulative_seed1_gpu_cap']!=167200:raise ValueError('Unchanged cumulative ceiling') if c['pending_rule']!='median_current_started_projections_else_train_remaining_div5' or c['fold_stop_rule']!='actual_charged_runaway_only' or c['runaway_gpu_seconds']!=40000:raise ValueError('Frozen total-projection rule') if any(j['gpu_seconds']!=40000 or j['cpu_seconds']!=40000 for j in c['jobs']):raise ValueError('Actual per-fold runaway guard') if c['comparison_contract_sha256']!=oc['comparison_contract_sha256']:raise ValueError('Scientific comparison unchanged') pilot=read(c['pilot_contract']) import numpy as np,torch if np.__version__!=pilot['versions']['numpy'] or torch.__version__!=pilot['versions']['torch']:raise ValueError('Original library versions') if c['phase3_prior_gpu_once']+c['gpu_cap']>c['phase3_gpu_cap']:raise ValueError('Aggregate phase3 reserve') tests=[] for j in c['jobs']: f=fold_view(pilot,j['id'])['folds'][0];tests.extend(f['test']) for k in ['learning','validation','test','learning_nodes','validation_nodes','clock_path','clock_sha256','c']: if j[k]!=f[k]:raise ValueError('Original roster/clock/geometry changed: '+k) if j['target_steps']!=200*len(f['learning']):raise ValueError('Approved all epochs') if len(tests)!=552 or len(set(tests))!=552:raise ValueError('Original test partition') return c,pilot def project_cell(job,observation): e=observation['epoch'];train=observation['max_epoch_seconds'];charged=observation['charged'] validation=max(job['validation_seconds_bound'],observation.get('max_validation_seconds',0)) if type(e) is not int or e not in range(1,201) or any(not math.isfinite(x) or x<0 for x in [train,charged,validation]) or train<=0:raise ValueError('Measured epoch timing') future=1.25*((200-e)*train+(20-e//10)*validation)+120 value=charged+future return dict(id=job['id'],projected_gpu_seconds=value,validation_seconds_bound=validation,actual_charged_gpu_seconds=charged,actual_within_runaway=charged<=job['gpu_seconds'],projected_above_runaway=value>job['gpu_seconds']) def project_total(c,estimates,terminal,active_charges=None): active_charges=active_charges or {};ids={j['id'] for j in c['jobs']} if (set(estimates)|set(terminal)|set(active_charges))-ids:raise ValueError('Unknown job') def valid(v): if not math.isfinite(v) or v<0:raise ValueError('Invalid ledger') return v values={} for i in ids: if i in terminal:values[i]=valid(terminal[i]['gpu_seconds_charged']) elif i in estimates:values[i]=max(valid(active_charges.get(i,estimates[i].get('actual_charged_gpu_seconds',0))),valid(estimates[i]['projected_gpu_seconds'])) # Measured started folds (including finished actuals) determine the median. # An active fold before its first measured epoch is conservatively reserved at # the larger of the initial equal share and this current measured median. initial=c['train_gpu_cap']/len(c['jobs']) median=statistics.median(values.values()) if values else initial for i,charged in active_charges.items(): if i not in values:values[i]=max(valid(charged),initial,median) pending_reserve=statistics.median(values.values()) if values else initial rows=[];total=c['postfit_gpu_reserve'];actuals=[] for j in c['jobs']: i=j['id'];v=values.get(i,pending_reserve) actual=valid(terminal[i]['gpu_seconds_charged'] if i in terminal else active_charges.get(i,estimates.get(i,{}).get('actual_charged_gpu_seconds',0))) total+=v;actuals.append(actual) rows.append(dict(id=i,projected_gpu_seconds=v,actual_charged_gpu_seconds=actual,actual_within_runaway=actual<=c['runaway_gpu_seconds'],basis='terminal_actual' if i in terminal else 'measured_projection' if i in estimates else 'active_unmeasured_reserve' if i in active_charges else 'pending_median_reserve')) ok=total<=c['gpu_cap'] and c['phase3_prior_gpu_once']+total<=c['phase3_gpu_cap'] and all(r['actual_within_runaway'] for r in rows) return dict(status='PASS' if ok else 'STOP_BUDGET_PROJECTION',rows=rows,projected_gpu_seconds=total,phase3_prior_gpu_once=c['phase3_prior_gpu_once'],projected_phase3_gpu_seconds=c['phase3_prior_gpu_once']+total,postfit_reserve=c['postfit_gpu_reserve'],pending_reserve_gpu_seconds=pending_reserve,actual_runaway_gpu_seconds=c['runaway_gpu_seconds'],total_projection_within_cap=total<=c['gpu_cap']) def first_epoch_release(root,jobid): if (root/'global-stop.json').exists():raise RuntimeError('Seed1 global stop') p=root/'jobs'/str(jobid)/'first-epoch-projection.json' if not p.exists():return False r=read(p) if r['status']!='PASS' or r['contract_sha256']!=sha(root/'contract.json'):raise RuntimeError('Cost gate rejected') return True def dispatchable(pending,active,available): occupied={j['gpu_uuid'] for j in active} if len(occupied)!=len(active) or len(active)>4 or not occupied<=set(GPUS) or not set(available)<=set(GPUS):raise ValueError('GPU collision or unknown GPU') free=[u for u in GPUS if u in available and u not in occupied] return [dict(j,gpu_uuid=u) for j,u in zip(pending,free)] def idle_eligible_gpus(eligible,query=None): if not set(eligible)<=set(GPUS):raise ValueError('Unknown eligible GPU') if query is None: from s2_precision_continuation_guard import device_usage query=device_usage idle=set();observations=[] for u in GPUS: if u not in eligible:continue mem,util=query(u) if not all(math.isfinite(x) and x>=0 for x in [mem,util]):raise ValueError('Invalid device occupancy') ok=mem<=128*2**20 and util==0 observations.append(dict(gpu_uuid=u,memory_bytes=mem,utilization=util,idle=ok)) if ok:idle.add(u) return idle,observations def assigned_job(root,c,index,digest): p=checked(root/'jobs'/str(index)/'assignment.json',digest);a=read(p) if a['id']!=index or a['contract_sha256']!=sha(root/'contract.json') or a['gpu_uuid'] not in GPUS:raise ValueError('Actual assignment identity') return dict(c['jobs'][index],gpu_uuid=a['gpu_uuid']) def available_gpus(c): import psutil available={GPUS[0]} item=Path(c['item1_recovery_root']);track=Path(c['track_a_training_root']) names={Path(a).name for p in psutil.process_iter(['cmdline']) for a in p.info['cmdline'] or []} if (item/'complete.json').exists() and not names & {'phase3_item1_postfit_worker.py','phase3_item1_resume.py'}: available.add(GPUS[1]) if (track/'complete.json').exists() and not names & {'track_a_h2_queue.py','track_a_h2_guard.py','track_a_h2_worker.py'}: # GPU2 stays reserved for the already-prioritized Track A postfit until its terminal. post=Path(c['track_a_postfit_root']) available.add(GPUS[3]) if (post/'complete.json').exists() and not names & {'track_a_postfit.py','track_a_postfit_worker.py'}: available.add(GPUS[2]) return available