"""Five original grouped folds, new seed1 only; fixed scope and bounded parallel cost gates.""" import math from pathlib import Path from phase2_h1_worker_v2 import checked,sha from phase2_home_freeze_io import verify from track_a_h2_common import read GPUS=['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c','GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36','GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76','GPU-8b71b032-1b75-2891-935c-ce106a53034a'] def execution_command(args): return any(Path(x).name in {'phase3_seed1_queue.py','phase3_seed1_guard.py','phase3_seed1_worker.py','phase3_seed1_continuation_queue.py','phase3_seed1_continuation_guard.py','phase3_seed1_continuation_worker.py'} for x in args) def fold_view(pilot,fold): if type(fold) is not int or fold not in range(5):raise PermissionError('Exactly five original grouped folds') f=pilot['folds'][fold] if f['fold']!=fold:raise ValueError('Actual original fold identity') roles=[set(f[k]) for k in ('learning','validation','test')] if any(roles[i]&roles[j] for i in range(3) for j in range(i)):raise ValueError('Role overlap') return dict(pilot,folds=[f]) def config(root): c=read(root/'contract.json');verify(root,'implementation') if c['seed']!=1 or c['epochs']!=200 or c['new_fits']!=5 or [j['id'] for j in c['jobs']]!=list(range(5)) or not c['no_retry']:raise PermissionError('Seed1 five-fold scope') if c['dispatch_approval']!='1794983024.445549' or c['approval']!='1794984996.938059' or c['accounting_approval']!='1794985731.591689' or c['gpu_cap']!=167200-c['old_seed1_gpu_once'] or c['train_gpu_cap']!=160000-c['old_seed1_gpu_once'] or c['postfit_gpu_reserve']!=7200 or c['phase3_gpu_cap']!=612000:raise PermissionError('Frozen seed1 and phase3 ceilings') for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h) old=Path(c['original_seed1_root']) if str(old)!='/home/frank14f/MeltMind-runs/phase3-seed1-2026-11-18' or root.resolve()!=old.with_name('phase3-seed1-continuation-2026-11-18'):raise PermissionError('Sole labelled continuation root') if c['continuation_number']!=1 or c['fresh_restart_folds']!=[0,1] or c['previously_unattempted_folds']!=[2,3,4]:raise PermissionError('Fixed continuation scope') d=read(old/'complete.json');oc=read(old/'contract.json') if d['status']!='STOPPED' or d['never_attempted']!=[2,3,4] or [e['id'] for e in d['entries']]!=[0,1]:raise PermissionError('Immutable original stop') spent={e['id']:e['gpu_seconds_charged'] for e in d['entries']} if c['old_seed1_gpu_once']!=d['training_gpu_seconds'] or sum(spent.values())!=c['old_seed1_gpu_once']:raise ValueError('Old seed1 charged exactly once') if c['phase3_prior_gpu_once']!=oc['phase3_prior_gpu_once']+c['old_seed1_gpu_once']:raise ValueError('Phase3 original item plus old seed1 once') if c['cumulative_seed1_gpu_cap']!=167200 or c['cumulative_fold_gpu_cap']!=32000:raise ValueError('Final cumulative ceilings') if any(j['gpu_seconds']!=32000-spent.get(j['id'],0) or j['cpu_seconds']!=j['gpu_seconds'] for j in c['jobs']):raise ValueError('Per-fold remaining caps') if c['comparison_contract_sha256']!=oc['comparison_contract_sha256']:raise ValueError('Scientific comparison unchanged') pilot=read(c['pilot_contract']) import numpy as np,torch if np.__version__!=pilot['versions']['numpy'] or torch.__version__!=pilot['versions']['torch']:raise ValueError('Original library versions') if sum(j['gpu_seconds'] for j in c['jobs'])!=c['train_gpu_cap']:raise ValueError('Partitioned training budget') if c['phase3_prior_gpu_once']+c['gpu_cap']>c['phase3_gpu_cap']:raise ValueError('Aggregate phase3 reserve') tests=[] for j in c['jobs']: f=fold_view(pilot,j['id'])['folds'][0];tests.extend(f['test']) for k in ['learning','validation','test','learning_nodes','validation_nodes','clock_path','clock_sha256','c']: if j[k]!=f[k]:raise ValueError('Original roster/clock/geometry changed: '+k) if j['target_steps']!=200*len(f['learning']):raise ValueError('Approved GPU assignment and all epochs') if len(tests)!=552 or len(set(tests))!=552:raise ValueError('Original test partition') return c,pilot def project_cell(job,observation): e=observation['epoch'];train=observation['max_epoch_seconds'];charged=observation['charged'] validation=max(job['validation_seconds_bound'],observation.get('max_validation_seconds',0)) if type(e) is not int or e not in range(1,201) or any(not math.isfinite(x) or x<0 for x in [train,charged,validation]) or train<=0:raise ValueError('Measured epoch timing') future=1.25*((200-e)*train+(20-e//10)*validation)+120 value=charged+future return dict(id=job['id'],projected_gpu_seconds=value,validation_seconds_bound=validation,within_cap=value<=job['gpu_seconds']) def project_total(c,estimates,terminal,active_charges=None): active_charges=active_charges or {};ids={j['id'] for j in c['jobs']} if (set(estimates)|set(terminal)|set(active_charges))-ids:raise ValueError('Unknown job') rows=[];total=c['postfit_gpu_reserve'] for j in c['jobs']: i=j['id'];v=terminal[i]['gpu_seconds_charged'] if i in terminal else max(active_charges.get(i,0),estimates.get(i,{}).get('projected_gpu_seconds',j['gpu_seconds'])) if not math.isfinite(v) or v<0:raise ValueError('Invalid ledger') total+=v;rows.append(dict(id=i,projected_gpu_seconds=v,within_cap=v<=j['gpu_seconds'])) ok=total<=c['gpu_cap'] and c['phase3_prior_gpu_once']+total<=c['phase3_gpu_cap'] and all(r['within_cap'] for r in rows) return dict(status='PASS' if ok else 'STOP_BUDGET_PROJECTION',rows=rows,projected_gpu_seconds=total,phase3_prior_gpu_once=c['phase3_prior_gpu_once'],projected_phase3_gpu_seconds=c['phase3_prior_gpu_once']+total,postfit_reserve=c['postfit_gpu_reserve']) def first_epoch_release(root,jobid): if (root/'global-stop.json').exists():raise RuntimeError('Seed1 global stop') p=root/'jobs'/str(jobid)/'first-epoch-projection.json' if not p.exists():return False r=read(p) if r['status']!='PASS' or r['contract_sha256']!=sha(root/'contract.json'):raise RuntimeError('Cost gate rejected') return True def dispatchable(pending,active,available): occupied={j['gpu_uuid'] for j in active} if len(occupied)!=len(active) or len(active)>4 or not occupied<=set(GPUS) or not set(available)<=set(GPUS):raise ValueError('GPU collision or unknown GPU') free=[u for u in GPUS if u in available and u not in occupied] return [dict(j,gpu_uuid=u) for j,u in zip(pending,free)] def idle_eligible_gpus(eligible,query=None): if not set(eligible)<=set(GPUS):raise ValueError('Unknown eligible GPU') if query is None: from s2_precision_continuation_guard import device_usage query=device_usage idle=set();observations=[] for u in GPUS: if u not in eligible:continue mem,util=query(u) if not all(math.isfinite(x) and x>=0 for x in [mem,util]):raise ValueError('Invalid device occupancy') ok=mem<=128*2**20 and util==0 observations.append(dict(gpu_uuid=u,memory_bytes=mem,utilization=util,idle=ok)) if ok:idle.add(u) return idle,observations def assigned_job(root,c,index,digest): p=checked(root/'jobs'/str(index)/'assignment.json',digest);a=read(p) if a['id']!=index or a['contract_sha256']!=sha(root/'contract.json') or a['gpu_uuid'] not in GPUS:raise ValueError('Actual assignment identity') return dict(c['jobs'][index],gpu_uuid=a['gpu_uuid']) def available_gpus(c): import psutil available={GPUS[0]} item=Path(c['item1_recovery_root']);track=Path(c['track_a_training_root']) names={Path(a).name for p in psutil.process_iter(['cmdline']) for a in p.info['cmdline'] or []} if (item/'complete.json').exists() and not names & {'phase3_item1_postfit_worker.py','phase3_item1_resume.py'}: available.add(GPUS[1]) if (track/'complete.json').exists() and not names & {'track_a_h2_queue.py','track_a_h2_guard.py','track_a_h2_worker.py'}: # GPU2 stays reserved for the already-prioritized Track A postfit until its terminal. post=Path(c['track_a_postfit_root']) available.add(GPUS[3]) if (post/'complete.json').exists() and not names & {'track_a_postfit.py','track_a_postfit_worker.py'}: available.add(GPUS[2]) return available