Files
MeltMind/tools/phase3_data_sufficiency_common.py
T

59 lines
4.0 KiB
Python

"""Item1 frozen roles, schedules and costs. No model execution or target access."""
import json,math
from pathlib import Path
from phase2_fixed_io import BASE
from phase2_h1_worker_v2 import checked,sha
from phase2_home_freeze_io import verify
def read(p):return json.loads(Path(p).read_text())
def lr_step(step,total=64000,warmup=3200):
if not 1<=step<=total or not 1<warmup<total:raise ValueError('1-based fixed update domain')
if step<=warmup:return 1e-4+(step-1)*9e-4/(warmup-1)
return 1e-5+(1e-3-1e-5)*(1+math.cos(math.pi*(step-warmup)/(total-warmup)))/2
def execution_command(args):
return any(Path(x).name in {'phase3_data_sufficiency_queue.py','phase3_data_sufficiency_guard.py','phase3_data_sufficiency_worker.py'} for x in args)
def config(root):
c=read(root/'contract.json');verify(root,'implementation')
if c['approval']!='1794946961.453149' or c['new_fits']!=4 or c['item_gpu_cap']!=86400 or c['phase3_gpu_cap']!=612000 or c['train_gpu_cap']+c['postfit_gpu_reserve']!=86400:raise PermissionError('Frozen item1 scope')
if c['U']!=32000 or c['delta']!=.001 or c['seed']!=0 or c['fold']!=0 or not c['no_retry']:raise ValueError('Frozen science identity')
for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h)
pilot=read(c['pilot_contract'])
import torch,numpy as np
if torch.__version__!=pilot['versions']['torch'] or np.__version__!=pilot['versions']['numpy']:raise ValueError('Original library identity')
if len({j['gpu_uuid'] for j in c['jobs']})!=4 or sum(j['gpu_seconds'] for j in c['jobs'])!=c['train_gpu_cap']:raise ValueError('Job resources')
pools=c['pools'];sets=[set(pools[str(p)]['learning']) for p in [25,50,100]]
if not sets[0]<sets[1]<sets[2] or sets[2]!=set(pilot['folds'][0]['learning']):raise ValueError('Original nested roles')
for p in [25,50,100]:
q=pools[str(p)]
if set(q['learning'])&(set(q['validation'])|set(q['test'])) or q['validation']!=pilot['folds'][0]['validation'] or q['test']!=pilot['folds'][0]['test']:raise ValueError('Role separation')
if read(q['clock_path'])['parts']!=q['learning'] or q['learning_nodes']!=sum(pilot['parts'][x]['nodes'] for x in q['learning']):raise ValueError('Learning clock/node identity')
for j in c['jobs']:
q=pools[str(j['pool'])];target=200*len(q['learning']) if j['mode']=='epoch' else 2*c['U']
if j['target_steps']!=target or j['learning_parts']!=len(q['learning']) or j['learning_nodes']!=q['learning_nodes']:raise ValueError('Cell identity')
return c,pilot
def view_for(c,pilot,job):return dict(pilot,folds=[c['pools'][str(job['pool'])]])
def project(c,observations):
if set(observations)!=set(range(4)):raise ValueError('Four first epochs required')
rows=[]
for j in c['jobs']:
v=observations[j['id']];steps=v['steps'];per=v['max_epoch_seconds']/j['learning_parts'];val=max(v['max_validation_seconds'],j['validation_seconds_bound'])
if not 0<steps<=j['target_steps'] or not all(math.isfinite(x) and x>=0 for x in [per,val,v['charged']]) or per<=0:raise ValueError('Invalid cost evidence')
# Up to20 validation diagnostics, plus U checkpoint and final save overhead.
future=1.25*((j['target_steps']-steps)*per+20*val)+120
cost=v['charged']+future
rows.append(dict(id=j['id'],pool=j['pool'],mode=j['mode'],projected_gpu_seconds=cost,job_cap=j['gpu_seconds'],within_cap=cost<=j['gpu_seconds']))
total=sum(r['projected_gpu_seconds'] for r in rows)+c['postfit_gpu_reserve']
return dict(status='PASS' if total<=c['item_gpu_cap'] and all(r['within_cap'] for r in rows) else 'STOP_BUDGET_PROJECTION',rows=rows,projected_item_gpu_seconds=total,postfit_reserve=c['postfit_gpu_reserve'],phase3_previous_items_gpu_seconds=0,phase3_cap=c['phase3_gpu_cap'],safety_factor=1.25)
def first_epoch_release(root):
if (root/'global-stop.json').exists():raise RuntimeError('Global item1 stop')
p=root/'first-epoch-projection.json'
if not p.exists():return False
d=read(p)
if d['status']!='PASS' or d['contract_sha256']!=sha(root/'contract.json'):raise RuntimeError('First-epoch projection rejected')
return True