Files
MeltMind/tools/prepare_phase3_data_sufficiency.py
T

44 lines
9.8 KiB
Python

"""Freeze item1 metadata, identity, costs and comparison contract; no target/GPU execution."""
import json,time,shutil,math
from pathlib import Path
from collections import Counter
from phase2_h1_worker_v2 import sha,save,checked
from phase2_h2_prepare import closure
from phase2_home_freeze_io import freeze,verify
B=Path('/home/frank14f/optane/MeltMind');R=Path('/home/frank14f/MeltMind-runs/phase3-data-sufficiency-2026-11-18')
def read(p):return json.loads(Path(p).read_text())
def main():
import psutil
from phase3_data_sufficiency_common import execution_command,config
if R.exists():raise FileExistsError('Existing item1 root; inspect before any action')
if any(execution_command(p.info['cmdline'] or []) for p in psutil.process_iter(['cmdline'])):raise RuntimeError('Existing item1 task')
start=time.process_time();src={}
def source(p):p=Path(p);src[str(p)]=sha(p);return read(p)
pilotpath=B/'artifacts/phase2-h2-pilot-2026-11-17/contract.json';pilot=source(pilotpath);hp=B/'artifacts/phase2-h1-execution-2026-11-17';h=source(hp/'contract.json');models=source(hp/'models-terminal.json')['models'];S=Path('/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17');S2=Path('/home/frank14f/MeltMind-runs/phase2-h2-s2-postfit-2026-11-17');s2=source(S2/'score/result.json');assert s2['decision']=='CONFIRMED';verify(S2,'terminal');sp=source(S/'contract.json');s1result=source(S/'score/result.json');s1complete=source(S/'complete.json');s1train=B/'artifacts/phase2-h2-s1-2026-11-17';tr=source(s1train/'worker/result.json');tc=source(s1train/'contract.json');f0=pilot['folds'][0];pools={}
for pool in [25,50,100]:
job=next(j for j in h['jobs'] if j['protocol']=='grouped' and j['fold']==0 and j['pool']==pool);m=next(x for x in models if x['index']==job['index']);clock=source(checked(m['clock_path'],m['clock_sha256']));assert clock['parts']==job['learning'];f=dict(f0,learning=job['learning'],c=clock['c'],clock_path=m['clock_path'],clock_sha256=m['clock_sha256']);f['learning_nodes']=sum(pilot['parts'][k]['nodes'] for k in f['learning']);f['learning_edges']=sum(pilot['geometry'][k]['directed_edges'] for k in f['learning']);f['learning_component_ids']=sorted({h['grouped_rows'][k]['component_id'] for k in f['learning']});f['learning_components']=len(f['learning_component_ids']);f['exposure_counts']=dict(Counter(h['grouped_rows'][k]['exposure']['inventory_state'] for k in f['learning']));f['geometry_category_coverage']='UNAVAILABLE_NO_FROZEN_CATEGORY_ONTOLOGY';pools[str(pool)]=f
assert pools['100']['learning']==f0['learning'] and pools['100']['c']==f0['c'];assert all(set(pools[str(a)]['learning_component_ids'])<set(pools[str(b)]['learning_component_ids']) for a,b in [(25,50),(50,100)])
assert tr['checkpoint_sha256']==sp['checkpoint_sha256'] and tr['completed_epochs']==200 and tr['optimizer_steps']==64000 and tc['seed']==0
oldstats=source(s1train/'worker/standardization.json');assert oldstats['parts']==f0['learning'];source(S/'predictions-manifest.json');source(S/'post-terminal-cost.json');source(S/'terminal-git-freeze.json');source(S/'predictions-git-freeze.json');src[tr['checkpoint']]=sha(tr['checkpoint']);assert src[tr['checkpoint']]==tr['checkpoint_sha256']
history=[source(s1train/'worker/history'/f'{e:03d}.json') for e in range(1,201)];maxt=max(x['train_epoch_seconds'] for x in history);maxv=max(x.get('validation_seconds',0) for x in history)
gpu=['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c','GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36','GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76','GPU-8b71b032-1b75-2891-935c-ce106a53034a'];jobs=[];projections=[]
for i,(pool,mode,cap) in enumerate([(25,'epoch',10000),(50,'epoch',16000),(50,'step',26600),(100,'step',26600)]):
f=pools[str(pool)];n=len(f['learning']);steps=200*n if mode=='epoch' else 64000;ratio=max(f['learning_nodes']/f0['learning_nodes'],f['learning_edges']/f0['learning_edges'],n/320);est=1.25*(maxt*ratio/n*steps+20*maxv)+180
j=dict(id=i,pool=pool,mode=mode,target_steps=steps,learning_parts=n,learning_nodes=f['learning_nodes'],learning_edges=f['learning_edges'],gpu_uuid=gpu[i],worker_cpus=[2*i,2*i+1],guard_cpus=[8+i],gpu_seconds=cap,cpu_seconds=cap,setup_gpu_seconds=60,validation_seconds_bound=maxv,rss_bytes=16*2**30,vram_bytes=15*2**30,disk_bytes=20*2**30);jobs.append(j);projections.append(dict(id=i,pool=pool,mode=mode,updates=steps,nodes_per_epoch=f['learning_nodes'],edges_per_epoch=f['learning_edges'],expected_new_gpu_seconds=est,cap=cap,within_cap=est<=cap))
if not all(x['within_cap'] for x in projections) or sum(x['expected_new_gpu_seconds'] for x in projections)+7200>86400:raise ValueError('Prelaunch cost projection fails; do not create or launch')
R.mkdir();comparison=dict(spec_sha256='e24fff7c5cb91dff7acf1532f502daec60e4b7fec21b2d4d4dcde1e793be7fb4',fold=0,seed=0,pools=[25,50,100],common_validation=f0['validation'],common_test=f0['test'],external_fold_previously_observed=True,epochs=200,U=32000,twoU=64000,delta=.001,bootstrap=dict(replicates=2000,seed=2026111408,unit='paired evaluation component',joint_resampling_all_contrasts=True),endpoints=['epoch25','epoch50','S1_epoch100_reuse','step50_U','step50_2U','step100_U','step100_2U'],clock_baselines=['c25F','c50F','c100F'],domains=['own_learning','common_validation','common_test'],metrics=['relative_l2','last_location','kendall_tau_b'],component_counts={p:f['learning_components'] for p,f in pools.items()},slope='(E50_H-E100_H)/log2(n100/n50), H=U and2U',gain='E_n_U-E_n_2U for n50,n100',interaction='s_2U-s_U; joint component resampling',classifications=dict(data_limited='s2U low>delta and g100 entireCI in[-delta,delta]',updates_limited='s2U entireCI in[-delta,delta] and g100 low>delta',both='both low>delta',local_plateau='both entireCI in[-delta,delta]',otherwise='UNIDENTIFIED_OR_OVERFIT_SIGNS'),undefined='Report missing/undefined support; do not call wide/cross-zero CI plateau',not_training_randomness=True,no_3U_or_extrapolation=True,predictions_frozen_before_external_targets=True,new_exact_postfit_review_required=True)
save(R/'comparison-contract.json',comparison)
spec=B/'references/2026-11-17/phase2-design/mold-h2-data-sufficiency-e24fff7c.md';assert sha(spec)==comparison['spec_sha256'];src[str(spec)]=sha(spec)
codes={str(B/p):d for p,d in closure(['phase3_data_sufficiency_common','phase3_data_sufficiency_worker','phase3_data_sufficiency_guard','phase3_data_sufficiency_queue','prepare_phase3_data_sufficiency','test_phase3_data_sufficiency']).items()};doc=B/'docs/phase3-data-sufficiency-entry-2026-11-18.md';codes[str(doc)]=sha(doc)
reuse=dict(checkpoint=tr['checkpoint'],checkpoint_sha256=tr['checkpoint_sha256'],standardization=str(s1train/'worker/standardization.json'),standardization_sha256=sha(s1train/'worker/standardization.json'),test_score=str(S/'score/result.json'),test_score_sha256=sha(S/'score/result.json'),test_manifest=str(S/'predictions-manifest.json'),test_manifest_sha256=sha(S/'predictions-manifest.json'),exact_identity_checked=['roles','seed0','architecture','learning_clock','normalization_formula','original_lr_epoch','last200_selection','test_domain'],old_gpu_cost_not_recharged=True)
c=dict(version='phase3-item1-training-v1',approval='1794946961.453149',plan_commit='304f5771d9fdf1c764251b3b7d7f0b6ae7d0e854',fold=0,seed=0,U=32000,delta=.001,new_fits=4,pilot_contract=str(pilotpath),pools=pools,jobs=jobs,S1_reuse=reuse,item_gpu_cap=86400,phase3_gpu_cap=612000,phase3_previous_item_gpu_seconds=0,train_gpu_cap=79200,postfit_gpu_reserve=7200,postfit_cpu_cap=3600,coordinator_cpus=[12,13],code_sha256=codes,source_sha256=src,comparison_contract=str(R/'comparison-contract.json'),comparison_contract_sha256=sha(R/'comparison-contract.json'),no_retry=True,stop_after='TRAIN_COMPLETE_POSTFIT_PENDING',test_prediction_or_scoring_in_this_entry=False)
save(R/'contract.json',c);save(R/'prelaunch-projection.json',dict(cells=projections,training_gpu_seconds=sum(x['expected_new_gpu_seconds'] for x in projections),postfit_gpu_reserve=7200,total_gpu_seconds=sum(x['expected_new_gpu_seconds'] for x in projections)+7200,item_cap=86400,source_max_train_epoch_seconds=maxt,source_max_validation_seconds=maxv,not_actual_execution=True))
for i in [1,2]:shutil.copyfile('/tmp/phase3-data-sufficiency-tests'+str(i)+'.log',R/('synthetic-attempt'+str(i)+'.log'))
assert (R/'synthetic-attempt2.log').read_text().rstrip().endswith('OK')
save(R/'acceptance.json',dict(status='PASS',tests=13,first_failure='Test incorrectly assumed end-of-epoch LR differs; schedule equals old at32000 but differs within epoch; corrected assertion to31999. No real workloads.',targets_read=False,GPU_workload=False))
save(R/'preparation-audit.json',dict(status='PREPARED_NOT_STARTED',pools={p:dict(parts=len(f['learning']),components=f['learning_components'],nodes=f['learning_nodes'],edges=f['learning_edges']) for p,f in pools.items()},S1_reuse_identity_verified=True,metadata_cpu_seconds=time.process_time()-start,no_target_arrays_read=True))
freeze(R,'implementation',[R/'contract.json',R/'comparison-contract.json',R/'acceptance.json',R/'prelaunch-projection.json',R/'preparation-audit.json',R/'synthetic-attempt1.log',R/'synthetic-attempt2.log',*[Path(p) for p in codes]])
binding=dict(reviewed_commit=verify(R,'implementation')['commit'],contract_sha256=sha(R/'contract.json'),implementation_proof_sha256=sha(R/'implementation-git-freeze.json'),comparison_contract_sha256=c['comparison_contract_sha256'],code_sha256=codes);save(R/'expected-review-binding.json',binding);freeze(R,'review-binding',[R/'contract.json',R/'comparison-contract.json',R/'expected-review-binding.json',R/'implementation-git-freeze.json']);config(R)
print(json.dumps(dict(root=str(R),implementation=binding['reviewed_commit'],contract_sha256=binding['contract_sha256'],binding_sha256=sha(R/'expected-review-binding.json'),pools=read(R/'preparation-audit.json')['pools'],projected_gpu_seconds=read(R/'prelaunch-projection.json')['total_gpu_seconds'],started=False)))
if __name__=='__main__':main()