Files
MeltMind/tools/test_phase3_item_b.py
T

197 lines
14 KiB
Python

"""Synthetic b acceptance; no real data/model loading, GPU query or scientific run."""
import ast,copy,inspect,json,tempfile,unittest
from contextlib import ExitStack,contextmanager
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
import numpy as np
import torch
from phase3_item_b_common import project
from phase3_shared_capacity import GPUS,GIB
from phase3_item_b_data import valid_rows,moments,encode
from phase3_item_b_model import H2Context
from phase2_h2_model import H2Geo
class ContextTests(unittest.TestCase):
def test_joint_missing_is_zero_after_normalization(self):
x=np.full((3,2),np.nan);v=valid_rows(x,'SCIENTIFIC_MISSING',3)
self.assertTrue(np.array_equal(encode(x,v,dict(mean=[7,8],scale=[1,2])),np.zeros((3,3))))
def test_valid_mean_still_has_flag(self):
x=np.array([[7.,8.]])
self.assertTrue(np.array_equal(encode(x,valid_rows(x,'GEOMETRY_COMPLETE',1),dict(mean=[7,8],scale=[1,2])),[[0,0,1]]))
def test_partial_missing_stops(self):
for x in (np.array([[1.,np.nan]]),np.array([[np.nan,1.]])):
with self.assertRaises(ValueError):valid_rows(x,'SCIENTIFIC_MISSING',1)
def test_status_mismatch_not_scientific_missing(self):
for x,status in ((np.zeros((2,2)),'SCIENTIFIC_MISSING'),(np.full((2,2),np.nan),'GEOMETRY_COMPLETE'),(np.array([[1.,2.],[np.nan,np.nan]]),'GEOMETRY_COMPLETE')):
with self.assertRaises(ValueError):valid_rows(x,status,2)
def test_infinity_or_wrong_shape_stops(self):
for x in (np.array([[np.inf,np.inf]]),np.zeros((1,3))):
with self.assertRaises(ValueError):valid_rows(x,'GEOMETRY_COMPLETE',1)
def test_unit_node_moments_not_equal_part_means(self):
a=np.array([[1.,2.],[3.,4.],[5.,8.]]);b=np.array([[9.,12.]])
stats=moments([(a,np.ones(3,bool)),(b,np.ones(1,bool)),(np.full((2,2),np.nan),np.zeros(2,bool))])
x=np.concatenate([a,b]);np.testing.assert_allclose(stats['mean'],x.mean(0));np.testing.assert_allclose(stats['scale'],x.std(0));self.assertEqual(stats['valid_nodes'],4)
def test_zero_variance_scale_one(self):
stats=moments([(np.full((4,2),7.),np.ones(4,bool))]);self.assertEqual(stats['scale'],[1.,1.])
def test_all_learning_invalid_stops(self):
with self.assertRaises(ValueError):moments([(np.full((3,2),np.nan),np.zeros(3,bool))])
def test_learning_only_iteration(self):
import phase3_item_b_data as d
old=dict(folds=[dict(learning=['a','b'],validation=['v'],test=['t'])]);seen=[]
with tempfile.TemporaryDirectory() as tmp,patch.object(d,'original_standardization',return_value=dict(mean=[0]*19,scale=[1]*19)),patch.object(d,'read_context',side_effect=lambda o,c,k:(seen.append(k) or np.ones((2,2)),np.ones(2,bool))):
stats=d.standardization(old,{},Path(tmp))
self.assertEqual(seen,['a','b']);self.assertEqual(stats['context']['valid_nodes'],4)
def test_original_role_guard_precedes_context_read(self):
import phase3_item_b_data as d
with patch.object(d,'original_load',side_effect=PermissionError),patch.object(d,'read_context') as extra:
with self.assertRaises(PermissionError):d.load_part({}, {},'t',{},'test',Path('/synthetic'))
extra.assert_not_called()
class ModelTests(unittest.TestCase):
def pair(self):
torch.manual_seed(0);old=H2Geo();rng=torch.get_rng_state().clone()
torch.manual_seed(0);new=H2Context();self.assertTrue(torch.equal(rng,torch.get_rng_state()))
return old,new
def test_all_shared_parameters_and_rng_match(self):
old,new=self.pair()
for key,value in old.state_dict().items():
v=new.state_dict()[key]
if key in ('encoder.0.weight','head.0.weight'):
self.assertTrue(torch.equal(value,v[:,:value.shape[1]]));self.assertEqual(int(torch.count_nonzero(v[:,value.shape[1]:])),0)
else:self.assertTrue(torch.equal(value,v),key)
def test_exactly384_additional_parameters(self):
old,new=self.pair();self.assertEqual(sum(x.numel() for x in new.parameters())-sum(x.numel() for x in old.parameters()),384)
def inputs(self):
n=4;x=torch.randn(n,19);mass=torch.ones(n);r=torch.tensor([0,1,2,3]);s=torch.tensor([1,2,3,0]);a=torch.randn(n,4);ids=torch.zeros(n,dtype=torch.long)
return x,(mass,r,s,a,ids)
def test_nonzero_readout_still_matches_original_initial_function(self):
old,new=self.pair();x,args=self.inputs()
with torch.no_grad():old.head[-1].weight.fill_(.01);new.head[-1].weight.copy_(old.head[-1].weight)
old.eval();new.eval()
with torch.no_grad():
a=old(x,*args);b=new(torch.cat([x,torch.randn(4,3)],dim=1),*args)
torch.testing.assert_close(a,b,rtol=1e-5,atol=1e-6)
def test_new_columns_can_learn_without_changing_recipe(self):
from phase2_h2_model import optimizer
from phase2_h2_pilot_worker import train_step
_,new=self.pair();x,args=self.inputs();data=dict(zip(['mass','receiver','sender','attributes','component_ids'],args));data.update(x=torch.cat([x,torch.ones(4,3)],dim=1),target=torch.ones(4),nodes=4)
opt=optimizer(new)
# Only synchronize is replaced; real original FP32 train_step/backward is exercised on CPU.
with patch.object(torch.cuda,'synchronize',lambda:None):
train_step(new,opt,data,1,4);train_step(new,opt,data,1,4)
self.assertGreater(float(new.head[0].weight[:,-3:].abs().sum()),0)
def test_wrong_input_columns_rejected(self):
_,new=self.pair();x,args=self.inputs()
with self.assertRaises(ValueError):new(x,*args)
def test_worker_uses_original200_recipe(self):
s=(Path(__file__).parent/'phase3_item_b_worker.py').read_text()
self.assertIn('range(1,201)',s);self.assertIn('SeedSequence([0,epoch])',s);self.assertIn('if epoch%10==0:',s)
self.assertIn('from phase2_h2_model import lr_epoch',s);self.assertIn('from phase2_h2_pilot_worker import peaks,train_step',s)
self.assertNotIn('location_loss',s);self.assertNotIn('screen_decision',s)
def test_guard_accepts_only_complete200_epoch_identity(self):
s=(Path(__file__).parent/'phase3_item_b_guard.py').read_text()
self.assertIn("result.get('completed_epochs')!=200",s)
self.assertNotIn("result.get('completed_epochs')!=100",s)
class BudgetTests(unittest.TestCase):
def setUp(self):self.c=dict(gpu_cap=178200,postfit_gpu_reserve=7200,preparation_charge_gpu_seconds=600,runaway_gpu_seconds=60000,jobs=[dict(validation_seconds_bound=3) for _ in range(5)])
def test_initial_budget_includes_postfit(self):
p=project(self.c,{i:60 for i in range(5)},{},{})
self.assertEqual(p['status'],'PASS');self.assertEqual(p['projected_gpu_seconds'],178200)
def test_remaining_validation_charged(self):
p=project(self.c,{i:100 for i in range(5)},{0:[dict(epoch=1,train_epoch_seconds=50)]},{})
self.assertEqual(p['folds']['0']['projected'],100+1.5*(199*50+20*3)+120)
def test_late_validation_max_used(self):
h=[dict(epoch=e,train_epoch_seconds=50,validation_seconds=7 if e==10 else 0) for e in range(1,11)]
p=project(self.c,{i:1000 for i in range(5)},{0:h},{})
self.assertEqual(p['folds']['0']['projected'],1000+1.5*(190*50+19*7)+120)
def test_actual_runaway_stops(self):
self.assertEqual(project(self.c,{i:60000 if i==0 else 60 for i in range(5)},{},{})['status'],'STOP_BUDGET')
def test_no_single_fold_projection_gate(self):
p=project(self.c,{i:1000 for i in range(5)},{0:[dict(epoch=1,train_epoch_seconds=220)]},{1:{},2:{},3:{}})
self.assertGreater(p['folds']['0']['projected'],60000);self.assertEqual(p['status'],'PASS')
def test_incomplete_history_rejected(self):
with self.assertRaises(ValueError):project(self.c,{i:60 for i in range(5)},{0:[dict(epoch=2,train_epoch_seconds=50)]},{})
class SharedWrapperTests(unittest.TestCase):
def test_wrapper_changes_only_key_cap_gpu_set(self):
import phase3_item_b_capacity as new
import phase3_shared_capacity as old
for name in ('register_entry','acquire'):
s=inspect.getsource(getattr(new,name)).replace("'b'","'c'").replace('item b','item c').replace('cap=178200','cap=205200').replace('GPUS[:3]','GPUS[1:3]')
self.assertEqual(ast.dump(ast.parse(s)),ast.dump(ast.parse(inspect.getsource(getattr(old,name)))))
for name in ('admission','registry','transfer','release','validate_lease'):
self.assertIs(getattr(new,name),getattr(old,name))
@contextmanager
def mixed(self,reserved=False):
import phase3_item_b_capacity as m
import phase3_shared_capacity as shared
identity=lambda p:dict(pid=p,created=1.)
root=Path('/synthetic-1b');oldroot='/synthetic-c'
leases={}
for n,pid in enumerate([11,12]+([10] if reserved else [])):
leases[str(n)]=dict(token=str(n),state='RESERVED' if pid==10 else 'GUARD_BOUND',root=oldroot,fold=n,
contract_sha256='old',gpu_uuid=GPUS[1],worker_cpus=[16+n],owner=identity(pid),queue_owner=identity(10),vram_reserve=4*GIB,rss_reserve=4*GIB)
original_c=dict(root=oldroot,contract_sha256='old',owner=identity(10),cap=205200)
s=dict(devices={g:dict(total=16*GIB,used=4*GIB) for g in GPUS},processes=[dict(gpu_uuid=GPUS[1],pid=p,vram=2*GIB) for p in (101,102)]+[dict(gpu_uuid=GPUS[2],pid=p,vram=GIB) for p in (201,202,203)],ram_total=128*GIB,ram_available=96*GIB,cpu_percent=20,old_pending=True)
trees={11:{11,101},12:{12,102},10:{10,11,12,101,102}}
with tempfile.TemporaryDirectory() as tmp,ExitStack() as stack:
path=Path(tmp)/'registry.json';path.write_text(json.dumps(dict(version=2,entries={'c':original_c},leases=leases)))
stack.enter_context(patch.object(shared,'SHARED',Path(tmp)))
for name,value in dict(process_identity=lambda p:identity(20),alive=lambda p:True,descendants=lambda p:trees[p['pid']],host_snapshot=lambda:copy.deepcopy(s)).items():stack.enter_context(patch.object(m,name,value))
stack.enter_context(patch.object(m.psutil,'Process',lambda p:SimpleNamespace(memory_info=lambda:SimpleNamespace(rss=GIB//4))))
stack.enter_context(patch.object(m.os,'sched_getaffinity',lambda p:set(range(16,26))))
m.register_entry(root,'new')
yield m,root,path,original_c
def test_shared_c_and_b_registration_independent(self):
with self.mixed() as (m,root,path,old):
d=json.loads(path.read_text());self.assertEqual(d['entries']['c'],old);self.assertEqual(d['entries']['b']['cap'],178200)
with self.assertRaises(FileExistsError):m.register_entry(root,'new')
def test_mixed_pending_c_reservation_blocks_b(self):
with self.mixed(True) as (m,root,path,old):
before=json.loads(path.read_text());l,_=m.acquire(root,1,'new');self.assertIsNone(l)
self.assertEqual(json.loads(path.read_text()),before)
def test_mixed_two_visible_allows_only_one_new(self):
with self.mixed() as (m,root,path,old):
l,_=m.acquire(root,1,'new');self.assertIsNotNone(l);self.assertEqual(l['state'],'RESERVED');self.assertEqual(l['gpu_uuid'],GPUS[1])
second,_=m.acquire(root,0,'new');self.assertIsNone(second)
self.assertEqual(json.loads(path.read_text())['entries']['c'],old)
class QueueTests(unittest.TestCase):
def exercise(self,failed=False):
import phase3_item_b_queue as q
from phase2_h1_worker_v2 import save
with tempfile.TemporaryDirectory() as tmp,ExitStack() as stack:
root=Path(tmp);save(root/'contract.json',dict(synthetic=True));launched=[]
c=dict(gpu_cap=178200,postfit_gpu_reserve=7200,preparation_charge_gpu_seconds=600,runaway_gpu_seconds=60000,jobs=[dict(validation_seconds_bound=1) for _ in range(5)])
class Process:
def __init__(self,cmd,**kw):
self.pid=999991;self.returncode=int(failed);i=int(cmd[3]);launched.append(i);j=root/'jobs'/str(i);(j/'worker').mkdir()
save(j/'complete.json',dict(status='STOPPED_OR_FAILED' if failed else 'PASS'));p=j/'worker/last-epoch200.pt';p.write_bytes(b'synthetic')
save(j/'worker/result.json',dict(fold=i,checkpoint=str(p),checkpoint_sha256=q.sha(p),completed_epochs=200));save(j/'worker/standardization.json',{})
def poll(self):return self.returncode
def wait(self):return self.returncode
def terminate(self):pass
def kill(self):pass
replacements=dict(config=lambda r:(c,{}),validate_review=lambda r:None,register_entry=lambda *a:None,
acquire=lambda r,i,d:(dict(token=str(i),gpu_uuid=GPUS[1],worker_cpus=[20]),{}),transfer=lambda *a:None,
release=lambda *a:None,process_identity=lambda p:dict(pid=p,created=0),freeze=lambda *a:None)
for k,v in replacements.items():stack.enter_context(patch.object(q,k,v))
stack.enter_context(patch.object(q.subprocess,'Popen',Process));stack.enter_context(patch.object(q.time,'sleep',lambda t:None))
stack.enter_context(patch.object(q.signal,'signal',lambda *a:None));stack.enter_context(patch.object(q.psutil,'disk_usage',return_value=SimpleNamespace(free=200*GIB)))
stack.enter_context(patch.dict(q.os.environ,PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True'))
result=q.run(root);d=q.read(root/'complete.json')
if failed:self.assertEqual(result,1);self.assertEqual(launched,[0]);self.assertFalse((root/'models.json').exists())
else:
self.assertEqual(result,0);self.assertEqual(launched,[0,1,2,3,4]);self.assertEqual(len(q.read(root/'models.json')),5)
self.assertEqual(d['status'],'TRAIN_COMPLETE_POSTFIT_PENDING')
self.assertFalse(d['test_targets_read']);self.assertFalse(d['prediction_started']);self.assertFalse(d['scoring_started'])
def test_five_fits_stop_before_postfit(self):self.exercise()
def test_failure_no_retry_or_test(self):self.exercise(True)