197 lines
14 KiB
Python
197 lines
14 KiB
Python
"""Synthetic b acceptance; no real data/model loading, GPU query or scientific run."""
|
|
import ast,copy,inspect,json,tempfile,unittest
|
|
from contextlib import ExitStack,contextmanager
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
import numpy as np
|
|
import torch
|
|
from phase3_item_b_common import project
|
|
from phase3_shared_capacity import GPUS,GIB
|
|
from phase3_item_b_data import valid_rows,moments,encode
|
|
from phase3_item_b_model import H2Context
|
|
from phase2_h2_model import H2Geo
|
|
|
|
class ContextTests(unittest.TestCase):
|
|
def test_joint_missing_is_zero_after_normalization(self):
|
|
x=np.full((3,2),np.nan);v=valid_rows(x,'SCIENTIFIC_MISSING',3)
|
|
self.assertTrue(np.array_equal(encode(x,v,dict(mean=[7,8],scale=[1,2])),np.zeros((3,3))))
|
|
def test_valid_mean_still_has_flag(self):
|
|
x=np.array([[7.,8.]])
|
|
self.assertTrue(np.array_equal(encode(x,valid_rows(x,'GEOMETRY_COMPLETE',1),dict(mean=[7,8],scale=[1,2])),[[0,0,1]]))
|
|
def test_partial_missing_stops(self):
|
|
for x in (np.array([[1.,np.nan]]),np.array([[np.nan,1.]])):
|
|
with self.assertRaises(ValueError):valid_rows(x,'SCIENTIFIC_MISSING',1)
|
|
def test_status_mismatch_not_scientific_missing(self):
|
|
for x,status in ((np.zeros((2,2)),'SCIENTIFIC_MISSING'),(np.full((2,2),np.nan),'GEOMETRY_COMPLETE'),(np.array([[1.,2.],[np.nan,np.nan]]),'GEOMETRY_COMPLETE')):
|
|
with self.assertRaises(ValueError):valid_rows(x,status,2)
|
|
def test_infinity_or_wrong_shape_stops(self):
|
|
for x in (np.array([[np.inf,np.inf]]),np.zeros((1,3))):
|
|
with self.assertRaises(ValueError):valid_rows(x,'GEOMETRY_COMPLETE',1)
|
|
def test_unit_node_moments_not_equal_part_means(self):
|
|
a=np.array([[1.,2.],[3.,4.],[5.,8.]]);b=np.array([[9.,12.]])
|
|
stats=moments([(a,np.ones(3,bool)),(b,np.ones(1,bool)),(np.full((2,2),np.nan),np.zeros(2,bool))])
|
|
x=np.concatenate([a,b]);np.testing.assert_allclose(stats['mean'],x.mean(0));np.testing.assert_allclose(stats['scale'],x.std(0));self.assertEqual(stats['valid_nodes'],4)
|
|
def test_zero_variance_scale_one(self):
|
|
stats=moments([(np.full((4,2),7.),np.ones(4,bool))]);self.assertEqual(stats['scale'],[1.,1.])
|
|
def test_all_learning_invalid_stops(self):
|
|
with self.assertRaises(ValueError):moments([(np.full((3,2),np.nan),np.zeros(3,bool))])
|
|
def test_learning_only_iteration(self):
|
|
import phase3_item_b_data as d
|
|
old=dict(folds=[dict(learning=['a','b'],validation=['v'],test=['t'])]);seen=[]
|
|
with tempfile.TemporaryDirectory() as tmp,patch.object(d,'original_standardization',return_value=dict(mean=[0]*19,scale=[1]*19)),patch.object(d,'read_context',side_effect=lambda o,c,k:(seen.append(k) or np.ones((2,2)),np.ones(2,bool))):
|
|
stats=d.standardization(old,{},Path(tmp))
|
|
self.assertEqual(seen,['a','b']);self.assertEqual(stats['context']['valid_nodes'],4)
|
|
def test_original_role_guard_precedes_context_read(self):
|
|
import phase3_item_b_data as d
|
|
with patch.object(d,'original_load',side_effect=PermissionError),patch.object(d,'read_context') as extra:
|
|
with self.assertRaises(PermissionError):d.load_part({}, {},'t',{},'test',Path('/synthetic'))
|
|
extra.assert_not_called()
|
|
|
|
|
|
class ModelTests(unittest.TestCase):
|
|
def pair(self):
|
|
torch.manual_seed(0);old=H2Geo();rng=torch.get_rng_state().clone()
|
|
torch.manual_seed(0);new=H2Context();self.assertTrue(torch.equal(rng,torch.get_rng_state()))
|
|
return old,new
|
|
def test_all_shared_parameters_and_rng_match(self):
|
|
old,new=self.pair()
|
|
for key,value in old.state_dict().items():
|
|
v=new.state_dict()[key]
|
|
if key in ('encoder.0.weight','head.0.weight'):
|
|
self.assertTrue(torch.equal(value,v[:,:value.shape[1]]));self.assertEqual(int(torch.count_nonzero(v[:,value.shape[1]:])),0)
|
|
else:self.assertTrue(torch.equal(value,v),key)
|
|
def test_exactly384_additional_parameters(self):
|
|
old,new=self.pair();self.assertEqual(sum(x.numel() for x in new.parameters())-sum(x.numel() for x in old.parameters()),384)
|
|
def inputs(self):
|
|
n=4;x=torch.randn(n,19);mass=torch.ones(n);r=torch.tensor([0,1,2,3]);s=torch.tensor([1,2,3,0]);a=torch.randn(n,4);ids=torch.zeros(n,dtype=torch.long)
|
|
return x,(mass,r,s,a,ids)
|
|
def test_nonzero_readout_still_matches_original_initial_function(self):
|
|
old,new=self.pair();x,args=self.inputs()
|
|
with torch.no_grad():old.head[-1].weight.fill_(.01);new.head[-1].weight.copy_(old.head[-1].weight)
|
|
old.eval();new.eval()
|
|
with torch.no_grad():
|
|
a=old(x,*args);b=new(torch.cat([x,torch.randn(4,3)],dim=1),*args)
|
|
torch.testing.assert_close(a,b,rtol=1e-5,atol=1e-6)
|
|
def test_new_columns_can_learn_without_changing_recipe(self):
|
|
from phase2_h2_model import optimizer
|
|
from phase2_h2_pilot_worker import train_step
|
|
_,new=self.pair();x,args=self.inputs();data=dict(zip(['mass','receiver','sender','attributes','component_ids'],args));data.update(x=torch.cat([x,torch.ones(4,3)],dim=1),target=torch.ones(4),nodes=4)
|
|
opt=optimizer(new)
|
|
# Only synchronize is replaced; real original FP32 train_step/backward is exercised on CPU.
|
|
with patch.object(torch.cuda,'synchronize',lambda:None):
|
|
train_step(new,opt,data,1,4);train_step(new,opt,data,1,4)
|
|
self.assertGreater(float(new.head[0].weight[:,-3:].abs().sum()),0)
|
|
def test_wrong_input_columns_rejected(self):
|
|
_,new=self.pair();x,args=self.inputs()
|
|
with self.assertRaises(ValueError):new(x,*args)
|
|
def test_worker_uses_original200_recipe(self):
|
|
s=(Path(__file__).parent/'phase3_item_b_worker.py').read_text()
|
|
self.assertIn('range(1,201)',s);self.assertIn('SeedSequence([0,epoch])',s);self.assertIn('if epoch%10==0:',s)
|
|
self.assertIn('from phase2_h2_model import lr_epoch',s);self.assertIn('from phase2_h2_pilot_worker import peaks,train_step',s)
|
|
self.assertNotIn('location_loss',s);self.assertNotIn('screen_decision',s)
|
|
def test_guard_accepts_only_complete200_epoch_identity(self):
|
|
s=(Path(__file__).parent/'phase3_item_b_guard.py').read_text()
|
|
self.assertIn("result.get('completed_epochs')!=200",s)
|
|
self.assertNotIn("result.get('completed_epochs')!=100",s)
|
|
|
|
class BudgetTests(unittest.TestCase):
|
|
def setUp(self):self.c=dict(gpu_cap=178200,postfit_gpu_reserve=7200,preparation_charge_gpu_seconds=600,runaway_gpu_seconds=60000,jobs=[dict(validation_seconds_bound=3) for _ in range(5)])
|
|
def test_initial_budget_includes_postfit(self):
|
|
p=project(self.c,{i:60 for i in range(5)},{},{})
|
|
self.assertEqual(p['status'],'PASS');self.assertEqual(p['projected_gpu_seconds'],178200)
|
|
def test_remaining_validation_charged(self):
|
|
p=project(self.c,{i:100 for i in range(5)},{0:[dict(epoch=1,train_epoch_seconds=50)]},{})
|
|
self.assertEqual(p['folds']['0']['projected'],100+1.5*(199*50+20*3)+120)
|
|
def test_late_validation_max_used(self):
|
|
h=[dict(epoch=e,train_epoch_seconds=50,validation_seconds=7 if e==10 else 0) for e in range(1,11)]
|
|
p=project(self.c,{i:1000 for i in range(5)},{0:h},{})
|
|
self.assertEqual(p['folds']['0']['projected'],1000+1.5*(190*50+19*7)+120)
|
|
def test_actual_runaway_stops(self):
|
|
self.assertEqual(project(self.c,{i:60000 if i==0 else 60 for i in range(5)},{},{})['status'],'STOP_BUDGET')
|
|
def test_no_single_fold_projection_gate(self):
|
|
p=project(self.c,{i:1000 for i in range(5)},{0:[dict(epoch=1,train_epoch_seconds=220)]},{1:{},2:{},3:{}})
|
|
self.assertGreater(p['folds']['0']['projected'],60000);self.assertEqual(p['status'],'PASS')
|
|
def test_incomplete_history_rejected(self):
|
|
with self.assertRaises(ValueError):project(self.c,{i:60 for i in range(5)},{0:[dict(epoch=2,train_epoch_seconds=50)]},{})
|
|
|
|
|
|
class SharedWrapperTests(unittest.TestCase):
|
|
def test_wrapper_changes_only_key_cap_gpu_set(self):
|
|
import phase3_item_b_capacity as new
|
|
import phase3_shared_capacity as old
|
|
for name in ('register_entry','acquire'):
|
|
s=inspect.getsource(getattr(new,name)).replace("'b'","'c'").replace('item b','item c').replace('cap=178200','cap=205200').replace('GPUS[:3]','GPUS[1:3]')
|
|
self.assertEqual(ast.dump(ast.parse(s)),ast.dump(ast.parse(inspect.getsource(getattr(old,name)))))
|
|
for name in ('admission','registry','transfer','release','validate_lease'):
|
|
self.assertIs(getattr(new,name),getattr(old,name))
|
|
@contextmanager
|
|
def mixed(self,reserved=False):
|
|
import phase3_item_b_capacity as m
|
|
import phase3_shared_capacity as shared
|
|
identity=lambda p:dict(pid=p,created=1.)
|
|
root=Path('/synthetic-1b');oldroot='/synthetic-c'
|
|
leases={}
|
|
for n,pid in enumerate([11,12]+([10] if reserved else [])):
|
|
leases[str(n)]=dict(token=str(n),state='RESERVED' if pid==10 else 'GUARD_BOUND',root=oldroot,fold=n,
|
|
contract_sha256='old',gpu_uuid=GPUS[1],worker_cpus=[16+n],owner=identity(pid),queue_owner=identity(10),vram_reserve=4*GIB,rss_reserve=4*GIB)
|
|
original_c=dict(root=oldroot,contract_sha256='old',owner=identity(10),cap=205200)
|
|
s=dict(devices={g:dict(total=16*GIB,used=4*GIB) for g in GPUS},processes=[dict(gpu_uuid=GPUS[1],pid=p,vram=2*GIB) for p in (101,102)]+[dict(gpu_uuid=GPUS[2],pid=p,vram=GIB) for p in (201,202,203)],ram_total=128*GIB,ram_available=96*GIB,cpu_percent=20,old_pending=True)
|
|
trees={11:{11,101},12:{12,102},10:{10,11,12,101,102}}
|
|
with tempfile.TemporaryDirectory() as tmp,ExitStack() as stack:
|
|
path=Path(tmp)/'registry.json';path.write_text(json.dumps(dict(version=2,entries={'c':original_c},leases=leases)))
|
|
stack.enter_context(patch.object(shared,'SHARED',Path(tmp)))
|
|
for name,value in dict(process_identity=lambda p:identity(20),alive=lambda p:True,descendants=lambda p:trees[p['pid']],host_snapshot=lambda:copy.deepcopy(s)).items():stack.enter_context(patch.object(m,name,value))
|
|
stack.enter_context(patch.object(m.psutil,'Process',lambda p:SimpleNamespace(memory_info=lambda:SimpleNamespace(rss=GIB//4))))
|
|
stack.enter_context(patch.object(m.os,'sched_getaffinity',lambda p:set(range(16,26))))
|
|
m.register_entry(root,'new')
|
|
yield m,root,path,original_c
|
|
def test_shared_c_and_b_registration_independent(self):
|
|
with self.mixed() as (m,root,path,old):
|
|
d=json.loads(path.read_text());self.assertEqual(d['entries']['c'],old);self.assertEqual(d['entries']['b']['cap'],178200)
|
|
with self.assertRaises(FileExistsError):m.register_entry(root,'new')
|
|
def test_mixed_pending_c_reservation_blocks_b(self):
|
|
with self.mixed(True) as (m,root,path,old):
|
|
before=json.loads(path.read_text());l,_=m.acquire(root,1,'new');self.assertIsNone(l)
|
|
self.assertEqual(json.loads(path.read_text()),before)
|
|
def test_mixed_two_visible_allows_only_one_new(self):
|
|
with self.mixed() as (m,root,path,old):
|
|
l,_=m.acquire(root,1,'new');self.assertIsNotNone(l);self.assertEqual(l['state'],'RESERVED');self.assertEqual(l['gpu_uuid'],GPUS[1])
|
|
second,_=m.acquire(root,0,'new');self.assertIsNone(second)
|
|
self.assertEqual(json.loads(path.read_text())['entries']['c'],old)
|
|
|
|
|
|
class QueueTests(unittest.TestCase):
|
|
def exercise(self,failed=False):
|
|
import phase3_item_b_queue as q
|
|
from phase2_h1_worker_v2 import save
|
|
with tempfile.TemporaryDirectory() as tmp,ExitStack() as stack:
|
|
root=Path(tmp);save(root/'contract.json',dict(synthetic=True));launched=[]
|
|
c=dict(gpu_cap=178200,postfit_gpu_reserve=7200,preparation_charge_gpu_seconds=600,runaway_gpu_seconds=60000,jobs=[dict(validation_seconds_bound=1) for _ in range(5)])
|
|
class Process:
|
|
def __init__(self,cmd,**kw):
|
|
self.pid=999991;self.returncode=int(failed);i=int(cmd[3]);launched.append(i);j=root/'jobs'/str(i);(j/'worker').mkdir()
|
|
save(j/'complete.json',dict(status='STOPPED_OR_FAILED' if failed else 'PASS'));p=j/'worker/last-epoch200.pt';p.write_bytes(b'synthetic')
|
|
save(j/'worker/result.json',dict(fold=i,checkpoint=str(p),checkpoint_sha256=q.sha(p),completed_epochs=200));save(j/'worker/standardization.json',{})
|
|
def poll(self):return self.returncode
|
|
def wait(self):return self.returncode
|
|
def terminate(self):pass
|
|
def kill(self):pass
|
|
replacements=dict(config=lambda r:(c,{}),validate_review=lambda r:None,register_entry=lambda *a:None,
|
|
acquire=lambda r,i,d:(dict(token=str(i),gpu_uuid=GPUS[1],worker_cpus=[20]),{}),transfer=lambda *a:None,
|
|
release=lambda *a:None,process_identity=lambda p:dict(pid=p,created=0),freeze=lambda *a:None)
|
|
for k,v in replacements.items():stack.enter_context(patch.object(q,k,v))
|
|
stack.enter_context(patch.object(q.subprocess,'Popen',Process));stack.enter_context(patch.object(q.time,'sleep',lambda t:None))
|
|
stack.enter_context(patch.object(q.signal,'signal',lambda *a:None));stack.enter_context(patch.object(q.psutil,'disk_usage',return_value=SimpleNamespace(free=200*GIB)))
|
|
stack.enter_context(patch.dict(q.os.environ,PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True'))
|
|
result=q.run(root);d=q.read(root/'complete.json')
|
|
if failed:self.assertEqual(result,1);self.assertEqual(launched,[0]);self.assertFalse((root/'models.json').exists())
|
|
else:
|
|
self.assertEqual(result,0);self.assertEqual(launched,[0,1,2,3,4]);self.assertEqual(len(q.read(root/'models.json')),5)
|
|
self.assertEqual(d['status'],'TRAIN_COMPLETE_POSTFIT_PENDING')
|
|
self.assertFalse(d['test_targets_read']);self.assertFalse(d['prediction_started']);self.assertFalse(d['scoring_started'])
|
|
def test_five_fits_stop_before_postfit(self):self.exercise()
|
|
def test_failure_no_retry_or_test(self):self.exercise(True)
|
|
|
|
|