153 lines
11 KiB
Python
153 lines
11 KiB
Python
"""New integration/topology/guard tests; synthetic inputs only, no old test rerun."""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
from unittest.mock import Mock
|
|
import signal
|
|
import numpy as np
|
|
from scipy.sparse import csr_matrix
|
|
from phase2_h1_topology import graph,events,match
|
|
from phase2_h1_decision import branch_decision
|
|
from phase2_h1_execution import projected_fits,Runner,task
|
|
import phase2_h1_worker_v2 as worker
|
|
from r4bc_guard import supervise
|
|
|
|
|
|
def line(n):
|
|
a=np.arange(n-1);b=a+1
|
|
return csr_matrix((np.ones(2*(n-1)),(np.r_[a,b],np.r_[b,a])),shape=(n,n))
|
|
|
|
|
|
class TopologyTests(unittest.TestCase):
|
|
def test_tie_plateau_is_one_merge_and_position_mass_weighted(self):
|
|
s=np.array([4.,1.,1.,3.]);m=np.array([1.,1.,3.,1.]);raw=np.arange(10,14);x=np.c_[np.arange(4),np.zeros((4,2))]
|
|
r=events(s,m,raw,x,line(4));self.assertEqual(len(r['events']),1)
|
|
e=r['events'][0];self.assertEqual(e['raw_id'],12);self.assertEqual(e['support_nodes'],2);self.assertEqual(e['multiplicity'],1)
|
|
self.assertEqual(r['infinite_classes'],1);self.assertAlmostEqual(e['bars'][0]['persistence'],1/6)
|
|
perm=np.array([3,1,0,2]);a=line(4)[perm][:,perm]
|
|
q=events(s[perm],m[perm],raw[perm],x[perm],a)
|
|
self.assertEqual(q['events'][0]['raw_id'],12)
|
|
self.assertEqual(q['events'][0]['death'],e['death'])
|
|
def test_three_peaks_compound_multiplicity_and_increasing_map(self):
|
|
a=csr_matrix((np.ones(6),([0,0,0,1,2,3],[1,2,3,0,0,0])),shape=(4,4));s=np.array([0.,4.,3.,2.]);x=np.c_[np.arange(4),np.zeros((4,2))]
|
|
e=events(s,np.ones(4),np.arange(4),x,a)
|
|
self.assertEqual(len(e['events']),1);self.assertEqual(e['events'][0]['multiplicity'],2)
|
|
self.assertEqual(e,events(s*2+1,np.ones(4),np.arange(4),x,a))
|
|
def test_all_equal_no_finite_events_and_separate_infinite_classes(self):
|
|
a=csr_matrix((4,4));r=events(np.ones(4),np.ones(4),np.arange(4),np.zeros((4,3)),a)
|
|
self.assertEqual(r['events'],[]);self.assertEqual(r['infinite_classes'],4)
|
|
def test_true_tet_graph_no_cross_component_shortcut(self):
|
|
x=np.array([[0,0,0],[1,0,0],[0,1,0],[0,0,1],[.1,.1,.1],[1.1,.1,.1],[.1,1.1,.1],[.1,.1,1.1]])
|
|
a=graph(x,np.array([[0,1,2,3],[4,5,6,7]]));self.assertEqual(a.nnz,24);self.assertEqual(a[:4,4:].nnz,0)
|
|
def test_matching_is_maximum_one_to_one_and_lexicographic(self):
|
|
a=line(4);truth=dict(events=[dict(node=0,raw_id=10,death=.5),dict(node=3,raw_id=13,death=.5)])
|
|
pred=dict(events=[dict(node=1,raw_id=11,death=.5),dict(node=2,raw_id=12,death=.5)])
|
|
r=match(truth,pred,a,30.);self.assertEqual(r['TP'],2);self.assertEqual([(x['truth_raw_id'],x['prediction_raw_id']) for x in r['pairs']],[(10,11),(13,12)])
|
|
# Same objective and cardinality: representative IDs define the tie.
|
|
tied=csr_matrix((np.ones(8),([0,0,3,3,1,2,1,2],[1,2,1,2,0,0,3,3])),shape=(4,4))
|
|
r=match(truth,pred,tied,20.);self.assertEqual([(x['truth_raw_id'],x['prediction_raw_id']) for x in r['pairs']],[(10,11),(13,12)])
|
|
pred['events'][0]['death']=.56;r=match(truth,pred,a,30.);self.assertEqual(r['TP'],1)
|
|
def test_empty_matching_denominators_undefined(self):
|
|
r=match(dict(events=[]),dict(events=[]),line(2),1.)
|
|
self.assertIsNone(r['precision']);self.assertIsNone(r['recall']);self.assertEqual(r['empty_class'],'both_empty')
|
|
|
|
|
|
class PolicyTests(unittest.TestCase):
|
|
def test_branch_units_sign_and_strict_boundaries(self):
|
|
interval=lambda a,b:dict(low=a,high=b)
|
|
self.assertEqual(branch_decision(interval(-.01,.0009),interval(-.2,-.1),interval(-.1,.1))['decision'],'EXPAND')
|
|
self.assertEqual(branch_decision(interval(-.01,.001),interval(-.2,-.1),interval(-.1,.1))['decision'],'PARK')
|
|
self.assertEqual(branch_decision(interval(.001,.002),interval(-.1,.1),interval(-.1,.1))['decision'],'PRUNE')
|
|
self.assertEqual(branch_decision(interval(.001,.002),interval(-.1,0),interval(.01,.02))['decision'],'PARK')
|
|
self.assertEqual(branch_decision(interval(0,1),None,None)['decision'],'PARK')
|
|
def test_cost_gate_counts_failed_first_cell_and_factor(self):
|
|
jobs=[dict(index=i,protocol='grouped' if i<15 else 'v4_random',learning_nodes=10) for i in range(20)]
|
|
c=dict(jobs=jobs,budget=dict(fit=100,postfit=50))
|
|
r=[dict(job_index=i,guard=dict(status='PASS',cpu_seconds=10,guard_cpu_seconds=0)) for i in (0,15)]
|
|
q=projected_fits(c,r);self.assertFalse(q['pass_gate']);self.assertEqual(q['remaining_projection'],225)
|
|
r[0]['guard']['status']='FAILED';self.assertFalse(projected_fits(c,r)['pass_gate'])
|
|
def test_failed_worker_peak_checked_and_retry_denied(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
limits=dict(cpu_seconds=5,wall_seconds=5,rss_bytes=40*1024**2,disk_bytes=1024**3)
|
|
receipt=supervise([sys.executable,'-c','x=bytearray(80*1024*1024); raise RuntimeError("synthetic numerical failure")'],Path(d)/'guard',limits,poll=.2)
|
|
self.assertEqual(receipt['status'],'FAILED');self.assertGreater(receipt['peak_tree_rss_bytes'],limits['rss_bytes'])
|
|
self.assertTrue(receipt['stop_reason']);self.assertGreater(receipt['cpu_seconds'],0)
|
|
with self.assertRaises(FileExistsError):supervise([sys.executable,'-c','pass'],Path(d)/'guard',limits)
|
|
|
|
|
|
class WorkerIntegrationTests(unittest.TestCase):
|
|
def test_real_tiny_xgb_clock_fp64_and_prediction_target_denial(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
base=Path(tmp);root=base/'execution';root.mkdir();parts={}
|
|
for k in range(4):
|
|
case=f'p{k}';out=base/case;out.mkdir();f=np.array([0.,.10000000001,.5,.9]);x=np.zeros((4,19),np.float32);x[:,0]=f
|
|
physical=np.array([0.,1.,2.,float(k+3)])*2
|
|
for name,value in [('Vf',f),('features',x),('raw_ids',np.arange(4)),('seconds',physical),('target',physical/2)]:np.save(out/(name+'.npy'),value)
|
|
parts[case]=dict(case=case,output=str(out),nodes=4,t_set=2.,clock_group='g0' if k<2 else 'g1',output_sha256={p.name:worker.sha(p) for p in out.glob('*.npy')})
|
|
job=dict(index=0,protocol='v4_random',learning=['p0','p1','p2'],test=['p3'],seed=0,threads=1)
|
|
c=dict(parts={'v4_random':parts},jobs=[job]);worker.save(root/'contract.json',c)
|
|
worker.fit(root,c,job);d=root/'fits/job-00';result=json.loads((d/'result.json').read_text())
|
|
self.assertEqual(result['c'],4.25) # mean(mean(3,4),5)
|
|
job['baseline_model']=dict(path=result['checkpoint'],sha256=result['checkpoint_sha256'],format='ubj')
|
|
worker.save(root/'contract.json',c)
|
|
worker.save(root/'batch-complete.json',dict(global_stop=None,accounting_complete=True,attempted=1))
|
|
worker.save(root/'models-terminal.json',dict(models=[dict(index=0,status='PASS',checkpoint=result['checkpoint'],checkpoint_sha256=result['checkpoint_sha256'],clock_path=str(d/'clock.json'),clock_sha256=result['clock_sha256'])]))
|
|
# Remove test truth; prediction must succeed without opening it.
|
|
for name in ('target.npy','seconds.npy'):(Path(parts['p3']['output'])/name).unlink()
|
|
repo=base/'git';repo.mkdir();subprocess.run(['git','init','-q',str(repo)],check=True)
|
|
names=['contract.json','batch-complete.json','models-terminal.json']
|
|
for name in names:
|
|
dest=repo/'execution'/name;dest.parent.mkdir(exist_ok=True);shutil.copyfile(root/name,dest)
|
|
subprocess.run(['git','-C',str(repo),'add','.'],check=True)
|
|
subprocess.run(['git','-C',str(repo),'-c','user.name=Synthetic','-c','user.email=synthetic@localhost','commit','-qm','synthetic barrier'],check=True)
|
|
commit=subprocess.check_output(['git','-C',str(repo),'rev-parse','HEAD'],text=True).strip()
|
|
worker.save(root/'models-git-freeze.json',dict(commit=commit,checkout=str(repo),sha256={name:worker.sha(root/name) for name in names}))
|
|
with patch.object(Path,'cwd',return_value=base):worker.predict(root,c,job)
|
|
index=json.loads((root/'predictions/job-00/index.json').read_text());self.assertEqual(len(index),1)
|
|
with np.load(index[0]['path']) as z:np.testing.assert_array_equal(z['cF_seconds'],4.25*f*2)
|
|
def test_role_overlap_is_rejected_before_access(self):
|
|
with self.assertRaises(PermissionError):worker.role_rows({},dict(learning=['same'],test=['same']),'learning')
|
|
|
|
|
|
class CoordinatorBoundaryTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.signals={s:signal.getsignal(s) for s in (signal.SIGTERM,signal.SIGINT)}
|
|
self.tmp=tempfile.TemporaryDirectory();self.root=Path(self.tmp.name)
|
|
self.c=dict(budget=dict(fit=100.,geometry=100.,postfit=1000.,disk_bytes=10**9,total_rss_bytes=10**9))
|
|
def tearDown(self):
|
|
for s,h in self.signals.items():signal.signal(s,h)
|
|
self.tmp.cleanup()
|
|
def test_reservation_stop_never_spawns(self):
|
|
r=Runner(self.root,self.c);item=task('oversized',['-c','pass'],[8],101.)
|
|
with patch('phase2_h1_execution.subprocess.Popen') as spawn,patch('phase2_h1_execution.time.sleep'):
|
|
self.assertEqual(r.jobs('fit',[item]),[]);spawn.assert_not_called()
|
|
self.assertEqual(r.stop,'STAGE_RESERVATION_LIMIT')
|
|
def test_missing_guard_receipt_marks_incomplete_and_closes_children(self):
|
|
r=Runner(self.root,self.c);proc=Mock();proc.pid=123456;proc.poll.return_value=0;proc.wait.return_value=0
|
|
with patch('phase2_h1_execution.subprocess.Popen',return_value=proc),patch('phase2_h1_execution.time.sleep'):
|
|
with self.assertRaises(RuntimeError):r.jobs('fit',[task('missing',['-c','pass'],[8],1.)])
|
|
self.assertFalse(r.accounting_complete);self.assertEqual(r.active,{})
|
|
self.assertTrue((self.root/'global-stop.json').exists())
|
|
def test_monitor_failure_stops_before_spawn(self):
|
|
r=Runner(self.root,self.c)
|
|
with patch('phase2_h1_execution.disk_usage',side_effect=OSError('synthetic monitor failure')),patch('phase2_h1_execution.subprocess.Popen') as spawn:
|
|
self.assertEqual(r.jobs('fit',[task('never',['-c','pass'],[8],1.)]),[]);spawn.assert_not_called()
|
|
self.assertEqual(r.stop,'GLOBAL_MONITOR_ERROR')
|
|
def test_scoring_tamper_rejected_before_target_access(self):
|
|
import phase2_h1_postfit as post
|
|
(self.root/'contract.json').write_text('{}')
|
|
worker.save(self.root/'scoring-git-freeze.json',dict(sha256={'contract.json':'wrong'},commit='fake',checkout='unused'))
|
|
with patch.object(post,'contract',return_value={}),patch.object(post.np,'load') as load:
|
|
with self.assertRaises(ValueError):post.scoring_gate(self.root)
|
|
load.assert_not_called()
|
|
|
|
|
|
if __name__=='__main__':unittest.main()
|