160 lines
9.5 KiB
Python
160 lines
9.5 KiB
Python
"""Synthetic acceptance for roles, comparisons, dispatch, budgets and real fit APIs."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import numpy as np
|
|
from r4bc_common import *
|
|
from r4bc_fit import assemble, run as fit
|
|
from r4bc_guard import supervise
|
|
from r4bc_postfit import component_table, git_gate
|
|
from run_r4bc import eligible_cells, tracks, cost_check
|
|
|
|
|
|
def fixture(base):
|
|
rng=np.random.default_rng(44);parts=[]
|
|
for k,role in enumerate(('learning','learning','validation','test')):
|
|
case=f'case-{k}';p=base/case;p.mkdir(parents=True);v=p/'v5';v.mkdir()
|
|
n=80;raw=np.arange(n,dtype=np.int64)*2+10;ids=np.arange(2,n,2,dtype=np.int64)
|
|
counts=np.arange(len(ids))%3+1;scale=.4+.1*k
|
|
xx=rng.normal(size=(n,19));seconds=(xx[:,0]+4.)*scale
|
|
np.save(v/'features.npy',xx);np.save(v/'target.npy',seconds/scale);np.save(v/'raw_ids.npy',raw);np.save(v/'seconds.npy',seconds)
|
|
old=np.column_stack([xx+1.,np.full(n,scale)])
|
|
np.savez(p/'r2.npz',features=old)
|
|
np.savez(p/'queries.npz',eligible_s2=np.ones(len(ids),bool),anchor_ids=ids,raw_node_ids=raw[ids],multiplicity=counts,near_wall=np.zeros(len(ids),bool))
|
|
label=p/'label.npz';np.savez(label,seconds=seconds[ids],raw_ids=raw[ids],counts=counts)
|
|
parts.append(dict(case=case,component_id='group-'+str(k//2),nodes=n,
|
|
assignments=[dict(fold=0,role=role,learning_pools=['25','50','100'] if role=='learning' else [])],
|
|
preparation=dict(preparation_root=str(p),t_set=scale,eligible_unique=len(ids),
|
|
output_sha256={f.name:sha(f) for f in p.glob('*.npz') if f.name!='label.npz'}),
|
|
v5_cache=dict(output=str(v),nodes=n,t_scale_seconds=scale,output_sha256={f.name:sha(f) for f in v.glob('*.npy')}),
|
|
label=dict(path=str(label),sha256=sha(label))))
|
|
jobs=make_jobs()
|
|
c=dict(parts=parts,jobs=jobs,code_sha256={},source_sha256={},first_cell=2,
|
|
limits=dict(fit_cpu_seconds=122400),arm_order=list(ARMS))
|
|
save(base/'contract.json',c);return c
|
|
|
|
|
|
class ProtocolTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.root=Path(os.environ.get('R4BC_TEST_ROOT',tempfile.mkdtemp(prefix='r4bc-tests-')))
|
|
cls.root.mkdir(parents=True,exist_ok=True);cls.c=fixture(cls.root/'fixture')
|
|
|
|
def test_01_jobs_count_and_order(self):
|
|
jobs=make_jobs();self.assertEqual(len(jobs),1005)
|
|
self.assertEqual(sum(j['kind']=='HPO' for j in jobs),900)
|
|
self.assertEqual([j['arm'] for j in jobs[:67] if j['kind']=='FINAL'],list(ARMS))
|
|
self.assertEqual([jobs[i]['pool'] for i in (0,335,670)],[100,50,25])
|
|
|
|
def test_02_pool_barrier_waits_for_active_final(self):
|
|
c=dict(jobs=make_jobs(),first_cell=2);t=tracks(c);cursor={i:0 for i in t}
|
|
for cell in (2,5,8,11,14):cursor[cell]=len(t[cell])
|
|
self.assertEqual(eligible_cells(c,cursor,checkpoint_passed=True,active_cells={14}),[14])
|
|
self.assertEqual(eligible_cells(c,cursor,checkpoint_passed=True),[1,4,7,10,13])
|
|
|
|
def test_03_weight_scale_and_multiplicity(self):
|
|
counts=[np.array([1,3]),np.array([2,2]),np.array([1,1])];g=['a','a','b']
|
|
w=hierarchy_weights(counts,g);np.testing.assert_allclose(w,[.0625,.1875,.125,.125,.25,.25])
|
|
np.testing.assert_allclose(hierarchy_weights(counts,g,xgb=True),6*w)
|
|
|
|
def test_04_test_role_denied_before_any_target_load(self):
|
|
job=dict(fold=0,pool=100,kind='FINAL',arm='H0p');row=self.c['parts'][-1]
|
|
with patch('r4bc_common.np.load',side_effect=AssertionError('must not open labels')):
|
|
with self.assertRaises(InputViolation):targets(row,job,'learning',full=False,access=[])
|
|
with self.assertRaises(InputViolation):targets(row,job,'test',full=False,access=[])
|
|
|
|
def test_05_final_cannot_read_validation(self):
|
|
with self.assertRaises(InputViolation):targets(self.c['parts'][2],dict(fold=0,pool=100,kind='FINAL',arm='H1'),'validation',full=False,access=[])
|
|
|
|
def test_06_hash_violation_global_type(self):
|
|
with self.assertRaises(InputViolation):checked(self.root/'fixture/contract.json','0'*64)
|
|
|
|
def test_07_selection_complete_and_tie(self):
|
|
trials=[dict(config_index=i,status='PASS',score=1.) for i in range(12)]
|
|
trials[0]['status']='FAILED';self.assertEqual(selection(trials)['config_index'],1)
|
|
with self.assertRaises(InputViolation):selection(trials[:-1])
|
|
self.assertEqual(selection([dict(config_index=i,status='FAILED',score=None) for i in range(12)])['status'],'NA_ALL_HPO_FAILED')
|
|
|
|
def test_08_budget_reservation(self):
|
|
self.assertTrue(can_reserve(100,3,7300));self.assertFalse(can_reserve(101,3,7300))
|
|
self.assertFalse(can_reserve(float('nan'),0,7300))
|
|
|
|
def assembled(self,arm):
|
|
output=self.root/('arrays-'+arm);output.mkdir(exist_ok=True)
|
|
job=dict(fold=0,pool=100,arm=arm,kind='FINAL');access=[]
|
|
x,y,w,rows=assemble(self.c,job,output,access)
|
|
self.assertEqual({a['role'] for a in access},{'learning'})
|
|
return x,y,w
|
|
|
|
def test_09_drop_only_twentieth_column(self):
|
|
a,y,w=self.assembled('H0p');b,yb,wb=self.assembled('H1')
|
|
np.testing.assert_array_equal(a[:,:19],b);np.testing.assert_array_equal(y,yb);np.testing.assert_array_equal(w,wb)
|
|
self.assertEqual((a.shape[1],b.shape[1]),(20,19))
|
|
|
|
def test_10_query_weight_change_keeps_rows(self):
|
|
a,y,w=self.assembled('H2');b,z,u=self.assembled('H3')
|
|
np.testing.assert_array_equal(a,b);np.testing.assert_array_equal(y,z)
|
|
self.assertAlmostEqual(w.sum(),1.);np.testing.assert_array_equal(u,np.ones(len(u)))
|
|
|
|
def test_11_full_node_and_precision_interface(self):
|
|
h,y,w=self.assembled('H4');a,z,u=self.assembled('A')
|
|
self.assertEqual(len(y),160);np.testing.assert_array_equal(a,h.astype(np.float32));np.testing.assert_array_equal(z,y.astype(np.float32))
|
|
self.assertEqual(a.dtype,np.float32);np.testing.assert_array_equal(w,u)
|
|
q,_,qw=self.assembled('Q');self.assertAlmostEqual(qw.sum(),len(q))
|
|
|
|
def test_12_complete_components_not_partial_parts(self):
|
|
roster=[dict(case='a',component_id='g'),dict(case='b',component_id='g'),dict(case='c',component_id='h')]
|
|
rows=[dict(case=x,arm=a,relative_l2=dict(value=1.)) for x in ('a','b','c') for a in ('A','B') if (x,a)!=('b','B')]
|
|
table=component_table(rows,roster,('A','B'),'relative_l2')
|
|
self.assertEqual(table['component_ids'],['h']);self.assertEqual(table['parts'],1)
|
|
|
|
def test_13_bootstrap_absolute_direction(self):
|
|
r=paired_difference({'a':3.,'b':4.},{'a':1.,'b':2.},replicates=50)
|
|
self.assertEqual((r['value'],r['low'],r['high']),(2.,2.,2.))
|
|
|
|
def test_14_prediction_gate_requires_real_git_objects(self):
|
|
p=self.root/'gate';p.mkdir(exist_ok=True)
|
|
save(p/'fake.json',dict(commit='',checkout=''))
|
|
with self.assertRaises(InputViolation):git_gate(p,'fake.json',[])
|
|
|
|
def test_15_guard_wall_failure_and_no_retry(self):
|
|
d=self.root/'guard-wall'
|
|
limits=dict(cpu_seconds=5,wall_seconds=.15,rss_bytes=256*2**20,disk_bytes=2**30)
|
|
r=supervise([sys.executable,'-c','import time;time.sleep(2)'],d,limits,poll=.02)
|
|
self.assertEqual(r['status'],'FAILED');self.assertIn('wall_limit',r['stop_reason'])
|
|
with self.assertRaises(FileExistsError):supervise([sys.executable,'-c','pass'],d,limits)
|
|
|
|
def test_16_guard_failed_worker_still_checks_peak(self):
|
|
d=self.root/'guard-memory';limits=dict(cpu_seconds=5,wall_seconds=5,rss_bytes=8*2**20,disk_bytes=2**30)
|
|
r=supervise([sys.executable,'-c','a=bytearray(32*1024*1024);raise SystemExit(3)'],d,limits,poll=.2)
|
|
self.assertEqual(r['status'],'FAILED');self.assertGreater(r['peak_tree_rss_bytes'],limits['rss_bytes'])
|
|
self.assertIn('rss_limit',r['stop_reason'])
|
|
|
|
def test_17_actual_hist_hpo_then_learning_only_final(self):
|
|
root=self.root/'fixture';job=self.c['jobs'][0]
|
|
(job_dir(root,0)).mkdir(parents=True,exist_ok=True);fit(root,0)
|
|
result=read(job_dir(root,0)/'output/result.json');self.assertTrue(np.isfinite(result['validation_score']))
|
|
access=read(job_dir(root,0)/'output/access.json');self.assertEqual({x['role'] for x in access},{'learning','validation'})
|
|
final=next(j for j in self.c['jobs'] if j['arm']=='H0p' and j['kind']=='FINAL' and j['cell']==2)
|
|
receipt=self.root/'synthetic-receipt.json';save(receipt,{'synthetic':True})
|
|
trials=[dict(config_index=i,status='PASS',score=float(i),receipt_path=str(receipt),receipt_sha256=sha(receipt)) for i in range(12)]
|
|
path=selection_path(root,final);path.parent.mkdir(exist_ok=True);save(path,selection(trials))
|
|
job_dir(root,final['index']).mkdir(parents=True,exist_ok=True);fit(root,final['index'])
|
|
self.assertEqual({x['role'] for x in read(job_dir(root,final['index'])/'output/access.json')},{'learning'})
|
|
with self.assertRaises(FileExistsError):fit(root,final['index'])
|
|
|
|
def test_18_actual_xgb_fixed_query_variant(self):
|
|
root=self.root/'fixture';job=next(j for j in self.c['jobs'] if j['cell']==2 and j['arm']=='Q')
|
|
job_dir(root,job['index']).mkdir(parents=True,exist_ok=True);fit(root,job['index'])
|
|
r=read(job_dir(root,job['index'])/'output/result.json')
|
|
self.assertEqual(r['params']['n_estimators'],100);self.assertEqual(r['params']['max_depth'],8)
|
|
self.assertEqual({x['role'] for x in read(job_dir(root,job['index'])/'output/access.json')},{'learning'})
|
|
|
|
|
|
if __name__=='__main__':unittest.main(verbosity=2)
|