"""Synthetic acceptance for roles, comparisons, dispatch, budgets and real fit APIs.""" import json import os from pathlib import Path import sys import tempfile import unittest from unittest.mock import patch import numpy as np from r4bc_common import * from r4bc_fit import assemble, run as fit from r4bc_guard import supervise from r4bc_postfit import component_table, git_gate from run_r4bc import eligible_cells, tracks, cost_check def fixture(base): rng=np.random.default_rng(44);parts=[] for k,role in enumerate(('learning','learning','validation','test')): case=f'case-{k}';p=base/case;p.mkdir(parents=True);v=p/'v5';v.mkdir() n=80;raw=np.arange(n,dtype=np.int64)*2+10;ids=np.arange(2,n,2,dtype=np.int64) counts=np.arange(len(ids))%3+1;scale=.4+.1*k xx=rng.normal(size=(n,19));seconds=(xx[:,0]+4.)*scale np.save(v/'features.npy',xx);np.save(v/'target.npy',seconds/scale);np.save(v/'raw_ids.npy',raw);np.save(v/'seconds.npy',seconds) old=np.column_stack([xx+1.,np.full(n,scale)]) np.savez(p/'r2.npz',features=old) np.savez(p/'queries.npz',eligible_s2=np.ones(len(ids),bool),anchor_ids=ids,raw_node_ids=raw[ids],multiplicity=counts,near_wall=np.zeros(len(ids),bool)) label=p/'label.npz';np.savez(label,seconds=seconds[ids],raw_ids=raw[ids],counts=counts) parts.append(dict(case=case,component_id='group-'+str(k//2),nodes=n, assignments=[dict(fold=0,role=role,learning_pools=['25','50','100'] if role=='learning' else [])], preparation=dict(preparation_root=str(p),t_set=scale,eligible_unique=len(ids), output_sha256={f.name:sha(f) for f in p.glob('*.npz') if f.name!='label.npz'}), v5_cache=dict(output=str(v),nodes=n,t_scale_seconds=scale,output_sha256={f.name:sha(f) for f in v.glob('*.npy')}), label=dict(path=str(label),sha256=sha(label)))) jobs=make_jobs() c=dict(parts=parts,jobs=jobs,code_sha256={},source_sha256={},first_cell=2, limits=dict(fit_cpu_seconds=122400),arm_order=list(ARMS)) save(base/'contract.json',c);return c class ProtocolTests(unittest.TestCase): @classmethod def setUpClass(cls): cls.root=Path(os.environ.get('R4BC_TEST_ROOT',tempfile.mkdtemp(prefix='r4bc-tests-'))) cls.root.mkdir(parents=True,exist_ok=True);cls.c=fixture(cls.root/'fixture') def test_01_jobs_count_and_order(self): jobs=make_jobs();self.assertEqual(len(jobs),1005) self.assertEqual(sum(j['kind']=='HPO' for j in jobs),900) self.assertEqual([j['arm'] for j in jobs[:67] if j['kind']=='FINAL'],list(ARMS)) self.assertEqual([jobs[i]['pool'] for i in (0,335,670)],[100,50,25]) def test_02_pool_barrier_waits_for_active_final(self): c=dict(jobs=make_jobs(),first_cell=2);t=tracks(c);cursor={i:0 for i in t} for cell in (2,5,8,11,14):cursor[cell]=len(t[cell]) self.assertEqual(eligible_cells(c,cursor,checkpoint_passed=True,active_cells={14}),[14]) self.assertEqual(eligible_cells(c,cursor,checkpoint_passed=True),[1,4,7,10,13]) def test_03_weight_scale_and_multiplicity(self): counts=[np.array([1,3]),np.array([2,2]),np.array([1,1])];g=['a','a','b'] w=hierarchy_weights(counts,g);np.testing.assert_allclose(w,[.0625,.1875,.125,.125,.25,.25]) np.testing.assert_allclose(hierarchy_weights(counts,g,xgb=True),6*w) def test_04_test_role_denied_before_any_target_load(self): job=dict(fold=0,pool=100,kind='FINAL',arm='H0p');row=self.c['parts'][-1] with patch('r4bc_common.np.load',side_effect=AssertionError('must not open labels')): with self.assertRaises(InputViolation):targets(row,job,'learning',full=False,access=[]) with self.assertRaises(InputViolation):targets(row,job,'test',full=False,access=[]) def test_05_final_cannot_read_validation(self): with self.assertRaises(InputViolation):targets(self.c['parts'][2],dict(fold=0,pool=100,kind='FINAL',arm='H1'),'validation',full=False,access=[]) def test_06_hash_violation_global_type(self): with self.assertRaises(InputViolation):checked(self.root/'fixture/contract.json','0'*64) def test_07_selection_complete_and_tie(self): trials=[dict(config_index=i,status='PASS',score=1.) for i in range(12)] trials[0]['status']='FAILED';self.assertEqual(selection(trials)['config_index'],1) with self.assertRaises(InputViolation):selection(trials[:-1]) self.assertEqual(selection([dict(config_index=i,status='FAILED',score=None) for i in range(12)])['status'],'NA_ALL_HPO_FAILED') def test_08_budget_reservation(self): self.assertTrue(can_reserve(100,3,7300));self.assertFalse(can_reserve(101,3,7300)) self.assertFalse(can_reserve(float('nan'),0,7300)) def assembled(self,arm): output=self.root/('arrays-'+arm);output.mkdir(exist_ok=True) job=dict(fold=0,pool=100,arm=arm,kind='FINAL');access=[] x,y,w,rows=assemble(self.c,job,output,access) self.assertEqual({a['role'] for a in access},{'learning'}) return x,y,w def test_09_drop_only_twentieth_column(self): a,y,w=self.assembled('H0p');b,yb,wb=self.assembled('H1') np.testing.assert_array_equal(a[:,:19],b);np.testing.assert_array_equal(y,yb);np.testing.assert_array_equal(w,wb) self.assertEqual((a.shape[1],b.shape[1]),(20,19)) def test_10_query_weight_change_keeps_rows(self): a,y,w=self.assembled('H2');b,z,u=self.assembled('H3') np.testing.assert_array_equal(a,b);np.testing.assert_array_equal(y,z) self.assertAlmostEqual(w.sum(),1.);np.testing.assert_array_equal(u,np.ones(len(u))) def test_11_full_node_and_precision_interface(self): h,y,w=self.assembled('H4');a,z,u=self.assembled('A') self.assertEqual(len(y),160);np.testing.assert_array_equal(a,h.astype(np.float32));np.testing.assert_array_equal(z,y.astype(np.float32)) self.assertEqual(a.dtype,np.float32);np.testing.assert_array_equal(w,u) q,_,qw=self.assembled('Q');self.assertAlmostEqual(qw.sum(),len(q)) def test_12_complete_components_not_partial_parts(self): roster=[dict(case='a',component_id='g'),dict(case='b',component_id='g'),dict(case='c',component_id='h')] rows=[dict(case=x,arm=a,relative_l2=dict(value=1.)) for x in ('a','b','c') for a in ('A','B') if (x,a)!=('b','B')] table=component_table(rows,roster,('A','B'),'relative_l2') self.assertEqual(table['component_ids'],['h']);self.assertEqual(table['parts'],1) def test_13_bootstrap_absolute_direction(self): r=paired_difference({'a':3.,'b':4.},{'a':1.,'b':2.},replicates=50) self.assertEqual((r['value'],r['low'],r['high']),(2.,2.,2.)) def test_14_prediction_gate_requires_real_git_objects(self): p=self.root/'gate';p.mkdir(exist_ok=True) save(p/'fake.json',dict(commit='',checkout='')) with self.assertRaises(InputViolation):git_gate(p,'fake.json',[]) def test_15_guard_wall_failure_and_no_retry(self): d=self.root/'guard-wall' limits=dict(cpu_seconds=5,wall_seconds=.15,rss_bytes=256*2**20,disk_bytes=2**30) r=supervise([sys.executable,'-c','import time;time.sleep(2)'],d,limits,poll=.02) self.assertEqual(r['status'],'FAILED');self.assertIn('wall_limit',r['stop_reason']) with self.assertRaises(FileExistsError):supervise([sys.executable,'-c','pass'],d,limits) def test_16_guard_failed_worker_still_checks_peak(self): d=self.root/'guard-memory';limits=dict(cpu_seconds=5,wall_seconds=5,rss_bytes=8*2**20,disk_bytes=2**30) r=supervise([sys.executable,'-c','a=bytearray(32*1024*1024);raise SystemExit(3)'],d,limits,poll=.2) self.assertEqual(r['status'],'FAILED');self.assertGreater(r['peak_tree_rss_bytes'],limits['rss_bytes']) self.assertIn('rss_limit',r['stop_reason']) def test_17_actual_hist_hpo_then_learning_only_final(self): root=self.root/'fixture';job=self.c['jobs'][0] (job_dir(root,0)).mkdir(parents=True,exist_ok=True);fit(root,0) result=read(job_dir(root,0)/'output/result.json');self.assertTrue(np.isfinite(result['validation_score'])) access=read(job_dir(root,0)/'output/access.json');self.assertEqual({x['role'] for x in access},{'learning','validation'}) final=next(j for j in self.c['jobs'] if j['arm']=='H0p' and j['kind']=='FINAL' and j['cell']==2) receipt=self.root/'synthetic-receipt.json';save(receipt,{'synthetic':True}) trials=[dict(config_index=i,status='PASS',score=float(i),receipt_path=str(receipt),receipt_sha256=sha(receipt)) for i in range(12)] path=selection_path(root,final);path.parent.mkdir(exist_ok=True);save(path,selection(trials)) job_dir(root,final['index']).mkdir(parents=True,exist_ok=True);fit(root,final['index']) self.assertEqual({x['role'] for x in read(job_dir(root,final['index'])/'output/access.json')},{'learning'}) with self.assertRaises(FileExistsError):fit(root,final['index']) def test_18_actual_xgb_fixed_query_variant(self): root=self.root/'fixture';job=next(j for j in self.c['jobs'] if j['cell']==2 and j['arm']=='Q') job_dir(root,job['index']).mkdir(parents=True,exist_ok=True);fit(root,job['index']) r=read(job_dir(root,job['index'])/'output/result.json') self.assertEqual(r['params']['n_estimators'],100);self.assertEqual(r['params']['max_depth'],8) self.assertEqual({x['role'] for x in read(job_dir(root,job['index'])/'output/access.json')},{'learning'}) if __name__=='__main__':unittest.main(verbosity=2)