Files
MeltMind/tools/test_s2_r2.py

137 lines
9.4 KiB
Python

import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
import numpy as np
import run_s2_r2 as queue
import run_s2_r2_fit as fit
import s2_r2_data as data
from s2_models.objectives import hierarchy_weights
from prepare_s2_r2 import build_jobs
def synthetic_part(case,component,role,counts):
n=len(counts);x=np.zeros((n,20));x[:,-1]=2.
return dict(case=case,component_id=component,role=role,x=x,y=np.ones(n),seconds=np.ones(n)*2,counts=np.array(counts),t_set=2.,label_mode='synthetic',label_sha256='synthetic')
def fake_contract():
jobs=[];cells=[]
for cell in range(15):
ids=[]
for cfg in range(12):
ids.append(len(jobs));jobs.append(dict(index=len(jobs),cell=cell,fold=cell//3,pool=[25,50,100][cell%3],kind='HPO',config_index=cfg,cost_units=1,cost_queries=1))
cells.append(dict(index=cell,hpo_indices=ids))
for cell in range(15):jobs.append(dict(index=len(jobs),cell=cell,fold=cell//3,pool=[25,50,100][cell%3],kind='FINAL',cost_units=1,cost_queries=1))
return dict(jobs=jobs,cells=cells,worker_cpus=[8,10,12,14],coordinator_cpus=[30,31],roster_sha256='fixed',
limits=dict(worker_cpu_seconds=360000,per_fit_cpu_seconds=1800,workers=4,disk_bytes=100*2**30))
class R2Checks(unittest.TestCase):
def test_weights_match_frozen_hierarchy_without_rescaling(self):
parts=[synthetic_part('a','A','learning',[3,1]),synthetic_part('b','A','learning',[2]),synthetic_part('c','B','learning',[1,1])]
_,_,weights=data.learning_arrays(parts)
expected=hierarchy_weights([3,1,2,1,1],['a','a','b','c','c'],['A','A','A','B','B'])
np.testing.assert_array_equal(weights,expected);self.assertEqual(weights.sum(),1.)
def test_validation_is_component_equal_and_seconds_scale_correct(self):
parts=[synthetic_part('a','A','validation',[1]),synthetic_part('b','A','validation',[1]),synthetic_part('c','B','validation',[1])]
class Model:
def __init__(self):self.i=0
def predict(self,x):self.i+=1;return np.array([1.,3.,4.])[self.i-1:self.i]
value,errors=data.validation_score(Model(),parts)
self.assertEqual(errors,dict(a=0.,b=2.,c=3.));self.assertEqual(value,2.)
def test_test_case_rejected_before_feature_or_target_io(self):
roles=dict(learning_parts=['l'],validation_parts=['v'],sealed_test_parts=['t'])
with patch.object(data,'ensure_targets',side_effect=AssertionError('target IO')),patch.object(data,'sha',side_effect=AssertionError('feature IO')):
with self.assertRaises(PermissionError):data.read_part(Path('/none'),{},dict(case='t'),roles,{})
def test_final_loader_never_reads_validation(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);roster=root/'roster.json';old=root/'old.json'
roster.write_text('{}');old.write_text(json.dumps(dict(ready_parts=[])))
c=dict(roster=str(roster),roster_sha256=data.sha(roster),old_labels_manifest=str(old),old_labels_manifest_sha256=data.sha(old))
roles=dict(learning_parts=['l'],validation_parts=['v'],sealed_test_parts=['t']);rows={k:dict(case=k) for k in ['l','v']};seen=[]
def read(*args):
case=args[2]['case'];seen.append(case);return synthetic_part(case,case,'learning' if case=='l' else 'validation',[1])
with patch.object(data,'select_cell',return_value=(rows,roles)),patch.object(data,'read_part',read):
learning,validation,_=data.load_cell(root,c,dict(fold=0,pool=25,kind='FINAL'))
self.assertEqual(seen,['l']);self.assertEqual(validation,[])
def test_select_finished_configures_failure_and_tie_policy(self):
trials=[dict(config_index=i,status='STOPPED_OR_FAILED',score=None,reason='wall_limit') for i in range(12)]
self.assertEqual(queue.select_finished(0,trials)['status'],'NA_ALL_HPO_FAILED')
trials[3].update(status='PASS',score=.2);trials[1].update(status='PASS',score=.2)
r=queue.select_finished(0,trials);self.assertEqual(r['config_index'],1);self.assertEqual(r['finished_configurations'],2)
self.assertEqual(r['failed_configurations'],10)
def test_grid_plan_has_180_unique_trials_15_final_and_real_roster(self):
roster=json.loads(Path('artifacts/s2-main-preparation-2026-11-15/final-roster.json').read_text());jobs,cells=build_jobs(roster)
self.assertEqual((len(jobs),len(cells)),(195,15))
self.assertEqual(sum(j['kind']=='HPO' for j in jobs),180)
for cell in cells:self.assertEqual([jobs[i]['config_index'] for i in cell['hpo_indices']],list(range(12)))
self.assertTrue(all(j['kind']=='FINAL' and j['config_index'] is None for j in jobs[180:]))
def test_first_fit_projection_and_failed_attempt_cost_bound(self):
with tempfile.TemporaryDirectory() as t:
p=Path(t)/'result.json';p.write_text(json.dumps(dict(fit_cpu_seconds=1700)))
c=fake_contract();first=dict(worker=dict(status='PASS',cpu_seconds=1800),result_path=str(p))
self.assertFalse(queue.cost_check(c,first)['continue_within_cap'])
first['worker']['status']='STOPPED_OR_FAILED';check=queue.cost_check(c,first)
self.assertTrue(check['continue_within_cap']);self.assertEqual(check['capped_attempt_upper_bound_seconds'],351000)
self.assertIsNone(check['projected_worker_cpu_seconds'])
def test_guard_uses_cpu_cap_and_disjoint_affinity(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);seen=[]
def supervise(command,dest,name,limits,**kw):seen.append(limits);return dict(status='STOPPED_OR_FAILED',stop_reason='synthetic')
with patch.object(queue,'contract',return_value=fake_contract()),patch.object(queue.os,'sched_setaffinity') as affinity,patch.object(queue,'supervise',supervise):
self.assertEqual(queue.guard(root,0,0),1)
affinity.assert_called_once_with(0,[8])
self.assertEqual(seen[0]['job_cpu_seconds'],1800);self.assertEqual(seen[0]['max_processes'],1)
def test_actual_synthetic_histgbrt_fit_single_thread_and_fixed_params(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);(root/'contract.json').write_text('{}');(root/'jobs/job-000').mkdir(parents=True)
c=fake_contract();train=synthetic_part('learn','A','learning',[1]*240);val=synthetic_part('val','V','validation',[1]*40)
rng=np.random.default_rng(17)
for p in [train,val]:
p['x'][:,:19]=rng.normal(size=(len(p['y']),19));p['y']=1+.1*p['x'][:,0];p['seconds']=p['y']*2
with patch.object(queue,'contract',return_value=c),patch.object(fit,'load_cell',return_value=([train],[val],{})):
self.assertEqual(fit.run(root,0),0)
r=json.loads((root/'jobs/job-000/output/result.json').read_text())
self.assertEqual(r['params']['max_iter'],200);self.assertFalse(r['params']['early_stopping'])
self.assertTrue(all(p['num_threads']==1 for p in r['threadpools']));self.assertFalse(r['test_predictions'])
def test_queue_first_fit_barrier_and_unique_195_attempts(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);(root/'contract.json').write_text('{}');c=fake_contract();spawned=[];peak=[0]
class Process:
def __init__(self,index):self.index=index;self.pid=1000000+index;self.polls=0
def poll(self):
self.polls+=1
if self.polls==1:return None
job=c['jobs'][self.index];dest=root/'jobs'/f'job-{self.index:03d}';out=dest/'output';out.mkdir(exist_ok=True)
result=out/'result.json';result.write_text(json.dumps(dict(status='COMPLETE',fit_cpu_seconds=.001,validation_component_equal_relative_l2=job.get('config_index',0)/100,model_sha256='synthetic')))
(dest/'guard-result.json').write_text(json.dumps(dict(worker=dict(status='PASS',cpu_seconds=.01,returncode=0,stop_reason=None),
supervisor_cpu_seconds=.001,result_path=str(result),result_sha256=data.sha(result),job=job)))
return 0
def wait(self):return 0
def spawn(command,**kw):
i=int(command[-3])
if i>0:self.assertTrue((root/'first-fit-cost-check.json').exists())
if i>=180:self.assertTrue((root/'selections'/f"cell-{c['jobs'][i]['cell']:02d}.json").exists())
p=Process(i);spawned.append(p);peak[0]=max(peak[0],sum(p.polls<2 for p in spawned));return p
with patch.object(queue,'contract',return_value=c),patch.object(queue.os,'sched_setaffinity'),patch.object(queue.subprocess,'Popen',spawn),patch.object(queue,'tree_usage',return_value=(0.,1,1)),patch.object(queue.time,'sleep'),patch('builtins.print'):
self.assertEqual(queue.run(root),0)
r=json.loads((root/'batch-complete.json').read_text());self.assertEqual(len(spawned),195);self.assertEqual(len({p.index for p in spawned}),195)
self.assertLessEqual(peak[0],4);self.assertEqual(len(r['final_fits']),15);self.assertTrue(all(x['status']=='PASS' for x in r['final_fits']))
with patch.object(queue,'contract',return_value=c),patch.object(queue.os,'sched_setaffinity'),patch.object(queue.subprocess,'Popen') as spawn:
with self.assertRaises(AssertionError):queue.run(root)
spawn.assert_not_called()
if __name__=='__main__':unittest.main(verbosity=2)