"""S2 CPU synthetic boundaries; no real learning/validation/test arrays or GPU.""" import ast,copy,inspect,json,tempfile,unittest from pathlib import Path from unittest.mock import patch import numpy as np from phase2_h2_s2_common import fold_view,projection,first_epoch_release,confirmed,execution_command from phase2_h2_data import load_part,standardization import phase2_h2_s1_worker as oldworker import phase2_h2_s2_worker as newworker import phase2_h2_s2_guard as guard from phase2_h1_worker_v2 import save,sha from s2_main_memory_guard import PeakRecords class Roles(unittest.TestCase): def setUp(self): self.pilot=dict(folds=[dict(fold=i,learning=['learn'+str(i)],validation=['val'+str(i)],test=['test'+str(i)],learning_nodes=2) for i in range(5)],parts={'learn'+str(i):dict(nodes=2) for i in range(5)}) def test_all_new_folds_map_exact_original_roles_without_mutation(self): before=copy.deepcopy(self.pilot) for f in range(1,5): v=fold_view(self.pilot,f);self.assertEqual(v['folds'][0],self.pilot['folds'][f]);self.assertEqual(v['folds'][0]['fold'],f) self.assertEqual(self.pilot,before) def test_fold0_and_role_overlap_rejected(self): for f in [0,5,-1]: with self.assertRaises(PermissionError):fold_view(self.pilot,f) self.pilot['folds'][2]['test']=['learn2'] with self.assertRaises(ValueError):fold_view(self.pilot,2) def test_test_members_rejected_before_geometry_or_target_io(self): for f in range(1,5): for role in ['learning','validation']: with self.assertRaises(PermissionError):load_part(fold_view(self.pilot,f),'test'+str(f),{},role,Path('/unused')) def test_normalization_uses_only_actual_learning(self): for f in range(1,5): seen=[] def features(c,case,name): self.assertEqual(case,'learn'+str(f));self.assertEqual(name,'features');seen.append(case);return np.vstack([np.zeros(19),np.full(19,2.)]) with tempfile.TemporaryDirectory() as p,patch('phase2_h2_data.cached',features): v=standardization(fold_view(self.pilot,f),Path(p));self.assertEqual(v['parts'],['learn'+str(f)]);self.assertEqual(v['mean'],[1.]*19);self.assertEqual(v['scale'],[1.]*19);self.assertEqual(seen,['learn'+str(f)]) def test_scientific_epoch_loop_AST_unchanged_except_cost_wait(self): def loop(fn): t=ast.parse(inspect.getsource(fn));return next(n for n in ast.walk(t) if isinstance(n,ast.For) and isinstance(n.target,ast.Name) and n.target.id=='epoch') old=loop(oldworker.run);new=loop(newworker.run);new.body=[n for n in new.body if not(isinstance(n,ast.If) and ast.unparse(n.test)=='epoch == 1')] self.assertEqual(ast.dump(old),ast.dump(new)) class BudgetAndGate(unittest.TestCase): def setUp(self): self.c=dict(prior_gpu_seconds=18858.21490871499,total_h2_gpu_seconds=144000,prediction_gpu_reserve=2400,jobs=[dict(fold=f,gpu_seconds=(144000-18858.21490871499-2400)/4,validation_seconds_bound=12.) for f in range(1,5)]) self.obs={f:dict(epoch=1,charged_gpu_seconds=200.,max_train_epoch_seconds=90.,max_validation_seconds=0.) for f in range(1,5)} def test_old_cost_once_plus_all_four_waits_and_prediction_reserve(self): d=projection(self.c,self.obs);expected=18858.21490871499+4*(200+1.25*(199*90+20*12)+60)+2400;self.assertEqual(d['projected_total_gpu_seconds'],expected);self.assertEqual(d['status'],'PASS') def test_budget_and_per_job_limits_stop(self): o=copy.deepcopy(self.obs);o[1]['max_train_epoch_seconds']=150;self.assertEqual(projection(self.c,o)['status'],'STOP_BUDGET_PROJECTION') for v in o.values():v['max_train_epoch_seconds']=200 self.assertGreater(projection(self.c,o)['projected_total_gpu_seconds'],144000) def test_missing_epoch_invalid_timing_rejected(self): with self.assertRaises(ValueError):projection(self.c,{1:self.obs[1]}) for field,value in [('epoch',0),('max_train_epoch_seconds',float('nan')),('charged_gpu_seconds',-1)]: o=copy.deepcopy(self.obs);o[2][field]=value with self.assertRaises(ValueError):projection(self.c,o) def test_actual_validation_cost_can_raise_projection(self): a=projection(self.c,self.obs);o=copy.deepcopy(self.obs);o[1]['max_validation_seconds']=30;self.assertGreater(projection(self.c,o)['projected_total_gpu_seconds'],a['projected_total_gpu_seconds']) def test_epoch10_and_final_remaining_validation_count(self): o=copy.deepcopy(self.obs) for v in o.values():v['epoch']=200 self.assertAlmostEqual(projection(self.c,o)['projected_total_gpu_seconds'],18858.21490871499+4*(200+60)+2400) def test_first_epoch_barrier_requires_exact_pass(self): with tempfile.TemporaryDirectory() as d: p=Path(d);self.assertFalse(first_epoch_release(p,'fixed')) for result in [dict(status='STOP_BUDGET_PROJECTION',contract_sha256='fixed'),dict(status='PASS',contract_sha256='other')]: (p/'first-epoch-projection.json').write_text(json.dumps(result)) with self.assertRaises(RuntimeError):first_epoch_release(p,'fixed') (p/'first-epoch-projection.json').write_text(json.dumps(dict(status='PASS',contract_sha256='fixed')));self.assertTrue(first_epoch_release(p,'fixed')) (p/'global-stop.json').write_text('{}') with self.assertRaises(RuntimeError):first_epoch_release(p,'fixed') def test_confirmation_is_strict_and_complete_five_fold_only(self): self.assertTrue(confirmed(-.001,552,204)) for h,n,g in [(0,552,204),(.001,552,204),(None,552,204),(float('nan'),552,204),(-.001,121,37),(-.001,551,204),(-.001,552,203)]:self.assertFalse(confirmed(h,n,g)) def test_exact_process_identity_excludes_preparation(self): self.assertFalse(execution_command(['python','tools/prepare_phase2_h2_s2.py']));self.assertTrue(execution_command(['python','tools/phase2_h2_s2_queue.py'])) def test_failure_peak_remains_even_after_lower_final_peak(self): with tempfile.TemporaryDirectory() as d: p=Path(d);t=PeakRecords();(p/'failure.json').write_text(json.dumps(dict(peak_allocated_bytes=100,peak_reserved_bytes=16*2**30)));self.assertTrue(t.scan(p));(p/'failure.json').unlink();(p/'result.json').write_text(json.dumps(dict(peak_allocated_bytes=100,peak_reserved_bytes=200)));self.assertTrue(t.scan(p));self.assertEqual(t.maxima['peak_reserved_bytes'],16*2**30) class GuardLifecycle(unittest.TestCase): def exercise(self,peak): with tempfile.TemporaryDirectory() as d: q=Path(d);save(q/'contract.json',dict(synthetic=True));r=q/'folds/1';r.mkdir(parents=True) job=dict(fold=1,gpu_uuid='synthetic',guard_cpus=[0],gpu_seconds=600,cpu_seconds=600,setup_gpu_seconds=60,learning_parts=2,rss_bytes=16*2**30,vram_bytes=15*2**30,disk_bytes=20*2**30) calls=[] def worker(command,*args,**kwargs): calls.append(command);self.assertEqual(command[-2:],[str(q),'1']);w=r/'worker';w.mkdir();save(w/'result.json',dict(status='COMPLETE',fold=1,completed_epochs=200,optimizer_steps=400,peak_allocated_bytes=100,peak_reserved_bytes=peak));return dict(status='PASS',cpu_seconds=.01,monitor_errors=[]) with patch.object(guard,'config',return_value=(dict(jobs=[job]),{})),patch.object(guard.os,'sched_setaffinity'),patch.object(guard,'device_usage',return_value=(0,0)),patch.object(guard,'supervise',worker),patch.object(guard.time,'sleep'): rc=guard.run(q,1) self.assertEqual(len(calls),1);self.assertEqual(json.loads((r/'started.json').read_text())['contract_sha256'],sha(q/'contract.json'));return rc,json.loads((r/'complete.json').read_text()) def test_queue_contract_identity_and_complete_guard(self): rc,d=self.exercise(100);self.assertEqual(rc,0);self.assertEqual(d['status'],'PASS') def test_recorded_failure_peak_cannot_be_ignored(self): rc,d=self.exercise(16*2**30);self.assertEqual(rc,1);self.assertIn('RECORDED_VRAM_LIMIT',d['reasons']) if __name__=='__main__':unittest.main()