"""CPU synthetic seed/role/cost/release/guard tests; no real arrays or GPU.""" import ast,copy,inspect,json,tempfile,unittest from pathlib import Path from unittest.mock import patch import numpy as np from phase3_seed1_common import fold_view,project_cell,project_total,dispatchable,first_epoch_release,available_gpus,GPUS,assigned_job,idle_eligible_gpus from phase2_h2_data import load_part,standardization from phase2_h1_worker_v2 import save,sha import phase2_h2_s2_worker as original import phase3_seed1_worker as new import phase3_seed1_guard as guard class SeedTests(unittest.TestCase): def setUp(self): self.pilot=dict(folds=[dict(fold=i,learning=['l'+str(i)],validation=['v'+str(i)],test=['t'+str(i)],learning_nodes=2) for i in range(5)],parts={'l'+str(i):dict(nodes=2) for i in range(5)}) self.jobs=[dict(id=i,gpu_seconds=22320,validation_seconds_bound=20) for i in range(5)] self.c=dict(jobs=self.jobs,gpu_cap=118800,postfit_gpu_reserve=7200,phase3_gpu_cap=612000,phase3_prior_gpu_once=73000) def test_all_five_original_roles(self): old=copy.deepcopy(self.pilot) for i in range(5):self.assertEqual(fold_view(self.pilot,i)['folds'][0],self.pilot['folds'][i]) self.assertEqual(old,self.pilot) def test_foreign_and_overlap_rejected(self): for i in [True,-1,5]: with self.assertRaises(PermissionError):fold_view(self.pilot,i) self.pilot['folds'][0]['test']=['l0'] with self.assertRaises(ValueError):fold_view(self.pilot,0) def test_test_role_never_used_for_train_or_normalization(self): for i in range(5): v=fold_view(self.pilot,i) for role in ['learning','validation']: with self.assertRaises(PermissionError):load_part(v,'t'+str(i),{},role,Path('/unused')) def features(c,case,name): self.assertEqual((case,name),('l'+str(i),'features'));return np.vstack([np.zeros(19),np.full(19,2.)]) with tempfile.TemporaryDirectory() as d,patch('phase2_h2_data.cached',features): s=standardization(v,Path(d));self.assertEqual(s['parts'],['l'+str(i)]);self.assertEqual(s['mean'],[1.]*19) def test_epoch_math_identical_except_seed_and_cost_wait(self): def loop(fn): t=ast.parse(inspect.getsource(fn));n=next(n for n in ast.walk(t) if isinstance(n,ast.For) and isinstance(n.target,ast.Name) and n.target.id=='epoch') n.body=[b for b in n.body if not(isinstance(b,ast.If) and ast.unparse(b.test)=='epoch == 1')] return n a,b=loop(original.run),loop(new.run) seq=[n for n in ast.walk(b) if isinstance(n,ast.Call) and ast.unparse(n.func)=='np.random.SeedSequence'];self.assertEqual(len(seq),1);self.assertEqual(seq[0].args[0].elts[0].value,1);seq[0].args[0].elts[0].value=0 self.assertEqual(ast.dump(a),ast.dump(b)) def test_all_rng_seed1_and_final_checkpoint(self): tree=ast.parse(inspect.getsource(new.run));calls={ast.unparse(n.func):n for n in ast.walk(tree) if isinstance(n,ast.Call)} for name in ['random.seed','np.random.seed','torch.manual_seed','torch.cuda.manual_seed_all']:self.assertEqual(calls[name].args[0].value,1) saves=[n for n in ast.walk(tree) if isinstance(n,ast.Call) and ast.unparse(n.func)=='torch.save'];self.assertEqual(len(saves),1) fields={k.arg:ast.literal_eval(k.value) for k in saves[0].args[0].keywords if k.arg in ['seed','role']};self.assertEqual(fields,dict(seed=1,role='last_epoch_only')) def test_first_epoch_gate_not_all_folds_barrier(self): with tempfile.TemporaryDirectory() as d: p=Path(d);(p/'jobs/0').mkdir(parents=True);save(p/'contract.json',{});self.assertFalse(first_epoch_release(p,0));save(p/'jobs/0/first-epoch-projection.json',dict(status='PASS',contract_sha256=sha(p/'contract.json')));self.assertTrue(first_epoch_release(p,0));self.assertFalse(first_epoch_release(p,1));save(p/'global-stop.json',{}) with self.assertRaises(RuntimeError):first_epoch_release(p,0) def test_full_reserves_then_settled_cost_once(self): self.assertEqual(project_total(self.c,{}, {})['projected_gpu_seconds'],118800) d=project_total(self.c,{}, {0:dict(gpu_seconds_charged=20000)}) self.assertEqual(d['projected_gpu_seconds'],116480);self.assertEqual(d['projected_phase3_gpu_seconds'],189480) def test_validation_and_setup_in_projection(self): o=dict(epoch=1,charged=100,max_epoch_seconds=80,max_validation_seconds=30) d=project_cell(self.jobs[0],o);self.assertEqual(d['projected_gpu_seconds'],100+1.25*(199*80+20*30)+120) def test_boundary_overrun_stops(self): d=project_cell(self.jobs[0],dict(epoch=1,charged=100,max_epoch_seconds=100));self.assertFalse(d['within_cap']) self.assertEqual(project_total(self.c,{0:d},{})['status'],'STOP_BUDGET_PROJECTION') def test_parallel_dispatch_respects_held_gpus_and_identity(self): self.assertEqual([j['id'] for j in dispatchable(self.jobs,[],set(GPUS[:2]))],[0,1]) self.assertEqual([j['id'] for j in dispatchable(self.jobs[2:],[dict(self.jobs[0],gpu_uuid=GPUS[0]),dict(self.jobs[1],gpu_uuid=GPUS[1])],set(GPUS))],[2,3]) self.assertEqual([j['id'] for j in dispatchable(self.jobs[2:],[],{GPUS[0],GPUS[3]})],[2,3]) def test_gpu2_waits_for_prior_track_postfit(self): with tempfile.TemporaryDirectory() as d: r=Path(d);[(r/n).mkdir() for n in ['item','train','post']];c=dict(item1_recovery_root=str(r/'item'),track_a_training_root=str(r/'train'),track_a_postfit_root=str(r/'post')) with patch('psutil.process_iter',return_value=[]): self.assertEqual(available_gpus(c),{GPUS[0]});save(r/'item/complete.json',{});save(r/'train/complete.json',{}) self.assertEqual(available_gpus(c),{GPUS[0],GPUS[1],GPUS[3]});save(r/'post/complete.json',{});self.assertEqual(available_gpus(c),set(GPUS)) def test_busy_first_idle_second_does_not_consume_first_fold(self): pending=copy.deepcopy(self.jobs);queries=[] def query(u): queries.append(u);return (2*2**30,50) if u==GPUS[0] else (5*2**20,0) idle,observed=idle_eligible_gpus(set(GPUS[:2]),query) assigned=dispatchable(pending,[],idle) self.assertEqual(queries,GPUS[:2]);self.assertEqual([(j['id'],j['gpu_uuid']) for j in assigned],[(0,GPUS[1])]);self.assertEqual(pending,self.jobs);self.assertFalse(observed[0]['idle']) def test_all_busy_leaves_pending_without_assignment(self): pending=copy.deepcopy(self.jobs);idle,_=idle_eligible_gpus({GPUS[0]},lambda u:(200*2**20,0));self.assertEqual(dispatchable(pending,[],idle),[]);self.assertEqual(pending,self.jobs) def test_ineligible_gpu_not_queried_and_bad_probe_rejected(self): calls=[] def query(u):calls.append(u);return 0,0 idle,_=idle_eligible_gpus({GPUS[3]},query);self.assertEqual(calls,[GPUS[3]]);self.assertEqual(idle,{GPUS[3]}) with self.assertRaises(ValueError):idle_eligible_gpus({GPUS[0]},lambda u:(float('nan'),0)) def test_assignment_is_hash_bound_and_fold_specific(self): with tempfile.TemporaryDirectory() as d: r=Path(d);(r/'jobs/2').mkdir(parents=True);save(r/'contract.json',{});p=r/'jobs/2/assignment.json';save(p,dict(id=2,gpu_uuid=GPUS[3],contract_sha256=sha(r/'contract.json'))) h=sha(p);self.assertEqual(assigned_job(r,self.c,2,h)['gpu_uuid'],GPUS[3]);save(p,dict(id=2,gpu_uuid=GPUS[0],contract_sha256=sha(r/'contract.json'))) with self.assertRaises(ValueError):assigned_job(r,self.c,2,h) def guard_exercise(self,peak=100,monitor=0): with tempfile.TemporaryDirectory() as d: q=Path(d);save(q/'contract.json',{});r=q/'jobs/0';r.mkdir(parents=True) j=dict(id=0,gpu_uuid='synthetic-seed1',guard_cpus=[0],gpu_seconds=600,cpu_seconds=600,setup_gpu_seconds=60,learning_parts=2,target_steps=400,rss_bytes=16*2**30,vram_bytes=15*2**30,disk_bytes=20*2**30) def work(*args,**kwargs): (r/'worker').mkdir();save(r/'worker/result.json',dict(status='COMPLETE',fold=0,completed_epochs=200,optimizer_steps=400,peak_allocated_bytes=100,peak_reserved_bytes=peak));return dict(status='PASS',cpu_seconds=.01,monitor_errors=monitor) with patch.object(guard,'config',return_value=(dict(jobs=[j]),{})),patch.object(guard,'assigned_job',return_value=j),patch.object(guard.os,'sched_setaffinity'),patch.object(guard,'device_usage',return_value=(0,0)),patch.object(guard,'supervise',work),patch.object(guard.time,'sleep'): rc=guard.run(q,0,'synthetic') return rc,json.loads((r/'complete.json').read_text()) def test_guard_clean_complete(self):self.assertEqual(self.guard_exercise()[0],0) def test_guard_rejects_monitor_errors(self):self.assertIn('WORKER_MONITOR_ERRORS',self.guard_exercise(monitor=1)[1]['reasons']) def test_guard_preserves_excess_recorded_peak(self):self.assertIn('RECORDED_VRAM_LIMIT',self.guard_exercise(peak=16*2**30)[1]['reasons']) if __name__=='__main__':unittest.main()