106 lines
8.3 KiB
Python
106 lines
8.3 KiB
Python
"""CPU synthetic seed/role/cost/release/guard tests; no real arrays or GPU."""
|
|
import ast,copy,inspect,json,tempfile,unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
import numpy as np
|
|
from phase3_seed1_common import fold_view,project_cell,project_total,dispatchable,first_epoch_release,available_gpus,GPUS,assigned_job,idle_eligible_gpus
|
|
from phase2_h2_data import load_part,standardization
|
|
from phase2_h1_worker_v2 import save,sha
|
|
import phase2_h2_s2_worker as original
|
|
import phase3_seed1_worker as new
|
|
import phase3_seed1_guard as guard
|
|
|
|
class SeedTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.pilot=dict(folds=[dict(fold=i,learning=['l'+str(i)],validation=['v'+str(i)],test=['t'+str(i)],learning_nodes=2) for i in range(5)],parts={'l'+str(i):dict(nodes=2) for i in range(5)})
|
|
self.jobs=[dict(id=i,gpu_seconds=22320,validation_seconds_bound=20) for i in range(5)]
|
|
self.c=dict(jobs=self.jobs,gpu_cap=118800,postfit_gpu_reserve=7200,phase3_gpu_cap=612000,phase3_prior_gpu_once=73000)
|
|
def test_all_five_original_roles(self):
|
|
old=copy.deepcopy(self.pilot)
|
|
for i in range(5):self.assertEqual(fold_view(self.pilot,i)['folds'][0],self.pilot['folds'][i])
|
|
self.assertEqual(old,self.pilot)
|
|
def test_foreign_and_overlap_rejected(self):
|
|
for i in [True,-1,5]:
|
|
with self.assertRaises(PermissionError):fold_view(self.pilot,i)
|
|
self.pilot['folds'][0]['test']=['l0']
|
|
with self.assertRaises(ValueError):fold_view(self.pilot,0)
|
|
def test_test_role_never_used_for_train_or_normalization(self):
|
|
for i in range(5):
|
|
v=fold_view(self.pilot,i)
|
|
for role in ['learning','validation']:
|
|
with self.assertRaises(PermissionError):load_part(v,'t'+str(i),{},role,Path('/unused'))
|
|
def features(c,case,name):
|
|
self.assertEqual((case,name),('l'+str(i),'features'));return np.vstack([np.zeros(19),np.full(19,2.)])
|
|
with tempfile.TemporaryDirectory() as d,patch('phase2_h2_data.cached',features):
|
|
s=standardization(v,Path(d));self.assertEqual(s['parts'],['l'+str(i)]);self.assertEqual(s['mean'],[1.]*19)
|
|
def test_epoch_math_identical_except_seed_and_cost_wait(self):
|
|
def loop(fn):
|
|
t=ast.parse(inspect.getsource(fn));n=next(n for n in ast.walk(t) if isinstance(n,ast.For) and isinstance(n.target,ast.Name) and n.target.id=='epoch')
|
|
n.body=[b for b in n.body if not(isinstance(b,ast.If) and ast.unparse(b.test)=='epoch == 1')]
|
|
return n
|
|
a,b=loop(original.run),loop(new.run)
|
|
seq=[n for n in ast.walk(b) if isinstance(n,ast.Call) and ast.unparse(n.func)=='np.random.SeedSequence'];self.assertEqual(len(seq),1);self.assertEqual(seq[0].args[0].elts[0].value,1);seq[0].args[0].elts[0].value=0
|
|
self.assertEqual(ast.dump(a),ast.dump(b))
|
|
def test_all_rng_seed1_and_final_checkpoint(self):
|
|
tree=ast.parse(inspect.getsource(new.run));calls={ast.unparse(n.func):n for n in ast.walk(tree) if isinstance(n,ast.Call)}
|
|
for name in ['random.seed','np.random.seed','torch.manual_seed','torch.cuda.manual_seed_all']:self.assertEqual(calls[name].args[0].value,1)
|
|
saves=[n for n in ast.walk(tree) if isinstance(n,ast.Call) and ast.unparse(n.func)=='torch.save'];self.assertEqual(len(saves),1)
|
|
fields={k.arg:ast.literal_eval(k.value) for k in saves[0].args[0].keywords if k.arg in ['seed','role']};self.assertEqual(fields,dict(seed=1,role='last_epoch_only'))
|
|
def test_first_epoch_gate_not_all_folds_barrier(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
p=Path(d);(p/'jobs/0').mkdir(parents=True);save(p/'contract.json',{});self.assertFalse(first_epoch_release(p,0));save(p/'jobs/0/first-epoch-projection.json',dict(status='PASS',contract_sha256=sha(p/'contract.json')));self.assertTrue(first_epoch_release(p,0));self.assertFalse(first_epoch_release(p,1));save(p/'global-stop.json',{})
|
|
with self.assertRaises(RuntimeError):first_epoch_release(p,0)
|
|
def test_full_reserves_then_settled_cost_once(self):
|
|
self.assertEqual(project_total(self.c,{}, {})['projected_gpu_seconds'],118800)
|
|
d=project_total(self.c,{}, {0:dict(gpu_seconds_charged=20000)})
|
|
self.assertEqual(d['projected_gpu_seconds'],116480);self.assertEqual(d['projected_phase3_gpu_seconds'],189480)
|
|
def test_validation_and_setup_in_projection(self):
|
|
o=dict(epoch=1,charged=100,max_epoch_seconds=80,max_validation_seconds=30)
|
|
d=project_cell(self.jobs[0],o);self.assertEqual(d['projected_gpu_seconds'],100+1.25*(199*80+20*30)+120)
|
|
def test_boundary_overrun_stops(self):
|
|
d=project_cell(self.jobs[0],dict(epoch=1,charged=100,max_epoch_seconds=100));self.assertFalse(d['within_cap'])
|
|
self.assertEqual(project_total(self.c,{0:d},{})['status'],'STOP_BUDGET_PROJECTION')
|
|
def test_parallel_dispatch_respects_held_gpus_and_identity(self):
|
|
self.assertEqual([j['id'] for j in dispatchable(self.jobs,[],set(GPUS[:2]))],[0,1])
|
|
self.assertEqual([j['id'] for j in dispatchable(self.jobs[2:],[dict(self.jobs[0],gpu_uuid=GPUS[0]),dict(self.jobs[1],gpu_uuid=GPUS[1])],set(GPUS))],[2,3])
|
|
self.assertEqual([j['id'] for j in dispatchable(self.jobs[2:],[],{GPUS[0],GPUS[3]})],[2,3])
|
|
def test_gpu2_waits_for_prior_track_postfit(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
r=Path(d);[(r/n).mkdir() for n in ['item','train','post']];c=dict(item1_recovery_root=str(r/'item'),track_a_training_root=str(r/'train'),track_a_postfit_root=str(r/'post'))
|
|
with patch('psutil.process_iter',return_value=[]):
|
|
self.assertEqual(available_gpus(c),{GPUS[0]});save(r/'item/complete.json',{});save(r/'train/complete.json',{})
|
|
self.assertEqual(available_gpus(c),{GPUS[0],GPUS[1],GPUS[3]});save(r/'post/complete.json',{});self.assertEqual(available_gpus(c),set(GPUS))
|
|
def test_busy_first_idle_second_does_not_consume_first_fold(self):
|
|
pending=copy.deepcopy(self.jobs);queries=[]
|
|
def query(u):
|
|
queries.append(u);return (2*2**30,50) if u==GPUS[0] else (5*2**20,0)
|
|
idle,observed=idle_eligible_gpus(set(GPUS[:2]),query)
|
|
assigned=dispatchable(pending,[],idle)
|
|
self.assertEqual(queries,GPUS[:2]);self.assertEqual([(j['id'],j['gpu_uuid']) for j in assigned],[(0,GPUS[1])]);self.assertEqual(pending,self.jobs);self.assertFalse(observed[0]['idle'])
|
|
def test_all_busy_leaves_pending_without_assignment(self):
|
|
pending=copy.deepcopy(self.jobs);idle,_=idle_eligible_gpus({GPUS[0]},lambda u:(200*2**20,0));self.assertEqual(dispatchable(pending,[],idle),[]);self.assertEqual(pending,self.jobs)
|
|
def test_ineligible_gpu_not_queried_and_bad_probe_rejected(self):
|
|
calls=[]
|
|
def query(u):calls.append(u);return 0,0
|
|
idle,_=idle_eligible_gpus({GPUS[3]},query);self.assertEqual(calls,[GPUS[3]]);self.assertEqual(idle,{GPUS[3]})
|
|
with self.assertRaises(ValueError):idle_eligible_gpus({GPUS[0]},lambda u:(float('nan'),0))
|
|
def test_assignment_is_hash_bound_and_fold_specific(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
r=Path(d);(r/'jobs/2').mkdir(parents=True);save(r/'contract.json',{});p=r/'jobs/2/assignment.json';save(p,dict(id=2,gpu_uuid=GPUS[3],contract_sha256=sha(r/'contract.json')))
|
|
h=sha(p);self.assertEqual(assigned_job(r,self.c,2,h)['gpu_uuid'],GPUS[3]);save(p,dict(id=2,gpu_uuid=GPUS[0],contract_sha256=sha(r/'contract.json')))
|
|
with self.assertRaises(ValueError):assigned_job(r,self.c,2,h)
|
|
def guard_exercise(self,peak=100,monitor=0):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
q=Path(d);save(q/'contract.json',{});r=q/'jobs/0';r.mkdir(parents=True)
|
|
j=dict(id=0,gpu_uuid='synthetic-seed1',guard_cpus=[0],gpu_seconds=600,cpu_seconds=600,setup_gpu_seconds=60,learning_parts=2,target_steps=400,rss_bytes=16*2**30,vram_bytes=15*2**30,disk_bytes=20*2**30)
|
|
def work(*args,**kwargs):
|
|
(r/'worker').mkdir();save(r/'worker/result.json',dict(status='COMPLETE',fold=0,completed_epochs=200,optimizer_steps=400,peak_allocated_bytes=100,peak_reserved_bytes=peak));return dict(status='PASS',cpu_seconds=.01,monitor_errors=monitor)
|
|
with patch.object(guard,'config',return_value=(dict(jobs=[j]),{})),patch.object(guard,'assigned_job',return_value=j),patch.object(guard.os,'sched_setaffinity'),patch.object(guard,'device_usage',return_value=(0,0)),patch.object(guard,'supervise',work),patch.object(guard.time,'sleep'):
|
|
rc=guard.run(q,0,'synthetic')
|
|
return rc,json.loads((r/'complete.json').read_text())
|
|
def test_guard_clean_complete(self):self.assertEqual(self.guard_exercise()[0],0)
|
|
def test_guard_rejects_monitor_errors(self):self.assertIn('WORKER_MONITOR_ERRORS',self.guard_exercise(monitor=1)[1]['reasons'])
|
|
def test_guard_preserves_excess_recorded_peak(self):self.assertIn('RECORDED_VRAM_LIMIT',self.guard_exercise(peak=16*2**30)[1]['reasons'])
|
|
|
|
if __name__=='__main__':unittest.main()
|