98 lines
7.5 KiB
Python
98 lines
7.5 KiB
Python
"""S2 CPU synthetic boundaries; no real learning/validation/test arrays or GPU."""
|
|
import ast,copy,inspect,json,tempfile,unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
import numpy as np
|
|
from phase2_h2_s2_common import fold_view,projection,first_epoch_release,confirmed,execution_command
|
|
from phase2_h2_data import load_part,standardization
|
|
import phase2_h2_s1_worker as oldworker
|
|
import phase2_h2_s2_worker as newworker
|
|
import phase2_h2_s2_guard as guard
|
|
from phase2_h1_worker_v2 import save,sha
|
|
from s2_main_memory_guard import PeakRecords
|
|
|
|
class Roles(unittest.TestCase):
|
|
def setUp(self):
|
|
self.pilot=dict(folds=[dict(fold=i,learning=['learn'+str(i)],validation=['val'+str(i)],test=['test'+str(i)],learning_nodes=2) for i in range(5)],parts={'learn'+str(i):dict(nodes=2) for i in range(5)})
|
|
def test_all_new_folds_map_exact_original_roles_without_mutation(self):
|
|
before=copy.deepcopy(self.pilot)
|
|
for f in range(1,5):
|
|
v=fold_view(self.pilot,f);self.assertEqual(v['folds'][0],self.pilot['folds'][f]);self.assertEqual(v['folds'][0]['fold'],f)
|
|
self.assertEqual(self.pilot,before)
|
|
def test_fold0_and_role_overlap_rejected(self):
|
|
for f in [0,5,-1]:
|
|
with self.assertRaises(PermissionError):fold_view(self.pilot,f)
|
|
self.pilot['folds'][2]['test']=['learn2']
|
|
with self.assertRaises(ValueError):fold_view(self.pilot,2)
|
|
def test_test_members_rejected_before_geometry_or_target_io(self):
|
|
for f in range(1,5):
|
|
for role in ['learning','validation']:
|
|
with self.assertRaises(PermissionError):load_part(fold_view(self.pilot,f),'test'+str(f),{},role,Path('/unused'))
|
|
def test_normalization_uses_only_actual_learning(self):
|
|
for f in range(1,5):
|
|
seen=[]
|
|
def features(c,case,name):
|
|
self.assertEqual(case,'learn'+str(f));self.assertEqual(name,'features');seen.append(case);return np.vstack([np.zeros(19),np.full(19,2.)])
|
|
with tempfile.TemporaryDirectory() as p,patch('phase2_h2_data.cached',features):
|
|
v=standardization(fold_view(self.pilot,f),Path(p));self.assertEqual(v['parts'],['learn'+str(f)]);self.assertEqual(v['mean'],[1.]*19);self.assertEqual(v['scale'],[1.]*19);self.assertEqual(seen,['learn'+str(f)])
|
|
def test_scientific_epoch_loop_AST_unchanged_except_cost_wait(self):
|
|
def loop(fn):
|
|
t=ast.parse(inspect.getsource(fn));return next(n for n in ast.walk(t) if isinstance(n,ast.For) and isinstance(n.target,ast.Name) and n.target.id=='epoch')
|
|
old=loop(oldworker.run);new=loop(newworker.run);new.body=[n for n in new.body if not(isinstance(n,ast.If) and ast.unparse(n.test)=='epoch == 1')]
|
|
self.assertEqual(ast.dump(old),ast.dump(new))
|
|
|
|
class BudgetAndGate(unittest.TestCase):
|
|
def setUp(self):
|
|
self.c=dict(prior_gpu_seconds=18858.21490871499,total_h2_gpu_seconds=144000,prediction_gpu_reserve=2400,jobs=[dict(fold=f,gpu_seconds=(144000-18858.21490871499-2400)/4,validation_seconds_bound=12.) for f in range(1,5)])
|
|
self.obs={f:dict(epoch=1,charged_gpu_seconds=200.,max_train_epoch_seconds=90.,max_validation_seconds=0.) for f in range(1,5)}
|
|
def test_old_cost_once_plus_all_four_waits_and_prediction_reserve(self):
|
|
d=projection(self.c,self.obs);expected=18858.21490871499+4*(200+1.25*(199*90+20*12)+60)+2400;self.assertEqual(d['projected_total_gpu_seconds'],expected);self.assertEqual(d['status'],'PASS')
|
|
def test_budget_and_per_job_limits_stop(self):
|
|
o=copy.deepcopy(self.obs);o[1]['max_train_epoch_seconds']=150;self.assertEqual(projection(self.c,o)['status'],'STOP_BUDGET_PROJECTION')
|
|
for v in o.values():v['max_train_epoch_seconds']=200
|
|
self.assertGreater(projection(self.c,o)['projected_total_gpu_seconds'],144000)
|
|
def test_missing_epoch_invalid_timing_rejected(self):
|
|
with self.assertRaises(ValueError):projection(self.c,{1:self.obs[1]})
|
|
for field,value in [('epoch',0),('max_train_epoch_seconds',float('nan')),('charged_gpu_seconds',-1)]:
|
|
o=copy.deepcopy(self.obs);o[2][field]=value
|
|
with self.assertRaises(ValueError):projection(self.c,o)
|
|
def test_actual_validation_cost_can_raise_projection(self):
|
|
a=projection(self.c,self.obs);o=copy.deepcopy(self.obs);o[1]['max_validation_seconds']=30;self.assertGreater(projection(self.c,o)['projected_total_gpu_seconds'],a['projected_total_gpu_seconds'])
|
|
def test_epoch10_and_final_remaining_validation_count(self):
|
|
o=copy.deepcopy(self.obs)
|
|
for v in o.values():v['epoch']=200
|
|
self.assertAlmostEqual(projection(self.c,o)['projected_total_gpu_seconds'],18858.21490871499+4*(200+60)+2400)
|
|
def test_first_epoch_barrier_requires_exact_pass(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
p=Path(d);self.assertFalse(first_epoch_release(p,'fixed'))
|
|
for result in [dict(status='STOP_BUDGET_PROJECTION',contract_sha256='fixed'),dict(status='PASS',contract_sha256='other')]:
|
|
(p/'first-epoch-projection.json').write_text(json.dumps(result))
|
|
with self.assertRaises(RuntimeError):first_epoch_release(p,'fixed')
|
|
(p/'first-epoch-projection.json').write_text(json.dumps(dict(status='PASS',contract_sha256='fixed')));self.assertTrue(first_epoch_release(p,'fixed'))
|
|
(p/'global-stop.json').write_text('{}')
|
|
with self.assertRaises(RuntimeError):first_epoch_release(p,'fixed')
|
|
def test_confirmation_is_strict_and_complete_five_fold_only(self):
|
|
self.assertTrue(confirmed(-.001,552,204))
|
|
for h,n,g in [(0,552,204),(.001,552,204),(None,552,204),(float('nan'),552,204),(-.001,121,37),(-.001,551,204),(-.001,552,203)]:self.assertFalse(confirmed(h,n,g))
|
|
def test_exact_process_identity_excludes_preparation(self):
|
|
self.assertFalse(execution_command(['python','tools/prepare_phase2_h2_s2.py']));self.assertTrue(execution_command(['python','tools/phase2_h2_s2_queue.py']))
|
|
def test_failure_peak_remains_even_after_lower_final_peak(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
p=Path(d);t=PeakRecords();(p/'failure.json').write_text(json.dumps(dict(peak_allocated_bytes=100,peak_reserved_bytes=16*2**30)));self.assertTrue(t.scan(p));(p/'failure.json').unlink();(p/'result.json').write_text(json.dumps(dict(peak_allocated_bytes=100,peak_reserved_bytes=200)));self.assertTrue(t.scan(p));self.assertEqual(t.maxima['peak_reserved_bytes'],16*2**30)
|
|
class GuardLifecycle(unittest.TestCase):
|
|
def exercise(self,peak):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
q=Path(d);save(q/'contract.json',dict(synthetic=True));r=q/'folds/1';r.mkdir(parents=True)
|
|
job=dict(fold=1,gpu_uuid='synthetic',guard_cpus=[0],gpu_seconds=600,cpu_seconds=600,setup_gpu_seconds=60,learning_parts=2,rss_bytes=16*2**30,vram_bytes=15*2**30,disk_bytes=20*2**30)
|
|
calls=[]
|
|
def worker(command,*args,**kwargs):
|
|
calls.append(command);self.assertEqual(command[-2:],[str(q),'1']);w=r/'worker';w.mkdir();save(w/'result.json',dict(status='COMPLETE',fold=1,completed_epochs=200,optimizer_steps=400,peak_allocated_bytes=100,peak_reserved_bytes=peak));return dict(status='PASS',cpu_seconds=.01,monitor_errors=[])
|
|
with patch.object(guard,'config',return_value=(dict(jobs=[job]),{})),patch.object(guard.os,'sched_setaffinity'),patch.object(guard,'device_usage',return_value=(0,0)),patch.object(guard,'supervise',worker),patch.object(guard.time,'sleep'):
|
|
rc=guard.run(q,1)
|
|
self.assertEqual(len(calls),1);self.assertEqual(json.loads((r/'started.json').read_text())['contract_sha256'],sha(q/'contract.json'));return rc,json.loads((r/'complete.json').read_text())
|
|
def test_queue_contract_identity_and_complete_guard(self):
|
|
rc,d=self.exercise(100);self.assertEqual(rc,0);self.assertEqual(d['status'],'PASS')
|
|
def test_recorded_failure_peak_cannot_be_ignored(self):
|
|
rc,d=self.exercise(16*2**30);self.assertEqual(rc,1);self.assertIn('RECORDED_VRAM_LIMIT',d['reasons'])
|
|
if __name__=='__main__':unittest.main()
|