160 lines
11 KiB
Python
160 lines
11 KiB
Python
"""Synthetic item1b acceptance; no real data/model loading or GPU access."""
|
|
import ast
|
|
import copy
|
|
from contextlib import ExitStack,contextmanager
|
|
import inspect
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import patch
|
|
from phase3_item1b_schedule import lr_epoch,selected_checkpoint,screen_decision,test_confirmation
|
|
from phase3_item1b_common import project
|
|
from phase3_shared_capacity import GPUS,GIB
|
|
|
|
|
|
class ScheduleTests(unittest.TestCase):
|
|
def test_exact_endpoints(self):
|
|
self.assertAlmostEqual(lr_epoch(1),1e-4,15)
|
|
self.assertAlmostEqual(lr_epoch(5),1e-3,15)
|
|
self.assertAlmostEqual(lr_epoch(100),1e-5,15)
|
|
def test_complete_warmup_and_decay(self):
|
|
rates=[lr_epoch(i) for i in range(1,101)]
|
|
self.assertTrue(all(a<b for a,b in zip(rates[:4],rates[1:5])))
|
|
self.assertTrue(all(a>b for a,b in zip(rates[4:-1],rates[5:])))
|
|
def test_not_original_midpoint(self):
|
|
from phase2_h2_model import lr_epoch as old_lr
|
|
self.assertGreater(old_lr(100),lr_epoch(100)*10)
|
|
self.assertEqual(lr_epoch(1),old_lr(1))
|
|
def test_only_fixed_last_checkpoint(self):
|
|
self.assertEqual(selected_checkpoint(100),'last-epoch100.pt')
|
|
for e in (99,200,100.,True):
|
|
with self.assertRaises(ValueError):selected_checkpoint(e)
|
|
def test_invalid_epoch(self):
|
|
for e in (0,101,1.5,True):
|
|
with self.assertRaises(ValueError):lr_epoch(e)
|
|
def test_screen_exact_boundary(self):
|
|
self.assertEqual(screen_decision(-.001,.0009),'PASS')
|
|
self.assertEqual(screen_decision(-.001,-.001),'SCREENED_OUT')
|
|
self.assertEqual(screen_decision(-.0009,0),'SCREENED_OUT')
|
|
def test_no_seed_no_decision(self):
|
|
self.assertEqual(screen_decision(-.02,None),'WAIT_MATCHED_VALIDATION_AND_SEED_REFERENCE')
|
|
def test_test_strict_boundary_and_seed(self):
|
|
self.assertEqual(test_confirmation(-.002,-.001,0,552,204),'NOT_CONFIRMED')
|
|
self.assertEqual(test_confirmation(-.002,-.0011,-.002,552,204),'NOT_CONFIRMED')
|
|
self.assertEqual(test_confirmation(-.002,-.0011,.001,552,204),'PASS')
|
|
def test_incomplete_test_not_confirmation(self):
|
|
self.assertEqual(test_confirmation(-.03,-.02,0,551,204),'INCOMPLETE_NO_CONFIRMATION')
|
|
def test_nonfinite_comparisons_rejected(self):
|
|
with self.assertRaises(ValueError):screen_decision(float('nan'),0)
|
|
with self.assertRaises(ValueError):test_confirmation(-.02,float('inf'),0,552,204)
|
|
def test_worker_original_train_step_and_seed0(self):
|
|
p=Path(__file__).parent/'phase3_item1b_worker.py';s=p.read_text();tree=ast.parse(s)
|
|
imports=[n for n in ast.walk(tree) if isinstance(n,ast.ImportFrom)]
|
|
self.assertTrue(any(n.module=='phase2_h2_pilot_worker' and 'train_step' in [a.name for a in n.names] for n in imports))
|
|
self.assertIn('SeedSequence([0,epoch])',s);self.assertIn('range(1,101)',s);self.assertIn('if epoch%10==0:',s)
|
|
self.assertNotIn('location_loss',s);self.assertNotIn('phase3_item_c_loss',s)
|
|
|
|
|
|
class BudgetTests(unittest.TestCase):
|
|
def setUp(self):self.c=dict(gpu_cap=88560,postfit_gpu_reserve=7200,preparation_charge_gpu_seconds=600,runaway_gpu_seconds=30000,jobs=[dict(validation_seconds_bound=3) for _ in range(5)])
|
|
def test_initial_budget_includes_postfit(self):
|
|
p=project(self.c,{i:60 for i in range(5)},{},{})
|
|
self.assertEqual(p['status'],'PASS');self.assertEqual(p['projected_gpu_seconds'],88560)
|
|
def test_remaining_validation_charged(self):
|
|
p=project(self.c,{i:100 for i in range(5)},{0:[dict(epoch=1,train_epoch_seconds=50)]},{})
|
|
self.assertEqual(p['folds']['0']['projected'],100+1.5*(99*50+10*3)+120)
|
|
def test_late_validation_max_used(self):
|
|
h=[dict(epoch=e,train_epoch_seconds=50,validation_seconds=7 if e==10 else 0) for e in range(1,11)]
|
|
p=project(self.c,{i:1000 for i in range(5)},{0:h},{})
|
|
self.assertEqual(p['folds']['0']['projected'],1000+1.5*(90*50+9*7)+120)
|
|
def test_actual_runaway_stops(self):
|
|
self.assertEqual(project(self.c,{i:30000 if i==0 else 60 for i in range(5)},{},{})['status'],'STOP_BUDGET')
|
|
def test_no_single_fold_projection_gate(self):
|
|
p=project(self.c,{i:1000 for i in range(5)},{0:[dict(epoch=1,train_epoch_seconds=220)]},{1:{},2:{},3:{}})
|
|
self.assertGreater(p['folds']['0']['projected'],30000);self.assertEqual(p['status'],'PASS')
|
|
def test_incomplete_history_rejected(self):
|
|
with self.assertRaises(ValueError):project(self.c,{i:60 for i in range(5)},{0:[dict(epoch=2,train_epoch_seconds=50)]},{})
|
|
|
|
|
|
class SharedWrapperTests(unittest.TestCase):
|
|
def test_wrapper_changes_only_key_cap_gpu_set(self):
|
|
import phase3_item1b_capacity as new
|
|
import phase3_shared_capacity as old
|
|
for name in ('register_entry','acquire'):
|
|
s=inspect.getsource(getattr(new,name)).replace("'item1b'","'c'").replace('item 1b','item c').replace('cap=88560','cap=205200').replace('GPUS[:3]','GPUS[1:3]')
|
|
self.assertEqual(ast.dump(ast.parse(s)),ast.dump(ast.parse(inspect.getsource(getattr(old,name)))))
|
|
for name in ('admission','registry','transfer','release','validate_lease'):
|
|
self.assertIs(getattr(new,name),getattr(old,name))
|
|
@contextmanager
|
|
def mixed(self,reserved=False):
|
|
import phase3_item1b_capacity as m
|
|
import phase3_shared_capacity as shared
|
|
identity=lambda p:dict(pid=p,created=1.)
|
|
root=Path('/synthetic-1b');oldroot='/synthetic-c'
|
|
leases={}
|
|
for n,pid in enumerate([11,12]+([10] if reserved else [])):
|
|
leases[str(n)]=dict(token=str(n),state='RESERVED' if pid==10 else 'GUARD_BOUND',root=oldroot,fold=n,
|
|
contract_sha256='old',gpu_uuid=GPUS[1],worker_cpus=[16+n],owner=identity(pid),queue_owner=identity(10),vram_reserve=4*GIB,rss_reserve=4*GIB)
|
|
original_c=dict(root=oldroot,contract_sha256='old',owner=identity(10),cap=205200)
|
|
s=dict(devices={g:dict(total=16*GIB,used=4*GIB) for g in GPUS},processes=[dict(gpu_uuid=GPUS[1],pid=p,vram=2*GIB) for p in (101,102)]+[dict(gpu_uuid=GPUS[2],pid=p,vram=GIB) for p in (201,202,203)],ram_total=128*GIB,ram_available=96*GIB,cpu_percent=20,old_pending=True)
|
|
trees={11:{11,101},12:{12,102},10:{10,11,12,101,102}}
|
|
with tempfile.TemporaryDirectory() as tmp,ExitStack() as stack:
|
|
path=Path(tmp)/'registry.json';path.write_text(json.dumps(dict(version=2,entries={'c':original_c},leases=leases)))
|
|
stack.enter_context(patch.object(shared,'SHARED',Path(tmp)))
|
|
for name,value in dict(process_identity=lambda p:identity(20),alive=lambda p:True,descendants=lambda p:trees[p['pid']],host_snapshot=lambda:copy.deepcopy(s)).items():stack.enter_context(patch.object(m,name,value))
|
|
stack.enter_context(patch.object(m.psutil,'Process',lambda p:SimpleNamespace(memory_info=lambda:SimpleNamespace(rss=GIB//4))))
|
|
stack.enter_context(patch.object(m.os,'sched_getaffinity',lambda p:set(range(16,26))))
|
|
m.register_entry(root,'new')
|
|
yield m,root,path,original_c
|
|
def test_shared_c_and_1b_registration_independent(self):
|
|
with self.mixed() as (m,root,path,old):
|
|
d=json.loads(path.read_text());self.assertEqual(d['entries']['c'],old);self.assertEqual(d['entries']['item1b']['cap'],88560)
|
|
with self.assertRaises(FileExistsError):m.register_entry(root,'new')
|
|
def test_mixed_pending_c_reservation_blocks_1b(self):
|
|
with self.mixed(True) as (m,root,path,old):
|
|
before=json.loads(path.read_text());l,_=m.acquire(root,1,'new');self.assertIsNone(l)
|
|
self.assertEqual(json.loads(path.read_text()),before)
|
|
def test_mixed_two_visible_allows_only_one_new(self):
|
|
with self.mixed() as (m,root,path,old):
|
|
l,_=m.acquire(root,1,'new');self.assertIsNotNone(l);self.assertEqual(l['state'],'RESERVED');self.assertEqual(l['gpu_uuid'],GPUS[1])
|
|
second,_=m.acquire(root,0,'new');self.assertIsNone(second)
|
|
self.assertEqual(json.loads(path.read_text())['entries']['c'],old)
|
|
|
|
|
|
class QueueTests(unittest.TestCase):
|
|
def exercise(self,failed=False):
|
|
import phase3_item1b_queue as q
|
|
from phase2_h1_worker_v2 import save
|
|
with tempfile.TemporaryDirectory() as tmp,ExitStack() as stack:
|
|
root=Path(tmp);save(root/'contract.json',dict(synthetic=True));launched=[]
|
|
c=dict(gpu_cap=88560,postfit_gpu_reserve=7200,preparation_charge_gpu_seconds=600,runaway_gpu_seconds=30000,jobs=[dict(validation_seconds_bound=1) for _ in range(5)])
|
|
class Process:
|
|
def __init__(self,cmd,**kw):
|
|
self.pid=999991;self.returncode=int(failed);i=int(cmd[3]);launched.append(i);j=root/'jobs'/str(i);(j/'worker').mkdir()
|
|
save(j/'complete.json',dict(status='STOPPED_OR_FAILED' if failed else 'PASS'));p=j/'worker/last-epoch100.pt';p.write_bytes(b'synthetic')
|
|
save(j/'worker/result.json',dict(fold=i,checkpoint=str(p),checkpoint_sha256=q.sha(p),completed_epochs=100));save(j/'worker/standardization.json',{})
|
|
def poll(self):return self.returncode
|
|
def wait(self):return self.returncode
|
|
def terminate(self):pass
|
|
def kill(self):pass
|
|
replacements=dict(config=lambda r:(c,{}),validate_review=lambda r:None,register_entry=lambda *a:None,
|
|
acquire=lambda r,i,d:(dict(token=str(i),gpu_uuid=GPUS[1],worker_cpus=[20]),{}),transfer=lambda *a:None,
|
|
release=lambda *a:None,process_identity=lambda p:dict(pid=p,created=0),freeze=lambda *a:None)
|
|
for k,v in replacements.items():stack.enter_context(patch.object(q,k,v))
|
|
stack.enter_context(patch.object(q.subprocess,'Popen',Process));stack.enter_context(patch.object(q.time,'sleep',lambda t:None))
|
|
stack.enter_context(patch.object(q.signal,'signal',lambda *a:None));stack.enter_context(patch.object(q.psutil,'disk_usage',return_value=SimpleNamespace(free=200*GIB)))
|
|
stack.enter_context(patch.dict(q.os.environ,PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True'))
|
|
result=q.run(root);d=q.read(root/'complete.json')
|
|
if failed:self.assertEqual(result,1);self.assertEqual(launched,[1]);self.assertFalse((root/'models.json').exists())
|
|
else:
|
|
self.assertEqual(result,0);self.assertEqual(launched,[1,0,2,3,4]);self.assertEqual(len(q.read(root/'models.json')),5)
|
|
self.assertEqual(d['status'],'TRAIN_COMPLETE_VALIDATION_SCREEN_PENDING')
|
|
self.assertFalse(d['test_targets_read']);self.assertFalse(d['prediction_started']);self.assertFalse(d['scoring_started'])
|
|
def test_five_fits_before_screen_but_no_test(self):self.exercise()
|
|
def test_failure_no_retry_or_test(self):self.exercise(True)
|
|
|
|
|
|
if __name__=='__main__':unittest.main(verbosity=2)
|