176 lines
13 KiB
Python
176 lines
13 KiB
Python
"""CPU-only ledger/projection/observation tests for the new stopping rule."""
|
|
import ast
|
|
import io
|
|
import os
|
|
import time
|
|
import copy
|
|
import inspect
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
import phase3_seed1_continuation_common as prior
|
|
import phase3_seed1_v2_common as new
|
|
import phase3_seed1_v2_observation as observation
|
|
from prepare_phase3_seed1_v2 import remaining_contract,OLD,PRIOR
|
|
|
|
class V2Tests(unittest.TestCase):
|
|
def setUp(self):
|
|
original=dict(jobs=[dict(id=i,gpu_seconds=22320,cpu_seconds=22320,validation_seconds_bound=12) for i in range(5)],phase3_prior_gpu_once=72421.31599938683,phase3_gpu_cap=612000,postfit_gpu_reserve=7200)
|
|
def stopped(a,b):return dict(status='STOPPED',never_attempted=[2,3,4],entries=[dict(id=0,gpu_seconds_charged=a),dict(id=1,gpu_seconds_charged=b)],training_gpu_seconds=a+b)
|
|
original_copy=copy.deepcopy(original)
|
|
self.c=remaining_contract(original,[(OLD,stopped(269.53611274203286,269.5359442579793)),(PRIOR,stopped(4317.711953648948,3625.5293972480576))])
|
|
self.assertEqual(original,original_copy)
|
|
def est(self,v,charged=100):return dict(projected_gpu_seconds=v,actual_charged_gpu_seconds=charged)
|
|
def test_both_old_attempts_once_and_initial_budget(self):
|
|
p=new.project_total(self.c,{},{})
|
|
self.assertEqual(p['status'],'PASS')
|
|
self.assertAlmostEqual(self.c['old_seed1_gpu_once'],8482.313407897018)
|
|
self.assertAlmostEqual(p['projected_gpu_seconds']+self.c['old_seed1_gpu_once'],167200)
|
|
self.assertAlmostEqual(p['projected_phase3_gpu_seconds'],72421.31599938683+167200)
|
|
self.assertAlmostEqual(p['pending_reserve_gpu_seconds'],self.c['train_gpu_cap']/5)
|
|
def test_one_measured_fold_sets_pending_median(self):
|
|
p=new.project_total(self.c,{0:self.est(24000)},{},{0:100})
|
|
self.assertEqual(p['projected_gpu_seconds'],5*24000+7200)
|
|
self.assertEqual(p['pending_reserve_gpu_seconds'],24000)
|
|
def test_even_median_and_two_started_count_once(self):
|
|
p=new.project_total(self.c,{0:self.est(20000),1:self.est(30000)},{},{0:1000,1:2000})
|
|
self.assertEqual(p['pending_reserve_gpu_seconds'],25000)
|
|
self.assertEqual(p['projected_gpu_seconds'],20000+30000+3*25000+7200)
|
|
def test_total_exceed_stops_even_below_runaway(self):
|
|
p=new.project_total(self.c,{0:self.est(31000)},{},{0:100})
|
|
self.assertEqual(p['status'],'STOP_BUDGET_PROJECTION')
|
|
self.assertTrue(all(r['actual_within_runaway'] for r in p['rows']))
|
|
def test_projected_above40000_does_not_alone_stop(self):
|
|
p=new.project_total(self.c,{0:self.est(45000)},{i:dict(gpu_seconds_charged=1000) for i in [1,2,3,4]},{0:100})
|
|
self.assertEqual(p['status'],'PASS')
|
|
self.assertEqual(p['projected_gpu_seconds'],56200)
|
|
def test_actual_runaway_stops_within_total(self):
|
|
p=new.project_total(self.c,{0:self.est(10000)},{i:dict(gpu_seconds_charged=1000) for i in [1,2,3,4]},{0:40000.01})
|
|
self.assertEqual(p['status'],'STOP_BUDGET_PROJECTION')
|
|
self.assertTrue(p['total_projection_within_cap']);self.assertFalse(p['rows'][0]['actual_within_runaway'])
|
|
def test_terminal_actual_replaces_prior_estimate(self):
|
|
p=new.project_total(self.c,{0:self.est(45000)},{0:dict(gpu_seconds_charged=20000)},{})
|
|
self.assertEqual(p['projected_gpu_seconds'],107200)
|
|
self.assertEqual(p['rows'][0]['basis'],'terminal_actual')
|
|
def test_active_unmeasured_not_free_and_actual_not_ignored(self):
|
|
p=new.project_total(self.c,{0:self.est(20000)},{},{0:100,1:100})
|
|
initial=self.c['train_gpu_cap']/5
|
|
self.assertEqual(p['rows'][1]['projected_gpu_seconds'],initial)
|
|
self.assertEqual(p['pending_reserve_gpu_seconds'],(20000+initial)/2)
|
|
q=new.project_total(self.c,{0:self.est(20000)},{},{0:35000})
|
|
self.assertEqual(q['rows'][0]['projected_gpu_seconds'],35000)
|
|
with self.assertRaises(ValueError):new.project_total(self.c,{0:self.est(float('nan'))},{})
|
|
with self.assertRaises(ValueError):new.project_total(self.c,{}, {},{5:100})
|
|
def test_projection_math_unchanged(self):
|
|
j=self.c['jobs'][0];obs=dict(epoch=34,charged=3620,max_epoch_seconds=134.2113543249434,max_validation_seconds=15)
|
|
self.assertEqual(new.project_cell(j,obs)['projected_gpu_seconds'],prior.project_cell(j,obs)['projected_gpu_seconds'])
|
|
def test_scientific_entry_and_guard_only_names_changed(self):
|
|
b=Path(__file__).resolve().parent
|
|
for suffix in ['worker','guard']:
|
|
s=(b/f'phase3_seed1_v2_{suffix}.py').read_text().replace('phase3_seed1_v2_common','phase3_seed1_continuation_common').replace('phase3_seed1_v2_worker.py','phase3_seed1_continuation_worker.py')
|
|
self.assertEqual(s,(b/f'phase3_seed1_continuation_{suffix}.py').read_text())
|
|
for name in ['fold_view','first_epoch_release','dispatchable','idle_eligible_gpus','assigned_job','available_gpus']:
|
|
self.assertEqual(inspect.getsource(getattr(new,name)),inspect.getsource(getattr(prior,name)))
|
|
for prefix in ['phase3_seed1','phase3_seed1_continuation','phase3_seed1_v2']:
|
|
for suffix in ['worker','guard','queue']:self.assertTrue(new.execution_command(['python',f'tools/{prefix}_{suffix}.py']))
|
|
def test_epoch_snapshot_marks_foreign_without_intervening(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
root=Path(d);own=SimpleNamespace(pid=10,info=dict(cmdline=['python','tools/phase3_seed1_v2_worker.py',d]))
|
|
response=SimpleNamespace(stdout='GPU-A, 10, python, 800\nGPU-A, 20, pilot.py, 900\nGPU-B, 30, python, 1000\n')
|
|
with patch.object(observation.psutil,'process_iter',return_value=[own]),patch.object(observation.subprocess,'run',return_value=response) as run:
|
|
r=observation.epoch_host_observation(root,0,1,'GPU-A')
|
|
self.assertEqual(r['status'],'OBSERVED');self.assertEqual([p['pid'] for p in r['gpu_processes'] if p['foreign_on_assigned_gpu']],[20])
|
|
self.assertEqual(run.call_args.kwargs['timeout'],5)
|
|
self.assertTrue((root/'jobs/0/epoch-host-observations/001.json').exists())
|
|
def test_snapshot_error_retained_observation_only(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
with patch.object(observation.psutil,'process_iter',return_value=[]),patch.object(observation.subprocess,'run',side_effect=subprocess.TimeoutExpired('nvidia-smi',5)):
|
|
r=observation.epoch_host_observation(Path(d),1,2,'GPU-A')
|
|
self.assertEqual(r['status'],'OBSERVATION_INCOMPLETE');self.assertTrue(r['observation_only']);self.assertEqual(r['errors'][0]['stage'],'gpu_process_snapshot')
|
|
|
|
class ObservationCoverageTests(unittest.TestCase):
|
|
def queue_fragments(self,root,exited,covered,last_cost,requested=None):
|
|
# Execute the actual coordinator's observe/collect and first poll loop.
|
|
from phase2_h1_worker_v2 import save,sha
|
|
tree=ast.parse((Path(__file__).resolve().parent/'phase3_seed1_v2_queue.py').read_text())
|
|
fn=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run')
|
|
nested=[n for n in fn.body if isinstance(n,ast.FunctionDef) and n.name in ['observe_completed','collect']]
|
|
main_try=next(n for n in fn.body if isinstance(n,ast.Try))
|
|
poll_loop=main_try.body[0].body[0]
|
|
self.assertIsInstance(poll_loop,ast.For)
|
|
v=dict(process=SimpleNamespace(poll=lambda:0 if exited else None),start=time.monotonic(),log=io.StringIO(),job=dict(gpu_uuid='SYNTHETIC-GPU'))
|
|
env=dict(root=root,active={0:v},terminal={},observed_epochs={0:set(covered)},last_epoch={0:last_cost},requested=requested or [],c=dict(jobs=[dict(setup_gpu_seconds=60)]),time=time,read=lambda p:json.loads(p.read_text()),save=save,sha=sha,stop=lambda reason:None,drain_epoch_observations=observation.drain_epoch_observations)
|
|
code=compile(ast.fix_missing_locations(ast.Module(body=nested+[poll_loop],type_ignores=[])),'actual_queue_poll','exec')
|
|
exec(code,env)
|
|
return env
|
|
|
|
def receipts(self,root,epochs):
|
|
from phase2_h1_worker_v2 import save
|
|
d=root/'jobs/0';(d/'worker/history').mkdir(parents=True)
|
|
save(d/'assignment.json',{})
|
|
save(d/'complete.json',dict(status='PASS',monitor_errors=[],worker=dict(monitor_errors=0),gpu_seconds_charged=1000))
|
|
for e in epochs:save(d/'worker/history'/f'{e:03d}.json',dict(epoch=e,train_epoch_seconds=1))
|
|
return d
|
|
|
|
def test_terminal_before_next_poll_keeps_epoch200(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
root=Path(d);job=self.receipts(root,[199,200])
|
|
with patch.object(observation.subprocess,'run') as query:
|
|
env=self.queue_fragments(root,True,range(1,200),199)
|
|
query.assert_not_called()
|
|
record=json.loads((job/'epoch-host-observations/200.json').read_text())
|
|
self.assertEqual(record['status'],'OBSERVATION_INCOMPLETE')
|
|
self.assertEqual(record['incomplete_reason'],'TERMINAL_BEFORE_OBSERVATION')
|
|
self.assertIsNone(record['snapshot_started_unix']);self.assertEqual(record['gpu_processes'],[])
|
|
self.assertEqual(env['active'],{});self.assertEqual(env['observed_epochs'][0],set(range(1,201)));self.assertEqual(env['last_epoch'][0],199)
|
|
|
|
def test_two_unseen_receipts_each_covered_no_historical_snapshot(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
root=Path(d);job=self.receipts(root,[199,200])
|
|
with patch.object(observation.subprocess,'run') as query:
|
|
env=self.queue_fragments(root,False,range(1,199),198)
|
|
self.assertEqual(observation.drain_epoch_observations(root,0,'SYNTHETIC-GPU',env['observed_epochs'][0]),[])
|
|
query.assert_not_called()
|
|
for epoch in [199,200]:
|
|
record=json.loads((job/'epoch-host-observations'/f'{epoch:03d}.json').read_text())
|
|
self.assertEqual(record['incomplete_reason'],'MULTIPLE_COMPLETED_RECEIPTS_BACKLOG');self.assertIsNone(record['snapshot_started_unix'])
|
|
self.assertIn(0,env['active']);self.assertEqual(env['last_epoch'][0],198);self.assertEqual(env['observed_epochs'][0],set(range(1,201)))
|
|
|
|
def test_global_stop_poll_covers_completed_receipt(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
root=Path(d);job=self.receipts(root,[12]);(root/'global-stop.json').write_text('{}')
|
|
with patch.object(observation.subprocess,'run') as query:
|
|
env=self.queue_fragments(root,False,range(1,12),11)
|
|
query.assert_not_called()
|
|
record=json.loads((job/'epoch-host-observations/012.json').read_text())
|
|
self.assertEqual(record['incomplete_reason'],'GLOBAL_STOP_BEFORE_OBSERVATION');self.assertIn(12,env['observed_epochs'][0])
|
|
|
|
def test_delayed_single_receipt_is_incomplete_without_query(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
root=Path(d);job=self.receipts(root,[1]);p=job/'worker/history/001.json';old=time.time()-20;os.utime(p,(old,old))
|
|
with patch.object(observation.subprocess,'run') as query:
|
|
records=observation.drain_epoch_observations(root,0,'GPU-A',set())
|
|
query.assert_not_called();self.assertEqual(records[0]['incomplete_reason'],'DELAYED_RECEIPT_OVER_5_SECONDS');self.assertGreaterEqual(records[0]['receipt_age_seconds'],20)
|
|
|
|
def test_fresh_single_receipt_has_real_snapshot_times_and_hash(self):
|
|
from phase2_h1_worker_v2 import sha
|
|
with tempfile.TemporaryDirectory() as d:
|
|
root=Path(d);job=self.receipts(root,[1]);start=time.time()
|
|
own=SimpleNamespace(pid=10,info=dict(cmdline=['python','tools/phase3_seed1_v2_worker.py',d]))
|
|
with patch.object(observation.psutil,'process_iter',return_value=[own]),patch.object(observation.subprocess,'run',return_value=SimpleNamespace(stdout='GPU-A, 10, python, 800\n')):
|
|
records=observation.drain_epoch_observations(root,0,'GPU-A',set())
|
|
r=records[0];self.assertEqual(r['status'],'OBSERVED');self.assertGreaterEqual(r['snapshot_started_unix'],start);self.assertGreaterEqual(r['snapshot_finished_unix'],r['snapshot_started_unix']);self.assertFalse(r['historical_gpu_occupancy_observed']);self.assertEqual(r['receipt_sha256'],sha(job/'worker/history/001.json'))
|
|
|
|
def test_worker_disappearing_between_poll_and_snapshot_is_incomplete(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
root=Path(d);self.receipts(root,[200])
|
|
with patch.object(observation.psutil,'process_iter',return_value=[]),patch.object(observation.subprocess,'run',return_value=SimpleNamespace(stdout='')):
|
|
records=observation.drain_epoch_observations(root,0,'GPU-A',set())
|
|
self.assertEqual(records[0]['status'],'OBSERVATION_INCOMPLETE');self.assertEqual(records[0]['errors'][0]['stage'],'epoch_worker_visibility');self.assertIsNotNone(records[0]['snapshot_started_unix'])
|
|
|
|
if __name__=='__main__':unittest.main()
|