Files
MeltMind/tools/test_phase3_seed1_v2.py
T

176 lines
13 KiB
Python

"""CPU-only ledger/projection/observation tests for the new stopping rule."""
import ast
import io
import os
import time
import copy
import inspect
import json
from pathlib import Path
import subprocess
import tempfile
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import phase3_seed1_continuation_common as prior
import phase3_seed1_v2_common as new
import phase3_seed1_v2_observation as observation
from prepare_phase3_seed1_v2 import remaining_contract,OLD,PRIOR
class V2Tests(unittest.TestCase):
def setUp(self):
original=dict(jobs=[dict(id=i,gpu_seconds=22320,cpu_seconds=22320,validation_seconds_bound=12) for i in range(5)],phase3_prior_gpu_once=72421.31599938683,phase3_gpu_cap=612000,postfit_gpu_reserve=7200)
def stopped(a,b):return dict(status='STOPPED',never_attempted=[2,3,4],entries=[dict(id=0,gpu_seconds_charged=a),dict(id=1,gpu_seconds_charged=b)],training_gpu_seconds=a+b)
original_copy=copy.deepcopy(original)
self.c=remaining_contract(original,[(OLD,stopped(269.53611274203286,269.5359442579793)),(PRIOR,stopped(4317.711953648948,3625.5293972480576))])
self.assertEqual(original,original_copy)
def est(self,v,charged=100):return dict(projected_gpu_seconds=v,actual_charged_gpu_seconds=charged)
def test_both_old_attempts_once_and_initial_budget(self):
p=new.project_total(self.c,{},{})
self.assertEqual(p['status'],'PASS')
self.assertAlmostEqual(self.c['old_seed1_gpu_once'],8482.313407897018)
self.assertAlmostEqual(p['projected_gpu_seconds']+self.c['old_seed1_gpu_once'],167200)
self.assertAlmostEqual(p['projected_phase3_gpu_seconds'],72421.31599938683+167200)
self.assertAlmostEqual(p['pending_reserve_gpu_seconds'],self.c['train_gpu_cap']/5)
def test_one_measured_fold_sets_pending_median(self):
p=new.project_total(self.c,{0:self.est(24000)},{},{0:100})
self.assertEqual(p['projected_gpu_seconds'],5*24000+7200)
self.assertEqual(p['pending_reserve_gpu_seconds'],24000)
def test_even_median_and_two_started_count_once(self):
p=new.project_total(self.c,{0:self.est(20000),1:self.est(30000)},{},{0:1000,1:2000})
self.assertEqual(p['pending_reserve_gpu_seconds'],25000)
self.assertEqual(p['projected_gpu_seconds'],20000+30000+3*25000+7200)
def test_total_exceed_stops_even_below_runaway(self):
p=new.project_total(self.c,{0:self.est(31000)},{},{0:100})
self.assertEqual(p['status'],'STOP_BUDGET_PROJECTION')
self.assertTrue(all(r['actual_within_runaway'] for r in p['rows']))
def test_projected_above40000_does_not_alone_stop(self):
p=new.project_total(self.c,{0:self.est(45000)},{i:dict(gpu_seconds_charged=1000) for i in [1,2,3,4]},{0:100})
self.assertEqual(p['status'],'PASS')
self.assertEqual(p['projected_gpu_seconds'],56200)
def test_actual_runaway_stops_within_total(self):
p=new.project_total(self.c,{0:self.est(10000)},{i:dict(gpu_seconds_charged=1000) for i in [1,2,3,4]},{0:40000.01})
self.assertEqual(p['status'],'STOP_BUDGET_PROJECTION')
self.assertTrue(p['total_projection_within_cap']);self.assertFalse(p['rows'][0]['actual_within_runaway'])
def test_terminal_actual_replaces_prior_estimate(self):
p=new.project_total(self.c,{0:self.est(45000)},{0:dict(gpu_seconds_charged=20000)},{})
self.assertEqual(p['projected_gpu_seconds'],107200)
self.assertEqual(p['rows'][0]['basis'],'terminal_actual')
def test_active_unmeasured_not_free_and_actual_not_ignored(self):
p=new.project_total(self.c,{0:self.est(20000)},{},{0:100,1:100})
initial=self.c['train_gpu_cap']/5
self.assertEqual(p['rows'][1]['projected_gpu_seconds'],initial)
self.assertEqual(p['pending_reserve_gpu_seconds'],(20000+initial)/2)
q=new.project_total(self.c,{0:self.est(20000)},{},{0:35000})
self.assertEqual(q['rows'][0]['projected_gpu_seconds'],35000)
with self.assertRaises(ValueError):new.project_total(self.c,{0:self.est(float('nan'))},{})
with self.assertRaises(ValueError):new.project_total(self.c,{}, {},{5:100})
def test_projection_math_unchanged(self):
j=self.c['jobs'][0];obs=dict(epoch=34,charged=3620,max_epoch_seconds=134.2113543249434,max_validation_seconds=15)
self.assertEqual(new.project_cell(j,obs)['projected_gpu_seconds'],prior.project_cell(j,obs)['projected_gpu_seconds'])
def test_scientific_entry_and_guard_only_names_changed(self):
b=Path(__file__).resolve().parent
for suffix in ['worker','guard']:
s=(b/f'phase3_seed1_v2_{suffix}.py').read_text().replace('phase3_seed1_v2_common','phase3_seed1_continuation_common').replace('phase3_seed1_v2_worker.py','phase3_seed1_continuation_worker.py')
self.assertEqual(s,(b/f'phase3_seed1_continuation_{suffix}.py').read_text())
for name in ['fold_view','first_epoch_release','dispatchable','idle_eligible_gpus','assigned_job','available_gpus']:
self.assertEqual(inspect.getsource(getattr(new,name)),inspect.getsource(getattr(prior,name)))
for prefix in ['phase3_seed1','phase3_seed1_continuation','phase3_seed1_v2']:
for suffix in ['worker','guard','queue']:self.assertTrue(new.execution_command(['python',f'tools/{prefix}_{suffix}.py']))
def test_epoch_snapshot_marks_foreign_without_intervening(self):
with tempfile.TemporaryDirectory() as d:
root=Path(d);own=SimpleNamespace(pid=10,info=dict(cmdline=['python','tools/phase3_seed1_v2_worker.py',d]))
response=SimpleNamespace(stdout='GPU-A, 10, python, 800\nGPU-A, 20, pilot.py, 900\nGPU-B, 30, python, 1000\n')
with patch.object(observation.psutil,'process_iter',return_value=[own]),patch.object(observation.subprocess,'run',return_value=response) as run:
r=observation.epoch_host_observation(root,0,1,'GPU-A')
self.assertEqual(r['status'],'OBSERVED');self.assertEqual([p['pid'] for p in r['gpu_processes'] if p['foreign_on_assigned_gpu']],[20])
self.assertEqual(run.call_args.kwargs['timeout'],5)
self.assertTrue((root/'jobs/0/epoch-host-observations/001.json').exists())
def test_snapshot_error_retained_observation_only(self):
with tempfile.TemporaryDirectory() as d:
with patch.object(observation.psutil,'process_iter',return_value=[]),patch.object(observation.subprocess,'run',side_effect=subprocess.TimeoutExpired('nvidia-smi',5)):
r=observation.epoch_host_observation(Path(d),1,2,'GPU-A')
self.assertEqual(r['status'],'OBSERVATION_INCOMPLETE');self.assertTrue(r['observation_only']);self.assertEqual(r['errors'][0]['stage'],'gpu_process_snapshot')
class ObservationCoverageTests(unittest.TestCase):
def queue_fragments(self,root,exited,covered,last_cost,requested=None):
# Execute the actual coordinator's observe/collect and first poll loop.
from phase2_h1_worker_v2 import save,sha
tree=ast.parse((Path(__file__).resolve().parent/'phase3_seed1_v2_queue.py').read_text())
fn=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run')
nested=[n for n in fn.body if isinstance(n,ast.FunctionDef) and n.name in ['observe_completed','collect']]
main_try=next(n for n in fn.body if isinstance(n,ast.Try))
poll_loop=main_try.body[0].body[0]
self.assertIsInstance(poll_loop,ast.For)
v=dict(process=SimpleNamespace(poll=lambda:0 if exited else None),start=time.monotonic(),log=io.StringIO(),job=dict(gpu_uuid='SYNTHETIC-GPU'))
env=dict(root=root,active={0:v},terminal={},observed_epochs={0:set(covered)},last_epoch={0:last_cost},requested=requested or [],c=dict(jobs=[dict(setup_gpu_seconds=60)]),time=time,read=lambda p:json.loads(p.read_text()),save=save,sha=sha,stop=lambda reason:None,drain_epoch_observations=observation.drain_epoch_observations)
code=compile(ast.fix_missing_locations(ast.Module(body=nested+[poll_loop],type_ignores=[])),'actual_queue_poll','exec')
exec(code,env)
return env
def receipts(self,root,epochs):
from phase2_h1_worker_v2 import save
d=root/'jobs/0';(d/'worker/history').mkdir(parents=True)
save(d/'assignment.json',{})
save(d/'complete.json',dict(status='PASS',monitor_errors=[],worker=dict(monitor_errors=0),gpu_seconds_charged=1000))
for e in epochs:save(d/'worker/history'/f'{e:03d}.json',dict(epoch=e,train_epoch_seconds=1))
return d
def test_terminal_before_next_poll_keeps_epoch200(self):
with tempfile.TemporaryDirectory() as d:
root=Path(d);job=self.receipts(root,[199,200])
with patch.object(observation.subprocess,'run') as query:
env=self.queue_fragments(root,True,range(1,200),199)
query.assert_not_called()
record=json.loads((job/'epoch-host-observations/200.json').read_text())
self.assertEqual(record['status'],'OBSERVATION_INCOMPLETE')
self.assertEqual(record['incomplete_reason'],'TERMINAL_BEFORE_OBSERVATION')
self.assertIsNone(record['snapshot_started_unix']);self.assertEqual(record['gpu_processes'],[])
self.assertEqual(env['active'],{});self.assertEqual(env['observed_epochs'][0],set(range(1,201)));self.assertEqual(env['last_epoch'][0],199)
def test_two_unseen_receipts_each_covered_no_historical_snapshot(self):
with tempfile.TemporaryDirectory() as d:
root=Path(d);job=self.receipts(root,[199,200])
with patch.object(observation.subprocess,'run') as query:
env=self.queue_fragments(root,False,range(1,199),198)
self.assertEqual(observation.drain_epoch_observations(root,0,'SYNTHETIC-GPU',env['observed_epochs'][0]),[])
query.assert_not_called()
for epoch in [199,200]:
record=json.loads((job/'epoch-host-observations'/f'{epoch:03d}.json').read_text())
self.assertEqual(record['incomplete_reason'],'MULTIPLE_COMPLETED_RECEIPTS_BACKLOG');self.assertIsNone(record['snapshot_started_unix'])
self.assertIn(0,env['active']);self.assertEqual(env['last_epoch'][0],198);self.assertEqual(env['observed_epochs'][0],set(range(1,201)))
def test_global_stop_poll_covers_completed_receipt(self):
with tempfile.TemporaryDirectory() as d:
root=Path(d);job=self.receipts(root,[12]);(root/'global-stop.json').write_text('{}')
with patch.object(observation.subprocess,'run') as query:
env=self.queue_fragments(root,False,range(1,12),11)
query.assert_not_called()
record=json.loads((job/'epoch-host-observations/012.json').read_text())
self.assertEqual(record['incomplete_reason'],'GLOBAL_STOP_BEFORE_OBSERVATION');self.assertIn(12,env['observed_epochs'][0])
def test_delayed_single_receipt_is_incomplete_without_query(self):
with tempfile.TemporaryDirectory() as d:
root=Path(d);job=self.receipts(root,[1]);p=job/'worker/history/001.json';old=time.time()-20;os.utime(p,(old,old))
with patch.object(observation.subprocess,'run') as query:
records=observation.drain_epoch_observations(root,0,'GPU-A',set())
query.assert_not_called();self.assertEqual(records[0]['incomplete_reason'],'DELAYED_RECEIPT_OVER_5_SECONDS');self.assertGreaterEqual(records[0]['receipt_age_seconds'],20)
def test_fresh_single_receipt_has_real_snapshot_times_and_hash(self):
from phase2_h1_worker_v2 import sha
with tempfile.TemporaryDirectory() as d:
root=Path(d);job=self.receipts(root,[1]);start=time.time()
own=SimpleNamespace(pid=10,info=dict(cmdline=['python','tools/phase3_seed1_v2_worker.py',d]))
with patch.object(observation.psutil,'process_iter',return_value=[own]),patch.object(observation.subprocess,'run',return_value=SimpleNamespace(stdout='GPU-A, 10, python, 800\n')):
records=observation.drain_epoch_observations(root,0,'GPU-A',set())
r=records[0];self.assertEqual(r['status'],'OBSERVED');self.assertGreaterEqual(r['snapshot_started_unix'],start);self.assertGreaterEqual(r['snapshot_finished_unix'],r['snapshot_started_unix']);self.assertFalse(r['historical_gpu_occupancy_observed']);self.assertEqual(r['receipt_sha256'],sha(job/'worker/history/001.json'))
def test_worker_disappearing_between_poll_and_snapshot_is_incomplete(self):
with tempfile.TemporaryDirectory() as d:
root=Path(d);self.receipts(root,[200])
with patch.object(observation.psutil,'process_iter',return_value=[]),patch.object(observation.subprocess,'run',return_value=SimpleNamespace(stdout='')):
records=observation.drain_epoch_observations(root,0,'GPU-A',set())
self.assertEqual(records[0]['status'],'OBSERVATION_INCOMPLETE');self.assertEqual(records[0]['errors'][0]['stage'],'epoch_worker_visibility');self.assertIsNotNone(records[0]['snapshot_started_unix'])
if __name__=='__main__':unittest.main()