Files
MeltMind/tools/test_phase2_gate_outcome_continuation.py
T

66 lines
5.2 KiB
Python

import copy,json,tempfile,unittest
from pathlib import Path
from unittest.mock import patch
import phase2_gate_outcome_continuation as m
class Continuation(unittest.TestCase):
def fixture(self):
old=dict(parts=[dict(row=dict(nodes=10)) for _ in range(493)],setup_seconds=600,worker_cpus=list(range(16,28)))
done=dict(status='STOPPED',reason='RuntimeError: Full per-part reservation exceeds remaining budget',accounting_complete=True,new_v2_labels_read=0,cumulative_cpu_seconds=66936.020178167,records=[dict(index=i,guard=dict(status='PASS',returncode=0)) for i in range(333)])
p=m.project(done['cumulative_cpu_seconds'],600,3330,1600)
c=dict(approval='1794927353.190299',new_indices=list(range(333,493)),reused_indices=list(range(333)),prior_cpu_seconds=done['cumulative_cpu_seconds'],projection=p,total_cpu_seconds=p['total_cpu_seconds'],geometry_cpu_cap=p['total_cpu_seconds']-3600,worker_cpus=list(range(16,28)),postfit_reserve=3600,setup_seconds=600,terminal_reserve=120)
return c,old,done
def test_actual_node_projection_includes_old_once_and_postfit(self):
p=m.project(66936.020178167,600,9360267,4096096)
self.assertAlmostEqual(p['projected_total_cpu_seconds'],104639.31073428741)
self.assertEqual(p['total_cpu_seconds'],p['projected_total_cpu_seconds']);self.assertTrue(p['within_final_ceiling'])
self.assertEqual(p['projected_total_cpu_seconds']-p['remaining_variable_cpu_seconds'],66936.020178167+600+120+3600)
def test_over110k_stops_not_silently_clipped_launch(self):
p=m.project(66936.020178167,600,100,1000);self.assertFalse(p['within_final_ceiling']);self.assertEqual(p['total_cpu_seconds'],110000)
c,o,d=self.fixture();o['parts'][-1]['row']['nodes']=100000
with self.assertRaises(PermissionError):m.scope(c,o,d)
def test_actual333_scope(self):
c,o,d=self.fixture();self.assertEqual(m.scope(c,o,d),c['projection'])
def test_duplicate_old_terminal_or_stale332_rejected(self):
for mode in ['duplicate','stale']:
c,o,d=self.fixture()
if mode=='duplicate':d['records'][-1]=d['records'][0]
else:d['records'].pop()
with self.assertRaises(ValueError):m.scope(c,o,d)
def test_no_retry_or_budget_bump(self):
for key,value in [('new_indices',list(range(332,493))),('total_cpu_seconds',110000),('prior_cpu_seconds',2*66936.020178167),('worker_cpus',list(range(12)))]:
c,o,d=self.fixture();c[key]=value
with self.assertRaises((ValueError,PermissionError)):m.scope(c,o,d)
def test_nonbudget_stop_and_incomplete_account_rejected(self):
for key,value in [('reason','numerical failure'),('accounting_complete',False),('new_v2_labels_read',1)]:
c,o,d=self.fixture();d[key]=value
with self.assertRaises(PermissionError):m.scope(c,o,d)
def test_full1800_reservation_never_shortens_job(self):
self.assertTrue(m.reservation(100,2,5620));self.assertFalse(m.reservation(100.001,2,5620));self.assertFalse(m.reservation(100,0,1919))
def test_worker_calls_unchanged_science_with_original_contract(self):
c,o,d=self.fixture();root=Path('/synthetic')
with patch.object(m,'config',return_value=(c,o,d)),patch.object(m,'prepare') as p:
m.worker(root,333);p.assert_called_once_with(root,o,333)
with self.assertRaises(PermissionError):m.worker(root,332)
def test_guard_retains_original_caps_and_cpu_assignment(self):
c,o,d=self.fixture()
with tempfile.TemporaryDirectory() as t:
root=Path(t);dest=root/'parts/0333';dest.mkdir(parents=True)
with patch.object(m,'config',return_value=(c,o,d)),patch.object(m.os,'sched_setaffinity') as affinity,patch.object(m,'supervise',return_value=dict(status='PASS',cpu_seconds=.1)) as g:
self.assertEqual(m.guard(root,333,0),0);affinity.assert_called_once_with(0,[16]);self.assertEqual(g.call_args.args[2],dict(cpu_seconds=1800,wall_seconds=1800,rss_bytes=4*2**30,disk_bytes=2*2**30));self.assertEqual(g.call_args.kwargs['global_stop'],root/'global-stop.json')
def test_real_hash_reuse_rejects_changed_output(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);(root/'features.npy').write_bytes(b'synthetic immutable bytes')
result=dict(status='FEATURES_COMPLETE_NO_PREDICTIONS',case='fixture',index=0,gate_raw_ids=[3],new_labels_read=False,new_fits=0,output_sha256={'features.npy':m.sha(root/'features.npy')});receipt=dict(index=0,guard=dict(status='PASS',returncode=0))
m.save(root/'result.json',result);m.save(root/'guard-result.json',receipt);e=dict(path=str(root),case='fixture',index=0,result_sha256=m.sha(root/'result.json'),guard_result_sha256=m.sha(root/'guard-result.json'),output_sha256=result['output_sha256']);item=dict(row=dict(case='fixture'),gate_raw_ids=[3])
m.reused_identity(e,item,receipt);(root/'features.npy').write_bytes(b'changed')
with self.assertRaises(ValueError):m.reused_identity(e,item,receipt)
def test_original_active_process_blocks(self):
class P:info=dict(pid=7,cmdline=['python','tools/phase2_gate_outcome_queue.py','worker','/original','333'])
with patch.object(m.psutil,'process_iter',return_value=[P()]):
with self.assertRaises(PermissionError):m.original_exited(Path('/original'))
def test_gpu_training_is_not_stopped_or_blocked(self):
class P:info=dict(pid=8,cmdline=['python','tools/phase2_h2_s2_queue.py','/training'])
with patch.object(m.psutil,'process_iter',return_value=[P()]):m.original_exited(Path('/original'))
if __name__=='__main__':unittest.main()