66 lines
5.2 KiB
Python
66 lines
5.2 KiB
Python
import copy,json,tempfile,unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
import phase2_gate_outcome_continuation as m
|
|
|
|
class Continuation(unittest.TestCase):
|
|
def fixture(self):
|
|
old=dict(parts=[dict(row=dict(nodes=10)) for _ in range(493)],setup_seconds=600,worker_cpus=list(range(16,28)))
|
|
done=dict(status='STOPPED',reason='RuntimeError: Full per-part reservation exceeds remaining budget',accounting_complete=True,new_v2_labels_read=0,cumulative_cpu_seconds=66936.020178167,records=[dict(index=i,guard=dict(status='PASS',returncode=0)) for i in range(333)])
|
|
p=m.project(done['cumulative_cpu_seconds'],600,3330,1600)
|
|
c=dict(approval='1794927353.190299',new_indices=list(range(333,493)),reused_indices=list(range(333)),prior_cpu_seconds=done['cumulative_cpu_seconds'],projection=p,total_cpu_seconds=p['total_cpu_seconds'],geometry_cpu_cap=p['total_cpu_seconds']-3600,worker_cpus=list(range(16,28)),postfit_reserve=3600,setup_seconds=600,terminal_reserve=120)
|
|
return c,old,done
|
|
def test_actual_node_projection_includes_old_once_and_postfit(self):
|
|
p=m.project(66936.020178167,600,9360267,4096096)
|
|
self.assertAlmostEqual(p['projected_total_cpu_seconds'],104639.31073428741)
|
|
self.assertEqual(p['total_cpu_seconds'],p['projected_total_cpu_seconds']);self.assertTrue(p['within_final_ceiling'])
|
|
self.assertEqual(p['projected_total_cpu_seconds']-p['remaining_variable_cpu_seconds'],66936.020178167+600+120+3600)
|
|
def test_over110k_stops_not_silently_clipped_launch(self):
|
|
p=m.project(66936.020178167,600,100,1000);self.assertFalse(p['within_final_ceiling']);self.assertEqual(p['total_cpu_seconds'],110000)
|
|
c,o,d=self.fixture();o['parts'][-1]['row']['nodes']=100000
|
|
with self.assertRaises(PermissionError):m.scope(c,o,d)
|
|
def test_actual333_scope(self):
|
|
c,o,d=self.fixture();self.assertEqual(m.scope(c,o,d),c['projection'])
|
|
def test_duplicate_old_terminal_or_stale332_rejected(self):
|
|
for mode in ['duplicate','stale']:
|
|
c,o,d=self.fixture()
|
|
if mode=='duplicate':d['records'][-1]=d['records'][0]
|
|
else:d['records'].pop()
|
|
with self.assertRaises(ValueError):m.scope(c,o,d)
|
|
def test_no_retry_or_budget_bump(self):
|
|
for key,value in [('new_indices',list(range(332,493))),('total_cpu_seconds',110000),('prior_cpu_seconds',2*66936.020178167),('worker_cpus',list(range(12)))]:
|
|
c,o,d=self.fixture();c[key]=value
|
|
with self.assertRaises((ValueError,PermissionError)):m.scope(c,o,d)
|
|
def test_nonbudget_stop_and_incomplete_account_rejected(self):
|
|
for key,value in [('reason','numerical failure'),('accounting_complete',False),('new_v2_labels_read',1)]:
|
|
c,o,d=self.fixture();d[key]=value
|
|
with self.assertRaises(PermissionError):m.scope(c,o,d)
|
|
def test_full1800_reservation_never_shortens_job(self):
|
|
self.assertTrue(m.reservation(100,2,5620));self.assertFalse(m.reservation(100.001,2,5620));self.assertFalse(m.reservation(100,0,1919))
|
|
def test_worker_calls_unchanged_science_with_original_contract(self):
|
|
c,o,d=self.fixture();root=Path('/synthetic')
|
|
with patch.object(m,'config',return_value=(c,o,d)),patch.object(m,'prepare') as p:
|
|
m.worker(root,333);p.assert_called_once_with(root,o,333)
|
|
with self.assertRaises(PermissionError):m.worker(root,332)
|
|
def test_guard_retains_original_caps_and_cpu_assignment(self):
|
|
c,o,d=self.fixture()
|
|
with tempfile.TemporaryDirectory() as t:
|
|
root=Path(t);dest=root/'parts/0333';dest.mkdir(parents=True)
|
|
with patch.object(m,'config',return_value=(c,o,d)),patch.object(m.os,'sched_setaffinity') as affinity,patch.object(m,'supervise',return_value=dict(status='PASS',cpu_seconds=.1)) as g:
|
|
self.assertEqual(m.guard(root,333,0),0);affinity.assert_called_once_with(0,[16]);self.assertEqual(g.call_args.args[2],dict(cpu_seconds=1800,wall_seconds=1800,rss_bytes=4*2**30,disk_bytes=2*2**30));self.assertEqual(g.call_args.kwargs['global_stop'],root/'global-stop.json')
|
|
def test_real_hash_reuse_rejects_changed_output(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
root=Path(t);(root/'features.npy').write_bytes(b'synthetic immutable bytes')
|
|
result=dict(status='FEATURES_COMPLETE_NO_PREDICTIONS',case='fixture',index=0,gate_raw_ids=[3],new_labels_read=False,new_fits=0,output_sha256={'features.npy':m.sha(root/'features.npy')});receipt=dict(index=0,guard=dict(status='PASS',returncode=0))
|
|
m.save(root/'result.json',result);m.save(root/'guard-result.json',receipt);e=dict(path=str(root),case='fixture',index=0,result_sha256=m.sha(root/'result.json'),guard_result_sha256=m.sha(root/'guard-result.json'),output_sha256=result['output_sha256']);item=dict(row=dict(case='fixture'),gate_raw_ids=[3])
|
|
m.reused_identity(e,item,receipt);(root/'features.npy').write_bytes(b'changed')
|
|
with self.assertRaises(ValueError):m.reused_identity(e,item,receipt)
|
|
def test_original_active_process_blocks(self):
|
|
class P:info=dict(pid=7,cmdline=['python','tools/phase2_gate_outcome_queue.py','worker','/original','333'])
|
|
with patch.object(m.psutil,'process_iter',return_value=[P()]):
|
|
with self.assertRaises(PermissionError):m.original_exited(Path('/original'))
|
|
def test_gpu_training_is_not_stopped_or_blocked(self):
|
|
class P:info=dict(pid=8,cmdline=['python','tools/phase2_h2_s2_queue.py','/training'])
|
|
with patch.object(m.psutil,'process_iter',return_value=[P()]):m.original_exited(Path('/original'))
|
|
if __name__=='__main__':unittest.main()
|