71 lines
5.0 KiB
Python
71 lines
5.0 KiB
Python
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
import run_s2_pilot_when_ready as waiter
|
|
from s2_pilot_terminal import merge,terminal_batch
|
|
from freeze_g1_components import sha
|
|
|
|
|
|
class WaiterTests(unittest.TestCase):
|
|
def test_terminal_gate_requires_full_roster_no_global_stop(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
source=Path(tmp);self.assertFalse(waiter.preparation_terminal(source))
|
|
batch=dict(status='ATTEMPTS_COMPLETE',stop_reason=None,not_run_indices=[],jobs=[dict(index=i) for i in range(499)])
|
|
p=source/'batch-complete.json';p.write_text(json.dumps(batch));self.assertTrue(waiter.preparation_terminal(source))
|
|
for change in [dict(status='STOPPED'),dict(stop_reason='GLOBAL_CPU_LIMIT'),dict(not_run_indices=[498]),dict(jobs=batch['jobs'][:-1]),dict(jobs=[dict(index=0)]*499)]:
|
|
p.write_text(json.dumps(dict(batch,**change)))
|
|
with self.assertRaises((RuntimeError,ValueError)):waiter.preparation_terminal(source)
|
|
def test_stopped_preparation_does_not_finalize_or_use_gpu(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
source=Path(tmp)/'source';source.mkdir();(source/'batch-started.json').write_text('{}');(source/'contract.json').write_text('{}')
|
|
(source/'batch-complete.json').write_text('{"status":"STOPPED"}')
|
|
watch=Path(tmp)/'watch';root=Path(tmp)/'pilot'
|
|
with patch.object(waiter,'finalize') as finalize,patch.object(waiter,'pilot') as pilot:
|
|
with self.assertRaises(RuntimeError):waiter.run(source,root,watch,'synthetic')
|
|
finalize.assert_not_called();pilot.assert_not_called()
|
|
with self.assertRaises(FileExistsError):waiter.run(source,root,watch,'synthetic')
|
|
self.assertTrue((watch/'failure.json').exists());self.assertFalse(root.exists())
|
|
|
|
|
|
class MergedTerminalTests(unittest.TestCase):
|
|
def fixture(self,root):
|
|
source=root/'original';later=root/'continuation';source.mkdir();later.mkdir()
|
|
cases=[dict(case=str(i),tet_nodes=i+1) for i in range(499)]
|
|
def job(i,case):return dict(index=i,case=str(case),worker=dict(status='STOPPED_OR_FAILED' if case==0 else 'PASS',cpu_seconds=1.))
|
|
original=dict(cases=cases,role_manifest_sha256='synthetic-roles')
|
|
before=dict(status='STOPPED',stop_reason='FIRST_50_PROJECTION_OVER_CAP',not_run_indices=list(range(50,499)),jobs=[job(i,i) for i in range(50)],guard_cpu_seconds=1.)
|
|
(source/'contract.json').write_text(json.dumps(original));(source/'batch-complete.json').write_text(json.dumps(before))
|
|
(source/'first-50-checkpoint.json').write_text(json.dumps(dict(projected_worker_cpu_seconds=70*3600,projected_disk_bytes=10*2**30)))
|
|
proof=dict(root=str(source),contract_sha256=sha(source/'contract.json'),batch_sha256=sha(source/'batch-complete.json'))
|
|
(later/'contract.json').write_text(json.dumps(dict(cases=cases[50:],role_manifest_sha256='synthetic-roles',continuation_of=proof)))
|
|
(later/'batch-complete.json').write_text(json.dumps(dict(status='ATTEMPTS_COMPLETE',stop_reason=None,not_run_indices=[],jobs=[job(i,i+50) for i in range(449)],guard_cpu_seconds=1.)))
|
|
return source,later
|
|
def test_waits_for_approved_merge_accepts_failed_parts_and_binds_both_sources(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
source,later=self.fixture(Path(tmp));self.assertFalse(waiter.preparation_terminal(source))
|
|
batch=merge(source,later);self.assertEqual(batch['failed'],1);self.assertEqual(len(batch['jobs']),499)
|
|
self.assertEqual(batch['jobs'][49]['ready_source_root'],str(source));self.assertEqual(batch['jobs'][50]['ready_source_root'],str(later))
|
|
self.assertEqual(batch['jobs'][50]['source_index'],0);self.assertEqual(batch['worker_cpu_seconds'],499.)
|
|
self.assertTrue(waiter.preparation_terminal(source))
|
|
fixed=sha(source/'merged-terminal.json')
|
|
with self.assertRaises(FileExistsError):merge(source,later)
|
|
self.assertEqual(sha(source/'merged-terminal.json'),fixed)
|
|
(later/'contract.json').write_text('{}')
|
|
with self.assertRaises(AssertionError):terminal_batch(source)
|
|
def test_merge_rejects_retries_changed_specs_and_cumulative_overrun(self):
|
|
for change in ['retry','spec','cpu']:
|
|
with self.subTest(change=change),tempfile.TemporaryDirectory() as tmp:
|
|
source,later=self.fixture(Path(tmp))
|
|
p=later/('batch-complete.json' if change=='cpu' else 'contract.json');data=json.loads(p.read_text())
|
|
if change=='retry':data['cases'][0]['case']='0'
|
|
elif change=='spec':data['cases'][0]['tet_nodes']+=1
|
|
else:data['jobs'][0]['worker']['cpu_seconds']=81*3600
|
|
p.write_text(json.dumps(data))
|
|
with self.assertRaises(AssertionError):merge(source,later)
|
|
self.assertFalse((source/'merged-terminal.json').exists())
|
|
|
|
|
|
if __name__=='__main__':unittest.main(verbosity=2)
|