Files
MeltMind/tools/test_s2_pilot_waiter.py

71 lines
5.0 KiB
Python

import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
import run_s2_pilot_when_ready as waiter
from s2_pilot_terminal import merge,terminal_batch
from freeze_g1_components import sha
class WaiterTests(unittest.TestCase):
def test_terminal_gate_requires_full_roster_no_global_stop(self):
with tempfile.TemporaryDirectory() as tmp:
source=Path(tmp);self.assertFalse(waiter.preparation_terminal(source))
batch=dict(status='ATTEMPTS_COMPLETE',stop_reason=None,not_run_indices=[],jobs=[dict(index=i) for i in range(499)])
p=source/'batch-complete.json';p.write_text(json.dumps(batch));self.assertTrue(waiter.preparation_terminal(source))
for change in [dict(status='STOPPED'),dict(stop_reason='GLOBAL_CPU_LIMIT'),dict(not_run_indices=[498]),dict(jobs=batch['jobs'][:-1]),dict(jobs=[dict(index=0)]*499)]:
p.write_text(json.dumps(dict(batch,**change)))
with self.assertRaises((RuntimeError,ValueError)):waiter.preparation_terminal(source)
def test_stopped_preparation_does_not_finalize_or_use_gpu(self):
with tempfile.TemporaryDirectory() as tmp:
source=Path(tmp)/'source';source.mkdir();(source/'batch-started.json').write_text('{}');(source/'contract.json').write_text('{}')
(source/'batch-complete.json').write_text('{"status":"STOPPED"}')
watch=Path(tmp)/'watch';root=Path(tmp)/'pilot'
with patch.object(waiter,'finalize') as finalize,patch.object(waiter,'pilot') as pilot:
with self.assertRaises(RuntimeError):waiter.run(source,root,watch,'synthetic')
finalize.assert_not_called();pilot.assert_not_called()
with self.assertRaises(FileExistsError):waiter.run(source,root,watch,'synthetic')
self.assertTrue((watch/'failure.json').exists());self.assertFalse(root.exists())
class MergedTerminalTests(unittest.TestCase):
def fixture(self,root):
source=root/'original';later=root/'continuation';source.mkdir();later.mkdir()
cases=[dict(case=str(i),tet_nodes=i+1) for i in range(499)]
def job(i,case):return dict(index=i,case=str(case),worker=dict(status='STOPPED_OR_FAILED' if case==0 else 'PASS',cpu_seconds=1.))
original=dict(cases=cases,role_manifest_sha256='synthetic-roles')
before=dict(status='STOPPED',stop_reason='FIRST_50_PROJECTION_OVER_CAP',not_run_indices=list(range(50,499)),jobs=[job(i,i) for i in range(50)],guard_cpu_seconds=1.)
(source/'contract.json').write_text(json.dumps(original));(source/'batch-complete.json').write_text(json.dumps(before))
(source/'first-50-checkpoint.json').write_text(json.dumps(dict(projected_worker_cpu_seconds=70*3600,projected_disk_bytes=10*2**30)))
proof=dict(root=str(source),contract_sha256=sha(source/'contract.json'),batch_sha256=sha(source/'batch-complete.json'))
(later/'contract.json').write_text(json.dumps(dict(cases=cases[50:],role_manifest_sha256='synthetic-roles',continuation_of=proof)))
(later/'batch-complete.json').write_text(json.dumps(dict(status='ATTEMPTS_COMPLETE',stop_reason=None,not_run_indices=[],jobs=[job(i,i+50) for i in range(449)],guard_cpu_seconds=1.)))
return source,later
def test_waits_for_approved_merge_accepts_failed_parts_and_binds_both_sources(self):
with tempfile.TemporaryDirectory() as tmp:
source,later=self.fixture(Path(tmp));self.assertFalse(waiter.preparation_terminal(source))
batch=merge(source,later);self.assertEqual(batch['failed'],1);self.assertEqual(len(batch['jobs']),499)
self.assertEqual(batch['jobs'][49]['ready_source_root'],str(source));self.assertEqual(batch['jobs'][50]['ready_source_root'],str(later))
self.assertEqual(batch['jobs'][50]['source_index'],0);self.assertEqual(batch['worker_cpu_seconds'],499.)
self.assertTrue(waiter.preparation_terminal(source))
fixed=sha(source/'merged-terminal.json')
with self.assertRaises(FileExistsError):merge(source,later)
self.assertEqual(sha(source/'merged-terminal.json'),fixed)
(later/'contract.json').write_text('{}')
with self.assertRaises(AssertionError):terminal_batch(source)
def test_merge_rejects_retries_changed_specs_and_cumulative_overrun(self):
for change in ['retry','spec','cpu']:
with self.subTest(change=change),tempfile.TemporaryDirectory() as tmp:
source,later=self.fixture(Path(tmp))
p=later/('batch-complete.json' if change=='cpu' else 'contract.json');data=json.loads(p.read_text())
if change=='retry':data['cases'][0]['case']='0'
elif change=='spec':data['cases'][0]['tet_nodes']+=1
else:data['jobs'][0]['worker']['cpu_seconds']=81*3600
p.write_text(json.dumps(data))
with self.assertRaises(AssertionError):merge(source,later)
self.assertFalse((source/'merged-terminal.json').exists())
if __name__=='__main__':unittest.main(verbosity=2)