101 lines
5.3 KiB
Python
101 lines
5.3 KiB
Python
"""Synthetic scope, accounting and stop paths of the continuation coordinator."""
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
spec = importlib.util.spec_from_file_location('gre24_continuation', Path(__file__).parent / 'gre24_continuation/run_gre24_protocol_preparation.py')
|
|
module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module)
|
|
|
|
|
|
def record(index, status='PASS', cpu=2.):
|
|
return dict(index=index, case=str(index), worker=dict(status=status, cpu_seconds=cpu,
|
|
stop_reason=None if status == 'PASS' else 'wall_limit'))
|
|
|
|
|
|
class ContinuationTests(unittest.TestCase):
|
|
def test_original_failures_are_not_retried(self):
|
|
self.assertEqual(module.partition([0, 1, 2], [record(0), record(1, 'STOPPED_OR_FAILED')], [2]), [2])
|
|
with self.assertRaises(ValueError):
|
|
module.partition([0, 1, 2], [record(0), record(1, 'STOPPED_OR_FAILED')], [1, 2])
|
|
|
|
def test_duplicate_reuse_rejected(self):
|
|
with self.assertRaises(ValueError):
|
|
module.partition([0, 1], [record(0), record(0)], [1])
|
|
|
|
def test_order_and_coverage_enforced(self):
|
|
for remaining in [[2, 1], [1], [1, 1, 2]]:
|
|
with self.assertRaises(ValueError):
|
|
module.partition([0, 1, 2], [record(0)], remaining)
|
|
|
|
def test_canonical_accounting_and_failure_preservation(self):
|
|
c = dict(parts=[0, 1, 2], limits={'preparation_initial_cpu_reserve_seconds': 10})
|
|
m = dict(remaining_indices=[2], engineering_test_cpu_seconds=.5, original_audit_sha256='audit')
|
|
result = module.terminal(c, m, [record(0), record(1, 'STOPPED_OR_FAILED')], [record(2, cpu=3)], [2], None, 1)
|
|
self.assertEqual(result['status'], 'COMPLETE')
|
|
self.assertEqual(result['worker_cpu_seconds'] + result['initial_cpu_reserve_seconds'], 17.5)
|
|
self.assertEqual(result['records'][1]['worker']['stop_reason'], 'wall_limit')
|
|
self.assertEqual(result['old_worker_cpu_seconds'], 4)
|
|
self.assertEqual(result['new_worker_cpu_seconds'], 3)
|
|
|
|
def test_missing_attempt_cannot_claim_complete(self):
|
|
c = dict(parts=[0, 1], limits={'preparation_initial_cpu_reserve_seconds': 0})
|
|
m = dict(remaining_indices=[1], engineering_test_cpu_seconds=0, original_audit_sha256='audit')
|
|
result = module.terminal(c, m, [record(0)], [], [1], 'INTERRUPTED', 1)
|
|
self.assertEqual(result['status'], 'STOPPED')
|
|
self.assertFalse(result['accounting_complete'])
|
|
self.assertEqual(result['missing_attempt_receipts'], [1])
|
|
self.assertEqual(result['never_attempted'], [])
|
|
|
|
def exercise(self, cap, missing_receipt=False):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
for name in ['contract.json', 'continuation-manifest.json']:
|
|
(root / name).write_text('{}')
|
|
c = dict(parts=[{'case':str(i)} for i in range(3)], coordinator_cpus=[0],
|
|
limits=dict(preparation_initial_cpu_reserve_seconds=1, total_worker_cpu_seconds=cap,
|
|
per_part_cpu_seconds=10, workers=2))
|
|
m = dict(remaining_indices=[1, 2], engineering_test_cpu_seconds=.5,
|
|
original_worker_cpu_seconds=2, original_audit_sha256='audit')
|
|
launched = []
|
|
class FakeProcess:
|
|
pid = 99999999
|
|
def poll(self): return 0
|
|
def wait(self): return 0
|
|
def fake_launch(command, **_):
|
|
index = int(command[command.index('--guard') + 1]); launched.append(index)
|
|
if not missing_receipt:
|
|
(root/'parts'/str(index)/'guard-result.json').write_text(json.dumps(record(index)))
|
|
return FakeProcess()
|
|
with patch.object(module, 'load_scope', return_value=(c, m, [record(0)])), \
|
|
patch.object(module.os, 'sched_setaffinity'), \
|
|
patch.object(module.signal, 'signal'), \
|
|
patch.object(module.subprocess, 'Popen', side_effect=fake_launch), \
|
|
patch.object(module.time, 'sleep'), patch.object(module, 'stop_group'):
|
|
code = module.run_locked(root)
|
|
result = json.loads((root/'complete.json').read_text())
|
|
with self.assertRaises(FileExistsError): module.run_locked(root)
|
|
return code, launched, result
|
|
|
|
def test_actual_coordinator_only_dispatches_remaining(self):
|
|
code, launched, result = self.exercise(100)
|
|
self.assertEqual((code, launched, result['status']), (0, [1, 2], 'COMPLETE'))
|
|
self.assertEqual(result['worker_cpu_seconds'] + result['initial_cpu_reserve_seconds'], 7.5)
|
|
|
|
def test_budget_reserves_old_cost_before_any_launch(self):
|
|
code, launched, result = self.exercise(15)
|
|
self.assertEqual((code, launched), (1, []))
|
|
self.assertEqual(result['stop_reason'], 'REMAINING_CPU_INSUFFICIENT_FOR_ANOTHER_CAPPED_ATTEMPT')
|
|
|
|
def test_missing_guard_stops_without_retry(self):
|
|
code, launched, result = self.exercise(100, missing_receipt=True)
|
|
self.assertEqual((code, launched), (1, [1, 2]))
|
|
self.assertFalse(result['accounting_complete'])
|
|
self.assertEqual(result['stop_reason'], 'CONTINUATION_INFRASTRUCTURE_EXCEPTION')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|