59 lines
4.0 KiB
Python
59 lines
4.0 KiB
Python
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
from prepare_s2_pilot_training import quantile_probe,finalize
|
|
from s2_pilot_train import common_epoch_limit
|
|
from run_s2_pilot import freeze_training_contract,run
|
|
import s2_pilot_gpu_guard as guard
|
|
|
|
|
|
class OrchestrationTests(unittest.TestCase):
|
|
def test_quantiles_only_learning_and_include_size_extremes(self):
|
|
records=[dict(case=str(i),nodes=i+1) for i in range(40)]
|
|
roles=dict(learning_parts=[str(i) for i in range(30)],validation_parts=[str(i) for i in range(30,40)],sealed_test_parts=['test'])
|
|
ids=quantile_probe(records,roles)
|
|
self.assertEqual(len(set(ids)),20);self.assertEqual(ids[0],'0');self.assertEqual(ids[-1],'29')
|
|
self.assertTrue(set(ids)<=set(roles['learning_parts']))
|
|
def test_partial_preparation_never_starts_pilot(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
source=Path(tmp);(source/'contract.json').write_text('{}');(source/'batch-complete.json').write_text('{"status":"STOPPED"}')
|
|
with self.assertRaises(AssertionError):finalize(source,source/'never-created','synthetic')
|
|
self.assertFalse((source/'never-created').exists())
|
|
def test_contract_records_two_node_rates_totals_and_cannot_overwrite(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root=Path(tmp);(root/'timing-probe').mkdir()
|
|
rows=[dict(case=str(i),nodes=(i+1)*100,training_seconds=(i+1)*.1,inference_seconds=(i+1)*.05) for i in range(20)]
|
|
nodes=sum(r['nodes'] for r in rows)
|
|
train=sum(r['training_seconds'] for r in rows)/nodes;infer=sum(r['inference_seconds'] for r in rows)/nodes
|
|
plan=dict(probe_cases=[r['case'] for r in rows],learning_nodes=38700,validation_nodes=11200)
|
|
probe=dict(status='COMPLETE',rows=rows,probe_nodes=nodes,labels_read=False,metrics_evaluated=False,standardizer_fingerprint='synthetic',
|
|
**common_epoch_limit(train,infer,38700,11200))
|
|
(root/'timing-probe/result.json').write_text(json.dumps(probe));(root/'timing-probe-guard-receipt.json').write_text('{"total_run_wall_seconds":10}')
|
|
c=freeze_training_contract(root,plan,probe)
|
|
self.assertEqual(c['E_max'],33);self.assertEqual(c['cosine_T_max'],33)
|
|
self.assertEqual(c['learning_nodes'],38700);self.assertEqual(c['validation_nodes'],11200)
|
|
self.assertAlmostEqual(c['training_seconds_per_node'],.001);self.assertAlmostEqual(c['inference_seconds_per_node'],.0005)
|
|
with self.assertRaises(FileExistsError):freeze_training_contract(root,plan,probe)
|
|
def test_failed_probe_stops_before_training(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root=Path(tmp);(root/'pilot-plan.json').write_text('{"code_sha256":{},"gpu_uuid":"synthetic"}')
|
|
with patch('run_s2_pilot.guarded_worker',return_value=({'status':'FAIL'},None)),patch('run_s2_pilot.train_runs') as train:
|
|
with self.assertRaises(RuntimeError):run(root)
|
|
train.assert_not_called()
|
|
self.assertTrue((root/'orchestrator-failure.json').exists());self.assertFalse((root/'training-contract.json').exists())
|
|
def test_probe_time_reduces_total_budget(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root=Path(tmp);c=dict(gpu_uuid='synthetic',code_sha256={},probe_guard_wall_seconds=7190.,
|
|
runs=[dict(name=f'{arm}{width}',arm=arm,width=width,seed=0) for width in [128,256] for arm in ['N','E']])
|
|
(root/'training-contract.json').write_text(json.dumps(c))
|
|
with patch.object(guard,'guarded_worker',return_value=({'status':'FAIL'},None)) as worker:
|
|
guard.run(root)
|
|
self.assertEqual(worker.call_count,1);self.assertLessEqual(worker.call_args.args[4],10.)
|
|
result=json.loads((root/'pilot-complete.json').read_text());self.assertGreaterEqual(result['pilot_wall_seconds'],7190.)
|
|
self.assertIsNone(result['selected_global_width'])
|
|
|
|
|
|
if __name__=='__main__':unittest.main(verbosity=2)
|