import json import tempfile import unittest from pathlib import Path from unittest.mock import patch from prepare_s2_pilot_training import quantile_probe,finalize from s2_pilot_train import common_epoch_limit from run_s2_pilot import freeze_training_contract,run import s2_pilot_gpu_guard as guard class OrchestrationTests(unittest.TestCase): def test_quantiles_only_learning_and_include_size_extremes(self): records=[dict(case=str(i),nodes=i+1) for i in range(40)] roles=dict(learning_parts=[str(i) for i in range(30)],validation_parts=[str(i) for i in range(30,40)],sealed_test_parts=['test']) ids=quantile_probe(records,roles) self.assertEqual(len(set(ids)),20);self.assertEqual(ids[0],'0');self.assertEqual(ids[-1],'29') self.assertTrue(set(ids)<=set(roles['learning_parts'])) def test_partial_preparation_never_starts_pilot(self): with tempfile.TemporaryDirectory() as tmp: source=Path(tmp);(source/'contract.json').write_text('{}');(source/'batch-complete.json').write_text('{"status":"STOPPED"}') with self.assertRaises(AssertionError):finalize(source,source/'never-created','synthetic') self.assertFalse((source/'never-created').exists()) def test_contract_records_two_node_rates_totals_and_cannot_overwrite(self): with tempfile.TemporaryDirectory() as tmp: root=Path(tmp);(root/'timing-probe').mkdir() rows=[dict(case=str(i),nodes=(i+1)*100,training_seconds=(i+1)*.1,inference_seconds=(i+1)*.05) for i in range(20)] nodes=sum(r['nodes'] for r in rows) train=sum(r['training_seconds'] for r in rows)/nodes;infer=sum(r['inference_seconds'] for r in rows)/nodes plan=dict(probe_cases=[r['case'] for r in rows],learning_nodes=38700,validation_nodes=11200) probe=dict(status='COMPLETE',rows=rows,probe_nodes=nodes,labels_read=False,metrics_evaluated=False,standardizer_fingerprint='synthetic', **common_epoch_limit(train,infer,38700,11200)) (root/'timing-probe/result.json').write_text(json.dumps(probe));(root/'timing-probe-guard-receipt.json').write_text('{"total_run_wall_seconds":10}') c=freeze_training_contract(root,plan,probe) self.assertEqual(c['E_max'],33);self.assertEqual(c['cosine_T_max'],33) self.assertEqual(c['learning_nodes'],38700);self.assertEqual(c['validation_nodes'],11200) self.assertAlmostEqual(c['training_seconds_per_node'],.001);self.assertAlmostEqual(c['inference_seconds_per_node'],.0005) with self.assertRaises(FileExistsError):freeze_training_contract(root,plan,probe) def test_failed_probe_stops_before_training(self): with tempfile.TemporaryDirectory() as tmp: root=Path(tmp);(root/'pilot-plan.json').write_text('{"code_sha256":{},"gpu_uuid":"synthetic"}') with patch('run_s2_pilot.guarded_worker',return_value=({'status':'FAIL'},None)),patch('run_s2_pilot.train_runs') as train: with self.assertRaises(RuntimeError):run(root) train.assert_not_called() self.assertTrue((root/'orchestrator-failure.json').exists());self.assertFalse((root/'training-contract.json').exists()) def test_probe_time_reduces_total_budget(self): with tempfile.TemporaryDirectory() as tmp: root=Path(tmp);c=dict(gpu_uuid='synthetic',code_sha256={},probe_guard_wall_seconds=7190., runs=[dict(name=f'{arm}{width}',arm=arm,width=width,seed=0) for width in [128,256] for arm in ['N','E']]) (root/'training-contract.json').write_text(json.dumps(c)) with patch.object(guard,'guarded_worker',return_value=({'status':'FAIL'},None)) as worker: guard.run(root) self.assertEqual(worker.call_count,1);self.assertLessEqual(worker.call_args.args[4],10.) result=json.loads((root/'pilot-complete.json').read_text());self.assertGreaterEqual(result['pilot_wall_seconds'],7190.) self.assertIsNone(result['selected_global_width']) if __name__=='__main__':unittest.main(verbosity=2)