Files
MeltMind/tools/test_s2_pilot_orchestration.py
T

59 lines
4.0 KiB
Python

import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from prepare_s2_pilot_training import quantile_probe,finalize
from s2_pilot_train import common_epoch_limit
from run_s2_pilot import freeze_training_contract,run
import s2_pilot_gpu_guard as guard
class OrchestrationTests(unittest.TestCase):
def test_quantiles_only_learning_and_include_size_extremes(self):
records=[dict(case=str(i),nodes=i+1) for i in range(40)]
roles=dict(learning_parts=[str(i) for i in range(30)],validation_parts=[str(i) for i in range(30,40)],sealed_test_parts=['test'])
ids=quantile_probe(records,roles)
self.assertEqual(len(set(ids)),20);self.assertEqual(ids[0],'0');self.assertEqual(ids[-1],'29')
self.assertTrue(set(ids)<=set(roles['learning_parts']))
def test_partial_preparation_never_starts_pilot(self):
with tempfile.TemporaryDirectory() as tmp:
source=Path(tmp);(source/'contract.json').write_text('{}');(source/'batch-complete.json').write_text('{"status":"STOPPED"}')
with self.assertRaises(AssertionError):finalize(source,source/'never-created','synthetic')
self.assertFalse((source/'never-created').exists())
def test_contract_records_two_node_rates_totals_and_cannot_overwrite(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);(root/'timing-probe').mkdir()
rows=[dict(case=str(i),nodes=(i+1)*100,training_seconds=(i+1)*.1,inference_seconds=(i+1)*.05) for i in range(20)]
nodes=sum(r['nodes'] for r in rows)
train=sum(r['training_seconds'] for r in rows)/nodes;infer=sum(r['inference_seconds'] for r in rows)/nodes
plan=dict(probe_cases=[r['case'] for r in rows],learning_nodes=38700,validation_nodes=11200)
probe=dict(status='COMPLETE',rows=rows,probe_nodes=nodes,labels_read=False,metrics_evaluated=False,standardizer_fingerprint='synthetic',
**common_epoch_limit(train,infer,38700,11200))
(root/'timing-probe/result.json').write_text(json.dumps(probe));(root/'timing-probe-guard-receipt.json').write_text('{"total_run_wall_seconds":10}')
c=freeze_training_contract(root,plan,probe)
self.assertEqual(c['E_max'],33);self.assertEqual(c['cosine_T_max'],33)
self.assertEqual(c['learning_nodes'],38700);self.assertEqual(c['validation_nodes'],11200)
self.assertAlmostEqual(c['training_seconds_per_node'],.001);self.assertAlmostEqual(c['inference_seconds_per_node'],.0005)
with self.assertRaises(FileExistsError):freeze_training_contract(root,plan,probe)
def test_failed_probe_stops_before_training(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);(root/'pilot-plan.json').write_text('{"code_sha256":{},"gpu_uuid":"synthetic"}')
with patch('run_s2_pilot.guarded_worker',return_value=({'status':'FAIL'},None)),patch('run_s2_pilot.train_runs') as train:
with self.assertRaises(RuntimeError):run(root)
train.assert_not_called()
self.assertTrue((root/'orchestrator-failure.json').exists());self.assertFalse((root/'training-contract.json').exists())
def test_probe_time_reduces_total_budget(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);c=dict(gpu_uuid='synthetic',code_sha256={},probe_guard_wall_seconds=7190.,
runs=[dict(name=f'{arm}{width}',arm=arm,width=width,seed=0) for width in [128,256] for arm in ['N','E']])
(root/'training-contract.json').write_text(json.dumps(c))
with patch.object(guard,'guarded_worker',return_value=({'status':'FAIL'},None)) as worker:
guard.run(root)
self.assertEqual(worker.call_count,1);self.assertLessEqual(worker.call_args.args[4],10.)
result=json.loads((root/'pilot-complete.json').read_text());self.assertGreaterEqual(result['pilot_wall_seconds'],7190.)
self.assertIsNone(result['selected_global_width'])
if __name__=='__main__':unittest.main(verbosity=2)