"""Execution acceptance with synthetic data/processes; no real dataset fitting.""" import copy import json from pathlib import Path import tempfile import unittest from unittest.mock import patch import numpy as np import run_s3a_queue as queue import run_s3a_fit as fit import s3a_runtime as runtime from freeze_g1_components import sha from s2_custody_common import read_json def example_contract(): return copy.deepcopy(read_json('artifacts/s3a-planning-2026-11-15/contract.json')) class QueueChecks(unittest.TestCase): def simulated(self, root, c, fail=None, missing=None, budget_stop=False): spawned=[];peak=[0] (root/'contract.json').write_text('{}') (root/'implementation-freeze.json').write_text('{}') owner=self class Process: def __init__(self,index):self.index=index;self.pid=900000+index;self.polls=0 def poll(self): self.polls+=1 if self.polls<2:return None if self.index==missing:return 1 job=c['jobs'][self.index];dest=root/'jobs'/f'job-{self.index:03d}';out=dest/'output';out.mkdir(exist_ok=True) model=out/'model.joblib';model.write_bytes(b'synthetic') result=out/'result.json' result.write_text(json.dumps(dict(status='COMPLETE',fit_cpu_seconds=.01, validation_component_equal_relative_l2=job.get('config_index',0)/100 if job['kind']=='HPO' else None,model_sha256=sha(model)))) failed=self.index==fail receipt=dict(worker=dict(status='STOPPED_OR_FAILED' if failed else 'PASS',cpu_seconds=.02,returncode=1 if failed else 0,stop_reason='synthetic_failure' if failed else None), supervisor_cpu_seconds=.001,result_path=str(result),result_sha256=sha(result),job=job) (dest/'guard-result.json').write_text(json.dumps(receipt)) return 1 if failed else 0 def wait(self):self.polls=2;return self.poll() def spawn(command,**kwargs): i=int(command[-3]) if len(spawned)==1:owner.assertEqual(i,12) if len(spawned)>1:owner.assertTrue((root/'first-fit-cost-check.json').exists()) if i>=360:owner.assertTrue((root/'selections'/f"cell-{c['jobs'][i]['cell']:02d}.json").exists()) proc=Process(i);spawned.append(proc);peak[0]=max(peak[0],sum(p.polls<2 for p in spawned));return proc def dispatch(*args):return len(spawned)<2 if budget_stop else True with patch.object(queue,'contract',return_value=c),patch.object(queue.os,'sched_setaffinity'),patch.object(queue.subprocess,'Popen',spawn),patch.object(queue,'tree_usage',return_value=(0.,1,1)),patch.object(queue.time,'sleep'),patch.object(queue,'terminate'),patch.object(queue,'can_dispatch',dispatch),patch('builtins.print'): code=queue.run(root) return code,spawned,peak[0] def test_390_unique_attempts_two_sequential_witnesses_and_four_workers(self): with tempfile.TemporaryDirectory() as t: root=Path(t);c=example_contract();code,spawned,peak=self.simulated(root,c) self.assertEqual(code,0);self.assertEqual([p.index for p in spawned[:2]],[0,12]) self.assertEqual(len(spawned),390);self.assertEqual(len({p.index for p in spawned}),390);self.assertEqual(peak,4) final=read_json(root/'terminal-freeze.json');self.assertEqual(len(final['final_fits']),30) self.assertEqual(final['hpo_terminal_indices'],list(range(360))) self.assertTrue(all(f['status']=='PASS' for f in final['final_fits'])) with patch.object(queue,'contract',return_value=c),patch.object(queue.subprocess,'Popen') as spawn: with self.assertRaises(FileExistsError):queue.run(root) spawn.assert_not_called() def test_first_failure_stops_without_second_trial_or_retry(self): with tempfile.TemporaryDirectory() as t: root=Path(t);code,spawned,_=self.simulated(root,example_contract(),fail=0) self.assertEqual(code,1);self.assertEqual(len(spawned),1) self.assertFalse((root/'terminal-freeze.json').exists()) result=read_json(root/'batch-complete.json') self.assertEqual(result['stop_reason'],'FIRST_TRIAL_FAILED_NO_RETRY') self.assertEqual(result['worker_cpu_seconds'],.02) def test_missing_guard_is_infrastructure_stop_not_completed_batch(self): with tempfile.TemporaryDirectory() as t: root=Path(t);code,spawned,_=self.simulated(root,example_contract(),missing=0) self.assertEqual(code,1);self.assertEqual(len(spawned),1) self.assertFalse(read_json(root/'batch-complete.json')['accounting_complete']) self.assertFalse((root/'terminal-freeze.json').exists()) def test_nonfirst_hpo_failure_is_NA_config_and_final_fits_remaining(self): with tempfile.TemporaryDirectory() as t: root=Path(t);code,spawned,_=self.simulated(root,example_contract(),fail=1) self.assertEqual(code,0);decision=read_json(root/'selections/cell-00.json') self.assertEqual(decision['finished_configurations'],11) self.assertEqual(decision['config_index'],0) self.assertEqual(read_json(root/'terminal-freeze.json')['final_fits'][0]['status'],'PASS') def test_failed_final_kept_as_cell_NA_in_terminal_freeze(self): with tempfile.TemporaryDirectory() as t: root=Path(t);code,_,_=self.simulated(root,example_contract(),fail=360) self.assertEqual(code,0) rows=read_json(root/'terminal-freeze.json')['final_fits'] self.assertEqual(rows[0]['status'],'NA');self.assertEqual(rows[0]['reason'],'synthetic_failure') self.assertEqual(sum(f['status']=='PASS' for f in rows),29) def test_reservation_stop_never_starts_next_task_or_freezes_complete(self): with tempfile.TemporaryDirectory() as t: root=Path(t);code,spawned,_=self.simulated(root,example_contract(),budget_stop=True) self.assertEqual(code,1);self.assertEqual(len(spawned),2) self.assertEqual(read_json(root/'batch-complete.json')['stop_reason'],'INSUFFICIENT_REMAINING_FIT_CAP_NO_LAUNCH') self.assertFalse((root/'terminal-freeze.json').exists()) def test_guard_uses_approved_limits_and_one_process(self): with tempfile.TemporaryDirectory() as t: root=Path(t);seen=[] def supervise(command,dest,name,limits,**kwargs): seen.append((command,limits));return dict(status='STOPPED_OR_FAILED',stop_reason='synthetic') with patch.object(queue,'contract',return_value=example_contract()),patch.object(queue.os,'sched_setaffinity'),patch.object(queue,'supervise',supervise): self.assertEqual(queue.guard(root,0,0),1) self.assertIn('tools/run_s3a_fit.py',seen[0][0]) self.assertEqual(seen[0][1]['job_cpu_seconds'],1800) self.assertEqual(seen[0][1]['max_processes'],1) def test_actual_synthetic_HistGBRT_has24columns_fixedgrid_and_singlethread(self): with tempfile.TemporaryDirectory() as t: root=Path(t);(root/'contract.json').write_text('{}');(root/'jobs/job-000').mkdir(parents=True) rng=np.random.default_rng(88) def part(case,role,n): x=rng.normal(size=(n,24));y=1+.1*x[:,0] return dict(case=case,component_id=case,role=role,x=x,y=y,seconds=y*2,counts=np.ones(n),t_set=2.,label_mode='synthetic',label_sha256='synthetic') with patch.object(runtime,'contract',return_value=example_contract()),patch.object(fit,'load_cell',return_value=([part('a','learning',240)],[part('b','validation',40)],{})),patch('builtins.print'): self.assertEqual(fit.run(root,0),0) r=read_json(root/'jobs/job-000/output/result.json') self.assertEqual(len(r['columns']),24);self.assertEqual(r['params']['max_iter'],200) self.assertFalse(r['params']['early_stopping']);self.assertTrue(all(p['num_threads']==1 for p in r['threadpools'])) self.assertFalse(r['test_labels_read']) if __name__=='__main__': unittest.main(verbosity=2)