133 lines
8.1 KiB
Python
133 lines
8.1 KiB
Python
"""Execution acceptance with synthetic data/processes; no real dataset fitting."""
|
|
import copy
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
import numpy as np
|
|
import run_s3a_queue as queue
|
|
import run_s3a_fit as fit
|
|
import s3a_runtime as runtime
|
|
from freeze_g1_components import sha
|
|
from s2_custody_common import read_json
|
|
|
|
|
|
def example_contract():
|
|
return copy.deepcopy(read_json('artifacts/s3a-planning-2026-11-15/contract.json'))
|
|
|
|
|
|
class QueueChecks(unittest.TestCase):
|
|
def simulated(self, root, c, fail=None, missing=None, budget_stop=False):
|
|
spawned=[];peak=[0]
|
|
(root/'contract.json').write_text('{}')
|
|
(root/'implementation-freeze.json').write_text('{}')
|
|
owner=self
|
|
class Process:
|
|
def __init__(self,index):self.index=index;self.pid=900000+index;self.polls=0
|
|
def poll(self):
|
|
self.polls+=1
|
|
if self.polls<2:return None
|
|
if self.index==missing:return 1
|
|
job=c['jobs'][self.index];dest=root/'jobs'/f'job-{self.index:03d}';out=dest/'output';out.mkdir(exist_ok=True)
|
|
model=out/'model.joblib';model.write_bytes(b'synthetic')
|
|
result=out/'result.json'
|
|
result.write_text(json.dumps(dict(status='COMPLETE',fit_cpu_seconds=.01,
|
|
validation_component_equal_relative_l2=job.get('config_index',0)/100 if job['kind']=='HPO' else None,model_sha256=sha(model))))
|
|
failed=self.index==fail
|
|
receipt=dict(worker=dict(status='STOPPED_OR_FAILED' if failed else 'PASS',cpu_seconds=.02,returncode=1 if failed else 0,stop_reason='synthetic_failure' if failed else None),
|
|
supervisor_cpu_seconds=.001,result_path=str(result),result_sha256=sha(result),job=job)
|
|
(dest/'guard-result.json').write_text(json.dumps(receipt))
|
|
return 1 if failed else 0
|
|
def wait(self):self.polls=2;return self.poll()
|
|
def spawn(command,**kwargs):
|
|
i=int(command[-3])
|
|
if len(spawned)==1:owner.assertEqual(i,12)
|
|
if len(spawned)>1:owner.assertTrue((root/'first-fit-cost-check.json').exists())
|
|
if i>=360:owner.assertTrue((root/'selections'/f"cell-{c['jobs'][i]['cell']:02d}.json").exists())
|
|
proc=Process(i);spawned.append(proc);peak[0]=max(peak[0],sum(p.polls<2 for p in spawned));return proc
|
|
def dispatch(*args):return len(spawned)<2 if budget_stop else True
|
|
with patch.object(queue,'contract',return_value=c),patch.object(queue.os,'sched_setaffinity'),patch.object(queue.subprocess,'Popen',spawn),patch.object(queue,'tree_usage',return_value=(0.,1,1)),patch.object(queue.time,'sleep'),patch.object(queue,'terminate'),patch.object(queue,'can_dispatch',dispatch),patch('builtins.print'):
|
|
code=queue.run(root)
|
|
return code,spawned,peak[0]
|
|
|
|
def test_390_unique_attempts_two_sequential_witnesses_and_four_workers(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
root=Path(t);c=example_contract();code,spawned,peak=self.simulated(root,c)
|
|
self.assertEqual(code,0);self.assertEqual([p.index for p in spawned[:2]],[0,12])
|
|
self.assertEqual(len(spawned),390);self.assertEqual(len({p.index for p in spawned}),390);self.assertEqual(peak,4)
|
|
final=read_json(root/'terminal-freeze.json');self.assertEqual(len(final['final_fits']),30)
|
|
self.assertEqual(final['hpo_terminal_indices'],list(range(360)))
|
|
self.assertTrue(all(f['status']=='PASS' for f in final['final_fits']))
|
|
with patch.object(queue,'contract',return_value=c),patch.object(queue.subprocess,'Popen') as spawn:
|
|
with self.assertRaises(FileExistsError):queue.run(root)
|
|
spawn.assert_not_called()
|
|
|
|
def test_first_failure_stops_without_second_trial_or_retry(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
root=Path(t);code,spawned,_=self.simulated(root,example_contract(),fail=0)
|
|
self.assertEqual(code,1);self.assertEqual(len(spawned),1)
|
|
self.assertFalse((root/'terminal-freeze.json').exists())
|
|
result=read_json(root/'batch-complete.json')
|
|
self.assertEqual(result['stop_reason'],'FIRST_TRIAL_FAILED_NO_RETRY')
|
|
self.assertEqual(result['worker_cpu_seconds'],.02)
|
|
|
|
def test_missing_guard_is_infrastructure_stop_not_completed_batch(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
root=Path(t);code,spawned,_=self.simulated(root,example_contract(),missing=0)
|
|
self.assertEqual(code,1);self.assertEqual(len(spawned),1)
|
|
self.assertFalse(read_json(root/'batch-complete.json')['accounting_complete'])
|
|
self.assertFalse((root/'terminal-freeze.json').exists())
|
|
|
|
def test_nonfirst_hpo_failure_is_NA_config_and_final_fits_remaining(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
root=Path(t);code,spawned,_=self.simulated(root,example_contract(),fail=1)
|
|
self.assertEqual(code,0);decision=read_json(root/'selections/cell-00.json')
|
|
self.assertEqual(decision['finished_configurations'],11)
|
|
self.assertEqual(decision['config_index'],0)
|
|
self.assertEqual(read_json(root/'terminal-freeze.json')['final_fits'][0]['status'],'PASS')
|
|
|
|
def test_failed_final_kept_as_cell_NA_in_terminal_freeze(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
root=Path(t);code,_,_=self.simulated(root,example_contract(),fail=360)
|
|
self.assertEqual(code,0)
|
|
rows=read_json(root/'terminal-freeze.json')['final_fits']
|
|
self.assertEqual(rows[0]['status'],'NA');self.assertEqual(rows[0]['reason'],'synthetic_failure')
|
|
self.assertEqual(sum(f['status']=='PASS' for f in rows),29)
|
|
|
|
def test_reservation_stop_never_starts_next_task_or_freezes_complete(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
root=Path(t);code,spawned,_=self.simulated(root,example_contract(),budget_stop=True)
|
|
self.assertEqual(code,1);self.assertEqual(len(spawned),2)
|
|
self.assertEqual(read_json(root/'batch-complete.json')['stop_reason'],'INSUFFICIENT_REMAINING_FIT_CAP_NO_LAUNCH')
|
|
self.assertFalse((root/'terminal-freeze.json').exists())
|
|
|
|
def test_guard_uses_approved_limits_and_one_process(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
root=Path(t);seen=[]
|
|
def supervise(command,dest,name,limits,**kwargs):
|
|
seen.append((command,limits));return dict(status='STOPPED_OR_FAILED',stop_reason='synthetic')
|
|
with patch.object(queue,'contract',return_value=example_contract()),patch.object(queue.os,'sched_setaffinity'),patch.object(queue,'supervise',supervise):
|
|
self.assertEqual(queue.guard(root,0,0),1)
|
|
self.assertIn('tools/run_s3a_fit.py',seen[0][0])
|
|
self.assertEqual(seen[0][1]['job_cpu_seconds'],1800)
|
|
self.assertEqual(seen[0][1]['max_processes'],1)
|
|
|
|
def test_actual_synthetic_HistGBRT_has24columns_fixedgrid_and_singlethread(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
root=Path(t);(root/'contract.json').write_text('{}');(root/'jobs/job-000').mkdir(parents=True)
|
|
rng=np.random.default_rng(88)
|
|
def part(case,role,n):
|
|
x=rng.normal(size=(n,24));y=1+.1*x[:,0]
|
|
return dict(case=case,component_id=case,role=role,x=x,y=y,seconds=y*2,counts=np.ones(n),t_set=2.,label_mode='synthetic',label_sha256='synthetic')
|
|
with patch.object(runtime,'contract',return_value=example_contract()),patch.object(fit,'load_cell',return_value=([part('a','learning',240)],[part('b','validation',40)],{})),patch('builtins.print'):
|
|
self.assertEqual(fit.run(root,0),0)
|
|
r=read_json(root/'jobs/job-000/output/result.json')
|
|
self.assertEqual(len(r['columns']),24);self.assertEqual(r['params']['max_iter'],200)
|
|
self.assertFalse(r['params']['early_stopping']);self.assertTrue(all(p['num_threads']==1 for p in r['threadpools']))
|
|
self.assertFalse(r['test_labels_read'])
|
|
|
|
|
|
if __name__=='__main__':
|
|
unittest.main(verbosity=2)
|