import io import json from pathlib import Path import tempfile import unittest from unittest.mock import patch from zipfile import ZipFile import numpy as np import run_s2_main_preparation as coordinator import s2_main_prepare as worker from s2_preprocess_pilot import load_selected,GEOMETRY_MEMBERS from g1_candidate_fingerprints import read_tets from freeze_s2_main_roster import assemble def synthetic_contract(root): return dict(cases=[dict(case=str(i),attempt=dict(wall_limit_seconds=3600 if i<6 else 1800)) for i in range(131)], prior_worker_cpu_seconds=125134.74286,limits=dict(worker_cpu_hours=20,workers=16,disk_bytes=100*2**30),budget_roots=[str(root)]) class MainPreparationChecks(unittest.TestCase): def test_label_free_adapter_accepts_test_case_without_label_io(self): with tempfile.TemporaryDirectory() as t: root=Path(t);case='synthetic_sealed_test' (root/'contract.json').write_text(json.dumps(dict(cases=[dict(case=case,attempt=dict(decode_labels=False,fold0_role='test'))]))) def geometric_run(r,i): d=r/'parts'/case;d.mkdir(parents=True) (d/'result.json').write_text(json.dumps(dict(status='COMPLETE',label_arrays_decoded=[],eligible_unique=1,eligible_slots=2000))) return 0 with patch.object(worker.s2_preprocess_pilot,'run',side_effect=geometric_run) as numerical: self.assertEqual(worker.run(root,0),0) numerical.assert_called_once_with(root,0) self.assertFalse(list(root.rglob('*label*'))) def test_actual_geometry_cache_whitelist_never_opens_label(self): with tempfile.TemporaryDirectory() as t: path=Path(t)/'mixed.npz';np.savez(path,points_raw=np.ones((2,3)),fill_time_raw=np.array([999.])) opened=[];original=ZipFile.open def spy(z,name,*a,**kw):opened.append(name);return original(z,name,*a,**kw) with patch.object(ZipFile,'open',spy):load_selected(path,['points_raw'],GEOMETRY_MEMBERS) self.assertEqual(opened,['points_raw.npy']) with self.assertRaises(ValueError):load_selected(path,['fill_time_raw'],GEOMETRY_MEMBERS) def test_zero_eligible_queries_fail_without_fabricating_ready(self): with tempfile.TemporaryDirectory() as t: root=Path(t);case='empty' (root/'contract.json').write_text(json.dumps(dict(cases=[dict(case=case,attempt=dict(decode_labels=False))]))) d=root/'parts'/case;d.mkdir(parents=True) (d/'result.json').write_text(json.dumps(dict(status='COMPLETE',label_arrays_decoded=[],eligible_unique=0,eligible_slots=0))) with patch.object(worker.s2_preprocess_pilot,'run',return_value=0):self.assertEqual(worker.run(root,0),3) self.assertEqual(json.loads((d/'eligibility-failure.json').read_text())['status'],'NO_COMMON_ELIGIBLE_QUERIES') def test_geometry_prefix_rejects_field_before_geometry_without_decode(self): xml=b'' with patch('g1_candidate_fingerprints.decode_array',side_effect=AssertionError('label decode')) as decode: with self.assertRaisesRegex(ValueError,'Field association'):read_tets(io.BytesIO(xml)) decode.assert_not_called() def test_guards_apply_60_and_30_minutes_to_label_free_worker(self): with tempfile.TemporaryDirectory() as t: root=Path(t);c=synthetic_contract(root);calls=[] def supervise(command,job,name,limits,**kw): calls.append((command,limits));return dict(status='PASS') with patch.object(coordinator,'contract',return_value=c),patch.object(coordinator,'supervise',supervise): self.assertEqual(coordinator.guard(root,0),0) self.assertEqual(coordinator.guard(root,6),0) self.assertEqual([x[1]['job_wall_seconds'] for x in calls],[3600,1800]) self.assertTrue(all(x[0][1]=='tools/s2_main_prepare.py' and x[1]['tree_rss_bytes']==1342177280 for x in calls)) def run_fake_batch(self,root,*,over_cpu=False,over_disk=False,monitor_fault=False): c=synthetic_contract(root);(root/'contract.json').write_text(json.dumps(c)) spawned=[];fault=False;terminated=[];peak=0 class FakeProcess: def __init__(self,i):self.index=i;self.pid=1000000+i;self.polls=0 def poll(self): self.polls+=1 if self.polls==1:return None p=root/'jobs'/f'job-{self.index:04d}'/'guard-result.json' p.write_text(json.dumps(dict(worker=dict(status='PASS',returncode=0,stop_reason=None,cpu_seconds=.01),supervisor_cpu_seconds=.001))) return 0 def wait(self):return 0 def spawn(command,**kwargs): nonlocal peak p=FakeProcess(int(command[-1]));spawned.append(p) peak=max(peak,sum(p.polls<2 for p in spawned));return p def disk(_): nonlocal fault if monitor_fault and spawned and not fault:fault=True;raise OSError('atomic replacement transient') return 100*2**30 if over_disk else 100 with patch.object(coordinator,'contract',return_value=c),patch.object(coordinator.subprocess,'Popen',spawn),\ patch.object(coordinator,'disk_usage',disk),patch.object(coordinator,'tree_usage',return_value=(5000 if over_cpu else .01,100,1)),\ patch.object(coordinator.time,'sleep'),patch.object(coordinator,'terminate',side_effect=lambda a:terminated.append(list(a))),\ patch('freeze_s2_main_roster.freeze') as freeze,patch('builtins.print'): coordinator.run(root) return json.loads((root/'batch-complete.json').read_text()),spawned,peak,freeze.call_count def test_131_terminal_and_transient_monitor_failure_without_retry(self): with tempfile.TemporaryDirectory() as t: r,spawned,peak,freezes=self.run_fake_batch(Path(t),monitor_fault=True) self.assertEqual(r['status'],'ATTEMPTS_COMPLETE');self.assertEqual(len(spawned),131) self.assertLessEqual(peak,16);self.assertEqual(r['monitor_errors'],1);self.assertEqual(freezes,1) self.assertAlmostEqual(r['worker_cpu_seconds'],1.31) self.assertAlmostEqual(r['cumulative_worker_cpu_seconds'],125134.74286+1.31) def test_new_20h_cap_stops_and_does_not_charge_prior_34h(self): with tempfile.TemporaryDirectory() as t: r,spawned,_,freezes=self.run_fake_batch(Path(t),over_cpu=True) self.assertEqual(r['stop_reason'],'GLOBAL_CPU_LIMIT');self.assertEqual(len(spawned),16) self.assertEqual(len(r['not_run_indices']),115);self.assertEqual(freezes,0) def test_cumulative_disk_cap_blocks_dispatch(self): with tempfile.TemporaryDirectory() as t: r,spawned,_,freezes=self.run_fake_batch(Path(t),over_disk=True) self.assertEqual(r['stop_reason'],'GLOBAL_DISK_LIMIT');self.assertEqual(spawned,[]);self.assertEqual(freezes,0) def test_existing_batch_marker_refuses_restart(self): with tempfile.TemporaryDirectory() as t: root=Path(t);(root/'batch-started.json').write_text('{}') with patch.object(coordinator,'contract',return_value=synthetic_contract(root)),patch.object(coordinator.subprocess,'Popen') as spawn: with self.assertRaises(AssertionError):coordinator.run(root) spawn.assert_not_called() def test_failed_part_excluded_globally_original_component_and_nested_pools(self): parts=[dict(case=str(i),nodes=i+10,component_id='unchanged-'+str(i),assignments=[dict(fold=f,role='learning',learning_pools=['50','100']) for f in range(5)]) for i in range(3)] jobs=[dict(case='1',worker=dict(status='PASS')),dict(case='2',worker=dict(status='STOPPED_OR_FAILED',stop_reason='wall_limit'))] rows,counts=assemble(parts,{'0':dict(nodes=10)},jobs,{'1':dict(nodes=11)}) self.assertFalse(rows[2]['included']);self.assertEqual(rows[2]['component_id'],'unchanged-2') self.assertEqual(rows[2]['assignments'],parts[2]['assignments']) for row in counts:self.assertEqual(row['learning']['parts'],0 if row['pool']==25 else 2) if __name__=='__main__':unittest.main(verbosity=2)