133 lines
8.3 KiB
Python
133 lines
8.3 KiB
Python
import io
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
from zipfile import ZipFile
|
|
import numpy as np
|
|
import run_s2_main_preparation as coordinator
|
|
import s2_main_prepare as worker
|
|
from s2_preprocess_pilot import load_selected,GEOMETRY_MEMBERS
|
|
from g1_candidate_fingerprints import read_tets
|
|
from freeze_s2_main_roster import assemble
|
|
|
|
|
|
def synthetic_contract(root):
|
|
return dict(cases=[dict(case=str(i),attempt=dict(wall_limit_seconds=3600 if i<6 else 1800)) for i in range(131)],
|
|
prior_worker_cpu_seconds=125134.74286,limits=dict(worker_cpu_hours=20,workers=16,disk_bytes=100*2**30),budget_roots=[str(root)])
|
|
|
|
|
|
class MainPreparationChecks(unittest.TestCase):
|
|
def test_label_free_adapter_accepts_test_case_without_label_io(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
root=Path(t);case='synthetic_sealed_test'
|
|
(root/'contract.json').write_text(json.dumps(dict(cases=[dict(case=case,attempt=dict(decode_labels=False,fold0_role='test'))])))
|
|
def geometric_run(r,i):
|
|
d=r/'parts'/case;d.mkdir(parents=True)
|
|
(d/'result.json').write_text(json.dumps(dict(status='COMPLETE',label_arrays_decoded=[],eligible_unique=1,eligible_slots=2000)))
|
|
return 0
|
|
with patch.object(worker.s2_preprocess_pilot,'run',side_effect=geometric_run) as numerical:
|
|
self.assertEqual(worker.run(root,0),0)
|
|
numerical.assert_called_once_with(root,0)
|
|
self.assertFalse(list(root.rglob('*label*')))
|
|
|
|
def test_actual_geometry_cache_whitelist_never_opens_label(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
path=Path(t)/'mixed.npz';np.savez(path,points_raw=np.ones((2,3)),fill_time_raw=np.array([999.]))
|
|
opened=[];original=ZipFile.open
|
|
def spy(z,name,*a,**kw):opened.append(name);return original(z,name,*a,**kw)
|
|
with patch.object(ZipFile,'open',spy):load_selected(path,['points_raw'],GEOMETRY_MEMBERS)
|
|
self.assertEqual(opened,['points_raw.npy'])
|
|
with self.assertRaises(ValueError):load_selected(path,['fill_time_raw'],GEOMETRY_MEMBERS)
|
|
|
|
def test_zero_eligible_queries_fail_without_fabricating_ready(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
root=Path(t);case='empty'
|
|
(root/'contract.json').write_text(json.dumps(dict(cases=[dict(case=case,attempt=dict(decode_labels=False))])))
|
|
d=root/'parts'/case;d.mkdir(parents=True)
|
|
(d/'result.json').write_text(json.dumps(dict(status='COMPLETE',label_arrays_decoded=[],eligible_unique=0,eligible_slots=0)))
|
|
with patch.object(worker.s2_preprocess_pilot,'run',return_value=0):self.assertEqual(worker.run(root,0),3)
|
|
self.assertEqual(json.loads((d/'eligibility-failure.json').read_text())['status'],'NO_COMMON_ELIGIBLE_QUERIES')
|
|
|
|
def test_geometry_prefix_rejects_field_before_geometry_without_decode(self):
|
|
xml=b'<VTKFile byte_order="LittleEndian" compressor="vtkZLibDataCompressor"><UnstructuredGrid><Piece><PointData><DataArray Name="Fill_time"/></PointData></Piece></UnstructuredGrid></VTKFile>'
|
|
with patch('g1_candidate_fingerprints.decode_array',side_effect=AssertionError('label decode')) as decode:
|
|
with self.assertRaisesRegex(ValueError,'Field association'):read_tets(io.BytesIO(xml))
|
|
decode.assert_not_called()
|
|
|
|
def test_guards_apply_60_and_30_minutes_to_label_free_worker(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
root=Path(t);c=synthetic_contract(root);calls=[]
|
|
def supervise(command,job,name,limits,**kw):
|
|
calls.append((command,limits));return dict(status='PASS')
|
|
with patch.object(coordinator,'contract',return_value=c),patch.object(coordinator,'supervise',supervise):
|
|
self.assertEqual(coordinator.guard(root,0),0)
|
|
self.assertEqual(coordinator.guard(root,6),0)
|
|
self.assertEqual([x[1]['job_wall_seconds'] for x in calls],[3600,1800])
|
|
self.assertTrue(all(x[0][1]=='tools/s2_main_prepare.py' and x[1]['tree_rss_bytes']==1342177280 for x in calls))
|
|
|
|
def run_fake_batch(self,root,*,over_cpu=False,over_disk=False,monitor_fault=False):
|
|
c=synthetic_contract(root);(root/'contract.json').write_text(json.dumps(c))
|
|
spawned=[];fault=False;terminated=[];peak=0
|
|
class FakeProcess:
|
|
def __init__(self,i):self.index=i;self.pid=1000000+i;self.polls=0
|
|
def poll(self):
|
|
self.polls+=1
|
|
if self.polls==1:return None
|
|
p=root/'jobs'/f'job-{self.index:04d}'/'guard-result.json'
|
|
p.write_text(json.dumps(dict(worker=dict(status='PASS',returncode=0,stop_reason=None,cpu_seconds=.01),supervisor_cpu_seconds=.001)))
|
|
return 0
|
|
def wait(self):return 0
|
|
def spawn(command,**kwargs):
|
|
nonlocal peak
|
|
p=FakeProcess(int(command[-1]));spawned.append(p)
|
|
peak=max(peak,sum(p.polls<2 for p in spawned));return p
|
|
def disk(_):
|
|
nonlocal fault
|
|
if monitor_fault and spawned and not fault:fault=True;raise OSError('atomic replacement transient')
|
|
return 100*2**30 if over_disk else 100
|
|
with patch.object(coordinator,'contract',return_value=c),patch.object(coordinator.subprocess,'Popen',spawn),\
|
|
patch.object(coordinator,'disk_usage',disk),patch.object(coordinator,'tree_usage',return_value=(5000 if over_cpu else .01,100,1)),\
|
|
patch.object(coordinator.time,'sleep'),patch.object(coordinator,'terminate',side_effect=lambda a:terminated.append(list(a))),\
|
|
patch('freeze_s2_main_roster.freeze') as freeze,patch('builtins.print'):
|
|
coordinator.run(root)
|
|
return json.loads((root/'batch-complete.json').read_text()),spawned,peak,freeze.call_count
|
|
|
|
def test_131_terminal_and_transient_monitor_failure_without_retry(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
r,spawned,peak,freezes=self.run_fake_batch(Path(t),monitor_fault=True)
|
|
self.assertEqual(r['status'],'ATTEMPTS_COMPLETE');self.assertEqual(len(spawned),131)
|
|
self.assertLessEqual(peak,16);self.assertEqual(r['monitor_errors'],1);self.assertEqual(freezes,1)
|
|
self.assertAlmostEqual(r['worker_cpu_seconds'],1.31)
|
|
self.assertAlmostEqual(r['cumulative_worker_cpu_seconds'],125134.74286+1.31)
|
|
|
|
def test_new_20h_cap_stops_and_does_not_charge_prior_34h(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
r,spawned,_,freezes=self.run_fake_batch(Path(t),over_cpu=True)
|
|
self.assertEqual(r['stop_reason'],'GLOBAL_CPU_LIMIT');self.assertEqual(len(spawned),16)
|
|
self.assertEqual(len(r['not_run_indices']),115);self.assertEqual(freezes,0)
|
|
|
|
def test_cumulative_disk_cap_blocks_dispatch(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
r,spawned,_,freezes=self.run_fake_batch(Path(t),over_disk=True)
|
|
self.assertEqual(r['stop_reason'],'GLOBAL_DISK_LIMIT');self.assertEqual(spawned,[]);self.assertEqual(freezes,0)
|
|
|
|
def test_existing_batch_marker_refuses_restart(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
root=Path(t);(root/'batch-started.json').write_text('{}')
|
|
with patch.object(coordinator,'contract',return_value=synthetic_contract(root)),patch.object(coordinator.subprocess,'Popen') as spawn:
|
|
with self.assertRaises(AssertionError):coordinator.run(root)
|
|
spawn.assert_not_called()
|
|
|
|
def test_failed_part_excluded_globally_original_component_and_nested_pools(self):
|
|
parts=[dict(case=str(i),nodes=i+10,component_id='unchanged-'+str(i),assignments=[dict(fold=f,role='learning',learning_pools=['50','100']) for f in range(5)]) for i in range(3)]
|
|
jobs=[dict(case='1',worker=dict(status='PASS')),dict(case='2',worker=dict(status='STOPPED_OR_FAILED',stop_reason='wall_limit'))]
|
|
rows,counts=assemble(parts,{'0':dict(nodes=10)},jobs,{'1':dict(nodes=11)})
|
|
self.assertFalse(rows[2]['included']);self.assertEqual(rows[2]['component_id'],'unchanged-2')
|
|
self.assertEqual(rows[2]['assignments'],parts[2]['assignments'])
|
|
for row in counts:self.assertEqual(row['learning']['parts'],0 if row['pool']==25 else 2)
|
|
|
|
|
|
if __name__=='__main__':unittest.main(verbosity=2)
|