Files
MeltMind/tools/check_phase3_followup_recovery_guard.py
T

62 lines
3.6 KiB
Python

"""Pass actual recovery contracts through the actual guarded dispatch with mock hardware."""
import ast
import resource
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest.mock import patch
from phase2_h1_worker_v2 import save
from phase3_followup_postfit_recovery_common import read
from phase3_followup_postfit_recovery_guard import run_prediction
GUARD = Path(__file__).with_name('phase3_followup_postfit_recovery_guard.py')
MODULE = 'phase3_followup_postfit_recovery_guard'
def run(contract_path):
c = read(contract_path)
guard_fields = sorted({n.slice.value for n in ast.walk(ast.parse(GUARD.read_text()))
if isinstance(n,ast.Subscript) and isinstance(n.value,ast.Name)
and n.value.id=='c' and isinstance(n.slice,ast.Constant)})
if set(guard_fields)!= {'jobs','gpu_uuid','idle_wait_seconds'} or set(guard_fields)-set(c):
raise ValueError('Actual guard read fields differ from the covered contract fields')
if c['idle_wait_seconds']!=20:
raise ValueError('Must use the fixed 20-second idle wait')
with TemporaryDirectory(prefix='meltmind-guard-synthetic-') as dirname:
temp = Path(dirname)
passed = []
for fold,job in enumerate(c['jobs']):
if job['id']!=fold or type(job['expected_parts']) is not int or job['expected_parts']<1:
raise ValueError('Actual nested guard fields missing or invalid')
jr = temp/'jobs'/str(fold)
jr.mkdir(parents=True)
def fake_worker(command, path, mode, limits, *, poll, gpu_uuid):
if (Path(command[1]).name!='phase3_followup_postfit_recovery_worker.py'
or command[2:]!=['predict',str(temp),str(fold)]
or path!=jr/'prediction-guard' or mode!='predict' or gpu_uuid!=c['gpu_uuid']
or limits['job_cpu_seconds']!=240 or not 0<limits['job_wall_seconds']<900):
raise ValueError('Actual guard dispatch contract mismatch')
(jr/'prediction-records').mkdir()
save(jr/'prediction-records/result.json',dict(status='COMPLETE',parts=job['expected_parts'],
peak_allocated_bytes=1024,peak_reserved_bytes=2048))
return dict(status='PASS',returncode=0,monitor_errors=0,cpu_seconds=.01)
with (patch(MODULE+'.device_usage',return_value=(5*2**20,0)) as device,
patch(MODULE+'.time.sleep'),
patch(MODULE+'.tree_usage',return_value=(0.,0,0)),
patch(MODULE+'.disk_usage',return_value=0),
patch(MODULE+'.supervise',side_effect=fake_worker) as dispatch):
result=run_prediction(temp,c,fold,0.,300)
if result['status']!='PASS' or dispatch.call_count!=1:
raise RuntimeError(f'Synthetic guard did not pass: {result}')
if any(call.args!=(c['gpu_uuid'],) for call in device.call_args_list):
raise ValueError('Actual GPU identity not propagated')
if read(jr/'recovery-idle-wait.json')['limit_seconds']!=20 or len(read(jr/'prediction-idle-polls.json'))!=3:
raise ValueError('Idle timing receipt mismatch')
passed.append(dict(fold=fold,expected_parts=job['expected_parts'],guard='PASS'))
u=resource.getrusage(resource.RUSAGE_SELF)
return dict(status='PASS',actual_contract=str(contract_path),guard_fields=guard_fields,
nested_guard_fields=['jobs[].expected_parts'],folds=passed,
real_gpu_or_worker_started=False,scientific_arrays_read=False,process_cpu_seconds=u.ru_utime+u.ru_stime)