62 lines
3.6 KiB
Python
62 lines
3.6 KiB
Python
"""Pass actual recovery contracts through the actual guarded dispatch with mock hardware."""
|
|
import ast
|
|
import resource
|
|
from pathlib import Path
|
|
from tempfile import TemporaryDirectory
|
|
from unittest.mock import patch
|
|
|
|
from phase2_h1_worker_v2 import save
|
|
from phase3_followup_postfit_recovery_common import read
|
|
from phase3_followup_postfit_recovery_guard import run_prediction
|
|
|
|
GUARD = Path(__file__).with_name('phase3_followup_postfit_recovery_guard.py')
|
|
MODULE = 'phase3_followup_postfit_recovery_guard'
|
|
|
|
|
|
def run(contract_path):
|
|
c = read(contract_path)
|
|
guard_fields = sorted({n.slice.value for n in ast.walk(ast.parse(GUARD.read_text()))
|
|
if isinstance(n,ast.Subscript) and isinstance(n.value,ast.Name)
|
|
and n.value.id=='c' and isinstance(n.slice,ast.Constant)})
|
|
if set(guard_fields)!= {'jobs','gpu_uuid','idle_wait_seconds'} or set(guard_fields)-set(c):
|
|
raise ValueError('Actual guard read fields differ from the covered contract fields')
|
|
if c['idle_wait_seconds']!=20:
|
|
raise ValueError('Must use the fixed 20-second idle wait')
|
|
with TemporaryDirectory(prefix='meltmind-guard-synthetic-') as dirname:
|
|
temp = Path(dirname)
|
|
passed = []
|
|
for fold,job in enumerate(c['jobs']):
|
|
if job['id']!=fold or type(job['expected_parts']) is not int or job['expected_parts']<1:
|
|
raise ValueError('Actual nested guard fields missing or invalid')
|
|
jr = temp/'jobs'/str(fold)
|
|
jr.mkdir(parents=True)
|
|
|
|
def fake_worker(command, path, mode, limits, *, poll, gpu_uuid):
|
|
if (Path(command[1]).name!='phase3_followup_postfit_recovery_worker.py'
|
|
or command[2:]!=['predict',str(temp),str(fold)]
|
|
or path!=jr/'prediction-guard' or mode!='predict' or gpu_uuid!=c['gpu_uuid']
|
|
or limits['job_cpu_seconds']!=240 or not 0<limits['job_wall_seconds']<900):
|
|
raise ValueError('Actual guard dispatch contract mismatch')
|
|
(jr/'prediction-records').mkdir()
|
|
save(jr/'prediction-records/result.json',dict(status='COMPLETE',parts=job['expected_parts'],
|
|
peak_allocated_bytes=1024,peak_reserved_bytes=2048))
|
|
return dict(status='PASS',returncode=0,monitor_errors=0,cpu_seconds=.01)
|
|
|
|
with (patch(MODULE+'.device_usage',return_value=(5*2**20,0)) as device,
|
|
patch(MODULE+'.time.sleep'),
|
|
patch(MODULE+'.tree_usage',return_value=(0.,0,0)),
|
|
patch(MODULE+'.disk_usage',return_value=0),
|
|
patch(MODULE+'.supervise',side_effect=fake_worker) as dispatch):
|
|
result=run_prediction(temp,c,fold,0.,300)
|
|
if result['status']!='PASS' or dispatch.call_count!=1:
|
|
raise RuntimeError(f'Synthetic guard did not pass: {result}')
|
|
if any(call.args!=(c['gpu_uuid'],) for call in device.call_args_list):
|
|
raise ValueError('Actual GPU identity not propagated')
|
|
if read(jr/'recovery-idle-wait.json')['limit_seconds']!=20 or len(read(jr/'prediction-idle-polls.json'))!=3:
|
|
raise ValueError('Idle timing receipt mismatch')
|
|
passed.append(dict(fold=fold,expected_parts=job['expected_parts'],guard='PASS'))
|
|
u=resource.getrusage(resource.RUSAGE_SELF)
|
|
return dict(status='PASS',actual_contract=str(contract_path),guard_fields=guard_fields,
|
|
nested_guard_fields=['jobs[].expected_parts'],folds=passed,
|
|
real_gpu_or_worker_started=False,scientific_arrays_read=False,process_cpu_seconds=u.ru_utime+u.ru_stime)
|