Prepare one labelled item1 postfit recovery with bounded idle wait

This commit is contained in:
MeltAstra committed 2026-11-18 14:04:43 +08:00
1 parent 4f646232f4
commit 581208098f
24 files changed
+83789

No files matched your search

+141
View File
@@ -0,0 +1,141 @@
"""Single labelled resume: reuse0, predict1..6, freeze all, unchanged score."""
import fcntl
import os
from pathlib import Path
import resource
import shutil
import sys
import time
import psutil
from phase2_gate_outcome_review import validate_review
from phase2_h1_worker_v2 import checked, save, sha
from phase2_home_freeze_io import freeze, verify
from phase3_item1_postfit_common import read, manifest_identity, require_budget
from phase3_item1_resume_common import config, adoption, reuse_entries, cumulative_cost
from phase3_item1_resume_guard import run_prediction
from r4bc_guard import supervise
def install_reuse(root, entries):
result = []
for e in entries:
original = checked(e['path'], e['sha256'])
target = root / 'predictions' / e['arm'] / (e['case'] + '.npz')
target.parent.mkdir(parents=True, exist_ok=True)
if target.exists():
raise FileExistsError('No duplicate prediction adoption')
shutil.copyfile(original, target)
checked(target, e['sha256'])
result.append(dict(e, path=str(target), reuse_original_path=str(original),
reused_without_prediction=True))
out = root / 'jobs/0/prediction-records'
out.mkdir(parents=True, exist_ok=False)
save(out / 'reuse.json', dict(status='REUSED_BY_HASH_NO_WORKER', entries=result, targets_read=False))
return result
def run(root):
gpu_clock = time.monotonic()
start = time.process_time()
u0 = resource.getrusage(resource.RUSAGE_CHILDREN)
c, tc, pilot = config(root)
validate_review(root)
adopted = adoption(c, tc)
lock = (root / 'run.lock').open('a')
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (root / 'started.json').exists():
raise FileExistsError('One labelled recovery, no repeat launch')
for p in psutil.process_iter(['pid', 'cmdline']):
if p.pid != os.getpid() and any(Path(a).name in {'phase3_item1_postfit.py', 'phase3_item1_resume.py',
'phase3_item1_postfit_worker.py'} for a in p.info['cmdline'] or []):
raise RuntimeError('Existing original or resumed postfit')
gpu_lock = Path('/tmp/meltmind-item1-resume-' + c['gpu_uuid'] + '.lock').open('a')
fcntl.flock(gpu_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
os.sched_setaffinity(0, c['coordinator_cpus'])
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
new_gpu = 0.
charged_children = 0.
records = []
prediction_start = None
def costs():
u = resource.getrusage(resource.RUSAGE_CHILDREN)
children = u.ru_utime + u.ru_stime - u0.ru_utime - u0.ru_stime
new_cpu = 180 + time.process_time() - start + max(charged_children, children)
return cumulative_cost(c, new_gpu, new_cpu)
def budget(remaining):
d = costs()
require_budget(c['training_gpu_once'], d['prediction_gpu_seconds'], d['postfit_cpu_seconds'], remaining)
return d
save(root / 'started.json', dict(pid=os.getpid(), unix=time.time(), recovery_attempt=1,
original_root=c['original_root'], prior_postfit_gpu=c['prior_postfit_gpu'],
prior_postfit_cpu=c['prior_postfit_cpu'], training_gpu_seconds_once=c['training_gpu_once'],
cpu_setup_allowance=180, review_sha256=sha(root / 'review.json'), indices=c['resume_indices'],
gpu_uuid=c['gpu_uuid'], new_fits=0, new_decodes=0))
try:
budget(6)
entries = install_reuse(root, reuse_entries(c, tc))
save(root / 'models-adopted.json', adopted)
freeze(root, 'models', [root / 'contract.json', root / 'models-adopted.json',
Path(c['original_root']) / 'models-adopted.json',
*[Path(m['checkpoint']) for m in adopted['models']],
*[Path(m['standardization']) for m in adopted['models']]])
for index in c['resume_indices']:
new_gpu = max(new_gpu, 60 + time.monotonic() - gpu_clock)
budget(7 - index)
(root / 'jobs' / str(index)).mkdir(parents=True, exist_ok=False)
prediction_start = time.monotonic()
g = run_prediction(root, c, pilot, index, c['training_gpu_once'] + costs()['prediction_gpu_seconds'], 300)
cost = max(60 + time.monotonic() - prediction_start, g['gpu_seconds_charged'])
prediction_start = None
new_gpu += cost
charged_children += g['worker']['cpu_seconds'] if g['worker'] else 0.
records.append(dict(mode='predict', index=index, charged_gpu_seconds=cost, guard=g))
d = costs()
save(root / 'resource-progress.json', dict(records=records, **d, original_complete_sha256=c['original_complete_sha256']))
if g['status'] != 'PASS' or cost > 900 or d['prediction_gpu_seconds'] > 7200 or d['cumulative_item_gpu_seconds'] > 120000:
raise RuntimeError('Recovery prediction/resource guard failure')
entries.extend(read(root / 'jobs' / str(index) / 'prediction-records/manifest.json')['entries'])
manifest_identity(entries, c, tc, pilot)
save(root / 'predictions-manifest.json', dict(entries=entries, predict0_reused_by_hash=686,
S1_test_predictions_reused=121, new_fits=0, targets_read=False))
freeze(root, 'predictions', [root / 'contract.json', root / 'models-adopted.json',
root / 'predictions-manifest.json', root / 'resource-progress.json',
root / 'jobs/0/prediction-records/reuse.json', *[Path(p) for p in c['code_sha256']]])
new_gpu = max(new_gpu, 60 + time.monotonic() - gpu_clock)
budget(0)
validate_review(root)
verify(root, 'models')
verify(root, 'predictions')
g = supervise([sys.executable, 'tools/phase3_item1_postfit_worker.py', 'score', str(root)],
root / 'score-guard', dict(cpu_seconds=900, wall_seconds=1200, rss_bytes=16*2**30, disk_bytes=20*2**30))
charged_children += g['cpu_seconds']
records.append(dict(mode='score', guard=g))
if g['status'] != 'PASS' or costs()['postfit_cpu_seconds'] + 120 > 3600:
raise RuntimeError('Score/resource failure')
save(root / 'complete.json', dict(status='COMPLETE', records=records, **costs(),
original_complete_sha256=c['original_complete_sha256'], predict0_reused=True,
resumed_indices=c['resume_indices'], item_gpu_cap=120000, new_fits=0, new_decodes=0,
future_phase3_started=False, no_retry=True))
freeze(root, 'terminal', [root / 'contract.json', root / 'complete.json', root / 'score/result.json', root / 'review.json'])
final = costs()
within = final['postfit_cpu_seconds'] <= 3600 and final['prediction_gpu_seconds'] <= 7200 and final['cumulative_item_gpu_seconds'] <= 120000
save(root / 'post-terminal-cost.json', dict(**final, within_cap=within))
if not within:
raise RuntimeError('Final cumulative resource cap')
except BaseException as exc:
if prediction_start is not None:
new_gpu += 60 + time.monotonic() - prediction_start
if (root / 'complete.json').exists():
save(root / 'pre-failure-complete.json', read(root / 'complete.json'))
save(root / 'complete.json', dict(status='STOPPED', reason=type(exc).__name__ + ': ' + str(exc),
records=records, **costs(), original_complete_sha256=c['original_complete_sha256'],
new_fits=0, no_retry=True, recovery_attempt=1))
raise
if __name__ == '__main__':
run(Path(sys.argv[1]))
+120
View File
@@ -0,0 +1,120 @@
"""One labelled infrastructure continuation; original scientific modules unchanged."""
import math
from pathlib import Path
from phase2_h1_worker_v2 import checked, sha
from phase2_home_freeze_io import verify
from phase3_item1_postfit_common import config as original_config, read, terminal_inputs
ALLOWED_GPUS = {'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36'}
def validate_scope(c, old):
if c['recovery_approval'] != '1794980643.205679' or c['resume_indices'] != list(range(1, 7)):
raise PermissionError('Exactly one recovery for jobs1 through6')
if (c['gpu_uuid'] not in ALLOWED_GPUS or c['recovery_attempt'] != 1
or c['idle_wait_seconds'] != 20 or c['idle_wait_approval'] != '1794981566.532619'):
raise PermissionError('Reserved Track A GPU or extra recovery')
for key in ('jobs', 'rows', 'training_root', 'training_contract', 'comparison_contract_sha256',
'postfit_gpu_cap', 'postfit_cpu_cap', 'item_gpu_cap', 'new_fits', 'new_decodes'):
if c[key] != old[key]:
raise ValueError('Original science/budget differs: ' + key)
if c['postfit_root'] == old['postfit_root']:
raise ValueError('Original stopped root must remain untouched')
for p, h in old['code_sha256'].items():
if c['code_sha256'].get(p) != h:
raise ValueError('Original pinned source changed')
def validate_stop(done):
records = done['records']
if done['status'] != 'STOPPED' or len(records) != 2 or [x['index'] for x in records] != [0, 1]:
raise ValueError('Unexpected original stop scope')
first, failed = [x['guard'] for x in records]
if first['status'] != 'PASS' or first['monitor_errors'] or not first['worker'] or first['worker']['status'] != 'PASS' or first['worker'].get('monitor_errors') != 0:
raise ValueError('Predict0 was not a clean PASS')
if failed['status'] != 'STOPPED' or failed['worker'] is not None or 'RuntimeError: SELECTED_GPU_BUSY_NO_RETRY' not in failed['reasons']:
raise ValueError('Not the authorized zero-worker infrastructure stop')
for key in ('prediction_gpu_seconds', 'postfit_cpu_seconds', 'training_gpu_seconds_once'):
if not math.isfinite(done[key]) or done[key] < 0:
raise ValueError('Invalid original ledger')
def six_receipts(c):
source = Path(c['original_root']) / 'tron-six-training-receipts-check.json'
check = read(source)
if check['status'] != 'PASS' or [r['job'] for r in check['rows']] != list(range(6)):
raise ValueError('Original six-receipt check missing')
for row in check['rows']:
p = Path(c['training_root']) / 'jobs' / str(row['job']) / 'guards/train.receipt.json'
if str(p) != row['path']:
raise ValueError('Original training receipt path mismatch')
receipt = read(checked(p, row['sha256']))
if type(receipt.get('monitor_errors')) is not int or receipt['monitor_errors'] != 0 or receipt['status'] != 'PASS' or receipt['returncode'] != 0:
raise ValueError('Training monitoring receipt not clean')
guard = read(p.parent.parent / 'complete.json')
if guard['worker'] != receipt:
raise ValueError('Embedded training receipt differs')
return check
def config(root):
c, tc, pilot = original_config(root)
original = Path(c['original_root'])
old, _, _ = original_config(original)
validate_scope(c, old)
done = read(checked(original / 'complete.json', c['original_complete_sha256']))
validate_stop(done)
if any((original / n).exists() for n in ('score', 'predictions-manifest.json')):
raise ValueError('Original scoring or full prediction barrier already exists')
for k, name in [('prior_postfit_gpu', 'prediction_gpu_seconds'), ('prior_postfit_cpu', 'postfit_cpu_seconds'),
('training_gpu_once', 'training_gpu_seconds_once')]:
if c[k] != done[name]:
raise ValueError('Original cost must be retained exactly once')
six_receipts(c)
return c, tc, pilot
def adoption(c, tc):
current = terminal_inputs(c, tc)
original = Path(c['original_root'])
verify(original, 'models')
old = read(original / 'models-adopted.json')
if old != current or current['training_gpu_seconds_once'] != c['training_gpu_once']:
raise ValueError('Predict0 and resumed jobs must use identical frozen models')
return current
def reuse_entries(c, tc, *, verify_bytes=True):
original = Path(c['original_root'])
manifest = read(checked(original / 'jobs/0/prediction-records/manifest.json', c['predict0_manifest_sha256']))
result = read(original / 'jobs/0/prediction-records/result.json')
if result['status'] != 'COMPLETE' or result['targets_read'] or result['parts'] != c['jobs'][0]['expected_parts']:
raise ValueError('Predict0 result mismatch')
entries = manifest['entries']
f = tc['pools']['25']
domains = {'learning': 'own_learning', 'validation': 'common_validation', 'test': 'common_test'}
expected = {(arm, case): domain for role, domain in domains.items() for case in f[role]
for arm in ('epoch25', 'c25F')}
if len(entries) != len(expected) or {(e['arm'], e['case']) for e in entries} != set(expected):
raise ValueError('Complete predict0 model and clock coverage required')
if manifest['targets_read'] or result['entries'] != len(entries):
raise ValueError('Predict0 targets/entry count')
for e in entries:
p = original / 'predictions' / e['arm'] / (e['case'] + '.npz')
if (e['path'] != str(p) or e['pool'] != 25 or e['domain'] != expected[(e['arm'], e['case'])]
or e['component_id'] != c['rows'][e['case']]['component_id']
or e['origin'] != 'ITEM1_FROZEN_ENDPOINT_OR_CLOCK'):
raise ValueError('Predict0 original identity')
if verify_bytes:
checked(p, e['sha256'])
return entries
def cumulative_cost(c, new_gpu, new_cpu):
if any(not math.isfinite(v) or v < 0 for v in [new_gpu, new_cpu]):
raise ValueError('Invalid new cost ledger')
gpu = c['prior_postfit_gpu'] + new_gpu
return dict(prediction_gpu_seconds=gpu, postfit_cpu_seconds=c['prior_postfit_cpu'] + new_cpu,
training_gpu_seconds_once=c['training_gpu_once'],
cumulative_item_gpu_seconds=c['training_gpu_once'] + gpu)
+85
View File
@@ -0,0 +1,85 @@
"""Recovery-only guard: bounded pre-worker idle wait inside original900s cap."""
import json,os,signal,sys,threading,time
import psutil
from phase2_h1_worker_v2 import save
from s2_main_memory_guard import PeakRecords
from s2_precision_continuation_guard import device_usage
from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage
def final_reasons(reasons,receipt,result,tracker,gpu_seconds,cpu_seconds,cpu_cap,expected_parts):
out=list(reasons)
def add(s):
if s not in out:out.append(s)
if not receipt or receipt['status']!='PASS':add('WORKER_NOT_PASS')
if receipt and receipt.get('monitor_errors'):add('WORKER_MONITOR_ERRORS')
if not result or result.get('status')!='COMPLETE' or result.get('parts')!=expected_parts:add('INCOMPLETE_FOLD')
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):add('MISSING_PEAKS')
if any(v>15*2**30 for v in tracker.maxima.values()):add('RECORDED_VRAM_LIMIT')
if gpu_seconds>900 or cpu_seconds>cpu_cap:add('FINAL_RESOURCE_LIMIT')
return out
def run_prediction(queue,c,pilot,fold,prior_gpu,cpu_cap):
root=queue/'jobs'/str(fold);gpu_uuid=c['gpu_uuid'];expected_parts=c['jobs'][fold]['expected_parts']
start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None
def elapsed():return 60+time.monotonic()-start
def reason(x):
if x not in reasons:reasons.append(x)
command=[sys.executable,'tools/phase3_item1_postfit_worker.py','predict',str(queue),str(fold)]
def kill():
for p in psutil.Process().children(recursive=True):
try:
if p.cmdline()==command:stop_group(p.pid)
except psutil.NoSuchProcess:pass
def scan():
if tracker.scan(root/'prediction-records'):reason('RECORDED_VRAM_LIMIT')
def watch():
while not done.is_set():
try:
scan();mem,util=device_usage(gpu_uuid);_,rss,_=tree_usage(os.getpid())
if mem>15*2**30:reason('DEVICE_VRAM_LIMIT')
if rss>16*2**30:reason('TREE_RSS_LIMIT')
if disk_usage(queue)>20*2**30:reason('DISK_LIMIT')
if elapsed()>=895:reason('PREDICTION_TIME_LIMIT')
for x in requested:reason(x)
save(root/'prediction-live.json',dict(gpu_seconds_charged=elapsed(),device_bytes=mem,device_utilization=util,rss_bytes=rss,peaks=tracker.maxima,reasons=reasons,unix=time.time()))
except Exception as exc:reason('MONITOR_ERROR');errors.append(str(exc))
if reasons:kill()
done.wait(.5)
old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGTERM,signal.SIGINT]};thread=None
try:
wait_start=time.monotonic();waiting=[]
while True:
mem,util=device_usage(gpu_uuid);waiting.append(dict(unix=time.time(),bytes=mem,utilization=util))
save(root/'recovery-idle-wait.json',dict(samples=waiting,limit_seconds=c['idle_wait_seconds'],worker_started=False))
if requested:raise RuntimeError('Signal received during idle wait')
duration=time.monotonic()-wait_start
if duration>c['idle_wait_seconds']:raise RuntimeError('SELECTED_GPU_BUSY_AFTER_BOUNDED_WAIT_NO_RETRY')
if mem<=128*2**20 and util==0:break
if duration>=c['idle_wait_seconds']:raise RuntimeError('SELECTED_GPU_BUSY_AFTER_BOUNDED_WAIT_NO_RETRY')
time.sleep(min(.5,max(0.,c['idle_wait_seconds']-(time.monotonic()-wait_start))))
for i in range(3):
mem,util=device_usage(gpu_uuid);samples.append(dict(unix=time.time(),bytes=mem,utilization=util))
if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY_NO_RETRY')
if i<2:time.sleep(2)
save(root/'prediction-idle-polls.json',samples)
if requested:raise RuntimeError('Signal received during setup')
thread=threading.Thread(target=watch,daemon=True);thread.start()
remaining=900-elapsed()-10
if remaining<=0:raise RuntimeError('Setup exhausted prediction budget')
receipt=supervise(command,root/'prediction-guard','predict',dict(total_cpu_seconds=cpu_cap-60,job_cpu_seconds=cpu_cap-60,job_wall_seconds=remaining,tree_rss_bytes=16*2**30,disk_bytes=20*2**30,max_processes=1),poll=.2,gpu_uuid=gpu_uuid)
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill()
finally:
done.set()
if thread:thread.join(timeout=4)
if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT')
try:scan()
except Exception as exc:reason('FINAL_SCAN_ERROR');errors.append(str(exc))
for s,h in old.items():signal.signal(s,h)
result=None;p=root/'prediction-records/result.json'
try:
if p.exists():result=json.loads(p.read_text())
except Exception as exc:reason('FINAL_RESULT_READ_ERROR');errors.append(str(exc))
totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu
duration=elapsed();reasons=final_reasons(reasons,receipt,result,tracker,duration,totalcpu,cpu_cap,expected_parts)
out=dict(status='PASS' if not reasons else 'STOPPED',reasons=reasons,monitor_errors=errors,worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=duration,cpu_seconds_charged=totalcpu,cumulative_item1_gpu_seconds=prior_gpu+duration,no_retry=True)
save(root/'prediction-guard.json',out);return out
+69
View File
@@ -0,0 +1,69 @@
"""Metadata and opaque-hash preparation; no new prediction or target access."""
from pathlib import Path
import time
from phase2_h1_worker_v2 import checked, save, sha
from phase2_gate_outcome_review import validate_review
from phase2_home_freeze_io import verify
from phase3_item1_postfit_common import config as old_config, read
from phase3_item1_resume_common import validate_scope, validate_stop, six_receipts, adoption, reuse_entries
from track_a_h2_prepare import local_code_closure
B = Path(__file__).resolve().parents[1]
O = Path('/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18')
R = O.parent / 'phase3-item1-postfit-recovery-2026-11-18'
def prepare():
start = time.process_time()
if R.exists():
raise FileExistsError('Existing recovery entry: inspect before any further action')
old, tc, pilot = old_config(O)
validate_review(O)
done = read(O / 'complete.json')
validate_stop(done)
sources = dict(old['source_sha256'])
for name in ['contract.json', 'complete.json', 'started.json', 'resource-progress.json',
'tron-six-training-receipts-check.json', 'models-adopted.json', 'models-git-freeze.json',
'review.json', 'implementation-git-freeze.json',
'jobs/0/prediction-guard.json', 'jobs/0/prediction-records/manifest.json',
'jobs/0/prediction-records/result.json', 'jobs/0/prediction-records/environment.json']:
sources[str(O / name)] = sha(O / name)
codes = dict(old['code_sha256'])
additional = local_code_closure([B / 'tools' / n for n in ['phase3_item1_resume.py',
'phase3_item1_resume_common.py', 'phase3_item1_resume_guard.py', 'prepare_phase3_item1_resume.py',
'test_phase3_item1_resume.py']])
for p, h in additional.items():
if p in codes and codes[p] != h:
raise ValueError('Old pinned code changed')
codes[p] = h
c = dict(old, postfit_root=str(R), code_sha256=codes, source_sha256=sources,
gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
original_root=str(O), original_complete_sha256=sha(O / 'complete.json'),
recovery_approval='1794980643.205679', recovery_attempt=1, resume_indices=list(range(1, 7)),
idle_wait_approval='1794981566.532619', idle_wait_seconds=20,
predict0_manifest_sha256=sha(O / 'jobs/0/prediction-records/manifest.json'),
prior_postfit_gpu=done['prediction_gpu_seconds'], prior_postfit_cpu=done['postfit_cpu_seconds'],
training_gpu_once=done['training_gpu_seconds_once'],
engineering_revision='Single labelled predict1..6 continuation; immutable predict0 by hash',
same_independent_review=False)
validate_scope(c, old)
six_receipts(c)
adopted = adoption(c, tc)
entries = reuse_entries(c, tc)
env = read(O / 'jobs/0/prediction-records/environment.json')
if env['checkpoint_sha256'] != adopted['models'][0]['checkpoint_sha256']:
raise ValueError('Predict0 checkpoint differs')
R.mkdir()
save(R / 'contract.json', c)
save(R / 'preparation-receipt.json', dict(status='PREPARED_PENDING_NARROW_DELTA_REVIEW',
metadata_and_hash_cpu_seconds=time.process_time() - start, reuse_files=len(entries),
reuse_bytes=sum(Path(e['path']).stat().st_size for e in entries), original_models_exact_match=True,
six_monitor_errors_all_integer_zero=True, old_sources_unchanged=True,
original_complete_sha256=c['original_complete_sha256'], contract_sha256=sha(R / 'contract.json'),
no_model_deserialization=True, no_prediction_array_decode=True, target_values_read=False,
new_prediction=False, new_score=False, launched=False))
print(read(R / 'preparation-receipt.json'))
if __name__ == '__main__':
prepare()
+125
View File
@@ -0,0 +1,125 @@
"""Narrow recovery delta checks: synthetic files/receipts, no GPU or target arrays."""
import copy
import hashlib
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from phase2_h1_worker_v2 import save, sha
from phase3_item1_postfit_common import read, require_budget
from phase3_item1_resume_common import validate_scope, validate_stop, cumulative_cost, reuse_entries, six_receipts
from phase3_item1_resume import install_reuse
import phase3_item1_resume_guard as guard
O = Path('/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18')
class RecoveryTests(unittest.TestCase):
def setUp(self):
self.old = read(O / 'contract.json')
self.done = read(O / 'complete.json')
self.c = dict(self.old, original_root=str(O), postfit_root='/tmp/synthetic-recovery',
recovery_approval='1794980643.205679', recovery_attempt=1, resume_indices=list(range(1,7)),
idle_wait_approval='1794981566.532619', idle_wait_seconds=20,
gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
prior_postfit_gpu=self.done['prediction_gpu_seconds'], prior_postfit_cpu=self.done['postfit_cpu_seconds'],
training_gpu_once=self.done['training_gpu_seconds_once'])
def test_scope_and_actual_original_stop(self):
validate_scope(self.c, self.old)
validate_stop(self.done)
def test_no_predict0_rerun(self):
self.c['resume_indices'] = list(range(7))
with self.assertRaises(PermissionError): validate_scope(self.c, self.old)
def test_gpu1_only(self):
for uuid in ['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c', 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76']:
self.c['gpu_uuid'] = uuid
with self.assertRaises(PermissionError): validate_scope(self.c, self.old)
def test_no_budget_change(self):
self.c['postfit_gpu_cap'] = 7201
with self.assertRaises(ValueError): validate_scope(self.c, self.old)
def test_failed_job_worker_must_be_null(self):
self.done['records'][1]['guard']['worker'] = {'status':'FAILED'}
with self.assertRaises(ValueError): validate_stop(self.done)
def test_predict0_must_be_pass(self):
self.done['records'][0]['guard']['worker']['monitor_errors'] = 1
with self.assertRaises(ValueError): validate_stop(self.done)
def test_old_cost_once_not_per_job(self):
a = cumulative_cost(self.c, 100., 200.)
b = cumulative_cost(self.c, 300., 500.)
self.assertAlmostEqual(b['prediction_gpu_seconds']-a['prediction_gpu_seconds'],200)
self.assertAlmostEqual(b['postfit_cpu_seconds']-a['postfit_cpu_seconds'],300)
self.assertEqual(b['cumulative_item_gpu_seconds'],self.c['training_gpu_once']+self.c['prior_postfit_gpu']+300)
def test_old_cost_limits_remaining_reserve(self):
d = cumulative_cost(self.c,60,240)
require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],6)
with self.assertRaises(RuntimeError):
require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],7)
def test_original_six_receipts_currently_clean(self):
self.assertEqual(len(six_receipts(self.c)['rows']),6)
def test_reuse_copy_preserves_original_and_rejects_duplicate(self):
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp);original=p/'original.npz';original.write_bytes(b'opaque synthetic prediction')
root=p/'new';root.mkdir();e=dict(path=str(original),sha256=sha(original),arm='epoch25',case='sample')
adopted=install_reuse(root,[e]);self.assertEqual(original.read_bytes(),b'opaque synthetic prediction')
self.assertEqual(sha(adopted[0]['path']),e['sha256']);self.assertTrue(adopted[0]['reused_without_prediction'])
with self.assertRaises(FileExistsError):install_reuse(root,[e])
def test_reuse_bad_hash_rejected_before_write(self):
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp);original=p/'old.npz';original.write_bytes(b'a');root=p/'new';root.mkdir()
with self.assertRaises(ValueError):
install_reuse(root,[dict(path=str(original),sha256='0'*64,arm='epoch25',case='sample')])
self.assertEqual(list(root.iterdir()),[])
def guarded(self, mode):
timer=[0.];calls=[0]
def sleep(seconds): timer[0]+=seconds
def probe(uuid):
calls[0]+=1
if mode=='timeout' or (mode=='settles' and calls[0]<=2):return 256*2**20,90
return 0,0
class Thread:
def __init__(self,*a,**k):pass
def start(self):pass
def join(self,**k):pass
def is_alive(self):return False
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);out=root/'jobs/1';out.mkdir(parents=True)
c=dict(gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',idle_wait_seconds=20,jobs=[{},dict(expected_parts=1)])
def supervisor(*a,**k):
r=out/'prediction-records';r.mkdir();save(r/'result.json',dict(status='COMPLETE',parts=1,peak_allocated_bytes=1024,peak_reserved_bytes=1024))
return dict(status='PASS',cpu_seconds=0,monitor_errors=1 if mode=='monitor_error' else 0)
with patch.object(guard.time,'monotonic',side_effect=lambda:timer[0]),patch.object(guard.time,'sleep',side_effect=sleep),patch.object(guard,'device_usage',side_effect=probe),patch.object(guard.threading,'Thread',Thread),patch.object(guard,'supervise',side_effect=supervisor) as sup:
result=guard.run_prediction(root,c,{},1,0,300)
return result,sup.call_count,read(out/'recovery-idle-wait.json')
def test_busy_then_idle_counts_wait_and_three_polls(self):
r,n,w=self.guarded('settles');self.assertEqual(r['status'],'PASS');self.assertEqual(n,1)
self.assertEqual(r['gpu_seconds_charged'],65.)
self.assertEqual(len(w['samples']),3)
def test_busy_timeout_stops_without_worker(self):
r,n,w=self.guarded('timeout');self.assertEqual(r['status'],'STOPPED');self.assertEqual(n,0)
self.assertEqual(r['gpu_seconds_charged'],80.)
self.assertTrue(any('BOUNDED_WAIT' in x for x in r['reasons']))
def test_supervisor_monitor_error_still_rejected(self):
r,n,w=self.guarded('monitor_error');self.assertEqual(n,1);self.assertIn('WORKER_MONITOR_ERRORS',r['reasons'])
def test_wait_limit_cannot_be_relaxed(self):
self.c['idle_wait_seconds']=21
with self.assertRaises(PermissionError):validate_scope(self.c,self.old)
if __name__ == '__main__':unittest.main()