Prepare one labelled item1 postfit recovery with bounded idle wait
This commit is contained in:
1 parent
4f646232f4
commit
581208098f
24 files changed
+83789
No files matched your search
@@ -0,0 +1,141 @@
|
||||
"""Single labelled resume: reuse0, predict1..6, freeze all, unchanged score."""
|
||||
import fcntl
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
|
||||
import psutil
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from phase2_home_freeze_io import freeze, verify
|
||||
from phase3_item1_postfit_common import read, manifest_identity, require_budget
|
||||
from phase3_item1_resume_common import config, adoption, reuse_entries, cumulative_cost
|
||||
from phase3_item1_resume_guard import run_prediction
|
||||
from r4bc_guard import supervise
|
||||
|
||||
|
||||
def install_reuse(root, entries):
|
||||
result = []
|
||||
for e in entries:
|
||||
original = checked(e['path'], e['sha256'])
|
||||
target = root / 'predictions' / e['arm'] / (e['case'] + '.npz')
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
if target.exists():
|
||||
raise FileExistsError('No duplicate prediction adoption')
|
||||
shutil.copyfile(original, target)
|
||||
checked(target, e['sha256'])
|
||||
result.append(dict(e, path=str(target), reuse_original_path=str(original),
|
||||
reused_without_prediction=True))
|
||||
out = root / 'jobs/0/prediction-records'
|
||||
out.mkdir(parents=True, exist_ok=False)
|
||||
save(out / 'reuse.json', dict(status='REUSED_BY_HASH_NO_WORKER', entries=result, targets_read=False))
|
||||
return result
|
||||
|
||||
|
||||
def run(root):
|
||||
gpu_clock = time.monotonic()
|
||||
start = time.process_time()
|
||||
u0 = resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
c, tc, pilot = config(root)
|
||||
validate_review(root)
|
||||
adopted = adoption(c, tc)
|
||||
lock = (root / 'run.lock').open('a')
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if (root / 'started.json').exists():
|
||||
raise FileExistsError('One labelled recovery, no repeat launch')
|
||||
for p in psutil.process_iter(['pid', 'cmdline']):
|
||||
if p.pid != os.getpid() and any(Path(a).name in {'phase3_item1_postfit.py', 'phase3_item1_resume.py',
|
||||
'phase3_item1_postfit_worker.py'} for a in p.info['cmdline'] or []):
|
||||
raise RuntimeError('Existing original or resumed postfit')
|
||||
gpu_lock = Path('/tmp/meltmind-item1-resume-' + c['gpu_uuid'] + '.lock').open('a')
|
||||
fcntl.flock(gpu_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
os.sched_setaffinity(0, c['coordinator_cpus'])
|
||||
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
|
||||
new_gpu = 0.
|
||||
charged_children = 0.
|
||||
records = []
|
||||
prediction_start = None
|
||||
|
||||
def costs():
|
||||
u = resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
children = u.ru_utime + u.ru_stime - u0.ru_utime - u0.ru_stime
|
||||
new_cpu = 180 + time.process_time() - start + max(charged_children, children)
|
||||
return cumulative_cost(c, new_gpu, new_cpu)
|
||||
|
||||
def budget(remaining):
|
||||
d = costs()
|
||||
require_budget(c['training_gpu_once'], d['prediction_gpu_seconds'], d['postfit_cpu_seconds'], remaining)
|
||||
return d
|
||||
|
||||
save(root / 'started.json', dict(pid=os.getpid(), unix=time.time(), recovery_attempt=1,
|
||||
original_root=c['original_root'], prior_postfit_gpu=c['prior_postfit_gpu'],
|
||||
prior_postfit_cpu=c['prior_postfit_cpu'], training_gpu_seconds_once=c['training_gpu_once'],
|
||||
cpu_setup_allowance=180, review_sha256=sha(root / 'review.json'), indices=c['resume_indices'],
|
||||
gpu_uuid=c['gpu_uuid'], new_fits=0, new_decodes=0))
|
||||
try:
|
||||
budget(6)
|
||||
entries = install_reuse(root, reuse_entries(c, tc))
|
||||
save(root / 'models-adopted.json', adopted)
|
||||
freeze(root, 'models', [root / 'contract.json', root / 'models-adopted.json',
|
||||
Path(c['original_root']) / 'models-adopted.json',
|
||||
*[Path(m['checkpoint']) for m in adopted['models']],
|
||||
*[Path(m['standardization']) for m in adopted['models']]])
|
||||
for index in c['resume_indices']:
|
||||
new_gpu = max(new_gpu, 60 + time.monotonic() - gpu_clock)
|
||||
budget(7 - index)
|
||||
(root / 'jobs' / str(index)).mkdir(parents=True, exist_ok=False)
|
||||
prediction_start = time.monotonic()
|
||||
g = run_prediction(root, c, pilot, index, c['training_gpu_once'] + costs()['prediction_gpu_seconds'], 300)
|
||||
cost = max(60 + time.monotonic() - prediction_start, g['gpu_seconds_charged'])
|
||||
prediction_start = None
|
||||
new_gpu += cost
|
||||
charged_children += g['worker']['cpu_seconds'] if g['worker'] else 0.
|
||||
records.append(dict(mode='predict', index=index, charged_gpu_seconds=cost, guard=g))
|
||||
d = costs()
|
||||
save(root / 'resource-progress.json', dict(records=records, **d, original_complete_sha256=c['original_complete_sha256']))
|
||||
if g['status'] != 'PASS' or cost > 900 or d['prediction_gpu_seconds'] > 7200 or d['cumulative_item_gpu_seconds'] > 120000:
|
||||
raise RuntimeError('Recovery prediction/resource guard failure')
|
||||
entries.extend(read(root / 'jobs' / str(index) / 'prediction-records/manifest.json')['entries'])
|
||||
manifest_identity(entries, c, tc, pilot)
|
||||
save(root / 'predictions-manifest.json', dict(entries=entries, predict0_reused_by_hash=686,
|
||||
S1_test_predictions_reused=121, new_fits=0, targets_read=False))
|
||||
freeze(root, 'predictions', [root / 'contract.json', root / 'models-adopted.json',
|
||||
root / 'predictions-manifest.json', root / 'resource-progress.json',
|
||||
root / 'jobs/0/prediction-records/reuse.json', *[Path(p) for p in c['code_sha256']]])
|
||||
new_gpu = max(new_gpu, 60 + time.monotonic() - gpu_clock)
|
||||
budget(0)
|
||||
validate_review(root)
|
||||
verify(root, 'models')
|
||||
verify(root, 'predictions')
|
||||
g = supervise([sys.executable, 'tools/phase3_item1_postfit_worker.py', 'score', str(root)],
|
||||
root / 'score-guard', dict(cpu_seconds=900, wall_seconds=1200, rss_bytes=16*2**30, disk_bytes=20*2**30))
|
||||
charged_children += g['cpu_seconds']
|
||||
records.append(dict(mode='score', guard=g))
|
||||
if g['status'] != 'PASS' or costs()['postfit_cpu_seconds'] + 120 > 3600:
|
||||
raise RuntimeError('Score/resource failure')
|
||||
save(root / 'complete.json', dict(status='COMPLETE', records=records, **costs(),
|
||||
original_complete_sha256=c['original_complete_sha256'], predict0_reused=True,
|
||||
resumed_indices=c['resume_indices'], item_gpu_cap=120000, new_fits=0, new_decodes=0,
|
||||
future_phase3_started=False, no_retry=True))
|
||||
freeze(root, 'terminal', [root / 'contract.json', root / 'complete.json', root / 'score/result.json', root / 'review.json'])
|
||||
final = costs()
|
||||
within = final['postfit_cpu_seconds'] <= 3600 and final['prediction_gpu_seconds'] <= 7200 and final['cumulative_item_gpu_seconds'] <= 120000
|
||||
save(root / 'post-terminal-cost.json', dict(**final, within_cap=within))
|
||||
if not within:
|
||||
raise RuntimeError('Final cumulative resource cap')
|
||||
except BaseException as exc:
|
||||
if prediction_start is not None:
|
||||
new_gpu += 60 + time.monotonic() - prediction_start
|
||||
if (root / 'complete.json').exists():
|
||||
save(root / 'pre-failure-complete.json', read(root / 'complete.json'))
|
||||
save(root / 'complete.json', dict(status='STOPPED', reason=type(exc).__name__ + ': ' + str(exc),
|
||||
records=records, **costs(), original_complete_sha256=c['original_complete_sha256'],
|
||||
new_fits=0, no_retry=True, recovery_attempt=1))
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(Path(sys.argv[1]))
|
||||
@@ -0,0 +1,120 @@
|
||||
"""One labelled infrastructure continuation; original scientific modules unchanged."""
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
from phase2_h1_worker_v2 import checked, sha
|
||||
from phase2_home_freeze_io import verify
|
||||
from phase3_item1_postfit_common import config as original_config, read, terminal_inputs
|
||||
|
||||
ALLOWED_GPUS = {'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36'}
|
||||
|
||||
|
||||
def validate_scope(c, old):
|
||||
if c['recovery_approval'] != '1794980643.205679' or c['resume_indices'] != list(range(1, 7)):
|
||||
raise PermissionError('Exactly one recovery for jobs1 through6')
|
||||
if (c['gpu_uuid'] not in ALLOWED_GPUS or c['recovery_attempt'] != 1
|
||||
or c['idle_wait_seconds'] != 20 or c['idle_wait_approval'] != '1794981566.532619'):
|
||||
raise PermissionError('Reserved Track A GPU or extra recovery')
|
||||
for key in ('jobs', 'rows', 'training_root', 'training_contract', 'comparison_contract_sha256',
|
||||
'postfit_gpu_cap', 'postfit_cpu_cap', 'item_gpu_cap', 'new_fits', 'new_decodes'):
|
||||
if c[key] != old[key]:
|
||||
raise ValueError('Original science/budget differs: ' + key)
|
||||
if c['postfit_root'] == old['postfit_root']:
|
||||
raise ValueError('Original stopped root must remain untouched')
|
||||
for p, h in old['code_sha256'].items():
|
||||
if c['code_sha256'].get(p) != h:
|
||||
raise ValueError('Original pinned source changed')
|
||||
|
||||
|
||||
def validate_stop(done):
|
||||
records = done['records']
|
||||
if done['status'] != 'STOPPED' or len(records) != 2 or [x['index'] for x in records] != [0, 1]:
|
||||
raise ValueError('Unexpected original stop scope')
|
||||
first, failed = [x['guard'] for x in records]
|
||||
if first['status'] != 'PASS' or first['monitor_errors'] or not first['worker'] or first['worker']['status'] != 'PASS' or first['worker'].get('monitor_errors') != 0:
|
||||
raise ValueError('Predict0 was not a clean PASS')
|
||||
if failed['status'] != 'STOPPED' or failed['worker'] is not None or 'RuntimeError: SELECTED_GPU_BUSY_NO_RETRY' not in failed['reasons']:
|
||||
raise ValueError('Not the authorized zero-worker infrastructure stop')
|
||||
for key in ('prediction_gpu_seconds', 'postfit_cpu_seconds', 'training_gpu_seconds_once'):
|
||||
if not math.isfinite(done[key]) or done[key] < 0:
|
||||
raise ValueError('Invalid original ledger')
|
||||
|
||||
|
||||
def six_receipts(c):
|
||||
source = Path(c['original_root']) / 'tron-six-training-receipts-check.json'
|
||||
check = read(source)
|
||||
if check['status'] != 'PASS' or [r['job'] for r in check['rows']] != list(range(6)):
|
||||
raise ValueError('Original six-receipt check missing')
|
||||
for row in check['rows']:
|
||||
p = Path(c['training_root']) / 'jobs' / str(row['job']) / 'guards/train.receipt.json'
|
||||
if str(p) != row['path']:
|
||||
raise ValueError('Original training receipt path mismatch')
|
||||
receipt = read(checked(p, row['sha256']))
|
||||
if type(receipt.get('monitor_errors')) is not int or receipt['monitor_errors'] != 0 or receipt['status'] != 'PASS' or receipt['returncode'] != 0:
|
||||
raise ValueError('Training monitoring receipt not clean')
|
||||
guard = read(p.parent.parent / 'complete.json')
|
||||
if guard['worker'] != receipt:
|
||||
raise ValueError('Embedded training receipt differs')
|
||||
return check
|
||||
|
||||
|
||||
def config(root):
|
||||
c, tc, pilot = original_config(root)
|
||||
original = Path(c['original_root'])
|
||||
old, _, _ = original_config(original)
|
||||
validate_scope(c, old)
|
||||
done = read(checked(original / 'complete.json', c['original_complete_sha256']))
|
||||
validate_stop(done)
|
||||
if any((original / n).exists() for n in ('score', 'predictions-manifest.json')):
|
||||
raise ValueError('Original scoring or full prediction barrier already exists')
|
||||
for k, name in [('prior_postfit_gpu', 'prediction_gpu_seconds'), ('prior_postfit_cpu', 'postfit_cpu_seconds'),
|
||||
('training_gpu_once', 'training_gpu_seconds_once')]:
|
||||
if c[k] != done[name]:
|
||||
raise ValueError('Original cost must be retained exactly once')
|
||||
six_receipts(c)
|
||||
return c, tc, pilot
|
||||
|
||||
|
||||
def adoption(c, tc):
|
||||
current = terminal_inputs(c, tc)
|
||||
original = Path(c['original_root'])
|
||||
verify(original, 'models')
|
||||
old = read(original / 'models-adopted.json')
|
||||
if old != current or current['training_gpu_seconds_once'] != c['training_gpu_once']:
|
||||
raise ValueError('Predict0 and resumed jobs must use identical frozen models')
|
||||
return current
|
||||
|
||||
|
||||
def reuse_entries(c, tc, *, verify_bytes=True):
|
||||
original = Path(c['original_root'])
|
||||
manifest = read(checked(original / 'jobs/0/prediction-records/manifest.json', c['predict0_manifest_sha256']))
|
||||
result = read(original / 'jobs/0/prediction-records/result.json')
|
||||
if result['status'] != 'COMPLETE' or result['targets_read'] or result['parts'] != c['jobs'][0]['expected_parts']:
|
||||
raise ValueError('Predict0 result mismatch')
|
||||
entries = manifest['entries']
|
||||
f = tc['pools']['25']
|
||||
domains = {'learning': 'own_learning', 'validation': 'common_validation', 'test': 'common_test'}
|
||||
expected = {(arm, case): domain for role, domain in domains.items() for case in f[role]
|
||||
for arm in ('epoch25', 'c25F')}
|
||||
if len(entries) != len(expected) or {(e['arm'], e['case']) for e in entries} != set(expected):
|
||||
raise ValueError('Complete predict0 model and clock coverage required')
|
||||
if manifest['targets_read'] or result['entries'] != len(entries):
|
||||
raise ValueError('Predict0 targets/entry count')
|
||||
for e in entries:
|
||||
p = original / 'predictions' / e['arm'] / (e['case'] + '.npz')
|
||||
if (e['path'] != str(p) or e['pool'] != 25 or e['domain'] != expected[(e['arm'], e['case'])]
|
||||
or e['component_id'] != c['rows'][e['case']]['component_id']
|
||||
or e['origin'] != 'ITEM1_FROZEN_ENDPOINT_OR_CLOCK'):
|
||||
raise ValueError('Predict0 original identity')
|
||||
if verify_bytes:
|
||||
checked(p, e['sha256'])
|
||||
return entries
|
||||
|
||||
|
||||
def cumulative_cost(c, new_gpu, new_cpu):
|
||||
if any(not math.isfinite(v) or v < 0 for v in [new_gpu, new_cpu]):
|
||||
raise ValueError('Invalid new cost ledger')
|
||||
gpu = c['prior_postfit_gpu'] + new_gpu
|
||||
return dict(prediction_gpu_seconds=gpu, postfit_cpu_seconds=c['prior_postfit_cpu'] + new_cpu,
|
||||
training_gpu_seconds_once=c['training_gpu_once'],
|
||||
cumulative_item_gpu_seconds=c['training_gpu_once'] + gpu)
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Recovery-only guard: bounded pre-worker idle wait inside original900s cap."""
|
||||
import json,os,signal,sys,threading,time
|
||||
import psutil
|
||||
from phase2_h1_worker_v2 import save
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
from s2_precision_continuation_guard import device_usage
|
||||
from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage
|
||||
|
||||
def final_reasons(reasons,receipt,result,tracker,gpu_seconds,cpu_seconds,cpu_cap,expected_parts):
|
||||
out=list(reasons)
|
||||
def add(s):
|
||||
if s not in out:out.append(s)
|
||||
if not receipt or receipt['status']!='PASS':add('WORKER_NOT_PASS')
|
||||
if receipt and receipt.get('monitor_errors'):add('WORKER_MONITOR_ERRORS')
|
||||
if not result or result.get('status')!='COMPLETE' or result.get('parts')!=expected_parts:add('INCOMPLETE_FOLD')
|
||||
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):add('MISSING_PEAKS')
|
||||
if any(v>15*2**30 for v in tracker.maxima.values()):add('RECORDED_VRAM_LIMIT')
|
||||
if gpu_seconds>900 or cpu_seconds>cpu_cap:add('FINAL_RESOURCE_LIMIT')
|
||||
return out
|
||||
|
||||
def run_prediction(queue,c,pilot,fold,prior_gpu,cpu_cap):
|
||||
root=queue/'jobs'/str(fold);gpu_uuid=c['gpu_uuid'];expected_parts=c['jobs'][fold]['expected_parts']
|
||||
start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None
|
||||
def elapsed():return 60+time.monotonic()-start
|
||||
def reason(x):
|
||||
if x not in reasons:reasons.append(x)
|
||||
command=[sys.executable,'tools/phase3_item1_postfit_worker.py','predict',str(queue),str(fold)]
|
||||
def kill():
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:stop_group(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
def scan():
|
||||
if tracker.scan(root/'prediction-records'):reason('RECORDED_VRAM_LIMIT')
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
try:
|
||||
scan();mem,util=device_usage(gpu_uuid);_,rss,_=tree_usage(os.getpid())
|
||||
if mem>15*2**30:reason('DEVICE_VRAM_LIMIT')
|
||||
if rss>16*2**30:reason('TREE_RSS_LIMIT')
|
||||
if disk_usage(queue)>20*2**30:reason('DISK_LIMIT')
|
||||
if elapsed()>=895:reason('PREDICTION_TIME_LIMIT')
|
||||
for x in requested:reason(x)
|
||||
save(root/'prediction-live.json',dict(gpu_seconds_charged=elapsed(),device_bytes=mem,device_utilization=util,rss_bytes=rss,peaks=tracker.maxima,reasons=reasons,unix=time.time()))
|
||||
except Exception as exc:reason('MONITOR_ERROR');errors.append(str(exc))
|
||||
if reasons:kill()
|
||||
done.wait(.5)
|
||||
old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGTERM,signal.SIGINT]};thread=None
|
||||
try:
|
||||
wait_start=time.monotonic();waiting=[]
|
||||
while True:
|
||||
mem,util=device_usage(gpu_uuid);waiting.append(dict(unix=time.time(),bytes=mem,utilization=util))
|
||||
save(root/'recovery-idle-wait.json',dict(samples=waiting,limit_seconds=c['idle_wait_seconds'],worker_started=False))
|
||||
if requested:raise RuntimeError('Signal received during idle wait')
|
||||
duration=time.monotonic()-wait_start
|
||||
if duration>c['idle_wait_seconds']:raise RuntimeError('SELECTED_GPU_BUSY_AFTER_BOUNDED_WAIT_NO_RETRY')
|
||||
if mem<=128*2**20 and util==0:break
|
||||
if duration>=c['idle_wait_seconds']:raise RuntimeError('SELECTED_GPU_BUSY_AFTER_BOUNDED_WAIT_NO_RETRY')
|
||||
time.sleep(min(.5,max(0.,c['idle_wait_seconds']-(time.monotonic()-wait_start))))
|
||||
for i in range(3):
|
||||
mem,util=device_usage(gpu_uuid);samples.append(dict(unix=time.time(),bytes=mem,utilization=util))
|
||||
if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY_NO_RETRY')
|
||||
if i<2:time.sleep(2)
|
||||
save(root/'prediction-idle-polls.json',samples)
|
||||
if requested:raise RuntimeError('Signal received during setup')
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
remaining=900-elapsed()-10
|
||||
if remaining<=0:raise RuntimeError('Setup exhausted prediction budget')
|
||||
receipt=supervise(command,root/'prediction-guard','predict',dict(total_cpu_seconds=cpu_cap-60,job_cpu_seconds=cpu_cap-60,job_wall_seconds=remaining,tree_rss_bytes=16*2**30,disk_bytes=20*2**30,max_processes=1),poll=.2,gpu_uuid=gpu_uuid)
|
||||
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill()
|
||||
finally:
|
||||
done.set()
|
||||
if thread:thread.join(timeout=4)
|
||||
if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT')
|
||||
try:scan()
|
||||
except Exception as exc:reason('FINAL_SCAN_ERROR');errors.append(str(exc))
|
||||
for s,h in old.items():signal.signal(s,h)
|
||||
result=None;p=root/'prediction-records/result.json'
|
||||
try:
|
||||
if p.exists():result=json.loads(p.read_text())
|
||||
except Exception as exc:reason('FINAL_RESULT_READ_ERROR');errors.append(str(exc))
|
||||
totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu
|
||||
duration=elapsed();reasons=final_reasons(reasons,receipt,result,tracker,duration,totalcpu,cpu_cap,expected_parts)
|
||||
out=dict(status='PASS' if not reasons else 'STOPPED',reasons=reasons,monitor_errors=errors,worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=duration,cpu_seconds_charged=totalcpu,cumulative_item1_gpu_seconds=prior_gpu+duration,no_retry=True)
|
||||
save(root/'prediction-guard.json',out);return out
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Metadata and opaque-hash preparation; no new prediction or target access."""
|
||||
from pathlib import Path
|
||||
import time
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_home_freeze_io import verify
|
||||
from phase3_item1_postfit_common import config as old_config, read
|
||||
from phase3_item1_resume_common import validate_scope, validate_stop, six_receipts, adoption, reuse_entries
|
||||
from track_a_h2_prepare import local_code_closure
|
||||
|
||||
B = Path(__file__).resolve().parents[1]
|
||||
O = Path('/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18')
|
||||
R = O.parent / 'phase3-item1-postfit-recovery-2026-11-18'
|
||||
|
||||
|
||||
def prepare():
|
||||
start = time.process_time()
|
||||
if R.exists():
|
||||
raise FileExistsError('Existing recovery entry: inspect before any further action')
|
||||
old, tc, pilot = old_config(O)
|
||||
validate_review(O)
|
||||
done = read(O / 'complete.json')
|
||||
validate_stop(done)
|
||||
sources = dict(old['source_sha256'])
|
||||
for name in ['contract.json', 'complete.json', 'started.json', 'resource-progress.json',
|
||||
'tron-six-training-receipts-check.json', 'models-adopted.json', 'models-git-freeze.json',
|
||||
'review.json', 'implementation-git-freeze.json',
|
||||
'jobs/0/prediction-guard.json', 'jobs/0/prediction-records/manifest.json',
|
||||
'jobs/0/prediction-records/result.json', 'jobs/0/prediction-records/environment.json']:
|
||||
sources[str(O / name)] = sha(O / name)
|
||||
codes = dict(old['code_sha256'])
|
||||
additional = local_code_closure([B / 'tools' / n for n in ['phase3_item1_resume.py',
|
||||
'phase3_item1_resume_common.py', 'phase3_item1_resume_guard.py', 'prepare_phase3_item1_resume.py',
|
||||
'test_phase3_item1_resume.py']])
|
||||
for p, h in additional.items():
|
||||
if p in codes and codes[p] != h:
|
||||
raise ValueError('Old pinned code changed')
|
||||
codes[p] = h
|
||||
c = dict(old, postfit_root=str(R), code_sha256=codes, source_sha256=sources,
|
||||
gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
|
||||
original_root=str(O), original_complete_sha256=sha(O / 'complete.json'),
|
||||
recovery_approval='1794980643.205679', recovery_attempt=1, resume_indices=list(range(1, 7)),
|
||||
idle_wait_approval='1794981566.532619', idle_wait_seconds=20,
|
||||
predict0_manifest_sha256=sha(O / 'jobs/0/prediction-records/manifest.json'),
|
||||
prior_postfit_gpu=done['prediction_gpu_seconds'], prior_postfit_cpu=done['postfit_cpu_seconds'],
|
||||
training_gpu_once=done['training_gpu_seconds_once'],
|
||||
engineering_revision='Single labelled predict1..6 continuation; immutable predict0 by hash',
|
||||
same_independent_review=False)
|
||||
validate_scope(c, old)
|
||||
six_receipts(c)
|
||||
adopted = adoption(c, tc)
|
||||
entries = reuse_entries(c, tc)
|
||||
env = read(O / 'jobs/0/prediction-records/environment.json')
|
||||
if env['checkpoint_sha256'] != adopted['models'][0]['checkpoint_sha256']:
|
||||
raise ValueError('Predict0 checkpoint differs')
|
||||
R.mkdir()
|
||||
save(R / 'contract.json', c)
|
||||
save(R / 'preparation-receipt.json', dict(status='PREPARED_PENDING_NARROW_DELTA_REVIEW',
|
||||
metadata_and_hash_cpu_seconds=time.process_time() - start, reuse_files=len(entries),
|
||||
reuse_bytes=sum(Path(e['path']).stat().st_size for e in entries), original_models_exact_match=True,
|
||||
six_monitor_errors_all_integer_zero=True, old_sources_unchanged=True,
|
||||
original_complete_sha256=c['original_complete_sha256'], contract_sha256=sha(R / 'contract.json'),
|
||||
no_model_deserialization=True, no_prediction_array_decode=True, target_values_read=False,
|
||||
new_prediction=False, new_score=False, launched=False))
|
||||
print(read(R / 'preparation-receipt.json'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
prepare()
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Narrow recovery delta checks: synthetic files/receipts, no GPU or target arrays."""
|
||||
import copy
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from phase2_h1_worker_v2 import save, sha
|
||||
from phase3_item1_postfit_common import read, require_budget
|
||||
from phase3_item1_resume_common import validate_scope, validate_stop, cumulative_cost, reuse_entries, six_receipts
|
||||
from phase3_item1_resume import install_reuse
|
||||
import phase3_item1_resume_guard as guard
|
||||
|
||||
O = Path('/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18')
|
||||
|
||||
|
||||
class RecoveryTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.old = read(O / 'contract.json')
|
||||
self.done = read(O / 'complete.json')
|
||||
self.c = dict(self.old, original_root=str(O), postfit_root='/tmp/synthetic-recovery',
|
||||
recovery_approval='1794980643.205679', recovery_attempt=1, resume_indices=list(range(1,7)),
|
||||
idle_wait_approval='1794981566.532619', idle_wait_seconds=20,
|
||||
gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
|
||||
prior_postfit_gpu=self.done['prediction_gpu_seconds'], prior_postfit_cpu=self.done['postfit_cpu_seconds'],
|
||||
training_gpu_once=self.done['training_gpu_seconds_once'])
|
||||
|
||||
def test_scope_and_actual_original_stop(self):
|
||||
validate_scope(self.c, self.old)
|
||||
validate_stop(self.done)
|
||||
|
||||
def test_no_predict0_rerun(self):
|
||||
self.c['resume_indices'] = list(range(7))
|
||||
with self.assertRaises(PermissionError): validate_scope(self.c, self.old)
|
||||
|
||||
def test_gpu1_only(self):
|
||||
for uuid in ['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c', 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76']:
|
||||
self.c['gpu_uuid'] = uuid
|
||||
with self.assertRaises(PermissionError): validate_scope(self.c, self.old)
|
||||
|
||||
def test_no_budget_change(self):
|
||||
self.c['postfit_gpu_cap'] = 7201
|
||||
with self.assertRaises(ValueError): validate_scope(self.c, self.old)
|
||||
|
||||
def test_failed_job_worker_must_be_null(self):
|
||||
self.done['records'][1]['guard']['worker'] = {'status':'FAILED'}
|
||||
with self.assertRaises(ValueError): validate_stop(self.done)
|
||||
|
||||
def test_predict0_must_be_pass(self):
|
||||
self.done['records'][0]['guard']['worker']['monitor_errors'] = 1
|
||||
with self.assertRaises(ValueError): validate_stop(self.done)
|
||||
|
||||
def test_old_cost_once_not_per_job(self):
|
||||
a = cumulative_cost(self.c, 100., 200.)
|
||||
b = cumulative_cost(self.c, 300., 500.)
|
||||
self.assertAlmostEqual(b['prediction_gpu_seconds']-a['prediction_gpu_seconds'],200)
|
||||
self.assertAlmostEqual(b['postfit_cpu_seconds']-a['postfit_cpu_seconds'],300)
|
||||
self.assertEqual(b['cumulative_item_gpu_seconds'],self.c['training_gpu_once']+self.c['prior_postfit_gpu']+300)
|
||||
|
||||
def test_old_cost_limits_remaining_reserve(self):
|
||||
d = cumulative_cost(self.c,60,240)
|
||||
require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],6)
|
||||
with self.assertRaises(RuntimeError):
|
||||
require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],7)
|
||||
|
||||
def test_original_six_receipts_currently_clean(self):
|
||||
self.assertEqual(len(six_receipts(self.c)['rows']),6)
|
||||
|
||||
def test_reuse_copy_preserves_original_and_rejects_duplicate(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
p=Path(tmp);original=p/'original.npz';original.write_bytes(b'opaque synthetic prediction')
|
||||
root=p/'new';root.mkdir();e=dict(path=str(original),sha256=sha(original),arm='epoch25',case='sample')
|
||||
adopted=install_reuse(root,[e]);self.assertEqual(original.read_bytes(),b'opaque synthetic prediction')
|
||||
self.assertEqual(sha(adopted[0]['path']),e['sha256']);self.assertTrue(adopted[0]['reused_without_prediction'])
|
||||
with self.assertRaises(FileExistsError):install_reuse(root,[e])
|
||||
|
||||
def test_reuse_bad_hash_rejected_before_write(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
p=Path(tmp);original=p/'old.npz';original.write_bytes(b'a');root=p/'new';root.mkdir()
|
||||
with self.assertRaises(ValueError):
|
||||
install_reuse(root,[dict(path=str(original),sha256='0'*64,arm='epoch25',case='sample')])
|
||||
self.assertEqual(list(root.iterdir()),[])
|
||||
|
||||
def guarded(self, mode):
|
||||
timer=[0.];calls=[0]
|
||||
def sleep(seconds): timer[0]+=seconds
|
||||
def probe(uuid):
|
||||
calls[0]+=1
|
||||
if mode=='timeout' or (mode=='settles' and calls[0]<=2):return 256*2**20,90
|
||||
return 0,0
|
||||
class Thread:
|
||||
def __init__(self,*a,**k):pass
|
||||
def start(self):pass
|
||||
def join(self,**k):pass
|
||||
def is_alive(self):return False
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);out=root/'jobs/1';out.mkdir(parents=True)
|
||||
c=dict(gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',idle_wait_seconds=20,jobs=[{},dict(expected_parts=1)])
|
||||
def supervisor(*a,**k):
|
||||
r=out/'prediction-records';r.mkdir();save(r/'result.json',dict(status='COMPLETE',parts=1,peak_allocated_bytes=1024,peak_reserved_bytes=1024))
|
||||
return dict(status='PASS',cpu_seconds=0,monitor_errors=1 if mode=='monitor_error' else 0)
|
||||
with patch.object(guard.time,'monotonic',side_effect=lambda:timer[0]),patch.object(guard.time,'sleep',side_effect=sleep),patch.object(guard,'device_usage',side_effect=probe),patch.object(guard.threading,'Thread',Thread),patch.object(guard,'supervise',side_effect=supervisor) as sup:
|
||||
result=guard.run_prediction(root,c,{},1,0,300)
|
||||
return result,sup.call_count,read(out/'recovery-idle-wait.json')
|
||||
|
||||
def test_busy_then_idle_counts_wait_and_three_polls(self):
|
||||
r,n,w=self.guarded('settles');self.assertEqual(r['status'],'PASS');self.assertEqual(n,1)
|
||||
self.assertEqual(r['gpu_seconds_charged'],65.)
|
||||
self.assertEqual(len(w['samples']),3)
|
||||
|
||||
def test_busy_timeout_stops_without_worker(self):
|
||||
r,n,w=self.guarded('timeout');self.assertEqual(r['status'],'STOPPED');self.assertEqual(n,0)
|
||||
self.assertEqual(r['gpu_seconds_charged'],80.)
|
||||
self.assertTrue(any('BOUNDED_WAIT' in x for x in r['reasons']))
|
||||
|
||||
def test_supervisor_monitor_error_still_rejected(self):
|
||||
r,n,w=self.guarded('monitor_error');self.assertEqual(n,1);self.assertIn('WORKER_MONITOR_ERRORS',r['reasons'])
|
||||
|
||||
def test_wait_limit_cannot_be_relaxed(self):
|
||||
self.c['idle_wait_seconds']=21
|
||||
with self.assertRaises(PermissionError):validate_scope(self.c,self.old)
|
||||
|
||||
|
||||
if __name__ == '__main__':unittest.main()
|
||||
Reference in new issue
Block a user