Files
MeltMind/tools/phase3_item1_resume.py
T

142 lines
7.4 KiB
Python

"""Single labelled resume: reuse0, predict1..6, freeze all, unchanged score."""
import fcntl
import os
from pathlib import Path
import resource
import shutil
import sys
import time
import psutil
from phase2_gate_outcome_review import validate_review
from phase2_h1_worker_v2 import checked, save, sha
from phase2_home_freeze_io import freeze, verify
from phase3_item1_postfit_common import read, manifest_identity, require_budget
from phase3_item1_resume_common import config, adoption, reuse_entries, cumulative_cost
from phase3_item1_resume_guard import run_prediction
from r4bc_guard import supervise
def install_reuse(root, entries):
result = []
for e in entries:
original = checked(e['path'], e['sha256'])
target = root / 'predictions' / e['arm'] / (e['case'] + '.npz')
target.parent.mkdir(parents=True, exist_ok=True)
if target.exists():
raise FileExistsError('No duplicate prediction adoption')
shutil.copyfile(original, target)
checked(target, e['sha256'])
result.append(dict(e, path=str(target), reuse_original_path=str(original),
reused_without_prediction=True))
out = root / 'jobs/0/prediction-records'
out.mkdir(parents=True, exist_ok=False)
save(out / 'reuse.json', dict(status='REUSED_BY_HASH_NO_WORKER', entries=result, targets_read=False))
return result
def run(root):
gpu_clock = time.monotonic()
start = time.process_time()
u0 = resource.getrusage(resource.RUSAGE_CHILDREN)
c, tc, pilot = config(root)
validate_review(root)
adopted = adoption(c, tc)
lock = (root / 'run.lock').open('a')
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (root / 'started.json').exists():
raise FileExistsError('One labelled recovery, no repeat launch')
for p in psutil.process_iter(['pid', 'cmdline']):
if p.pid != os.getpid() and any(Path(a).name in {'phase3_item1_postfit.py', 'phase3_item1_resume.py',
'phase3_item1_postfit_worker.py'} for a in p.info['cmdline'] or []):
raise RuntimeError('Existing original or resumed postfit')
gpu_lock = Path('/tmp/meltmind-item1-resume-' + c['gpu_uuid'] + '.lock').open('a')
fcntl.flock(gpu_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
os.sched_setaffinity(0, c['coordinator_cpus'])
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
new_gpu = 0.
charged_children = 0.
records = []
prediction_start = None
def costs():
u = resource.getrusage(resource.RUSAGE_CHILDREN)
children = u.ru_utime + u.ru_stime - u0.ru_utime - u0.ru_stime
new_cpu = 180 + time.process_time() - start + max(charged_children, children)
return cumulative_cost(c, new_gpu, new_cpu)
def budget(remaining):
d = costs()
require_budget(c['training_gpu_once'], d['prediction_gpu_seconds'], d['postfit_cpu_seconds'], remaining)
return d
save(root / 'started.json', dict(pid=os.getpid(), unix=time.time(), recovery_attempt=1,
original_root=c['original_root'], prior_postfit_gpu=c['prior_postfit_gpu'],
prior_postfit_cpu=c['prior_postfit_cpu'], training_gpu_seconds_once=c['training_gpu_once'],
cpu_setup_allowance=180, review_sha256=sha(root / 'review.json'), indices=c['resume_indices'],
gpu_uuid=c['gpu_uuid'], new_fits=0, new_decodes=0))
try:
budget(6)
entries = install_reuse(root, reuse_entries(c, tc))
save(root / 'models-adopted.json', adopted)
freeze(root, 'models', [root / 'contract.json', root / 'models-adopted.json',
Path(c['original_root']) / 'models-adopted.json',
*[Path(m['checkpoint']) for m in adopted['models']],
*[Path(m['standardization']) for m in adopted['models']]])
for index in c['resume_indices']:
new_gpu = max(new_gpu, 60 + time.monotonic() - gpu_clock)
budget(7 - index)
(root / 'jobs' / str(index)).mkdir(parents=True, exist_ok=False)
prediction_start = time.monotonic()
g = run_prediction(root, c, pilot, index, c['training_gpu_once'] + costs()['prediction_gpu_seconds'], 300)
cost = max(60 + time.monotonic() - prediction_start, g['gpu_seconds_charged'])
prediction_start = None
new_gpu += cost
charged_children += g['worker']['cpu_seconds'] if g['worker'] else 0.
records.append(dict(mode='predict', index=index, charged_gpu_seconds=cost, guard=g))
d = costs()
save(root / 'resource-progress.json', dict(records=records, **d, original_complete_sha256=c['original_complete_sha256']))
if g['status'] != 'PASS' or cost > 900 or d['prediction_gpu_seconds'] > 7200 or d['cumulative_item_gpu_seconds'] > 120000:
raise RuntimeError('Recovery prediction/resource guard failure')
entries.extend(read(root / 'jobs' / str(index) / 'prediction-records/manifest.json')['entries'])
manifest_identity(entries, c, tc, pilot)
save(root / 'predictions-manifest.json', dict(entries=entries, predict0_reused_by_hash=686,
S1_test_predictions_reused=121, new_fits=0, targets_read=False))
freeze(root, 'predictions', [root / 'contract.json', root / 'models-adopted.json',
root / 'predictions-manifest.json', root / 'resource-progress.json',
root / 'jobs/0/prediction-records/reuse.json', *[Path(p) for p in c['code_sha256']]])
new_gpu = max(new_gpu, 60 + time.monotonic() - gpu_clock)
budget(0)
validate_review(root)
verify(root, 'models')
verify(root, 'predictions')
g = supervise([sys.executable, 'tools/phase3_item1_postfit_worker.py', 'score', str(root)],
root / 'score-guard', dict(cpu_seconds=900, wall_seconds=1200, rss_bytes=16*2**30, disk_bytes=20*2**30))
charged_children += g['cpu_seconds']
records.append(dict(mode='score', guard=g))
if g['status'] != 'PASS' or costs()['postfit_cpu_seconds'] + 120 > 3600:
raise RuntimeError('Score/resource failure')
save(root / 'complete.json', dict(status='COMPLETE', records=records, **costs(),
original_complete_sha256=c['original_complete_sha256'], predict0_reused=True,
resumed_indices=c['resume_indices'], item_gpu_cap=120000, new_fits=0, new_decodes=0,
future_phase3_started=False, no_retry=True))
freeze(root, 'terminal', [root / 'contract.json', root / 'complete.json', root / 'score/result.json', root / 'review.json'])
final = costs()
within = final['postfit_cpu_seconds'] <= 3600 and final['prediction_gpu_seconds'] <= 7200 and final['cumulative_item_gpu_seconds'] <= 120000
save(root / 'post-terminal-cost.json', dict(**final, within_cap=within))
if not within:
raise RuntimeError('Final cumulative resource cap')
except BaseException as exc:
if prediction_start is not None:
new_gpu += 60 + time.monotonic() - prediction_start
if (root / 'complete.json').exists():
save(root / 'pre-failure-complete.json', read(root / 'complete.json'))
save(root / 'complete.json', dict(status='STOPPED', reason=type(exc).__name__ + ': ' + str(exc),
records=records, **costs(), original_complete_sha256=c['original_complete_sha256'],
new_fits=0, no_retry=True, recovery_attempt=1))
raise
if __name__ == '__main__':
run(Path(sys.argv[1]))