142 lines
7.4 KiB
Python
142 lines
7.4 KiB
Python
"""Single labelled resume: reuse0, predict1..6, freeze all, unchanged score."""
|
|
import fcntl
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import shutil
|
|
import sys
|
|
import time
|
|
|
|
import psutil
|
|
from phase2_gate_outcome_review import validate_review
|
|
from phase2_h1_worker_v2 import checked, save, sha
|
|
from phase2_home_freeze_io import freeze, verify
|
|
from phase3_item1_postfit_common import read, manifest_identity, require_budget
|
|
from phase3_item1_resume_common import config, adoption, reuse_entries, cumulative_cost
|
|
from phase3_item1_resume_guard import run_prediction
|
|
from r4bc_guard import supervise
|
|
|
|
|
|
def install_reuse(root, entries):
|
|
result = []
|
|
for e in entries:
|
|
original = checked(e['path'], e['sha256'])
|
|
target = root / 'predictions' / e['arm'] / (e['case'] + '.npz')
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
if target.exists():
|
|
raise FileExistsError('No duplicate prediction adoption')
|
|
shutil.copyfile(original, target)
|
|
checked(target, e['sha256'])
|
|
result.append(dict(e, path=str(target), reuse_original_path=str(original),
|
|
reused_without_prediction=True))
|
|
out = root / 'jobs/0/prediction-records'
|
|
out.mkdir(parents=True, exist_ok=False)
|
|
save(out / 'reuse.json', dict(status='REUSED_BY_HASH_NO_WORKER', entries=result, targets_read=False))
|
|
return result
|
|
|
|
|
|
def run(root):
|
|
gpu_clock = time.monotonic()
|
|
start = time.process_time()
|
|
u0 = resource.getrusage(resource.RUSAGE_CHILDREN)
|
|
c, tc, pilot = config(root)
|
|
validate_review(root)
|
|
adopted = adoption(c, tc)
|
|
lock = (root / 'run.lock').open('a')
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
if (root / 'started.json').exists():
|
|
raise FileExistsError('One labelled recovery, no repeat launch')
|
|
for p in psutil.process_iter(['pid', 'cmdline']):
|
|
if p.pid != os.getpid() and any(Path(a).name in {'phase3_item1_postfit.py', 'phase3_item1_resume.py',
|
|
'phase3_item1_postfit_worker.py'} for a in p.info['cmdline'] or []):
|
|
raise RuntimeError('Existing original or resumed postfit')
|
|
gpu_lock = Path('/tmp/meltmind-item1-resume-' + c['gpu_uuid'] + '.lock').open('a')
|
|
fcntl.flock(gpu_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
os.sched_setaffinity(0, c['coordinator_cpus'])
|
|
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
|
|
new_gpu = 0.
|
|
charged_children = 0.
|
|
records = []
|
|
prediction_start = None
|
|
|
|
def costs():
|
|
u = resource.getrusage(resource.RUSAGE_CHILDREN)
|
|
children = u.ru_utime + u.ru_stime - u0.ru_utime - u0.ru_stime
|
|
new_cpu = 180 + time.process_time() - start + max(charged_children, children)
|
|
return cumulative_cost(c, new_gpu, new_cpu)
|
|
|
|
def budget(remaining):
|
|
d = costs()
|
|
require_budget(c['training_gpu_once'], d['prediction_gpu_seconds'], d['postfit_cpu_seconds'], remaining)
|
|
return d
|
|
|
|
save(root / 'started.json', dict(pid=os.getpid(), unix=time.time(), recovery_attempt=1,
|
|
original_root=c['original_root'], prior_postfit_gpu=c['prior_postfit_gpu'],
|
|
prior_postfit_cpu=c['prior_postfit_cpu'], training_gpu_seconds_once=c['training_gpu_once'],
|
|
cpu_setup_allowance=180, review_sha256=sha(root / 'review.json'), indices=c['resume_indices'],
|
|
gpu_uuid=c['gpu_uuid'], new_fits=0, new_decodes=0))
|
|
try:
|
|
budget(6)
|
|
entries = install_reuse(root, reuse_entries(c, tc))
|
|
save(root / 'models-adopted.json', adopted)
|
|
freeze(root, 'models', [root / 'contract.json', root / 'models-adopted.json',
|
|
Path(c['original_root']) / 'models-adopted.json',
|
|
*[Path(m['checkpoint']) for m in adopted['models']],
|
|
*[Path(m['standardization']) for m in adopted['models']]])
|
|
for index in c['resume_indices']:
|
|
new_gpu = max(new_gpu, 60 + time.monotonic() - gpu_clock)
|
|
budget(7 - index)
|
|
(root / 'jobs' / str(index)).mkdir(parents=True, exist_ok=False)
|
|
prediction_start = time.monotonic()
|
|
g = run_prediction(root, c, pilot, index, c['training_gpu_once'] + costs()['prediction_gpu_seconds'], 300)
|
|
cost = max(60 + time.monotonic() - prediction_start, g['gpu_seconds_charged'])
|
|
prediction_start = None
|
|
new_gpu += cost
|
|
charged_children += g['worker']['cpu_seconds'] if g['worker'] else 0.
|
|
records.append(dict(mode='predict', index=index, charged_gpu_seconds=cost, guard=g))
|
|
d = costs()
|
|
save(root / 'resource-progress.json', dict(records=records, **d, original_complete_sha256=c['original_complete_sha256']))
|
|
if g['status'] != 'PASS' or cost > 900 or d['prediction_gpu_seconds'] > 7200 or d['cumulative_item_gpu_seconds'] > 120000:
|
|
raise RuntimeError('Recovery prediction/resource guard failure')
|
|
entries.extend(read(root / 'jobs' / str(index) / 'prediction-records/manifest.json')['entries'])
|
|
manifest_identity(entries, c, tc, pilot)
|
|
save(root / 'predictions-manifest.json', dict(entries=entries, predict0_reused_by_hash=686,
|
|
S1_test_predictions_reused=121, new_fits=0, targets_read=False))
|
|
freeze(root, 'predictions', [root / 'contract.json', root / 'models-adopted.json',
|
|
root / 'predictions-manifest.json', root / 'resource-progress.json',
|
|
root / 'jobs/0/prediction-records/reuse.json', *[Path(p) for p in c['code_sha256']]])
|
|
new_gpu = max(new_gpu, 60 + time.monotonic() - gpu_clock)
|
|
budget(0)
|
|
validate_review(root)
|
|
verify(root, 'models')
|
|
verify(root, 'predictions')
|
|
g = supervise([sys.executable, 'tools/phase3_item1_postfit_worker.py', 'score', str(root)],
|
|
root / 'score-guard', dict(cpu_seconds=900, wall_seconds=1200, rss_bytes=16*2**30, disk_bytes=20*2**30))
|
|
charged_children += g['cpu_seconds']
|
|
records.append(dict(mode='score', guard=g))
|
|
if g['status'] != 'PASS' or costs()['postfit_cpu_seconds'] + 120 > 3600:
|
|
raise RuntimeError('Score/resource failure')
|
|
save(root / 'complete.json', dict(status='COMPLETE', records=records, **costs(),
|
|
original_complete_sha256=c['original_complete_sha256'], predict0_reused=True,
|
|
resumed_indices=c['resume_indices'], item_gpu_cap=120000, new_fits=0, new_decodes=0,
|
|
future_phase3_started=False, no_retry=True))
|
|
freeze(root, 'terminal', [root / 'contract.json', root / 'complete.json', root / 'score/result.json', root / 'review.json'])
|
|
final = costs()
|
|
within = final['postfit_cpu_seconds'] <= 3600 and final['prediction_gpu_seconds'] <= 7200 and final['cumulative_item_gpu_seconds'] <= 120000
|
|
save(root / 'post-terminal-cost.json', dict(**final, within_cap=within))
|
|
if not within:
|
|
raise RuntimeError('Final cumulative resource cap')
|
|
except BaseException as exc:
|
|
if prediction_start is not None:
|
|
new_gpu += 60 + time.monotonic() - prediction_start
|
|
if (root / 'complete.json').exists():
|
|
save(root / 'pre-failure-complete.json', read(root / 'complete.json'))
|
|
save(root / 'complete.json', dict(status='STOPPED', reason=type(exc).__name__ + ': ' + str(exc),
|
|
records=records, **costs(), original_complete_sha256=c['original_complete_sha256'],
|
|
new_fits=0, no_retry=True, recovery_attempt=1))
|
|
raise
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(Path(sys.argv[1]))
|