122 lines
6.4 KiB
Python
122 lines
6.4 KiB
Python
"""Run one exact-reviewed candidate postfit within the original cumulative budgets."""
|
|
import fcntl
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import sys
|
|
import time
|
|
|
|
from phase2_h1_worker_v2 import save, sha
|
|
from phase2_home_freeze_io import freeze
|
|
from phase3_followup_postfit_common import (
|
|
CLEANUP_CPU, FOLD_CPU, FOLD_GPU, POSTFIT_CPU, POSTFIT_GPU, SCORE_CPU, SETUP_CPU,
|
|
TRAINING, budget, prediction_identity, read, training_terminal,
|
|
)
|
|
from phase3_followup_postfit_recovery_common import config, prior_stop
|
|
from phase3_followup_postfit_recovery_guard import run_prediction
|
|
from r4bc_guard import supervise
|
|
|
|
|
|
def run(root):
|
|
started_wall = time.monotonic()
|
|
started_cpu = time.process_time()
|
|
child_start = resource.getrusage(resource.RUSAGE_CHILDREN)
|
|
c, tc, _, by = config(root)
|
|
item = c['item']
|
|
old_gpu, old_cpu = prior_stop(root)
|
|
evidence = training_terminal(item, tc)
|
|
if evidence != c['training'] or c['item_gpu_cap'] != tc['gpu_cap']:
|
|
raise ValueError('Original training-to-postfit evidence changed')
|
|
lock = (root/'run.lock').open('a')
|
|
fcntl.flock(lock, fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
gpu_lock = Path(f"/tmp/meltmind-candidate-postfit-{c['gpu_uuid']}.lock").open('a')
|
|
fcntl.flock(gpu_lock, fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if (root/'started.json').exists():
|
|
raise FileExistsError('One candidate postfit attempt')
|
|
os.sched_setaffinity(0,c['coordinator_cpus'])
|
|
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
|
|
counters = dict(self_process_time=started_cpu, children_user=child_start.ru_utime,
|
|
children_system=child_start.ru_stime)
|
|
train = TRAINING[item][0]
|
|
prior = evidence['training_gpu_seconds']
|
|
accrued_gpu = 0.
|
|
accrued_child = 0.
|
|
in_flight = None
|
|
records = []
|
|
|
|
def cpu():
|
|
child = resource.getrusage(resource.RUSAGE_CHILDREN)
|
|
reaped = child.ru_utime + child.ru_stime - child_start.ru_utime - child_start.ru_stime
|
|
return old_cpu + SETUP_CPU + time.process_time() - started_cpu + max(accrued_child,reaped)
|
|
|
|
def ledger():
|
|
return dict(records=records, prediction_gpu_seconds=old_gpu+accrued_gpu,
|
|
continuation_gpu_seconds=accrued_gpu, prior_stop_gpu_seconds_once=old_gpu,
|
|
postfit_cpu_seconds=cpu(), prior_stop_cpu_seconds_once=old_cpu, training_gpu_seconds_once=prior,
|
|
cumulative_item_gpu_seconds=prior+old_gpu+accrued_gpu,new_fits=0,new_decodes=0,no_retry=True)
|
|
|
|
budget(c,old_gpu,cpu(),5)
|
|
save(root/'started.json',dict(item=item,pid=os.getpid(),unix=time.time(),training_gpu_seconds_once=prior,
|
|
postfit_gpu_cap=POSTFIT_GPU,postfit_cpu_cap=POSTFIT_CPU,cpu_start_counters=counters,
|
|
prior_stop_gpu_seconds_once=old_gpu,prior_stop_cpu_seconds_once=old_cpu,recovery_label=True,
|
|
review_sha256=sha(root/'review.json'),new_fits=0,new_decodes=0))
|
|
try:
|
|
save(root/'models-adopted.json',evidence)
|
|
freeze(root,'models',[root/'contract.json',root/'models-adopted.json',train/'complete.json',
|
|
train/'training-terminal-git-freeze.json',train/'post-terminal-cost.json',root/'recovery-accounting.json',
|
|
*[Path(m[k]) for m in evidence['models'] for k in ('path','standardization')]])
|
|
predictions = []
|
|
for fold in range(5):
|
|
accrued_gpu = max(accrued_gpu,60+time.monotonic()-started_wall)
|
|
budget(c,old_gpu+accrued_gpu,cpu(),5-fold)
|
|
(root/'jobs'/str(fold)).mkdir(parents=True,exist_ok=False)
|
|
in_flight = time.monotonic()
|
|
guard = run_prediction(root,c,fold,prior+old_gpu+accrued_gpu,FOLD_CPU)
|
|
elapsed = max(60+time.monotonic()-in_flight,guard['gpu_seconds_charged'])
|
|
in_flight = None
|
|
accrued_gpu += elapsed
|
|
accrued_child += guard['worker']['cpu_seconds'] if guard['worker'] else 0.
|
|
records.append(dict(fold=fold,charged_gpu_seconds=elapsed,guard=guard))
|
|
save(root/'resource-progress.json',ledger())
|
|
if guard['status'] != 'PASS' or guard['monitor_errors'] or elapsed > FOLD_GPU:
|
|
raise RuntimeError(f'Fold {fold} prediction stop')
|
|
p = root/'jobs'/str(fold)/'prediction-records/manifest.json'
|
|
predictions.extend(read(p)['entries'])
|
|
prediction_identity(root,c,tc,by,predictions)
|
|
path = root/'predictions-manifest.json'
|
|
save(path,dict(entries=predictions,test_targets_read=False,new_fits=0))
|
|
freeze(root,'predictions',[root/'contract.json',root/'models-adopted.json',path,
|
|
root/'resource-progress.json',*map(Path,c['code_sha256'])])
|
|
accrued_gpu = max(accrued_gpu,60+time.monotonic()-started_wall)
|
|
budget(c,old_gpu+accrued_gpu,cpu(),0)
|
|
g = supervise([sys.executable,'tools/phase3_followup_postfit_recovery_worker.py','score',str(root)],
|
|
root/'score-guard',dict(cpu_seconds=SCORE_CPU,wall_seconds=1200,
|
|
rss_bytes=16*2**30,disk_bytes=20*2**30))
|
|
accrued_child += g['cpu_seconds']
|
|
records.append(dict(mode='score',guard=g))
|
|
if (g['status']!='PASS' or g['returncode']!=0 or g['monitor_errors']
|
|
or cpu()+CLEANUP_CPU>POSTFIT_CPU):
|
|
raise RuntimeError('Score or CPU limit')
|
|
save(root/'complete.json',dict(status='COMPLETE',**ledger()))
|
|
freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'score/result.json',
|
|
root/'score/target-access.jsonl',root/'review.json'])
|
|
final = cpu()
|
|
save(root/'post-terminal-cost.json',dict(**(ledger() | {'postfit_cpu_seconds': final}),
|
|
within_cap=final<=POSTFIT_CPU,training_complete_sha256=evidence['training_complete_sha256'],
|
|
cpu_start_counters=counters))
|
|
if final>POSTFIT_CPU:
|
|
raise RuntimeError('CPU cap reached during terminal freeze')
|
|
except BaseException as exc:
|
|
if in_flight is not None:
|
|
accrued_gpu += 60+time.monotonic()-in_flight
|
|
if not any(r.get('mode')=='score' for r in records):
|
|
accrued_gpu = max(accrued_gpu,60+time.monotonic()-started_wall)
|
|
if (root/'complete.json').exists():
|
|
save(root/'pre-failure-complete.json',read(root/'complete.json'))
|
|
save(root/'complete.json',dict(status='STOPPED',reason=f'{type(exc).__name__}: {exc}',**ledger()))
|
|
raise
|
|
|
|
|
|
if __name__=='__main__':
|
|
run(Path(sys.argv[1]))
|