Files
MeltMind/tools/phase3_followup_postfit.py

118 lines
6.0 KiB
Python

"""Run one exact-reviewed candidate postfit within the original cumulative budgets."""
import fcntl
import os
from pathlib import Path
import resource
import sys
import time
from phase2_h1_worker_v2 import save, sha
from phase2_home_freeze_io import freeze
from phase3_followup_postfit_common import (
CLEANUP_CPU, FOLD_CPU, FOLD_GPU, POSTFIT_CPU, POSTFIT_GPU, SCORE_CPU, SETUP_CPU,
TRAINING, budget, config, prediction_identity, read, training_terminal,
)
from phase3_followup_postfit_guard import run_prediction
from r4bc_guard import supervise
def run(root):
started_wall = time.monotonic()
started_cpu = time.process_time()
child_start = resource.getrusage(resource.RUSAGE_CHILDREN)
c, tc, _, by = config(root)
item = c['item']
evidence = training_terminal(item, tc)
if evidence != c['training'] or c['item_gpu_cap'] != tc['gpu_cap']:
raise ValueError('Original training-to-postfit evidence changed')
lock = (root/'run.lock').open('a')
fcntl.flock(lock, fcntl.LOCK_EX|fcntl.LOCK_NB)
gpu_lock = Path(f"/tmp/meltmind-candidate-postfit-{c['gpu_uuid']}.lock").open('a')
fcntl.flock(gpu_lock, fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():
raise FileExistsError('One candidate postfit attempt')
os.sched_setaffinity(0,c['coordinator_cpus'])
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
counters = dict(self_process_time=started_cpu, children_user=child_start.ru_utime,
children_system=child_start.ru_stime)
train = TRAINING[item][0]
prior = evidence['training_gpu_seconds']
accrued_gpu = 0.
accrued_child = 0.
in_flight = None
records = []
def cpu():
child = resource.getrusage(resource.RUSAGE_CHILDREN)
reaped = child.ru_utime + child.ru_stime - child_start.ru_utime - child_start.ru_stime
return SETUP_CPU + time.process_time() - started_cpu + max(accrued_child,reaped)
def ledger():
return dict(records=records,prediction_gpu_seconds=accrued_gpu,
postfit_cpu_seconds=cpu(),training_gpu_seconds_once=prior,
cumulative_item_gpu_seconds=prior+accrued_gpu,new_fits=0,new_decodes=0,no_retry=True)
budget(c,0,cpu(),5)
save(root/'started.json',dict(item=item,pid=os.getpid(),unix=time.time(),training_gpu_seconds_once=prior,
postfit_gpu_cap=POSTFIT_GPU,postfit_cpu_cap=POSTFIT_CPU,cpu_start_counters=counters,
review_sha256=sha(root/'review.json'),new_fits=0,new_decodes=0))
try:
save(root/'models-adopted.json',evidence)
freeze(root,'models',[root/'contract.json',root/'models-adopted.json',train/'complete.json',
train/'training-terminal-git-freeze.json',train/'post-terminal-cost.json',
*[Path(m[k]) for m in evidence['models'] for k in ('path','standardization')]])
predictions = []
for fold in range(5):
accrued_gpu = max(accrued_gpu,60+time.monotonic()-started_wall)
budget(c,accrued_gpu,cpu(),5-fold)
(root/'jobs'/str(fold)).mkdir(parents=True,exist_ok=False)
in_flight = time.monotonic()
guard = run_prediction(root,c,fold,prior+accrued_gpu,FOLD_CPU)
elapsed = max(60+time.monotonic()-in_flight,guard['gpu_seconds_charged'])
in_flight = None
accrued_gpu += elapsed
accrued_child += guard['worker']['cpu_seconds'] if guard['worker'] else 0.
records.append(dict(fold=fold,charged_gpu_seconds=elapsed,guard=guard))
save(root/'resource-progress.json',ledger())
if guard['status'] != 'PASS' or guard['monitor_errors'] or elapsed > FOLD_GPU:
raise RuntimeError(f'Fold {fold} prediction stop')
p = root/'jobs'/str(fold)/'prediction-records/manifest.json'
predictions.extend(read(p)['entries'])
prediction_identity(root,c,tc,by,predictions)
path = root/'predictions-manifest.json'
save(path,dict(entries=predictions,test_targets_read=False,new_fits=0))
freeze(root,'predictions',[root/'contract.json',root/'models-adopted.json',path,
root/'resource-progress.json',*map(Path,c['code_sha256'])])
accrued_gpu = max(accrued_gpu,60+time.monotonic()-started_wall)
budget(c,accrued_gpu,cpu(),0)
g = supervise([sys.executable,'tools/phase3_followup_postfit_worker.py','score',str(root)],
root/'score-guard',dict(cpu_seconds=SCORE_CPU,wall_seconds=1200,
rss_bytes=16*2**30,disk_bytes=20*2**30))
accrued_child += g['cpu_seconds']
records.append(dict(mode='score',guard=g))
if (g['status']!='PASS' or g['returncode']!=0 or g['monitor_errors']
or cpu()+CLEANUP_CPU>POSTFIT_CPU):
raise RuntimeError('Score or CPU limit')
save(root/'complete.json',dict(status='COMPLETE',**ledger()))
freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'score/result.json',
root/'score/target-access.jsonl',root/'review.json'])
final = cpu()
save(root/'post-terminal-cost.json',dict(**(ledger() | {'postfit_cpu_seconds': final}),
within_cap=final<=POSTFIT_CPU,training_complete_sha256=evidence['training_complete_sha256'],
cpu_start_counters=counters))
if final>POSTFIT_CPU:
raise RuntimeError('CPU cap reached during terminal freeze')
except BaseException as exc:
if in_flight is not None:
accrued_gpu += 60+time.monotonic()-in_flight
if not any(r.get('mode')=='score' for r in records):
accrued_gpu = max(accrued_gpu,60+time.monotonic()-started_wall)
if (root/'complete.json').exists():
save(root/'pre-failure-complete.json',read(root/'complete.json'))
save(root/'complete.json',dict(status='STOPPED',reason=f'{type(exc).__name__}: {exc}',**ledger()))
raise
if __name__=='__main__':
run(Path(sys.argv[1]))