Files
MeltMind/tools/track_a_postfit_shared.py
T

185 lines
9.5 KiB
Python

"""Serial, shared-slot prediction followed by the original paired scoring barrier."""
import fcntl
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
from phase2_gate_outcome_review import validate_review
from phase2_h1_worker_v2 import save, sha
from phase2_home_freeze_io import freeze, verify
from phase3_shared_capacity import alive, process_identity, release, transfer
from r4bc_guard import supervise
from track_a_postfit_common import prediction_identity
from track_a_postfit_rebind_common import RECOVERY, SPLITS, selected_training, terminal
from track_a_postfit_shared_capacity import acquire, register
from track_a_postfit_shared_common import ROOT, SETUP, budget, config, read
def stop_guard(guard):
if guard.poll() is not None:
return
guard.terminate()
try:
guard.wait(timeout=45)
except subprocess.TimeoutExpired:
guard.kill()
guard.wait()
def predict_fold(root, job, reservation, queue_identity, wait_seconds, resource_check):
folder = root/'jobs'/str(job['id'])
token = reservation['token']
guard = guard_id = None
transferred = False
try:
folder.mkdir(exist_ok=False)
path = folder/'assignment.json'
save(path, dict(id=job['id'], contract_sha256=sha(root/'contract.json'), token=token,
gpu_uuid=reservation['gpu_uuid'], worker_cpus=reservation['worker_cpus'],
queue_identity=queue_identity, pre_dispatch_wait_seconds=wait_seconds))
with (folder/'guard.log').open('x') as log:
guard = subprocess.Popen([sys.executable, 'tools/track_a_postfit_shared_guard.py',
str(root), str(job['id']), sha(path)],
stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
guard_id = process_identity(guard.pid)
transfer(token, guard_id)
transferred = True
save(folder/'lease-ready.json', dict(token=token, owner=guard_id))
while guard.poll() is None:
resource_check()
time.sleep(0.5)
receipt = read(folder/'complete.json')
if (guard.returncode != 0 or receipt['status'] != 'PASS'
or receipt['gpu_seconds_charged'] > job['gpu_seconds'] or receipt['monitor_policy']['failed']):
raise RuntimeError(f'Shared prediction guard rejected fold {job["id"]}')
return receipt
finally:
if guard is not None:
stop_guard(guard)
worker = folder/'prediction-records/worker-identity.json'
if worker.exists() and alive(read(worker)):
raise RuntimeError('Prediction worker outlived guard; retained own lease')
release(token, guard_id if transferred else None)
def run(root):
gpu_clock = time.monotonic()
started_cpu = time.process_time()
initial_children = resource.getrusage(resource.RUSAGE_CHILDREN)
c, tc, _ = config(root)
validate_review(root)
adopted = terminal(tc)
if adopted != c['training_evidence']:
raise ValueError('Mixed-source training evidence changed')
prior = adopted['training_gpu_seconds_once']
charged_cpu = 0.
frozen_gpu = None
attempted = 0
records = []
stop_requests = []
original_affinity = os.sched_getaffinity(0)
def gpu():
return (SETUP + attempted*SETUP + time.monotonic()-gpu_clock
if frozen_gpu is None else frozen_gpu)
def cpu():
children = resource.getrusage(resource.RUSAGE_CHILDREN)
child_time = children.ru_utime + children.ru_stime - initial_children.ru_utime - initial_children.ru_stime
return 180 + time.process_time() - started_cpu + max(charged_cpu, child_time)
def resource_check(pending):
if stop_requests:
raise RuntimeError(stop_requests[0])
budget(c, gpu(), cpu(), pending)
with (root/'run.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (root/'started.json').exists():
raise FileExistsError('One Track A shared postfit, zero retries')
identity = process_identity(os.getpid())
digest = sha(root/'contract.json')
register(root, digest, c)
os.sched_setaffinity(0, c['coordinator_cpus'])
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
for sig in (signal.SIGINT, signal.SIGTERM):
signal.signal(sig, lambda n, f: stop_requests.append(f'COORDINATOR_SIGNAL_{n}'))
cpu_counters = dict(self_process_time=started_cpu, children_user=initial_children.ru_utime,
children_system=initial_children.ru_stime)
save(root/'started.json', dict(identity=identity, unix=time.time(), contract_sha256=digest,
review_sha256=sha(root/'review.json'), training_gpu_seconds_once=prior, shared_slot_limit=1,
postfit_gpu_cap=c['postfit_gpu_cap'], track_a_gpu_cap=c['gpu_cap'],
cpu_start_counters=cpu_counters, cpu_setup_allowance=180, new_fits=0, new_decodes=0))
try:
save(root/'models-adopted.json', adopted)
train = Path(c['training_root'])
freeze(root, 'models', [root/'contract.json', root/'models-adopted.json', train/'complete.json',
RECOVERY/'complete.json', RECOVERY/'available-models-incomplete.json',
RECOVERY/'incomplete-models-git-freeze.json', RECOVERY/'terminal-git-freeze.json',
*[Path(m['path']) for m in adopted['models']],
*[Path(m['standardization']) for m in adopted['models']]])
entries = []
(root/'jobs').mkdir(exist_ok=False)
jobs = c['predictions']
for pos, job in enumerate(jobs):
wait_started = time.monotonic()
while True:
resource_check(jobs[pos:])
lease, observation = acquire(root, job['id'], digest, original_affinity)
if lease is not None:
break
save(root/'latest-admission.json', observation)
time.sleep(1)
attempted += 1
receipt = predict_fold(root, job, lease, identity, time.monotonic()-wait_started,
lambda: resource_check(jobs[pos+1:]))
charged_cpu += receipt['worker']['cpu_seconds']
records.append(dict(mode='predict', split=job['id'], gpu_uuid=lease['gpu_uuid'], guard=receipt))
save(root/'resource-progress.json', dict(records=records, prediction_gpu_seconds=gpu(),
postfit_cpu_seconds=cpu(), cumulative_track_a_gpu_seconds=prior+gpu()))
entries.extend(read(root/'jobs'/str(job['id'])/'prediction-records/manifest.json')['entries'])
prediction_identity(root, c, selected_training(tc), entries)
save(root/'predictions-manifest.json', dict(entries=entries, A_predictions_reused=c['expected_pair_visits'],
evaluated_splits=list(SPLITS), excluded_split=c['excluded_split'], new_fits=0, targets_read=False))
freeze(root, 'predictions', [root/'contract.json', root/'models-adopted.json',
root/'predictions-manifest.json', root/'resource-progress.json', *map(Path,c['code_sha256'])])
resource_check([])
frozen_gpu = gpu()
validate_review(root)
verify(root, 'models')
verify(root, 'predictions')
score = supervise([sys.executable, 'tools/track_a_postfit_shared_worker.py', 'score', str(root)],
root/'score-guard', dict(cpu_seconds=900, wall_seconds=1200,
rss_bytes=16*2**30, disk_bytes=20*2**30))
charged_cpu += score['cpu_seconds']
records.append(dict(mode='score', guard=score))
if score['status'] != 'PASS' or cpu()+120 > c['postfit_cpu_cap']:
raise RuntimeError('Original Track A scoring cap or guard')
save(root/'complete.json', dict(status='COMPLETE_FOUR_OF_FIVE', evaluated_splits=list(SPLITS),
excluded_split=c['excluded_split'], records=records, prediction_gpu_seconds=gpu(),
postfit_cpu_seconds=cpu(), training_gpu_seconds_once=prior,
cumulative_track_a_gpu_seconds=prior+gpu(), track_a_gpu_cap=c['gpu_cap'],
new_fits=0, new_decodes=0, no_retry=True))
freeze(root, 'terminal', [root/'contract.json', root/'complete.json', root/'score/result.json', root/'review.json'])
within = cpu()<=c['postfit_cpu_cap'] and gpu()<=c['postfit_gpu_cap'] and prior+gpu()<=c['gpu_cap']
save(root/'post-terminal-cost.json', dict(postfit_cpu_seconds=cpu(), prediction_gpu_seconds=gpu(),
cumulative_track_a_gpu_seconds=prior+gpu(), within_cap=within, cpu_start_counters=cpu_counters))
if not within:
raise RuntimeError('Shared Track A final resource cap')
except BaseException as exc:
if (root/'complete.json').exists():
save(root/'pre-failure-complete.json', read(root/'complete.json'))
save(root/'complete.json', dict(status='STOPPED', reason=f'{type(exc).__name__}: {exc}',
records=records, postfit_cpu_seconds=cpu(), prediction_gpu_seconds=gpu(),
training_gpu_seconds_once=prior, cumulative_track_a_gpu_seconds=prior+gpu(), new_fits=0, no_retry=True))
raise
if __name__ == '__main__':
run(ROOT)