185 lines
9.5 KiB
Python
185 lines
9.5 KiB
Python
"""Serial, shared-slot prediction followed by the original paired scoring barrier."""
|
|
|
|
import fcntl
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
from phase2_gate_outcome_review import validate_review
|
|
from phase2_h1_worker_v2 import save, sha
|
|
from phase2_home_freeze_io import freeze, verify
|
|
from phase3_shared_capacity import alive, process_identity, release, transfer
|
|
from r4bc_guard import supervise
|
|
from track_a_postfit_common import prediction_identity
|
|
from track_a_postfit_rebind_common import RECOVERY, SPLITS, selected_training, terminal
|
|
from track_a_postfit_shared_capacity import acquire, register
|
|
from track_a_postfit_shared_common import ROOT, SETUP, budget, config, read
|
|
|
|
|
|
def stop_guard(guard):
|
|
if guard.poll() is not None:
|
|
return
|
|
guard.terminate()
|
|
try:
|
|
guard.wait(timeout=45)
|
|
except subprocess.TimeoutExpired:
|
|
guard.kill()
|
|
guard.wait()
|
|
|
|
|
|
def predict_fold(root, job, reservation, queue_identity, wait_seconds, resource_check):
|
|
folder = root/'jobs'/str(job['id'])
|
|
token = reservation['token']
|
|
guard = guard_id = None
|
|
transferred = False
|
|
try:
|
|
folder.mkdir(exist_ok=False)
|
|
path = folder/'assignment.json'
|
|
save(path, dict(id=job['id'], contract_sha256=sha(root/'contract.json'), token=token,
|
|
gpu_uuid=reservation['gpu_uuid'], worker_cpus=reservation['worker_cpus'],
|
|
queue_identity=queue_identity, pre_dispatch_wait_seconds=wait_seconds))
|
|
with (folder/'guard.log').open('x') as log:
|
|
guard = subprocess.Popen([sys.executable, 'tools/track_a_postfit_shared_guard.py',
|
|
str(root), str(job['id']), sha(path)],
|
|
stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
|
guard_id = process_identity(guard.pid)
|
|
transfer(token, guard_id)
|
|
transferred = True
|
|
save(folder/'lease-ready.json', dict(token=token, owner=guard_id))
|
|
while guard.poll() is None:
|
|
resource_check()
|
|
time.sleep(0.5)
|
|
receipt = read(folder/'complete.json')
|
|
if (guard.returncode != 0 or receipt['status'] != 'PASS'
|
|
or receipt['gpu_seconds_charged'] > job['gpu_seconds'] or receipt['monitor_policy']['failed']):
|
|
raise RuntimeError(f'Shared prediction guard rejected fold {job["id"]}')
|
|
return receipt
|
|
finally:
|
|
if guard is not None:
|
|
stop_guard(guard)
|
|
worker = folder/'prediction-records/worker-identity.json'
|
|
if worker.exists() and alive(read(worker)):
|
|
raise RuntimeError('Prediction worker outlived guard; retained own lease')
|
|
release(token, guard_id if transferred else None)
|
|
|
|
|
|
def run(root):
|
|
gpu_clock = time.monotonic()
|
|
started_cpu = time.process_time()
|
|
initial_children = resource.getrusage(resource.RUSAGE_CHILDREN)
|
|
c, tc, _ = config(root)
|
|
validate_review(root)
|
|
adopted = terminal(tc)
|
|
if adopted != c['training_evidence']:
|
|
raise ValueError('Mixed-source training evidence changed')
|
|
prior = adopted['training_gpu_seconds_once']
|
|
charged_cpu = 0.
|
|
frozen_gpu = None
|
|
attempted = 0
|
|
records = []
|
|
stop_requests = []
|
|
original_affinity = os.sched_getaffinity(0)
|
|
|
|
def gpu():
|
|
return (SETUP + attempted*SETUP + time.monotonic()-gpu_clock
|
|
if frozen_gpu is None else frozen_gpu)
|
|
|
|
def cpu():
|
|
children = resource.getrusage(resource.RUSAGE_CHILDREN)
|
|
child_time = children.ru_utime + children.ru_stime - initial_children.ru_utime - initial_children.ru_stime
|
|
return 180 + time.process_time() - started_cpu + max(charged_cpu, child_time)
|
|
|
|
def resource_check(pending):
|
|
if stop_requests:
|
|
raise RuntimeError(stop_requests[0])
|
|
budget(c, gpu(), cpu(), pending)
|
|
|
|
with (root/'run.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
if (root/'started.json').exists():
|
|
raise FileExistsError('One Track A shared postfit, zero retries')
|
|
identity = process_identity(os.getpid())
|
|
digest = sha(root/'contract.json')
|
|
register(root, digest, c)
|
|
os.sched_setaffinity(0, c['coordinator_cpus'])
|
|
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
|
|
for sig in (signal.SIGINT, signal.SIGTERM):
|
|
signal.signal(sig, lambda n, f: stop_requests.append(f'COORDINATOR_SIGNAL_{n}'))
|
|
cpu_counters = dict(self_process_time=started_cpu, children_user=initial_children.ru_utime,
|
|
children_system=initial_children.ru_stime)
|
|
save(root/'started.json', dict(identity=identity, unix=time.time(), contract_sha256=digest,
|
|
review_sha256=sha(root/'review.json'), training_gpu_seconds_once=prior, shared_slot_limit=1,
|
|
postfit_gpu_cap=c['postfit_gpu_cap'], track_a_gpu_cap=c['gpu_cap'],
|
|
cpu_start_counters=cpu_counters, cpu_setup_allowance=180, new_fits=0, new_decodes=0))
|
|
try:
|
|
save(root/'models-adopted.json', adopted)
|
|
train = Path(c['training_root'])
|
|
freeze(root, 'models', [root/'contract.json', root/'models-adopted.json', train/'complete.json',
|
|
RECOVERY/'complete.json', RECOVERY/'available-models-incomplete.json',
|
|
RECOVERY/'incomplete-models-git-freeze.json', RECOVERY/'terminal-git-freeze.json',
|
|
*[Path(m['path']) for m in adopted['models']],
|
|
*[Path(m['standardization']) for m in adopted['models']]])
|
|
entries = []
|
|
(root/'jobs').mkdir(exist_ok=False)
|
|
jobs = c['predictions']
|
|
for pos, job in enumerate(jobs):
|
|
wait_started = time.monotonic()
|
|
while True:
|
|
resource_check(jobs[pos:])
|
|
lease, observation = acquire(root, job['id'], digest, original_affinity)
|
|
if lease is not None:
|
|
break
|
|
save(root/'latest-admission.json', observation)
|
|
time.sleep(1)
|
|
attempted += 1
|
|
receipt = predict_fold(root, job, lease, identity, time.monotonic()-wait_started,
|
|
lambda: resource_check(jobs[pos+1:]))
|
|
charged_cpu += receipt['worker']['cpu_seconds']
|
|
records.append(dict(mode='predict', split=job['id'], gpu_uuid=lease['gpu_uuid'], guard=receipt))
|
|
save(root/'resource-progress.json', dict(records=records, prediction_gpu_seconds=gpu(),
|
|
postfit_cpu_seconds=cpu(), cumulative_track_a_gpu_seconds=prior+gpu()))
|
|
entries.extend(read(root/'jobs'/str(job['id'])/'prediction-records/manifest.json')['entries'])
|
|
prediction_identity(root, c, selected_training(tc), entries)
|
|
save(root/'predictions-manifest.json', dict(entries=entries, A_predictions_reused=c['expected_pair_visits'],
|
|
evaluated_splits=list(SPLITS), excluded_split=c['excluded_split'], new_fits=0, targets_read=False))
|
|
freeze(root, 'predictions', [root/'contract.json', root/'models-adopted.json',
|
|
root/'predictions-manifest.json', root/'resource-progress.json', *map(Path,c['code_sha256'])])
|
|
resource_check([])
|
|
frozen_gpu = gpu()
|
|
validate_review(root)
|
|
verify(root, 'models')
|
|
verify(root, 'predictions')
|
|
score = supervise([sys.executable, 'tools/track_a_postfit_shared_worker.py', 'score', str(root)],
|
|
root/'score-guard', dict(cpu_seconds=900, wall_seconds=1200,
|
|
rss_bytes=16*2**30, disk_bytes=20*2**30))
|
|
charged_cpu += score['cpu_seconds']
|
|
records.append(dict(mode='score', guard=score))
|
|
if score['status'] != 'PASS' or cpu()+120 > c['postfit_cpu_cap']:
|
|
raise RuntimeError('Original Track A scoring cap or guard')
|
|
save(root/'complete.json', dict(status='COMPLETE_FOUR_OF_FIVE', evaluated_splits=list(SPLITS),
|
|
excluded_split=c['excluded_split'], records=records, prediction_gpu_seconds=gpu(),
|
|
postfit_cpu_seconds=cpu(), training_gpu_seconds_once=prior,
|
|
cumulative_track_a_gpu_seconds=prior+gpu(), track_a_gpu_cap=c['gpu_cap'],
|
|
new_fits=0, new_decodes=0, no_retry=True))
|
|
freeze(root, 'terminal', [root/'contract.json', root/'complete.json', root/'score/result.json', root/'review.json'])
|
|
within = cpu()<=c['postfit_cpu_cap'] and gpu()<=c['postfit_gpu_cap'] and prior+gpu()<=c['gpu_cap']
|
|
save(root/'post-terminal-cost.json', dict(postfit_cpu_seconds=cpu(), prediction_gpu_seconds=gpu(),
|
|
cumulative_track_a_gpu_seconds=prior+gpu(), within_cap=within, cpu_start_counters=cpu_counters))
|
|
if not within:
|
|
raise RuntimeError('Shared Track A final resource cap')
|
|
except BaseException as exc:
|
|
if (root/'complete.json').exists():
|
|
save(root/'pre-failure-complete.json', read(root/'complete.json'))
|
|
save(root/'complete.json', dict(status='STOPPED', reason=f'{type(exc).__name__}: {exc}',
|
|
records=records, postfit_cpu_seconds=cpu(), prediction_gpu_seconds=gpu(),
|
|
training_gpu_seconds_once=prior, cumulative_track_a_gpu_seconds=prior+gpu(), new_fits=0, no_retry=True))
|
|
raise
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(ROOT)
|