Publish Track A shared-slot dispatch delta for narrow independent review
This commit is contained in:
1 parent
3438020be4
commit
3e62e08c76
15 files changed
+10123
No files matched your search
@@ -0,0 +1,184 @@
|
||||
"""Serial, shared-slot prediction followed by the original paired scoring barrier."""
|
||||
|
||||
import fcntl
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_h1_worker_v2 import save, sha
|
||||
from phase2_home_freeze_io import freeze, verify
|
||||
from phase3_shared_capacity import alive, process_identity, release, transfer
|
||||
from r4bc_guard import supervise
|
||||
from track_a_postfit_common import prediction_identity
|
||||
from track_a_postfit_rebind_common import RECOVERY, SPLITS, selected_training, terminal
|
||||
from track_a_postfit_shared_capacity import acquire, register
|
||||
from track_a_postfit_shared_common import ROOT, SETUP, budget, config, read
|
||||
|
||||
|
||||
def stop_guard(guard):
|
||||
if guard.poll() is not None:
|
||||
return
|
||||
guard.terminate()
|
||||
try:
|
||||
guard.wait(timeout=45)
|
||||
except subprocess.TimeoutExpired:
|
||||
guard.kill()
|
||||
guard.wait()
|
||||
|
||||
|
||||
def predict_fold(root, job, reservation, queue_identity, wait_seconds, resource_check):
|
||||
folder = root/'jobs'/str(job['id'])
|
||||
token = reservation['token']
|
||||
guard = guard_id = None
|
||||
transferred = False
|
||||
try:
|
||||
folder.mkdir(exist_ok=False)
|
||||
path = folder/'assignment.json'
|
||||
save(path, dict(id=job['id'], contract_sha256=sha(root/'contract.json'), token=token,
|
||||
gpu_uuid=reservation['gpu_uuid'], worker_cpus=reservation['worker_cpus'],
|
||||
queue_identity=queue_identity, pre_dispatch_wait_seconds=wait_seconds))
|
||||
with (folder/'guard.log').open('x') as log:
|
||||
guard = subprocess.Popen([sys.executable, 'tools/track_a_postfit_shared_guard.py',
|
||||
str(root), str(job['id']), sha(path)],
|
||||
stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
guard_id = process_identity(guard.pid)
|
||||
transfer(token, guard_id)
|
||||
transferred = True
|
||||
save(folder/'lease-ready.json', dict(token=token, owner=guard_id))
|
||||
while guard.poll() is None:
|
||||
resource_check()
|
||||
time.sleep(0.5)
|
||||
receipt = read(folder/'complete.json')
|
||||
if (guard.returncode != 0 or receipt['status'] != 'PASS'
|
||||
or receipt['gpu_seconds_charged'] > job['gpu_seconds'] or receipt['monitor_policy']['failed']):
|
||||
raise RuntimeError(f'Shared prediction guard rejected fold {job["id"]}')
|
||||
return receipt
|
||||
finally:
|
||||
if guard is not None:
|
||||
stop_guard(guard)
|
||||
worker = folder/'prediction-records/worker-identity.json'
|
||||
if worker.exists() and alive(read(worker)):
|
||||
raise RuntimeError('Prediction worker outlived guard; retained own lease')
|
||||
release(token, guard_id if transferred else None)
|
||||
|
||||
|
||||
def run(root):
|
||||
gpu_clock = time.monotonic()
|
||||
started_cpu = time.process_time()
|
||||
initial_children = resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
c, tc, _ = config(root)
|
||||
validate_review(root)
|
||||
adopted = terminal(tc)
|
||||
if adopted != c['training_evidence']:
|
||||
raise ValueError('Mixed-source training evidence changed')
|
||||
prior = adopted['training_gpu_seconds_once']
|
||||
charged_cpu = 0.
|
||||
frozen_gpu = None
|
||||
attempted = 0
|
||||
records = []
|
||||
stop_requests = []
|
||||
original_affinity = os.sched_getaffinity(0)
|
||||
|
||||
def gpu():
|
||||
return (SETUP + attempted*SETUP + time.monotonic()-gpu_clock
|
||||
if frozen_gpu is None else frozen_gpu)
|
||||
|
||||
def cpu():
|
||||
children = resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
child_time = children.ru_utime + children.ru_stime - initial_children.ru_utime - initial_children.ru_stime
|
||||
return 180 + time.process_time() - started_cpu + max(charged_cpu, child_time)
|
||||
|
||||
def resource_check(pending):
|
||||
if stop_requests:
|
||||
raise RuntimeError(stop_requests[0])
|
||||
budget(c, gpu(), cpu(), pending)
|
||||
|
||||
with (root/'run.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():
|
||||
raise FileExistsError('One Track A shared postfit, zero retries')
|
||||
identity = process_identity(os.getpid())
|
||||
digest = sha(root/'contract.json')
|
||||
register(root, digest, c)
|
||||
os.sched_setaffinity(0, c['coordinator_cpus'])
|
||||
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
signal.signal(sig, lambda n, f: stop_requests.append(f'COORDINATOR_SIGNAL_{n}'))
|
||||
cpu_counters = dict(self_process_time=started_cpu, children_user=initial_children.ru_utime,
|
||||
children_system=initial_children.ru_stime)
|
||||
save(root/'started.json', dict(identity=identity, unix=time.time(), contract_sha256=digest,
|
||||
review_sha256=sha(root/'review.json'), training_gpu_seconds_once=prior, shared_slot_limit=1,
|
||||
postfit_gpu_cap=c['postfit_gpu_cap'], track_a_gpu_cap=c['gpu_cap'],
|
||||
cpu_start_counters=cpu_counters, cpu_setup_allowance=180, new_fits=0, new_decodes=0))
|
||||
try:
|
||||
save(root/'models-adopted.json', adopted)
|
||||
train = Path(c['training_root'])
|
||||
freeze(root, 'models', [root/'contract.json', root/'models-adopted.json', train/'complete.json',
|
||||
RECOVERY/'complete.json', RECOVERY/'available-models-incomplete.json',
|
||||
RECOVERY/'incomplete-models-git-freeze.json', RECOVERY/'terminal-git-freeze.json',
|
||||
*[Path(m['path']) for m in adopted['models']],
|
||||
*[Path(m['standardization']) for m in adopted['models']]])
|
||||
entries = []
|
||||
(root/'jobs').mkdir(exist_ok=False)
|
||||
jobs = c['predictions']
|
||||
for pos, job in enumerate(jobs):
|
||||
wait_started = time.monotonic()
|
||||
while True:
|
||||
resource_check(jobs[pos:])
|
||||
lease, observation = acquire(root, job['id'], digest, original_affinity)
|
||||
if lease is not None:
|
||||
break
|
||||
save(root/'latest-admission.json', observation)
|
||||
time.sleep(1)
|
||||
attempted += 1
|
||||
receipt = predict_fold(root, job, lease, identity, time.monotonic()-wait_started,
|
||||
lambda: resource_check(jobs[pos+1:]))
|
||||
charged_cpu += receipt['worker']['cpu_seconds']
|
||||
records.append(dict(mode='predict', split=job['id'], gpu_uuid=lease['gpu_uuid'], guard=receipt))
|
||||
save(root/'resource-progress.json', dict(records=records, prediction_gpu_seconds=gpu(),
|
||||
postfit_cpu_seconds=cpu(), cumulative_track_a_gpu_seconds=prior+gpu()))
|
||||
entries.extend(read(root/'jobs'/str(job['id'])/'prediction-records/manifest.json')['entries'])
|
||||
prediction_identity(root, c, selected_training(tc), entries)
|
||||
save(root/'predictions-manifest.json', dict(entries=entries, A_predictions_reused=c['expected_pair_visits'],
|
||||
evaluated_splits=list(SPLITS), excluded_split=c['excluded_split'], new_fits=0, targets_read=False))
|
||||
freeze(root, 'predictions', [root/'contract.json', root/'models-adopted.json',
|
||||
root/'predictions-manifest.json', root/'resource-progress.json', *map(Path,c['code_sha256'])])
|
||||
resource_check([])
|
||||
frozen_gpu = gpu()
|
||||
validate_review(root)
|
||||
verify(root, 'models')
|
||||
verify(root, 'predictions')
|
||||
score = supervise([sys.executable, 'tools/track_a_postfit_shared_worker.py', 'score', str(root)],
|
||||
root/'score-guard', dict(cpu_seconds=900, wall_seconds=1200,
|
||||
rss_bytes=16*2**30, disk_bytes=20*2**30))
|
||||
charged_cpu += score['cpu_seconds']
|
||||
records.append(dict(mode='score', guard=score))
|
||||
if score['status'] != 'PASS' or cpu()+120 > c['postfit_cpu_cap']:
|
||||
raise RuntimeError('Original Track A scoring cap or guard')
|
||||
save(root/'complete.json', dict(status='COMPLETE_FOUR_OF_FIVE', evaluated_splits=list(SPLITS),
|
||||
excluded_split=c['excluded_split'], records=records, prediction_gpu_seconds=gpu(),
|
||||
postfit_cpu_seconds=cpu(), training_gpu_seconds_once=prior,
|
||||
cumulative_track_a_gpu_seconds=prior+gpu(), track_a_gpu_cap=c['gpu_cap'],
|
||||
new_fits=0, new_decodes=0, no_retry=True))
|
||||
freeze(root, 'terminal', [root/'contract.json', root/'complete.json', root/'score/result.json', root/'review.json'])
|
||||
within = cpu()<=c['postfit_cpu_cap'] and gpu()<=c['postfit_gpu_cap'] and prior+gpu()<=c['gpu_cap']
|
||||
save(root/'post-terminal-cost.json', dict(postfit_cpu_seconds=cpu(), prediction_gpu_seconds=gpu(),
|
||||
cumulative_track_a_gpu_seconds=prior+gpu(), within_cap=within, cpu_start_counters=cpu_counters))
|
||||
if not within:
|
||||
raise RuntimeError('Shared Track A final resource cap')
|
||||
except BaseException as exc:
|
||||
if (root/'complete.json').exists():
|
||||
save(root/'pre-failure-complete.json', read(root/'complete.json'))
|
||||
save(root/'complete.json', dict(status='STOPPED', reason=f'{type(exc).__name__}: {exc}',
|
||||
records=records, postfit_cpu_seconds=cpu(), prediction_gpu_seconds=gpu(),
|
||||
training_gpu_seconds_once=prior, cumulative_track_a_gpu_seconds=prior+gpu(), new_fits=0, no_retry=True))
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(ROOT)
|
||||
Reference in new issue
Block a user