98 lines
4.0 KiB
Python
98 lines
4.0 KiB
Python
"""One Track A slot in the same registry and admission policy used by item b."""
|
|
|
|
import os
|
|
from pathlib import Path
|
|
import time
|
|
import uuid
|
|
|
|
import psutil
|
|
from phase3_shared_capacity import GIB, GPUS, admission, alive, descendants, host_snapshot, process_identity, registry
|
|
from track_a_postfit_shared_common import ROOT, read
|
|
|
|
|
|
KEY = 'track_a_postfit_shared'
|
|
OLDER = ('phase3-item-c-2026-11-18', 'phase3-item-b-shared-2026-11-19')
|
|
|
|
|
|
def older_shared_pending(base, now):
|
|
pending = []
|
|
for name in OLDER:
|
|
root = base/name
|
|
if (root/'complete.json').exists():
|
|
read(root/'complete.json')
|
|
continue
|
|
live = read(root/'live-progress.json')
|
|
if now-live['unix'] > 120:
|
|
raise RuntimeError(f'Stale earlier queue receipt {name}')
|
|
if live['pending']:
|
|
pending.append(name)
|
|
return pending
|
|
|
|
|
|
def available(data, allowed_cpus):
|
|
pending = older_shared_pending(ROOT.parent, time.time())
|
|
if pending:
|
|
return None, dict(reason='OLDER_SHARED_FOLDS_PENDING', queues=pending)
|
|
leases = []
|
|
for lease in data['leases'].values():
|
|
if not alive(lease['owner']):
|
|
return None, dict(reason='STALE_LEASE_AWAIT_OWNER_COLLECTION')
|
|
if lease['state'] == 'RESERVED':
|
|
leases.append(dict(lease, observed_pids=[], observed_rss=0))
|
|
continue
|
|
if lease['state'] != 'GUARD_BOUND':
|
|
raise ValueError('Unknown lease state')
|
|
try:
|
|
ids = descendants(lease['owner'])
|
|
except (psutil.NoSuchProcess, RuntimeError):
|
|
return None, dict(reason='LEASE_EXIT_RACE_AWAIT_OWNER_COLLECTION')
|
|
rss = 0
|
|
for pid in ids:
|
|
try:
|
|
rss += psutil.Process(pid).memory_info().rss
|
|
except psutil.NoSuchProcess:
|
|
pass
|
|
leases.append(dict(lease, observed_pids=sorted(ids), observed_rss=rss))
|
|
snapshot = host_snapshot()
|
|
if snapshot['old_pending']:
|
|
return None, dict(reason='OLDER_FIXED_FOLDS_PENDING', snapshot=snapshot)
|
|
occupied = {cpu for lease in leases for cpu in lease['worker_cpus']}
|
|
cpus = [cpu for cpu in range(16,26) if cpu in allowed_cpus and cpu not in occupied]
|
|
if not cpus:
|
|
return None, dict(reason='CPU_RESERVATIONS', snapshot=snapshot)
|
|
decisions = {}
|
|
for gpu in GPUS[:3]:
|
|
decisions[gpu] = admission(snapshot, leases, gpu)
|
|
if decisions[gpu][0]:
|
|
return (gpu, cpus[0]), dict(reason='ADMITTED', snapshot=snapshot, decisions=decisions)
|
|
return None, dict(reason='NO_CAPACITY', snapshot=snapshot, decisions=decisions)
|
|
|
|
|
|
def register(root, digest, contract):
|
|
with registry() as data:
|
|
if KEY in data['entries']:
|
|
raise FileExistsError('Only one shared Track A attempt')
|
|
data['entries'][KEY] = dict(root=str(root), contract_sha256=digest,
|
|
owner=process_identity(os.getpid()), cap=contract['postfit_gpu_cap'],
|
|
track_a_total_cap=contract['gpu_cap'], started_unix=time.time())
|
|
|
|
|
|
def acquire(root, fold, digest, allowed_cpus):
|
|
with registry() as data:
|
|
queue = data['entries'][KEY]
|
|
owner = process_identity(os.getpid())
|
|
if queue['root'] != str(root) or queue['contract_sha256'] != digest or queue['owner'] != owner:
|
|
raise PermissionError('Only the exact shared Track A queue may reserve')
|
|
if any(lease['root'] == str(root) for lease in data['leases'].values()):
|
|
raise RuntimeError('Track A already owns one shared slot')
|
|
selected, observation = available(data, allowed_cpus)
|
|
if selected is None:
|
|
return None, observation
|
|
gpu, cpu = selected
|
|
token = uuid.uuid4().hex
|
|
lease = dict(token=token, state='RESERVED', root=str(root), fold=fold,
|
|
contract_sha256=digest, gpu_uuid=gpu, worker_cpus=[cpu], owner=owner,
|
|
queue_owner=owner, vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time())
|
|
data['leases'][token] = lease
|
|
return lease, observation
|