Files
MeltMind/tools/track_a_postfit_shared_capacity.py
T

98 lines
4.0 KiB
Python

"""One Track A slot in the same registry and admission policy used by item b."""
import os
from pathlib import Path
import time
import uuid
import psutil
from phase3_shared_capacity import GIB, GPUS, admission, alive, descendants, host_snapshot, process_identity, registry
from track_a_postfit_shared_common import ROOT, read
KEY = 'track_a_postfit_shared'
OLDER = ('phase3-item-c-2026-11-18', 'phase3-item-b-shared-2026-11-19')
def older_shared_pending(base, now):
pending = []
for name in OLDER:
root = base/name
if (root/'complete.json').exists():
read(root/'complete.json')
continue
live = read(root/'live-progress.json')
if now-live['unix'] > 120:
raise RuntimeError(f'Stale earlier queue receipt {name}')
if live['pending']:
pending.append(name)
return pending
def available(data, allowed_cpus):
pending = older_shared_pending(ROOT.parent, time.time())
if pending:
return None, dict(reason='OLDER_SHARED_FOLDS_PENDING', queues=pending)
leases = []
for lease in data['leases'].values():
if not alive(lease['owner']):
return None, dict(reason='STALE_LEASE_AWAIT_OWNER_COLLECTION')
if lease['state'] == 'RESERVED':
leases.append(dict(lease, observed_pids=[], observed_rss=0))
continue
if lease['state'] != 'GUARD_BOUND':
raise ValueError('Unknown lease state')
try:
ids = descendants(lease['owner'])
except (psutil.NoSuchProcess, RuntimeError):
return None, dict(reason='LEASE_EXIT_RACE_AWAIT_OWNER_COLLECTION')
rss = 0
for pid in ids:
try:
rss += psutil.Process(pid).memory_info().rss
except psutil.NoSuchProcess:
pass
leases.append(dict(lease, observed_pids=sorted(ids), observed_rss=rss))
snapshot = host_snapshot()
if snapshot['old_pending']:
return None, dict(reason='OLDER_FIXED_FOLDS_PENDING', snapshot=snapshot)
occupied = {cpu for lease in leases for cpu in lease['worker_cpus']}
cpus = [cpu for cpu in range(16,26) if cpu in allowed_cpus and cpu not in occupied]
if not cpus:
return None, dict(reason='CPU_RESERVATIONS', snapshot=snapshot)
decisions = {}
for gpu in GPUS[:3]:
decisions[gpu] = admission(snapshot, leases, gpu)
if decisions[gpu][0]:
return (gpu, cpus[0]), dict(reason='ADMITTED', snapshot=snapshot, decisions=decisions)
return None, dict(reason='NO_CAPACITY', snapshot=snapshot, decisions=decisions)
def register(root, digest, contract):
with registry() as data:
if KEY in data['entries']:
raise FileExistsError('Only one shared Track A attempt')
data['entries'][KEY] = dict(root=str(root), contract_sha256=digest,
owner=process_identity(os.getpid()), cap=contract['postfit_gpu_cap'],
track_a_total_cap=contract['gpu_cap'], started_unix=time.time())
def acquire(root, fold, digest, allowed_cpus):
with registry() as data:
queue = data['entries'][KEY]
owner = process_identity(os.getpid())
if queue['root'] != str(root) or queue['contract_sha256'] != digest or queue['owner'] != owner:
raise PermissionError('Only the exact shared Track A queue may reserve')
if any(lease['root'] == str(root) for lease in data['leases'].values()):
raise RuntimeError('Track A already owns one shared slot')
selected, observation = available(data, allowed_cpus)
if selected is None:
return None, observation
gpu, cpu = selected
token = uuid.uuid4().hex
lease = dict(token=token, state='RESERVED', root=str(root), fold=fold,
contract_sha256=digest, gpu_uuid=gpu, worker_cpus=[cpu], owner=owner,
queue_owner=owner, vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time())
data['leases'][token] = lease
return lease, observation