Files
MeltMind/tools/phase3_shared_capacity.py
T

198 lines
8.5 KiB
Python

"""New-entry admission only; never changes or kills another entry's worker.
All participating future entries must use this same registry/lock. Unknown
processes consume real slots and a conservative reservation; stale leases fail
closed. Old fixed queues do not join this registry and remain untouched.
"""
from contextlib import contextmanager
import fcntl
import json
import math
import os
from pathlib import Path
import subprocess
import time
import uuid
import psutil
from phase2_h1_worker_v2 import save
GIB = 2**30
GPUS = ['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c', 'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76', 'GPU-8b71b032-1b75-2891-935c-ce106a53034a']
SHARED = Path('/home/frank14f/MeltMind-runs/phase3-shared-capacity-2026-11-18')
OLD = [Path('/home/frank14f/MeltMind-runs')/s for s in
['phase3-seed1-v2-2026-11-18', 'track-a-recovery-2026-11-18']]
PHASE_RESERVES = dict(item1=72421.31599938683, seed1=167200., c=205200., item1b=88560., b=178200.)
def process_identity(pid):
p = psutil.Process(pid)
return dict(pid=pid, created=p.create_time())
def alive(identity):
try:
p = psutil.Process(identity['pid'])
return p.create_time() == identity['created'] and p.status() != psutil.STATUS_ZOMBIE
except psutil.NoSuchProcess:
return False
def descendants(identity):
if not alive(identity):
raise RuntimeError('Stale lease: explicit reconciliation required')
return {identity['pid']} | {p.pid for p in psutil.Process(identity['pid']).children(recursive=True)}
def old_pending():
for root in OLD:
p = root/'complete.json'
if p.exists():
json.loads(p.read_text()) # corrupt terminal fails closed
continue
d = json.loads((root/'live-progress.json').read_text())
if time.time()-d['unix'] > 120:
raise RuntimeError('Stale old queue snapshot')
if d['pending']:
return True
return False
def host_snapshot():
def query(arg):
return subprocess.check_output(['nvidia-smi', arg, '--format=csv,noheader,nounits'], text=True, timeout=10)
devices = {}
for line in query('--query-gpu=uuid,memory.total,memory.used').strip().splitlines():
u, total, used = [s.strip() for s in line.split(',')]
devices[u] = dict(total=int(total)*2**20, used=int(used)*2**20)
processes = []
for line in query('--query-compute-apps=gpu_uuid,pid,used_gpu_memory').strip().splitlines():
if not line.strip():
continue
u, pid, mem = [s.strip() for s in line.split(',')]
processes.append(dict(gpu_uuid=u, pid=int(pid), vram=int(mem)*2**20))
if set(devices) != set(GPUS) or len({(p['gpu_uuid'],p['pid']) for p in processes}) != len(processes):
raise ValueError('Unknown GPU topology or duplicate compute PID')
vm = psutil.virtual_memory()
return dict(devices=devices, processes=processes, ram_total=vm.total, ram_available=vm.available,
cpu_percent=psutil.cpu_percent(interval=.1), old_pending=old_pending(), unix=time.time())
def admission(snapshot, leases, gpu, *, vram_reserve=4*GIB, rss_reserve=4*GIB):
"""Pure admission rule. leases include observed descendant PIDs and RSS."""
if gpu not in GPUS:
raise ValueError('Unknown GPU')
if gpu == GPUS[0] and snapshot['old_pending']:
return False, 'GPU0_RESERVED_FOR_OLD_PENDING'
processes = snapshot['processes']
card = [p for p in processes if p['gpu_uuid'] == gpu]
same = [l for l in leases if l['gpu_uuid'] == gpu]
real = {p['pid'] for p in card}
occupied = len(real) + sum(not (real & set(l['observed_pids'])) for l in same)
if occupied >= (1 if gpu == GPUS[3] else 3):
return False, 'GPU_SLOT_LIMIT'
owned = set().union(*(set(l['observed_pids']) for l in same)) if same else set()
# Charge the unseen part of each reserved peak; do not double-count visible memory.
missing = sum(max(0, l['vram_reserve']-sum(p['vram'] for p in card if p['pid'] in l['observed_pids'])) for l in same)
missing += sum(max(0, 4*GIB-p['vram']) for p in card if p['pid'] not in owned)
d = snapshot['devices'][gpu]
if d['used'] + missing + vram_reserve > .8*d['total']:
return False, 'GPU_HEADROOM'
ram_missing = sum(max(0,l['rss_reserve']-l['observed_rss']) for l in leases)
# Extra8GiB preserves future original folds; visible old RSS already in available.
if snapshot['ram_available']-ram_missing-rss_reserve-8*GIB < .2*snapshot['ram_total']:
return False, 'HOST_RAM_HEADROOM'
if snapshot['cpu_percent'] > 80:
return False, 'HOST_CPU_HEADROOM'
return True, 'ADMITTED'
@contextmanager
def registry():
SHARED.mkdir(exist_ok=True)
with (SHARED/'capacity.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
p = SHARED/'registry.json'
d = json.loads(p.read_text()) if p.exists() else dict(version=1, entries={}, leases={})
if d['version'] != 1:
raise ValueError('Shared registry version')
yield d
save(p, d)
def register_entry(root, digest):
with registry() as d:
if 'c' in d['entries']:
raise FileExistsError('Sole item c entry already registered; no resume')
if sum(PHASE_RESERVES.values()) > 720000:
raise RuntimeError('Whole phase3 reserved cap')
d['entries']['c'] = dict(root=str(root), contract_sha256=digest, owner=process_identity(os.getpid()),
cap=205200, phase_reservations=PHASE_RESERVES, started_unix=time.time())
def acquire(root, fold, digest):
with registry() as d:
entry = d['entries']['c']
if entry['root'] != str(root) or entry['contract_sha256'] != digest or not alive(entry['owner']):
raise PermissionError('Entry owner/identity')
leases = []
for l in d['leases'].values():
# Normal guard exit can race this observation. Block admission until
# its owning coordinator collects it; never reclaim or kill peers.
if not alive(l['owner']):
return None, dict(reason='STALE_LEASE_AWAIT_OWNER_COLLECTION',token=l['token'])
try:
ids = descendants(l['owner'])
except (psutil.NoSuchProcess,RuntimeError):
return None, dict(reason='LEASE_EXIT_RACE_AWAIT_OWNER_COLLECTION',token=l['token'])
rss=0
for pid in ids:
try:rss+=psutil.Process(pid).memory_info().rss
except psutil.NoSuchProcess:pass
leases.append(dict(l, observed_pids=list(ids), observed_rss=rss))
s = host_snapshot() # fresh inside inter-entry lock, before making reservation
used_cpu = {x for l in leases for x in l['worker_cpus']}
cpus = [x for x in range(16,26) if x not in used_cpu and x in os.sched_getaffinity(0)]
if not cpus:
return None, dict(reason='CPU_RESERVATIONS', snapshot=s)
for gpu in GPUS[1:3]:
ok, reason = admission(s, leases, gpu)
if not ok:
continue
token = uuid.uuid4().hex
l = dict(token=token, root=str(root), fold=fold, contract_sha256=digest, gpu_uuid=gpu,
worker_cpus=[cpus[0]], owner=process_identity(os.getpid()),
vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time())
d['leases'][token] = l
return l, dict(reason='ADMITTED', snapshot=s)
return None, dict(reason='NO_CAPACITY', snapshot=s)
def transfer(token, owner):
with registry() as d:
l = d['leases'][token]
if l['owner']['pid'] != os.getpid() or not alive(owner):
raise PermissionError('Only queue transfers its own lease to live guard')
l['owner'] = owner
def release(token, guard_identity=None):
with registry() as d:
l = d['leases'][token]
if guard_identity is None:
if l['owner']['pid'] != os.getpid():
raise PermissionError('Only unused own lease')
elif l['owner'] != guard_identity or alive(guard_identity):
raise PermissionError('Guard must be reaped before release')
del d['leases'][token]
def validate_lease(token, root, fold, digest):
with registry() as d:
l = d['leases'][token]
if (l['root'], l['fold'], l['contract_sha256']) != (str(root), fold, digest):
raise PermissionError('Lease identity')
if l['owner'] != process_identity(os.getpid()):
raise PermissionError('Lease not transferred to this guard')
return dict(l)