Files
MeltMind/tools/phase3_shared_capacity.py
T

226 lines
10 KiB
Python

"""New-entry admission only; never changes or kills another entry's worker.
All participating future entries must use this same registry/lock. Unknown
processes consume real slots and a conservative reservation; stale leases fail
closed. Old fixed queues do not join this registry and remain untouched.
"""
from contextlib import contextmanager
import fcntl
import json
import math
import os
from pathlib import Path
import subprocess
import time
import uuid
import psutil
from phase2_h1_worker_v2 import save
GIB = 2**30
GPUS = ['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c', 'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76', 'GPU-8b71b032-1b75-2891-935c-ce106a53034a']
SHARED = Path('/home/frank14f/MeltMind-runs/phase3-shared-capacity-2026-11-18')
OLD = [Path('/home/frank14f/MeltMind-runs')/s for s in
['phase3-seed1-v2-2026-11-18', 'track-a-recovery-2026-11-18']]
PHASE_RESERVES = dict(item1=72421.31599938683, seed1=167200., c=205200., item1b=88560., b=178200.)
def process_identity(pid):
p = psutil.Process(pid)
return dict(pid=pid, created=p.create_time())
def alive(identity):
try:
p = psutil.Process(identity['pid'])
return p.create_time() == identity['created'] and p.status() != psutil.STATUS_ZOMBIE
except psutil.NoSuchProcess:
return False
def descendants(identity):
if not alive(identity):
raise RuntimeError('Stale lease: explicit reconciliation required')
return {identity['pid']} | {p.pid for p in psutil.Process(identity['pid']).children(recursive=True)}
def old_pending():
for root in OLD:
p = root/'complete.json'
if p.exists():
json.loads(p.read_text()) # corrupt terminal fails closed
continue
d = json.loads((root/'live-progress.json').read_text())
if time.time()-d['unix'] > 120:
raise RuntimeError('Stale old queue snapshot')
if d['pending']:
return True
return False
def host_snapshot():
def query(arg):
return subprocess.check_output(['nvidia-smi', arg, '--format=csv,noheader,nounits'], text=True, timeout=10)
devices = {}
for line in query('--query-gpu=uuid,memory.total,memory.used').strip().splitlines():
u, total, used = [s.strip() for s in line.split(',')]
devices[u] = dict(total=int(total)*2**20, used=int(used)*2**20)
processes = []
for line in query('--query-compute-apps=gpu_uuid,pid,used_gpu_memory').strip().splitlines():
if not line.strip():
continue
u, pid, mem = [s.strip() for s in line.split(',')]
processes.append(dict(gpu_uuid=u, pid=int(pid), vram=int(mem)*2**20))
if set(devices) != set(GPUS) or len({(p['gpu_uuid'],p['pid']) for p in processes}) != len(processes):
raise ValueError('Unknown GPU topology or duplicate compute PID')
vm = psutil.virtual_memory()
return dict(devices=devices, processes=processes, ram_total=vm.total, ram_available=vm.available,
cpu_percent=psutil.cpu_percent(interval=.1), old_pending=old_pending(), unix=time.time())
def admission(snapshot, leases, gpu, *, vram_reserve=4*GIB, rss_reserve=4*GIB):
"""Untransferred reservations never receive process or memory credits."""
observed=[];claimed=set()
for lease in leases:
if lease['state']=='RESERVED':
observed.append(dict(lease,observed_pids=[],observed_rss=0))
elif lease['state']=='GUARD_BOUND':
pids=set(lease['observed_pids'])
if claimed & pids:
return False, 'LEASE_PID_OWNERSHIP_OVERLAP'
claimed.update(pids)
observed.append(lease)
else:
raise ValueError('Unknown lease state')
leases=observed
if gpu not in GPUS:
raise ValueError('Unknown GPU')
if gpu == GPUS[0] and snapshot['old_pending']:
return False, 'GPU0_RESERVED_FOR_OLD_PENDING'
processes = snapshot['processes']
card = [p for p in processes if p['gpu_uuid'] == gpu]
same = [l for l in leases if l['gpu_uuid'] == gpu]
real = {p['pid'] for p in card}
occupied = len(real) + sum(not (real & set(l['observed_pids'])) for l in same)
if occupied >= (1 if gpu == GPUS[3] else 3):
return False, 'GPU_SLOT_LIMIT'
owned = set().union(*(set(l['observed_pids']) for l in same)) if same else set()
# Charge the unseen part of each reserved peak; do not double-count visible memory.
missing = sum(max(0, l['vram_reserve']-sum(p['vram'] for p in card if p['pid'] in l['observed_pids'])) for l in same)
missing += sum(max(0, 4*GIB-p['vram']) for p in card if p['pid'] not in owned)
d = snapshot['devices'][gpu]
if d['used'] + missing + vram_reserve > .8*d['total']:
return False, 'GPU_HEADROOM'
ram_missing = sum(max(0,l['rss_reserve']-l['observed_rss']) for l in leases)
# Extra8GiB preserves future original folds; visible old RSS already in available.
if snapshot['ram_available']-ram_missing-rss_reserve-8*GIB < .2*snapshot['ram_total']:
return False, 'HOST_RAM_HEADROOM'
if snapshot['cpu_percent'] > 80:
return False, 'HOST_CPU_HEADROOM'
return True, 'ADMITTED'
@contextmanager
def registry():
SHARED.mkdir(exist_ok=True)
with (SHARED/'capacity.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
p = SHARED/'registry.json'
d = json.loads(p.read_text()) if p.exists() else dict(version=2, entries={}, leases={})
if d['version'] != 2:
raise ValueError('Shared registry version')
yield d
save(p, d)
def register_entry(root, digest):
with registry() as d:
if 'c' in d['entries']:
raise FileExistsError('Sole item c entry already registered; no resume')
if sum(PHASE_RESERVES.values()) > 720000:
raise RuntimeError('Whole phase3 reserved cap')
d['entries']['c'] = dict(root=str(root), contract_sha256=digest, owner=process_identity(os.getpid()),
cap=205200, phase_reservations=PHASE_RESERVES, started_unix=time.time())
def acquire(root, fold, digest):
with registry() as d:
entry = d['entries']['c']
if entry['root'] != str(root) or entry['contract_sha256'] != digest or not alive(entry['owner']):
raise PermissionError('Entry owner/identity')
leases = []
for l in d['leases'].values():
# Normal guard exit can race this observation. Block admission until
# its owning coordinator collects it; never reclaim or kill peers.
if not alive(l['owner']):
return None, dict(reason='STALE_LEASE_AWAIT_OWNER_COLLECTION',token=l['token'])
if l['state']=='RESERVED':
# owner is a queue with other children, not this job's guard.
# One independent slot and full VRAM/RSS remain reserved.
leases.append(dict(l,observed_pids=[],observed_rss=0))
continue
if l['state']!='GUARD_BOUND':
raise ValueError('Unknown lease state')
try:
ids = descendants(l['owner'])
except (psutil.NoSuchProcess,RuntimeError):
return None, dict(reason='LEASE_EXIT_RACE_AWAIT_OWNER_COLLECTION',token=l['token'])
rss=0
for pid in ids:
try:rss+=psutil.Process(pid).memory_info().rss
except psutil.NoSuchProcess:pass
leases.append(dict(l, observed_pids=list(ids), observed_rss=rss))
s = host_snapshot() # fresh inside inter-entry lock, before making reservation
used_cpu = {x for l in leases for x in l['worker_cpus']}
cpus = [x for x in range(16,26) if x not in used_cpu and x in os.sched_getaffinity(0)]
if not cpus:
return None, dict(reason='CPU_RESERVATIONS', snapshot=s)
for gpu in GPUS[1:3]:
ok, reason = admission(s, leases, gpu)
if not ok:
continue
token = uuid.uuid4().hex
queue_owner=process_identity(os.getpid())
l = dict(token=token, state='RESERVED',root=str(root), fold=fold, contract_sha256=digest, gpu_uuid=gpu,
worker_cpus=[cpus[0]], owner=queue_owner,queue_owner=queue_owner,
vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time())
d['leases'][token] = l
return l, dict(reason='ADMITTED', snapshot=s)
return None, dict(reason='NO_CAPACITY', snapshot=s)
def transfer(token, owner):
with registry() as d:
l = d['leases'][token]
caller=process_identity(os.getpid())
if l['state']!='RESERVED' or l['owner']!=caller or l['queue_owner']!=caller or not alive(owner):
raise PermissionError('Only queue transfers its own lease to live guard')
if owner==caller or psutil.Process(owner['pid']).ppid()!=caller['pid']:
raise PermissionError('Guard must be this queue direct child, never queue itself')
if any(x['state']=='GUARD_BOUND' and x['owner']==owner for x in d['leases'].values()):
raise PermissionError('Guard already bound to another lease')
l.update(owner=owner,state='GUARD_BOUND')
def release(token, guard_identity=None):
with registry() as d:
l = d['leases'][token]
if l['queue_owner']!=process_identity(os.getpid()):
raise PermissionError('Only original queue releases its lease')
if guard_identity is None:
if l['state']!='RESERVED' or l['owner']!=l['queue_owner']:
raise PermissionError('Only unused own lease')
elif l['state']!='GUARD_BOUND' or l['owner'] != guard_identity or alive(guard_identity):
raise PermissionError('Guard must be reaped before release')
del d['leases'][token]
def validate_lease(token, root, fold, digest):
with registry() as d:
l = d['leases'][token]
if (l['root'], l['fold'], l['contract_sha256']) != (str(root), fold, digest):
raise PermissionError('Lease identity')
if l['state']!='GUARD_BOUND' or l['owner'] != process_identity(os.getpid()):
raise PermissionError('Lease not transferred to this guard')
return dict(l)