226 lines
10 KiB
Python
226 lines
10 KiB
Python
"""New-entry admission only; never changes or kills another entry's worker.
|
|
|
|
All participating future entries must use this same registry/lock. Unknown
|
|
processes consume real slots and a conservative reservation; stale leases fail
|
|
closed. Old fixed queues do not join this registry and remain untouched.
|
|
"""
|
|
from contextlib import contextmanager
|
|
import fcntl
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
import psutil
|
|
from phase2_h1_worker_v2 import save
|
|
|
|
GIB = 2**30
|
|
GPUS = ['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c', 'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
|
|
'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76', 'GPU-8b71b032-1b75-2891-935c-ce106a53034a']
|
|
SHARED = Path('/home/frank14f/MeltMind-runs/phase3-shared-capacity-2026-11-18')
|
|
OLD = [Path('/home/frank14f/MeltMind-runs')/s for s in
|
|
['phase3-seed1-v2-2026-11-18', 'track-a-recovery-2026-11-18']]
|
|
PHASE_RESERVES = dict(item1=72421.31599938683, seed1=167200., c=205200., item1b=88560., b=178200.)
|
|
|
|
|
|
def process_identity(pid):
|
|
p = psutil.Process(pid)
|
|
return dict(pid=pid, created=p.create_time())
|
|
|
|
|
|
def alive(identity):
|
|
try:
|
|
p = psutil.Process(identity['pid'])
|
|
return p.create_time() == identity['created'] and p.status() != psutil.STATUS_ZOMBIE
|
|
except psutil.NoSuchProcess:
|
|
return False
|
|
|
|
|
|
def descendants(identity):
|
|
if not alive(identity):
|
|
raise RuntimeError('Stale lease: explicit reconciliation required')
|
|
return {identity['pid']} | {p.pid for p in psutil.Process(identity['pid']).children(recursive=True)}
|
|
|
|
|
|
def old_pending():
|
|
for root in OLD:
|
|
p = root/'complete.json'
|
|
if p.exists():
|
|
json.loads(p.read_text()) # corrupt terminal fails closed
|
|
continue
|
|
d = json.loads((root/'live-progress.json').read_text())
|
|
if time.time()-d['unix'] > 120:
|
|
raise RuntimeError('Stale old queue snapshot')
|
|
if d['pending']:
|
|
return True
|
|
return False
|
|
|
|
|
|
def host_snapshot():
|
|
def query(arg):
|
|
return subprocess.check_output(['nvidia-smi', arg, '--format=csv,noheader,nounits'], text=True, timeout=10)
|
|
devices = {}
|
|
for line in query('--query-gpu=uuid,memory.total,memory.used').strip().splitlines():
|
|
u, total, used = [s.strip() for s in line.split(',')]
|
|
devices[u] = dict(total=int(total)*2**20, used=int(used)*2**20)
|
|
processes = []
|
|
for line in query('--query-compute-apps=gpu_uuid,pid,used_gpu_memory').strip().splitlines():
|
|
if not line.strip():
|
|
continue
|
|
u, pid, mem = [s.strip() for s in line.split(',')]
|
|
processes.append(dict(gpu_uuid=u, pid=int(pid), vram=int(mem)*2**20))
|
|
if set(devices) != set(GPUS) or len({(p['gpu_uuid'],p['pid']) for p in processes}) != len(processes):
|
|
raise ValueError('Unknown GPU topology or duplicate compute PID')
|
|
vm = psutil.virtual_memory()
|
|
return dict(devices=devices, processes=processes, ram_total=vm.total, ram_available=vm.available,
|
|
cpu_percent=psutil.cpu_percent(interval=.1), old_pending=old_pending(), unix=time.time())
|
|
|
|
|
|
def admission(snapshot, leases, gpu, *, vram_reserve=4*GIB, rss_reserve=4*GIB):
|
|
"""Untransferred reservations never receive process or memory credits."""
|
|
observed=[];claimed=set()
|
|
for lease in leases:
|
|
if lease['state']=='RESERVED':
|
|
observed.append(dict(lease,observed_pids=[],observed_rss=0))
|
|
elif lease['state']=='GUARD_BOUND':
|
|
pids=set(lease['observed_pids'])
|
|
if claimed & pids:
|
|
return False, 'LEASE_PID_OWNERSHIP_OVERLAP'
|
|
claimed.update(pids)
|
|
observed.append(lease)
|
|
else:
|
|
raise ValueError('Unknown lease state')
|
|
leases=observed
|
|
if gpu not in GPUS:
|
|
raise ValueError('Unknown GPU')
|
|
if gpu == GPUS[0] and snapshot['old_pending']:
|
|
return False, 'GPU0_RESERVED_FOR_OLD_PENDING'
|
|
processes = snapshot['processes']
|
|
card = [p for p in processes if p['gpu_uuid'] == gpu]
|
|
same = [l for l in leases if l['gpu_uuid'] == gpu]
|
|
real = {p['pid'] for p in card}
|
|
occupied = len(real) + sum(not (real & set(l['observed_pids'])) for l in same)
|
|
if occupied >= (1 if gpu == GPUS[3] else 3):
|
|
return False, 'GPU_SLOT_LIMIT'
|
|
owned = set().union(*(set(l['observed_pids']) for l in same)) if same else set()
|
|
# Charge the unseen part of each reserved peak; do not double-count visible memory.
|
|
missing = sum(max(0, l['vram_reserve']-sum(p['vram'] for p in card if p['pid'] in l['observed_pids'])) for l in same)
|
|
missing += sum(max(0, 4*GIB-p['vram']) for p in card if p['pid'] not in owned)
|
|
d = snapshot['devices'][gpu]
|
|
if d['used'] + missing + vram_reserve > .8*d['total']:
|
|
return False, 'GPU_HEADROOM'
|
|
ram_missing = sum(max(0,l['rss_reserve']-l['observed_rss']) for l in leases)
|
|
# Extra8GiB preserves future original folds; visible old RSS already in available.
|
|
if snapshot['ram_available']-ram_missing-rss_reserve-8*GIB < .2*snapshot['ram_total']:
|
|
return False, 'HOST_RAM_HEADROOM'
|
|
if snapshot['cpu_percent'] > 80:
|
|
return False, 'HOST_CPU_HEADROOM'
|
|
return True, 'ADMITTED'
|
|
|
|
|
|
@contextmanager
|
|
def registry():
|
|
SHARED.mkdir(exist_ok=True)
|
|
with (SHARED/'capacity.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX)
|
|
p = SHARED/'registry.json'
|
|
d = json.loads(p.read_text()) if p.exists() else dict(version=2, entries={}, leases={})
|
|
if d['version'] != 2:
|
|
raise ValueError('Shared registry version')
|
|
yield d
|
|
save(p, d)
|
|
|
|
|
|
def register_entry(root, digest):
|
|
with registry() as d:
|
|
if 'c' in d['entries']:
|
|
raise FileExistsError('Sole item c entry already registered; no resume')
|
|
if sum(PHASE_RESERVES.values()) > 720000:
|
|
raise RuntimeError('Whole phase3 reserved cap')
|
|
d['entries']['c'] = dict(root=str(root), contract_sha256=digest, owner=process_identity(os.getpid()),
|
|
cap=205200, phase_reservations=PHASE_RESERVES, started_unix=time.time())
|
|
|
|
|
|
def acquire(root, fold, digest):
|
|
with registry() as d:
|
|
entry = d['entries']['c']
|
|
if entry['root'] != str(root) or entry['contract_sha256'] != digest or not alive(entry['owner']):
|
|
raise PermissionError('Entry owner/identity')
|
|
leases = []
|
|
for l in d['leases'].values():
|
|
# Normal guard exit can race this observation. Block admission until
|
|
# its owning coordinator collects it; never reclaim or kill peers.
|
|
if not alive(l['owner']):
|
|
return None, dict(reason='STALE_LEASE_AWAIT_OWNER_COLLECTION',token=l['token'])
|
|
if l['state']=='RESERVED':
|
|
# owner is a queue with other children, not this job's guard.
|
|
# One independent slot and full VRAM/RSS remain reserved.
|
|
leases.append(dict(l,observed_pids=[],observed_rss=0))
|
|
continue
|
|
if l['state']!='GUARD_BOUND':
|
|
raise ValueError('Unknown lease state')
|
|
try:
|
|
ids = descendants(l['owner'])
|
|
except (psutil.NoSuchProcess,RuntimeError):
|
|
return None, dict(reason='LEASE_EXIT_RACE_AWAIT_OWNER_COLLECTION',token=l['token'])
|
|
rss=0
|
|
for pid in ids:
|
|
try:rss+=psutil.Process(pid).memory_info().rss
|
|
except psutil.NoSuchProcess:pass
|
|
leases.append(dict(l, observed_pids=list(ids), observed_rss=rss))
|
|
s = host_snapshot() # fresh inside inter-entry lock, before making reservation
|
|
used_cpu = {x for l in leases for x in l['worker_cpus']}
|
|
cpus = [x for x in range(16,26) if x not in used_cpu and x in os.sched_getaffinity(0)]
|
|
if not cpus:
|
|
return None, dict(reason='CPU_RESERVATIONS', snapshot=s)
|
|
for gpu in GPUS[1:3]:
|
|
ok, reason = admission(s, leases, gpu)
|
|
if not ok:
|
|
continue
|
|
token = uuid.uuid4().hex
|
|
queue_owner=process_identity(os.getpid())
|
|
l = dict(token=token, state='RESERVED',root=str(root), fold=fold, contract_sha256=digest, gpu_uuid=gpu,
|
|
worker_cpus=[cpus[0]], owner=queue_owner,queue_owner=queue_owner,
|
|
vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time())
|
|
d['leases'][token] = l
|
|
return l, dict(reason='ADMITTED', snapshot=s)
|
|
return None, dict(reason='NO_CAPACITY', snapshot=s)
|
|
|
|
|
|
def transfer(token, owner):
|
|
with registry() as d:
|
|
l = d['leases'][token]
|
|
caller=process_identity(os.getpid())
|
|
if l['state']!='RESERVED' or l['owner']!=caller or l['queue_owner']!=caller or not alive(owner):
|
|
raise PermissionError('Only queue transfers its own lease to live guard')
|
|
if owner==caller or psutil.Process(owner['pid']).ppid()!=caller['pid']:
|
|
raise PermissionError('Guard must be this queue direct child, never queue itself')
|
|
if any(x['state']=='GUARD_BOUND' and x['owner']==owner for x in d['leases'].values()):
|
|
raise PermissionError('Guard already bound to another lease')
|
|
l.update(owner=owner,state='GUARD_BOUND')
|
|
|
|
|
|
def release(token, guard_identity=None):
|
|
with registry() as d:
|
|
l = d['leases'][token]
|
|
if l['queue_owner']!=process_identity(os.getpid()):
|
|
raise PermissionError('Only original queue releases its lease')
|
|
if guard_identity is None:
|
|
if l['state']!='RESERVED' or l['owner']!=l['queue_owner']:
|
|
raise PermissionError('Only unused own lease')
|
|
elif l['state']!='GUARD_BOUND' or l['owner'] != guard_identity or alive(guard_identity):
|
|
raise PermissionError('Guard must be reaped before release')
|
|
del d['leases'][token]
|
|
|
|
|
|
def validate_lease(token, root, fold, digest):
|
|
with registry() as d:
|
|
l = d['leases'][token]
|
|
if (l['root'], l['fold'], l['contract_sha256']) != (str(root), fold, digest):
|
|
raise PermissionError('Lease identity')
|
|
if l['state']!='GUARD_BOUND' or l['owner'] != process_identity(os.getpid()):
|
|
raise PermissionError('Lease not transferred to this guard')
|
|
return dict(l)
|