"""New-entry admission only; never changes or kills another entry's worker. All participating future entries must use this same registry/lock. Unknown processes consume real slots and a conservative reservation; stale leases fail closed. Old fixed queues do not join this registry and remain untouched. """ from contextlib import contextmanager import fcntl import json import math import os from pathlib import Path import subprocess import time import uuid import psutil from phase2_h1_worker_v2 import save GIB = 2**30 GPUS = ['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c', 'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36', 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76', 'GPU-8b71b032-1b75-2891-935c-ce106a53034a'] SHARED = Path('/home/frank14f/MeltMind-runs/phase3-shared-capacity-2026-11-18') OLD = [Path('/home/frank14f/MeltMind-runs')/s for s in ['phase3-seed1-v2-2026-11-18', 'track-a-recovery-2026-11-18']] PHASE_RESERVES = dict(item1=72421.31599938683, seed1=167200., c=205200., item1b=88560., b=178200.) def process_identity(pid): p = psutil.Process(pid) return dict(pid=pid, created=p.create_time()) def alive(identity): try: p = psutil.Process(identity['pid']) return p.create_time() == identity['created'] and p.status() != psutil.STATUS_ZOMBIE except psutil.NoSuchProcess: return False def descendants(identity): if not alive(identity): raise RuntimeError('Stale lease: explicit reconciliation required') return {identity['pid']} | {p.pid for p in psutil.Process(identity['pid']).children(recursive=True)} def old_pending(): for root in OLD: p = root/'complete.json' if p.exists(): json.loads(p.read_text()) # corrupt terminal fails closed continue d = json.loads((root/'live-progress.json').read_text()) if time.time()-d['unix'] > 120: raise RuntimeError('Stale old queue snapshot') if d['pending']: return True return False def host_snapshot(): def query(arg): return subprocess.check_output(['nvidia-smi', arg, '--format=csv,noheader,nounits'], text=True, timeout=10) devices = {} for line in query('--query-gpu=uuid,memory.total,memory.used').strip().splitlines(): u, total, used = [s.strip() for s in line.split(',')] devices[u] = dict(total=int(total)*2**20, used=int(used)*2**20) processes = [] for line in query('--query-compute-apps=gpu_uuid,pid,used_gpu_memory').strip().splitlines(): if not line.strip(): continue u, pid, mem = [s.strip() for s in line.split(',')] processes.append(dict(gpu_uuid=u, pid=int(pid), vram=int(mem)*2**20)) if set(devices) != set(GPUS) or len({(p['gpu_uuid'],p['pid']) for p in processes}) != len(processes): raise ValueError('Unknown GPU topology or duplicate compute PID') vm = psutil.virtual_memory() return dict(devices=devices, processes=processes, ram_total=vm.total, ram_available=vm.available, cpu_percent=psutil.cpu_percent(interval=.1), old_pending=old_pending(), unix=time.time()) def admission(snapshot, leases, gpu, *, vram_reserve=4*GIB, rss_reserve=4*GIB): """Untransferred reservations never receive process or memory credits.""" observed=[];claimed=set() for lease in leases: if lease['state']=='RESERVED': observed.append(dict(lease,observed_pids=[],observed_rss=0)) elif lease['state']=='GUARD_BOUND': pids=set(lease['observed_pids']) if claimed & pids: return False, 'LEASE_PID_OWNERSHIP_OVERLAP' claimed.update(pids) observed.append(lease) else: raise ValueError('Unknown lease state') leases=observed if gpu not in GPUS: raise ValueError('Unknown GPU') if gpu == GPUS[0] and snapshot['old_pending']: return False, 'GPU0_RESERVED_FOR_OLD_PENDING' processes = snapshot['processes'] card = [p for p in processes if p['gpu_uuid'] == gpu] same = [l for l in leases if l['gpu_uuid'] == gpu] real = {p['pid'] for p in card} occupied = len(real) + sum(not (real & set(l['observed_pids'])) for l in same) if occupied >= (1 if gpu == GPUS[3] else 3): return False, 'GPU_SLOT_LIMIT' owned = set().union(*(set(l['observed_pids']) for l in same)) if same else set() # Charge the unseen part of each reserved peak; do not double-count visible memory. missing = sum(max(0, l['vram_reserve']-sum(p['vram'] for p in card if p['pid'] in l['observed_pids'])) for l in same) missing += sum(max(0, 4*GIB-p['vram']) for p in card if p['pid'] not in owned) d = snapshot['devices'][gpu] if d['used'] + missing + vram_reserve > .8*d['total']: return False, 'GPU_HEADROOM' ram_missing = sum(max(0,l['rss_reserve']-l['observed_rss']) for l in leases) # Extra8GiB preserves future original folds; visible old RSS already in available. if snapshot['ram_available']-ram_missing-rss_reserve-8*GIB < .2*snapshot['ram_total']: return False, 'HOST_RAM_HEADROOM' if snapshot['cpu_percent'] > 80: return False, 'HOST_CPU_HEADROOM' return True, 'ADMITTED' @contextmanager def registry(): SHARED.mkdir(exist_ok=True) with (SHARED/'capacity.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX) p = SHARED/'registry.json' d = json.loads(p.read_text()) if p.exists() else dict(version=2, entries={}, leases={}) if d['version'] != 2: raise ValueError('Shared registry version') yield d save(p, d) def register_entry(root, digest): with registry() as d: if 'c' in d['entries']: raise FileExistsError('Sole item c entry already registered; no resume') if sum(PHASE_RESERVES.values()) > 720000: raise RuntimeError('Whole phase3 reserved cap') d['entries']['c'] = dict(root=str(root), contract_sha256=digest, owner=process_identity(os.getpid()), cap=205200, phase_reservations=PHASE_RESERVES, started_unix=time.time()) def acquire(root, fold, digest): with registry() as d: entry = d['entries']['c'] if entry['root'] != str(root) or entry['contract_sha256'] != digest or not alive(entry['owner']): raise PermissionError('Entry owner/identity') leases = [] for l in d['leases'].values(): # Normal guard exit can race this observation. Block admission until # its owning coordinator collects it; never reclaim or kill peers. if not alive(l['owner']): return None, dict(reason='STALE_LEASE_AWAIT_OWNER_COLLECTION',token=l['token']) if l['state']=='RESERVED': # owner is a queue with other children, not this job's guard. # One independent slot and full VRAM/RSS remain reserved. leases.append(dict(l,observed_pids=[],observed_rss=0)) continue if l['state']!='GUARD_BOUND': raise ValueError('Unknown lease state') try: ids = descendants(l['owner']) except (psutil.NoSuchProcess,RuntimeError): return None, dict(reason='LEASE_EXIT_RACE_AWAIT_OWNER_COLLECTION',token=l['token']) rss=0 for pid in ids: try:rss+=psutil.Process(pid).memory_info().rss except psutil.NoSuchProcess:pass leases.append(dict(l, observed_pids=list(ids), observed_rss=rss)) s = host_snapshot() # fresh inside inter-entry lock, before making reservation used_cpu = {x for l in leases for x in l['worker_cpus']} cpus = [x for x in range(16,26) if x not in used_cpu and x in os.sched_getaffinity(0)] if not cpus: return None, dict(reason='CPU_RESERVATIONS', snapshot=s) for gpu in GPUS[1:3]: ok, reason = admission(s, leases, gpu) if not ok: continue token = uuid.uuid4().hex queue_owner=process_identity(os.getpid()) l = dict(token=token, state='RESERVED',root=str(root), fold=fold, contract_sha256=digest, gpu_uuid=gpu, worker_cpus=[cpus[0]], owner=queue_owner,queue_owner=queue_owner, vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time()) d['leases'][token] = l return l, dict(reason='ADMITTED', snapshot=s) return None, dict(reason='NO_CAPACITY', snapshot=s) def transfer(token, owner): with registry() as d: l = d['leases'][token] caller=process_identity(os.getpid()) if l['state']!='RESERVED' or l['owner']!=caller or l['queue_owner']!=caller or not alive(owner): raise PermissionError('Only queue transfers its own lease to live guard') if owner==caller or psutil.Process(owner['pid']).ppid()!=caller['pid']: raise PermissionError('Guard must be this queue direct child, never queue itself') if any(x['state']=='GUARD_BOUND' and x['owner']==owner for x in d['leases'].values()): raise PermissionError('Guard already bound to another lease') l.update(owner=owner,state='GUARD_BOUND') def release(token, guard_identity=None): with registry() as d: l = d['leases'][token] if l['queue_owner']!=process_identity(os.getpid()): raise PermissionError('Only original queue releases its lease') if guard_identity is None: if l['state']!='RESERVED' or l['owner']!=l['queue_owner']: raise PermissionError('Only unused own lease') elif l['state']!='GUARD_BOUND' or l['owner'] != guard_identity or alive(guard_identity): raise PermissionError('Guard must be reaped before release') del d['leases'][token] def validate_lease(token, root, fold, digest): with registry() as d: l = d['leases'][token] if (l['root'], l['fold'], l['contract_sha256']) != (str(root), fold, digest): raise PermissionError('Lease identity') if l['state']!='GUARD_BOUND' or l['owner'] != process_identity(os.getpid()): raise PermissionError('Lease not transferred to this guard') return dict(l)