Files
MeltMind/tools/phase3_item1b_capacity.py
T

66 lines
3.3 KiB
Python

"""1b registration wrapper; unchanged reviewed shared admission and leases.
Only entry key/cap and eligibleGPU0..2 differ from c's registration/acquire.
The live c module is imported without mutation, never patched or rewritten.
"""
import os,time,uuid
from pathlib import Path
import psutil
from phase3_shared_capacity import (registry,PHASE_RESERVES,process_identity,alive,descendants,
host_snapshot,admission,GPUS,GIB,transfer,release,validate_lease)
def register_entry(root, digest):
with registry() as d:
if 'item1b' in d['entries']:
raise FileExistsError('Sole item 1b entry already registered; no resume')
if sum(PHASE_RESERVES.values()) > 720000:
raise RuntimeError('Whole phase3 reserved cap')
d['entries']['item1b'] = dict(root=str(root), contract_sha256=digest, owner=process_identity(os.getpid()),
cap=88560, phase_reservations=PHASE_RESERVES, started_unix=time.time())
def acquire(root, fold, digest):
with registry() as d:
entry = d['entries']['item1b']
if entry['root'] != str(root) or entry['contract_sha256'] != digest or not alive(entry['owner']):
raise PermissionError('Entry owner/identity')
leases = []
for l in d['leases'].values():
# Normal guard exit can race this observation. Block admission until
# its owning coordinator collects it; never reclaim or kill peers.
if not alive(l['owner']):
return None, dict(reason='STALE_LEASE_AWAIT_OWNER_COLLECTION',token=l['token'])
if l['state']=='RESERVED':
# owner is a queue with other children, not this job's guard.
# One independent slot and full VRAM/RSS remain reserved.
leases.append(dict(l,observed_pids=[],observed_rss=0))
continue
if l['state']!='GUARD_BOUND':
raise ValueError('Unknown lease state')
try:
ids = descendants(l['owner'])
except (psutil.NoSuchProcess,RuntimeError):
return None, dict(reason='LEASE_EXIT_RACE_AWAIT_OWNER_COLLECTION',token=l['token'])
rss=0
for pid in ids:
try:rss+=psutil.Process(pid).memory_info().rss
except psutil.NoSuchProcess:pass
leases.append(dict(l, observed_pids=list(ids), observed_rss=rss))
s = host_snapshot() # fresh inside inter-entry lock, before making reservation
used_cpu = {x for l in leases for x in l['worker_cpus']}
cpus = [x for x in range(16,26) if x not in used_cpu and x in os.sched_getaffinity(0)]
if not cpus:
return None, dict(reason='CPU_RESERVATIONS', snapshot=s)
for gpu in GPUS[:3]:
ok, reason = admission(s, leases, gpu)
if not ok:
continue
token = uuid.uuid4().hex
queue_owner=process_identity(os.getpid())
l = dict(token=token, state='RESERVED',root=str(root), fold=fold, contract_sha256=digest, gpu_uuid=gpu,
worker_cpus=[cpus[0]], owner=queue_owner,queue_owner=queue_owner,
vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time())
d['leases'][token] = l
return l, dict(reason='ADMITTED', snapshot=s)
return None, dict(reason='NO_CAPACITY', snapshot=s)