68 lines
3.5 KiB
Python
68 lines
3.5 KiB
Python
"""b registration wrapper; unchanged reviewed shared admission and leases.
|
|
|
|
Only b's registered cap/ledger changes; the existing admission and reservations remain in use.
|
|
The live c module is imported without mutation, never patched or rewritten.
|
|
"""
|
|
import os,time,uuid
|
|
from pathlib import Path
|
|
import psutil
|
|
from phase3_shared_capacity import (registry,process_identity,alive,descendants,
|
|
host_snapshot,admission,GPUS,GIB,transfer,release,validate_lease)
|
|
|
|
def register_entry(root, digest, contract):
|
|
with registry() as d:
|
|
if 'b' in d['entries']:
|
|
raise FileExistsError('Sole item b entry already registered; no resume')
|
|
reservations = contract['phase_reservations']
|
|
cap = contract['gpu_cap']
|
|
if reservations['b'] != cap or sum(reservations.values()) > 720000:
|
|
raise RuntimeError('Whole phase3 reserved cap')
|
|
d['entries']['b'] = dict(root=str(root), contract_sha256=digest, owner=process_identity(os.getpid()),
|
|
cap=cap, phase_reservations=reservations, started_unix=time.time())
|
|
|
|
|
|
def acquire(root, fold, digest):
|
|
with registry() as d:
|
|
entry = d['entries']['b']
|
|
if entry['root'] != str(root) or entry['contract_sha256'] != digest or not alive(entry['owner']):
|
|
raise PermissionError('Entry owner/identity')
|
|
leases = []
|
|
for l in d['leases'].values():
|
|
# Normal guard exit can race this observation. Block admission until
|
|
# its owning coordinator collects it; never reclaim or kill peers.
|
|
if not alive(l['owner']):
|
|
return None, dict(reason='STALE_LEASE_AWAIT_OWNER_COLLECTION',token=l['token'])
|
|
if l['state']=='RESERVED':
|
|
# owner is a queue with other children, not this job's guard.
|
|
# One independent slot and full VRAM/RSS remain reserved.
|
|
leases.append(dict(l,observed_pids=[],observed_rss=0))
|
|
continue
|
|
if l['state']!='GUARD_BOUND':
|
|
raise ValueError('Unknown lease state')
|
|
try:
|
|
ids = descendants(l['owner'])
|
|
except (psutil.NoSuchProcess,RuntimeError):
|
|
return None, dict(reason='LEASE_EXIT_RACE_AWAIT_OWNER_COLLECTION',token=l['token'])
|
|
rss=0
|
|
for pid in ids:
|
|
try:rss+=psutil.Process(pid).memory_info().rss
|
|
except psutil.NoSuchProcess:pass
|
|
leases.append(dict(l, observed_pids=list(ids), observed_rss=rss))
|
|
s = host_snapshot() # fresh inside inter-entry lock, before making reservation
|
|
used_cpu = {x for l in leases for x in l['worker_cpus']}
|
|
cpus = [x for x in range(16,26) if x not in used_cpu and x in os.sched_getaffinity(0)]
|
|
if not cpus:
|
|
return None, dict(reason='CPU_RESERVATIONS', snapshot=s)
|
|
for gpu in GPUS[:3]:
|
|
ok, reason = admission(s, leases, gpu)
|
|
if not ok:
|
|
continue
|
|
token = uuid.uuid4().hex
|
|
queue_owner=process_identity(os.getpid())
|
|
l = dict(token=token, state='RESERVED',root=str(root), fold=fold, contract_sha256=digest, gpu_uuid=gpu,
|
|
worker_cpus=[cpus[0]], owner=queue_owner,queue_owner=queue_owner,
|
|
vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time())
|
|
d['leases'][token] = l
|
|
return l, dict(reason='ADMITTED', snapshot=s)
|
|
return None, dict(reason='NO_CAPACITY', snapshot=s)
|