Fix item c untransferred lease identity; preserve BLOCKED and submit narrow review delta
This commit is contained in:
1 parent
706608dfe0
commit
5d0dfabe1e
91 files changed
+18173
-46
No files matched your search
@@ -79,7 +79,20 @@ def host_snapshot():
|
||||
|
||||
|
||||
def admission(snapshot, leases, gpu, *, vram_reserve=4*GIB, rss_reserve=4*GIB):
|
||||
"""Pure admission rule. leases include observed descendant PIDs and RSS."""
|
||||
"""Untransferred reservations never receive process or memory credits."""
|
||||
observed=[];claimed=set()
|
||||
for lease in leases:
|
||||
if lease['state']=='RESERVED':
|
||||
observed.append(dict(lease,observed_pids=[],observed_rss=0))
|
||||
elif lease['state']=='GUARD_BOUND':
|
||||
pids=set(lease['observed_pids'])
|
||||
if claimed & pids:
|
||||
return False, 'LEASE_PID_OWNERSHIP_OVERLAP'
|
||||
claimed.update(pids)
|
||||
observed.append(lease)
|
||||
else:
|
||||
raise ValueError('Unknown lease state')
|
||||
leases=observed
|
||||
if gpu not in GPUS:
|
||||
raise ValueError('Unknown GPU')
|
||||
if gpu == GPUS[0] and snapshot['old_pending']:
|
||||
@@ -113,8 +126,8 @@ def registry():
|
||||
with (SHARED/'capacity.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
p = SHARED/'registry.json'
|
||||
d = json.loads(p.read_text()) if p.exists() else dict(version=1, entries={}, leases={})
|
||||
if d['version'] != 1:
|
||||
d = json.loads(p.read_text()) if p.exists() else dict(version=2, entries={}, leases={})
|
||||
if d['version'] != 2:
|
||||
raise ValueError('Shared registry version')
|
||||
yield d
|
||||
save(p, d)
|
||||
@@ -141,6 +154,13 @@ def acquire(root, fold, digest):
|
||||
# its owning coordinator collects it; never reclaim or kill peers.
|
||||
if not alive(l['owner']):
|
||||
return None, dict(reason='STALE_LEASE_AWAIT_OWNER_COLLECTION',token=l['token'])
|
||||
if l['state']=='RESERVED':
|
||||
# owner is a queue with other children, not this job's guard.
|
||||
# One independent slot and full VRAM/RSS remain reserved.
|
||||
leases.append(dict(l,observed_pids=[],observed_rss=0))
|
||||
continue
|
||||
if l['state']!='GUARD_BOUND':
|
||||
raise ValueError('Unknown lease state')
|
||||
try:
|
||||
ids = descendants(l['owner'])
|
||||
except (psutil.NoSuchProcess,RuntimeError):
|
||||
@@ -160,8 +180,9 @@ def acquire(root, fold, digest):
|
||||
if not ok:
|
||||
continue
|
||||
token = uuid.uuid4().hex
|
||||
l = dict(token=token, root=str(root), fold=fold, contract_sha256=digest, gpu_uuid=gpu,
|
||||
worker_cpus=[cpus[0]], owner=process_identity(os.getpid()),
|
||||
queue_owner=process_identity(os.getpid())
|
||||
l = dict(token=token, state='RESERVED',root=str(root), fold=fold, contract_sha256=digest, gpu_uuid=gpu,
|
||||
worker_cpus=[cpus[0]], owner=queue_owner,queue_owner=queue_owner,
|
||||
vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time())
|
||||
d['leases'][token] = l
|
||||
return l, dict(reason='ADMITTED', snapshot=s)
|
||||
@@ -171,18 +192,25 @@ def acquire(root, fold, digest):
|
||||
def transfer(token, owner):
|
||||
with registry() as d:
|
||||
l = d['leases'][token]
|
||||
if l['owner']['pid'] != os.getpid() or not alive(owner):
|
||||
caller=process_identity(os.getpid())
|
||||
if l['state']!='RESERVED' or l['owner']!=caller or l['queue_owner']!=caller or not alive(owner):
|
||||
raise PermissionError('Only queue transfers its own lease to live guard')
|
||||
l['owner'] = owner
|
||||
if owner==caller or psutil.Process(owner['pid']).ppid()!=caller['pid']:
|
||||
raise PermissionError('Guard must be this queue direct child, never queue itself')
|
||||
if any(x['state']=='GUARD_BOUND' and x['owner']==owner for x in d['leases'].values()):
|
||||
raise PermissionError('Guard already bound to another lease')
|
||||
l.update(owner=owner,state='GUARD_BOUND')
|
||||
|
||||
|
||||
def release(token, guard_identity=None):
|
||||
with registry() as d:
|
||||
l = d['leases'][token]
|
||||
if l['queue_owner']!=process_identity(os.getpid()):
|
||||
raise PermissionError('Only original queue releases its lease')
|
||||
if guard_identity is None:
|
||||
if l['owner']['pid'] != os.getpid():
|
||||
if l['state']!='RESERVED' or l['owner']!=l['queue_owner']:
|
||||
raise PermissionError('Only unused own lease')
|
||||
elif l['owner'] != guard_identity or alive(guard_identity):
|
||||
elif l['state']!='GUARD_BOUND' or l['owner'] != guard_identity or alive(guard_identity):
|
||||
raise PermissionError('Guard must be reaped before release')
|
||||
del d['leases'][token]
|
||||
|
||||
@@ -192,6 +220,6 @@ def validate_lease(token, root, fold, digest):
|
||||
l = d['leases'][token]
|
||||
if (l['root'], l['fold'], l['contract_sha256']) != (str(root), fold, digest):
|
||||
raise PermissionError('Lease identity')
|
||||
if l['owner'] != process_identity(os.getpid()):
|
||||
if l['state']!='GUARD_BOUND' or l['owner'] != process_identity(os.getpid()):
|
||||
raise PermissionError('Lease not transferred to this guard')
|
||||
return dict(l)
|
||||
Reference in new issue
Block a user