Fix item c untransferred lease identity; preserve BLOCKED and submit narrow review delta
This commit is contained in:
1 parent
706608dfe0
commit
5d0dfabe1e
91 files changed
+18173
-46
No files matched your search
@@ -79,7 +79,20 @@ def host_snapshot():
|
||||
|
||||
|
||||
def admission(snapshot, leases, gpu, *, vram_reserve=4*GIB, rss_reserve=4*GIB):
|
||||
"""Pure admission rule. leases include observed descendant PIDs and RSS."""
|
||||
"""Untransferred reservations never receive process or memory credits."""
|
||||
observed=[];claimed=set()
|
||||
for lease in leases:
|
||||
if lease['state']=='RESERVED':
|
||||
observed.append(dict(lease,observed_pids=[],observed_rss=0))
|
||||
elif lease['state']=='GUARD_BOUND':
|
||||
pids=set(lease['observed_pids'])
|
||||
if claimed & pids:
|
||||
return False, 'LEASE_PID_OWNERSHIP_OVERLAP'
|
||||
claimed.update(pids)
|
||||
observed.append(lease)
|
||||
else:
|
||||
raise ValueError('Unknown lease state')
|
||||
leases=observed
|
||||
if gpu not in GPUS:
|
||||
raise ValueError('Unknown GPU')
|
||||
if gpu == GPUS[0] and snapshot['old_pending']:
|
||||
@@ -113,8 +126,8 @@ def registry():
|
||||
with (SHARED/'capacity.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
p = SHARED/'registry.json'
|
||||
d = json.loads(p.read_text()) if p.exists() else dict(version=1, entries={}, leases={})
|
||||
if d['version'] != 1:
|
||||
d = json.loads(p.read_text()) if p.exists() else dict(version=2, entries={}, leases={})
|
||||
if d['version'] != 2:
|
||||
raise ValueError('Shared registry version')
|
||||
yield d
|
||||
save(p, d)
|
||||
@@ -141,6 +154,13 @@ def acquire(root, fold, digest):
|
||||
# its owning coordinator collects it; never reclaim or kill peers.
|
||||
if not alive(l['owner']):
|
||||
return None, dict(reason='STALE_LEASE_AWAIT_OWNER_COLLECTION',token=l['token'])
|
||||
if l['state']=='RESERVED':
|
||||
# owner is a queue with other children, not this job's guard.
|
||||
# One independent slot and full VRAM/RSS remain reserved.
|
||||
leases.append(dict(l,observed_pids=[],observed_rss=0))
|
||||
continue
|
||||
if l['state']!='GUARD_BOUND':
|
||||
raise ValueError('Unknown lease state')
|
||||
try:
|
||||
ids = descendants(l['owner'])
|
||||
except (psutil.NoSuchProcess,RuntimeError):
|
||||
@@ -160,8 +180,9 @@ def acquire(root, fold, digest):
|
||||
if not ok:
|
||||
continue
|
||||
token = uuid.uuid4().hex
|
||||
l = dict(token=token, root=str(root), fold=fold, contract_sha256=digest, gpu_uuid=gpu,
|
||||
worker_cpus=[cpus[0]], owner=process_identity(os.getpid()),
|
||||
queue_owner=process_identity(os.getpid())
|
||||
l = dict(token=token, state='RESERVED',root=str(root), fold=fold, contract_sha256=digest, gpu_uuid=gpu,
|
||||
worker_cpus=[cpus[0]], owner=queue_owner,queue_owner=queue_owner,
|
||||
vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time())
|
||||
d['leases'][token] = l
|
||||
return l, dict(reason='ADMITTED', snapshot=s)
|
||||
@@ -171,18 +192,25 @@ def acquire(root, fold, digest):
|
||||
def transfer(token, owner):
|
||||
with registry() as d:
|
||||
l = d['leases'][token]
|
||||
if l['owner']['pid'] != os.getpid() or not alive(owner):
|
||||
caller=process_identity(os.getpid())
|
||||
if l['state']!='RESERVED' or l['owner']!=caller or l['queue_owner']!=caller or not alive(owner):
|
||||
raise PermissionError('Only queue transfers its own lease to live guard')
|
||||
l['owner'] = owner
|
||||
if owner==caller or psutil.Process(owner['pid']).ppid()!=caller['pid']:
|
||||
raise PermissionError('Guard must be this queue direct child, never queue itself')
|
||||
if any(x['state']=='GUARD_BOUND' and x['owner']==owner for x in d['leases'].values()):
|
||||
raise PermissionError('Guard already bound to another lease')
|
||||
l.update(owner=owner,state='GUARD_BOUND')
|
||||
|
||||
|
||||
def release(token, guard_identity=None):
|
||||
with registry() as d:
|
||||
l = d['leases'][token]
|
||||
if l['queue_owner']!=process_identity(os.getpid()):
|
||||
raise PermissionError('Only original queue releases its lease')
|
||||
if guard_identity is None:
|
||||
if l['owner']['pid'] != os.getpid():
|
||||
if l['state']!='RESERVED' or l['owner']!=l['queue_owner']:
|
||||
raise PermissionError('Only unused own lease')
|
||||
elif l['owner'] != guard_identity or alive(guard_identity):
|
||||
elif l['state']!='GUARD_BOUND' or l['owner'] != guard_identity or alive(guard_identity):
|
||||
raise PermissionError('Guard must be reaped before release')
|
||||
del d['leases'][token]
|
||||
|
||||
@@ -192,6 +220,6 @@ def validate_lease(token, root, fold, digest):
|
||||
l = d['leases'][token]
|
||||
if (l['root'], l['fold'], l['contract_sha256']) != (str(root), fold, digest):
|
||||
raise PermissionError('Lease identity')
|
||||
if l['owner'] != process_identity(os.getpid()):
|
||||
if l['state']!='GUARD_BOUND' or l['owner'] != process_identity(os.getpid()):
|
||||
raise PermissionError('Lease not transferred to this guard')
|
||||
return dict(l)
|
||||
@@ -6,6 +6,8 @@ import multiprocessing
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import contextmanager,ExitStack
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -24,7 +26,7 @@ def snapshot():
|
||||
|
||||
|
||||
def lease(pid=102):
|
||||
return dict(gpu_uuid=GPUS[1],observed_pids=[pid],observed_rss=GIB,vram_reserve=4*GIB,rss_reserve=4*GIB)
|
||||
return dict(state='GUARD_BOUND',gpu_uuid=GPUS[1],observed_pids=[pid],observed_rss=GIB,vram_reserve=4*GIB,rss_reserve=4*GIB)
|
||||
|
||||
|
||||
def concurrent_admit(shared, barrier, results):
|
||||
@@ -120,7 +122,7 @@ class CapacityTests(unittest.TestCase):
|
||||
with tempfile.TemporaryDirectory() as tmp,patch.object(m,'SHARED',Path(tmp)):
|
||||
owner=m.process_identity(__import__('os').getpid())
|
||||
stale=dict(token='stale',owner=dict(pid=99999999,created=0))
|
||||
Path(tmp,'registry.json').write_text(json.dumps(dict(version=1,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=owner)},leases={'stale':stale})))
|
||||
Path(tmp,'registry.json').write_text(json.dumps(dict(version=2,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=owner)},leases={'stale':stale})))
|
||||
decision,reason=m.acquire(Path('/synthetic'),0,'abc')
|
||||
self.assertIsNone(decision);self.assertIn('STALE_LEASE',reason['reason'])
|
||||
self.assertIn('stale',json.loads(Path(tmp,'registry.json').read_text())['leases'])
|
||||
@@ -128,7 +130,7 @@ class CapacityTests(unittest.TestCase):
|
||||
import phase3_shared_capacity as m
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
parent=m.process_identity(__import__('os').getpid())
|
||||
Path(tmp,'registry.json').write_text(json.dumps(dict(version=1,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=parent)},leases={})))
|
||||
Path(tmp,'registry.json').write_text(json.dumps(dict(version=2,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=parent)},leases={})))
|
||||
ctx=multiprocessing.get_context('fork');barrier=ctx.Barrier(2);results=ctx.Queue()
|
||||
ps=[ctx.Process(target=concurrent_admit,args=(tmp,barrier,results)) for _ in range(2)]
|
||||
for p in ps:p.start()
|
||||
@@ -169,6 +171,89 @@ class BudgetTests(unittest.TestCase):
|
||||
self.assertEqual(tracker.maxima['peak_reserved_bytes'],3*GIB)
|
||||
|
||||
|
||||
class ReservationIdentityDeltaTests(unittest.TestCase):
|
||||
@contextmanager
|
||||
def interleaving(self):
|
||||
"""Assist's actual-acquire scenario, now with explicit lease states."""
|
||||
import phase3_shared_capacity as m
|
||||
ident=lambda p:dict(pid=p,created=1.)
|
||||
root=Path('/synthetic-item-c');q=ident(10)
|
||||
def l(token,pid,fold,state):
|
||||
return dict(token=token,state=state,root=str(root),fold=fold,contract_sha256='synthetic',
|
||||
gpu_uuid=GPUS[1],worker_cpus=[16+fold],owner=ident(pid),queue_owner=q,
|
||||
vram_reserve=4*GIB,rss_reserve=4*GIB,created_unix=0)
|
||||
leases={'first':l('first',11,0,'GUARD_BOUND'),'second':l('second',12,1,'GUARD_BOUND'),
|
||||
'pending':l('pending',10,2,'RESERVED')}
|
||||
trees={11:{11,101},12:{12,102},13:{13,103},10:{10,11,101,12,102}}
|
||||
s=snapshot();s['devices'][GPUS[1]]['used']=4*GIB
|
||||
s['processes']=[dict(gpu_uuid=GPUS[1],pid=p,vram=2*GIB) for p in (101,102)]
|
||||
s['processes'] += [dict(gpu_uuid=GPUS[2],pid=p,vram=GIB) for p in (201,202,203)]
|
||||
with tempfile.TemporaryDirectory() as tmp,ExitStack() as stack:
|
||||
shared=Path(tmp);p=shared/'registry.json'
|
||||
p.write_text(json.dumps(dict(version=2,entries={'c':dict(root=str(root),contract_sha256='synthetic',owner=q)},leases=leases)))
|
||||
mocks=dict(SHARED=shared,alive=lambda x:True,descendants=lambda x:trees[x['pid']],
|
||||
host_snapshot=lambda:copy.deepcopy(s),process_identity=lambda pid:q)
|
||||
for k,v in mocks.items():stack.enter_context(patch.object(m,k,v))
|
||||
stack.enter_context(patch.object(m.psutil,'Process',lambda pid:SimpleNamespace(ppid=lambda:10,memory_info=lambda:SimpleNamespace(rss=GIB//4))))
|
||||
stack.enter_context(patch.object(m.os,'sched_getaffinity',lambda pid:set(range(16,26))))
|
||||
yield m,root,p,ident,s
|
||||
def test_actual_acquire_two_visible_plus_reserved_refuses_fourth(self):
|
||||
with self.interleaving() as (m,root,p,ident,s):
|
||||
before=p.read_bytes()
|
||||
with patch.object(m,'descendants',wraps=m.descendants) as seen:
|
||||
decision,observation=m.acquire(root,3,'synthetic')
|
||||
self.assertNotIn(10,[call.args[0]['pid'] for call in seen.call_args_list])
|
||||
self.assertIsNone(decision);self.assertEqual(observation['reason'],'NO_CAPACITY')
|
||||
self.assertEqual(json.loads(before),json.loads(p.read_text()))
|
||||
def test_reserved_sibling_claims_ignored_for_slots(self):
|
||||
s=snapshot();s['processes'].append(dict(gpu_uuid=GPUS[1],pid=102,vram=2*GIB))
|
||||
pending=dict(lease(101),state='RESERVED',observed_pids=[101,102],observed_rss=99*GIB)
|
||||
self.assertEqual(admission(s,[lease(101),lease(102),pending],GPUS[1]),(False,'GPU_SLOT_LIMIT'))
|
||||
def test_reserved_holds_full_gpu_memory(self):
|
||||
s=snapshot();s['devices'][GPUS[1]]['used']=3*GIB
|
||||
pending=dict(lease(101),state='RESERVED')
|
||||
# 3GiB actual +2GiB visible-job deficit +4GiB reserved +4GiB proposed
|
||||
# =13GiB, above12.8GiB. Crediting the sibling would incorrectly give11.
|
||||
self.assertEqual(admission(s,[lease(101),pending],GPUS[1]),(False,'GPU_HEADROOM'))
|
||||
def test_reserved_holds_full_rss_memory(self):
|
||||
s=snapshot();s['ram_available']=44*GIB
|
||||
pending=dict(lease(101),state='RESERVED',observed_rss=99*GIB)
|
||||
self.assertEqual(admission(s,[lease(101),pending],GPUS[1]),(False,'HOST_RAM_HEADROOM'))
|
||||
def test_transfer_binds_only_new_guard_and_child(self):
|
||||
with self.interleaving() as (m,root,p,ident,s):
|
||||
m.transfer('pending',ident(13))
|
||||
d=json.loads(p.read_text());l=d['leases']['pending']
|
||||
self.assertEqual(l['state'],'GUARD_BOUND');self.assertEqual(l['owner'],ident(13))
|
||||
with patch.object(m,'descendants',wraps=m.descendants) as seen:
|
||||
decision,_=m.acquire(root,3,'synthetic')
|
||||
self.assertEqual({call.args[0]['pid'] for call in seen.call_args_list},{11,12,13})
|
||||
self.assertIsNone(decision);self.assertEqual(d,json.loads(p.read_text()))
|
||||
def test_visible_after_transfer_deduplicates_once(self):
|
||||
s=snapshot();s['devices'][GPUS[1]]['used']=4*GIB
|
||||
s['processes'].append(dict(gpu_uuid=GPUS[1],pid=102,vram=2*GIB))
|
||||
self.assertEqual(admission(s,[lease(101),lease(102)],GPUS[1]),(True,'ADMITTED'))
|
||||
def test_overlapping_bound_pid_rejected(self):
|
||||
self.assertEqual(admission(snapshot(),[lease(101),lease(101)],GPUS[1]),(False,'LEASE_PID_OWNERSHIP_OVERLAP'))
|
||||
def test_overlapping_non_gpu_pid_also_rejected(self):
|
||||
a=dict(lease(101),observed_pids=[11,101]);b=dict(lease(102),observed_pids=[11,102])
|
||||
self.assertEqual(admission(snapshot(),[a,b],GPUS[1]),(False,'LEASE_PID_OWNERSHIP_OVERLAP'))
|
||||
def test_guard_cannot_be_bound_twice(self):
|
||||
with self.interleaving() as (m,root,p,ident,s):
|
||||
before=p.read_bytes()
|
||||
with self.assertRaises(PermissionError):m.transfer('pending',ident(11))
|
||||
self.assertEqual(p.read_bytes(),before)
|
||||
def test_queue_cannot_be_its_own_guard(self):
|
||||
with self.interleaving() as (m,root,p,ident,s):
|
||||
before=p.read_bytes()
|
||||
with self.assertRaises(PermissionError):m.transfer('pending',ident(10))
|
||||
self.assertEqual(p.read_bytes(),before)
|
||||
def test_transfer_is_one_way(self):
|
||||
with self.interleaving() as (m,root,p,ident,s):
|
||||
m.transfer('pending',ident(13));before=p.read_bytes()
|
||||
with self.assertRaises(PermissionError):m.transfer('pending',ident(13))
|
||||
self.assertEqual(p.read_bytes(),before)
|
||||
|
||||
|
||||
class QueueSyntheticTests(unittest.TestCase):
|
||||
def exercise(self,failed=False,admission_errors=0):
|
||||
import phase3_item_c_queue as q
|
||||
|
||||
Reference in new issue
Block a user