Fix item c untransferred lease identity; preserve BLOCKED and submit narrow review delta
This commit is contained in:
1 parent
706608dfe0
commit
5d0dfabe1e
91 files changed
+18173
-46
No files matched your search
@@ -6,6 +6,8 @@ import multiprocessing
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import contextmanager,ExitStack
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -24,7 +26,7 @@ def snapshot():
|
||||
|
||||
|
||||
def lease(pid=102):
|
||||
return dict(gpu_uuid=GPUS[1],observed_pids=[pid],observed_rss=GIB,vram_reserve=4*GIB,rss_reserve=4*GIB)
|
||||
return dict(state='GUARD_BOUND',gpu_uuid=GPUS[1],observed_pids=[pid],observed_rss=GIB,vram_reserve=4*GIB,rss_reserve=4*GIB)
|
||||
|
||||
|
||||
def concurrent_admit(shared, barrier, results):
|
||||
@@ -120,7 +122,7 @@ class CapacityTests(unittest.TestCase):
|
||||
with tempfile.TemporaryDirectory() as tmp,patch.object(m,'SHARED',Path(tmp)):
|
||||
owner=m.process_identity(__import__('os').getpid())
|
||||
stale=dict(token='stale',owner=dict(pid=99999999,created=0))
|
||||
Path(tmp,'registry.json').write_text(json.dumps(dict(version=1,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=owner)},leases={'stale':stale})))
|
||||
Path(tmp,'registry.json').write_text(json.dumps(dict(version=2,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=owner)},leases={'stale':stale})))
|
||||
decision,reason=m.acquire(Path('/synthetic'),0,'abc')
|
||||
self.assertIsNone(decision);self.assertIn('STALE_LEASE',reason['reason'])
|
||||
self.assertIn('stale',json.loads(Path(tmp,'registry.json').read_text())['leases'])
|
||||
@@ -128,7 +130,7 @@ class CapacityTests(unittest.TestCase):
|
||||
import phase3_shared_capacity as m
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
parent=m.process_identity(__import__('os').getpid())
|
||||
Path(tmp,'registry.json').write_text(json.dumps(dict(version=1,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=parent)},leases={})))
|
||||
Path(tmp,'registry.json').write_text(json.dumps(dict(version=2,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=parent)},leases={})))
|
||||
ctx=multiprocessing.get_context('fork');barrier=ctx.Barrier(2);results=ctx.Queue()
|
||||
ps=[ctx.Process(target=concurrent_admit,args=(tmp,barrier,results)) for _ in range(2)]
|
||||
for p in ps:p.start()
|
||||
@@ -169,6 +171,89 @@ class BudgetTests(unittest.TestCase):
|
||||
self.assertEqual(tracker.maxima['peak_reserved_bytes'],3*GIB)
|
||||
|
||||
|
||||
class ReservationIdentityDeltaTests(unittest.TestCase):
|
||||
@contextmanager
|
||||
def interleaving(self):
|
||||
"""Assist's actual-acquire scenario, now with explicit lease states."""
|
||||
import phase3_shared_capacity as m
|
||||
ident=lambda p:dict(pid=p,created=1.)
|
||||
root=Path('/synthetic-item-c');q=ident(10)
|
||||
def l(token,pid,fold,state):
|
||||
return dict(token=token,state=state,root=str(root),fold=fold,contract_sha256='synthetic',
|
||||
gpu_uuid=GPUS[1],worker_cpus=[16+fold],owner=ident(pid),queue_owner=q,
|
||||
vram_reserve=4*GIB,rss_reserve=4*GIB,created_unix=0)
|
||||
leases={'first':l('first',11,0,'GUARD_BOUND'),'second':l('second',12,1,'GUARD_BOUND'),
|
||||
'pending':l('pending',10,2,'RESERVED')}
|
||||
trees={11:{11,101},12:{12,102},13:{13,103},10:{10,11,101,12,102}}
|
||||
s=snapshot();s['devices'][GPUS[1]]['used']=4*GIB
|
||||
s['processes']=[dict(gpu_uuid=GPUS[1],pid=p,vram=2*GIB) for p in (101,102)]
|
||||
s['processes'] += [dict(gpu_uuid=GPUS[2],pid=p,vram=GIB) for p in (201,202,203)]
|
||||
with tempfile.TemporaryDirectory() as tmp,ExitStack() as stack:
|
||||
shared=Path(tmp);p=shared/'registry.json'
|
||||
p.write_text(json.dumps(dict(version=2,entries={'c':dict(root=str(root),contract_sha256='synthetic',owner=q)},leases=leases)))
|
||||
mocks=dict(SHARED=shared,alive=lambda x:True,descendants=lambda x:trees[x['pid']],
|
||||
host_snapshot=lambda:copy.deepcopy(s),process_identity=lambda pid:q)
|
||||
for k,v in mocks.items():stack.enter_context(patch.object(m,k,v))
|
||||
stack.enter_context(patch.object(m.psutil,'Process',lambda pid:SimpleNamespace(ppid=lambda:10,memory_info=lambda:SimpleNamespace(rss=GIB//4))))
|
||||
stack.enter_context(patch.object(m.os,'sched_getaffinity',lambda pid:set(range(16,26))))
|
||||
yield m,root,p,ident,s
|
||||
def test_actual_acquire_two_visible_plus_reserved_refuses_fourth(self):
|
||||
with self.interleaving() as (m,root,p,ident,s):
|
||||
before=p.read_bytes()
|
||||
with patch.object(m,'descendants',wraps=m.descendants) as seen:
|
||||
decision,observation=m.acquire(root,3,'synthetic')
|
||||
self.assertNotIn(10,[call.args[0]['pid'] for call in seen.call_args_list])
|
||||
self.assertIsNone(decision);self.assertEqual(observation['reason'],'NO_CAPACITY')
|
||||
self.assertEqual(json.loads(before),json.loads(p.read_text()))
|
||||
def test_reserved_sibling_claims_ignored_for_slots(self):
|
||||
s=snapshot();s['processes'].append(dict(gpu_uuid=GPUS[1],pid=102,vram=2*GIB))
|
||||
pending=dict(lease(101),state='RESERVED',observed_pids=[101,102],observed_rss=99*GIB)
|
||||
self.assertEqual(admission(s,[lease(101),lease(102),pending],GPUS[1]),(False,'GPU_SLOT_LIMIT'))
|
||||
def test_reserved_holds_full_gpu_memory(self):
|
||||
s=snapshot();s['devices'][GPUS[1]]['used']=3*GIB
|
||||
pending=dict(lease(101),state='RESERVED')
|
||||
# 3GiB actual +2GiB visible-job deficit +4GiB reserved +4GiB proposed
|
||||
# =13GiB, above12.8GiB. Crediting the sibling would incorrectly give11.
|
||||
self.assertEqual(admission(s,[lease(101),pending],GPUS[1]),(False,'GPU_HEADROOM'))
|
||||
def test_reserved_holds_full_rss_memory(self):
|
||||
s=snapshot();s['ram_available']=44*GIB
|
||||
pending=dict(lease(101),state='RESERVED',observed_rss=99*GIB)
|
||||
self.assertEqual(admission(s,[lease(101),pending],GPUS[1]),(False,'HOST_RAM_HEADROOM'))
|
||||
def test_transfer_binds_only_new_guard_and_child(self):
|
||||
with self.interleaving() as (m,root,p,ident,s):
|
||||
m.transfer('pending',ident(13))
|
||||
d=json.loads(p.read_text());l=d['leases']['pending']
|
||||
self.assertEqual(l['state'],'GUARD_BOUND');self.assertEqual(l['owner'],ident(13))
|
||||
with patch.object(m,'descendants',wraps=m.descendants) as seen:
|
||||
decision,_=m.acquire(root,3,'synthetic')
|
||||
self.assertEqual({call.args[0]['pid'] for call in seen.call_args_list},{11,12,13})
|
||||
self.assertIsNone(decision);self.assertEqual(d,json.loads(p.read_text()))
|
||||
def test_visible_after_transfer_deduplicates_once(self):
|
||||
s=snapshot();s['devices'][GPUS[1]]['used']=4*GIB
|
||||
s['processes'].append(dict(gpu_uuid=GPUS[1],pid=102,vram=2*GIB))
|
||||
self.assertEqual(admission(s,[lease(101),lease(102)],GPUS[1]),(True,'ADMITTED'))
|
||||
def test_overlapping_bound_pid_rejected(self):
|
||||
self.assertEqual(admission(snapshot(),[lease(101),lease(101)],GPUS[1]),(False,'LEASE_PID_OWNERSHIP_OVERLAP'))
|
||||
def test_overlapping_non_gpu_pid_also_rejected(self):
|
||||
a=dict(lease(101),observed_pids=[11,101]);b=dict(lease(102),observed_pids=[11,102])
|
||||
self.assertEqual(admission(snapshot(),[a,b],GPUS[1]),(False,'LEASE_PID_OWNERSHIP_OVERLAP'))
|
||||
def test_guard_cannot_be_bound_twice(self):
|
||||
with self.interleaving() as (m,root,p,ident,s):
|
||||
before=p.read_bytes()
|
||||
with self.assertRaises(PermissionError):m.transfer('pending',ident(11))
|
||||
self.assertEqual(p.read_bytes(),before)
|
||||
def test_queue_cannot_be_its_own_guard(self):
|
||||
with self.interleaving() as (m,root,p,ident,s):
|
||||
before=p.read_bytes()
|
||||
with self.assertRaises(PermissionError):m.transfer('pending',ident(10))
|
||||
self.assertEqual(p.read_bytes(),before)
|
||||
def test_transfer_is_one_way(self):
|
||||
with self.interleaving() as (m,root,p,ident,s):
|
||||
m.transfer('pending',ident(13));before=p.read_bytes()
|
||||
with self.assertRaises(PermissionError):m.transfer('pending',ident(13))
|
||||
self.assertEqual(p.read_bytes(),before)
|
||||
|
||||
|
||||
class QueueSyntheticTests(unittest.TestCase):
|
||||
def exercise(self,failed=False,admission_errors=0):
|
||||
import phase3_item_c_queue as q
|
||||
|
||||
Reference in new issue
Block a user