Fix item c untransferred lease identity; preserve BLOCKED and submit narrow review delta

This commit is contained in:
MeltAstra committed 2026-11-18 21:52:16 +08:00
1 parent 706608dfe0
commit 5d0dfabe1e
91 files changed
+18173 -46

No files matched your search

+38 -10
View File
@@ -79,7 +79,20 @@ def host_snapshot():
def admission(snapshot, leases, gpu, *, vram_reserve=4*GIB, rss_reserve=4*GIB):
"""Pure admission rule. leases include observed descendant PIDs and RSS."""
"""Untransferred reservations never receive process or memory credits."""
observed=[];claimed=set()
for lease in leases:
if lease['state']=='RESERVED':
observed.append(dict(lease,observed_pids=[],observed_rss=0))
elif lease['state']=='GUARD_BOUND':
pids=set(lease['observed_pids'])
if claimed & pids:
return False, 'LEASE_PID_OWNERSHIP_OVERLAP'
claimed.update(pids)
observed.append(lease)
else:
raise ValueError('Unknown lease state')
leases=observed
if gpu not in GPUS:
raise ValueError('Unknown GPU')
if gpu == GPUS[0] and snapshot['old_pending']:
@@ -113,8 +126,8 @@ def registry():
with (SHARED/'capacity.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
p = SHARED/'registry.json'
d = json.loads(p.read_text()) if p.exists() else dict(version=1, entries={}, leases={})
if d['version'] != 1:
d = json.loads(p.read_text()) if p.exists() else dict(version=2, entries={}, leases={})
if d['version'] != 2:
raise ValueError('Shared registry version')
yield d
save(p, d)
@@ -141,6 +154,13 @@ def acquire(root, fold, digest):
# its owning coordinator collects it; never reclaim or kill peers.
if not alive(l['owner']):
return None, dict(reason='STALE_LEASE_AWAIT_OWNER_COLLECTION',token=l['token'])
if l['state']=='RESERVED':
# owner is a queue with other children, not this job's guard.
# One independent slot and full VRAM/RSS remain reserved.
leases.append(dict(l,observed_pids=[],observed_rss=0))
continue
if l['state']!='GUARD_BOUND':
raise ValueError('Unknown lease state')
try:
ids = descendants(l['owner'])
except (psutil.NoSuchProcess,RuntimeError):
@@ -160,8 +180,9 @@ def acquire(root, fold, digest):
if not ok:
continue
token = uuid.uuid4().hex
l = dict(token=token, root=str(root), fold=fold, contract_sha256=digest, gpu_uuid=gpu,
worker_cpus=[cpus[0]], owner=process_identity(os.getpid()),
queue_owner=process_identity(os.getpid())
l = dict(token=token, state='RESERVED',root=str(root), fold=fold, contract_sha256=digest, gpu_uuid=gpu,
worker_cpus=[cpus[0]], owner=queue_owner,queue_owner=queue_owner,
vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time())
d['leases'][token] = l
return l, dict(reason='ADMITTED', snapshot=s)
@@ -171,18 +192,25 @@ def acquire(root, fold, digest):
def transfer(token, owner):
with registry() as d:
l = d['leases'][token]
if l['owner']['pid'] != os.getpid() or not alive(owner):
caller=process_identity(os.getpid())
if l['state']!='RESERVED' or l['owner']!=caller or l['queue_owner']!=caller or not alive(owner):
raise PermissionError('Only queue transfers its own lease to live guard')
l['owner'] = owner
if owner==caller or psutil.Process(owner['pid']).ppid()!=caller['pid']:
raise PermissionError('Guard must be this queue direct child, never queue itself')
if any(x['state']=='GUARD_BOUND' and x['owner']==owner for x in d['leases'].values()):
raise PermissionError('Guard already bound to another lease')
l.update(owner=owner,state='GUARD_BOUND')
def release(token, guard_identity=None):
with registry() as d:
l = d['leases'][token]
if l['queue_owner']!=process_identity(os.getpid()):
raise PermissionError('Only original queue releases its lease')
if guard_identity is None:
if l['owner']['pid'] != os.getpid():
if l['state']!='RESERVED' or l['owner']!=l['queue_owner']:
raise PermissionError('Only unused own lease')
elif l['owner'] != guard_identity or alive(guard_identity):
elif l['state']!='GUARD_BOUND' or l['owner'] != guard_identity or alive(guard_identity):
raise PermissionError('Guard must be reaped before release')
del d['leases'][token]
@@ -192,6 +220,6 @@ def validate_lease(token, root, fold, digest):
l = d['leases'][token]
if (l['root'], l['fold'], l['contract_sha256']) != (str(root), fold, digest):
raise PermissionError('Lease identity')
if l['owner'] != process_identity(os.getpid()):
if l['state']!='GUARD_BOUND' or l['owner'] != process_identity(os.getpid()):
raise PermissionError('Lease not transferred to this guard')
return dict(l)
+88 -3
View File
@@ -6,6 +6,8 @@ import multiprocessing
from pathlib import Path
import tempfile
import unittest
from contextlib import contextmanager,ExitStack
from types import SimpleNamespace
from unittest.mock import patch
import numpy as np
import torch
@@ -24,7 +26,7 @@ def snapshot():
def lease(pid=102):
return dict(gpu_uuid=GPUS[1],observed_pids=[pid],observed_rss=GIB,vram_reserve=4*GIB,rss_reserve=4*GIB)
return dict(state='GUARD_BOUND',gpu_uuid=GPUS[1],observed_pids=[pid],observed_rss=GIB,vram_reserve=4*GIB,rss_reserve=4*GIB)
def concurrent_admit(shared, barrier, results):
@@ -120,7 +122,7 @@ class CapacityTests(unittest.TestCase):
with tempfile.TemporaryDirectory() as tmp,patch.object(m,'SHARED',Path(tmp)):
owner=m.process_identity(__import__('os').getpid())
stale=dict(token='stale',owner=dict(pid=99999999,created=0))
Path(tmp,'registry.json').write_text(json.dumps(dict(version=1,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=owner)},leases={'stale':stale})))
Path(tmp,'registry.json').write_text(json.dumps(dict(version=2,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=owner)},leases={'stale':stale})))
decision,reason=m.acquire(Path('/synthetic'),0,'abc')
self.assertIsNone(decision);self.assertIn('STALE_LEASE',reason['reason'])
self.assertIn('stale',json.loads(Path(tmp,'registry.json').read_text())['leases'])
@@ -128,7 +130,7 @@ class CapacityTests(unittest.TestCase):
import phase3_shared_capacity as m
with tempfile.TemporaryDirectory() as tmp:
parent=m.process_identity(__import__('os').getpid())
Path(tmp,'registry.json').write_text(json.dumps(dict(version=1,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=parent)},leases={})))
Path(tmp,'registry.json').write_text(json.dumps(dict(version=2,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=parent)},leases={})))
ctx=multiprocessing.get_context('fork');barrier=ctx.Barrier(2);results=ctx.Queue()
ps=[ctx.Process(target=concurrent_admit,args=(tmp,barrier,results)) for _ in range(2)]
for p in ps:p.start()
@@ -169,6 +171,89 @@ class BudgetTests(unittest.TestCase):
self.assertEqual(tracker.maxima['peak_reserved_bytes'],3*GIB)
class ReservationIdentityDeltaTests(unittest.TestCase):
@contextmanager
def interleaving(self):
"""Assist's actual-acquire scenario, now with explicit lease states."""
import phase3_shared_capacity as m
ident=lambda p:dict(pid=p,created=1.)
root=Path('/synthetic-item-c');q=ident(10)
def l(token,pid,fold,state):
return dict(token=token,state=state,root=str(root),fold=fold,contract_sha256='synthetic',
gpu_uuid=GPUS[1],worker_cpus=[16+fold],owner=ident(pid),queue_owner=q,
vram_reserve=4*GIB,rss_reserve=4*GIB,created_unix=0)
leases={'first':l('first',11,0,'GUARD_BOUND'),'second':l('second',12,1,'GUARD_BOUND'),
'pending':l('pending',10,2,'RESERVED')}
trees={11:{11,101},12:{12,102},13:{13,103},10:{10,11,101,12,102}}
s=snapshot();s['devices'][GPUS[1]]['used']=4*GIB
s['processes']=[dict(gpu_uuid=GPUS[1],pid=p,vram=2*GIB) for p in (101,102)]
s['processes'] += [dict(gpu_uuid=GPUS[2],pid=p,vram=GIB) for p in (201,202,203)]
with tempfile.TemporaryDirectory() as tmp,ExitStack() as stack:
shared=Path(tmp);p=shared/'registry.json'
p.write_text(json.dumps(dict(version=2,entries={'c':dict(root=str(root),contract_sha256='synthetic',owner=q)},leases=leases)))
mocks=dict(SHARED=shared,alive=lambda x:True,descendants=lambda x:trees[x['pid']],
host_snapshot=lambda:copy.deepcopy(s),process_identity=lambda pid:q)
for k,v in mocks.items():stack.enter_context(patch.object(m,k,v))
stack.enter_context(patch.object(m.psutil,'Process',lambda pid:SimpleNamespace(ppid=lambda:10,memory_info=lambda:SimpleNamespace(rss=GIB//4))))
stack.enter_context(patch.object(m.os,'sched_getaffinity',lambda pid:set(range(16,26))))
yield m,root,p,ident,s
def test_actual_acquire_two_visible_plus_reserved_refuses_fourth(self):
with self.interleaving() as (m,root,p,ident,s):
before=p.read_bytes()
with patch.object(m,'descendants',wraps=m.descendants) as seen:
decision,observation=m.acquire(root,3,'synthetic')
self.assertNotIn(10,[call.args[0]['pid'] for call in seen.call_args_list])
self.assertIsNone(decision);self.assertEqual(observation['reason'],'NO_CAPACITY')
self.assertEqual(json.loads(before),json.loads(p.read_text()))
def test_reserved_sibling_claims_ignored_for_slots(self):
s=snapshot();s['processes'].append(dict(gpu_uuid=GPUS[1],pid=102,vram=2*GIB))
pending=dict(lease(101),state='RESERVED',observed_pids=[101,102],observed_rss=99*GIB)
self.assertEqual(admission(s,[lease(101),lease(102),pending],GPUS[1]),(False,'GPU_SLOT_LIMIT'))
def test_reserved_holds_full_gpu_memory(self):
s=snapshot();s['devices'][GPUS[1]]['used']=3*GIB
pending=dict(lease(101),state='RESERVED')
# 3GiB actual +2GiB visible-job deficit +4GiB reserved +4GiB proposed
# =13GiB, above12.8GiB. Crediting the sibling would incorrectly give11.
self.assertEqual(admission(s,[lease(101),pending],GPUS[1]),(False,'GPU_HEADROOM'))
def test_reserved_holds_full_rss_memory(self):
s=snapshot();s['ram_available']=44*GIB
pending=dict(lease(101),state='RESERVED',observed_rss=99*GIB)
self.assertEqual(admission(s,[lease(101),pending],GPUS[1]),(False,'HOST_RAM_HEADROOM'))
def test_transfer_binds_only_new_guard_and_child(self):
with self.interleaving() as (m,root,p,ident,s):
m.transfer('pending',ident(13))
d=json.loads(p.read_text());l=d['leases']['pending']
self.assertEqual(l['state'],'GUARD_BOUND');self.assertEqual(l['owner'],ident(13))
with patch.object(m,'descendants',wraps=m.descendants) as seen:
decision,_=m.acquire(root,3,'synthetic')
self.assertEqual({call.args[0]['pid'] for call in seen.call_args_list},{11,12,13})
self.assertIsNone(decision);self.assertEqual(d,json.loads(p.read_text()))
def test_visible_after_transfer_deduplicates_once(self):
s=snapshot();s['devices'][GPUS[1]]['used']=4*GIB
s['processes'].append(dict(gpu_uuid=GPUS[1],pid=102,vram=2*GIB))
self.assertEqual(admission(s,[lease(101),lease(102)],GPUS[1]),(True,'ADMITTED'))
def test_overlapping_bound_pid_rejected(self):
self.assertEqual(admission(snapshot(),[lease(101),lease(101)],GPUS[1]),(False,'LEASE_PID_OWNERSHIP_OVERLAP'))
def test_overlapping_non_gpu_pid_also_rejected(self):
a=dict(lease(101),observed_pids=[11,101]);b=dict(lease(102),observed_pids=[11,102])
self.assertEqual(admission(snapshot(),[a,b],GPUS[1]),(False,'LEASE_PID_OWNERSHIP_OVERLAP'))
def test_guard_cannot_be_bound_twice(self):
with self.interleaving() as (m,root,p,ident,s):
before=p.read_bytes()
with self.assertRaises(PermissionError):m.transfer('pending',ident(11))
self.assertEqual(p.read_bytes(),before)
def test_queue_cannot_be_its_own_guard(self):
with self.interleaving() as (m,root,p,ident,s):
before=p.read_bytes()
with self.assertRaises(PermissionError):m.transfer('pending',ident(10))
self.assertEqual(p.read_bytes(),before)
def test_transfer_is_one_way(self):
with self.interleaving() as (m,root,p,ident,s):
m.transfer('pending',ident(13));before=p.read_bytes()
with self.assertRaises(PermissionError):m.transfer('pending',ident(13))
self.assertEqual(p.read_bytes(),before)
class QueueSyntheticTests(unittest.TestCase):
def exercise(self,failed=False,admission_errors=0):
import phase3_item_c_queue as q