309 lines
19 KiB
Python
309 lines
19 KiB
Python
"""CPU synthetic acceptance only: no original arrays, labels, CUDA or fits."""
|
|
import ast
|
|
import copy
|
|
import json
|
|
import multiprocessing
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from contextlib import contextmanager,ExitStack
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
import numpy as np
|
|
import torch
|
|
from phase3_item_c_loss import candidates,location_loss,objective,THETA
|
|
from phase3_item_c_data import require_learning
|
|
from phase3_item_c_common import project
|
|
from phase3_shared_capacity import admission,GPUS,GIB,PHASE_RESERVES
|
|
from s2_custody_statistics import last_location
|
|
from s2_main_memory_guard import PeakRecords
|
|
|
|
|
|
def snapshot():
|
|
return dict(devices={u:dict(total=16*GIB,used=2*GIB) for u in GPUS},
|
|
processes=[dict(gpu_uuid=GPUS[1],pid=101,vram=2*GIB)],
|
|
ram_total=128*GIB,ram_available=96*GIB,cpu_percent=20,old_pending=False)
|
|
|
|
|
|
def lease(pid=102):
|
|
return dict(state='GUARD_BOUND',gpu_uuid=GPUS[1],observed_pids=[pid],observed_rss=GIB,vram_reserve=4*GIB,rss_reserve=4*GIB)
|
|
|
|
|
|
def concurrent_admit(shared, barrier, results):
|
|
import phase3_shared_capacity as m
|
|
m.SHARED=Path(shared)
|
|
s=snapshot()
|
|
# two actual tasks leave one slot onGPU1; GPU2 already has three.
|
|
s['processes'].append(dict(gpu_uuid=GPUS[1],pid=103,vram=GIB))
|
|
s['processes'] += [dict(gpu_uuid=GPUS[2],pid=p,vram=GIB) for p in (201,202,203)]
|
|
m.host_snapshot=lambda:s
|
|
try:
|
|
l,_=m.acquire(Path('/synthetic'),0,'abc')
|
|
results.put(bool(l))
|
|
barrier.wait(timeout=10) # keep lease owners alive through both admissions
|
|
except BaseException as exc:
|
|
results.put(type(exc).__name__+': '+str(exc))
|
|
|
|
|
|
class ObjectiveTests(unittest.TestCase):
|
|
def test_exact_old_peak_tie(self):
|
|
raw=np.array([7,19,42]);xyz=np.array([[0.,0,0],[2.,0,0],[0.,3,0]])
|
|
truth=np.array([5.,5.,1.]);q=candidates(raw,raw,np.ones(3,dtype=int),xyz,truth)
|
|
self.assertEqual(q['true_peak_raw_id'],7)
|
|
for i in range(3):
|
|
p=np.zeros(3);p[i]=1
|
|
self.assertEqual(q['distances'][i],last_location(truth,p,raw,xyz))
|
|
def test_duplicate_order_does_not_vote(self):
|
|
full=np.array([1,2,3]);ids=np.array([3,1,3,2]);xyz=np.array([[2.,0,0],[0,0,0],[2,0,0],[1,0,0]])
|
|
q=candidates(full,ids,np.array([50,1,99,2]),xyz,np.array([9.,10,9,8]))
|
|
np.testing.assert_array_equal(q['indices'],[0,1,2]);np.testing.assert_array_equal(q['distances'],[0,1,2])
|
|
def test_bad_duplicate_fails(self):
|
|
with self.assertRaises(ValueError):candidates(np.array([1]),np.array([1,1]),np.array([1,1]),np.zeros((2,3)),np.array([1.,2.]))
|
|
def test_missing_id_fails(self):
|
|
with self.assertRaises(ValueError):candidates(np.array([1]),np.array([2]),np.array([1]),np.zeros((1,3)),np.array([1.]))
|
|
def test_nan_truth_not_undefined(self):
|
|
with self.assertRaises(ValueError):candidates(np.array([1]),np.array([1]),np.array([1]),np.zeros((1,3)),np.array([np.nan]))
|
|
def test_gradient_formula(self):
|
|
r=torch.tensor([.01,-.02,.03],dtype=torch.float64,requires_grad=True);prior=torch.tensor([.7,.8,.9],dtype=torch.float64)
|
|
d=torch.tensor([2.,1.,0.],dtype=torch.float64);idx=torch.arange(3)
|
|
loss=location_loss(r,prior,idx,d);loss.backward();p=torch.softmax((r.detach()+prior)/THETA,0)
|
|
torch.testing.assert_close(r.grad,p*(d-(p*d).sum())/THETA)
|
|
def test_constant_shift(self):
|
|
r=torch.tensor([.1,.2],dtype=torch.float64);p=torch.tensor([.7,.8],dtype=torch.float64);d=torch.tensor([2.,0.],dtype=torch.float64)
|
|
torch.testing.assert_close(location_loss(r,p,torch.arange(2),d),location_loss(r+12,p,torch.arange(2),d))
|
|
def test_degenerate_gradients_zero(self):
|
|
for d in (torch.tensor([2.],dtype=torch.float64),torch.tensor([2.,2.,2.],dtype=torch.float64)):
|
|
r=torch.arange(len(d),dtype=torch.float64,requires_grad=True);location_loss(r,torch.zeros_like(r),torch.arange(len(d)),d).backward()
|
|
torch.testing.assert_close(r.grad,torch.zeros_like(r))
|
|
def test_fp64_prior_preserved(self):
|
|
r=torch.zeros(2,dtype=torch.float32,requires_grad=True);p=torch.tensor([1.,1.+1e-8],dtype=torch.float64);d=torch.tensor([0.,1.],dtype=torch.float64)
|
|
loss=location_loss(r,p,torch.arange(2),d);self.assertGreater(float(loss),.5);loss.backward();self.assertTrue(torch.isfinite(r.grad).all())
|
|
with self.assertRaises(ValueError):location_loss(r,p.float(),torch.arange(2),d)
|
|
def test_original_weight_plus_one_location(self):
|
|
p=torch.tensor([.2,.4]);t=torch.tensor([.1,.7]);prior=torch.tensor([.7,.6],dtype=torch.float64);d=torch.tensor([0.,3.],dtype=torch.float64)
|
|
total,reg,loc=objective(p,t,7,2,19,prior,torch.arange(2),d)
|
|
self.assertEqual(float(reg),float(((p-t)**2).mean()*(14/19)))
|
|
torch.testing.assert_close(total-reg,.001*loc)
|
|
def test_no_validation_or_test_role(self):
|
|
view=dict(folds=[dict(learning=['a'],validation=['b'],test=['c'])])
|
|
require_learning(view,'a')
|
|
for case in ('b','c','missing'):
|
|
with self.assertRaises(PermissionError):require_learning(view,case)
|
|
def test_worker_only_loads_learning(self):
|
|
tree=ast.parse((Path(__file__).parent/'phase3_item_c_worker.py').read_text())
|
|
calls=[n for n in ast.walk(tree) if isinstance(n,ast.Call) and isinstance(n.func,ast.Name) and n.func.id=='load_part']
|
|
self.assertEqual(len(calls),1);self.assertEqual(calls[0].args[3].value,'learning')
|
|
|
|
|
|
class CapacityTests(unittest.TestCase):
|
|
def test_visible_reserved_process_counted_once(self):
|
|
self.assertTrue(admission(snapshot(),[lease(101)],GPUS[1])[0])
|
|
def test_unseen_reservations_block_fourth(self):
|
|
self.assertFalse(admission(snapshot(),[lease(102),lease(103)],GPUS[1])[0])
|
|
def test_foreign_processes_count(self):
|
|
s=snapshot();s['processes'] += [dict(gpu_uuid=GPUS[1],pid=i,vram=GIB) for i in (102,103)]
|
|
self.assertEqual(admission(s,[],GPUS[1])[1],'GPU_SLOT_LIMIT')
|
|
def test_gpu3_single(self):
|
|
s=snapshot();s['processes']=[dict(gpu_uuid=GPUS[3],pid=1,vram=1)]
|
|
self.assertFalse(admission(s,[],GPUS[3])[0])
|
|
def test_gpu0_old_pending(self):
|
|
s=snapshot();s['old_pending']=True;self.assertEqual(admission(s,[],GPUS[0])[1],'GPU0_RESERVED_FOR_OLD_PENDING')
|
|
def test_memory_reserves_future_peaks(self):
|
|
s=snapshot();s['devices'][GPUS[1]]['used']=9*GIB
|
|
self.assertEqual(admission(s,[lease(101)],GPUS[1])[1],'GPU_HEADROOM')
|
|
def test_host_ram_reserve(self):
|
|
s=snapshot();s['ram_available']=30*GIB;self.assertEqual(admission(s,[],GPUS[1])[1],'HOST_RAM_HEADROOM')
|
|
def test_cpu_headroom(self):
|
|
s=snapshot();s['cpu_percent']=81;self.assertEqual(admission(s,[],GPUS[1])[1],'HOST_CPU_HEADROOM')
|
|
def test_phase_reservation_once(self):
|
|
self.assertEqual(PHASE_RESERVES['seed1'],167200);self.assertLess(sum(PHASE_RESERVES.values()),720000)
|
|
def test_stale_lease_blocks_without_reclaim(self):
|
|
import phase3_shared_capacity as m
|
|
with tempfile.TemporaryDirectory() as tmp,patch.object(m,'SHARED',Path(tmp)):
|
|
owner=m.process_identity(__import__('os').getpid())
|
|
stale=dict(token='stale',owner=dict(pid=99999999,created=0))
|
|
Path(tmp,'registry.json').write_text(json.dumps(dict(version=2,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=owner)},leases={'stale':stale})))
|
|
decision,reason=m.acquire(Path('/synthetic'),0,'abc')
|
|
self.assertIsNone(decision);self.assertIn('STALE_LEASE',reason['reason'])
|
|
self.assertIn('stale',json.loads(Path(tmp,'registry.json').read_text())['leases'])
|
|
def test_two_admissions_atomic(self):
|
|
import phase3_shared_capacity as m
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
parent=m.process_identity(__import__('os').getpid())
|
|
Path(tmp,'registry.json').write_text(json.dumps(dict(version=2,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=parent)},leases={})))
|
|
ctx=multiprocessing.get_context('fork');barrier=ctx.Barrier(2);results=ctx.Queue()
|
|
ps=[ctx.Process(target=concurrent_admit,args=(tmp,barrier,results)) for _ in range(2)]
|
|
for p in ps:p.start()
|
|
for p in ps:p.join(15)
|
|
try:
|
|
self.assertTrue(all(p.exitcode==0 for p in ps))
|
|
self.assertEqual(sorted([results.get(timeout=1),results.get(timeout=1)]),[False,True])
|
|
finally:
|
|
for p in ps:
|
|
if p.is_alive():p.kill();p.join()
|
|
|
|
|
|
class BudgetTests(unittest.TestCase):
|
|
def setUp(self):self.c=dict(gpu_cap=205200,postfit_gpu_reserve=7200,preparation_charge_gpu_seconds=600,runaway_gpu_seconds=60000)
|
|
def test_initial_inclusive_cap(self):
|
|
p=project(self.c,{i:60 for i in range(5)},{},{})
|
|
self.assertEqual(p['projected_gpu_seconds'],205200);self.assertEqual(p['status'],'PASS')
|
|
def test_pending_wait_not_free(self):
|
|
p=project(self.c,{i:50000 for i in range(5)},{},{})
|
|
self.assertEqual(p['status'],'STOP_BUDGET')
|
|
def test_projected_fold_above_runaway_not_alone_stop(self):
|
|
h={0:[dict(epoch=1,train_epoch_seconds=220)]}
|
|
p=project(self.c,{i:1000 for i in range(5)},h,{1:{},2:{},3:{}})
|
|
self.assertGreater(p['folds']['0']['projected'],60000);self.assertEqual(p['status'],'PASS')
|
|
def test_actual_runaway_stops(self):
|
|
self.assertEqual(project(self.c,{i:60000 if i==0 else 1 for i in range(5)},{},{})['status'],'STOP_BUDGET')
|
|
def test_negative_nan_cost(self):
|
|
for v in (-1,float('nan')):
|
|
with self.assertRaises(ValueError):project(self.c,{i:v for i in range(5)},{},{})
|
|
def test_incomplete_epoch_history(self):
|
|
with self.assertRaises(ValueError):project(self.c,{i:60 for i in range(5)},{0:[dict(epoch=2,train_epoch_seconds=10)]},{})
|
|
def test_failed_worker_late_memory_peak(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
p=Path(tmp)/'failure.json';p.write_text(json.dumps(dict(peak_reserved_bytes=3*GIB,peak_allocated_bytes=GIB)))
|
|
tracker=PeakRecords();tracker.scan(tmp)
|
|
self.assertGreater(max(tracker.maxima.values()),2*GIB)
|
|
p.write_text(json.dumps(dict(peak_reserved_bytes=1)));tracker.scan(tmp)
|
|
self.assertEqual(tracker.maxima['peak_reserved_bytes'],3*GIB)
|
|
|
|
|
|
class ReservationIdentityDeltaTests(unittest.TestCase):
|
|
@contextmanager
|
|
def interleaving(self):
|
|
"""Assist's actual-acquire scenario, now with explicit lease states."""
|
|
import phase3_shared_capacity as m
|
|
ident=lambda p:dict(pid=p,created=1.)
|
|
root=Path('/synthetic-item-c');q=ident(10)
|
|
def l(token,pid,fold,state):
|
|
return dict(token=token,state=state,root=str(root),fold=fold,contract_sha256='synthetic',
|
|
gpu_uuid=GPUS[1],worker_cpus=[16+fold],owner=ident(pid),queue_owner=q,
|
|
vram_reserve=4*GIB,rss_reserve=4*GIB,created_unix=0)
|
|
leases={'first':l('first',11,0,'GUARD_BOUND'),'second':l('second',12,1,'GUARD_BOUND'),
|
|
'pending':l('pending',10,2,'RESERVED')}
|
|
trees={11:{11,101},12:{12,102},13:{13,103},10:{10,11,101,12,102}}
|
|
s=snapshot();s['devices'][GPUS[1]]['used']=4*GIB
|
|
s['processes']=[dict(gpu_uuid=GPUS[1],pid=p,vram=2*GIB) for p in (101,102)]
|
|
s['processes'] += [dict(gpu_uuid=GPUS[2],pid=p,vram=GIB) for p in (201,202,203)]
|
|
with tempfile.TemporaryDirectory() as tmp,ExitStack() as stack:
|
|
shared=Path(tmp);p=shared/'registry.json'
|
|
p.write_text(json.dumps(dict(version=2,entries={'c':dict(root=str(root),contract_sha256='synthetic',owner=q)},leases=leases)))
|
|
mocks=dict(SHARED=shared,alive=lambda x:True,descendants=lambda x:trees[x['pid']],
|
|
host_snapshot=lambda:copy.deepcopy(s),process_identity=lambda pid:q)
|
|
for k,v in mocks.items():stack.enter_context(patch.object(m,k,v))
|
|
stack.enter_context(patch.object(m.psutil,'Process',lambda pid:SimpleNamespace(ppid=lambda:10,memory_info=lambda:SimpleNamespace(rss=GIB//4))))
|
|
stack.enter_context(patch.object(m.os,'sched_getaffinity',lambda pid:set(range(16,26))))
|
|
yield m,root,p,ident,s
|
|
def test_actual_acquire_two_visible_plus_reserved_refuses_fourth(self):
|
|
with self.interleaving() as (m,root,p,ident,s):
|
|
before=p.read_bytes()
|
|
with patch.object(m,'descendants',wraps=m.descendants) as seen:
|
|
decision,observation=m.acquire(root,3,'synthetic')
|
|
self.assertNotIn(10,[call.args[0]['pid'] for call in seen.call_args_list])
|
|
self.assertIsNone(decision);self.assertEqual(observation['reason'],'NO_CAPACITY')
|
|
self.assertEqual(json.loads(before),json.loads(p.read_text()))
|
|
def test_reserved_sibling_claims_ignored_for_slots(self):
|
|
s=snapshot();s['processes'].append(dict(gpu_uuid=GPUS[1],pid=102,vram=2*GIB))
|
|
pending=dict(lease(101),state='RESERVED',observed_pids=[101,102],observed_rss=99*GIB)
|
|
self.assertEqual(admission(s,[lease(101),lease(102),pending],GPUS[1]),(False,'GPU_SLOT_LIMIT'))
|
|
def test_reserved_holds_full_gpu_memory(self):
|
|
s=snapshot();s['devices'][GPUS[1]]['used']=3*GIB
|
|
pending=dict(lease(101),state='RESERVED')
|
|
# 3GiB actual +2GiB visible-job deficit +4GiB reserved +4GiB proposed
|
|
# =13GiB, above12.8GiB. Crediting the sibling would incorrectly give11.
|
|
self.assertEqual(admission(s,[lease(101),pending],GPUS[1]),(False,'GPU_HEADROOM'))
|
|
def test_reserved_holds_full_rss_memory(self):
|
|
s=snapshot();s['ram_available']=44*GIB
|
|
pending=dict(lease(101),state='RESERVED',observed_rss=99*GIB)
|
|
self.assertEqual(admission(s,[lease(101),pending],GPUS[1]),(False,'HOST_RAM_HEADROOM'))
|
|
def test_transfer_binds_only_new_guard_and_child(self):
|
|
with self.interleaving() as (m,root,p,ident,s):
|
|
m.transfer('pending',ident(13))
|
|
d=json.loads(p.read_text());l=d['leases']['pending']
|
|
self.assertEqual(l['state'],'GUARD_BOUND');self.assertEqual(l['owner'],ident(13))
|
|
with patch.object(m,'descendants',wraps=m.descendants) as seen:
|
|
decision,_=m.acquire(root,3,'synthetic')
|
|
self.assertEqual({call.args[0]['pid'] for call in seen.call_args_list},{11,12,13})
|
|
self.assertIsNone(decision);self.assertEqual(d,json.loads(p.read_text()))
|
|
def test_visible_after_transfer_deduplicates_once(self):
|
|
s=snapshot();s['devices'][GPUS[1]]['used']=4*GIB
|
|
s['processes'].append(dict(gpu_uuid=GPUS[1],pid=102,vram=2*GIB))
|
|
self.assertEqual(admission(s,[lease(101),lease(102)],GPUS[1]),(True,'ADMITTED'))
|
|
def test_overlapping_bound_pid_rejected(self):
|
|
self.assertEqual(admission(snapshot(),[lease(101),lease(101)],GPUS[1]),(False,'LEASE_PID_OWNERSHIP_OVERLAP'))
|
|
def test_overlapping_non_gpu_pid_also_rejected(self):
|
|
a=dict(lease(101),observed_pids=[11,101]);b=dict(lease(102),observed_pids=[11,102])
|
|
self.assertEqual(admission(snapshot(),[a,b],GPUS[1]),(False,'LEASE_PID_OWNERSHIP_OVERLAP'))
|
|
def test_guard_cannot_be_bound_twice(self):
|
|
with self.interleaving() as (m,root,p,ident,s):
|
|
before=p.read_bytes()
|
|
with self.assertRaises(PermissionError):m.transfer('pending',ident(11))
|
|
self.assertEqual(p.read_bytes(),before)
|
|
def test_queue_cannot_be_its_own_guard(self):
|
|
with self.interleaving() as (m,root,p,ident,s):
|
|
before=p.read_bytes()
|
|
with self.assertRaises(PermissionError):m.transfer('pending',ident(10))
|
|
self.assertEqual(p.read_bytes(),before)
|
|
def test_transfer_is_one_way(self):
|
|
with self.interleaving() as (m,root,p,ident,s):
|
|
m.transfer('pending',ident(13));before=p.read_bytes()
|
|
with self.assertRaises(PermissionError):m.transfer('pending',ident(13))
|
|
self.assertEqual(p.read_bytes(),before)
|
|
|
|
|
|
class QueueSyntheticTests(unittest.TestCase):
|
|
def exercise(self,failed=False,admission_errors=0):
|
|
import phase3_item_c_queue as q
|
|
from contextlib import ExitStack
|
|
from phase2_h1_worker_v2 import save
|
|
with tempfile.TemporaryDirectory() as tmp, ExitStack() as stack:
|
|
root=Path(tmp);save(root/'contract.json',{'synthetic':True})
|
|
c=dict(gpu_cap=205200,postfit_gpu_reserve=7200,preparation_charge_gpu_seconds=600,runaway_gpu_seconds=60000)
|
|
class FakeProcess:
|
|
def __init__(self,cmd,**kw):
|
|
self.pid=999991;self.returncode=1 if failed else 0
|
|
i=int(cmd[3]);jr=root/'jobs'/str(i);(jr/'worker').mkdir()
|
|
save(jr/'complete.json',dict(status='STOPPED_OR_FAILED' if failed else 'PASS'))
|
|
checkpoint=jr/'worker/final.pt';checkpoint.write_bytes(b'synthetic')
|
|
save(jr/'worker/result.json',dict(fold=i,checkpoint=str(checkpoint),checkpoint_sha256=q.sha(checkpoint)))
|
|
save(jr/'worker/standardization.json',dict(synthetic=True))
|
|
def poll(self):return self.returncode
|
|
def wait(self):return self.returncode
|
|
def terminate(self):pass
|
|
def kill(self):pass
|
|
attempts=[]
|
|
def acquire(root,i,digest):
|
|
attempts.append(i)
|
|
if len(attempts)<=admission_errors:raise TimeoutError('synthetic admission query')
|
|
return dict(token=str(i),gpu_uuid=GPUS[1],worker_cpus=[16]),dict(synthetic=True)
|
|
patches=dict(config=lambda r:(c,{}),validate_review=lambda r:None,register_entry=lambda *a:None,
|
|
acquire=acquire,transfer=lambda *a:None,release=lambda *a:None,
|
|
process_identity=lambda pid:dict(pid=pid,created=0),freeze=lambda *a:None)
|
|
for k,v in patches.items():stack.enter_context(patch.object(q,k,v))
|
|
stack.enter_context(patch.object(q.subprocess,'Popen',FakeProcess))
|
|
stack.enter_context(patch.object(q.time,'sleep',lambda t:None))
|
|
stack.enter_context(patch.object(q.signal,'signal',lambda *a:None))
|
|
stack.enter_context(patch.object(q.psutil,'disk_usage',return_value=type('D',(),{'free':200*GIB})()))
|
|
stack.enter_context(patch.dict(q.os.environ,PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True'))
|
|
result=q.run(root);done=q.read(root/'complete.json')
|
|
if admission_errors>=3:
|
|
self.assertEqual(result,1);self.assertEqual(done['never_attempted'],[0,1,2,3,4])
|
|
self.assertEqual(len(attempts),3)
|
|
elif failed:
|
|
self.assertEqual(result,1);self.assertEqual(done['never_attempted'],[1,2,3,4]);self.assertFalse((root/'models.json').exists())
|
|
self.assertEqual(set(done['never_attempted_wait_charges']),{'1','2','3','4'})
|
|
else:
|
|
self.assertEqual(result,0);self.assertEqual(len(q.read(root/'models.json')),5)
|
|
self.assertEqual(done['status'],'TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING')
|
|
self.assertFalse(done['test_targets_read']);self.assertFalse(done['prediction_started'])
|
|
def test_all_five_terminate_before_model_freeze(self):self.exercise()
|
|
def test_failure_stops_without_retry(self):self.exercise(failed=True)
|
|
def test_two_failed_admission_samples_then_success(self):self.exercise(admission_errors=2)
|
|
def test_third_failed_admission_stops(self):self.exercise(admission_errors=3)
|
|
|
|
|
|
if __name__=='__main__':unittest.main(verbosity=2)
|