Files
MeltMind/tools/test_phase3_item_c.py
T

309 lines
19 KiB
Python

"""CPU synthetic acceptance only: no original arrays, labels, CUDA or fits."""
import ast
import copy
import json
import multiprocessing
from pathlib import Path
import tempfile
import unittest
from contextlib import contextmanager,ExitStack
from types import SimpleNamespace
from unittest.mock import patch
import numpy as np
import torch
from phase3_item_c_loss import candidates,location_loss,objective,THETA
from phase3_item_c_data import require_learning
from phase3_item_c_common import project
from phase3_shared_capacity import admission,GPUS,GIB,PHASE_RESERVES
from s2_custody_statistics import last_location
from s2_main_memory_guard import PeakRecords
def snapshot():
return dict(devices={u:dict(total=16*GIB,used=2*GIB) for u in GPUS},
processes=[dict(gpu_uuid=GPUS[1],pid=101,vram=2*GIB)],
ram_total=128*GIB,ram_available=96*GIB,cpu_percent=20,old_pending=False)
def lease(pid=102):
return dict(state='GUARD_BOUND',gpu_uuid=GPUS[1],observed_pids=[pid],observed_rss=GIB,vram_reserve=4*GIB,rss_reserve=4*GIB)
def concurrent_admit(shared, barrier, results):
import phase3_shared_capacity as m
m.SHARED=Path(shared)
s=snapshot()
# two actual tasks leave one slot onGPU1; GPU2 already has three.
s['processes'].append(dict(gpu_uuid=GPUS[1],pid=103,vram=GIB))
s['processes'] += [dict(gpu_uuid=GPUS[2],pid=p,vram=GIB) for p in (201,202,203)]
m.host_snapshot=lambda:s
try:
l,_=m.acquire(Path('/synthetic'),0,'abc')
results.put(bool(l))
barrier.wait(timeout=10) # keep lease owners alive through both admissions
except BaseException as exc:
results.put(type(exc).__name__+': '+str(exc))
class ObjectiveTests(unittest.TestCase):
def test_exact_old_peak_tie(self):
raw=np.array([7,19,42]);xyz=np.array([[0.,0,0],[2.,0,0],[0.,3,0]])
truth=np.array([5.,5.,1.]);q=candidates(raw,raw,np.ones(3,dtype=int),xyz,truth)
self.assertEqual(q['true_peak_raw_id'],7)
for i in range(3):
p=np.zeros(3);p[i]=1
self.assertEqual(q['distances'][i],last_location(truth,p,raw,xyz))
def test_duplicate_order_does_not_vote(self):
full=np.array([1,2,3]);ids=np.array([3,1,3,2]);xyz=np.array([[2.,0,0],[0,0,0],[2,0,0],[1,0,0]])
q=candidates(full,ids,np.array([50,1,99,2]),xyz,np.array([9.,10,9,8]))
np.testing.assert_array_equal(q['indices'],[0,1,2]);np.testing.assert_array_equal(q['distances'],[0,1,2])
def test_bad_duplicate_fails(self):
with self.assertRaises(ValueError):candidates(np.array([1]),np.array([1,1]),np.array([1,1]),np.zeros((2,3)),np.array([1.,2.]))
def test_missing_id_fails(self):
with self.assertRaises(ValueError):candidates(np.array([1]),np.array([2]),np.array([1]),np.zeros((1,3)),np.array([1.]))
def test_nan_truth_not_undefined(self):
with self.assertRaises(ValueError):candidates(np.array([1]),np.array([1]),np.array([1]),np.zeros((1,3)),np.array([np.nan]))
def test_gradient_formula(self):
r=torch.tensor([.01,-.02,.03],dtype=torch.float64,requires_grad=True);prior=torch.tensor([.7,.8,.9],dtype=torch.float64)
d=torch.tensor([2.,1.,0.],dtype=torch.float64);idx=torch.arange(3)
loss=location_loss(r,prior,idx,d);loss.backward();p=torch.softmax((r.detach()+prior)/THETA,0)
torch.testing.assert_close(r.grad,p*(d-(p*d).sum())/THETA)
def test_constant_shift(self):
r=torch.tensor([.1,.2],dtype=torch.float64);p=torch.tensor([.7,.8],dtype=torch.float64);d=torch.tensor([2.,0.],dtype=torch.float64)
torch.testing.assert_close(location_loss(r,p,torch.arange(2),d),location_loss(r+12,p,torch.arange(2),d))
def test_degenerate_gradients_zero(self):
for d in (torch.tensor([2.],dtype=torch.float64),torch.tensor([2.,2.,2.],dtype=torch.float64)):
r=torch.arange(len(d),dtype=torch.float64,requires_grad=True);location_loss(r,torch.zeros_like(r),torch.arange(len(d)),d).backward()
torch.testing.assert_close(r.grad,torch.zeros_like(r))
def test_fp64_prior_preserved(self):
r=torch.zeros(2,dtype=torch.float32,requires_grad=True);p=torch.tensor([1.,1.+1e-8],dtype=torch.float64);d=torch.tensor([0.,1.],dtype=torch.float64)
loss=location_loss(r,p,torch.arange(2),d);self.assertGreater(float(loss),.5);loss.backward();self.assertTrue(torch.isfinite(r.grad).all())
with self.assertRaises(ValueError):location_loss(r,p.float(),torch.arange(2),d)
def test_original_weight_plus_one_location(self):
p=torch.tensor([.2,.4]);t=torch.tensor([.1,.7]);prior=torch.tensor([.7,.6],dtype=torch.float64);d=torch.tensor([0.,3.],dtype=torch.float64)
total,reg,loc=objective(p,t,7,2,19,prior,torch.arange(2),d)
self.assertEqual(float(reg),float(((p-t)**2).mean()*(14/19)))
torch.testing.assert_close(total-reg,.001*loc)
def test_no_validation_or_test_role(self):
view=dict(folds=[dict(learning=['a'],validation=['b'],test=['c'])])
require_learning(view,'a')
for case in ('b','c','missing'):
with self.assertRaises(PermissionError):require_learning(view,case)
def test_worker_only_loads_learning(self):
tree=ast.parse((Path(__file__).parent/'phase3_item_c_worker.py').read_text())
calls=[n for n in ast.walk(tree) if isinstance(n,ast.Call) and isinstance(n.func,ast.Name) and n.func.id=='load_part']
self.assertEqual(len(calls),1);self.assertEqual(calls[0].args[3].value,'learning')
class CapacityTests(unittest.TestCase):
def test_visible_reserved_process_counted_once(self):
self.assertTrue(admission(snapshot(),[lease(101)],GPUS[1])[0])
def test_unseen_reservations_block_fourth(self):
self.assertFalse(admission(snapshot(),[lease(102),lease(103)],GPUS[1])[0])
def test_foreign_processes_count(self):
s=snapshot();s['processes'] += [dict(gpu_uuid=GPUS[1],pid=i,vram=GIB) for i in (102,103)]
self.assertEqual(admission(s,[],GPUS[1])[1],'GPU_SLOT_LIMIT')
def test_gpu3_single(self):
s=snapshot();s['processes']=[dict(gpu_uuid=GPUS[3],pid=1,vram=1)]
self.assertFalse(admission(s,[],GPUS[3])[0])
def test_gpu0_old_pending(self):
s=snapshot();s['old_pending']=True;self.assertEqual(admission(s,[],GPUS[0])[1],'GPU0_RESERVED_FOR_OLD_PENDING')
def test_memory_reserves_future_peaks(self):
s=snapshot();s['devices'][GPUS[1]]['used']=9*GIB
self.assertEqual(admission(s,[lease(101)],GPUS[1])[1],'GPU_HEADROOM')
def test_host_ram_reserve(self):
s=snapshot();s['ram_available']=30*GIB;self.assertEqual(admission(s,[],GPUS[1])[1],'HOST_RAM_HEADROOM')
def test_cpu_headroom(self):
s=snapshot();s['cpu_percent']=81;self.assertEqual(admission(s,[],GPUS[1])[1],'HOST_CPU_HEADROOM')
def test_phase_reservation_once(self):
self.assertEqual(PHASE_RESERVES['seed1'],167200);self.assertLess(sum(PHASE_RESERVES.values()),720000)
def test_stale_lease_blocks_without_reclaim(self):
import phase3_shared_capacity as m
with tempfile.TemporaryDirectory() as tmp,patch.object(m,'SHARED',Path(tmp)):
owner=m.process_identity(__import__('os').getpid())
stale=dict(token='stale',owner=dict(pid=99999999,created=0))
Path(tmp,'registry.json').write_text(json.dumps(dict(version=2,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=owner)},leases={'stale':stale})))
decision,reason=m.acquire(Path('/synthetic'),0,'abc')
self.assertIsNone(decision);self.assertIn('STALE_LEASE',reason['reason'])
self.assertIn('stale',json.loads(Path(tmp,'registry.json').read_text())['leases'])
def test_two_admissions_atomic(self):
import phase3_shared_capacity as m
with tempfile.TemporaryDirectory() as tmp:
parent=m.process_identity(__import__('os').getpid())
Path(tmp,'registry.json').write_text(json.dumps(dict(version=2,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=parent)},leases={})))
ctx=multiprocessing.get_context('fork');barrier=ctx.Barrier(2);results=ctx.Queue()
ps=[ctx.Process(target=concurrent_admit,args=(tmp,barrier,results)) for _ in range(2)]
for p in ps:p.start()
for p in ps:p.join(15)
try:
self.assertTrue(all(p.exitcode==0 for p in ps))
self.assertEqual(sorted([results.get(timeout=1),results.get(timeout=1)]),[False,True])
finally:
for p in ps:
if p.is_alive():p.kill();p.join()
class BudgetTests(unittest.TestCase):
def setUp(self):self.c=dict(gpu_cap=205200,postfit_gpu_reserve=7200,preparation_charge_gpu_seconds=600,runaway_gpu_seconds=60000)
def test_initial_inclusive_cap(self):
p=project(self.c,{i:60 for i in range(5)},{},{})
self.assertEqual(p['projected_gpu_seconds'],205200);self.assertEqual(p['status'],'PASS')
def test_pending_wait_not_free(self):
p=project(self.c,{i:50000 for i in range(5)},{},{})
self.assertEqual(p['status'],'STOP_BUDGET')
def test_projected_fold_above_runaway_not_alone_stop(self):
h={0:[dict(epoch=1,train_epoch_seconds=220)]}
p=project(self.c,{i:1000 for i in range(5)},h,{1:{},2:{},3:{}})
self.assertGreater(p['folds']['0']['projected'],60000);self.assertEqual(p['status'],'PASS')
def test_actual_runaway_stops(self):
self.assertEqual(project(self.c,{i:60000 if i==0 else 1 for i in range(5)},{},{})['status'],'STOP_BUDGET')
def test_negative_nan_cost(self):
for v in (-1,float('nan')):
with self.assertRaises(ValueError):project(self.c,{i:v for i in range(5)},{},{})
def test_incomplete_epoch_history(self):
with self.assertRaises(ValueError):project(self.c,{i:60 for i in range(5)},{0:[dict(epoch=2,train_epoch_seconds=10)]},{})
def test_failed_worker_late_memory_peak(self):
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp)/'failure.json';p.write_text(json.dumps(dict(peak_reserved_bytes=3*GIB,peak_allocated_bytes=GIB)))
tracker=PeakRecords();tracker.scan(tmp)
self.assertGreater(max(tracker.maxima.values()),2*GIB)
p.write_text(json.dumps(dict(peak_reserved_bytes=1)));tracker.scan(tmp)
self.assertEqual(tracker.maxima['peak_reserved_bytes'],3*GIB)
class ReservationIdentityDeltaTests(unittest.TestCase):
@contextmanager
def interleaving(self):
"""Assist's actual-acquire scenario, now with explicit lease states."""
import phase3_shared_capacity as m
ident=lambda p:dict(pid=p,created=1.)
root=Path('/synthetic-item-c');q=ident(10)
def l(token,pid,fold,state):
return dict(token=token,state=state,root=str(root),fold=fold,contract_sha256='synthetic',
gpu_uuid=GPUS[1],worker_cpus=[16+fold],owner=ident(pid),queue_owner=q,
vram_reserve=4*GIB,rss_reserve=4*GIB,created_unix=0)
leases={'first':l('first',11,0,'GUARD_BOUND'),'second':l('second',12,1,'GUARD_BOUND'),
'pending':l('pending',10,2,'RESERVED')}
trees={11:{11,101},12:{12,102},13:{13,103},10:{10,11,101,12,102}}
s=snapshot();s['devices'][GPUS[1]]['used']=4*GIB
s['processes']=[dict(gpu_uuid=GPUS[1],pid=p,vram=2*GIB) for p in (101,102)]
s['processes'] += [dict(gpu_uuid=GPUS[2],pid=p,vram=GIB) for p in (201,202,203)]
with tempfile.TemporaryDirectory() as tmp,ExitStack() as stack:
shared=Path(tmp);p=shared/'registry.json'
p.write_text(json.dumps(dict(version=2,entries={'c':dict(root=str(root),contract_sha256='synthetic',owner=q)},leases=leases)))
mocks=dict(SHARED=shared,alive=lambda x:True,descendants=lambda x:trees[x['pid']],
host_snapshot=lambda:copy.deepcopy(s),process_identity=lambda pid:q)
for k,v in mocks.items():stack.enter_context(patch.object(m,k,v))
stack.enter_context(patch.object(m.psutil,'Process',lambda pid:SimpleNamespace(ppid=lambda:10,memory_info=lambda:SimpleNamespace(rss=GIB//4))))
stack.enter_context(patch.object(m.os,'sched_getaffinity',lambda pid:set(range(16,26))))
yield m,root,p,ident,s
def test_actual_acquire_two_visible_plus_reserved_refuses_fourth(self):
with self.interleaving() as (m,root,p,ident,s):
before=p.read_bytes()
with patch.object(m,'descendants',wraps=m.descendants) as seen:
decision,observation=m.acquire(root,3,'synthetic')
self.assertNotIn(10,[call.args[0]['pid'] for call in seen.call_args_list])
self.assertIsNone(decision);self.assertEqual(observation['reason'],'NO_CAPACITY')
self.assertEqual(json.loads(before),json.loads(p.read_text()))
def test_reserved_sibling_claims_ignored_for_slots(self):
s=snapshot();s['processes'].append(dict(gpu_uuid=GPUS[1],pid=102,vram=2*GIB))
pending=dict(lease(101),state='RESERVED',observed_pids=[101,102],observed_rss=99*GIB)
self.assertEqual(admission(s,[lease(101),lease(102),pending],GPUS[1]),(False,'GPU_SLOT_LIMIT'))
def test_reserved_holds_full_gpu_memory(self):
s=snapshot();s['devices'][GPUS[1]]['used']=3*GIB
pending=dict(lease(101),state='RESERVED')
# 3GiB actual +2GiB visible-job deficit +4GiB reserved +4GiB proposed
# =13GiB, above12.8GiB. Crediting the sibling would incorrectly give11.
self.assertEqual(admission(s,[lease(101),pending],GPUS[1]),(False,'GPU_HEADROOM'))
def test_reserved_holds_full_rss_memory(self):
s=snapshot();s['ram_available']=44*GIB
pending=dict(lease(101),state='RESERVED',observed_rss=99*GIB)
self.assertEqual(admission(s,[lease(101),pending],GPUS[1]),(False,'HOST_RAM_HEADROOM'))
def test_transfer_binds_only_new_guard_and_child(self):
with self.interleaving() as (m,root,p,ident,s):
m.transfer('pending',ident(13))
d=json.loads(p.read_text());l=d['leases']['pending']
self.assertEqual(l['state'],'GUARD_BOUND');self.assertEqual(l['owner'],ident(13))
with patch.object(m,'descendants',wraps=m.descendants) as seen:
decision,_=m.acquire(root,3,'synthetic')
self.assertEqual({call.args[0]['pid'] for call in seen.call_args_list},{11,12,13})
self.assertIsNone(decision);self.assertEqual(d,json.loads(p.read_text()))
def test_visible_after_transfer_deduplicates_once(self):
s=snapshot();s['devices'][GPUS[1]]['used']=4*GIB
s['processes'].append(dict(gpu_uuid=GPUS[1],pid=102,vram=2*GIB))
self.assertEqual(admission(s,[lease(101),lease(102)],GPUS[1]),(True,'ADMITTED'))
def test_overlapping_bound_pid_rejected(self):
self.assertEqual(admission(snapshot(),[lease(101),lease(101)],GPUS[1]),(False,'LEASE_PID_OWNERSHIP_OVERLAP'))
def test_overlapping_non_gpu_pid_also_rejected(self):
a=dict(lease(101),observed_pids=[11,101]);b=dict(lease(102),observed_pids=[11,102])
self.assertEqual(admission(snapshot(),[a,b],GPUS[1]),(False,'LEASE_PID_OWNERSHIP_OVERLAP'))
def test_guard_cannot_be_bound_twice(self):
with self.interleaving() as (m,root,p,ident,s):
before=p.read_bytes()
with self.assertRaises(PermissionError):m.transfer('pending',ident(11))
self.assertEqual(p.read_bytes(),before)
def test_queue_cannot_be_its_own_guard(self):
with self.interleaving() as (m,root,p,ident,s):
before=p.read_bytes()
with self.assertRaises(PermissionError):m.transfer('pending',ident(10))
self.assertEqual(p.read_bytes(),before)
def test_transfer_is_one_way(self):
with self.interleaving() as (m,root,p,ident,s):
m.transfer('pending',ident(13));before=p.read_bytes()
with self.assertRaises(PermissionError):m.transfer('pending',ident(13))
self.assertEqual(p.read_bytes(),before)
class QueueSyntheticTests(unittest.TestCase):
def exercise(self,failed=False,admission_errors=0):
import phase3_item_c_queue as q
from contextlib import ExitStack
from phase2_h1_worker_v2 import save
with tempfile.TemporaryDirectory() as tmp, ExitStack() as stack:
root=Path(tmp);save(root/'contract.json',{'synthetic':True})
c=dict(gpu_cap=205200,postfit_gpu_reserve=7200,preparation_charge_gpu_seconds=600,runaway_gpu_seconds=60000)
class FakeProcess:
def __init__(self,cmd,**kw):
self.pid=999991;self.returncode=1 if failed else 0
i=int(cmd[3]);jr=root/'jobs'/str(i);(jr/'worker').mkdir()
save(jr/'complete.json',dict(status='STOPPED_OR_FAILED' if failed else 'PASS'))
checkpoint=jr/'worker/final.pt';checkpoint.write_bytes(b'synthetic')
save(jr/'worker/result.json',dict(fold=i,checkpoint=str(checkpoint),checkpoint_sha256=q.sha(checkpoint)))
save(jr/'worker/standardization.json',dict(synthetic=True))
def poll(self):return self.returncode
def wait(self):return self.returncode
def terminate(self):pass
def kill(self):pass
attempts=[]
def acquire(root,i,digest):
attempts.append(i)
if len(attempts)<=admission_errors:raise TimeoutError('synthetic admission query')
return dict(token=str(i),gpu_uuid=GPUS[1],worker_cpus=[16]),dict(synthetic=True)
patches=dict(config=lambda r:(c,{}),validate_review=lambda r:None,register_entry=lambda *a:None,
acquire=acquire,transfer=lambda *a:None,release=lambda *a:None,
process_identity=lambda pid:dict(pid=pid,created=0),freeze=lambda *a:None)
for k,v in patches.items():stack.enter_context(patch.object(q,k,v))
stack.enter_context(patch.object(q.subprocess,'Popen',FakeProcess))
stack.enter_context(patch.object(q.time,'sleep',lambda t:None))
stack.enter_context(patch.object(q.signal,'signal',lambda *a:None))
stack.enter_context(patch.object(q.psutil,'disk_usage',return_value=type('D',(),{'free':200*GIB})()))
stack.enter_context(patch.dict(q.os.environ,PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True'))
result=q.run(root);done=q.read(root/'complete.json')
if admission_errors>=3:
self.assertEqual(result,1);self.assertEqual(done['never_attempted'],[0,1,2,3,4])
self.assertEqual(len(attempts),3)
elif failed:
self.assertEqual(result,1);self.assertEqual(done['never_attempted'],[1,2,3,4]);self.assertFalse((root/'models.json').exists())
self.assertEqual(set(done['never_attempted_wait_charges']),{'1','2','3','4'})
else:
self.assertEqual(result,0);self.assertEqual(len(q.read(root/'models.json')),5)
self.assertEqual(done['status'],'TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING')
self.assertFalse(done['test_targets_read']);self.assertFalse(done['prediction_started'])
def test_all_five_terminate_before_model_freeze(self):self.exercise()
def test_failure_stops_without_retry(self):self.exercise(failed=True)
def test_two_failed_admission_samples_then_success(self):self.exercise(admission_errors=2)
def test_third_failed_admission_stops(self):self.exercise(admission_errors=3)
if __name__=='__main__':unittest.main(verbosity=2)