Files
MeltMind/tools/test_phase3_item_c.py
T

224 lines
14 KiB
Python

"""CPU synthetic acceptance only: no original arrays, labels, CUDA or fits."""
import ast
import copy
import json
import multiprocessing
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
import numpy as np
import torch
from phase3_item_c_loss import candidates,location_loss,objective,THETA
from phase3_item_c_data import require_learning
from phase3_item_c_common import project
from phase3_shared_capacity import admission,GPUS,GIB,PHASE_RESERVES
from s2_custody_statistics import last_location
from s2_main_memory_guard import PeakRecords
def snapshot():
return dict(devices={u:dict(total=16*GIB,used=2*GIB) for u in GPUS},
processes=[dict(gpu_uuid=GPUS[1],pid=101,vram=2*GIB)],
ram_total=128*GIB,ram_available=96*GIB,cpu_percent=20,old_pending=False)
def lease(pid=102):
return dict(gpu_uuid=GPUS[1],observed_pids=[pid],observed_rss=GIB,vram_reserve=4*GIB,rss_reserve=4*GIB)
def concurrent_admit(shared, barrier, results):
import phase3_shared_capacity as m
m.SHARED=Path(shared)
s=snapshot()
# two actual tasks leave one slot onGPU1; GPU2 already has three.
s['processes'].append(dict(gpu_uuid=GPUS[1],pid=103,vram=GIB))
s['processes'] += [dict(gpu_uuid=GPUS[2],pid=p,vram=GIB) for p in (201,202,203)]
m.host_snapshot=lambda:s
try:
l,_=m.acquire(Path('/synthetic'),0,'abc')
results.put(bool(l))
barrier.wait(timeout=10) # keep lease owners alive through both admissions
except BaseException as exc:
results.put(type(exc).__name__+': '+str(exc))
class ObjectiveTests(unittest.TestCase):
def test_exact_old_peak_tie(self):
raw=np.array([7,19,42]);xyz=np.array([[0.,0,0],[2.,0,0],[0.,3,0]])
truth=np.array([5.,5.,1.]);q=candidates(raw,raw,np.ones(3,dtype=int),xyz,truth)
self.assertEqual(q['true_peak_raw_id'],7)
for i in range(3):
p=np.zeros(3);p[i]=1
self.assertEqual(q['distances'][i],last_location(truth,p,raw,xyz))
def test_duplicate_order_does_not_vote(self):
full=np.array([1,2,3]);ids=np.array([3,1,3,2]);xyz=np.array([[2.,0,0],[0,0,0],[2,0,0],[1,0,0]])
q=candidates(full,ids,np.array([50,1,99,2]),xyz,np.array([9.,10,9,8]))
np.testing.assert_array_equal(q['indices'],[0,1,2]);np.testing.assert_array_equal(q['distances'],[0,1,2])
def test_bad_duplicate_fails(self):
with self.assertRaises(ValueError):candidates(np.array([1]),np.array([1,1]),np.array([1,1]),np.zeros((2,3)),np.array([1.,2.]))
def test_missing_id_fails(self):
with self.assertRaises(ValueError):candidates(np.array([1]),np.array([2]),np.array([1]),np.zeros((1,3)),np.array([1.]))
def test_nan_truth_not_undefined(self):
with self.assertRaises(ValueError):candidates(np.array([1]),np.array([1]),np.array([1]),np.zeros((1,3)),np.array([np.nan]))
def test_gradient_formula(self):
r=torch.tensor([.01,-.02,.03],dtype=torch.float64,requires_grad=True);prior=torch.tensor([.7,.8,.9],dtype=torch.float64)
d=torch.tensor([2.,1.,0.],dtype=torch.float64);idx=torch.arange(3)
loss=location_loss(r,prior,idx,d);loss.backward();p=torch.softmax((r.detach()+prior)/THETA,0)
torch.testing.assert_close(r.grad,p*(d-(p*d).sum())/THETA)
def test_constant_shift(self):
r=torch.tensor([.1,.2],dtype=torch.float64);p=torch.tensor([.7,.8],dtype=torch.float64);d=torch.tensor([2.,0.],dtype=torch.float64)
torch.testing.assert_close(location_loss(r,p,torch.arange(2),d),location_loss(r+12,p,torch.arange(2),d))
def test_degenerate_gradients_zero(self):
for d in (torch.tensor([2.],dtype=torch.float64),torch.tensor([2.,2.,2.],dtype=torch.float64)):
r=torch.arange(len(d),dtype=torch.float64,requires_grad=True);location_loss(r,torch.zeros_like(r),torch.arange(len(d)),d).backward()
torch.testing.assert_close(r.grad,torch.zeros_like(r))
def test_fp64_prior_preserved(self):
r=torch.zeros(2,dtype=torch.float32,requires_grad=True);p=torch.tensor([1.,1.+1e-8],dtype=torch.float64);d=torch.tensor([0.,1.],dtype=torch.float64)
loss=location_loss(r,p,torch.arange(2),d);self.assertGreater(float(loss),.5);loss.backward();self.assertTrue(torch.isfinite(r.grad).all())
with self.assertRaises(ValueError):location_loss(r,p.float(),torch.arange(2),d)
def test_original_weight_plus_one_location(self):
p=torch.tensor([.2,.4]);t=torch.tensor([.1,.7]);prior=torch.tensor([.7,.6],dtype=torch.float64);d=torch.tensor([0.,3.],dtype=torch.float64)
total,reg,loc=objective(p,t,7,2,19,prior,torch.arange(2),d)
self.assertEqual(float(reg),float(((p-t)**2).mean()*(14/19)))
torch.testing.assert_close(total-reg,.001*loc)
def test_no_validation_or_test_role(self):
view=dict(folds=[dict(learning=['a'],validation=['b'],test=['c'])])
require_learning(view,'a')
for case in ('b','c','missing'):
with self.assertRaises(PermissionError):require_learning(view,case)
def test_worker_only_loads_learning(self):
tree=ast.parse((Path(__file__).parent/'phase3_item_c_worker.py').read_text())
calls=[n for n in ast.walk(tree) if isinstance(n,ast.Call) and isinstance(n.func,ast.Name) and n.func.id=='load_part']
self.assertEqual(len(calls),1);self.assertEqual(calls[0].args[3].value,'learning')
class CapacityTests(unittest.TestCase):
def test_visible_reserved_process_counted_once(self):
self.assertTrue(admission(snapshot(),[lease(101)],GPUS[1])[0])
def test_unseen_reservations_block_fourth(self):
self.assertFalse(admission(snapshot(),[lease(102),lease(103)],GPUS[1])[0])
def test_foreign_processes_count(self):
s=snapshot();s['processes'] += [dict(gpu_uuid=GPUS[1],pid=i,vram=GIB) for i in (102,103)]
self.assertEqual(admission(s,[],GPUS[1])[1],'GPU_SLOT_LIMIT')
def test_gpu3_single(self):
s=snapshot();s['processes']=[dict(gpu_uuid=GPUS[3],pid=1,vram=1)]
self.assertFalse(admission(s,[],GPUS[3])[0])
def test_gpu0_old_pending(self):
s=snapshot();s['old_pending']=True;self.assertEqual(admission(s,[],GPUS[0])[1],'GPU0_RESERVED_FOR_OLD_PENDING')
def test_memory_reserves_future_peaks(self):
s=snapshot();s['devices'][GPUS[1]]['used']=9*GIB
self.assertEqual(admission(s,[lease(101)],GPUS[1])[1],'GPU_HEADROOM')
def test_host_ram_reserve(self):
s=snapshot();s['ram_available']=30*GIB;self.assertEqual(admission(s,[],GPUS[1])[1],'HOST_RAM_HEADROOM')
def test_cpu_headroom(self):
s=snapshot();s['cpu_percent']=81;self.assertEqual(admission(s,[],GPUS[1])[1],'HOST_CPU_HEADROOM')
def test_phase_reservation_once(self):
self.assertEqual(PHASE_RESERVES['seed1'],167200);self.assertLess(sum(PHASE_RESERVES.values()),720000)
def test_stale_lease_blocks_without_reclaim(self):
import phase3_shared_capacity as m
with tempfile.TemporaryDirectory() as tmp,patch.object(m,'SHARED',Path(tmp)):
owner=m.process_identity(__import__('os').getpid())
stale=dict(token='stale',owner=dict(pid=99999999,created=0))
Path(tmp,'registry.json').write_text(json.dumps(dict(version=1,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=owner)},leases={'stale':stale})))
decision,reason=m.acquire(Path('/synthetic'),0,'abc')
self.assertIsNone(decision);self.assertIn('STALE_LEASE',reason['reason'])
self.assertIn('stale',json.loads(Path(tmp,'registry.json').read_text())['leases'])
def test_two_admissions_atomic(self):
import phase3_shared_capacity as m
with tempfile.TemporaryDirectory() as tmp:
parent=m.process_identity(__import__('os').getpid())
Path(tmp,'registry.json').write_text(json.dumps(dict(version=1,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=parent)},leases={})))
ctx=multiprocessing.get_context('fork');barrier=ctx.Barrier(2);results=ctx.Queue()
ps=[ctx.Process(target=concurrent_admit,args=(tmp,barrier,results)) for _ in range(2)]
for p in ps:p.start()
for p in ps:p.join(15)
try:
self.assertTrue(all(p.exitcode==0 for p in ps))
self.assertEqual(sorted([results.get(timeout=1),results.get(timeout=1)]),[False,True])
finally:
for p in ps:
if p.is_alive():p.kill();p.join()
class BudgetTests(unittest.TestCase):
def setUp(self):self.c=dict(gpu_cap=205200,postfit_gpu_reserve=7200,preparation_charge_gpu_seconds=600,runaway_gpu_seconds=60000)
def test_initial_inclusive_cap(self):
p=project(self.c,{i:60 for i in range(5)},{},{})
self.assertEqual(p['projected_gpu_seconds'],205200);self.assertEqual(p['status'],'PASS')
def test_pending_wait_not_free(self):
p=project(self.c,{i:50000 for i in range(5)},{},{})
self.assertEqual(p['status'],'STOP_BUDGET')
def test_projected_fold_above_runaway_not_alone_stop(self):
h={0:[dict(epoch=1,train_epoch_seconds=220)]}
p=project(self.c,{i:1000 for i in range(5)},h,{1:{},2:{},3:{}})
self.assertGreater(p['folds']['0']['projected'],60000);self.assertEqual(p['status'],'PASS')
def test_actual_runaway_stops(self):
self.assertEqual(project(self.c,{i:60000 if i==0 else 1 for i in range(5)},{},{})['status'],'STOP_BUDGET')
def test_negative_nan_cost(self):
for v in (-1,float('nan')):
with self.assertRaises(ValueError):project(self.c,{i:v for i in range(5)},{},{})
def test_incomplete_epoch_history(self):
with self.assertRaises(ValueError):project(self.c,{i:60 for i in range(5)},{0:[dict(epoch=2,train_epoch_seconds=10)]},{})
def test_failed_worker_late_memory_peak(self):
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp)/'failure.json';p.write_text(json.dumps(dict(peak_reserved_bytes=3*GIB,peak_allocated_bytes=GIB)))
tracker=PeakRecords();tracker.scan(tmp)
self.assertGreater(max(tracker.maxima.values()),2*GIB)
p.write_text(json.dumps(dict(peak_reserved_bytes=1)));tracker.scan(tmp)
self.assertEqual(tracker.maxima['peak_reserved_bytes'],3*GIB)
class QueueSyntheticTests(unittest.TestCase):
def exercise(self,failed=False,admission_errors=0):
import phase3_item_c_queue as q
from contextlib import ExitStack
from phase2_h1_worker_v2 import save
with tempfile.TemporaryDirectory() as tmp, ExitStack() as stack:
root=Path(tmp);save(root/'contract.json',{'synthetic':True})
c=dict(gpu_cap=205200,postfit_gpu_reserve=7200,preparation_charge_gpu_seconds=600,runaway_gpu_seconds=60000)
class FakeProcess:
def __init__(self,cmd,**kw):
self.pid=999991;self.returncode=1 if failed else 0
i=int(cmd[3]);jr=root/'jobs'/str(i);(jr/'worker').mkdir()
save(jr/'complete.json',dict(status='STOPPED_OR_FAILED' if failed else 'PASS'))
checkpoint=jr/'worker/final.pt';checkpoint.write_bytes(b'synthetic')
save(jr/'worker/result.json',dict(fold=i,checkpoint=str(checkpoint),checkpoint_sha256=q.sha(checkpoint)))
save(jr/'worker/standardization.json',dict(synthetic=True))
def poll(self):return self.returncode
def wait(self):return self.returncode
def terminate(self):pass
def kill(self):pass
attempts=[]
def acquire(root,i,digest):
attempts.append(i)
if len(attempts)<=admission_errors:raise TimeoutError('synthetic admission query')
return dict(token=str(i),gpu_uuid=GPUS[1],worker_cpus=[16]),dict(synthetic=True)
patches=dict(config=lambda r:(c,{}),validate_review=lambda r:None,register_entry=lambda *a:None,
acquire=acquire,transfer=lambda *a:None,release=lambda *a:None,
process_identity=lambda pid:dict(pid=pid,created=0),freeze=lambda *a:None)
for k,v in patches.items():stack.enter_context(patch.object(q,k,v))
stack.enter_context(patch.object(q.subprocess,'Popen',FakeProcess))
stack.enter_context(patch.object(q.time,'sleep',lambda t:None))
stack.enter_context(patch.object(q.signal,'signal',lambda *a:None))
stack.enter_context(patch.object(q.psutil,'disk_usage',return_value=type('D',(),{'free':200*GIB})()))
stack.enter_context(patch.dict(q.os.environ,PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True'))
result=q.run(root);done=q.read(root/'complete.json')
if admission_errors>=3:
self.assertEqual(result,1);self.assertEqual(done['never_attempted'],[0,1,2,3,4])
self.assertEqual(len(attempts),3)
elif failed:
self.assertEqual(result,1);self.assertEqual(done['never_attempted'],[1,2,3,4]);self.assertFalse((root/'models.json').exists())
self.assertEqual(set(done['never_attempted_wait_charges']),{'1','2','3','4'})
else:
self.assertEqual(result,0);self.assertEqual(len(q.read(root/'models.json')),5)
self.assertEqual(done['status'],'TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING')
self.assertFalse(done['test_targets_read']);self.assertFalse(done['prediction_started'])
def test_all_five_terminate_before_model_freeze(self):self.exercise()
def test_failure_stops_without_retry(self):self.exercise(failed=True)
def test_two_failed_admission_samples_then_success(self):self.exercise(admission_errors=2)
def test_third_failed_admission_stops(self):self.exercise(admission_errors=3)
if __name__=='__main__':unittest.main(verbosity=2)