"""CPU synthetic acceptance only: no original arrays, labels, CUDA or fits.""" import ast import copy import json import multiprocessing from pathlib import Path import tempfile import unittest from unittest.mock import patch import numpy as np import torch from phase3_item_c_loss import candidates,location_loss,objective,THETA from phase3_item_c_data import require_learning from phase3_item_c_common import project from phase3_shared_capacity import admission,GPUS,GIB,PHASE_RESERVES from s2_custody_statistics import last_location from s2_main_memory_guard import PeakRecords def snapshot(): return dict(devices={u:dict(total=16*GIB,used=2*GIB) for u in GPUS}, processes=[dict(gpu_uuid=GPUS[1],pid=101,vram=2*GIB)], ram_total=128*GIB,ram_available=96*GIB,cpu_percent=20,old_pending=False) def lease(pid=102): return dict(gpu_uuid=GPUS[1],observed_pids=[pid],observed_rss=GIB,vram_reserve=4*GIB,rss_reserve=4*GIB) def concurrent_admit(shared, barrier, results): import phase3_shared_capacity as m m.SHARED=Path(shared) s=snapshot() # two actual tasks leave one slot onGPU1; GPU2 already has three. s['processes'].append(dict(gpu_uuid=GPUS[1],pid=103,vram=GIB)) s['processes'] += [dict(gpu_uuid=GPUS[2],pid=p,vram=GIB) for p in (201,202,203)] m.host_snapshot=lambda:s try: l,_=m.acquire(Path('/synthetic'),0,'abc') results.put(bool(l)) barrier.wait(timeout=10) # keep lease owners alive through both admissions except BaseException as exc: results.put(type(exc).__name__+': '+str(exc)) class ObjectiveTests(unittest.TestCase): def test_exact_old_peak_tie(self): raw=np.array([7,19,42]);xyz=np.array([[0.,0,0],[2.,0,0],[0.,3,0]]) truth=np.array([5.,5.,1.]);q=candidates(raw,raw,np.ones(3,dtype=int),xyz,truth) self.assertEqual(q['true_peak_raw_id'],7) for i in range(3): p=np.zeros(3);p[i]=1 self.assertEqual(q['distances'][i],last_location(truth,p,raw,xyz)) def test_duplicate_order_does_not_vote(self): full=np.array([1,2,3]);ids=np.array([3,1,3,2]);xyz=np.array([[2.,0,0],[0,0,0],[2,0,0],[1,0,0]]) q=candidates(full,ids,np.array([50,1,99,2]),xyz,np.array([9.,10,9,8])) np.testing.assert_array_equal(q['indices'],[0,1,2]);np.testing.assert_array_equal(q['distances'],[0,1,2]) def test_bad_duplicate_fails(self): with self.assertRaises(ValueError):candidates(np.array([1]),np.array([1,1]),np.array([1,1]),np.zeros((2,3)),np.array([1.,2.])) def test_missing_id_fails(self): with self.assertRaises(ValueError):candidates(np.array([1]),np.array([2]),np.array([1]),np.zeros((1,3)),np.array([1.])) def test_nan_truth_not_undefined(self): with self.assertRaises(ValueError):candidates(np.array([1]),np.array([1]),np.array([1]),np.zeros((1,3)),np.array([np.nan])) def test_gradient_formula(self): r=torch.tensor([.01,-.02,.03],dtype=torch.float64,requires_grad=True);prior=torch.tensor([.7,.8,.9],dtype=torch.float64) d=torch.tensor([2.,1.,0.],dtype=torch.float64);idx=torch.arange(3) loss=location_loss(r,prior,idx,d);loss.backward();p=torch.softmax((r.detach()+prior)/THETA,0) torch.testing.assert_close(r.grad,p*(d-(p*d).sum())/THETA) def test_constant_shift(self): r=torch.tensor([.1,.2],dtype=torch.float64);p=torch.tensor([.7,.8],dtype=torch.float64);d=torch.tensor([2.,0.],dtype=torch.float64) torch.testing.assert_close(location_loss(r,p,torch.arange(2),d),location_loss(r+12,p,torch.arange(2),d)) def test_degenerate_gradients_zero(self): for d in (torch.tensor([2.],dtype=torch.float64),torch.tensor([2.,2.,2.],dtype=torch.float64)): r=torch.arange(len(d),dtype=torch.float64,requires_grad=True);location_loss(r,torch.zeros_like(r),torch.arange(len(d)),d).backward() torch.testing.assert_close(r.grad,torch.zeros_like(r)) def test_fp64_prior_preserved(self): r=torch.zeros(2,dtype=torch.float32,requires_grad=True);p=torch.tensor([1.,1.+1e-8],dtype=torch.float64);d=torch.tensor([0.,1.],dtype=torch.float64) loss=location_loss(r,p,torch.arange(2),d);self.assertGreater(float(loss),.5);loss.backward();self.assertTrue(torch.isfinite(r.grad).all()) with self.assertRaises(ValueError):location_loss(r,p.float(),torch.arange(2),d) def test_original_weight_plus_one_location(self): p=torch.tensor([.2,.4]);t=torch.tensor([.1,.7]);prior=torch.tensor([.7,.6],dtype=torch.float64);d=torch.tensor([0.,3.],dtype=torch.float64) total,reg,loc=objective(p,t,7,2,19,prior,torch.arange(2),d) self.assertEqual(float(reg),float(((p-t)**2).mean()*(14/19))) torch.testing.assert_close(total-reg,.001*loc) def test_no_validation_or_test_role(self): view=dict(folds=[dict(learning=['a'],validation=['b'],test=['c'])]) require_learning(view,'a') for case in ('b','c','missing'): with self.assertRaises(PermissionError):require_learning(view,case) def test_worker_only_loads_learning(self): tree=ast.parse((Path(__file__).parent/'phase3_item_c_worker.py').read_text()) calls=[n for n in ast.walk(tree) if isinstance(n,ast.Call) and isinstance(n.func,ast.Name) and n.func.id=='load_part'] self.assertEqual(len(calls),1);self.assertEqual(calls[0].args[3].value,'learning') class CapacityTests(unittest.TestCase): def test_visible_reserved_process_counted_once(self): self.assertTrue(admission(snapshot(),[lease(101)],GPUS[1])[0]) def test_unseen_reservations_block_fourth(self): self.assertFalse(admission(snapshot(),[lease(102),lease(103)],GPUS[1])[0]) def test_foreign_processes_count(self): s=snapshot();s['processes'] += [dict(gpu_uuid=GPUS[1],pid=i,vram=GIB) for i in (102,103)] self.assertEqual(admission(s,[],GPUS[1])[1],'GPU_SLOT_LIMIT') def test_gpu3_single(self): s=snapshot();s['processes']=[dict(gpu_uuid=GPUS[3],pid=1,vram=1)] self.assertFalse(admission(s,[],GPUS[3])[0]) def test_gpu0_old_pending(self): s=snapshot();s['old_pending']=True;self.assertEqual(admission(s,[],GPUS[0])[1],'GPU0_RESERVED_FOR_OLD_PENDING') def test_memory_reserves_future_peaks(self): s=snapshot();s['devices'][GPUS[1]]['used']=9*GIB self.assertEqual(admission(s,[lease(101)],GPUS[1])[1],'GPU_HEADROOM') def test_host_ram_reserve(self): s=snapshot();s['ram_available']=30*GIB;self.assertEqual(admission(s,[],GPUS[1])[1],'HOST_RAM_HEADROOM') def test_cpu_headroom(self): s=snapshot();s['cpu_percent']=81;self.assertEqual(admission(s,[],GPUS[1])[1],'HOST_CPU_HEADROOM') def test_phase_reservation_once(self): self.assertEqual(PHASE_RESERVES['seed1'],167200);self.assertLess(sum(PHASE_RESERVES.values()),720000) def test_stale_lease_blocks_without_reclaim(self): import phase3_shared_capacity as m with tempfile.TemporaryDirectory() as tmp,patch.object(m,'SHARED',Path(tmp)): owner=m.process_identity(__import__('os').getpid()) stale=dict(token='stale',owner=dict(pid=99999999,created=0)) Path(tmp,'registry.json').write_text(json.dumps(dict(version=1,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=owner)},leases={'stale':stale}))) decision,reason=m.acquire(Path('/synthetic'),0,'abc') self.assertIsNone(decision);self.assertIn('STALE_LEASE',reason['reason']) self.assertIn('stale',json.loads(Path(tmp,'registry.json').read_text())['leases']) def test_two_admissions_atomic(self): import phase3_shared_capacity as m with tempfile.TemporaryDirectory() as tmp: parent=m.process_identity(__import__('os').getpid()) Path(tmp,'registry.json').write_text(json.dumps(dict(version=1,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=parent)},leases={}))) ctx=multiprocessing.get_context('fork');barrier=ctx.Barrier(2);results=ctx.Queue() ps=[ctx.Process(target=concurrent_admit,args=(tmp,barrier,results)) for _ in range(2)] for p in ps:p.start() for p in ps:p.join(15) try: self.assertTrue(all(p.exitcode==0 for p in ps)) self.assertEqual(sorted([results.get(timeout=1),results.get(timeout=1)]),[False,True]) finally: for p in ps: if p.is_alive():p.kill();p.join() class BudgetTests(unittest.TestCase): def setUp(self):self.c=dict(gpu_cap=205200,postfit_gpu_reserve=7200,preparation_charge_gpu_seconds=600,runaway_gpu_seconds=60000) def test_initial_inclusive_cap(self): p=project(self.c,{i:60 for i in range(5)},{},{}) self.assertEqual(p['projected_gpu_seconds'],205200);self.assertEqual(p['status'],'PASS') def test_pending_wait_not_free(self): p=project(self.c,{i:50000 for i in range(5)},{},{}) self.assertEqual(p['status'],'STOP_BUDGET') def test_projected_fold_above_runaway_not_alone_stop(self): h={0:[dict(epoch=1,train_epoch_seconds=220)]} p=project(self.c,{i:1000 for i in range(5)},h,{1:{},2:{},3:{}}) self.assertGreater(p['folds']['0']['projected'],60000);self.assertEqual(p['status'],'PASS') def test_actual_runaway_stops(self): self.assertEqual(project(self.c,{i:60000 if i==0 else 1 for i in range(5)},{},{})['status'],'STOP_BUDGET') def test_negative_nan_cost(self): for v in (-1,float('nan')): with self.assertRaises(ValueError):project(self.c,{i:v for i in range(5)},{},{}) def test_incomplete_epoch_history(self): with self.assertRaises(ValueError):project(self.c,{i:60 for i in range(5)},{0:[dict(epoch=2,train_epoch_seconds=10)]},{}) def test_failed_worker_late_memory_peak(self): with tempfile.TemporaryDirectory() as tmp: p=Path(tmp)/'failure.json';p.write_text(json.dumps(dict(peak_reserved_bytes=3*GIB,peak_allocated_bytes=GIB))) tracker=PeakRecords();tracker.scan(tmp) self.assertGreater(max(tracker.maxima.values()),2*GIB) p.write_text(json.dumps(dict(peak_reserved_bytes=1)));tracker.scan(tmp) self.assertEqual(tracker.maxima['peak_reserved_bytes'],3*GIB) class QueueSyntheticTests(unittest.TestCase): def exercise(self,failed=False,admission_errors=0): import phase3_item_c_queue as q from contextlib import ExitStack from phase2_h1_worker_v2 import save with tempfile.TemporaryDirectory() as tmp, ExitStack() as stack: root=Path(tmp);save(root/'contract.json',{'synthetic':True}) c=dict(gpu_cap=205200,postfit_gpu_reserve=7200,preparation_charge_gpu_seconds=600,runaway_gpu_seconds=60000) class FakeProcess: def __init__(self,cmd,**kw): self.pid=999991;self.returncode=1 if failed else 0 i=int(cmd[3]);jr=root/'jobs'/str(i);(jr/'worker').mkdir() save(jr/'complete.json',dict(status='STOPPED_OR_FAILED' if failed else 'PASS')) checkpoint=jr/'worker/final.pt';checkpoint.write_bytes(b'synthetic') save(jr/'worker/result.json',dict(fold=i,checkpoint=str(checkpoint),checkpoint_sha256=q.sha(checkpoint))) save(jr/'worker/standardization.json',dict(synthetic=True)) def poll(self):return self.returncode def wait(self):return self.returncode def terminate(self):pass def kill(self):pass attempts=[] def acquire(root,i,digest): attempts.append(i) if len(attempts)<=admission_errors:raise TimeoutError('synthetic admission query') return dict(token=str(i),gpu_uuid=GPUS[1],worker_cpus=[16]),dict(synthetic=True) patches=dict(config=lambda r:(c,{}),validate_review=lambda r:None,register_entry=lambda *a:None, acquire=acquire,transfer=lambda *a:None,release=lambda *a:None, process_identity=lambda pid:dict(pid=pid,created=0),freeze=lambda *a:None) for k,v in patches.items():stack.enter_context(patch.object(q,k,v)) stack.enter_context(patch.object(q.subprocess,'Popen',FakeProcess)) stack.enter_context(patch.object(q.time,'sleep',lambda t:None)) stack.enter_context(patch.object(q.signal,'signal',lambda *a:None)) stack.enter_context(patch.object(q.psutil,'disk_usage',return_value=type('D',(),{'free':200*GIB})())) stack.enter_context(patch.dict(q.os.environ,PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True')) result=q.run(root);done=q.read(root/'complete.json') if admission_errors>=3: self.assertEqual(result,1);self.assertEqual(done['never_attempted'],[0,1,2,3,4]) self.assertEqual(len(attempts),3) elif failed: self.assertEqual(result,1);self.assertEqual(done['never_attempted'],[1,2,3,4]);self.assertFalse((root/'models.json').exists()) self.assertEqual(set(done['never_attempted_wait_charges']),{'1','2','3','4'}) else: self.assertEqual(result,0);self.assertEqual(len(q.read(root/'models.json')),5) self.assertEqual(done['status'],'TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING') self.assertFalse(done['test_targets_read']);self.assertFalse(done['prediction_started']) def test_all_five_terminate_before_model_freeze(self):self.exercise() def test_failure_stops_without_retry(self):self.exercise(failed=True) def test_two_failed_admission_samples_then_success(self):self.exercise(admission_errors=2) def test_third_failed_admission_stops(self):self.exercise(admission_errors=3) if __name__=='__main__':unittest.main(verbosity=2)