Files
MeltMind/tests/test_track_b_gino_greif_combined.py
T

202 lines
11 KiB
Python

"""CPU replay for the reviewed Greif cycle, first-epoch continuity and one-read scorer."""
import json
from pathlib import Path
from tempfile import TemporaryDirectory
import time
import unittest
from unittest.mock import patch
import numpy as np
import torch
from s2_gpu_idle_gate import wait_idle
import track_b_gino_formal_guard as guard
import track_b_gino_greif_custody as custody
import track_b_gino_greif_entry as entry
import track_b_gino_greif_worker as worker
from track_b_gino_shapenet_entry import read, save, sha
EVIDENCE={}
class Combined(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.c=entry.contract()
cls.contract_path=entry.ROOT/'contract.json'
cls.comparison=read(entry.ROOT/'comparison-contract.json')
def test_replays_actual_production_guard_and_stage_order(self):
for busy in (False,True):
with self.subTest(busy=busy), TemporaryDirectory() as directory:
root=Path(directory)
(root/'contract.json').write_bytes(self.contract_path.read_bytes())
save(root/'stage-authorization.json',dict(test=True))
path=root/'track_b_gino_greif_worker.py'
path.write_text(
'import json,sys,pathlib\n'
'root=pathlib.Path(__file__).parent\n'
'fold,seed,phase=sys.argv[1:]\n'
'stage=root/f"fold-{fold}"/f"seed-{seed}"/phase\n'
'p=stage/"worker-records"; p.mkdir()\n'
'spec=json.loads((root/"contract.json").read_text())["folds"][fold]\n'
'obj=dict(learning_cases=spec["counts"]["learning"],learning_nodes=spec["learning_nodes"],'
'first_epoch_seconds=.01,files=[],peak_allocated_bytes=1,peak_reserved_bytes=2)\n'
'(p/"temp").write_text(json.dumps(obj)); (p/"temp").rename(p/"result.json")\n')
observed=[]
elapsed=[0.]
card=entry.DEVICES[0]
def idle(uuid):
return dict(uuid=uuid,idle=True,occupants=[],memory_MiB=0,unix=10.)
def device(uuid):
return dict(idle(uuid),idle=not busy)
def simulated_idle(uuid, limit, record, *, sample):
return wait_idle(uuid,limit,record,sample=sample,clock=lambda:elapsed[0],
sleep=lambda s:elapsed.__setitem__(0,elapsed[0]+s))
def validate(_,fold,seed,phase):
observed.append((fold,seed,phase))
return sha(entry.path(fold,seed,phase)/'worker-records/result.json')
with (patch.object(entry,'ROOT',root),patch.object(entry,'SOURCE',root),
patch.object(entry,'snapshot',side_effect=idle),
patch.object(entry,'validate_result',side_effect=validate),
patch.object(guard,'snapshot',side_effect=device),
patch.object(guard,'wait_idle',side_effect=simulated_idle),
patch.object(guard,'dispatch_environment',
side_effect=lambda u,env:(dict(env,CUDA_VISIBLE_DEVICES='0'),[]))):
result=entry.run(self.c,dict(dispatch_before_unix=time.time()+50),495434.9566447616)
with self.assertRaises(FileExistsError):
entry.run(self.c,dict(dispatch_before_unix=time.time()+50),495434.9566447616)
terminal=read(root/'complete.json')
p=root/'fold-0/seed-0/first'
if busy:
self.assertEqual(result,1)
self.assertIn('IDLE_DEADLINE',terminal['trigger'])
self.assertFalse((p/'actual-worker-spawn.json').is_file())
self.assertFalse((root/'predictions-frozen.json').exists())
self.assertEqual(observed,[])
else:
expected=[(0,0,'first'),(1,0,'first')]+[(f,s,p) for p in ('fit','predict') for f,s in entry.JOBS]
self.assertEqual(result,0)
self.assertEqual(observed,expected)
self.assertTrue(read(root/'first-epoch-cost-gate.json')['release_epoch2'])
for f,s,phase in observed:
event=read(root/f'fold-{f}'/f'seed-{s}'/phase/'complete.json')
self.assertEqual(event['physical_uuid'],card)
self.assertEqual([v['event'] for v in event['lifecycle']],['leader_reaped'])
self.assertFalse((root/'score').exists())
EVIDENCE['busy' if busy else 'nominal']=dict(
status=terminal['status'],trigger=terminal['trigger'],stages=observed,
cpu_child_spawns=len(list(root.rglob('actual-worker-spawn.json'))))
def test_first_cost_gate_counts_spent_and_blocks_over_budget(self):
common=dict(seed=0,epoch=1,heldout_target_reads=0,booked_gpu_seconds=400.)
p={k:dict(common,fold=int(k),epoch_seconds=70.,learning_nodes=r['learning_nodes'],
learning_cases=r['counts']['learning']) for k,r in self.c['folds'].items()}
gate=entry.cost_gate(self.c,p,495434.9566447616,820.)
self.assertTrue(gate['release_epoch2'])
self.assertEqual(gate['charged_first_epochs_and_setup_gpu_seconds'],800.)
self.assertEqual(gate['remaining_train_gpu_seconds'],(70+70)*399)
self.assertFalse(entry.cost_gate(self.c,{'0':p['0']},495434.9566447616,820.)['release_epoch2'])
p['1']['epoch_seconds']=185.
self.assertEqual(entry.cost_gate(self.c,p,495434.9566447616,820.)['status'],'STOP_PROJECTED_CAP')
p['1']['epoch_seconds']=70.
self.assertFalse(entry.cost_gate(self.c,p,680000.,820.)['release_epoch2'])
def test_actual_checkpoint_preserves_optimizer_for_second_update(self):
torch.manual_seed(7)
model=torch.nn.Linear(2,1)
opt=torch.optim.AdamW(model.parameters())
x=torch.tensor([[3.,7.]])
def step(m,o):
o.zero_grad()
m(x).square().sum().backward()
o.step()
step(model,opt)
with TemporaryDirectory() as directory, patch.object(torch.cuda,'get_rng_state',return_value=torch.get_rng_state()):
path=Path(directory)/'epoch-1.pt'
worker.checkpoint(path,model,opt,1,0,0,{'only_learning':True})
saved=worker.restore(dict(path=str(path),sha256=sha(path)),0,0,1)
copy=torch.nn.Linear(2,1)
copy.load_state_dict(saved['model'])
resumed=torch.optim.AdamW(copy.parameters())
resumed.load_state_dict(saved['optimizer'])
step(model,opt)
step(copy,resumed)
self.assertTrue(all(torch.equal(a,b) for a,b in zip(model.parameters(),copy.parameters())))
def test_scoring_decision_fixed_roster_and_named_hard_intersection(self):
def metrics(x):
return {k:dict(value=x,reason=None) for k in custody.METRICS}
for g,h,decision in ((.06,.07,'GINO_BETTER'),(.07,.07,'COMPARABLE'),(.08,.07,'H2_BETTER')):
rows=[dict(r,gino=[metrics(g+.001),metrics(g-.001)],h2=[metrics(h),metrics(h)])
for r in self.comparison['roster']]
result=custody.summary(rows,self.comparison)
self.assertEqual(result['field']['decision'],decision)
self.assertEqual(result['field']['components'],78)
self.assertEqual(result['hard_subset']['frozen_total'],34)
rows[0]['gino'][0]['relative_l2']['value']=None
self.assertEqual(custody.summary(rows,self.comparison)['field']['decision'],'INCOMPLETE_PAIRED_SUPPORT')
with self.assertRaises(ValueError):
custody.summary(rows[:-1],self.comparison)
def test_matching_drive_checksum_is_required(self):
with TemporaryDirectory() as directory, patch.object(entry,'ROOT',Path(directory)):
root=Path(directory)
for n in ('contract.json','models-frozen.json','predictions-frozen.json'):
save(root/n,[])
path=root/'gino-greif-models-and-predictions.zip'
path.write_bytes(b'fixture')
bundle=dict(path=str(path),sha256=sha(path),bytes=path.stat().st_size,contract_sha256=sha(root/'contract.json'),
model_index_sha256=sha(root/'models-frozen.json'),prediction_index_sha256=sha(root/'predictions-frozen.json'))
save(root/'drive-upload-index.json',bundle)
auth=dict(private_drive_folder_id='folder-fixture')
receipt=dict(sha256Checksum='wrong',size=str(bundle['bytes']),parent_id='folder-fixture',
file_id='fixture',project_private=True,contract_sha256=bundle['contract_sha256'])
save(root/'drive-readback.json',receipt)
with self.assertRaises(PermissionError):
custody.verify_drive(auth)
save(root/'drive-readback.json',dict(receipt,sha256Checksum=sha(path)))
self.assertEqual(custody.verify_drive(auth)['file_id'],'fixture')
def test_two_predictions_share_exactly_one_mock_target_open(self):
with TemporaryDirectory() as directory:
root=Path(directory)
raw=np.array([20,24,28])
coords=np.array([[0.,0.,0.],[1.,0.,0.],[2.,0.,0.]])
q=dict(raw_ids=raw,counts=np.array([1,1,1]),near_wall=np.array([True,False,False]),coordinates_over_Rg=coords)
row=dict(fold=0,case='0003_fixture',family='test-family')
forecasts={}
for seed in (0,1):
p=root/(str(seed)+'.npz')
np.savez(p,seconds=np.array([1.,2.,4.]),raw_ids=raw,query_raw_ids=raw,
counts=q['counts'],near_wall=q['near_wall'],coordinates_over_Rg=coords)
forecasts[0,seed,row['case']]=dict(family=row['family'],path=str(p),sha256=sha(p))
pilot=dict(parts={row['case']:dict(output_sha256={'seconds.npy':'fixture'})})
ref={row['case']:dict(fold=0,component_id=row['family'],seed0={},seed1={})}
with ((root/'log').open('x') as log, patch.object(custody,'prediction_inputs',return_value=q),
patch.object(custody,'cached',return_value=raw),
patch.object(custody,'target_array',return_value=np.array([1.,2.,3.])) as target):
result=custody.one_case(pilot,{},row,forecasts,ref,log)
target.assert_called_once_with(pilot,row['case'])
self.assertEqual(len(result['gino']),2)
self.assertEqual(len((root/'log').read_text().splitlines()),1)
def test_no_review_prevents_any_gpu_probe(self):
with patch.object(entry,'snapshot') as probe:
with self.assertRaises(FileNotFoundError):
entry.authorize(self.c)
probe.assert_not_called()
if __name__=='__main__':
unittest.main()