Bind one Greif training, custody and paired scoring entry for review
This commit is contained in:
1 parent
8810276beb
commit
3c82c074e8
9 files changed
+3419
No files matched your search
@@ -0,0 +1,201 @@
|
||||
"""CPU replay for the reviewed Greif cycle, first-epoch continuity and one-read scorer."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
import time
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from s2_gpu_idle_gate import wait_idle
|
||||
import track_b_gino_formal_guard as guard
|
||||
import track_b_gino_greif_custody as custody
|
||||
import track_b_gino_greif_entry as entry
|
||||
import track_b_gino_greif_worker as worker
|
||||
from track_b_gino_shapenet_entry import read, save, sha
|
||||
|
||||
|
||||
EVIDENCE={}
|
||||
|
||||
|
||||
class Combined(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.c=entry.contract()
|
||||
cls.contract_path=entry.ROOT/'contract.json'
|
||||
cls.comparison=read(entry.ROOT/'comparison-contract.json')
|
||||
|
||||
def test_replays_actual_production_guard_and_stage_order(self):
|
||||
for busy in (False,True):
|
||||
with self.subTest(busy=busy), TemporaryDirectory() as directory:
|
||||
root=Path(directory)
|
||||
(root/'contract.json').write_bytes(self.contract_path.read_bytes())
|
||||
save(root/'stage-authorization.json',dict(test=True))
|
||||
path=root/'track_b_gino_greif_worker.py'
|
||||
path.write_text(
|
||||
'import json,sys,pathlib\n'
|
||||
'root=pathlib.Path(__file__).parent\n'
|
||||
'fold,seed,phase=sys.argv[1:]\n'
|
||||
'stage=root/f"fold-{fold}"/f"seed-{seed}"/phase\n'
|
||||
'p=stage/"worker-records"; p.mkdir()\n'
|
||||
'spec=json.loads((root/"contract.json").read_text())["folds"][fold]\n'
|
||||
'obj=dict(learning_cases=spec["counts"]["learning"],learning_nodes=spec["learning_nodes"],'
|
||||
'first_epoch_seconds=.01,files=[],peak_allocated_bytes=1,peak_reserved_bytes=2)\n'
|
||||
'(p/"temp").write_text(json.dumps(obj)); (p/"temp").rename(p/"result.json")\n')
|
||||
observed=[]
|
||||
elapsed=[0.]
|
||||
card=entry.DEVICES[0]
|
||||
|
||||
def idle(uuid):
|
||||
return dict(uuid=uuid,idle=True,occupants=[],memory_MiB=0,unix=10.)
|
||||
|
||||
def device(uuid):
|
||||
return dict(idle(uuid),idle=not busy)
|
||||
|
||||
def simulated_idle(uuid, limit, record, *, sample):
|
||||
return wait_idle(uuid,limit,record,sample=sample,clock=lambda:elapsed[0],
|
||||
sleep=lambda s:elapsed.__setitem__(0,elapsed[0]+s))
|
||||
|
||||
def validate(_,fold,seed,phase):
|
||||
observed.append((fold,seed,phase))
|
||||
return sha(entry.path(fold,seed,phase)/'worker-records/result.json')
|
||||
|
||||
with (patch.object(entry,'ROOT',root),patch.object(entry,'SOURCE',root),
|
||||
patch.object(entry,'snapshot',side_effect=idle),
|
||||
patch.object(entry,'validate_result',side_effect=validate),
|
||||
patch.object(guard,'snapshot',side_effect=device),
|
||||
patch.object(guard,'wait_idle',side_effect=simulated_idle),
|
||||
patch.object(guard,'dispatch_environment',
|
||||
side_effect=lambda u,env:(dict(env,CUDA_VISIBLE_DEVICES='0'),[]))):
|
||||
result=entry.run(self.c,dict(dispatch_before_unix=time.time()+50),495434.9566447616)
|
||||
with self.assertRaises(FileExistsError):
|
||||
entry.run(self.c,dict(dispatch_before_unix=time.time()+50),495434.9566447616)
|
||||
terminal=read(root/'complete.json')
|
||||
p=root/'fold-0/seed-0/first'
|
||||
if busy:
|
||||
self.assertEqual(result,1)
|
||||
self.assertIn('IDLE_DEADLINE',terminal['trigger'])
|
||||
self.assertFalse((p/'actual-worker-spawn.json').is_file())
|
||||
self.assertFalse((root/'predictions-frozen.json').exists())
|
||||
self.assertEqual(observed,[])
|
||||
else:
|
||||
expected=[(0,0,'first'),(1,0,'first')]+[(f,s,p) for p in ('fit','predict') for f,s in entry.JOBS]
|
||||
self.assertEqual(result,0)
|
||||
self.assertEqual(observed,expected)
|
||||
self.assertTrue(read(root/'first-epoch-cost-gate.json')['release_epoch2'])
|
||||
for f,s,phase in observed:
|
||||
event=read(root/f'fold-{f}'/f'seed-{s}'/phase/'complete.json')
|
||||
self.assertEqual(event['physical_uuid'],card)
|
||||
self.assertEqual([v['event'] for v in event['lifecycle']],['leader_reaped'])
|
||||
self.assertFalse((root/'score').exists())
|
||||
EVIDENCE['busy' if busy else 'nominal']=dict(
|
||||
status=terminal['status'],trigger=terminal['trigger'],stages=observed,
|
||||
cpu_child_spawns=len(list(root.rglob('actual-worker-spawn.json'))))
|
||||
|
||||
def test_first_cost_gate_counts_spent_and_blocks_over_budget(self):
|
||||
common=dict(seed=0,epoch=1,heldout_target_reads=0,booked_gpu_seconds=400.)
|
||||
p={k:dict(common,fold=int(k),epoch_seconds=70.,learning_nodes=r['learning_nodes'],
|
||||
learning_cases=r['counts']['learning']) for k,r in self.c['folds'].items()}
|
||||
gate=entry.cost_gate(self.c,p,495434.9566447616,820.)
|
||||
self.assertTrue(gate['release_epoch2'])
|
||||
self.assertEqual(gate['charged_first_epochs_and_setup_gpu_seconds'],800.)
|
||||
self.assertEqual(gate['remaining_train_gpu_seconds'],(70+70)*399)
|
||||
self.assertFalse(entry.cost_gate(self.c,{'0':p['0']},495434.9566447616,820.)['release_epoch2'])
|
||||
p['1']['epoch_seconds']=185.
|
||||
self.assertEqual(entry.cost_gate(self.c,p,495434.9566447616,820.)['status'],'STOP_PROJECTED_CAP')
|
||||
p['1']['epoch_seconds']=70.
|
||||
self.assertFalse(entry.cost_gate(self.c,p,680000.,820.)['release_epoch2'])
|
||||
|
||||
def test_actual_checkpoint_preserves_optimizer_for_second_update(self):
|
||||
torch.manual_seed(7)
|
||||
model=torch.nn.Linear(2,1)
|
||||
opt=torch.optim.AdamW(model.parameters())
|
||||
x=torch.tensor([[3.,7.]])
|
||||
def step(m,o):
|
||||
o.zero_grad()
|
||||
m(x).square().sum().backward()
|
||||
o.step()
|
||||
|
||||
step(model,opt)
|
||||
with TemporaryDirectory() as directory, patch.object(torch.cuda,'get_rng_state',return_value=torch.get_rng_state()):
|
||||
path=Path(directory)/'epoch-1.pt'
|
||||
worker.checkpoint(path,model,opt,1,0,0,{'only_learning':True})
|
||||
saved=worker.restore(dict(path=str(path),sha256=sha(path)),0,0,1)
|
||||
copy=torch.nn.Linear(2,1)
|
||||
copy.load_state_dict(saved['model'])
|
||||
resumed=torch.optim.AdamW(copy.parameters())
|
||||
resumed.load_state_dict(saved['optimizer'])
|
||||
step(model,opt)
|
||||
step(copy,resumed)
|
||||
self.assertTrue(all(torch.equal(a,b) for a,b in zip(model.parameters(),copy.parameters())))
|
||||
|
||||
def test_scoring_decision_fixed_roster_and_named_hard_intersection(self):
|
||||
def metrics(x):
|
||||
return {k:dict(value=x,reason=None) for k in custody.METRICS}
|
||||
|
||||
for g,h,decision in ((.06,.07,'GINO_BETTER'),(.07,.07,'COMPARABLE'),(.08,.07,'H2_BETTER')):
|
||||
rows=[dict(r,gino=[metrics(g+.001),metrics(g-.001)],h2=[metrics(h),metrics(h)])
|
||||
for r in self.comparison['roster']]
|
||||
result=custody.summary(rows,self.comparison)
|
||||
self.assertEqual(result['field']['decision'],decision)
|
||||
self.assertEqual(result['field']['components'],78)
|
||||
self.assertEqual(result['hard_subset']['frozen_total'],34)
|
||||
rows[0]['gino'][0]['relative_l2']['value']=None
|
||||
self.assertEqual(custody.summary(rows,self.comparison)['field']['decision'],'INCOMPLETE_PAIRED_SUPPORT')
|
||||
with self.assertRaises(ValueError):
|
||||
custody.summary(rows[:-1],self.comparison)
|
||||
|
||||
def test_matching_drive_checksum_is_required(self):
|
||||
with TemporaryDirectory() as directory, patch.object(entry,'ROOT',Path(directory)):
|
||||
root=Path(directory)
|
||||
for n in ('contract.json','models-frozen.json','predictions-frozen.json'):
|
||||
save(root/n,[])
|
||||
path=root/'gino-greif-models-and-predictions.zip'
|
||||
path.write_bytes(b'fixture')
|
||||
bundle=dict(path=str(path),sha256=sha(path),bytes=path.stat().st_size,contract_sha256=sha(root/'contract.json'),
|
||||
model_index_sha256=sha(root/'models-frozen.json'),prediction_index_sha256=sha(root/'predictions-frozen.json'))
|
||||
save(root/'drive-upload-index.json',bundle)
|
||||
auth=dict(private_drive_folder_id='folder-fixture')
|
||||
receipt=dict(sha256Checksum='wrong',size=str(bundle['bytes']),parent_id='folder-fixture',
|
||||
file_id='fixture',project_private=True,contract_sha256=bundle['contract_sha256'])
|
||||
save(root/'drive-readback.json',receipt)
|
||||
with self.assertRaises(PermissionError):
|
||||
custody.verify_drive(auth)
|
||||
save(root/'drive-readback.json',dict(receipt,sha256Checksum=sha(path)))
|
||||
self.assertEqual(custody.verify_drive(auth)['file_id'],'fixture')
|
||||
|
||||
def test_two_predictions_share_exactly_one_mock_target_open(self):
|
||||
with TemporaryDirectory() as directory:
|
||||
root=Path(directory)
|
||||
raw=np.array([20,24,28])
|
||||
coords=np.array([[0.,0.,0.],[1.,0.,0.],[2.,0.,0.]])
|
||||
q=dict(raw_ids=raw,counts=np.array([1,1,1]),near_wall=np.array([True,False,False]),coordinates_over_Rg=coords)
|
||||
row=dict(fold=0,case='0003_fixture',family='test-family')
|
||||
forecasts={}
|
||||
for seed in (0,1):
|
||||
p=root/(str(seed)+'.npz')
|
||||
np.savez(p,seconds=np.array([1.,2.,4.]),raw_ids=raw,query_raw_ids=raw,
|
||||
counts=q['counts'],near_wall=q['near_wall'],coordinates_over_Rg=coords)
|
||||
forecasts[0,seed,row['case']]=dict(family=row['family'],path=str(p),sha256=sha(p))
|
||||
pilot=dict(parts={row['case']:dict(output_sha256={'seconds.npy':'fixture'})})
|
||||
ref={row['case']:dict(fold=0,component_id=row['family'],seed0={},seed1={})}
|
||||
with ((root/'log').open('x') as log, patch.object(custody,'prediction_inputs',return_value=q),
|
||||
patch.object(custody,'cached',return_value=raw),
|
||||
patch.object(custody,'target_array',return_value=np.array([1.,2.,3.])) as target):
|
||||
result=custody.one_case(pilot,{},row,forecasts,ref,log)
|
||||
target.assert_called_once_with(pilot,row['case'])
|
||||
self.assertEqual(len(result['gino']),2)
|
||||
self.assertEqual(len((root/'log').read_text().splitlines()),1)
|
||||
|
||||
def test_no_review_prevents_any_gpu_probe(self):
|
||||
with patch.object(entry,'snapshot') as probe:
|
||||
with self.assertRaises(FileNotFoundError):
|
||||
entry.authorize(self.c)
|
||||
probe.assert_not_called()
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user