Files
MeltMind/tools/track_b_gino_formal_worker.py

144 lines
6.5 KiB
Python

"""Run one formally bound GINO fit or the corresponding one-pass score."""
import argparse
import ctypes
import os
import signal
import sys
from pathlib import Path
from tempfile import TemporaryDirectory
import time
import track_b_gino_formal as formal
from track_b_gino_paper_keyword_patch import extract_patched_paper_code
from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha
from track_b_gino_shapenet_worker import (
OriginalData, case_scores, cuda_peaks, paper_model, test_environment, train_seed,
)
def attach(root, seed, phase):
ticket = read(root / 'dispatch.json')
parent = os.getppid()
if (ticket['coordinator'] != process_identity(parent)
or ticket['contract_sha256'] != sha(formal.ROOT / 'contract.json')
or (ticket['seed'], ticket['phase']) != (seed, phase)
or time.monotonic() >= ticket['start_by_monotonic']):
raise PermissionError('Worker is not attached to the authorized guard')
if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) or os.getppid() != parent:
raise RuntimeError('Could not attach worker lifetime to its guard')
c = formal.contract()
formal.authorize(c)
if str(Path(sys.executable).absolute()) != c['environment']['python']:
raise PermissionError('Unreviewed interpreter')
return c, ticket
def bind_device(root, c, ticket, torch):
ordinal = ticket['ordinal']
uuid = formal.DEVICES[str(ticket['seed'])]
dispatch = read(root / 'ordinal-dispatch.json')
if (dispatch['ordinal'] != ordinal or dispatch['physical_uuid'] != uuid
or os.environ.get('CUDA_VISIBLE_DEVICES') != ordinal
or os.environ.get('CUDA_DEVICE_ORDER') != 'PCI_BUS_ID'
or torch.cuda.device_count() != 1 or torch.version.cuda != c['environment']['torch_cuda']):
raise PermissionError('One reviewed CUDA PCI ordinal is required')
torch.cuda.set_device(0)
name = torch.cuda.get_device_name(0)
if 'V100' not in name:
raise PermissionError('The reviewed device must be a V100')
probe = torch.empty(1, dtype=torch.uint8, device='cuda:0')
current = process_identity(os.getpid())
save(root / 'worker-device-witness.json', dict(
process=current, ordinal=ordinal, device_count=1, device_name=name,
physical_uuid_expected=uuid, first_allocation_bytes=probe.numel()))
verified = root / 'worker-device-verified.json'
while time.monotonic() < ticket['start_by_monotonic']:
if verified.exists():
record = read(verified)
expected = dict(process=current, physical_uuid=uuid, ordinal=ordinal, observed_uuids=[uuid])
if record != expected:
raise PermissionError('Actual worker physical-device witness differs')
return
time.sleep(.1)
raise RuntimeError('Physical device was not verified in the admission window')
def train(c, seed, data, root, records, torch):
learning = data.data(data.train_ids, 'train')
epochs = c['protocol']['epochs_per_seed']
model, times = train_seed(c, learning, seed, epochs, records / 'progress.json')
if len(times) != epochs:
raise ValueError('Full training did not finish')
formal.epoch100(root, seed, c)
path = root / 'epoch-100.pt'
temporary = root / 'epoch-100.pt.tmp'
with temporary.open('xb') as stream:
torch.save(dict(state=model.state_dict(), pressure_mean=data.normalizer.mean,
pressure_std=data.normalizer.std), stream)
temporary.rename(path)
return dict(checkpoint=dict(path=str(path), sha256=sha(path)), target_reads=0,
epochs=epochs, training_examples=500, epoch_seconds=times)
def score(c, seed, data, root, torch):
# Targets are still unopened while rechecking all three finished fits,
# initializing the exact model and loading its immutable checkpoint.
manifest = root / 'checkpoints-frozen.json'
manifest_sha = sha(manifest)
roster = read(manifest)
if roster != formal.all_checkpoints(c):
raise PermissionError('Three complete, pinned checkpoints must precede custody')
checkpoint = roster[str(seed)]['checkpoint']
state = torch.load(checked(checkpoint['path'], checkpoint['sha256']), map_location='cpu')
model = paper_model(c).to('cuda')
model.load_state_dict(state['state'], strict=True)
# The pinned normalizer's encode/decode and device methods use these
# fitted tensors and epsilon. No learning or target dataset is refitted.
normalizer = object.__new__(data.normalizer_class)
normalizer.mean = state['pressure_mean']
normalizer.std = state['pressure_std']
normalizer.eps = c['protocol']['pressure_epsilon']
if any(t.shape != (1,) or not bool(torch.isfinite(t).all())
for t in (normalizer.mean, normalizer.std)) or bool(normalizer.std < 0):
raise ValueError('Invalid frozen learning-only pressure statistics')
data.normalizer = normalizer
access = root / 'heldout-target-access.jsonl'
with access.open('x') as log:
heldout = data.data(data.test_ids, 'heldout', log)
rows = case_scores(model, heldout, data.test_ids, data.normalizer)
torch.cuda.synchronize()
return dict(cases=rows, mean_percent=formal.score_mean(rows, data.test_ids),
target_reads=len(rows), target_access_sha256=sha(access),
checkpoints_manifest_sha256=manifest_sha, checkpoint_sha256=checkpoint['sha256'])
def run(seed, phase):
root = formal.run_root(seed, phase)
c, ticket = attach(root, seed, phase)
test_environment(c)
import torch
torch.set_num_threads(c['limits']['torch_threads'])
torch.set_num_interop_threads(1)
bind_device(root, c, ticket, torch)
torch.cuda.reset_peak_memory_stats()
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
records = root / 'worker-records'
records.mkdir(exist_ok=False)
with TemporaryDirectory(prefix='formal-gino-', dir=root) as scratch:
extract_patched_paper_code(c, scratch)
data = OriginalData(c, scratch)
result = (train(c, seed, data, root, records, torch) if phase == 'fit'
else score(c, seed, data, root, torch))
save(records / 'result.json', dict(result, phase=phase, seed=seed, status='COMPLETE', **cuda_peaks(torch)))
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('seed', type=int, choices=(0, 1, 2))
parser.add_argument('phase', choices=('fit', 'score'))
args = parser.parse_args()
run(args.seed, args.phase)