144 lines
6.5 KiB
Python
144 lines
6.5 KiB
Python
"""Run one formally bound GINO fit or the corresponding one-pass score."""
|
|
|
|
import argparse
|
|
import ctypes
|
|
import os
|
|
import signal
|
|
import sys
|
|
from pathlib import Path
|
|
from tempfile import TemporaryDirectory
|
|
import time
|
|
|
|
import track_b_gino_formal as formal
|
|
from track_b_gino_paper_keyword_patch import extract_patched_paper_code
|
|
from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha
|
|
from track_b_gino_shapenet_worker import (
|
|
OriginalData, case_scores, cuda_peaks, paper_model, test_environment, train_seed,
|
|
)
|
|
|
|
|
|
def attach(root, seed, phase):
|
|
ticket = read(root / 'dispatch.json')
|
|
parent = os.getppid()
|
|
if (ticket['coordinator'] != process_identity(parent)
|
|
or ticket['contract_sha256'] != sha(formal.ROOT / 'contract.json')
|
|
or (ticket['seed'], ticket['phase']) != (seed, phase)
|
|
or time.monotonic() >= ticket['start_by_monotonic']):
|
|
raise PermissionError('Worker is not attached to the authorized guard')
|
|
if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) or os.getppid() != parent:
|
|
raise RuntimeError('Could not attach worker lifetime to its guard')
|
|
c = formal.contract()
|
|
formal.authorize(c)
|
|
if str(Path(sys.executable).absolute()) != c['environment']['python']:
|
|
raise PermissionError('Unreviewed interpreter')
|
|
return c, ticket
|
|
|
|
|
|
def bind_device(root, c, ticket, torch):
|
|
ordinal = ticket['ordinal']
|
|
uuid = formal.DEVICES[str(ticket['seed'])]
|
|
dispatch = read(root / 'ordinal-dispatch.json')
|
|
if (dispatch['ordinal'] != ordinal or dispatch['physical_uuid'] != uuid
|
|
or os.environ.get('CUDA_VISIBLE_DEVICES') != ordinal
|
|
or os.environ.get('CUDA_DEVICE_ORDER') != 'PCI_BUS_ID'
|
|
or torch.cuda.device_count() != 1 or torch.version.cuda != c['environment']['torch_cuda']):
|
|
raise PermissionError('One reviewed CUDA PCI ordinal is required')
|
|
torch.cuda.set_device(0)
|
|
name = torch.cuda.get_device_name(0)
|
|
if 'V100' not in name:
|
|
raise PermissionError('The reviewed device must be a V100')
|
|
probe = torch.empty(1, dtype=torch.uint8, device='cuda:0')
|
|
current = process_identity(os.getpid())
|
|
save(root / 'worker-device-witness.json', dict(
|
|
process=current, ordinal=ordinal, device_count=1, device_name=name,
|
|
physical_uuid_expected=uuid, first_allocation_bytes=probe.numel()))
|
|
verified = root / 'worker-device-verified.json'
|
|
while time.monotonic() < ticket['start_by_monotonic']:
|
|
if verified.exists():
|
|
record = read(verified)
|
|
expected = dict(process=current, physical_uuid=uuid, ordinal=ordinal, observed_uuids=[uuid])
|
|
if record != expected:
|
|
raise PermissionError('Actual worker physical-device witness differs')
|
|
return
|
|
time.sleep(.1)
|
|
raise RuntimeError('Physical device was not verified in the admission window')
|
|
|
|
|
|
def train(c, seed, data, root, records, torch):
|
|
learning = data.data(data.train_ids, 'train')
|
|
epochs = c['protocol']['epochs_per_seed']
|
|
model, times = train_seed(c, learning, seed, epochs, records / 'progress.json')
|
|
if len(times) != epochs:
|
|
raise ValueError('Full training did not finish')
|
|
formal.epoch100(root, seed, c)
|
|
path = root / 'epoch-100.pt'
|
|
temporary = root / 'epoch-100.pt.tmp'
|
|
with temporary.open('xb') as stream:
|
|
torch.save(dict(state=model.state_dict(), pressure_mean=data.normalizer.mean,
|
|
pressure_std=data.normalizer.std), stream)
|
|
temporary.rename(path)
|
|
return dict(checkpoint=dict(path=str(path), sha256=sha(path)), target_reads=0,
|
|
epochs=epochs, training_examples=500, epoch_seconds=times)
|
|
|
|
|
|
def score(c, seed, data, root, torch):
|
|
# Targets are still unopened while rechecking all three finished fits,
|
|
# initializing the exact model and loading its immutable checkpoint.
|
|
manifest = root / 'checkpoints-frozen.json'
|
|
manifest_sha = sha(manifest)
|
|
roster = read(manifest)
|
|
if roster != formal.all_checkpoints(c):
|
|
raise PermissionError('Three complete, pinned checkpoints must precede custody')
|
|
checkpoint = roster[str(seed)]['checkpoint']
|
|
state = torch.load(checked(checkpoint['path'], checkpoint['sha256']), map_location='cpu')
|
|
model = paper_model(c).to('cuda')
|
|
model.load_state_dict(state['state'], strict=True)
|
|
# The pinned normalizer's encode/decode and device methods use these
|
|
# fitted tensors and epsilon. No learning or target dataset is refitted.
|
|
normalizer = object.__new__(data.normalizer_class)
|
|
normalizer.mean = state['pressure_mean']
|
|
normalizer.std = state['pressure_std']
|
|
normalizer.eps = c['protocol']['pressure_epsilon']
|
|
if any(t.shape != (1,) or not bool(torch.isfinite(t).all())
|
|
for t in (normalizer.mean, normalizer.std)) or bool(normalizer.std < 0):
|
|
raise ValueError('Invalid frozen learning-only pressure statistics')
|
|
data.normalizer = normalizer
|
|
access = root / 'heldout-target-access.jsonl'
|
|
with access.open('x') as log:
|
|
heldout = data.data(data.test_ids, 'heldout', log)
|
|
rows = case_scores(model, heldout, data.test_ids, data.normalizer)
|
|
torch.cuda.synchronize()
|
|
return dict(cases=rows, mean_percent=formal.score_mean(rows, data.test_ids),
|
|
target_reads=len(rows), target_access_sha256=sha(access),
|
|
checkpoints_manifest_sha256=manifest_sha, checkpoint_sha256=checkpoint['sha256'])
|
|
|
|
|
|
def run(seed, phase):
|
|
root = formal.run_root(seed, phase)
|
|
c, ticket = attach(root, seed, phase)
|
|
test_environment(c)
|
|
import torch
|
|
|
|
torch.set_num_threads(c['limits']['torch_threads'])
|
|
torch.set_num_interop_threads(1)
|
|
bind_device(root, c, ticket, torch)
|
|
torch.cuda.reset_peak_memory_stats()
|
|
torch.backends.cuda.matmul.allow_tf32 = False
|
|
torch.backends.cudnn.allow_tf32 = False
|
|
records = root / 'worker-records'
|
|
records.mkdir(exist_ok=False)
|
|
with TemporaryDirectory(prefix='formal-gino-', dir=root) as scratch:
|
|
extract_patched_paper_code(c, scratch)
|
|
data = OriginalData(c, scratch)
|
|
result = (train(c, seed, data, root, records, torch) if phase == 'fit'
|
|
else score(c, seed, data, root, torch))
|
|
save(records / 'result.json', dict(result, phase=phase, seed=seed, status='COMPLETE', **cuda_peaks(torch)))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('seed', type=int, choices=(0, 1, 2))
|
|
parser.add_argument('phase', choices=('fit', 'score'))
|
|
args = parser.parse_args()
|
|
run(args.seed, args.phase)
|