"""Run one formally bound GINO fit or the corresponding one-pass score.""" import argparse import ctypes import os import signal import sys from pathlib import Path from tempfile import TemporaryDirectory import time import track_b_gino_formal as formal from track_b_gino_paper_keyword_patch import extract_patched_paper_code from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha from track_b_gino_shapenet_worker import ( OriginalData, case_scores, cuda_peaks, paper_model, test_environment, train_seed, ) def attach(root, seed, phase): ticket = read(root / 'dispatch.json') parent = os.getppid() if (ticket['coordinator'] != process_identity(parent) or ticket['contract_sha256'] != sha(formal.ROOT / 'contract.json') or (ticket['seed'], ticket['phase']) != (seed, phase) or time.monotonic() >= ticket['start_by_monotonic']): raise PermissionError('Worker is not attached to the authorized guard') if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) or os.getppid() != parent: raise RuntimeError('Could not attach worker lifetime to its guard') c = formal.contract() formal.authorize(c) if str(Path(sys.executable).absolute()) != c['environment']['python']: raise PermissionError('Unreviewed interpreter') return c, ticket def bind_device(root, c, ticket, torch): ordinal = ticket['ordinal'] uuid = formal.DEVICES[str(ticket['seed'])] dispatch = read(root / 'ordinal-dispatch.json') if (dispatch['ordinal'] != ordinal or dispatch['physical_uuid'] != uuid or os.environ.get('CUDA_VISIBLE_DEVICES') != ordinal or os.environ.get('CUDA_DEVICE_ORDER') != 'PCI_BUS_ID' or torch.cuda.device_count() != 1 or torch.version.cuda != c['environment']['torch_cuda']): raise PermissionError('One reviewed CUDA PCI ordinal is required') torch.cuda.set_device(0) name = torch.cuda.get_device_name(0) if 'V100' not in name: raise PermissionError('The reviewed device must be a V100') probe = torch.empty(1, dtype=torch.uint8, device='cuda:0') current = process_identity(os.getpid()) save(root / 'worker-device-witness.json', dict( process=current, ordinal=ordinal, device_count=1, device_name=name, physical_uuid_expected=uuid, first_allocation_bytes=probe.numel())) verified = root / 'worker-device-verified.json' while time.monotonic() < ticket['start_by_monotonic']: if verified.exists(): record = read(verified) expected = dict(process=current, physical_uuid=uuid, ordinal=ordinal, observed_uuids=[uuid]) if record != expected: raise PermissionError('Actual worker physical-device witness differs') return time.sleep(.1) raise RuntimeError('Physical device was not verified in the admission window') def train(c, seed, data, root, records, torch): learning = data.data(data.train_ids, 'train') epochs = c['protocol']['epochs_per_seed'] model, times = train_seed(c, learning, seed, epochs, records / 'progress.json') if len(times) != epochs: raise ValueError('Full training did not finish') formal.epoch100(root, seed, c) path = root / 'epoch-100.pt' temporary = root / 'epoch-100.pt.tmp' with temporary.open('xb') as stream: torch.save(dict(state=model.state_dict(), pressure_mean=data.normalizer.mean, pressure_std=data.normalizer.std), stream) temporary.rename(path) return dict(checkpoint=dict(path=str(path), sha256=sha(path)), target_reads=0, epochs=epochs, training_examples=500, epoch_seconds=times) def score(c, seed, data, root, torch): # Targets are still unopened while rechecking all three finished fits, # initializing the exact model and loading its immutable checkpoint. manifest = root / 'checkpoints-frozen.json' manifest_sha = sha(manifest) roster = read(manifest) if roster != formal.all_checkpoints(c): raise PermissionError('Three complete, pinned checkpoints must precede custody') checkpoint = roster[str(seed)]['checkpoint'] state = torch.load(checked(checkpoint['path'], checkpoint['sha256']), map_location='cpu') model = paper_model(c).to('cuda') model.load_state_dict(state['state'], strict=True) # The pinned normalizer's encode/decode and device methods use these # fitted tensors and epsilon. No learning or target dataset is refitted. normalizer = object.__new__(data.normalizer_class) normalizer.mean = state['pressure_mean'] normalizer.std = state['pressure_std'] normalizer.eps = c['protocol']['pressure_epsilon'] if any(t.shape != (1,) or not bool(torch.isfinite(t).all()) for t in (normalizer.mean, normalizer.std)) or bool(normalizer.std < 0): raise ValueError('Invalid frozen learning-only pressure statistics') data.normalizer = normalizer access = root / 'heldout-target-access.jsonl' with access.open('x') as log: heldout = data.data(data.test_ids, 'heldout', log) rows = case_scores(model, heldout, data.test_ids, data.normalizer) torch.cuda.synchronize() return dict(cases=rows, mean_percent=formal.score_mean(rows, data.test_ids), target_reads=len(rows), target_access_sha256=sha(access), checkpoints_manifest_sha256=manifest_sha, checkpoint_sha256=checkpoint['sha256']) def run(seed, phase): root = formal.run_root(seed, phase) c, ticket = attach(root, seed, phase) test_environment(c) import torch torch.set_num_threads(c['limits']['torch_threads']) torch.set_num_interop_threads(1) bind_device(root, c, ticket, torch) torch.cuda.reset_peak_memory_stats() torch.backends.cuda.matmul.allow_tf32 = False torch.backends.cudnn.allow_tf32 = False records = root / 'worker-records' records.mkdir(exist_ok=False) with TemporaryDirectory(prefix='formal-gino-', dir=root) as scratch: extract_patched_paper_code(c, scratch) data = OriginalData(c, scratch) result = (train(c, seed, data, root, records, torch) if phase == 'fit' else score(c, seed, data, root, torch)) save(records / 'result.json', dict(result, phase=phase, seed=seed, status='COMPLETE', **cuda_peaks(torch))) if __name__ == '__main__': parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('seed', type=int, choices=(0, 1, 2)) parser.add_argument('phase', choices=('fit', 'score')) args = parser.parse_args() run(args.seed, args.phase)