Bind formal three-seed GINO with CPU guard and custody replay

This commit is contained in:
NewMeltAssist committed 2026-11-19 16:06:50 +08:00
1 parent 93d7262e83
commit 9a4f8563ee
11 files changed
+1809

No files matched your search

+314
View File
@@ -0,0 +1,314 @@
"""Exact review, custody and resource binding for formal three-seed GINO."""
import argparse
from datetime import datetime
import fcntl
import json
import math
import os
from pathlib import Path
import signal
import time
from zoneinfo import ZoneInfo
from phase2_gate_outcome_review import validate_review
from track_b_gino_formal_guard import guard
from track_b_gino_import_checked_entry import accepted_environment
from track_b_gino_ordinal_recovery import ROOT as MEASUREMENT
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
ROOT = MEASUREMENT.with_name('track-b-gino-shapenet611-formal-2026-11-19')
SOURCES = Path(__file__).resolve().parent
OLD_FILES = {
'contract.json': 'ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a',
'complete.json': 'db3605159c5ade3deda5313ab409e2f1c8bf630511a617f527177da06d6c9a2d',
'recovery-settlement.json': '851eaedf716f01c1d80ce637f7de3edd0664c734bcbfed5622623d99f572fcec',
'worker-records/progress.json': '8f8fb1902c44e2f9c3198608cbecb3a74967ed00fdbcaf1e990d146c894b0daf',
}
DEVICES = {
'0': 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76',
'1': 'GPU-8b71b032-1b75-2891-935c-ce106a53034a',
'2': 'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c',
}
ORDINALS = {'0': 2, '1': 3, '2': 0}
INVENTORY = BASE / 'artifacts/track-b-gino-gpu2-ordinal-preparation-2026-11-19/physical-inventory.json'
INVENTORY_SHA = 'da2a7b1bf168c7cc4d45b6eaa2e007ae90cb3640eeadde94c10fd1d32fa30dd8'
TZ = ZoneInfo('Asia/Hong_Kong')
RESERVATION_START = datetime(2026, 11, 19, tzinfo=TZ)
RESERVATION_END = datetime(2026, 11, 20, tzinfo=TZ)
NEW_SOURCES = ('track_b_gino_formal.py', 'track_b_gino_formal_guard.py',
'track_b_gino_formal_worker.py')
def expected_contract():
old = {name: read(checked(MEASUREMENT / name, digest)) for name, digest in OLD_FILES.items()}
c = old['contract.json']
stop, epoch = old['complete.json'], old['worker-records/progress.json']
account = old['recovery-settlement.json']
if (stop['status'], stop['trigger'], stop['result_sha256']) != ('STOPPED', 'FOREIGN_GPU_OCCUPANT', None):
raise ValueError('Historical guard STOP differs')
if (epoch['stage'], epoch['seed'], epoch['epoch'], epoch['examples']) != ('EPOCH_COMPLETE', 0, 1, 500):
raise ValueError('Only the completed 500-example epoch can size this reproduction')
seconds = epoch['seconds']
inventory = read(checked(INVENTORY, INVENTORY_SHA))['inventory']
physical = {(int(index), uuid) for index, uuid, _, name in inventory if 'V100' in name}
if not {(ORDINALS[seed], uuid) for seed, uuid in DEVICES.items()} <= physical:
raise ValueError('Reviewed physical V100 inventory differs')
budgets = {seed: {'fit': 13033 + (seed == '0'), 'score': 300} for seed in DEVICES}
fit_reference = 100 * seconds * 1.25
if (not math.isfinite(fit_reference) or fit_reference <= 0
or min(b['fit'] for b in budgets.values()) <= fit_reference
or sum(b['fit'] + b['score'] for b in budgets.values()) != 40000):
raise ValueError('Formal booking must cover the sizing reference and score reserve')
c['mode'] = 'formal-gino-reproduction'
for key in ('gpu_uuid', 'measurement_allowed_heldout_labels', 'gpu_ruling', 'ordinal_recovery'):
del c[key]
c['code_sha256'].update({str(SOURCES / name): sha(SOURCES / name) for name in NEW_SOURCES})
c['formal'] = dict(
handoff='h-4e4e680b', ruling_slack_ts='1795074651.222599', devices=DEVICES,
device_pci_ordinals=ORDINALS,
reservation=dict(handoff='h-a429935e', timezone=TZ.key,
start_inclusive=RESERVATION_START.isoformat(),
end_exclusive=RESERVATION_END.isoformat(), gpu3_single_task=True,
historical_inventory_path=str(INVENTORY), historical_inventory_sha256=INVENTORY_SHA,
runtime_idle_check_required=True),
budget_per_seed_stage_gpu_seconds=budgets, total_gpu_wall_seconds=40000,
timing=dict(record_paths={str(MEASUREMENT / k): v for k, v in OLD_FILES.items()},
status='CONTENTION_AFFECTED_NOT_GUARD_CERTIFIED', sizing_only=True,
epoch_seconds=seconds, train_sizing_gpu_seconds=300 * seconds * 1.25,
train_sizing_multiplier=1.25, score_reserved_gpu_seconds=900,
prior_measurement_gpu_seconds=account['cumulative_measurement_gpu_wall_seconds']),
heldout_policy=dict(preparation_reads=0, training_reads=0, score_reads_per_seed=111,
all_epoch100_checkpoints_required=True, retry=False),
)
del c['limits']['gpu_wall_seconds']
c['limits']['cleanup_seconds'] = 20
del c['limits']['cpu_seconds']
c['limits']['cpu_seconds_per_gpu_second'] = 8
return c
def contract():
c = read(ROOT / 'contract.json')
if c != expected_contract():
raise PermissionError('The sole formal contract or its implementation changed')
for path, digest in c['code_sha256'].items():
checked(path, digest)
for name in ('paper_supplement', 'split_receipt', 'mapping_audit'):
pin = c[name]
checked(pin['path'], pin['sha256'])
# Archive members are opened only in the scoped workers, which verify each
# member before using it. Preparation does not hash the archive payload.
comparison = read(checked(c['comparison_contract'], c['comparison_contract_sha256']))
split = read(c['split_receipt']['path'])
train, test = split['train'], split['test']
if (c['protocol'] != comparison['protocol'] or len(train) != 500 or len(test) != 111
or len(set(train) | set(test)) != 611
or any(c['split'][k] != split[k] for k in ('train', 'test', 'raw_global_bounds'))
or c['protocol']['epochs_per_seed'] != 100 or c['protocol']['seeds'] != [0, 1, 2]
or c['protocol']['acceptance_percent'] != [6.62, 7.62]
or c['paper_compatibility']['identity'] != 'supplement code + 2-keyword patch'
or not Path(c['paper_data']['path']).is_file()):
raise ValueError('Original GINO comparison, split or pinned data path differs')
accepted_environment(c)
return c
def authorize(c):
validate_review(ROOT)
authorization = read(ROOT / 'stage-authorization.json')
required = dict(contract_sha256=sha(ROOT / 'contract.json'),
implementation_proof_sha256=sha(ROOT / 'implementation-git-freeze.json'),
decision='APPROVED', mode=c['mode'], reserved_devices=DEVICES,
reservation_end_exclusive=RESERVATION_END.isoformat())
if any(authorization.get(k) != v for k, v in required.items()):
raise PermissionError('Exact execution and card-reservation authorization required')
def check_reservation(unix, duration):
if (unix < RESERVATION_START.timestamp()
or unix + duration >= RESERVATION_END.timestamp()):
raise PermissionError('The entire booking must fit the November 19 Hong Kong reservation')
def run_root(seed, phase):
return ROOT / f'seed-{seed}' / phase
def epoch100(root, seed, c):
progress = read(root / 'worker-records/progress.json')
if (progress['stage'], progress['seed'], progress['epoch'], progress['examples']) != (
'EPOCH_COMPLETE', seed, c['protocol']['epochs_per_seed'], 500):
raise ValueError('No completed 100th full training epoch for this seed')
return progress
def checkpoint(seed, result):
path = run_root(seed, 'fit') / 'epoch-100.pt'
if result['checkpoint']['path'] != str(path):
raise ValueError('Checkpoint path differs from this seed')
return checked(path, result['checkpoint']['sha256'])
def score_mean(rows, test_ids):
if [r['case'] for r in rows] != test_ids:
raise ValueError('Scoring requires the original 111-case order')
values = [r['relative_l2'] for r in rows]
if any(type(v) not in (int, float) or not math.isfinite(v) or v < 0 for v in values):
raise ValueError('Incomplete or invalid relative L2')
return 100 * (math.fsum(values) / len(values))
def validate_result(c, seed, phase):
root = run_root(seed, phase)
path = root / 'worker-records/result.json'
result = read(path)
spawned = read(root / 'actual-worker-spawn.json')
device = DEVICES[str(seed)]
ordinal = str(ORDINALS[str(seed)])
if (spawned['physical_uuid'], spawned['ordinal']) != (device, ordinal) or (
read(root / 'worker-device-verified.json') != dict(
process=spawned['process'], physical_uuid=device, ordinal=ordinal, observed_uuids=[device])):
raise ValueError('Worker has no matching physical-device witness')
if (result['phase'], result['seed'], result['status']) != (phase, seed, 'COMPLETE'):
raise ValueError('Worker result identity differs')
if phase == 'fit':
epoch100(root, seed, c)
checkpoint(seed, result)
if (result['target_reads'], result['epochs'], result['training_examples']) != (0, 100, 500):
raise ValueError('Fit must complete all 100 epochs with zero held-out reads')
else:
fit_terminal = read(run_root(seed, 'fit') / 'complete.json')
fit_path = run_root(seed, 'fit') / 'worker-records/result.json'
original = read(checked(fit_path, fit_terminal['result_sha256']))
access = root / 'heldout-target-access.jsonl'
mapping = {row['case']: row['pressure_sha256'] for row in read(c['mapping_audit']['path'])['rows']}
actual = [json.loads(line) for line in access.read_text().splitlines()]
expected = [dict(case=i, role='heldout', sha256=mapping[i]) for i in c['split']['test']]
if (actual != expected or result['target_reads'] != 111
or result['target_access_sha256'] != sha(access)
or result['checkpoints_manifest_sha256'] != sha(root / 'checkpoints-frozen.json')
or read(root / 'checkpoints-frozen.json') != all_checkpoints(c)
or result['checkpoint_sha256'] != original['checkpoint']['sha256']
or score_mean(result['cases'], c['split']['test']) != result['mean_percent']):
raise ValueError('Score or held-out target audit differs')
return sha(path)
def accepted_terminal(c, seed, phase):
root = run_root(seed, phase)
record = read(root / 'complete.json')
if (record['status'], record['seed'], record['phase'], record['contract_sha256'], record['physical_uuid']) != (
'COMPLETE', seed, phase, sha(ROOT / 'contract.json'), DEVICES[str(seed)]):
raise PermissionError(f'Seed {seed} {phase} has no accepted terminal result')
booked = c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase]
if (record['booked_gpu_seconds'] != booked or record['charged_gpu_seconds'] > booked
or record['result_sha256'] != validate_result(c, seed, phase)):
raise ValueError('Terminal result bytes or resource accounting differ')
return record
def all_checkpoints(c):
checkpoints = {}
for seed in c['protocol']['seeds']:
accepted_terminal(c, seed, 'fit')
root = run_root(seed, 'fit')
checkpoints[str(seed)] = dict(
result_sha256=sha(root / 'worker-records/result.json'),
terminal_sha256=sha(root / 'complete.json'),
checkpoint=read(root / 'worker-records/result.json')['checkpoint'])
return checkpoints
def dispatch(c, seed, phase):
began = time.monotonic()
unix = time.time()
authorize(c)
budget = c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase]
check_reservation(unix, budget)
seed_root = run_root(seed, phase).parent
seed_root.mkdir(exist_ok=True)
with (seed_root / 'execution.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
roster = all_checkpoints(c) if phase == 'score' else None
root = run_root(seed, phase)
root.mkdir(exist_ok=False)
if roster is not None:
save(root / 'checkpoints-frozen.json', roster)
ticket = dict(seed=seed, phase=phase, ordinal=str(ORDINALS[str(seed)]),
booked_gpu_seconds=budget, unix=unix,
contract_sha256=sha(ROOT / 'contract.json'),
guard_started_monotonic=began, booked_deadline_monotonic=began + budget)
save(root / 'started.json', ticket)
command = [c['environment']['python'], str(SOURCES / 'track_b_gino_formal_worker.py'),
str(seed), phase]
def request_stop(signum, _frame):
raise KeyboardInterrupt(signal.Signals(signum).name)
previous = signal.signal(signal.SIGTERM, request_stop)
try:
limits = dict(c['limits'], cpu_seconds=c['limits']['cpu_seconds_per_gpu_second'] * budget)
result = guard(root, command, uuid=DEVICES[str(seed)], limits=limits,
deadline=began + budget, ticket=ticket,
validate_result=lambda: validate_result(c, seed, phase))
finally:
signal.signal(signal.SIGTERM, previous)
return 0 if result['status'] == 'COMPLETE' else 1
def pool(means, limits):
if len(means) != 3 or any(v is None for v in means):
return None, None
mean = math.fsum(means) / 3
return mean, limits[0] <= mean <= limits[1]
def collect(c):
seeds = {}
accounted = unsettled = 0
for seed in c['protocol']['seeds']:
rows = {}
for phase in ('fit', 'score'):
root = run_root(seed, phase)
terminal = root / 'complete.json'
if terminal.exists():
record = read(terminal)
if record['status'] == 'COMPLETE':
accepted_terminal(c, seed, phase)
accounted += record['charged_gpu_seconds']
rows[phase] = dict(status=record['status'], trigger=record['trigger'],
terminal_sha256=sha(terminal))
if phase == 'score' and record['status'] == 'COMPLETE':
rows['mean_percent'] = read(root / 'worker-records/result.json')['mean_percent']
elif (root / 'started.json').exists():
rows[phase] = dict(status='NO_TERMINAL_RECEIPT')
unsettled += c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase]
else:
rows[phase] = dict(status='NOT_STARTED')
seeds[str(seed)] = rows
mean, verdict = pool([row.get('mean_percent') for row in seeds.values()],
c['protocol']['acceptance_percent'])
prior = c['formal']['timing']['prior_measurement_gpu_seconds']
return dict(seeds=seeds, mean_percent=mean, within_registered_range=verdict,
reproduction_identity=c['paper_compatibility']['identity'],
prior_measurement_gpu_seconds=prior, settled_formal_gpu_seconds=accounted,
settled_combined_gpu_seconds=accounted + prior,
unsettled_formal_bookings_gpu_seconds=unsettled, formal_gpu_cap_seconds=40000)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('fit', 'score', 'collect'))
parser.add_argument('--seed', type=int, choices=(0, 1, 2))
args = parser.parse_args()
c = contract()
if args.action == 'collect':
if args.seed is not None:
parser.error('collect reports all three seeds')
print(json.dumps(collect(c), indent=2, allow_nan=False))
else:
if args.seed is None:
parser.error('--seed is required for fit or score')
raise SystemExit(dispatch(c, args.seed, args.action))
+175
View File
@@ -0,0 +1,175 @@
"""Supervise a formal GINO fit or score on one physical V100."""
import os
import shutil
import subprocess
import threading
import time
from types import FunctionType, SimpleNamespace
import psutil
from s2_gpu_idle_gate import snapshot, wait_idle
from s2_main_memory_guard import PeakRecords
from s2_recovery_gpu_supervisor import stop_group, supervise
from track_b_gino_ordinal_recovery import dispatch_environment, monitor_identity
from track_b_gino_shapenet_entry import process_identity, save
def current_identity(pid):
try:
return process_identity(pid)
except psutil.NoSuchProcess:
return None
def process_tree(worker):
if current_identity(worker['pid']) != worker:
return []
try:
children = psutil.Process(worker['pid']).children(recursive=True)
except psutil.NoSuchProcess:
return []
result = []
for child in children:
try:
result.append(dict(pid=child.pid, created=child.create_time()))
except psutil.NoSuchProcess:
continue
return result
class OccupancyMonitor:
def __init__(self, root, uuid, worker, device_vram_bytes):
self.root = root
self.uuid = uuid
self.worker = worker
self.known = {worker['pid']: worker}
self.device_vram_bytes = device_vram_bytes
self.peak_device_bytes = 0
def check(self):
before = process_tree(self.worker)
state = snapshot(self.uuid)
after = process_tree(self.worker)
for identity in before + after:
if identity['pid'] != self.worker['pid']:
self.known[identity['pid']] = identity
foreign = []
for row in state['occupants']:
pid = row['pid']
current = current_identity(pid)
if pid not in self.known or (current is not None and current != self.known[pid]):
foreign.append(dict(row, observed_process_identity=current))
if foreign:
save(self.root / 'foreign-occupant.json', dict(
trigger='FOREIGN_GPU_OCCUPANT', unix=state['unix'], physical_uuid=self.uuid,
occupants=foreign, own_spawn=self.worker, tree_before=before, tree_after=after))
raise RuntimeError('FOREIGN_GPU_OCCUPANT')
self.peak_device_bytes = max(self.peak_device_bytes, state['memory_MiB'] * 2**20)
if self.peak_device_bytes > self.device_vram_bytes:
raise RuntimeError('DEVICE_VRAM_LIMIT')
def watch(root, uuid, worker, limits, done, tracker, state):
occupancy = OccupancyMonitor(root, uuid, worker, limits['device_vram_bytes'])
while not done.is_set():
try:
occupancy.check()
state['peak_device_bytes'] = occupancy.peak_device_bytes
tracker.scan(root / 'worker-records')
if max(tracker.maxima.values()) > limits['recorded_vram_bytes']:
raise RuntimeError('RECORDED_VRAM_LIMIT')
monitor_identity(root, uuid)
except Exception as exc:
state['trigger'] = str(exc) or type(exc).__name__
# Signal only the recorded worker group while its leader has the same identity.
if current_identity(worker['pid']) == worker:
stop_group(worker['pid'])
return
done.wait(.5)
def guard(root, command, *, uuid, limits, deadline, ticket, validate_result):
began = time.monotonic()
admission = min(deadline - limits['cleanup_seconds'], began + limits['idle_wait_seconds'])
done = threading.Event()
tracker = PeakRecords()
state = dict(trigger=None, peak_device_bytes=0)
watcher = None
worker_receipt = None
status, trigger, result_hash = 'STOPPED', None, None
def spawn(args, **kwargs):
nonlocal watcher
env, inventory = dispatch_environment(uuid, kwargs['env'])
ordinal = env['CUDA_VISIBLE_DEVICES']
if ordinal != ticket['ordinal']:
raise RuntimeError('PINNED_PCI_ORDINAL_DIFFERS')
if not snapshot(uuid)['idle']:
raise RuntimeError('SELECTED_GPU_BECAME_BUSY')
save(root / 'ordinal-dispatch.json', dict(ordinal=ordinal, physical_uuid=uuid, inventory=inventory))
if time.monotonic() >= admission:
raise RuntimeError('ADMISSION_DEADLINE')
proc = subprocess.Popen(args, **dict(kwargs, env=env))
try:
worker = process_identity(proc.pid)
save(root / 'actual-worker-spawn.json', dict(process=worker, ordinal=ordinal, physical_uuid=uuid))
watcher = threading.Thread(target=watch, daemon=True,
args=(root, uuid, worker, limits, done, tracker, state))
watcher.start()
except (Exception, KeyboardInterrupt):
# The shared supervisor has not yet received the process to reap.
stop_group(proc.pid)
proc.wait()
raise
return proc
try:
if shutil.disk_usage(root).free < limits['disk_bytes']:
raise RuntimeError('DISK_HEADROOM')
idle = wait_idle(uuid, max(0., admission - time.monotonic()), lambda _: None, sample=snapshot)
save(root / 'idle-gate.json', idle)
if idle['status'] != 'IDLE_CONFIRMED' or time.monotonic() >= admission:
raise RuntimeError('IDLE_DEADLINE')
save(root / 'dispatch.json', dict(ticket, coordinator=process_identity(os.getpid()),
start_by_monotonic=admission))
remaining = deadline - time.monotonic() - limits['cleanup_seconds']
if remaining <= 0:
raise RuntimeError('INSUFFICIENT_REMAINING_BUDGET')
ns = dict(supervise.__globals__, subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT))
run_worker = FunctionType(supervise.__code__, ns, supervise.__name__)
try:
worker_receipt = run_worker(command, root, 'gino', dict(
total_cpu_seconds=limits['cpu_seconds'], job_cpu_seconds=limits['cpu_seconds'],
job_wall_seconds=remaining, tree_rss_bytes=limits['rss_bytes'],
disk_bytes=limits['disk_bytes'], max_processes=limits['max_processes']),
poll=.2, gpu_uuid=uuid)
finally:
done.set()
if watcher is not None:
watcher.join(timeout=8)
if watcher is not None and watcher.is_alive():
raise RuntimeError('MONITOR_DID_NOT_EXIT')
tracker.scan(root / 'worker-records')
if state['trigger']:
raise RuntimeError(state['trigger'])
if max(tracker.maxima.values()) > limits['recorded_vram_bytes']:
raise RuntimeError('RECORDED_VRAM_LIMIT')
if worker_receipt['status'] != 'PASS' or worker_receipt['monitor_errors']:
raise RuntimeError(worker_receipt.get('stop_reason') or 'WORKER_FAILED')
if set(tracker.witnesses) != set(tracker.maxima):
raise RuntimeError('MISSING_RECORDED_MEMORY_PEAKS')
result_hash = validate_result()
status = 'COMPLETE'
except (Exception, KeyboardInterrupt) as exc:
trigger = str(exc) or type(exc).__name__
end = time.monotonic()
if end > deadline and status == 'COMPLETE':
status, trigger, result_hash = 'STOPPED', 'BOOKED_GPU_WALL_CAP', None
terminal = dict(ticket, status=status, trigger=trigger, result_sha256=result_hash, worker=worker_receipt,
physical_uuid=uuid, guard_wall_seconds=end - began, monitor=state,
charged_gpu_seconds=end - ticket['guard_started_monotonic'],
booked_gpu_cap_exceeded=end > deadline, recorded_peaks=tracker.maxima, no_retry=True)
save(root / 'complete.json', terminal)
return terminal
+143
View File
@@ -0,0 +1,143 @@
"""Run one formally bound GINO fit or the corresponding one-pass score."""
import argparse
import ctypes
import os
import signal
import sys
from pathlib import Path
from tempfile import TemporaryDirectory
import time
import track_b_gino_formal as formal
from track_b_gino_paper_keyword_patch import extract_patched_paper_code
from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha
from track_b_gino_shapenet_worker import (
OriginalData, case_scores, cuda_peaks, paper_model, test_environment, train_seed,
)
def attach(root, seed, phase):
ticket = read(root / 'dispatch.json')
parent = os.getppid()
if (ticket['coordinator'] != process_identity(parent)
or ticket['contract_sha256'] != sha(formal.ROOT / 'contract.json')
or (ticket['seed'], ticket['phase']) != (seed, phase)
or time.monotonic() >= ticket['start_by_monotonic']):
raise PermissionError('Worker is not attached to the authorized guard')
if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) or os.getppid() != parent:
raise RuntimeError('Could not attach worker lifetime to its guard')
c = formal.contract()
formal.authorize(c)
if str(Path(sys.executable).absolute()) != c['environment']['python']:
raise PermissionError('Unreviewed interpreter')
return c, ticket
def bind_device(root, c, ticket, torch):
ordinal = ticket['ordinal']
uuid = formal.DEVICES[str(ticket['seed'])]
dispatch = read(root / 'ordinal-dispatch.json')
if (dispatch['ordinal'] != ordinal or dispatch['physical_uuid'] != uuid
or os.environ.get('CUDA_VISIBLE_DEVICES') != ordinal
or os.environ.get('CUDA_DEVICE_ORDER') != 'PCI_BUS_ID'
or torch.cuda.device_count() != 1 or torch.version.cuda != c['environment']['torch_cuda']):
raise PermissionError('One reviewed CUDA PCI ordinal is required')
torch.cuda.set_device(0)
name = torch.cuda.get_device_name(0)
if 'V100' not in name:
raise PermissionError('The reviewed device must be a V100')
probe = torch.empty(1, dtype=torch.uint8, device='cuda:0')
current = process_identity(os.getpid())
save(root / 'worker-device-witness.json', dict(
process=current, ordinal=ordinal, device_count=1, device_name=name,
physical_uuid_expected=uuid, first_allocation_bytes=probe.numel()))
verified = root / 'worker-device-verified.json'
while time.monotonic() < ticket['start_by_monotonic']:
if verified.exists():
record = read(verified)
expected = dict(process=current, physical_uuid=uuid, ordinal=ordinal, observed_uuids=[uuid])
if record != expected:
raise PermissionError('Actual worker physical-device witness differs')
return
time.sleep(.1)
raise RuntimeError('Physical device was not verified in the admission window')
def train(c, seed, data, root, records, torch):
learning = data.data(data.train_ids, 'train')
epochs = c['protocol']['epochs_per_seed']
model, times = train_seed(c, learning, seed, epochs, records / 'progress.json')
if len(times) != epochs:
raise ValueError('Full training did not finish')
formal.epoch100(root, seed, c)
path = root / 'epoch-100.pt'
temporary = root / 'epoch-100.pt.tmp'
with temporary.open('xb') as stream:
torch.save(dict(state=model.state_dict(), pressure_mean=data.normalizer.mean,
pressure_std=data.normalizer.std), stream)
temporary.rename(path)
return dict(checkpoint=dict(path=str(path), sha256=sha(path)), target_reads=0,
epochs=epochs, training_examples=500, epoch_seconds=times)
def score(c, seed, data, root, torch):
# Targets are still unopened while rechecking all three finished fits,
# initializing the exact model and loading its immutable checkpoint.
manifest = root / 'checkpoints-frozen.json'
manifest_sha = sha(manifest)
roster = read(manifest)
if roster != formal.all_checkpoints(c):
raise PermissionError('Three complete, pinned checkpoints must precede custody')
checkpoint = roster[str(seed)]['checkpoint']
state = torch.load(checked(checkpoint['path'], checkpoint['sha256']), map_location='cpu')
model = paper_model(c).to('cuda')
model.load_state_dict(state['state'], strict=True)
# The pinned normalizer's encode/decode and device methods use these
# fitted tensors and epsilon. No learning or target dataset is refitted.
normalizer = object.__new__(data.normalizer_class)
normalizer.mean = state['pressure_mean']
normalizer.std = state['pressure_std']
normalizer.eps = c['protocol']['pressure_epsilon']
if any(t.shape != (1,) or not bool(torch.isfinite(t).all())
for t in (normalizer.mean, normalizer.std)) or bool(normalizer.std < 0):
raise ValueError('Invalid frozen learning-only pressure statistics')
data.normalizer = normalizer
access = root / 'heldout-target-access.jsonl'
with access.open('x') as log:
heldout = data.data(data.test_ids, 'heldout', log)
rows = case_scores(model, heldout, data.test_ids, data.normalizer)
torch.cuda.synchronize()
return dict(cases=rows, mean_percent=formal.score_mean(rows, data.test_ids),
target_reads=len(rows), target_access_sha256=sha(access),
checkpoints_manifest_sha256=manifest_sha, checkpoint_sha256=checkpoint['sha256'])
def run(seed, phase):
root = formal.run_root(seed, phase)
c, ticket = attach(root, seed, phase)
test_environment(c)
import torch
torch.set_num_threads(c['limits']['torch_threads'])
torch.set_num_interop_threads(1)
bind_device(root, c, ticket, torch)
torch.cuda.reset_peak_memory_stats()
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
records = root / 'worker-records'
records.mkdir(exist_ok=False)
with TemporaryDirectory(prefix='formal-gino-', dir=root) as scratch:
extract_patched_paper_code(c, scratch)
data = OriginalData(c, scratch)
result = (train(c, seed, data, root, records, torch) if phase == 'fit'
else score(c, seed, data, root, torch))
save(records / 'result.json', dict(result, phase=phase, seed=seed, status='COMPLETE', **cuda_peaks(torch)))
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('seed', type=int, choices=(0, 1, 2))
parser.add_argument('phase', choices=('fit', 'score'))
args = parser.parse_args()
run(args.seed, args.phase)