Bind formal three-seed GINO with CPU guard and custody replay
This commit is contained in:
1 parent
93d7262e83
commit
9a4f8563ee
11 files changed
+1809
No files matched your search
@@ -0,0 +1,314 @@
|
||||
"""Exact review, custody and resource binding for formal three-seed GINO."""
|
||||
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
import fcntl
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import time
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from track_b_gino_formal_guard import guard
|
||||
from track_b_gino_import_checked_entry import accepted_environment
|
||||
from track_b_gino_ordinal_recovery import ROOT as MEASUREMENT
|
||||
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
|
||||
|
||||
|
||||
ROOT = MEASUREMENT.with_name('track-b-gino-shapenet611-formal-2026-11-19')
|
||||
SOURCES = Path(__file__).resolve().parent
|
||||
OLD_FILES = {
|
||||
'contract.json': 'ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a',
|
||||
'complete.json': 'db3605159c5ade3deda5313ab409e2f1c8bf630511a617f527177da06d6c9a2d',
|
||||
'recovery-settlement.json': '851eaedf716f01c1d80ce637f7de3edd0664c734bcbfed5622623d99f572fcec',
|
||||
'worker-records/progress.json': '8f8fb1902c44e2f9c3198608cbecb3a74967ed00fdbcaf1e990d146c894b0daf',
|
||||
}
|
||||
DEVICES = {
|
||||
'0': 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76',
|
||||
'1': 'GPU-8b71b032-1b75-2891-935c-ce106a53034a',
|
||||
'2': 'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c',
|
||||
}
|
||||
ORDINALS = {'0': 2, '1': 3, '2': 0}
|
||||
INVENTORY = BASE / 'artifacts/track-b-gino-gpu2-ordinal-preparation-2026-11-19/physical-inventory.json'
|
||||
INVENTORY_SHA = 'da2a7b1bf168c7cc4d45b6eaa2e007ae90cb3640eeadde94c10fd1d32fa30dd8'
|
||||
TZ = ZoneInfo('Asia/Hong_Kong')
|
||||
RESERVATION_START = datetime(2026, 11, 19, tzinfo=TZ)
|
||||
RESERVATION_END = datetime(2026, 11, 20, tzinfo=TZ)
|
||||
NEW_SOURCES = ('track_b_gino_formal.py', 'track_b_gino_formal_guard.py',
|
||||
'track_b_gino_formal_worker.py')
|
||||
|
||||
|
||||
def expected_contract():
|
||||
old = {name: read(checked(MEASUREMENT / name, digest)) for name, digest in OLD_FILES.items()}
|
||||
c = old['contract.json']
|
||||
stop, epoch = old['complete.json'], old['worker-records/progress.json']
|
||||
account = old['recovery-settlement.json']
|
||||
if (stop['status'], stop['trigger'], stop['result_sha256']) != ('STOPPED', 'FOREIGN_GPU_OCCUPANT', None):
|
||||
raise ValueError('Historical guard STOP differs')
|
||||
if (epoch['stage'], epoch['seed'], epoch['epoch'], epoch['examples']) != ('EPOCH_COMPLETE', 0, 1, 500):
|
||||
raise ValueError('Only the completed 500-example epoch can size this reproduction')
|
||||
seconds = epoch['seconds']
|
||||
inventory = read(checked(INVENTORY, INVENTORY_SHA))['inventory']
|
||||
physical = {(int(index), uuid) for index, uuid, _, name in inventory if 'V100' in name}
|
||||
if not {(ORDINALS[seed], uuid) for seed, uuid in DEVICES.items()} <= physical:
|
||||
raise ValueError('Reviewed physical V100 inventory differs')
|
||||
budgets = {seed: {'fit': 13033 + (seed == '0'), 'score': 300} for seed in DEVICES}
|
||||
fit_reference = 100 * seconds * 1.25
|
||||
if (not math.isfinite(fit_reference) or fit_reference <= 0
|
||||
or min(b['fit'] for b in budgets.values()) <= fit_reference
|
||||
or sum(b['fit'] + b['score'] for b in budgets.values()) != 40000):
|
||||
raise ValueError('Formal booking must cover the sizing reference and score reserve')
|
||||
c['mode'] = 'formal-gino-reproduction'
|
||||
for key in ('gpu_uuid', 'measurement_allowed_heldout_labels', 'gpu_ruling', 'ordinal_recovery'):
|
||||
del c[key]
|
||||
c['code_sha256'].update({str(SOURCES / name): sha(SOURCES / name) for name in NEW_SOURCES})
|
||||
c['formal'] = dict(
|
||||
handoff='h-4e4e680b', ruling_slack_ts='1795074651.222599', devices=DEVICES,
|
||||
device_pci_ordinals=ORDINALS,
|
||||
reservation=dict(handoff='h-a429935e', timezone=TZ.key,
|
||||
start_inclusive=RESERVATION_START.isoformat(),
|
||||
end_exclusive=RESERVATION_END.isoformat(), gpu3_single_task=True,
|
||||
historical_inventory_path=str(INVENTORY), historical_inventory_sha256=INVENTORY_SHA,
|
||||
runtime_idle_check_required=True),
|
||||
budget_per_seed_stage_gpu_seconds=budgets, total_gpu_wall_seconds=40000,
|
||||
timing=dict(record_paths={str(MEASUREMENT / k): v for k, v in OLD_FILES.items()},
|
||||
status='CONTENTION_AFFECTED_NOT_GUARD_CERTIFIED', sizing_only=True,
|
||||
epoch_seconds=seconds, train_sizing_gpu_seconds=300 * seconds * 1.25,
|
||||
train_sizing_multiplier=1.25, score_reserved_gpu_seconds=900,
|
||||
prior_measurement_gpu_seconds=account['cumulative_measurement_gpu_wall_seconds']),
|
||||
heldout_policy=dict(preparation_reads=0, training_reads=0, score_reads_per_seed=111,
|
||||
all_epoch100_checkpoints_required=True, retry=False),
|
||||
)
|
||||
del c['limits']['gpu_wall_seconds']
|
||||
c['limits']['cleanup_seconds'] = 20
|
||||
del c['limits']['cpu_seconds']
|
||||
c['limits']['cpu_seconds_per_gpu_second'] = 8
|
||||
return c
|
||||
|
||||
|
||||
def contract():
|
||||
c = read(ROOT / 'contract.json')
|
||||
if c != expected_contract():
|
||||
raise PermissionError('The sole formal contract or its implementation changed')
|
||||
for path, digest in c['code_sha256'].items():
|
||||
checked(path, digest)
|
||||
for name in ('paper_supplement', 'split_receipt', 'mapping_audit'):
|
||||
pin = c[name]
|
||||
checked(pin['path'], pin['sha256'])
|
||||
# Archive members are opened only in the scoped workers, which verify each
|
||||
# member before using it. Preparation does not hash the archive payload.
|
||||
comparison = read(checked(c['comparison_contract'], c['comparison_contract_sha256']))
|
||||
split = read(c['split_receipt']['path'])
|
||||
train, test = split['train'], split['test']
|
||||
if (c['protocol'] != comparison['protocol'] or len(train) != 500 or len(test) != 111
|
||||
or len(set(train) | set(test)) != 611
|
||||
or any(c['split'][k] != split[k] for k in ('train', 'test', 'raw_global_bounds'))
|
||||
or c['protocol']['epochs_per_seed'] != 100 or c['protocol']['seeds'] != [0, 1, 2]
|
||||
or c['protocol']['acceptance_percent'] != [6.62, 7.62]
|
||||
or c['paper_compatibility']['identity'] != 'supplement code + 2-keyword patch'
|
||||
or not Path(c['paper_data']['path']).is_file()):
|
||||
raise ValueError('Original GINO comparison, split or pinned data path differs')
|
||||
accepted_environment(c)
|
||||
return c
|
||||
|
||||
|
||||
def authorize(c):
|
||||
validate_review(ROOT)
|
||||
authorization = read(ROOT / 'stage-authorization.json')
|
||||
required = dict(contract_sha256=sha(ROOT / 'contract.json'),
|
||||
implementation_proof_sha256=sha(ROOT / 'implementation-git-freeze.json'),
|
||||
decision='APPROVED', mode=c['mode'], reserved_devices=DEVICES,
|
||||
reservation_end_exclusive=RESERVATION_END.isoformat())
|
||||
if any(authorization.get(k) != v for k, v in required.items()):
|
||||
raise PermissionError('Exact execution and card-reservation authorization required')
|
||||
|
||||
|
||||
def check_reservation(unix, duration):
|
||||
if (unix < RESERVATION_START.timestamp()
|
||||
or unix + duration >= RESERVATION_END.timestamp()):
|
||||
raise PermissionError('The entire booking must fit the November 19 Hong Kong reservation')
|
||||
|
||||
|
||||
def run_root(seed, phase):
|
||||
return ROOT / f'seed-{seed}' / phase
|
||||
|
||||
|
||||
def epoch100(root, seed, c):
|
||||
progress = read(root / 'worker-records/progress.json')
|
||||
if (progress['stage'], progress['seed'], progress['epoch'], progress['examples']) != (
|
||||
'EPOCH_COMPLETE', seed, c['protocol']['epochs_per_seed'], 500):
|
||||
raise ValueError('No completed 100th full training epoch for this seed')
|
||||
return progress
|
||||
|
||||
|
||||
def checkpoint(seed, result):
|
||||
path = run_root(seed, 'fit') / 'epoch-100.pt'
|
||||
if result['checkpoint']['path'] != str(path):
|
||||
raise ValueError('Checkpoint path differs from this seed')
|
||||
return checked(path, result['checkpoint']['sha256'])
|
||||
|
||||
|
||||
def score_mean(rows, test_ids):
|
||||
if [r['case'] for r in rows] != test_ids:
|
||||
raise ValueError('Scoring requires the original 111-case order')
|
||||
values = [r['relative_l2'] for r in rows]
|
||||
if any(type(v) not in (int, float) or not math.isfinite(v) or v < 0 for v in values):
|
||||
raise ValueError('Incomplete or invalid relative L2')
|
||||
return 100 * (math.fsum(values) / len(values))
|
||||
|
||||
|
||||
def validate_result(c, seed, phase):
|
||||
root = run_root(seed, phase)
|
||||
path = root / 'worker-records/result.json'
|
||||
result = read(path)
|
||||
spawned = read(root / 'actual-worker-spawn.json')
|
||||
device = DEVICES[str(seed)]
|
||||
ordinal = str(ORDINALS[str(seed)])
|
||||
if (spawned['physical_uuid'], spawned['ordinal']) != (device, ordinal) or (
|
||||
read(root / 'worker-device-verified.json') != dict(
|
||||
process=spawned['process'], physical_uuid=device, ordinal=ordinal, observed_uuids=[device])):
|
||||
raise ValueError('Worker has no matching physical-device witness')
|
||||
if (result['phase'], result['seed'], result['status']) != (phase, seed, 'COMPLETE'):
|
||||
raise ValueError('Worker result identity differs')
|
||||
if phase == 'fit':
|
||||
epoch100(root, seed, c)
|
||||
checkpoint(seed, result)
|
||||
if (result['target_reads'], result['epochs'], result['training_examples']) != (0, 100, 500):
|
||||
raise ValueError('Fit must complete all 100 epochs with zero held-out reads')
|
||||
else:
|
||||
fit_terminal = read(run_root(seed, 'fit') / 'complete.json')
|
||||
fit_path = run_root(seed, 'fit') / 'worker-records/result.json'
|
||||
original = read(checked(fit_path, fit_terminal['result_sha256']))
|
||||
access = root / 'heldout-target-access.jsonl'
|
||||
mapping = {row['case']: row['pressure_sha256'] for row in read(c['mapping_audit']['path'])['rows']}
|
||||
actual = [json.loads(line) for line in access.read_text().splitlines()]
|
||||
expected = [dict(case=i, role='heldout', sha256=mapping[i]) for i in c['split']['test']]
|
||||
if (actual != expected or result['target_reads'] != 111
|
||||
or result['target_access_sha256'] != sha(access)
|
||||
or result['checkpoints_manifest_sha256'] != sha(root / 'checkpoints-frozen.json')
|
||||
or read(root / 'checkpoints-frozen.json') != all_checkpoints(c)
|
||||
or result['checkpoint_sha256'] != original['checkpoint']['sha256']
|
||||
or score_mean(result['cases'], c['split']['test']) != result['mean_percent']):
|
||||
raise ValueError('Score or held-out target audit differs')
|
||||
return sha(path)
|
||||
|
||||
|
||||
def accepted_terminal(c, seed, phase):
|
||||
root = run_root(seed, phase)
|
||||
record = read(root / 'complete.json')
|
||||
if (record['status'], record['seed'], record['phase'], record['contract_sha256'], record['physical_uuid']) != (
|
||||
'COMPLETE', seed, phase, sha(ROOT / 'contract.json'), DEVICES[str(seed)]):
|
||||
raise PermissionError(f'Seed {seed} {phase} has no accepted terminal result')
|
||||
booked = c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase]
|
||||
if (record['booked_gpu_seconds'] != booked or record['charged_gpu_seconds'] > booked
|
||||
or record['result_sha256'] != validate_result(c, seed, phase)):
|
||||
raise ValueError('Terminal result bytes or resource accounting differ')
|
||||
return record
|
||||
|
||||
|
||||
def all_checkpoints(c):
|
||||
checkpoints = {}
|
||||
for seed in c['protocol']['seeds']:
|
||||
accepted_terminal(c, seed, 'fit')
|
||||
root = run_root(seed, 'fit')
|
||||
checkpoints[str(seed)] = dict(
|
||||
result_sha256=sha(root / 'worker-records/result.json'),
|
||||
terminal_sha256=sha(root / 'complete.json'),
|
||||
checkpoint=read(root / 'worker-records/result.json')['checkpoint'])
|
||||
return checkpoints
|
||||
|
||||
|
||||
def dispatch(c, seed, phase):
|
||||
began = time.monotonic()
|
||||
unix = time.time()
|
||||
authorize(c)
|
||||
budget = c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase]
|
||||
check_reservation(unix, budget)
|
||||
seed_root = run_root(seed, phase).parent
|
||||
seed_root.mkdir(exist_ok=True)
|
||||
with (seed_root / 'execution.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
roster = all_checkpoints(c) if phase == 'score' else None
|
||||
root = run_root(seed, phase)
|
||||
root.mkdir(exist_ok=False)
|
||||
if roster is not None:
|
||||
save(root / 'checkpoints-frozen.json', roster)
|
||||
ticket = dict(seed=seed, phase=phase, ordinal=str(ORDINALS[str(seed)]),
|
||||
booked_gpu_seconds=budget, unix=unix,
|
||||
contract_sha256=sha(ROOT / 'contract.json'),
|
||||
guard_started_monotonic=began, booked_deadline_monotonic=began + budget)
|
||||
save(root / 'started.json', ticket)
|
||||
command = [c['environment']['python'], str(SOURCES / 'track_b_gino_formal_worker.py'),
|
||||
str(seed), phase]
|
||||
|
||||
def request_stop(signum, _frame):
|
||||
raise KeyboardInterrupt(signal.Signals(signum).name)
|
||||
|
||||
previous = signal.signal(signal.SIGTERM, request_stop)
|
||||
try:
|
||||
limits = dict(c['limits'], cpu_seconds=c['limits']['cpu_seconds_per_gpu_second'] * budget)
|
||||
result = guard(root, command, uuid=DEVICES[str(seed)], limits=limits,
|
||||
deadline=began + budget, ticket=ticket,
|
||||
validate_result=lambda: validate_result(c, seed, phase))
|
||||
finally:
|
||||
signal.signal(signal.SIGTERM, previous)
|
||||
return 0 if result['status'] == 'COMPLETE' else 1
|
||||
|
||||
|
||||
def pool(means, limits):
|
||||
if len(means) != 3 or any(v is None for v in means):
|
||||
return None, None
|
||||
mean = math.fsum(means) / 3
|
||||
return mean, limits[0] <= mean <= limits[1]
|
||||
|
||||
|
||||
def collect(c):
|
||||
seeds = {}
|
||||
accounted = unsettled = 0
|
||||
for seed in c['protocol']['seeds']:
|
||||
rows = {}
|
||||
for phase in ('fit', 'score'):
|
||||
root = run_root(seed, phase)
|
||||
terminal = root / 'complete.json'
|
||||
if terminal.exists():
|
||||
record = read(terminal)
|
||||
if record['status'] == 'COMPLETE':
|
||||
accepted_terminal(c, seed, phase)
|
||||
accounted += record['charged_gpu_seconds']
|
||||
rows[phase] = dict(status=record['status'], trigger=record['trigger'],
|
||||
terminal_sha256=sha(terminal))
|
||||
if phase == 'score' and record['status'] == 'COMPLETE':
|
||||
rows['mean_percent'] = read(root / 'worker-records/result.json')['mean_percent']
|
||||
elif (root / 'started.json').exists():
|
||||
rows[phase] = dict(status='NO_TERMINAL_RECEIPT')
|
||||
unsettled += c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase]
|
||||
else:
|
||||
rows[phase] = dict(status='NOT_STARTED')
|
||||
seeds[str(seed)] = rows
|
||||
mean, verdict = pool([row.get('mean_percent') for row in seeds.values()],
|
||||
c['protocol']['acceptance_percent'])
|
||||
prior = c['formal']['timing']['prior_measurement_gpu_seconds']
|
||||
return dict(seeds=seeds, mean_percent=mean, within_registered_range=verdict,
|
||||
reproduction_identity=c['paper_compatibility']['identity'],
|
||||
prior_measurement_gpu_seconds=prior, settled_formal_gpu_seconds=accounted,
|
||||
settled_combined_gpu_seconds=accounted + prior,
|
||||
unsettled_formal_bookings_gpu_seconds=unsettled, formal_gpu_cap_seconds=40000)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('action', choices=('fit', 'score', 'collect'))
|
||||
parser.add_argument('--seed', type=int, choices=(0, 1, 2))
|
||||
args = parser.parse_args()
|
||||
c = contract()
|
||||
if args.action == 'collect':
|
||||
if args.seed is not None:
|
||||
parser.error('collect reports all three seeds')
|
||||
print(json.dumps(collect(c), indent=2, allow_nan=False))
|
||||
else:
|
||||
if args.seed is None:
|
||||
parser.error('--seed is required for fit or score')
|
||||
raise SystemExit(dispatch(c, args.seed, args.action))
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Supervise a formal GINO fit or score on one physical V100."""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from types import FunctionType, SimpleNamespace
|
||||
|
||||
import psutil
|
||||
|
||||
from s2_gpu_idle_gate import snapshot, wait_idle
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
from s2_recovery_gpu_supervisor import stop_group, supervise
|
||||
from track_b_gino_ordinal_recovery import dispatch_environment, monitor_identity
|
||||
from track_b_gino_shapenet_entry import process_identity, save
|
||||
|
||||
|
||||
def current_identity(pid):
|
||||
try:
|
||||
return process_identity(pid)
|
||||
except psutil.NoSuchProcess:
|
||||
return None
|
||||
|
||||
|
||||
def process_tree(worker):
|
||||
if current_identity(worker['pid']) != worker:
|
||||
return []
|
||||
try:
|
||||
children = psutil.Process(worker['pid']).children(recursive=True)
|
||||
except psutil.NoSuchProcess:
|
||||
return []
|
||||
result = []
|
||||
for child in children:
|
||||
try:
|
||||
result.append(dict(pid=child.pid, created=child.create_time()))
|
||||
except psutil.NoSuchProcess:
|
||||
continue
|
||||
return result
|
||||
|
||||
|
||||
class OccupancyMonitor:
|
||||
def __init__(self, root, uuid, worker, device_vram_bytes):
|
||||
self.root = root
|
||||
self.uuid = uuid
|
||||
self.worker = worker
|
||||
self.known = {worker['pid']: worker}
|
||||
self.device_vram_bytes = device_vram_bytes
|
||||
self.peak_device_bytes = 0
|
||||
|
||||
def check(self):
|
||||
before = process_tree(self.worker)
|
||||
state = snapshot(self.uuid)
|
||||
after = process_tree(self.worker)
|
||||
for identity in before + after:
|
||||
if identity['pid'] != self.worker['pid']:
|
||||
self.known[identity['pid']] = identity
|
||||
foreign = []
|
||||
for row in state['occupants']:
|
||||
pid = row['pid']
|
||||
current = current_identity(pid)
|
||||
if pid not in self.known or (current is not None and current != self.known[pid]):
|
||||
foreign.append(dict(row, observed_process_identity=current))
|
||||
if foreign:
|
||||
save(self.root / 'foreign-occupant.json', dict(
|
||||
trigger='FOREIGN_GPU_OCCUPANT', unix=state['unix'], physical_uuid=self.uuid,
|
||||
occupants=foreign, own_spawn=self.worker, tree_before=before, tree_after=after))
|
||||
raise RuntimeError('FOREIGN_GPU_OCCUPANT')
|
||||
self.peak_device_bytes = max(self.peak_device_bytes, state['memory_MiB'] * 2**20)
|
||||
if self.peak_device_bytes > self.device_vram_bytes:
|
||||
raise RuntimeError('DEVICE_VRAM_LIMIT')
|
||||
|
||||
|
||||
def watch(root, uuid, worker, limits, done, tracker, state):
|
||||
occupancy = OccupancyMonitor(root, uuid, worker, limits['device_vram_bytes'])
|
||||
while not done.is_set():
|
||||
try:
|
||||
occupancy.check()
|
||||
state['peak_device_bytes'] = occupancy.peak_device_bytes
|
||||
tracker.scan(root / 'worker-records')
|
||||
if max(tracker.maxima.values()) > limits['recorded_vram_bytes']:
|
||||
raise RuntimeError('RECORDED_VRAM_LIMIT')
|
||||
monitor_identity(root, uuid)
|
||||
except Exception as exc:
|
||||
state['trigger'] = str(exc) or type(exc).__name__
|
||||
# Signal only the recorded worker group while its leader has the same identity.
|
||||
if current_identity(worker['pid']) == worker:
|
||||
stop_group(worker['pid'])
|
||||
return
|
||||
done.wait(.5)
|
||||
|
||||
|
||||
def guard(root, command, *, uuid, limits, deadline, ticket, validate_result):
|
||||
began = time.monotonic()
|
||||
admission = min(deadline - limits['cleanup_seconds'], began + limits['idle_wait_seconds'])
|
||||
done = threading.Event()
|
||||
tracker = PeakRecords()
|
||||
state = dict(trigger=None, peak_device_bytes=0)
|
||||
watcher = None
|
||||
worker_receipt = None
|
||||
status, trigger, result_hash = 'STOPPED', None, None
|
||||
|
||||
def spawn(args, **kwargs):
|
||||
nonlocal watcher
|
||||
env, inventory = dispatch_environment(uuid, kwargs['env'])
|
||||
ordinal = env['CUDA_VISIBLE_DEVICES']
|
||||
if ordinal != ticket['ordinal']:
|
||||
raise RuntimeError('PINNED_PCI_ORDINAL_DIFFERS')
|
||||
if not snapshot(uuid)['idle']:
|
||||
raise RuntimeError('SELECTED_GPU_BECAME_BUSY')
|
||||
save(root / 'ordinal-dispatch.json', dict(ordinal=ordinal, physical_uuid=uuid, inventory=inventory))
|
||||
if time.monotonic() >= admission:
|
||||
raise RuntimeError('ADMISSION_DEADLINE')
|
||||
proc = subprocess.Popen(args, **dict(kwargs, env=env))
|
||||
try:
|
||||
worker = process_identity(proc.pid)
|
||||
save(root / 'actual-worker-spawn.json', dict(process=worker, ordinal=ordinal, physical_uuid=uuid))
|
||||
watcher = threading.Thread(target=watch, daemon=True,
|
||||
args=(root, uuid, worker, limits, done, tracker, state))
|
||||
watcher.start()
|
||||
except (Exception, KeyboardInterrupt):
|
||||
# The shared supervisor has not yet received the process to reap.
|
||||
stop_group(proc.pid)
|
||||
proc.wait()
|
||||
raise
|
||||
return proc
|
||||
|
||||
try:
|
||||
if shutil.disk_usage(root).free < limits['disk_bytes']:
|
||||
raise RuntimeError('DISK_HEADROOM')
|
||||
idle = wait_idle(uuid, max(0., admission - time.monotonic()), lambda _: None, sample=snapshot)
|
||||
save(root / 'idle-gate.json', idle)
|
||||
if idle['status'] != 'IDLE_CONFIRMED' or time.monotonic() >= admission:
|
||||
raise RuntimeError('IDLE_DEADLINE')
|
||||
save(root / 'dispatch.json', dict(ticket, coordinator=process_identity(os.getpid()),
|
||||
start_by_monotonic=admission))
|
||||
remaining = deadline - time.monotonic() - limits['cleanup_seconds']
|
||||
if remaining <= 0:
|
||||
raise RuntimeError('INSUFFICIENT_REMAINING_BUDGET')
|
||||
ns = dict(supervise.__globals__, subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT))
|
||||
run_worker = FunctionType(supervise.__code__, ns, supervise.__name__)
|
||||
try:
|
||||
worker_receipt = run_worker(command, root, 'gino', dict(
|
||||
total_cpu_seconds=limits['cpu_seconds'], job_cpu_seconds=limits['cpu_seconds'],
|
||||
job_wall_seconds=remaining, tree_rss_bytes=limits['rss_bytes'],
|
||||
disk_bytes=limits['disk_bytes'], max_processes=limits['max_processes']),
|
||||
poll=.2, gpu_uuid=uuid)
|
||||
finally:
|
||||
done.set()
|
||||
if watcher is not None:
|
||||
watcher.join(timeout=8)
|
||||
if watcher is not None and watcher.is_alive():
|
||||
raise RuntimeError('MONITOR_DID_NOT_EXIT')
|
||||
tracker.scan(root / 'worker-records')
|
||||
if state['trigger']:
|
||||
raise RuntimeError(state['trigger'])
|
||||
if max(tracker.maxima.values()) > limits['recorded_vram_bytes']:
|
||||
raise RuntimeError('RECORDED_VRAM_LIMIT')
|
||||
if worker_receipt['status'] != 'PASS' or worker_receipt['monitor_errors']:
|
||||
raise RuntimeError(worker_receipt.get('stop_reason') or 'WORKER_FAILED')
|
||||
if set(tracker.witnesses) != set(tracker.maxima):
|
||||
raise RuntimeError('MISSING_RECORDED_MEMORY_PEAKS')
|
||||
result_hash = validate_result()
|
||||
status = 'COMPLETE'
|
||||
except (Exception, KeyboardInterrupt) as exc:
|
||||
trigger = str(exc) or type(exc).__name__
|
||||
end = time.monotonic()
|
||||
if end > deadline and status == 'COMPLETE':
|
||||
status, trigger, result_hash = 'STOPPED', 'BOOKED_GPU_WALL_CAP', None
|
||||
terminal = dict(ticket, status=status, trigger=trigger, result_sha256=result_hash, worker=worker_receipt,
|
||||
physical_uuid=uuid, guard_wall_seconds=end - began, monitor=state,
|
||||
charged_gpu_seconds=end - ticket['guard_started_monotonic'],
|
||||
booked_gpu_cap_exceeded=end > deadline, recorded_peaks=tracker.maxima, no_retry=True)
|
||||
save(root / 'complete.json', terminal)
|
||||
return terminal
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Run one formally bound GINO fit or the corresponding one-pass score."""
|
||||
|
||||
import argparse
|
||||
import ctypes
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
import time
|
||||
|
||||
import track_b_gino_formal as formal
|
||||
from track_b_gino_paper_keyword_patch import extract_patched_paper_code
|
||||
from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha
|
||||
from track_b_gino_shapenet_worker import (
|
||||
OriginalData, case_scores, cuda_peaks, paper_model, test_environment, train_seed,
|
||||
)
|
||||
|
||||
|
||||
def attach(root, seed, phase):
|
||||
ticket = read(root / 'dispatch.json')
|
||||
parent = os.getppid()
|
||||
if (ticket['coordinator'] != process_identity(parent)
|
||||
or ticket['contract_sha256'] != sha(formal.ROOT / 'contract.json')
|
||||
or (ticket['seed'], ticket['phase']) != (seed, phase)
|
||||
or time.monotonic() >= ticket['start_by_monotonic']):
|
||||
raise PermissionError('Worker is not attached to the authorized guard')
|
||||
if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) or os.getppid() != parent:
|
||||
raise RuntimeError('Could not attach worker lifetime to its guard')
|
||||
c = formal.contract()
|
||||
formal.authorize(c)
|
||||
if str(Path(sys.executable).absolute()) != c['environment']['python']:
|
||||
raise PermissionError('Unreviewed interpreter')
|
||||
return c, ticket
|
||||
|
||||
|
||||
def bind_device(root, c, ticket, torch):
|
||||
ordinal = ticket['ordinal']
|
||||
uuid = formal.DEVICES[str(ticket['seed'])]
|
||||
dispatch = read(root / 'ordinal-dispatch.json')
|
||||
if (dispatch['ordinal'] != ordinal or dispatch['physical_uuid'] != uuid
|
||||
or os.environ.get('CUDA_VISIBLE_DEVICES') != ordinal
|
||||
or os.environ.get('CUDA_DEVICE_ORDER') != 'PCI_BUS_ID'
|
||||
or torch.cuda.device_count() != 1 or torch.version.cuda != c['environment']['torch_cuda']):
|
||||
raise PermissionError('One reviewed CUDA PCI ordinal is required')
|
||||
torch.cuda.set_device(0)
|
||||
name = torch.cuda.get_device_name(0)
|
||||
if 'V100' not in name:
|
||||
raise PermissionError('The reviewed device must be a V100')
|
||||
probe = torch.empty(1, dtype=torch.uint8, device='cuda:0')
|
||||
current = process_identity(os.getpid())
|
||||
save(root / 'worker-device-witness.json', dict(
|
||||
process=current, ordinal=ordinal, device_count=1, device_name=name,
|
||||
physical_uuid_expected=uuid, first_allocation_bytes=probe.numel()))
|
||||
verified = root / 'worker-device-verified.json'
|
||||
while time.monotonic() < ticket['start_by_monotonic']:
|
||||
if verified.exists():
|
||||
record = read(verified)
|
||||
expected = dict(process=current, physical_uuid=uuid, ordinal=ordinal, observed_uuids=[uuid])
|
||||
if record != expected:
|
||||
raise PermissionError('Actual worker physical-device witness differs')
|
||||
return
|
||||
time.sleep(.1)
|
||||
raise RuntimeError('Physical device was not verified in the admission window')
|
||||
|
||||
|
||||
def train(c, seed, data, root, records, torch):
|
||||
learning = data.data(data.train_ids, 'train')
|
||||
epochs = c['protocol']['epochs_per_seed']
|
||||
model, times = train_seed(c, learning, seed, epochs, records / 'progress.json')
|
||||
if len(times) != epochs:
|
||||
raise ValueError('Full training did not finish')
|
||||
formal.epoch100(root, seed, c)
|
||||
path = root / 'epoch-100.pt'
|
||||
temporary = root / 'epoch-100.pt.tmp'
|
||||
with temporary.open('xb') as stream:
|
||||
torch.save(dict(state=model.state_dict(), pressure_mean=data.normalizer.mean,
|
||||
pressure_std=data.normalizer.std), stream)
|
||||
temporary.rename(path)
|
||||
return dict(checkpoint=dict(path=str(path), sha256=sha(path)), target_reads=0,
|
||||
epochs=epochs, training_examples=500, epoch_seconds=times)
|
||||
|
||||
|
||||
def score(c, seed, data, root, torch):
|
||||
# Targets are still unopened while rechecking all three finished fits,
|
||||
# initializing the exact model and loading its immutable checkpoint.
|
||||
manifest = root / 'checkpoints-frozen.json'
|
||||
manifest_sha = sha(manifest)
|
||||
roster = read(manifest)
|
||||
if roster != formal.all_checkpoints(c):
|
||||
raise PermissionError('Three complete, pinned checkpoints must precede custody')
|
||||
checkpoint = roster[str(seed)]['checkpoint']
|
||||
state = torch.load(checked(checkpoint['path'], checkpoint['sha256']), map_location='cpu')
|
||||
model = paper_model(c).to('cuda')
|
||||
model.load_state_dict(state['state'], strict=True)
|
||||
# The pinned normalizer's encode/decode and device methods use these
|
||||
# fitted tensors and epsilon. No learning or target dataset is refitted.
|
||||
normalizer = object.__new__(data.normalizer_class)
|
||||
normalizer.mean = state['pressure_mean']
|
||||
normalizer.std = state['pressure_std']
|
||||
normalizer.eps = c['protocol']['pressure_epsilon']
|
||||
if any(t.shape != (1,) or not bool(torch.isfinite(t).all())
|
||||
for t in (normalizer.mean, normalizer.std)) or bool(normalizer.std < 0):
|
||||
raise ValueError('Invalid frozen learning-only pressure statistics')
|
||||
data.normalizer = normalizer
|
||||
access = root / 'heldout-target-access.jsonl'
|
||||
with access.open('x') as log:
|
||||
heldout = data.data(data.test_ids, 'heldout', log)
|
||||
rows = case_scores(model, heldout, data.test_ids, data.normalizer)
|
||||
torch.cuda.synchronize()
|
||||
return dict(cases=rows, mean_percent=formal.score_mean(rows, data.test_ids),
|
||||
target_reads=len(rows), target_access_sha256=sha(access),
|
||||
checkpoints_manifest_sha256=manifest_sha, checkpoint_sha256=checkpoint['sha256'])
|
||||
|
||||
|
||||
def run(seed, phase):
|
||||
root = formal.run_root(seed, phase)
|
||||
c, ticket = attach(root, seed, phase)
|
||||
test_environment(c)
|
||||
import torch
|
||||
|
||||
torch.set_num_threads(c['limits']['torch_threads'])
|
||||
torch.set_num_interop_threads(1)
|
||||
bind_device(root, c, ticket, torch)
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
torch.backends.cuda.matmul.allow_tf32 = False
|
||||
torch.backends.cudnn.allow_tf32 = False
|
||||
records = root / 'worker-records'
|
||||
records.mkdir(exist_ok=False)
|
||||
with TemporaryDirectory(prefix='formal-gino-', dir=root) as scratch:
|
||||
extract_patched_paper_code(c, scratch)
|
||||
data = OriginalData(c, scratch)
|
||||
result = (train(c, seed, data, root, records, torch) if phase == 'fit'
|
||||
else score(c, seed, data, root, torch))
|
||||
save(records / 'result.json', dict(result, phase=phase, seed=seed, status='COMPLETE', **cuda_peaks(torch)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('seed', type=int, choices=(0, 1, 2))
|
||||
parser.add_argument('phase', choices=('fit', 'score'))
|
||||
args = parser.parse_args()
|
||||
run(args.seed, args.phase)
|
||||
Reference in new issue
Block a user