Bind H2-P ShapeNet training, first-epoch gate and paired scoring for review

This commit is contained in:
NewMeltAssist committed 2026-11-19 18:12:25 +08:00
1 parent db87b0f54b
commit 70a85dffb8
10 files changed
+7577

No files matched your search

@@ -0,0 +1,54 @@
"""Freeze H2-P comparison metadata and audit only original triangle mesh members."""
from pathlib import Path
from tempfile import TemporaryDirectory
from zipfile import ZipFile
import torch
import track_b_h2_pressure_entry as entry
from track_b_gino_paper_keyword_patch import extract_patched_paper_code
from track_b_gino_shapenet_entry import read, save
from track_b_h2_pressure_worker import SurfaceData, mesh_graph
def prepare():
c = entry.reference.contract()
entry.ROOT.mkdir(exist_ok=False)
torch.set_num_threads(1)
with TemporaryDirectory(prefix='h2-p-mesh-only-') as folder:
extract_patched_paper_code(c, folder)
path = Path(folder)
data = SurfaceData(c, path)
rows = []
class MeshOnly:
def __init__(self, archive):
self.archive = archive
def read(self, member):
if not member.startswith('car-pressure-data/data/mesh_') or not member.endswith('.ply'):
raise PermissionError('Preparation may only open original meshes')
return self.archive.read(member)
with ZipFile(c['paper_data']['path']) as archive:
for case in data.train_ids + data.test_ids:
graph, faces = mesh_graph(c, path, MeshOnly(archive), data.rows[case], data.paper)
x, mass, receivers, _, _, components = graph
rows.append(dict(case=case, vertices=len(x), faces=faces,
edge_count=len(receivers), components=int(components.max()) + 1,
total_positive_area=float(mass.double().sum())))
save(entry.ROOT/'geometry-audit.json', dict(status='PASS', mesh_members_opened=len(rows),
target_members_opened=0, actual_gpu_queries=0, rows=rows))
save(entry.ROOT/'comparison-contract.json', entry.expected_comparison(c))
save(entry.ROOT/'contract.json', entry.expected_contract())
assert entry.contract()['identity'] == 'H2-P'
print(f'{len(rows)} original shape meshes audited, zero pressure members')
if __name__ == '__main__':
import os
if os.environ.get('CUDA_VISIBLE_DEVICES') != '':
raise PermissionError('Preparation is CPU only')
prepare()
+346
View File
@@ -0,0 +1,346 @@
"""One reviewed, cost-gated H2-P pipeline with frozen three-seed custody."""
import argparse
from collections import defaultdict
import fcntl
import json
import math
from pathlib import Path
import signal
import time
from phase2_gate_outcome_review import validate_review
from s2_gpu_idle_gate import snapshot
import track_b_gino_formal as reference
from track_b_gino_formal_guard import guard
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
ROOT = BASE / 'artifacts/track-b-h2-pressure-combined-2026-11-19'
REFERENCE_SHA = 'c3c5c5f0fab049c757032c343673104eb0f99cbaead5e0f78247f2a3aa6173c3'
DEVICES = {'0': reference.DEVICES['2'], '1': 'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
'2': reference.DEVICES['0']}
SEEDS = (0, 1, 2)
CAP = 40000
SCORE_RESERVE = 300
BOOTSTRAP_REPS = 2000
BOOTSTRAP_SEED = 20261119
SOURCES = Path(__file__).resolve().parent
def geometry_components(c):
mapping = read(c['mapping_audit']['path'])['rows']
by_case = {row['case']: row['mesh_sha256'] for row in mapping}
if set(by_case) != set(c['split']['train'] + c['split']['test']):
raise ValueError('Mesh identity roster differs')
if {by_case[i] for i in c['split']['train']} & {by_case[i] for i in c['split']['test']}:
raise ValueError('Identical raw mesh would cross the fixed learning and held-out split')
groups = defaultdict(list)
for case in c['split']['test']:
groups[by_case[case]].append(case)
return list(groups.values())
def expected_comparison(c):
return dict(handoff='h-b5985d03', identity='H2-P', seeds=list(SEEDS),
train_cases=c['split']['train'], heldout_cases=c['split']['test'],
pressure_selection='original raw rows [0:16] and [112:] in original vertex order',
metric=c['protocol']['metric'], summary='equal cases, then equal matched seeds',
component_definition='exact source triangle-mesh SHA-256, geometry-only; no known family identifiers',
heldout_components=geometry_components(c),
bootstrap=dict(replicates=BOOTSTRAP_REPS, seed=BOOTSTRAP_SEED, confidence=.95,
method='paired component resampling with replacement; case weighted; all three paired seeds retained'),
paired_delta='H2-P minus GINO in percentage points',
gate='all three exact seed pairs valid, GINO reproduction mean in [6.62,7.62], paired CI high < 0',
no_test_selection=True)
def expected_contract():
base = reference.contract()
base_path = checked(reference.ROOT / 'contract.json', REFERENCE_SHA)
ref = dict(path=str(base_path), sha256=REFERENCE_SHA,
comparison_sha256=base['comparison_contract_sha256'])
geometry = ROOT / 'geometry-audit.json'
audit = read(geometry)
if (audit['status'] != 'PASS' or audit['mesh_members_opened'] != 611 or audit['target_members_opened'] != 0
or [r['case'] for r in audit['rows']] != base['split']['train'] + base['split']['test']):
raise ValueError('Complete geometry-only coverage is required')
comparison = ROOT / 'comparison-contract.json'
if read(comparison) != expected_comparison(base):
raise ValueError('Pre-registered comparison differs')
c = {key: base[key] for key in (
'paper_data', 'paper_supplement', 'paper_code_zip_sha256', 'paper_compatibility',
'split', 'split_receipt', 'mapping_audit', 'protocol', 'environment', 'environment_ruling',
'environment_acceptance', 'limits', 'code_sha256')}
c['code_sha256'].update({str(SOURCES / name): sha(SOURCES / name) for name in (
'track_b_h2_pressure_entry.py', 'track_b_h2_pressure_worker.py',
'track_b_h2_pressure_prepare.py', 'phase2_h2_model.py')})
c.update(mode='h2-pressure-combined', identity='H2-P', handoff='h-b5985d03', reference=ref,
geometry_audit=dict(path=str(geometry), sha256=sha(geometry)),
comparison_contract=str(comparison), comparison_contract_sha256=sha(comparison),
devices=DEVICES, first_epoch_seed=0, total_gpu_seconds=CAP, scoring_seconds_per_seed=SCORE_RESERVE,
timing_margin=1.25, future_fit_setup_floor_seconds=120, gpu3_eligible=False,
model=dict(inputs=['x','y','z'], edges='original triangle edges both directions',
mass='one-third incident area', message_components='surface edge connectivity',
architecture='shared four 64-wide H2 blocks and direct pressure head',
optimizer_and_case_loss='same Adam, StepLR and LpLoss as fixed GINO training body'),
custody=dict(train_reads_per_seed=500, heldout_reads_per_seed=111,
require_all_frozen_checkpoints=True, retries=0))
return c
def contract():
c = read(ROOT / 'contract.json')
if c != expected_contract():
raise PermissionError('H2-P contract or source binding changed')
for path, digest in c['code_sha256'].items():
checked(path, digest)
return c
def authorize(c):
review = validate_review(ROOT)
auth = read(ROOT / 'stage-authorization.json')
approved = auth.get('eligible_ordinals')
expires = auth.get('dispatch_before_unix')
if ((auth['decision'], auth['mode'], auth['contract_sha256'], auth['review_sha256']) !=
('APPROVED', c['mode'], sha(ROOT / 'contract.json'), sha(ROOT / 'review.json'))
or review['verdict'] != 'PASS'
or not isinstance(approved, list) or not approved or len(set(approved)) != len(approved)
or not set(approved) <= set(DEVICES)
or type(expires) not in (float, int) or not math.isfinite(expires)):
raise PermissionError('Exact independent PASS and stage/card authorization required')
return auth
def first_epoch_projection(ticket, progress, now):
if tuple(progress[k] for k in ('stage', 'seed', 'epoch', 'examples')) != ('EPOCH_COMPLETE', 0, 1, 500):
raise ValueError('Only seed 0 first complete 500-case epoch measures cost')
epoch = progress['seconds']
spent = now - ticket['guard_started_monotonic']
setup = spent - epoch
if not math.isfinite(epoch) or not math.isfinite(setup) or epoch <= 0 or setup < 0:
raise ValueError('First full epoch and setup clock differ')
future_setup = max(120, math.ceil(1.25 * setup))
remaining = math.ceil(299 * epoch * 1.25) + 2 * future_setup + 3 * SCORE_RESERVE + 60
projected = spent + remaining
return dict(seed=0, epoch=1, examples=500, epoch_seconds=epoch, booked_seconds_so_far=spent,
measured_setup_seconds=setup, future_fit_setup_reserve_seconds=2 * future_setup,
remaining_train_epochs=299, safety_factor=1.25, scoring_reserve_seconds=3 * SCORE_RESERVE,
fit_cleanup_reserve_seconds=60, projected_total_gpu_seconds=projected,
formal_cap_gpu_seconds=CAP, within_cap=projected <= CAP)
def run_root(seed, phase):
return ROOT / f'seed-{seed}' / phase
score_mean = reference.score_mean
def validate_result(c, seed, phase):
root = run_root(seed, phase)
path = root / 'worker-records/result.json'
result = read(path)
ticket = read(root / 'started.json')
ordinal, uuid = ticket['ordinal'], ticket['physical_uuid']
spawn = read(root / 'actual-worker-spawn.json')
if (DEVICES[ordinal] != uuid or spawn['physical_uuid'] != uuid or spawn['ordinal'] != ordinal
or read(root / 'worker-device-verified.json') !=
dict(process=spawn['process'], physical_uuid=uuid, ordinal=ordinal, observed_uuids=[uuid])):
raise ValueError('Actual H2-P worker/device witness differs')
if (result['status'], result['seed'], result['phase']) != ('COMPLETE', seed, phase):
raise ValueError('Worker result identity differs')
if phase == 'fit':
reference.epoch100(root, seed, c)
checkpoint = result['checkpoint']
if (checkpoint['path'] != str(root / 'epoch-100.pt')
or (result['epochs'], result['training_examples'], result['target_reads']) != (100, 500, 0)
or (root / 'heldout-target-access.jsonl').exists()):
raise ValueError('Fit checkpoint or held-out seal differs')
checked(checkpoint['path'], checkpoint['sha256'])
if seed == 0 and read(root / 'first-epoch-projection.json')['within_cap'] is not True:
raise ValueError('Missing first epoch cost PASS')
else:
manifest = root / 'checkpoints-frozen.json'
roster = all_checkpoints(c)
access = root / 'heldout-target-access.jsonl'
mapping = {row['case']: row['pressure_sha256'] for row in read(c['mapping_audit']['path'])['rows']}
rows = [json.loads(line) for line in access.read_text().splitlines()]
expected = [dict(case=i, role='heldout', sha256=mapping[i]) for i in c['split']['test']]
if (read(manifest) != roster or result['checkpoints_manifest_sha256'] != sha(manifest)
or result['checkpoint_sha256'] != roster[str(seed)]['checkpoint']['sha256']
or result['target_reads'] != 111 or result['target_access_sha256'] != sha(access)
or rows != expected or score_mean(result['cases'], c['split']['test']) != result['mean_percent']):
raise ValueError('Held-out custody or score differs')
return sha(path)
def accepted_terminal(c, seed, phase):
root = run_root(seed, phase)
terminal = read(root / 'complete.json')
if (terminal['status'], terminal['seed'], terminal['phase'], terminal['contract_sha256']) != (
'COMPLETE', seed, phase, sha(ROOT / 'contract.json')):
raise PermissionError('No exact accepted stage terminal')
if (terminal['booked_gpu_cap_exceeded'] or terminal['charged_gpu_seconds'] > terminal['booked_gpu_seconds']
or terminal['result_sha256'] != validate_result(c, seed, phase)):
raise ValueError('H2-P terminal budget or result differs')
return terminal
def all_checkpoints(c):
roster = {}
for seed in SEEDS:
terminal = accepted_terminal(c, seed, 'fit')
root = run_root(seed, 'fit')
result = read(root / 'worker-records/result.json')
roster[str(seed)] = dict(checkpoint=result['checkpoint'], terminal_sha256=sha(root / 'complete.json'),
result_sha256=terminal['result_sha256'])
return roster
def run_stage(c, seed, phase, ordinal, auth, began, deadline, roster=None):
root = run_root(seed, phase)
root.mkdir(parents=True, exist_ok=False)
if phase == 'score':
save(root / 'checkpoints-frozen.json', roster)
uuid = DEVICES[ordinal]
ticket = dict(seed=seed, phase=phase, ordinal=ordinal, physical_uuid=uuid, unix=time.time(),
contract_sha256=sha(ROOT / 'contract.json'), dispatch_before_unix=auth['dispatch_before_unix'],
guard_started_monotonic=began, booked_deadline_monotonic=deadline,
booked_gpu_seconds=deadline - began)
save(root / 'started.json', ticket)
command = [c['environment']['python'], '-B', str(SOURCES / 'track_b_h2_pressure_worker.py'), str(seed), phase]
limits = dict(c['limits'], cpu_seconds=8 * (deadline - began))
return guard(root, command, uuid=uuid, limits=limits, deadline=deadline, ticket=ticket,
validate_result=lambda: validate_result(c, seed, phase))
def run(c):
auth = authorize(c)
with (ROOT / 'combined.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (ROOT / 'started.json').exists() or any(ROOT.glob('seed-*')):
raise FileExistsError('H2-P combined attempt already exists')
if time.time() >= auth['dispatch_before_unix']:
raise PermissionError('Card reservation for new dispatch has expired')
ordinal = next((i for i, uuid in DEVICES.items()
if i in auth['eligible_ordinals'] and snapshot(uuid)['idle']), None)
if ordinal is None:
raise RuntimeError('NO_IDLE_APPROVED_GPU0_TO_2')
began = time.monotonic()
deadline = began + CAP
save(ROOT / 'started.json', dict(ordinal=ordinal, physical_uuid=DEVICES[ordinal],
started_monotonic=began, contract_sha256=sha(ROOT / 'contract.json'), phase_budget=CAP))
def request_stop(number, _frame):
raise KeyboardInterrupt(signal.Signals(number).name)
old = signal.signal(signal.SIGTERM, request_stop)
status, trigger = 'STOPPED', None
try:
fit_deadline = deadline - len(SEEDS) * SCORE_RESERVE
for seed in SEEDS:
epoch_start = began if seed == 0 else time.monotonic()
if epoch_start >= fit_deadline:
raise RuntimeError('INSUFFICIENT_FIT_BUDGET')
receipt = run_stage(c, seed, 'fit', ordinal, auth, epoch_start, fit_deadline)
if receipt['status'] != 'COMPLETE':
cost = run_root(0, 'fit') / 'first-epoch-projection.json'
if seed == 0 and cost.exists() and read(cost)['within_cap'] is False:
raise RuntimeError('STOP_PROJECTED_CAP')
raise RuntimeError(f"Seed {seed} fit HOLD: {receipt['trigger']}")
roster = all_checkpoints(c)
save(ROOT / 'checkpoints-frozen.json', roster)
for seed in SEEDS:
score_start = time.monotonic()
score_deadline = min(deadline, score_start + SCORE_RESERVE)
if score_deadline - score_start <= c['limits']['cleanup_seconds']:
raise RuntimeError('INSUFFICIENT_SCORE_BUDGET')
receipt = run_stage(c, seed, 'score', ordinal, auth, score_start, score_deadline, roster)
if receipt['status'] != 'COMPLETE':
raise RuntimeError(f"Seed {seed} score HOLD: {receipt['trigger']}")
status = 'COMPLETE'
except (Exception, KeyboardInterrupt) as exc:
trigger = str(exc) or type(exc).__name__
finally:
signal.signal(signal.SIGTERM, old)
charged = time.monotonic() - began
if charged > CAP and status == 'COMPLETE':
status, trigger = 'STOPPED', 'GPU_BUDGET_OVERAGE'
save(ROOT / 'complete.json', dict(status=status, trigger=trigger, charged_gpu_seconds=charged,
cap_gpu_seconds=CAP, no_retry=True))
return 0 if status == 'COMPLETE' else 1
def paired_bootstrap(comparison, h2, gino):
import numpy as np
ids = comparison['heldout_cases']
groups = comparison['heldout_components']
if sorted(case for group in groups for case in group) != sorted(ids) or min(map(len, groups)) < 1:
raise ValueError('Fixed paired test component support differs')
if len(h2) != 3 or len(gino) != 3:
raise ValueError('All three exact matched seeds are required')
h2_means = [score_mean(rows, ids) for rows in h2]
gino_means = [score_mean(rows, ids) for rows in gino]
deltas = np.array([[a['relative_l2'] - b['relative_l2'] for a, b in zip(hr, gr)]
for hr, gr in zip(h2, gino)], dtype=np.float64).mean(axis=0)
per_case = dict(zip(ids, deltas))
masses = np.array([len(group) for group in groups])
sums = np.array([sum(per_case[case] for case in group) for group in groups])
drawn = np.random.default_rng(BOOTSTRAP_SEED).integers(len(groups), size=(BOOTSTRAP_REPS, len(groups)))
sampled = 100 * sums[drawn].sum(axis=1) / masses[drawn].sum(axis=1)
low, high = map(float, np.quantile(sampled, [.025, .975]))
reference_mean, within = reference.pool(gino_means, [6.62, 7.62])
return dict(h2_percent_by_seed=dict(zip(SEEDS, h2_means)), gino_percent_by_seed=dict(zip(SEEDS, gino_means)),
gino_mean_percent=reference_mean, gino_reproduction_pass=within,
paired_mean_difference_pp=float(100 * deltas.mean()), paired_ci95_pp=[low, high],
components=len(groups), decision=('GINO_REPRODUCTION_GATE_NOT_MET' if not within else
'H2_P_IMPROVEMENT_PASS' if high < 0 else 'H2_P_IMPROVEMENT_NOT_ESTABLISHED'))
def collect(c):
seeds, scores = {}, []
for seed in SEEDS:
rows = {}
for phase in ('fit', 'score'):
root = run_root(seed, phase)
if (root / 'complete.json').exists():
terminal = read(root / 'complete.json')
rows[phase] = dict(status=terminal['status'], trigger=terminal['trigger'])
if phase == 'score' and terminal['status'] == 'COMPLETE':
accepted_terminal(c, seed, phase)
result = read(root / 'worker-records/result.json')
rows['mean_percent'] = result['mean_percent']
scores.append(result['cases'])
else:
rows[phase] = dict(status='UNSETTLED' if (root / 'started.json').exists() else 'NOT_STARTED')
seeds[str(seed)] = rows
report = dict(identity='H2-P', seeds=seeds, paired_status='PENDING_ALL_ACCEPTED_SEEDS')
if len(scores) != 3 or not (ROOT / 'complete.json').exists() or read(ROOT / 'complete.json')['status'] != 'COMPLETE':
return report
gino = reference.contract()
validate_review(reference.ROOT)
refs = []
for seed in SEEDS:
path = reference.run_root(seed, 'score')
terminal = path / 'complete.json'
if not terminal.exists():
return report
if read(terminal)['status'] != 'COMPLETE':
return dict(report, paired_status='GINO_SEED_SCORE_UNAVAILABLE')
reference.accepted_terminal(gino, seed, 'score')
refs.append(read(path / 'worker-records/result.json')['cases'])
stats = paired_bootstrap(read(c['comparison_contract']), scores, refs)
return dict(report, paired_status=stats['decision'], comparison=stats)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('run', 'collect'))
args = parser.parse_args()
c = contract()
if args.action == 'run':
raise SystemExit(run(c))
print(json.dumps(collect(c), indent=2, allow_nan=False))
+189
View File
@@ -0,0 +1,189 @@
"""H2-P worker using the original ShapeNet case-level training and scoring protocol."""
import argparse
import ctypes
from hashlib import sha256
import io
import json
import os
from pathlib import Path
import signal
import sys
from tempfile import TemporaryDirectory
import time
from types import FunctionType, SimpleNamespace
from zipfile import ZipFile
import numpy as np
import torch
import track_b_gino_formal_worker as gino
import track_b_h2_pressure_entry as entry
from track_b_h2_pressure_prepare import PressureH2, surface
from track_b_gino_paper_keyword_patch import extract_patched_paper_code
from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha
from track_b_gino_shapenet_worker import cuda_peaks, test_environment, train_seed
def mesh_graph(c, folder, archive, row, paper):
case = row['case']
member = archive.read(f'car-pressure-data/data/mesh_{case:03d}.ply')
if sha256(member).hexdigest() != row['mesh_sha256']:
raise ValueError(f'Original mesh hash differs for case {case}')
path = folder / f'mesh_{case:03d}.ply'
path.write_bytes(member)
try:
mesh = paper.load_mesh(path)
xyz = mesh.vertex.positions.numpy()
faces = mesh.triangle.indices.numpy()
finally:
path.unlink()
if len(xyz) != row['mesh_vertices'] or len(xyz) != 3586:
raise ValueError(f'Original vertex support differs for case {case}')
lower, upper = c['split']['raw_global_bounds']
margin = c['protocol']['bounds_expansion']
lo, hi = [v - margin for v in lower], [v + margin for v in upper]
graph = list(surface(xyz, faces, lo, hi))
graph[0] = paper.location_normalization(
torch.as_tensor(xyz, dtype=torch.float32),
torch.tensor(lo, dtype=torch.float32), torch.tensor(hi, dtype=torch.float32),
)
return tuple(graph), len(faces)
class SurfaceData:
def __init__(self, c, folder):
from neuralop.utils import UnitGaussianNormalizer
from src.data.cfd_datamodule import CFDSDFDataModule
self.c = c
self.folder = folder
self.paper = object.__new__(CFDSDFDataModule)
self.normalizer_class = UnitGaussianNormalizer
self.normalizer = None
self.train_ids = c['split']['train']
self.test_ids = c['split']['test']
self.rows = {r['case']: r for r in read(c['mapping_audit']['path'])['rows']}
def data(self, ids, role, target_log=None):
if role == 'train':
allowed = ids == self.train_ids and self.normalizer is None and target_log is None
else:
allowed = role == 'heldout' and ids == self.test_ids and self.normalizer is not None and target_log is not None
if not allowed:
raise PermissionError('Unexpected ShapeNet input or target scope')
graphs, pressures = [], []
folder = self.folder / role
folder.mkdir(exist_ok=False)
with ZipFile(self.c['paper_data']['path']) as archive:
for case in ids:
row = self.rows[case]
graph, _ = mesh_graph(self.c, folder, archive, row, self.paper)
graphs.append(graph)
if role == 'heldout':
target_log.write(json.dumps(dict(case=case, role=role, sha256=row['pressure_sha256'])) + '\n')
target_log.flush()
blob = archive.read(f'car-pressure-data/data/press_{case:03d}.npy')
if sha256(blob).hexdigest() != row['pressure_sha256']:
raise ValueError(f'Original pressure hash differs for case {case}')
raw = np.load(io.BytesIO(blob), allow_pickle=False).reshape(-1).astype(np.float32)
retained = np.concatenate((raw[:16], raw[112:]))
if (raw.size != 3682 or retained.size != 3586 or not np.isfinite(retained).all()
or sha256(retained.tobytes()).hexdigest() != row['retained_pressure_f32_sha256']):
raise ValueError(f'Original pressure mapping differs for case {case}')
pressures.append(torch.from_numpy(retained))
target = torch.stack(pressures)
if role == 'train':
self.normalizer = self.normalizer_class(
target, eps=self.c['protocol']['pressure_epsilon'], reduce_dim=[0, 1], verbose=False)
target = self.normalizer.encode(target)
return [dict(graph=graph, pressure=values) for graph, values in zip(graphs, target)]
class PressureModel(PressureH2):
def data_dict_to_input(self, item):
device = self.encoder[0].weight.device
return tuple(value[0].to(device) for value in item['graph']), None
def forward(self, graph, _unused=None):
return super().forward(*graph)
def loss_dict(self, item, loss_fn):
graph, _ = self.data_dict_to_input(item)
prediction = self(graph).reshape(1, -1)
truth = item['pressure'].to(prediction.device).reshape(1, -1)
return dict(loss=loss_fn(prediction, truth))
def paper_model(_c):
return PressureModel()
def attach(c, root, seed, phase):
ticket = read(root / 'dispatch.json')
owner = os.getppid()
if (ticket['coordinator'] != process_identity(owner)
or ticket['contract_sha256'] != sha(entry.ROOT / 'contract.json')
or (ticket['seed'], ticket['phase']) != (seed, phase)
or time.monotonic() >= ticket['start_by_monotonic']
or str(Path(sys.executable).absolute()) != c['environment']['python']):
raise PermissionError('Worker is not the exact admitted child')
if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) or os.getppid() != owner:
raise RuntimeError('Could not bind worker lifetime to the supervisor')
authorization = entry.authorize(c)
if (entry.DEVICES.get(ticket['ordinal']) != ticket['physical_uuid']
or ticket['ordinal'] not in authorization['eligible_ordinals']):
raise PermissionError('Unapproved physical V100')
return ticket
def fit(c, seed, data, root, records, ticket):
def progress(path, row):
save(path, row)
if seed == 0 and row['epoch'] == 1:
result = entry.first_epoch_projection(ticket, row, time.monotonic())
save(root / 'first-epoch-projection.json', result)
if not result['within_cap']:
raise RuntimeError('STOP_PROJECTED_CAP')
trainer = FunctionType(train_seed.__code__, dict(train_seed.__globals__, paper_model=paper_model,
save=progress), train_seed.__name__)
fit_from_gino = FunctionType(gino.train.__code__, dict(gino.train.__globals__, train_seed=trainer),
gino.train.__name__)
return fit_from_gino(c, seed, data, root, records, torch)
def score(c, seed, data, root):
scorer = FunctionType(gino.score.__code__,
dict(gino.score.__globals__, formal=entry, paper_model=paper_model), gino.score.__name__)
return scorer(c, seed, data, root, torch)
def run(seed, phase):
c = entry.contract()
root = entry.run_root(seed, phase)
ticket = attach(c, root, seed, phase)
test_environment(c)
torch.set_num_threads(c['limits']['torch_threads'])
torch.set_num_interop_threads(1)
binder = FunctionType(gino.bind_device.__code__, dict(gino.bind_device.__globals__,
formal=SimpleNamespace(DEVICES={str(seed): ticket['physical_uuid']})), gino.bind_device.__name__)
binder(root, c, ticket, torch)
torch.cuda.reset_peak_memory_stats()
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
records = root / 'worker-records'
records.mkdir(exist_ok=False)
with TemporaryDirectory(prefix='h2-pressure-', dir=root) as work:
extract_patched_paper_code(c, work)
data = SurfaceData(c, Path(work))
result = fit(c, seed, data, root, records, ticket) if phase == 'fit' else score(c, seed, data, root)
save(records / 'result.json', dict(result, phase=phase, seed=seed, status='COMPLETE', **cuda_peaks(torch)))
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('seed', type=int, choices=(0, 1, 2))
parser.add_argument('phase', choices=('fit', 'score'))
args = parser.parse_args()
run(args.seed, args.phase)