Bind H2-P ShapeNet training, first-epoch gate and paired scoring for review
This commit is contained in:
1 parent
db87b0f54b
commit
70a85dffb8
10 files changed
+7577
No files matched your search
@@ -0,0 +1,54 @@
|
||||
"""Freeze H2-P comparison metadata and audit only original triangle mesh members."""
|
||||
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from zipfile import ZipFile
|
||||
|
||||
import torch
|
||||
|
||||
import track_b_h2_pressure_entry as entry
|
||||
from track_b_gino_paper_keyword_patch import extract_patched_paper_code
|
||||
from track_b_gino_shapenet_entry import read, save
|
||||
from track_b_h2_pressure_worker import SurfaceData, mesh_graph
|
||||
|
||||
|
||||
def prepare():
|
||||
c = entry.reference.contract()
|
||||
entry.ROOT.mkdir(exist_ok=False)
|
||||
torch.set_num_threads(1)
|
||||
with TemporaryDirectory(prefix='h2-p-mesh-only-') as folder:
|
||||
extract_patched_paper_code(c, folder)
|
||||
path = Path(folder)
|
||||
data = SurfaceData(c, path)
|
||||
rows = []
|
||||
|
||||
class MeshOnly:
|
||||
def __init__(self, archive):
|
||||
self.archive = archive
|
||||
|
||||
def read(self, member):
|
||||
if not member.startswith('car-pressure-data/data/mesh_') or not member.endswith('.ply'):
|
||||
raise PermissionError('Preparation may only open original meshes')
|
||||
return self.archive.read(member)
|
||||
|
||||
with ZipFile(c['paper_data']['path']) as archive:
|
||||
for case in data.train_ids + data.test_ids:
|
||||
graph, faces = mesh_graph(c, path, MeshOnly(archive), data.rows[case], data.paper)
|
||||
x, mass, receivers, _, _, components = graph
|
||||
rows.append(dict(case=case, vertices=len(x), faces=faces,
|
||||
edge_count=len(receivers), components=int(components.max()) + 1,
|
||||
total_positive_area=float(mass.double().sum())))
|
||||
save(entry.ROOT/'geometry-audit.json', dict(status='PASS', mesh_members_opened=len(rows),
|
||||
target_members_opened=0, actual_gpu_queries=0, rows=rows))
|
||||
save(entry.ROOT/'comparison-contract.json', entry.expected_comparison(c))
|
||||
save(entry.ROOT/'contract.json', entry.expected_contract())
|
||||
assert entry.contract()['identity'] == 'H2-P'
|
||||
print(f'{len(rows)} original shape meshes audited, zero pressure members')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import os
|
||||
|
||||
if os.environ.get('CUDA_VISIBLE_DEVICES') != '':
|
||||
raise PermissionError('Preparation is CPU only')
|
||||
prepare()
|
||||
@@ -0,0 +1,346 @@
|
||||
"""One reviewed, cost-gated H2-P pipeline with frozen three-seed custody."""
|
||||
|
||||
import argparse
|
||||
from collections import defaultdict
|
||||
import fcntl
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import time
|
||||
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from s2_gpu_idle_gate import snapshot
|
||||
import track_b_gino_formal as reference
|
||||
from track_b_gino_formal_guard import guard
|
||||
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
|
||||
|
||||
|
||||
ROOT = BASE / 'artifacts/track-b-h2-pressure-combined-2026-11-19'
|
||||
REFERENCE_SHA = 'c3c5c5f0fab049c757032c343673104eb0f99cbaead5e0f78247f2a3aa6173c3'
|
||||
DEVICES = {'0': reference.DEVICES['2'], '1': 'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
|
||||
'2': reference.DEVICES['0']}
|
||||
SEEDS = (0, 1, 2)
|
||||
CAP = 40000
|
||||
SCORE_RESERVE = 300
|
||||
BOOTSTRAP_REPS = 2000
|
||||
BOOTSTRAP_SEED = 20261119
|
||||
SOURCES = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def geometry_components(c):
|
||||
mapping = read(c['mapping_audit']['path'])['rows']
|
||||
by_case = {row['case']: row['mesh_sha256'] for row in mapping}
|
||||
if set(by_case) != set(c['split']['train'] + c['split']['test']):
|
||||
raise ValueError('Mesh identity roster differs')
|
||||
if {by_case[i] for i in c['split']['train']} & {by_case[i] for i in c['split']['test']}:
|
||||
raise ValueError('Identical raw mesh would cross the fixed learning and held-out split')
|
||||
groups = defaultdict(list)
|
||||
for case in c['split']['test']:
|
||||
groups[by_case[case]].append(case)
|
||||
return list(groups.values())
|
||||
|
||||
|
||||
def expected_comparison(c):
|
||||
return dict(handoff='h-b5985d03', identity='H2-P', seeds=list(SEEDS),
|
||||
train_cases=c['split']['train'], heldout_cases=c['split']['test'],
|
||||
pressure_selection='original raw rows [0:16] and [112:] in original vertex order',
|
||||
metric=c['protocol']['metric'], summary='equal cases, then equal matched seeds',
|
||||
component_definition='exact source triangle-mesh SHA-256, geometry-only; no known family identifiers',
|
||||
heldout_components=geometry_components(c),
|
||||
bootstrap=dict(replicates=BOOTSTRAP_REPS, seed=BOOTSTRAP_SEED, confidence=.95,
|
||||
method='paired component resampling with replacement; case weighted; all three paired seeds retained'),
|
||||
paired_delta='H2-P minus GINO in percentage points',
|
||||
gate='all three exact seed pairs valid, GINO reproduction mean in [6.62,7.62], paired CI high < 0',
|
||||
no_test_selection=True)
|
||||
|
||||
|
||||
def expected_contract():
|
||||
base = reference.contract()
|
||||
base_path = checked(reference.ROOT / 'contract.json', REFERENCE_SHA)
|
||||
ref = dict(path=str(base_path), sha256=REFERENCE_SHA,
|
||||
comparison_sha256=base['comparison_contract_sha256'])
|
||||
geometry = ROOT / 'geometry-audit.json'
|
||||
audit = read(geometry)
|
||||
if (audit['status'] != 'PASS' or audit['mesh_members_opened'] != 611 or audit['target_members_opened'] != 0
|
||||
or [r['case'] for r in audit['rows']] != base['split']['train'] + base['split']['test']):
|
||||
raise ValueError('Complete geometry-only coverage is required')
|
||||
comparison = ROOT / 'comparison-contract.json'
|
||||
if read(comparison) != expected_comparison(base):
|
||||
raise ValueError('Pre-registered comparison differs')
|
||||
c = {key: base[key] for key in (
|
||||
'paper_data', 'paper_supplement', 'paper_code_zip_sha256', 'paper_compatibility',
|
||||
'split', 'split_receipt', 'mapping_audit', 'protocol', 'environment', 'environment_ruling',
|
||||
'environment_acceptance', 'limits', 'code_sha256')}
|
||||
c['code_sha256'].update({str(SOURCES / name): sha(SOURCES / name) for name in (
|
||||
'track_b_h2_pressure_entry.py', 'track_b_h2_pressure_worker.py',
|
||||
'track_b_h2_pressure_prepare.py', 'phase2_h2_model.py')})
|
||||
c.update(mode='h2-pressure-combined', identity='H2-P', handoff='h-b5985d03', reference=ref,
|
||||
geometry_audit=dict(path=str(geometry), sha256=sha(geometry)),
|
||||
comparison_contract=str(comparison), comparison_contract_sha256=sha(comparison),
|
||||
devices=DEVICES, first_epoch_seed=0, total_gpu_seconds=CAP, scoring_seconds_per_seed=SCORE_RESERVE,
|
||||
timing_margin=1.25, future_fit_setup_floor_seconds=120, gpu3_eligible=False,
|
||||
model=dict(inputs=['x','y','z'], edges='original triangle edges both directions',
|
||||
mass='one-third incident area', message_components='surface edge connectivity',
|
||||
architecture='shared four 64-wide H2 blocks and direct pressure head',
|
||||
optimizer_and_case_loss='same Adam, StepLR and LpLoss as fixed GINO training body'),
|
||||
custody=dict(train_reads_per_seed=500, heldout_reads_per_seed=111,
|
||||
require_all_frozen_checkpoints=True, retries=0))
|
||||
return c
|
||||
|
||||
|
||||
def contract():
|
||||
c = read(ROOT / 'contract.json')
|
||||
if c != expected_contract():
|
||||
raise PermissionError('H2-P contract or source binding changed')
|
||||
for path, digest in c['code_sha256'].items():
|
||||
checked(path, digest)
|
||||
return c
|
||||
|
||||
|
||||
def authorize(c):
|
||||
review = validate_review(ROOT)
|
||||
auth = read(ROOT / 'stage-authorization.json')
|
||||
approved = auth.get('eligible_ordinals')
|
||||
expires = auth.get('dispatch_before_unix')
|
||||
if ((auth['decision'], auth['mode'], auth['contract_sha256'], auth['review_sha256']) !=
|
||||
('APPROVED', c['mode'], sha(ROOT / 'contract.json'), sha(ROOT / 'review.json'))
|
||||
or review['verdict'] != 'PASS'
|
||||
or not isinstance(approved, list) or not approved or len(set(approved)) != len(approved)
|
||||
or not set(approved) <= set(DEVICES)
|
||||
or type(expires) not in (float, int) or not math.isfinite(expires)):
|
||||
raise PermissionError('Exact independent PASS and stage/card authorization required')
|
||||
return auth
|
||||
|
||||
|
||||
def first_epoch_projection(ticket, progress, now):
|
||||
if tuple(progress[k] for k in ('stage', 'seed', 'epoch', 'examples')) != ('EPOCH_COMPLETE', 0, 1, 500):
|
||||
raise ValueError('Only seed 0 first complete 500-case epoch measures cost')
|
||||
epoch = progress['seconds']
|
||||
spent = now - ticket['guard_started_monotonic']
|
||||
setup = spent - epoch
|
||||
if not math.isfinite(epoch) or not math.isfinite(setup) or epoch <= 0 or setup < 0:
|
||||
raise ValueError('First full epoch and setup clock differ')
|
||||
future_setup = max(120, math.ceil(1.25 * setup))
|
||||
remaining = math.ceil(299 * epoch * 1.25) + 2 * future_setup + 3 * SCORE_RESERVE + 60
|
||||
projected = spent + remaining
|
||||
return dict(seed=0, epoch=1, examples=500, epoch_seconds=epoch, booked_seconds_so_far=spent,
|
||||
measured_setup_seconds=setup, future_fit_setup_reserve_seconds=2 * future_setup,
|
||||
remaining_train_epochs=299, safety_factor=1.25, scoring_reserve_seconds=3 * SCORE_RESERVE,
|
||||
fit_cleanup_reserve_seconds=60, projected_total_gpu_seconds=projected,
|
||||
formal_cap_gpu_seconds=CAP, within_cap=projected <= CAP)
|
||||
|
||||
|
||||
def run_root(seed, phase):
|
||||
return ROOT / f'seed-{seed}' / phase
|
||||
|
||||
|
||||
score_mean = reference.score_mean
|
||||
|
||||
|
||||
def validate_result(c, seed, phase):
|
||||
root = run_root(seed, phase)
|
||||
path = root / 'worker-records/result.json'
|
||||
result = read(path)
|
||||
ticket = read(root / 'started.json')
|
||||
ordinal, uuid = ticket['ordinal'], ticket['physical_uuid']
|
||||
spawn = read(root / 'actual-worker-spawn.json')
|
||||
if (DEVICES[ordinal] != uuid or spawn['physical_uuid'] != uuid or spawn['ordinal'] != ordinal
|
||||
or read(root / 'worker-device-verified.json') !=
|
||||
dict(process=spawn['process'], physical_uuid=uuid, ordinal=ordinal, observed_uuids=[uuid])):
|
||||
raise ValueError('Actual H2-P worker/device witness differs')
|
||||
if (result['status'], result['seed'], result['phase']) != ('COMPLETE', seed, phase):
|
||||
raise ValueError('Worker result identity differs')
|
||||
if phase == 'fit':
|
||||
reference.epoch100(root, seed, c)
|
||||
checkpoint = result['checkpoint']
|
||||
if (checkpoint['path'] != str(root / 'epoch-100.pt')
|
||||
or (result['epochs'], result['training_examples'], result['target_reads']) != (100, 500, 0)
|
||||
or (root / 'heldout-target-access.jsonl').exists()):
|
||||
raise ValueError('Fit checkpoint or held-out seal differs')
|
||||
checked(checkpoint['path'], checkpoint['sha256'])
|
||||
if seed == 0 and read(root / 'first-epoch-projection.json')['within_cap'] is not True:
|
||||
raise ValueError('Missing first epoch cost PASS')
|
||||
else:
|
||||
manifest = root / 'checkpoints-frozen.json'
|
||||
roster = all_checkpoints(c)
|
||||
access = root / 'heldout-target-access.jsonl'
|
||||
mapping = {row['case']: row['pressure_sha256'] for row in read(c['mapping_audit']['path'])['rows']}
|
||||
rows = [json.loads(line) for line in access.read_text().splitlines()]
|
||||
expected = [dict(case=i, role='heldout', sha256=mapping[i]) for i in c['split']['test']]
|
||||
if (read(manifest) != roster or result['checkpoints_manifest_sha256'] != sha(manifest)
|
||||
or result['checkpoint_sha256'] != roster[str(seed)]['checkpoint']['sha256']
|
||||
or result['target_reads'] != 111 or result['target_access_sha256'] != sha(access)
|
||||
or rows != expected or score_mean(result['cases'], c['split']['test']) != result['mean_percent']):
|
||||
raise ValueError('Held-out custody or score differs')
|
||||
return sha(path)
|
||||
|
||||
|
||||
def accepted_terminal(c, seed, phase):
|
||||
root = run_root(seed, phase)
|
||||
terminal = read(root / 'complete.json')
|
||||
if (terminal['status'], terminal['seed'], terminal['phase'], terminal['contract_sha256']) != (
|
||||
'COMPLETE', seed, phase, sha(ROOT / 'contract.json')):
|
||||
raise PermissionError('No exact accepted stage terminal')
|
||||
if (terminal['booked_gpu_cap_exceeded'] or terminal['charged_gpu_seconds'] > terminal['booked_gpu_seconds']
|
||||
or terminal['result_sha256'] != validate_result(c, seed, phase)):
|
||||
raise ValueError('H2-P terminal budget or result differs')
|
||||
return terminal
|
||||
|
||||
|
||||
def all_checkpoints(c):
|
||||
roster = {}
|
||||
for seed in SEEDS:
|
||||
terminal = accepted_terminal(c, seed, 'fit')
|
||||
root = run_root(seed, 'fit')
|
||||
result = read(root / 'worker-records/result.json')
|
||||
roster[str(seed)] = dict(checkpoint=result['checkpoint'], terminal_sha256=sha(root / 'complete.json'),
|
||||
result_sha256=terminal['result_sha256'])
|
||||
return roster
|
||||
|
||||
|
||||
def run_stage(c, seed, phase, ordinal, auth, began, deadline, roster=None):
|
||||
root = run_root(seed, phase)
|
||||
root.mkdir(parents=True, exist_ok=False)
|
||||
if phase == 'score':
|
||||
save(root / 'checkpoints-frozen.json', roster)
|
||||
uuid = DEVICES[ordinal]
|
||||
ticket = dict(seed=seed, phase=phase, ordinal=ordinal, physical_uuid=uuid, unix=time.time(),
|
||||
contract_sha256=sha(ROOT / 'contract.json'), dispatch_before_unix=auth['dispatch_before_unix'],
|
||||
guard_started_monotonic=began, booked_deadline_monotonic=deadline,
|
||||
booked_gpu_seconds=deadline - began)
|
||||
save(root / 'started.json', ticket)
|
||||
command = [c['environment']['python'], '-B', str(SOURCES / 'track_b_h2_pressure_worker.py'), str(seed), phase]
|
||||
limits = dict(c['limits'], cpu_seconds=8 * (deadline - began))
|
||||
return guard(root, command, uuid=uuid, limits=limits, deadline=deadline, ticket=ticket,
|
||||
validate_result=lambda: validate_result(c, seed, phase))
|
||||
|
||||
|
||||
def run(c):
|
||||
auth = authorize(c)
|
||||
with (ROOT / 'combined.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if (ROOT / 'started.json').exists() or any(ROOT.glob('seed-*')):
|
||||
raise FileExistsError('H2-P combined attempt already exists')
|
||||
if time.time() >= auth['dispatch_before_unix']:
|
||||
raise PermissionError('Card reservation for new dispatch has expired')
|
||||
ordinal = next((i for i, uuid in DEVICES.items()
|
||||
if i in auth['eligible_ordinals'] and snapshot(uuid)['idle']), None)
|
||||
if ordinal is None:
|
||||
raise RuntimeError('NO_IDLE_APPROVED_GPU0_TO_2')
|
||||
began = time.monotonic()
|
||||
deadline = began + CAP
|
||||
save(ROOT / 'started.json', dict(ordinal=ordinal, physical_uuid=DEVICES[ordinal],
|
||||
started_monotonic=began, contract_sha256=sha(ROOT / 'contract.json'), phase_budget=CAP))
|
||||
|
||||
def request_stop(number, _frame):
|
||||
raise KeyboardInterrupt(signal.Signals(number).name)
|
||||
|
||||
old = signal.signal(signal.SIGTERM, request_stop)
|
||||
status, trigger = 'STOPPED', None
|
||||
try:
|
||||
fit_deadline = deadline - len(SEEDS) * SCORE_RESERVE
|
||||
for seed in SEEDS:
|
||||
epoch_start = began if seed == 0 else time.monotonic()
|
||||
if epoch_start >= fit_deadline:
|
||||
raise RuntimeError('INSUFFICIENT_FIT_BUDGET')
|
||||
receipt = run_stage(c, seed, 'fit', ordinal, auth, epoch_start, fit_deadline)
|
||||
if receipt['status'] != 'COMPLETE':
|
||||
cost = run_root(0, 'fit') / 'first-epoch-projection.json'
|
||||
if seed == 0 and cost.exists() and read(cost)['within_cap'] is False:
|
||||
raise RuntimeError('STOP_PROJECTED_CAP')
|
||||
raise RuntimeError(f"Seed {seed} fit HOLD: {receipt['trigger']}")
|
||||
roster = all_checkpoints(c)
|
||||
save(ROOT / 'checkpoints-frozen.json', roster)
|
||||
for seed in SEEDS:
|
||||
score_start = time.monotonic()
|
||||
score_deadline = min(deadline, score_start + SCORE_RESERVE)
|
||||
if score_deadline - score_start <= c['limits']['cleanup_seconds']:
|
||||
raise RuntimeError('INSUFFICIENT_SCORE_BUDGET')
|
||||
receipt = run_stage(c, seed, 'score', ordinal, auth, score_start, score_deadline, roster)
|
||||
if receipt['status'] != 'COMPLETE':
|
||||
raise RuntimeError(f"Seed {seed} score HOLD: {receipt['trigger']}")
|
||||
status = 'COMPLETE'
|
||||
except (Exception, KeyboardInterrupt) as exc:
|
||||
trigger = str(exc) or type(exc).__name__
|
||||
finally:
|
||||
signal.signal(signal.SIGTERM, old)
|
||||
charged = time.monotonic() - began
|
||||
if charged > CAP and status == 'COMPLETE':
|
||||
status, trigger = 'STOPPED', 'GPU_BUDGET_OVERAGE'
|
||||
save(ROOT / 'complete.json', dict(status=status, trigger=trigger, charged_gpu_seconds=charged,
|
||||
cap_gpu_seconds=CAP, no_retry=True))
|
||||
return 0 if status == 'COMPLETE' else 1
|
||||
|
||||
|
||||
def paired_bootstrap(comparison, h2, gino):
|
||||
import numpy as np
|
||||
|
||||
ids = comparison['heldout_cases']
|
||||
groups = comparison['heldout_components']
|
||||
if sorted(case for group in groups for case in group) != sorted(ids) or min(map(len, groups)) < 1:
|
||||
raise ValueError('Fixed paired test component support differs')
|
||||
if len(h2) != 3 or len(gino) != 3:
|
||||
raise ValueError('All three exact matched seeds are required')
|
||||
h2_means = [score_mean(rows, ids) for rows in h2]
|
||||
gino_means = [score_mean(rows, ids) for rows in gino]
|
||||
deltas = np.array([[a['relative_l2'] - b['relative_l2'] for a, b in zip(hr, gr)]
|
||||
for hr, gr in zip(h2, gino)], dtype=np.float64).mean(axis=0)
|
||||
per_case = dict(zip(ids, deltas))
|
||||
masses = np.array([len(group) for group in groups])
|
||||
sums = np.array([sum(per_case[case] for case in group) for group in groups])
|
||||
drawn = np.random.default_rng(BOOTSTRAP_SEED).integers(len(groups), size=(BOOTSTRAP_REPS, len(groups)))
|
||||
sampled = 100 * sums[drawn].sum(axis=1) / masses[drawn].sum(axis=1)
|
||||
low, high = map(float, np.quantile(sampled, [.025, .975]))
|
||||
reference_mean, within = reference.pool(gino_means, [6.62, 7.62])
|
||||
return dict(h2_percent_by_seed=dict(zip(SEEDS, h2_means)), gino_percent_by_seed=dict(zip(SEEDS, gino_means)),
|
||||
gino_mean_percent=reference_mean, gino_reproduction_pass=within,
|
||||
paired_mean_difference_pp=float(100 * deltas.mean()), paired_ci95_pp=[low, high],
|
||||
components=len(groups), decision=('GINO_REPRODUCTION_GATE_NOT_MET' if not within else
|
||||
'H2_P_IMPROVEMENT_PASS' if high < 0 else 'H2_P_IMPROVEMENT_NOT_ESTABLISHED'))
|
||||
|
||||
|
||||
def collect(c):
|
||||
seeds, scores = {}, []
|
||||
for seed in SEEDS:
|
||||
rows = {}
|
||||
for phase in ('fit', 'score'):
|
||||
root = run_root(seed, phase)
|
||||
if (root / 'complete.json').exists():
|
||||
terminal = read(root / 'complete.json')
|
||||
rows[phase] = dict(status=terminal['status'], trigger=terminal['trigger'])
|
||||
if phase == 'score' and terminal['status'] == 'COMPLETE':
|
||||
accepted_terminal(c, seed, phase)
|
||||
result = read(root / 'worker-records/result.json')
|
||||
rows['mean_percent'] = result['mean_percent']
|
||||
scores.append(result['cases'])
|
||||
else:
|
||||
rows[phase] = dict(status='UNSETTLED' if (root / 'started.json').exists() else 'NOT_STARTED')
|
||||
seeds[str(seed)] = rows
|
||||
report = dict(identity='H2-P', seeds=seeds, paired_status='PENDING_ALL_ACCEPTED_SEEDS')
|
||||
if len(scores) != 3 or not (ROOT / 'complete.json').exists() or read(ROOT / 'complete.json')['status'] != 'COMPLETE':
|
||||
return report
|
||||
gino = reference.contract()
|
||||
validate_review(reference.ROOT)
|
||||
refs = []
|
||||
for seed in SEEDS:
|
||||
path = reference.run_root(seed, 'score')
|
||||
terminal = path / 'complete.json'
|
||||
if not terminal.exists():
|
||||
return report
|
||||
if read(terminal)['status'] != 'COMPLETE':
|
||||
return dict(report, paired_status='GINO_SEED_SCORE_UNAVAILABLE')
|
||||
reference.accepted_terminal(gino, seed, 'score')
|
||||
refs.append(read(path / 'worker-records/result.json')['cases'])
|
||||
stats = paired_bootstrap(read(c['comparison_contract']), scores, refs)
|
||||
return dict(report, paired_status=stats['decision'], comparison=stats)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('action', choices=('run', 'collect'))
|
||||
args = parser.parse_args()
|
||||
c = contract()
|
||||
if args.action == 'run':
|
||||
raise SystemExit(run(c))
|
||||
print(json.dumps(collect(c), indent=2, allow_nan=False))
|
||||
@@ -0,0 +1,189 @@
|
||||
"""H2-P worker using the original ShapeNet case-level training and scoring protocol."""
|
||||
|
||||
import argparse
|
||||
import ctypes
|
||||
from hashlib import sha256
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import sys
|
||||
from tempfile import TemporaryDirectory
|
||||
import time
|
||||
from types import FunctionType, SimpleNamespace
|
||||
from zipfile import ZipFile
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
import track_b_gino_formal_worker as gino
|
||||
import track_b_h2_pressure_entry as entry
|
||||
from track_b_h2_pressure_prepare import PressureH2, surface
|
||||
from track_b_gino_paper_keyword_patch import extract_patched_paper_code
|
||||
from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha
|
||||
from track_b_gino_shapenet_worker import cuda_peaks, test_environment, train_seed
|
||||
|
||||
|
||||
def mesh_graph(c, folder, archive, row, paper):
|
||||
case = row['case']
|
||||
member = archive.read(f'car-pressure-data/data/mesh_{case:03d}.ply')
|
||||
if sha256(member).hexdigest() != row['mesh_sha256']:
|
||||
raise ValueError(f'Original mesh hash differs for case {case}')
|
||||
path = folder / f'mesh_{case:03d}.ply'
|
||||
path.write_bytes(member)
|
||||
try:
|
||||
mesh = paper.load_mesh(path)
|
||||
xyz = mesh.vertex.positions.numpy()
|
||||
faces = mesh.triangle.indices.numpy()
|
||||
finally:
|
||||
path.unlink()
|
||||
if len(xyz) != row['mesh_vertices'] or len(xyz) != 3586:
|
||||
raise ValueError(f'Original vertex support differs for case {case}')
|
||||
lower, upper = c['split']['raw_global_bounds']
|
||||
margin = c['protocol']['bounds_expansion']
|
||||
lo, hi = [v - margin for v in lower], [v + margin for v in upper]
|
||||
graph = list(surface(xyz, faces, lo, hi))
|
||||
graph[0] = paper.location_normalization(
|
||||
torch.as_tensor(xyz, dtype=torch.float32),
|
||||
torch.tensor(lo, dtype=torch.float32), torch.tensor(hi, dtype=torch.float32),
|
||||
)
|
||||
return tuple(graph), len(faces)
|
||||
|
||||
|
||||
class SurfaceData:
|
||||
def __init__(self, c, folder):
|
||||
from neuralop.utils import UnitGaussianNormalizer
|
||||
from src.data.cfd_datamodule import CFDSDFDataModule
|
||||
|
||||
self.c = c
|
||||
self.folder = folder
|
||||
self.paper = object.__new__(CFDSDFDataModule)
|
||||
self.normalizer_class = UnitGaussianNormalizer
|
||||
self.normalizer = None
|
||||
self.train_ids = c['split']['train']
|
||||
self.test_ids = c['split']['test']
|
||||
self.rows = {r['case']: r for r in read(c['mapping_audit']['path'])['rows']}
|
||||
|
||||
def data(self, ids, role, target_log=None):
|
||||
if role == 'train':
|
||||
allowed = ids == self.train_ids and self.normalizer is None and target_log is None
|
||||
else:
|
||||
allowed = role == 'heldout' and ids == self.test_ids and self.normalizer is not None and target_log is not None
|
||||
if not allowed:
|
||||
raise PermissionError('Unexpected ShapeNet input or target scope')
|
||||
graphs, pressures = [], []
|
||||
folder = self.folder / role
|
||||
folder.mkdir(exist_ok=False)
|
||||
with ZipFile(self.c['paper_data']['path']) as archive:
|
||||
for case in ids:
|
||||
row = self.rows[case]
|
||||
graph, _ = mesh_graph(self.c, folder, archive, row, self.paper)
|
||||
graphs.append(graph)
|
||||
if role == 'heldout':
|
||||
target_log.write(json.dumps(dict(case=case, role=role, sha256=row['pressure_sha256'])) + '\n')
|
||||
target_log.flush()
|
||||
blob = archive.read(f'car-pressure-data/data/press_{case:03d}.npy')
|
||||
if sha256(blob).hexdigest() != row['pressure_sha256']:
|
||||
raise ValueError(f'Original pressure hash differs for case {case}')
|
||||
raw = np.load(io.BytesIO(blob), allow_pickle=False).reshape(-1).astype(np.float32)
|
||||
retained = np.concatenate((raw[:16], raw[112:]))
|
||||
if (raw.size != 3682 or retained.size != 3586 or not np.isfinite(retained).all()
|
||||
or sha256(retained.tobytes()).hexdigest() != row['retained_pressure_f32_sha256']):
|
||||
raise ValueError(f'Original pressure mapping differs for case {case}')
|
||||
pressures.append(torch.from_numpy(retained))
|
||||
target = torch.stack(pressures)
|
||||
if role == 'train':
|
||||
self.normalizer = self.normalizer_class(
|
||||
target, eps=self.c['protocol']['pressure_epsilon'], reduce_dim=[0, 1], verbose=False)
|
||||
target = self.normalizer.encode(target)
|
||||
return [dict(graph=graph, pressure=values) for graph, values in zip(graphs, target)]
|
||||
|
||||
|
||||
class PressureModel(PressureH2):
|
||||
def data_dict_to_input(self, item):
|
||||
device = self.encoder[0].weight.device
|
||||
return tuple(value[0].to(device) for value in item['graph']), None
|
||||
|
||||
def forward(self, graph, _unused=None):
|
||||
return super().forward(*graph)
|
||||
|
||||
def loss_dict(self, item, loss_fn):
|
||||
graph, _ = self.data_dict_to_input(item)
|
||||
prediction = self(graph).reshape(1, -1)
|
||||
truth = item['pressure'].to(prediction.device).reshape(1, -1)
|
||||
return dict(loss=loss_fn(prediction, truth))
|
||||
|
||||
|
||||
def paper_model(_c):
|
||||
return PressureModel()
|
||||
|
||||
|
||||
def attach(c, root, seed, phase):
|
||||
ticket = read(root / 'dispatch.json')
|
||||
owner = os.getppid()
|
||||
if (ticket['coordinator'] != process_identity(owner)
|
||||
or ticket['contract_sha256'] != sha(entry.ROOT / 'contract.json')
|
||||
or (ticket['seed'], ticket['phase']) != (seed, phase)
|
||||
or time.monotonic() >= ticket['start_by_monotonic']
|
||||
or str(Path(sys.executable).absolute()) != c['environment']['python']):
|
||||
raise PermissionError('Worker is not the exact admitted child')
|
||||
if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) or os.getppid() != owner:
|
||||
raise RuntimeError('Could not bind worker lifetime to the supervisor')
|
||||
authorization = entry.authorize(c)
|
||||
if (entry.DEVICES.get(ticket['ordinal']) != ticket['physical_uuid']
|
||||
or ticket['ordinal'] not in authorization['eligible_ordinals']):
|
||||
raise PermissionError('Unapproved physical V100')
|
||||
return ticket
|
||||
|
||||
|
||||
def fit(c, seed, data, root, records, ticket):
|
||||
def progress(path, row):
|
||||
save(path, row)
|
||||
if seed == 0 and row['epoch'] == 1:
|
||||
result = entry.first_epoch_projection(ticket, row, time.monotonic())
|
||||
save(root / 'first-epoch-projection.json', result)
|
||||
if not result['within_cap']:
|
||||
raise RuntimeError('STOP_PROJECTED_CAP')
|
||||
|
||||
trainer = FunctionType(train_seed.__code__, dict(train_seed.__globals__, paper_model=paper_model,
|
||||
save=progress), train_seed.__name__)
|
||||
fit_from_gino = FunctionType(gino.train.__code__, dict(gino.train.__globals__, train_seed=trainer),
|
||||
gino.train.__name__)
|
||||
return fit_from_gino(c, seed, data, root, records, torch)
|
||||
|
||||
|
||||
def score(c, seed, data, root):
|
||||
scorer = FunctionType(gino.score.__code__,
|
||||
dict(gino.score.__globals__, formal=entry, paper_model=paper_model), gino.score.__name__)
|
||||
return scorer(c, seed, data, root, torch)
|
||||
|
||||
|
||||
def run(seed, phase):
|
||||
c = entry.contract()
|
||||
root = entry.run_root(seed, phase)
|
||||
ticket = attach(c, root, seed, phase)
|
||||
test_environment(c)
|
||||
torch.set_num_threads(c['limits']['torch_threads'])
|
||||
torch.set_num_interop_threads(1)
|
||||
binder = FunctionType(gino.bind_device.__code__, dict(gino.bind_device.__globals__,
|
||||
formal=SimpleNamespace(DEVICES={str(seed): ticket['physical_uuid']})), gino.bind_device.__name__)
|
||||
binder(root, c, ticket, torch)
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
torch.backends.cuda.matmul.allow_tf32 = False
|
||||
torch.backends.cudnn.allow_tf32 = False
|
||||
records = root / 'worker-records'
|
||||
records.mkdir(exist_ok=False)
|
||||
with TemporaryDirectory(prefix='h2-pressure-', dir=root) as work:
|
||||
extract_patched_paper_code(c, work)
|
||||
data = SurfaceData(c, Path(work))
|
||||
result = fit(c, seed, data, root, records, ticket) if phase == 'fit' else score(c, seed, data, root)
|
||||
save(records / 'result.json', dict(result, phase=phase, seed=seed, status='COMPLETE', **cuda_peaks(torch)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('seed', type=int, choices=(0, 1, 2))
|
||||
parser.add_argument('phase', choices=('fit', 'score'))
|
||||
args = parser.parse_args()
|
||||
run(args.seed, args.phase)
|
||||
Reference in new issue
Block a user