"""Opt-in, frozen N64 space-time GPU trajectory; no convergence certification.""" import argparse import hashlib import json import os from pathlib import Path import resource import subprocess import sys import time import numpy as np ROOT = Path(__file__).resolve().parents[3] sys.path.insert(0, str(ROOT/'src')) sys.path.insert(0, str(Path(__file__).parents[1]/'taylor_green')) from CelerisLab.config import LBMConfig import oracle as tg import spacetime_gpu as gpu import spacetime_table_guard as table_guard ref = gpu.ref CPU_SHA = '180eef90a1c3f43113fe18cc54981ee4723cfc0cc2724334907ff188022fa770' UNIFORM_SHA = '01e7415ae39456f2de7f0c3d331afb971c49d19353b43038650e32e80737f1e7' CUDA_SHA = 'afea11ea4a1c2c3800b2f6a7a8e81f5568989fc9e1cc02fb52d7e1ea3864e003' AUDIT_SHA = 'fba340b1533bec308b666ee84666df0ac9d08e2872be31ee379e2d1447565158' STEPS = (256, 512, 768, 1024) TIMES = (.12, .24, .36, .48) FACTORS = dict(velocity_l2=.8, pressure_l2=1., energy_relative=1.) LIMITS = dict(velocity_l2=.05, pressure_l2=.15, energy_relative=.05, mass_relative=1e-5) HEALTH = ('boundary_pre_nonfinite', 'boundary_pre_nonpositive', 'boundary_stored_nonfinite', 'boundary_stored_nonpositive', 'physical_population_nonfinite', 'physical_population_nonpositive') def sha(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() def load_references(cpu_path, uniform_path): for path, expected in ((cpu_path, CPU_SHA), (uniform_path, UNIFORM_SHA)): if sha(path) != expected: raise ValueError(f'pinned reference receipt differs: {path}') cpu, uniform = [json.loads(Path(p).read_text()) for p in (cpu_path, uniform_path)] assert cpu['completed'] and cpu['experimental_pass'] and not cpu['source']['dirty'] assert cpu['source']['commit'] == '07879bc52bfef8d95503146c55d3823c8b6f8d0a' assert [r['mode'] for r in cpu['runs']] == ['constant_ivp', 'spacetime'] assert all(tuple(row['tstar'] for row in r['rows']) == TIMES for r in cpu['runs']) assert uniform['completed'] and not uniform['source']['dirty'] assert uniform['source']['revision'] == '9805383f9c50d867e8555471baf5dbda8333a652' assert (uniform['n'], uniform['depth'], uniform['viscosity']) == (64, 1, .0192) assert uniform['limits'] == tg.LIMITS == LIMITS return cpu, uniform def total(state, maps): arrays = [a.astype(np.float64) for a in (state.coarse_f, state.fine_f)] return ref.cpu.MOMENTS.T @ ref.directional(ref.cpu.State(*arrays), maps) def differing_fields(actual, expected): """Compare serialized sample numerics exactly, including the sign of zero.""" if actual.keys() != expected.keys(): return ['field_set'] return [key for key in actual if np.asarray(actual[key]).dtype != np.asarray(expected[key]).dtype or np.asarray(actual[key]).shape != np.asarray(expected[key]).shape or np.asarray(actual[key]).tobytes() != np.asarray(expected[key]).tobytes()] def sample_difference(actual, expected): """Descriptive comparison only; never an equivalence gate for different codecs.""" return {key:dict(reference=value,signed=(np.asarray(actual[key])-value).tolist(), absolute=np.abs(np.asarray(actual[key])-value).tolist()) for key,value in expected.items()} class LedgerTotals: """Compensated directional sums; retain absolute step sums to expose cancellation.""" names = ('net_change','physical_collision','coarse_collision','fine0_collision','fine1_collision', 'child_rounding','child_interpolation','child_cast','register_rounding','receiver_rounding','unexplained') def __init__(self): self.sums = np.zeros((len(self.names),9)); self.correction = self.sums.copy() self.absolute = self.sums.copy(); self.absolute_moments = np.zeros((len(self.names),3)) self.maxima = self.sums.copy(); self.max_steps = np.zeros(self.sums.shape,np.int64) self.cells = np.zeros((3,4)); self.cell_correction = self.cells.copy() self.steps = 0; self.bound = 0.; self.maximum = 0. def add(self, ledger): values = np.array([ledger[name] for name in self.names],np.float64) cells = np.asarray(ledger['collision_cell_mass'],np.float64) if (values.shape != self.sums.shape or not np.isfinite(values).all() or cells.shape != (3,4) or not np.isfinite(cells).all() or not np.isfinite(ledger['unexplained_bound']) or ledger['unexplained_bound'] <= 0): raise AssertionError('invalid diagnostic ledger') delta = values-self.correction updated = self.sums+delta self.correction = (updated-self.sums)-delta self.sums = updated self.absolute += abs(values) self.absolute_moments += abs(values @ ref.cpu.MOMENTS) larger = abs(values) > self.maxima self.max_steps[larger] = self.steps+1; self.maxima = np.maximum(self.maxima,abs(values)) delta = cells-self.cell_correction updated = self.cells+delta self.cell_correction = (updated-self.cells)-delta; self.cells = updated self.steps += 1; self.bound += ledger['unexplained_bound'] self.maximum = max(self.maximum,float(abs(values[-1]).max())) def snapshot(self, endpoint_change=None): values = dict(zip(self.names,self.sums)) accounted = sum(values[key] for key in ('physical_collision','child_rounding','register_rounding','receiver_rounding')) closure = None if endpoint_change is None else endpoint_change-accounted telescoping = None if endpoint_change is None else endpoint_change-values['net_change'] closure_pass = None if closure is None else max(float(abs(closure).max()),float(abs(telescoping).max())) <= self.bound collision_split = values['physical_collision']-sum(values[key] for key in ('coarse_collision','fine0_collision','fine1_collision')) child_split = values['child_rounding']-values['child_interpolation']-values['child_cast'] split_pass = max(float(abs(collision_split).max()),float(abs(child_split).max())) <= self.bound return dict(intervals=self.steps, summed_per_step_bound=self.bound, max_step_unexplained=self.maximum, closure_pass=closure_pass,component_split_pass=split_pass, endpoint_status='not_measured' if endpoint_change is None else 'measured', endpoint_directional_change=None if endpoint_change is None else endpoint_change.tolist(), closure_residual=None if closure is None else closure.tolist(), telescoping_residual=None if telescoping is None else telescoping.tolist(), closure_mass_momentum=None if closure is None else (ref.cpu.MOMENTS.T@closure).tolist(), collision_split_residual=collision_split.tolist(),child_split_residual=child_split.tolist(), collision_cell_mass=dict(levels=['coarse','fine0','fine1'],columns=['net','positive','negative','l1'], applied_volume_weights=[1.,.25,.25],values=self.cells.tolist()), components={key:dict(directional=self.sums[i].tolist(),mass_momentum=(self.sums[i]@ref.cpu.MOMENTS).tolist(), absolute_step_directional_sum=self.absolute[i].tolist(), absolute_step_mass_momentum_sum=self.absolute_moments[i].tolist(), max_absolute_interval=self.maxima[i].tolist(),max_absolute_interval_step=self.max_steps[i].tolist()) for i,key in enumerate(self.names)}) def warm_and_reset(run, initial, check_cap, *, audit=False): """Time 32 synchronized intervals, then restore exact uploaded arrival F.""" run.initialize(initial) before = run.download() started = time.monotonic() for _ in range(32): check_cap() run.step(audit=audit) seconds = time.monotonic()-started warm_health = run.health_counts.tolist() run.initialize(initial) after = run.download() for a, b in zip((before.coarse_f, before.fine_f), (after.coarse_f, after.fine_f)): np.testing.assert_array_equal(a, b) if run.step_count != 0 or after.tick != 0 or run.health_counts.any(): raise AssertionError('warm-up reset did not restore tick/counters') return after, dict(intervals=32, seconds=seconds, estimated_1024_seconds=32*seconds, health_counts=warm_health, reset_verified=True, ledger_audit=audit) def assess(rows, cpu, uniform, health_counts): if tuple(row['tstar'] for row in rows) != TIMES: raise ValueError('four frozen sample times required') worst = {key: max(row[key] for row in rows) for key in LIMITS} campaign = {key: all(np.isfinite(row[key]) and 0 <= row[key] <= limit for row in rows) for key, limit in LIMITS.items()} campaign['population_health'] = not any(health_counts) campaign['sampled_positive_finite'] = all( np.isfinite(row[key]) and row[key] > 0 for row in rows for key in ('minimum_population', 'minimum_boundary_pre', 'minimum_boundary_stored')) baseline, candidate = cpu['runs'] thresholds = {key: factor*baseline['worst'][key] for key, factor in FACTORS.items()} experimental = {key: all(np.isfinite(row[key]) and row[key] >= 0 for row in rows) and worst[key] <= threshold for key, threshold in thresholds.items()} differences = [] for actual, expected in zip(rows, candidate['rows']): delta = {} for key, value in expected.items(): d = np.asarray(actual[key])-np.asarray(value) delta[key] = dict(cpu=value, signed=d.tolist(), absolute=np.abs(d).tolist()) differences.append(dict(tstar=actual['tstar'], fields=delta)) historical_worst = max(row['pressure_l2'] for row in uniform['rows'] if row['tstar'] > 0) historical_limit = max(2*historical_worst, .02) assert historical_limit == .024506347612699403 return dict(worst=worst, campaign_checks=campaign, campaign_pass=all(campaign.values()), experimental_thresholds=thresholds, experimental_checks=experimental, experimental_pass=all(experimental.values()), gpu_minus_cpu=dict(status='descriptive_only_no_equivalence_tolerance', rows=differences), historical_pressure_gate=dict(status='historical_shifted_FP32_uniform_vs_unshifted_FP32_candidate', matched_storage=False, uniform_worst=historical_worst, threshold=historical_limit, candidate_worst=worst['pressure_l2'], passed=worst['pressure_l2'] <= historical_limit)) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--gpu-admitted', action='store_true') parser.add_argument('--cpu-receipt', type=Path, required=True) parser.add_argument('--uniform-receipt', type=Path, required=True) parser.add_argument('--audit-against', type=Path, help='audit every interval and compare exact saved 1779cf2 samples') parser.add_argument('--shifted', action='store_true', help='opt-in shifted storage with all interval ledgers') parser.add_argument('--unshifted-receipt', type=Path, help='pinned1779 comparison, descriptive only') parser.add_argument('--output', type=Path, required=True) args = parser.parse_args() if not args.gpu_admitted or not os.environ.get('CUDA_VISIBLE_DEVICES'): parser.error('explicit --gpu-admitted and CUDA_VISIBLE_DEVICES required') if args.output.exists(): parser.error('output exists; preserve evidence and choose a new path') if args.shifted != (args.unshifted_receipt is not None) or (args.shifted and args.audit_against): parser.error('--shifted requires --unshifted-receipt and cannot use exact --audit-against') exact_repeat = args.audit_against is not None cpu, uniform = load_references(args.cpu_receipt, args.uniform_receipt) previous = None comparison_path = args.audit_against or args.unshifted_receipt if comparison_path is not None: if sha(comparison_path) != AUDIT_SHA: raise ValueError('pinned GPU trajectory receipt differs') previous = json.loads(comparison_path.read_text()) if previous['source']['numpy'] != np.__version__: raise RuntimeError('audited repeat requires unchanged NumPy environment') gpu.check_sources() if sha(Path(gpu.__file__).with_suffix('.cu')) != CUDA_SHA: raise RuntimeError('reviewed CUDA source changed') source = dict(commit=subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(), dirty=bool(subprocess.check_output(['git', 'status', '--porcelain'], cwd=ROOT, text=True)), hashes={p.name: sha(p) for p in (Path(__file__), Path(gpu.__file__), Path(gpu.__file__).with_suffix('.cu'), Path(__file__).with_name('test_spacetime_gpu.py'), Path(table_guard.__file__))}, reference_hashes=gpu.PINS, numpy=np.__version__, checkout=str(ROOT)) if source['dirty']: raise RuntimeError('clean committed checkout required') started = time.monotonic() report = dict(scope='N64 experimental space-time GPU; no grid-convergence qualification', completed=False, passed=False, source=source, rows=[], health_counts=[0]*6, health_labels=HEALTH, setup=dict(n=64, bounds=[8,8,56,48], re_L=100, U=.03, viscosity=.0192, tau=[.5576,.6152], phase='arrival_f', storage='shifted_FP32' if args.shifted else 'unshifted_FP32', collision='SRT', interval_count=1024, initialization='common analytic FP64 field, encoded using production FP64 weights before FP32 upload' if args.shifted else 'same physical FP64 equilibrium as CPU, rounded to FP32 on upload', cpu_storage='FP64', sample_steps=STEPS, sample_times=TIMES), references=dict(cpu_receipt_sha256=CPU_SHA, cpu_source=cpu['source'], cpu_archive_drive_id='197F9nwDOGf_YROK1a46YpVvNrSldnH_w', cpu_archive_sha256='850ddce3f5203ef38e76e5153e718a9b92e137d1fc388ba7cd12f5d47eab83b4', uniform_receipt_sha256=UNIFORM_SHA, uniform_source=uniform['source'], uniform_storage='historical_shifted_FP32'), frozen_screen=dict(campaign_limits=LIMITS, experimental_factors=FACTORS, gpu_cpu_equivalence_tolerance=None, historical_pressure_threshold=.024506347612699403), boundary_budget_status='pending_measurement' if previous else 'not_measured', max_directional_residual=None, minima_scope='initial and four sampled physical arrival states; boundary minima at four samples', limits=dict(seconds=180., peak_rss_mib=512.), cuda_visible_devices=os.environ['CUDA_VISIBLE_DEVICES']) run = context = None ledger_totals = LedgerTotals() if previous else None if previous: report['diagnostic'] = dict(completed=False, passed=False, reference_sha256=AUDIT_SHA, reference_source=previous['source']['commit'], sample_checks=[], prefixes=[], scope='net drift attribution; exact saved diagnostics only, not full-field trajectory certification' if exact_repeat else 'raw stored-value ledger; unshifted and CPU comparisons descriptive, initial codecs differ') phase = 'setup' def save(): report['resources'] = dict(seconds=time.monotonic()-started, peak_rss_mib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024, device_allocations_bytes=None if run is None else run.device_bytes, device_memory_scope='owner allocations; excludes compiler, driver and context') if run is not None: report['health_counts'] = run.health_counts.tolist() report['completed_intervals'] = run.step_count if ledger_totals is not None: report['audited_intervals'] = ledger_totals.steps if ledger_totals.steps: report['boundary_budget_status'] = 'measured' if ledger_totals.steps==1024 else 'measured_prefix' report['diagnostic']['accumulated'] = ledger_totals.snapshot() ref.write_receipt(args.output, report) def check_cap(): if time.monotonic()-started > 180 or resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024 > 512: raise RuntimeError('frozen resource cap exceeded') save() try: from CelerisLab.cuda.context import CudaContext context = CudaContext(0) config = LBMConfig(nx=64, ny=64, viscosity=.0192, velocity=0., ddf_shifting=args.shifted) run = gpu.SpaceTimeGPU(config, (8,8,56,48), context=context) report.update(device=context.name, sm_arch=context.sm_arch, build_metadata=run.build_metadata, table_sha256=gpu.table_hash(run.tables)) report['table_validation'] = table_guard.validate_tables(run.plan,run.tables) if previous: for key in ('device','sm_arch','table_sha256') + (('build_metadata',) if exact_repeat else ()): if report[key] != previous[key]: raise AssertionError(f'audited repeat identity differs: {key}') analytic = ref.cpu.State(tg.initial((64,64),0,64,.0192), tg.initial((128,128),1,64,.0192)) initial = analytic phase = 'warm_reset' initial, report['warmup'] = warm_and_reset(run, initial, check_cap, audit=previous is not None) maps = run.plan.maps masks = (maps.coarse_active, maps.fine_active) arrays = (initial.coarse_f, initial.fine_f) mass0 = tg.metrics(arrays, masks, 0, 64, .0192)['mass'] moment0 = total(initial, maps) report['initial'] = dict(mass=mass0, momentum=moment0[1:].tolist(), minimum_population=min(float(a[:,m].min()) for a,m in zip(arrays,masks))) if exact_repeat and differing_fields(report['initial'],previous['initial']): raise AssertionError('audited repeat initial state diagnostics differ') raw_initial = run._download_stored() if args.shifted else initial direction0 = ref.directional(ref.cpu.State(*(a.astype(np.float64) for a in (raw_initial.coarse_f,raw_initial.fine_f))),maps) if args.shifted: errors=[]; unshifted_errors=[] for raw,actual,wanted,mask in zip((raw_initial.coarse_f,raw_initial.fine_f),arrays, (analytic.coarse_f,analytic.fine_f),masks): encoded=gpu._encode_physical(wanted,True) np.testing.assert_array_equal(raw[:,mask].view(np.uint32),encoded[:,mask].view(np.uint32)) errors.append(float(abs(actual[:,mask]-wanted[:,mask]).max())) unshifted_errors.append(float(abs(gpu._encode_physical(wanted,False)[:,mask]-wanted[:,mask]).max())) report['warmup']['raw_reset_verified']=True report['initial_quantization']=dict(maximum_population_error_by_level=errors, unshifted_codec_error_by_level=unshifted_errors, moment_change_vs_analytic=(moment0-total(analytic,maps)).tolist(), cpu_start='pinned analytic FP64 trajectory, not reinitialized to decoded GPU upload', matched_cpu_initial_bits=False) report['unshifted_comparison']=dict(status='descriptive_only_codecs_have_different_initial_bits', reference_sha256=AUDIT_SHA,initial=sample_difference(report['initial'],previous['initial']),rows=[]) save() check_cap() phase = 'trajectory' trajectory_started = time.monotonic() for step in range(1,1025): check_cap() ledger = run.step(audit=previous is not None) if ledger_totals is not None: ledger_totals.add(ledger) report['max_directional_residual'] = ledger_totals.maximum if step in STEPS: state = run.download() if state.tick != 2*step: raise AssertionError('sample is not synchronized arrival state') arrays = (state.coarse_f, state.fine_f) row = tg.metrics(arrays, masks, step, 64, .0192, mass0) row.update(step=step, momentum_change=(total(state,maps)-moment0)[1:].tolist(), mass_change=row['mass']-mass0, minimum_population=min(float(a[:,m].min()) for a,m in zip(arrays,masks))) if ledger_totals is not None: row['minimum_boundary_pre'] = ledger['minimum_pre'] row['minimum_boundary_stored'] = ledger['minimum_stored'] else: with context.activate(): row['minimum_boundary_pre'] = float(run.read(run.pre,(len(run.plan.groups),4),np.float64).min()) row['minimum_boundary_stored'] = float(run.read(run.h,(len(run.plan.groups),4)).min()) if not all(np.isfinite(value).all() for value in row.values()): report['failed_sample'] = {key: repr(value) for key,value in row.items()} raise FloatingPointError('nonfinite sample diagnostic') report['rows'].append(row) if previous: expected=previous['rows'][len(report['rows'])-1] changed = differing_fields(row,expected) if exact_repeat else [] report['diagnostic']['sample_checks'].append(dict(step=step, status='exact_repeat' if exact_repeat else 'descriptive_only',differing_fields=changed if exact_repeat else None)) if args.shifted: report['unshifted_comparison']['rows'].append(dict(step=step,fields=sample_difference(row,expected))) raw_state=run._download_stored() if args.shifted else state endpoint = ref.directional(ref.cpu.State(*(a.astype(np.float64) for a in (raw_state.coarse_f,raw_state.fine_f))),maps)-direction0 prefix = ledger_totals.snapshot(endpoint) if args.shifted: prefix['physical_minus_raw_endpoint_moments'] = ( total(state,maps)-moment0-ref.cpu.MOMENTS.T@endpoint).tolist() report['diagnostic']['prefixes'].append(prefix) if not prefix['closure_pass'] or not prefix['component_split_pass']: raise AssertionError('cumulative ledger does not close within summed existing per-step bounds') if changed: raise AssertionError(f'audited repeat sample differs: {changed}') save() print(json.dumps(row, allow_nan=False), flush=True) report['trajectory_seconds'] = time.monotonic()-trajectory_started check_cap() report.update(assess(report['rows'], cpu, uniform, run.health_counts)) if args.shifted: report['historical_pressure_gate'].update(status='historical_shifted_FP32_uniform_vs_shifted_FP32_candidate', matched_storage=True,matched_control=False) report['completed'] = True report['passed'] = report['campaign_pass'] and report['experimental_pass'] if previous: report['diagnostic'].update(completed=True,passed=ledger_totals.steps==1024 and len(report['rows'])==4) except Exception as exc: report['failure'] = dict(phase=phase, error=repr(exc)) raise finally: # Preserve the primary failure and every completed sample even if cleanup fails. for owner in (run, context): if owner is not None: try: owner.close() except Exception as exc: report.setdefault('cleanup_errors', []).append(repr(exc)) report['passed'] = False save() return 0 if report['passed'] else 1 if __name__ == '__main__': raise SystemExit(main())