From 5dcd3e0dbe1c68b68cf06b44bcf575c4572147f7 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 18 Oct 2026 09:32:21 -0700 Subject: [PATCH] test: add bounded N64 space-time GPU trajectory harness --- .../multigrid/run_spacetime_gpu_n64.py | 235 ++++++++++++++++++ tests/validation/multigrid/spacetime_gpu.py | 37 ++- .../multigrid/test_spacetime_gpu.py | 91 +++++++ 3 files changed, 350 insertions(+), 13 deletions(-) create mode 100644 tests/validation/multigrid/run_spacetime_gpu_n64.py diff --git a/tests/validation/multigrid/run_spacetime_gpu_n64.py b/tests/validation/multigrid/run_spacetime_gpu_n64.py new file mode 100644 index 0000000..7eb1cda --- /dev/null +++ b/tests/validation/multigrid/run_spacetime_gpu_n64.py @@ -0,0 +1,235 @@ +"""Opt-in, frozen N64 space-time GPU trajectory; no convergence certification.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import resource +import subprocess +import sys +import time + +import numpy as np + +ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(ROOT/'src')) +sys.path.insert(0, str(Path(__file__).parents[1]/'taylor_green')) +from CelerisLab.config import LBMConfig +import oracle as tg +import spacetime_gpu as gpu +ref = gpu.ref + +CPU_SHA = '180eef90a1c3f43113fe18cc54981ee4723cfc0cc2724334907ff188022fa770' +UNIFORM_SHA = '01e7415ae39456f2de7f0c3d331afb971c49d19353b43038650e32e80737f1e7' +CUDA_SHA = 'c47869688f7ac2ec93411c0af233887fe8b77405838299e4ae52c8963fd97fe4' +TABLE_SHA = 'f2e62038dc620c5a10bb61d51a88be2afd80488f2f9f4a10c2677cfbe11009cf' +STEPS = (256, 512, 768, 1024) +TIMES = (.12, .24, .36, .48) +FACTORS = dict(velocity_l2=.8, pressure_l2=1., energy_relative=1.) +LIMITS = dict(velocity_l2=.05, pressure_l2=.15, energy_relative=.05, mass_relative=1e-5) +HEALTH = ('boundary_pre_nonfinite', 'boundary_pre_nonpositive', 'boundary_stored_nonfinite', + 'boundary_stored_nonpositive', 'physical_population_nonfinite', 'physical_population_nonpositive') + + +def sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def load_references(cpu_path, uniform_path): + for path, expected in ((cpu_path, CPU_SHA), (uniform_path, UNIFORM_SHA)): + if sha(path) != expected: + raise ValueError(f'pinned reference receipt differs: {path}') + cpu, uniform = [json.loads(Path(p).read_text()) for p in (cpu_path, uniform_path)] + assert cpu['completed'] and cpu['experimental_pass'] and not cpu['source']['dirty'] + assert cpu['source']['commit'] == '07879bc52bfef8d95503146c55d3823c8b6f8d0a' + assert [r['mode'] for r in cpu['runs']] == ['constant_ivp', 'spacetime'] + assert all(tuple(row['tstar'] for row in r['rows']) == TIMES for r in cpu['runs']) + assert uniform['completed'] and not uniform['source']['dirty'] + assert uniform['source']['revision'] == '9805383f9c50d867e8555471baf5dbda8333a652' + assert (uniform['n'], uniform['depth'], uniform['viscosity']) == (64, 1, .0192) + assert uniform['limits'] == tg.LIMITS == LIMITS + return cpu, uniform + + +def total(state, maps): + arrays = [a.astype(np.float64) for a in (state.coarse_f, state.fine_f)] + return ref.cpu.MOMENTS.T @ ref.directional(ref.cpu.State(*arrays), maps) + + +def warm_and_reset(run, initial, check_cap): + """Time 32 synchronized intervals, then restore exact uploaded arrival F.""" + run.initialize(initial) + before = run.download() + started = time.monotonic() + for _ in range(32): + check_cap() + run.step(audit=False) + seconds = time.monotonic()-started + warm_health = run.health_counts.tolist() + run.initialize(initial) + after = run.download() + for a, b in zip((before.coarse_f, before.fine_f), (after.coarse_f, after.fine_f)): + np.testing.assert_array_equal(a, b) + if run.step_count != 0 or after.tick != 0 or run.health_counts.any(): + raise AssertionError('warm-up reset did not restore tick/counters') + return after, dict(intervals=32, seconds=seconds, estimated_1024_seconds=32*seconds, + health_counts=warm_health, reset_verified=True) + + +def assess(rows, cpu, uniform, health_counts): + if tuple(row['tstar'] for row in rows) != TIMES: + raise ValueError('four frozen sample times required') + worst = {key: max(row[key] for row in rows) for key in LIMITS} + campaign = {key: all(np.isfinite(row[key]) and 0 <= row[key] <= limit for row in rows) + for key, limit in LIMITS.items()} + campaign['population_health'] = not any(health_counts) + campaign['sampled_positive_finite'] = all( + np.isfinite(row[key]) and row[key] > 0 for row in rows + for key in ('minimum_population', 'minimum_boundary_pre', 'minimum_boundary_stored')) + baseline, candidate = cpu['runs'] + thresholds = {key: factor*baseline['worst'][key] for key, factor in FACTORS.items()} + experimental = {key: all(np.isfinite(row[key]) and row[key] >= 0 for row in rows) + and worst[key] <= threshold for key, threshold in thresholds.items()} + differences = [] + for actual, expected in zip(rows, candidate['rows']): + delta = {} + for key, value in expected.items(): + d = np.asarray(actual[key])-np.asarray(value) + delta[key] = dict(cpu=value, signed=d.tolist(), absolute=np.abs(d).tolist()) + differences.append(dict(tstar=actual['tstar'], fields=delta)) + historical_worst = max(row['pressure_l2'] for row in uniform['rows'] if row['tstar'] > 0) + historical_limit = max(2*historical_worst, .02) + assert historical_limit == .024506347612699403 + return dict(worst=worst, campaign_checks=campaign, campaign_pass=all(campaign.values()), + experimental_thresholds=thresholds, experimental_checks=experimental, + experimental_pass=all(experimental.values()), + gpu_minus_cpu=dict(status='descriptive_only_no_equivalence_tolerance', rows=differences), + historical_pressure_gate=dict(status='historical_shifted_FP32_uniform_vs_unshifted_FP32_candidate', + matched_storage=False, uniform_worst=historical_worst, threshold=historical_limit, + candidate_worst=worst['pressure_l2'], passed=worst['pressure_l2'] <= historical_limit)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--gpu-admitted', action='store_true') + parser.add_argument('--cpu-receipt', type=Path, required=True) + parser.add_argument('--uniform-receipt', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + if not args.gpu_admitted or not os.environ.get('CUDA_VISIBLE_DEVICES'): + parser.error('explicit --gpu-admitted and CUDA_VISIBLE_DEVICES required') + if args.output.exists(): + parser.error('output exists; preserve evidence and choose a new path') + cpu, uniform = load_references(args.cpu_receipt, args.uniform_receipt) + gpu.check_sources() + if sha(Path(gpu.__file__).with_suffix('.cu')) != CUDA_SHA: + raise RuntimeError('reviewed CUDA source changed') + source = dict(commit=subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(), + dirty=bool(subprocess.check_output(['git', 'status', '--porcelain'], cwd=ROOT, text=True)), + hashes={p.name: sha(p) for p in (Path(__file__), Path(gpu.__file__), Path(gpu.__file__).with_suffix('.cu'), + Path(__file__).with_name('test_spacetime_gpu.py'))}, + reference_hashes=gpu.PINS, numpy=np.__version__, checkout=str(ROOT)) + if source['dirty']: + raise RuntimeError('clean committed checkout required') + started = time.monotonic() + report = dict(scope='N64 experimental space-time GPU; no grid-convergence qualification', + completed=False, passed=False, source=source, rows=[], health_counts=[0]*6, health_labels=HEALTH, + setup=dict(n=64, bounds=[8,8,56,48], re_L=100, U=.03, viscosity=.0192, tau=[.5576,.6152], + phase='arrival_f', storage='unshifted_FP32', collision='SRT', interval_count=1024, + initialization='same physical FP64 equilibrium as CPU, rounded to FP32 on upload', + cpu_storage='FP64', sample_steps=STEPS, sample_times=TIMES), + references=dict(cpu_receipt_sha256=CPU_SHA, cpu_source=cpu['source'], + cpu_archive_drive_id='197F9nwDOGf_YROK1a46YpVvNrSldnH_w', + cpu_archive_sha256='850ddce3f5203ef38e76e5153e718a9b92e137d1fc388ba7cd12f5d47eab83b4', + uniform_receipt_sha256=UNIFORM_SHA, uniform_source=uniform['source'], + uniform_storage='historical_shifted_FP32'), + frozen_screen=dict(campaign_limits=LIMITS, experimental_factors=FACTORS, + gpu_cpu_equivalence_tolerance=None, historical_pressure_threshold=.024506347612699403), + boundary_budget_status='not_measured', max_directional_residual=None, + minima_scope='initial and four sampled physical arrival states; boundary minima at four samples', + limits=dict(seconds=180., peak_rss_mib=512.), cuda_visible_devices=os.environ['CUDA_VISIBLE_DEVICES']) + run = context = None + phase = 'setup' + + def save(): + report['resources'] = dict(seconds=time.monotonic()-started, + peak_rss_mib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024, + device_allocations_bytes=None if run is None else run.device_bytes, + device_memory_scope='owner allocations; excludes compiler, driver and context') + if run is not None: + report['health_counts'] = run.health_counts.tolist() + report['completed_intervals'] = run.step_count + ref.write_receipt(args.output, report) + + def check_cap(): + if time.monotonic()-started > 180 or resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024 > 512: + raise RuntimeError('frozen resource cap exceeded') + + save() + try: + from CelerisLab.cuda.context import CudaContext + context = CudaContext(0) + config = LBMConfig(nx=64, ny=64, viscosity=.0192, velocity=0., ddf_shifting=False) + run = gpu.SpaceTimeGPU(config, (8,8,56,48), context=context) + report.update(device=context.name, sm_arch=context.sm_arch, build_metadata=run.build_metadata, + table_sha256=gpu.table_hash(run.tables)) + if report['table_sha256'] != TABLE_SHA: + raise RuntimeError('frozen coordinate tables changed') + initial = ref.cpu.State(tg.initial((64,64),0,64,.0192), tg.initial((128,128),1,64,.0192)) + phase = 'warm_reset' + initial, report['warmup'] = warm_and_reset(run, initial, check_cap) + maps = run.plan.maps + masks = (maps.coarse_active, maps.fine_active) + arrays = (initial.coarse_f, initial.fine_f) + mass0 = tg.metrics(arrays, masks, 0, 64, .0192)['mass'] + moment0 = total(initial, maps) + report['initial'] = dict(mass=mass0, momentum=moment0[1:].tolist(), + minimum_population=min(float(a[:,m].min()) for a,m in zip(arrays,masks))) + save() + check_cap() + phase = 'trajectory' + trajectory_started = time.monotonic() + for step in range(1,1025): + check_cap() + run.step(audit=False) + if step in STEPS: + state = run.download() + if state.tick != 2*step: + raise AssertionError('sample is not synchronized arrival state') + arrays = (state.coarse_f, state.fine_f) + row = tg.metrics(arrays, masks, step, 64, .0192, mass0) + row.update(step=step, momentum_change=(total(state,maps)-moment0)[1:].tolist(), + mass_change=row['mass']-mass0, + minimum_population=min(float(a[:,m].min()) for a,m in zip(arrays,masks))) + with context.activate(): + row['minimum_boundary_pre'] = float(run.read(run.pre,(len(run.plan.groups),4),np.float64).min()) + row['minimum_boundary_stored'] = float(run.read(run.h,(len(run.plan.groups),4)).min()) + if not all(np.isfinite(value).all() for value in row.values()): + report['failed_sample'] = {key: repr(value) for key,value in row.items()} + raise FloatingPointError('nonfinite sample diagnostic') + report['rows'].append(row) + save() + print(json.dumps(row, allow_nan=False), flush=True) + report['trajectory_seconds'] = time.monotonic()-trajectory_started + check_cap() + report.update(assess(report['rows'], cpu, uniform, run.health_counts)) + report['completed'] = True + report['passed'] = report['campaign_pass'] and report['experimental_pass'] + except Exception as exc: + report['failure'] = dict(phase=phase, error=repr(exc)) + raise + finally: + # Preserve the primary failure and every completed sample even if cleanup fails. + for owner in (run, context): + if owner is not None: + try: + owner.close() + except Exception as exc: + report.setdefault('cleanup_errors', []).append(repr(exc)) + report['passed'] = False + save() + return 0 if report['passed'] else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tests/validation/multigrid/spacetime_gpu.py b/tests/validation/multigrid/spacetime_gpu.py index ece4025..8fbcd1a 100644 --- a/tests/validation/multigrid/spacetime_gpu.py +++ b/tests/validation/multigrid/spacetime_gpu.py @@ -132,6 +132,7 @@ class SpaceTimeGPU: self.ctx, self.mode, self.identity = context, mode, identity self.allocations, self.modules, self.build_metadata = [], [], [] self.status, self.step_count, self.device_bytes = 'new', 0, 0 + self.health_counts = np.zeros(6, np.uint64) import pycuda.driver as cuda from CelerisLab.cuda import compiler_v2 as compiler root = Path(__file__).resolve().parents[3] @@ -208,6 +209,7 @@ class SpaceTimeGPU: self.status = 'failed' raise self.status, self.step_count = 'ready', 0 + self.health_counts.fill(0) def download(self): if self.status != 'ready': @@ -223,11 +225,12 @@ class SpaceTimeGPU: a[:,~mask] = np.nan return ref.cpu.State(*arrays, tick=2*self.step_count) - def step(self, *, mutation=0): - """One audited interval; mutation 1/2/3 drops packets, 4 omits a ghost.""" - if self.status != 'ready' or type(mutation) is not int or mutation not in range(5): + def step(self, *, mutation=0, audit=True): + """One interval; audit=False skips only host ledger snapshots, never health checks.""" + if (self.status != 'ready' or type(mutation) is not int or mutation not in range(5) + or type(audit) is not bool or (mutation and not audit)): raise RuntimeError("ready owner and known test mutation required") - before = self.download() + before = self.download() if audit else None n, nf, t = self.plan.n, self.plan.nf, self.device_tables ng, nr = len(self.plan.groups), len(self.plan.receivers) u32, null = np.uint32, np.uintp(0) @@ -243,13 +246,16 @@ class SpaceTimeGPU: self.launch(0, 'Collide', predicted,gp,null,omega[0],self.counts) self.launch(0, 'Interpolate', g0,gp,t['samples'],t['origins'],t['sizes'],t['weights'], self.pre,self.h,u32(ng),u32(self.mode=='frozen'),self.counts,count=ng) - coarse_g = self.read(g0,(9,n,n)); prediction = self.read(predicted,(9,n,n)) + if audit: + coarse_g = self.read(g0,(9,n,n)); prediction = self.read(predicted,(9,n,n)) fine_phases = [] for m in (0,1): f, out, g = self.fine - old_f = self.read(f,(9,nf,nf)) + if audit: + old_f = self.read(f,(9,nf,nf)) self.launch(1, 'Collide', f,g,self.active,omega[1],self.counts) - fine_phases.append((old_f,self.read(g,(9,nf,nf)))) + if audit: + fine_phases.append((old_f,self.read(g,(9,nf,nf)))) self.launch(0, 'Register', predicted,g,self.h,t['targets'],t['sources'],t['times'], self.registers,u32(nr),np.int32(m),np.int32(mutation),count=nr) rows = self.tables[f'incoming{m}'] @@ -258,14 +264,19 @@ class SpaceTimeGPU: self.fine[0], self.fine[1] = out, f self.launch(0, 'ApplyRegisters', predicted,self.registers,t['targets'],u32(nr),self.counts,count=nr) counts = self.read(self.counts,(6,),np.uint32) + self.health_counts += counts.astype(np.uint64) if counts.any(): raise AssertionError(f"boundary/leaf health failed: {counts.tolist()}") - coarse_after = self.read(predicted,(9,n,n)) - coarse_after[:,~self.plan.maps.coarse_active] = np.nan - after = ref.cpu.State(coarse_after,self.read(self.fine[0],(9,nf,nf)),tick=2*(self.step_count+1)) - ledger = rounding_ledger(self.plan,self.tables,before,after,coarse_g,prediction,fine_phases, - self.read(self.pre,(ng,4),np.float64),self.read(self.h,(ng,4)), - self.read(self.registers,(nr,),np.float64)) + ledger = dict(boundary_budget_status='not_measured',unexplained=None) + if audit: + coarse_after = self.read(predicted,(9,n,n)) + coarse_after[:,~self.plan.maps.coarse_active] = np.nan + after = ref.cpu.State(coarse_after,self.read(self.fine[0],(9,nf,nf)),tick=2*(self.step_count+1)) + ledger = rounding_ledger(self.plan,self.tables,before,after,coarse_g,prediction,fine_phases, + self.read(self.pre,(ng,4),np.float64),self.read(self.h,(ng,4)), + self.read(self.registers,(nr,),np.float64)) + ledger['boundary_budget_status'] = 'measured' + ledger['health_counts'] = counts.tolist() self.coarse[0], self.coarse[2] = predicted, fc self.step_count += 1 self.status = 'ready' diff --git a/tests/validation/multigrid/test_spacetime_gpu.py b/tests/validation/multigrid/test_spacetime_gpu.py index 3e26557..6bdff69 100644 --- a/tests/validation/multigrid/test_spacetime_gpu.py +++ b/tests/validation/multigrid/test_spacetime_gpu.py @@ -8,6 +8,7 @@ import os from pathlib import Path import subprocess import sys +import tempfile import time from types import SimpleNamespace import unittest @@ -19,6 +20,7 @@ ROOT = Path(__file__).resolve().parents[3] sys.path.insert(0, str(ROOT/'src')) from CelerisLab.config import LBMConfig import spacetime_gpu as gpu +import run_spacetime_gpu_n64 as trajectory ref = gpu.ref RECORDS = [] REPORT = None @@ -121,6 +123,76 @@ class Host(unittest.TestCase): self.assertEqual(run.status,'closed'); self.assertEqual(run.allocations,[]) second.free.assert_called_once() + def test_unmeasured_schedule_and_health_failure(self): + run=gpu.SpaceTimeGPU.__new__(gpu.SpaceTimeGPU) + run.status='ready'; run.step_count=0; run.identity=False; run.mode='spacetime' + run.ctx=SimpleNamespace(activate=nullcontext); run.cuda=Mock(); run.stream=Mock() + run.plan=self.plan; run.bounds=(3,3,9,8) + run.configs=gpu.validate_config(config(12)); run.coarse=list('abcd'); run.fine=list('efg') + run.tables=gpu.pack_plan(self.plan); run.device_tables={key:key for key in run.tables} + run.active='active'; run.pre='pre'; run.h='h'; run.registers='registers'; run.counts='counts' + run.health_counts=np.zeros(6,np.uint64) + run.launch=Mock(); run.read=Mock(return_value=np.zeros(6,np.uint32)); run.download=Mock() + names=['CoarseStart','Pull','Collide','Interpolate','Collide','Register','Inject','Pull', + 'Collide','Register','Inject','Pull','ApplyRegisters'] + for _ in range(2): + result=run.step(audit=False) + self.assertEqual(result['boundary_budget_status'],'not_measured') + self.assertIsNone(result['unexplained']) + self.assertEqual([call.args[1] for call in run.launch.call_args_list],names*2) + self.assertEqual(run.read.call_count,2); run.download.assert_not_called() + self.assertEqual(run.step_count,2); self.assertEqual(run.coarse,list('abcd')) + run.read.return_value=np.array([0,0,0,0,1,0],np.uint32) + with self.assertRaisesRegex(AssertionError,'health failed'): + run.step(audit=False) + self.assertEqual(run.status,'failed'); self.assertEqual(run.step_count,2) + self.assertEqual(run.health_counts.tolist(),[0,0,0,0,1,0]) + + def test_warmup_resets_arrivals_and_counters(self): + class Fake: + def initialize(self,state): + self.state=ref.cpu.State(state.coarse_f.copy(),state.fine_f.copy()) + self.step_count=0; self.health_counts=np.zeros(6,np.uint64) + def download(self): + return ref.cpu.State(self.state.coarse_f.copy(),self.state.fine_f.copy(),tick=2*self.step_count) + def step(self,*,audit): + assert audit is False + self.step_count+=1; self.state.coarse_f[:]+=1e-4 + fake=Fake(); check=Mock(); state=initial(self.plan) + actual,warm=trajectory.warm_and_reset(fake,state,check) + np.testing.assert_array_equal(actual.coarse_f,state.coarse_f) + self.assertEqual(check.call_count,32); self.assertEqual(fake.step_count,0) + self.assertTrue(warm['reset_verified']); self.assertEqual(warm['intervals'],32) + + def test_trajectory_screens_differences_and_json(self): + rows=[dict(tstar=t,velocity_l2=np.float64(.01),pressure_l2=.03,energy_relative=.01, + mass_relative=1e-7,momentum_change=[1e-6,-2e-6],minimum_population=.02, + minimum_boundary_pre=.02,minimum_boundary_stored=.02) for t in trajectory.TIMES] + cpu=dict(runs=[dict(worst=dict(velocity_l2=.02,pressure_l2=.04,energy_relative=.02)), + dict(rows=[dict(tstar=t,velocity_l2=.009,momentum_change=[0.,0.]) for t in trajectory.TIMES])]) + uniform=dict(rows=[dict(tstar=.12,pressure_l2=.012253173806349701)]) + result=trajectory.assess(rows,cpu,uniform,[0]*6) + self.assertTrue(result['campaign_pass']); self.assertTrue(result['experimental_pass']) + self.assertFalse(result['historical_pressure_gate']['passed']) + self.assertFalse(result['historical_pressure_gate']['matched_storage']) + self.assertAlmostEqual(result['gpu_minus_cpu']['rows'][0]['fields']['velocity_l2']['signed'],.001) + with tempfile.TemporaryDirectory() as directory: + path=Path(directory)/'receipt.json' + ref.write_receipt(path,dict(completed=True,**result)) + self.assertTrue(json.loads(path.read_text())['completed']) + rows[0]['velocity_l2']=.06 + failed=trajectory.assess(rows,cpu,uniform,[0,1,0,0,0,0]) + ref.write_receipt(path,dict(completed=True,**failed)) + saved=json.loads(path.read_text()) + self.assertFalse(saved['campaign_pass']); self.assertFalse(saved['experimental_pass']) + for invalid in (float('nan'),-.01): + rows[0]['velocity_l2']=.01; rows[-1]['velocity_l2']=invalid + failed=trajectory.assess(rows,cpu,uniform,[0]*6) + self.assertFalse(failed['campaign_pass']); self.assertFalse(failed['experimental_pass']) + path.write_text('changed reference') + with self.assertRaisesRegex(ValueError,'pinned reference'): + trajectory.load_references(path,path) + class Device(unittest.TestCase): @classmethod @@ -214,6 +286,25 @@ class Device(unittest.TestCase): device_bytes=run.device_bytes,table_sha256=gpu.table_hash(run.tables), build_metadata=run.build_metadata,device=self.context.name)) + def test_unmeasured_path_and_warm_reset(self): + with gpu.SpaceTimeGPU(config(12),(3,3,9,8),context=self.context) as run: + state=initial(run.plan) + run.initialize(state) + for _ in range(2): + run.step() + expected=run.download() + run.initialize(state) + for _ in range(2): + result=run.step(audit=False) + self.assertEqual(result['boundary_budget_status'],'not_measured') + actual=run.download() + for a,b in zip((actual.coarse_f,actual.fine_f),(expected.coarse_f,expected.fine_f)): + np.testing.assert_array_equal(a,b) + reset,warm=trajectory.warm_and_reset(run,state,lambda:None) + for a,b in zip((reset.coarse_f,reset.fine_f),(state.coarse_f,state.fine_f)): + np.testing.assert_array_equal(a,b.astype(np.float32)) + record(dict(case='unmeasured_bitwise_and_reset',warmup=warm,health_counts=run.health_counts.tolist())) + def main(): global REPORT,OUTPUT,GPU_ADMITTED