264 lines
17 KiB
Python
264 lines
17 KiB
Python
"""Four opt-in shifted GPU controls: U64/U128/U256 and refined base128/fine256."""
|
|
import argparse
|
|
import hashlib
|
|
import inspect
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import subprocess
|
|
import time
|
|
|
|
import numpy as np
|
|
import run_spacetime_gpu_n64 as old
|
|
|
|
gpu,ref,tg,ROOT = old.gpu,old.ref,old.tg,old.ROOT
|
|
BASE='26d0d3853c9d4d3cd6ae3f27653dced56f8c5fec'
|
|
REFERENCE_SHA='9d154c989b087bf288f86d413db0f3067b73767b618c329d4b9e53c4eee00032'
|
|
CASES=(('uniform64',64,False),('uniform128',128,False),('uniform256',256,False),('refined128',128,True))
|
|
LIMITS=old.LIMITS
|
|
SECONDS,RSS_MIB=600.,512.
|
|
|
|
|
|
def setup(n, refined):
|
|
return dict(base_n=n,finest_n=2*n if refined else n,normalized_domain=[0.,1.,0.,1.],
|
|
re_L=100,U=.03,mach=.03*np.sqrt(3),viscosity=.0003*n,
|
|
tau=[.5+.0009*n]+([.5+.0018*n] if refined else []),
|
|
bounds=[n//8,n//8,7*n//8,3*n//4] if refined else None,
|
|
normalized_patch=[.125,.125,.875,.75] if refined else None,
|
|
intervals=16*n,sample_steps=[4*n,8*n,12*n,16*n],sample_times=list(old.TIMES),
|
|
early_steps=list(range(0,4*n+1,n//16)),early_times=(np.arange(65)*.001875).tolist(),
|
|
phase='arrival_f',storage='shifted_FP32',collision='SRT',
|
|
initialization='common analytic FP64 equilibrium at actual cell centers, production shifted codec')
|
|
|
|
|
|
def pressure_projection(arrays,masks,t,n,viscosity):
|
|
"""One global gauge and a weighted Gram fit, including unequal leaf areas."""
|
|
rows=[]
|
|
for level,(a,mask) in enumerate(zip(arrays,masks)):
|
|
rho=np.asarray(a[:,mask],np.float64).sum(axis=0)
|
|
x,y=tg.centers(mask.shape,level); x,y=x[mask],y[mask]
|
|
exact=tg.exact(x,y,t,n,viscosity)[2]
|
|
angle=4*np.pi/n
|
|
basis=np.column_stack((np.cos(angle*x),np.sin(angle*x),np.cos(angle*y),np.sin(angle*y)))
|
|
rows.append(((rho-1)/3,exact,np.full(len(rho),.25**level),basis))
|
|
p,exact,w,b=[np.concatenate(x) for x in zip(*rows)]
|
|
volume=w.sum(); p-=w@p/volume; exact-=w@exact/volume; e=p-exact
|
|
b-=w@b/volume
|
|
gram=b.T@(w[:,None]*b); rhs=b.T@(w*e)
|
|
coefficients=np.linalg.solve(gram,rhs); residual=e-b@coefficients
|
|
norm=float(w@(e*e)); exact_norm=float(w@(exact*exact)); remaining=float(w@(residual*residual))
|
|
projected=float(coefficients@gram@coefficients)
|
|
return dict(step=t,tstar=t*.03/n,absolute_rms=np.sqrt(norm/volume),relative_rms=np.sqrt(norm/exact_norm),
|
|
coefficients=coefficients.tolist(),basis=['cos_4pi_x','sin_4pi_x','cos_4pi_y','sin_4pi_y'],
|
|
residual_rms=np.sqrt(remaining/volume),residual_relative=np.sqrt(remaining/exact_norm),
|
|
gram_condition=float(np.linalg.cond(gram)),
|
|
orthogonality=(b.T@(w*residual)/volume).tolist(),energy_closure=(norm-projected-remaining)/volume)
|
|
|
|
|
|
def load_reference(path):
|
|
if old.sha(path)!=REFERENCE_SHA:
|
|
raise ValueError('verified shifted N64 receipt differs')
|
|
data=json.loads(Path(path).read_text())
|
|
if (not data['completed'] or not data['passed'] or data['source']['commit']!=BASE
|
|
or data['setup']['storage']!='shifted_FP32' or data['setup']['bounds']!=[8,8,56,48]
|
|
or data['setup']['interval_count']!=1024 or len(data['rows'])!=4):
|
|
raise ValueError('N64 reference identity differs')
|
|
if tuple(row['tstar'] for row in data['rows'])!=old.TIMES:
|
|
raise ValueError('N64 reference sample times differ')
|
|
return data
|
|
|
|
|
|
def source_identity(reference):
|
|
git=lambda *args:subprocess.check_output(['git',*args],cwd=ROOT,text=True).strip()
|
|
if git('status','--porcelain'):
|
|
raise RuntimeError('clean committed checkout required')
|
|
unchanged=['src','tests/validation/taylor_green/oracle.py']+[f'tests/validation/multigrid/{name}' for name in
|
|
('spacetime_gpu.cu','spacetime_reference.py','cpu_reference.py','run_spacetime_gpu_n64.py','test_spacetime_gpu.py')]
|
|
subprocess.check_call(['git','diff','--exit-code',BASE,'--',*unchanged],cwd=ROOT)
|
|
previous=subprocess.check_output(['git','show',f'{BASE}:tests/validation/multigrid/spacetime_gpu.py'],cwd=ROOT,text=True)
|
|
# The raw interval schedule remains byte-for-byte unchanged despite the new owner.
|
|
method=inspect.getsource(gpu.SpaceTimeGPU.step)
|
|
if method not in previous or np.__version__!=reference['source']['numpy']:
|
|
raise RuntimeError('raw schedule or NumPy identity changed')
|
|
files=[Path(__file__),Path(gpu.__file__),Path(old.table_guard.__file__),Path(__file__).with_name('test_spacetime_gpu_ladder.py')]
|
|
return dict(commit=git('rev-parse','HEAD'),dirty=False,numpy=np.__version__,production_tree=git('rev-parse','HEAD:src'),
|
|
hashes={p.name:old.sha(p) for p in files},cuda_sha256=old.sha(Path(gpu.__file__).with_suffix('.cu')),
|
|
raw_schedule_sha256=hashlib.sha256(method.encode()).hexdigest(),unchanged_against=BASE,
|
|
reference_hashes=gpu.PINS)
|
|
|
|
|
|
def directional(arrays,masks):
|
|
return sum(.25**level*np.asarray(a[:,m],np.float64).sum(axis=1) for level,(a,m) in enumerate(zip(arrays,masks)))
|
|
|
|
|
|
def populations(run,refined,*,raw=False):
|
|
state=run._download_stored() if raw else run.download()
|
|
if not refined:
|
|
return (state,)
|
|
if state.phase!='arrival_f' or state.tick!=2*run.step_count:
|
|
raise AssertionError('refined sample is not synchronized arrival F')
|
|
return state.coarse_f,state.fine_f
|
|
|
|
|
|
def assess(rows,health):
|
|
checks={key:all(np.isfinite(row[key]) and 0<=row[key]<=limit for row in rows) for key,limit in LIMITS.items()}
|
|
checks['population_health']=not any(health)
|
|
checks['sampled_population_positive']=all(row['minimum_population']>0 for row in rows)
|
|
return dict(worst={key:max(row[key] for row in rows) for key in LIMITS},campaign_checks=checks,
|
|
campaign_pass=all(checks.values()),accuracy_comparison_status='descriptive_only_no_new_threshold')
|
|
|
|
|
|
def ratios(a,b,*,trend=False):
|
|
out=dict(ratio=a/b if b>0 else None,ratio_status='defined' if b>0 else 'zero_denominator')
|
|
if trend:
|
|
out.update(two_point_slope=np.log2(a/b) if a>0 and b>0 else None,
|
|
slope_status='defined' if a>0 and b>0 else 'zero_error_log_undefined')
|
|
return out
|
|
|
|
|
|
def comparisons(runs,reference):
|
|
data={r['name']:r for r in runs}; data['refined64']=reference
|
|
out=[]
|
|
for refined,base,fine in (('refined64','uniform64','uniform128'),('refined128','uniform128','uniform256')):
|
|
for control in (base,fine):
|
|
out.append(dict(candidate=refined,control=control,control_spacing='base' if control==base else 'finest',
|
|
samples=[dict(tstar=a['tstar'],**{key:dict(candidate=a[key],control=b[key],**ratios(a[key],b[key]))
|
|
for key in ('velocity_l2','pressure_l2','energy_relative')})
|
|
for a,b in zip(data[refined]['rows'],data[control]['rows'])]))
|
|
trends=[]
|
|
for coarse,fine in (('uniform64','uniform128'),('uniform128','uniform256'),('refined64','refined128')):
|
|
trends.append(dict(coarse=coarse,fine=fine,samples=[dict(tstar=a['tstar'],**{
|
|
key:ratios(a[key],b[key],trend=True)
|
|
for key in ('velocity_l2','pressure_l2','energy_relative')})
|
|
for a,b in zip(data[coarse]['rows'],data[fine]['rows'])]))
|
|
return dict(controls=out,trends=trends,caveat='norm differences are not interface error; fixed Mach and two refined sizes do not certify convergence')
|
|
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument('--gpu-admitted',action='store_true'); p.add_argument('--reference-n64',type=Path,required=True)
|
|
p.add_argument('--output',type=Path,required=True); args=p.parse_args()
|
|
if not args.gpu_admitted or not os.environ.get('CUDA_VISIBLE_DEVICES'):
|
|
p.error('explicit GPU admission and CUDA_VISIBLE_DEVICES required')
|
|
if args.output.exists() or args.output.with_suffix(args.output.suffix+'.tmp').exists(): p.error('output or partial receipt exists; preserve it')
|
|
reference=load_reference(args.reference_n64); source=source_identity(reference); gpu.check_sources()
|
|
started=time.monotonic()
|
|
report=dict(completed=False,passed=False,source=source,runs=[],limits=dict(seconds=SECONDS,peak_rss_mib=RSS_MIB),
|
|
campaign_limits=LIMITS,reference=dict(sha256=REFERENCE_SHA,source=reference['source'],
|
|
coverage='verified refined base64/fine128 four-time rows only; early pressure trace unavailable'),
|
|
historical_pressure_gate=dict(scope='reused_refined64_only',**reference['historical_pressure_gate']),
|
|
n64_experimental_screen=dict(scope='reused_refined64_only',checks=reference['experimental_checks']),
|
|
frozen_cases=[setup(n,r) for _,n,r in CASES],early_trace_scope='65 predeclared rows per new run, no field snapshots',
|
|
cuda_visible_devices=os.environ['CUDA_VISIBLE_DEVICES'])
|
|
run=context=ledger=None
|
|
case_started=None
|
|
def save():
|
|
report['resources']=dict(seconds=time.monotonic()-started,peak_rss_mib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024,
|
|
rss_scope='Python process peak; external wrapper monitors process group and wall time')
|
|
report['resources']['within_limits']=report['resources']['seconds']<=SECONDS and report['resources']['peak_rss_mib']<=RSS_MIB
|
|
if report['completed'] and not report['resources']['within_limits']: report['passed']=False
|
|
if run is not None:
|
|
report['runs'][-1].update(completed_intervals=run.step_count,health_counts=run.health_counts.tolist(),
|
|
seconds=time.monotonic()-case_started)
|
|
if ledger is not None:
|
|
report['runs'][-1].update(ledger=ledger.snapshot(),audited_intervals=ledger.steps)
|
|
temporary=args.output.with_suffix(args.output.suffix+'.tmp')
|
|
ref.write_receipt(temporary,report); os.replace(temporary,args.output)
|
|
def cap():
|
|
if time.monotonic()-started>SECONDS or resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024>RSS_MIB:
|
|
raise RuntimeError('frozen aggregate resource cap exceeded')
|
|
save()
|
|
try:
|
|
from CelerisLab.cuda.context import CudaContext
|
|
context=CudaContext(0)
|
|
if context.name!=reference['device'] or context.sm_arch!=reference['sm_arch']:
|
|
raise RuntimeError('matched reference device identity differs')
|
|
report.update(device=context.name,sm_arch=context.sm_arch)
|
|
for name,n,refined in CASES:
|
|
cap(); ledger=None; case_started=time.monotonic(); cfg=old.LBMConfig(nx=n,ny=n,viscosity=.0003*n,velocity=0.,ddf_shifting=True)
|
|
case=dict(name=name,setup=setup(n,refined),completed=False,phase='setup',rows=[],early_pressure=[],prefixes=[],health_counts=[0]*6,
|
|
ledger_scope='all intervals; fine/transfer components are mathematically absent for uniform runs')
|
|
report['runs'].append(case); save()
|
|
run=gpu.SpaceTimeGPU(cfg,tuple(case['setup']['bounds']),context=context,allow_n128=True) if refined else gpu.UniformGPU(cfg,context=context)
|
|
case.update(build_metadata=run.build_metadata,device_allocations_bytes=run.device_bytes,
|
|
device_memory_scope='owner allocations; excludes driver/context/compiler')
|
|
expected_modules=None
|
|
if not refined and n in (64,128): expected_modules=[reference['build_metadata'][0 if n==64 else 1]]
|
|
if refined: expected_modules=[report['runs'][1]['build_metadata'][0],report['runs'][2]['build_metadata'][0]]
|
|
if expected_modules is not None:
|
|
if run.build_metadata!=expected_modules: raise RuntimeError('matched source/config/PTX metadata differs')
|
|
case['matched_build_metadata']=True
|
|
if any(m['flags']!=reference['build_metadata'][0]['flags'] for m in run.build_metadata):
|
|
raise RuntimeError('compiler flags differ from reference')
|
|
if refined:
|
|
case['tables']=old.table_guard.validate_tables(run.plan,run.tables,allow_n128=True)
|
|
masks=(run.plan.maps.coarse_active,run.plan.maps.fine_active)
|
|
else: masks=(np.ones((n,n),bool),)
|
|
analytic=tuple(tg.initial(m.shape,l,n,cfg.viscosity) for l,m in enumerate(masks))
|
|
initial=ref.cpu.State(*analytic) if refined else analytic[0]
|
|
physical=lambda:populations(run,refined)
|
|
stored=lambda:populations(run,refined,raw=True)
|
|
run.initialize(initial); raw0=stored(); direction0=directional(raw0,masks)
|
|
case['phase']='warm_reset'; warm_started=time.monotonic()
|
|
for _ in range(32): cap(); run.step()
|
|
case['warmup_seconds']=time.monotonic()-warm_started
|
|
run.initialize(initial)
|
|
for a,b in zip(raw0,stored()): np.testing.assert_array_equal(a.view(np.uint32),b.view(np.uint32))
|
|
if run.step_count or run.health_counts.any(): raise AssertionError('warm reset failed')
|
|
case['raw_reset_verified']=True; arrays=physical()
|
|
first=tg.metrics(arrays,masks,0,n,cfg.viscosity); mass0=first['mass']; moment0=ref.cpu.MOMENTS.T@directional(arrays,masks)
|
|
case['initial']=dict(mass=mass0,momentum=moment0[1:].tolist(),minimum_population=min(float(a[:,m].min()) for a,m in zip(arrays,masks)))
|
|
case['initial_quantization']=[float(abs(a[:,m]-b[:,m]).max()) for a,b,m in zip(arrays,analytic,masks)]
|
|
case['early_pressure'].append(pressure_projection(arrays,masks,0,n,cfg.viscosity))
|
|
ledger=old.LedgerTotals(); case['phase']='trajectory'; trajectory_started=time.monotonic()
|
|
case['all_interval_mass']=dict(maximum_relative=0.,step=0,signed_change_at_maximum=0.,limit=LIMITS['mass_relative'],passed=True)
|
|
for step in range(1,16*n+1):
|
|
cap(); measured=run.step(); ledger.add(measured)
|
|
if run.step_count!=step: raise AssertionError('interval count differs')
|
|
mass_change=float(ledger.sums[0].sum()); drift=abs(mass_change)/mass0
|
|
if drift>case['all_interval_mass']['maximum_relative']:
|
|
case['all_interval_mass'].update(maximum_relative=drift,step=step,signed_change_at_maximum=mass_change,passed=drift<=LIMITS['mass_relative'])
|
|
if step in case['setup']['early_steps'] or step in case['setup']['sample_steps']:
|
|
arrays=physical()
|
|
if step in case['setup']['early_steps']:
|
|
case['early_pressure'].append(pressure_projection(arrays,masks,step,n,cfg.viscosity)); save()
|
|
if step in case['setup']['sample_steps']:
|
|
row=tg.metrics(arrays,masks,step,n,cfg.viscosity,mass0)
|
|
row.update(step=step,momentum_change=(ref.cpu.MOMENTS.T@directional(arrays,masks)-moment0)[1:].tolist(),
|
|
mass_change=row['mass']-mass0,minimum_population=min(float(a[:,m].min()) for a,m in zip(arrays,masks)))
|
|
if not all(np.isfinite(v).all() for v in row.values()): raise FloatingPointError('nonfinite sampled metric')
|
|
case['rows'].append(row); prefix=ledger.snapshot(directional(stored(),masks)-direction0)
|
|
prefix['physical_minus_raw_endpoint_moments']=(ref.cpu.MOMENTS.T@directional(arrays,masks)-moment0-ref.cpu.MOMENTS.T@np.array(prefix['endpoint_directional_change'])).tolist()
|
|
case['prefixes'].append(prefix)
|
|
if not prefix['closure_pass'] or not prefix['component_split_pass']: raise AssertionError('cumulative budget failed')
|
|
save(); print(json.dumps(dict(case=name,**row),allow_nan=False),flush=True)
|
|
case.update(assess(case['rows'],run.health_counts))
|
|
cap(); case.update(completed=True,phase='completed',health_counts=run.health_counts.tolist(),audited_intervals=ledger.steps,
|
|
ledger=ledger.snapshot(),trajectory_seconds=time.monotonic()-trajectory_started,seconds=time.monotonic()-case_started)
|
|
case['passed']=case['campaign_pass'] and case['all_interval_mass']['passed'] and ledger.steps==16*n and len(case['early_pressure'])==65
|
|
run.close(); run=None; save()
|
|
cap(); report.update(completed=True,passed=all(c['passed'] for c in report['runs']),comparisons=comparisons(report['runs'],reference))
|
|
except Exception as exc:
|
|
report['failure']=repr(exc)
|
|
raise
|
|
finally:
|
|
if run is not None and ledger is not None:
|
|
current=report['runs'][-1]
|
|
current['failure_endpoint_status']='unavailable_owner_not_ready'
|
|
if run.status=='ready':
|
|
try:
|
|
current['failure_endpoint']=ledger.snapshot(directional(populations(run,refined,raw=True),masks)-direction0)
|
|
current['failure_endpoint_status']='measured'
|
|
except Exception as exc: current['failure_endpoint_status']=repr(exc)
|
|
for owner in (run,context):
|
|
if owner is not None:
|
|
try: owner.close()
|
|
except Exception as exc: report.setdefault('cleanup_errors',[]).append(repr(exc)); report['passed']=False
|
|
save()
|
|
return 0 if report['passed'] else 1
|
|
|
|
|
|
if __name__=='__main__':
|
|
raise SystemExit(main())
|