117 lines
8.6 KiB
Python
117 lines
8.6 KiB
Python
"""Approved <=64-query/part timing-only calibration of the frozen integrator."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import resource
|
|
import time
|
|
import numpy as np
|
|
from scipy.optimize import nnls
|
|
if __package__:
|
|
from . import run_e6_real_kernel as e6
|
|
else:
|
|
import run_e6_real_kernel as e6
|
|
|
|
|
|
def sha(p):return hashlib.sha256(Path(p).read_bytes()).hexdigest()
|
|
|
|
|
|
def choose(counts,positive,old_ids):
|
|
rows=[]
|
|
for name,mask in [('boundary',~positive),('interior',positive)]:
|
|
ids=np.flatnonzero(mask);ids=ids[np.argsort(counts[ids],kind='stable')]
|
|
for j,chunk in enumerate(np.array_split(ids,16)):
|
|
first=int(chunk[len(chunk)//4]);second=int(chunk[3*len(chunk)//4])
|
|
overlap=chunk[np.isin(chunk,old_ids)]
|
|
if len(overlap):first=int(overlap[len(overlap)//2])
|
|
if j==15:second=int(chunk[-1])
|
|
if first==second: first=int(chunk[0] if chunk[0]!=second else chunk[1])
|
|
for node in [first,second]:
|
|
rows.append(dict(node=node,group=name,stratum=j,population=len(chunk),population_neighbors=int(counts[chunk].sum()),neighbors=int(counts[node])))
|
|
assert len(rows)<=64 and len({r['node'] for r in rows})==len(rows)
|
|
assert max(r['neighbors'] for r in rows)==int(counts.max())
|
|
return rows
|
|
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser();p.add_argument('output',type=Path);p.add_argument('phase',choices=['plan','run']);p.add_argument('--case-index',type=int);a=p.parse_args();out=a.output
|
|
contract=json.loads((out/'calibration_contract.json').read_text())
|
|
for path,h in contract['input_sha256'].items():
|
|
if sha(path)!=h:raise ValueError('Changed source '+path)
|
|
if a.phase=='plan':
|
|
cases=[]
|
|
for spec in contract['cases']:
|
|
cost=np.load(out/(spec['case']+'_counts.npz'));old=np.load(Path(contract['prior_directory'])/(spec['case']+'_tetra_four_point_anchors.npz'))
|
|
rows=choose(cost['neighbor_counts'],cost['positive_distance_mask'],old['unique_node_ids'])
|
|
cases.append(dict(case=spec['case'],queries=rows,neighbor_visits=sum(r['neighbors'] for r in rows)))
|
|
total=sum(r['neighbor_visits'] for r in cases)
|
|
if total>contract['total_visit_cap']:raise ValueError('Calibration visit cap exceeded')
|
|
result=dict(contract_sha256=sha(out/'calibration_contract.json'),cases=cases,total_neighbor_visits=total)
|
|
with (out/'calibration_plan.json').open('x') as f:json.dump(result,f,indent=2)
|
|
print(json.dumps(dict(total_visits=total,parts=[dict(case=c['case'],queries=len(c['queries']),visits=c['neighbor_visits']) for c in cases]),indent=2));return
|
|
plan=json.loads((out/'calibration_plan.json').read_text());assert plan['contract_sha256']==sha(out/'calibration_contract.json')
|
|
assert plan['total_neighbor_visits']<=contract['total_visit_cap']
|
|
spec=contract['cases'][a.case_index];case=spec['case'];queries=plan['cases'][a.case_index]['queries'];assert plan['cases'][a.case_index]['case']==case
|
|
with (out/(case+'.calibration.started.json')).open('x') as f:json.dump(dict(contract_sha256=sha(out/'calibration_contract.json')),f)
|
|
started=time.perf_counter()
|
|
with np.load(spec['sample']) as data:x=data['points_raw'];tet=data['tetra'];mass=data['node_volume_raw']
|
|
d=np.load(spec['distance']);loaded=time.perf_counter()
|
|
raw,_=e6.raw_field(x,tet,d);raw_done=time.perf_counter()
|
|
sx,sm,st=e6.samples(x,tet,mass,'tetra_four_point',raw);sources_done=time.perf_counter()
|
|
count_result=json.loads((out/(case+'.json')).read_text());R=count_result['radius']
|
|
ids=np.array([q['node'] for q in queries]);record=dict(tree_seconds=None,query_seconds=[],neighbor_counts=[],tree_calls=0)
|
|
original_tree=e6.cKDTree
|
|
class TimedTree:
|
|
def __init__(self,*args,**kwargs):
|
|
t=time.perf_counter();self.tree=original_tree(*args,**kwargs);record['tree_seconds']=time.perf_counter()-t;record['tree_calls']+=1;record['last_start']=None
|
|
def query_ball_point(self,*args,**kwargs):
|
|
now=time.perf_counter()
|
|
if record['last_start'] is not None:record['query_seconds'].append(now-record['last_start'])
|
|
record['last_start']=now
|
|
result=self.tree.query_ball_point(*args,**kwargs);record['neighbor_counts'].append(len(result));return result
|
|
e6.cKDTree=TimedTree
|
|
try:
|
|
tick=time.perf_counter();values=e6.integrate(x[ids],sx,sm,st,R);integrated=time.perf_counter()
|
|
record['query_seconds'].append(integrated-record['last_start'])
|
|
finally:e6.cKDTree=original_tree
|
|
assert record['tree_calls']==1 and len(record['query_seconds'])==len(queries)
|
|
np.testing.assert_array_equal(record['neighbor_counts'],[q['neighbors'] for q in queries])
|
|
# Reusing frozen code plus an exact overlap check avoids inventing a timing surrogate.
|
|
old=np.load(Path(contract['prior_directory'])/(case+'_tetra_four_point_fields.npz'));old_ids=old['unique_node_ids'];overlap=np.isin(ids,old_ids)
|
|
assert overlap.any()
|
|
for key in values:
|
|
np.testing.assert_allclose(values[key][overlap],old['original_'+key][np.searchsorted(old_ids,ids[overlap])],rtol=1e-11,atol=1e-12)
|
|
for q,t in zip(queries,record['query_seconds']):q['seconds']=t;q['amortized_seconds_per_neighbor']=t/q['neighbors']
|
|
strata=[]
|
|
for group in ['boundary','interior']:
|
|
for j in range(16):
|
|
v=[q for q in queries if q['group']==group and q['stratum']==j];assert len(v)==2
|
|
times=[q['seconds'] for q in v];n=v[0]['population']
|
|
strata.append(dict(group=group,stratum=j,nodes=n,neighbor_visits=v[0]['population_neighbors'],sample_nodes=[q['node'] for q in v],
|
|
sample_query_seconds=times,estimated_seconds=n*np.mean(times),fast_scenario_seconds=n*min(times),slow_scenario_seconds=n*max(times)))
|
|
fixed=dict(load_seconds=loaded-started,raw_field_seconds=raw_done-loaded,source_samples_seconds=sources_done-raw_done,tree_seconds=record['tree_seconds'])
|
|
groups={}
|
|
for name,allowed in [('all_nodes',['boundary','interior']),('positive_distance_nodes',['interior']),('boundary_nodes',['boundary'])]:
|
|
select=[s for s in strata if s['group'] in allowed]
|
|
groups[name]=dict(integration_seconds=sum(s['estimated_seconds'] for s in select),fast_scenario_seconds=sum(s['fast_scenario_seconds'] for s in select),slow_scenario_seconds=sum(s['slow_scenario_seconds'] for s in select))
|
|
groups[name]['total_seconds_with_fixed_setup']=sum(fixed.values())+groups[name]['integration_seconds']
|
|
counts=np.array([q['neighbors'] for q in queries]);times=np.array(record['query_seconds'])
|
|
coef,residual=nnls(np.c_[np.ones(len(counts)),counts/counts.max()],times)
|
|
beta=coef[1]/counts.max();fit={name:sum(fixed.values())+coef[0]*v['nodes']+beta*v['neighbor_sum'] for name,v in count_result['groups'].items()}
|
|
peak=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss*1024;N=len(x)
|
|
result=dict(case=case,contract_sha256=sha(out/'calibration_contract.json'),queries=queries,strata=strata,
|
|
query_count=len(ids),neighbor_visits=int(counts.sum()),fixed_setup=fixed,measured_integrate_seconds=integrated-tick,
|
|
measured_query_loop_seconds=float(times.sum()),amortized_seconds_per_neighbor=float(times.sum()/counts.sum()),
|
|
old_field_overlap_checked=int(overlap.sum()),estimates=groups,
|
|
nonnegative_two_term_diagnostic_fit=dict(seconds_per_query_intercept=float(coef[0]),seconds_per_neighbor_slope=float(beta),residual_l2_seconds=float(residual),estimated_total_seconds=fit),
|
|
memory=dict(measured_calibration_peak_bytes=peak,full_E6_output_addition_bytes=120*(N-len(ids)),
|
|
full_E6_peak_allowance_bytes=peak+120*(N-len(ids))+128*1024**2,
|
|
block4096_E6_peak_allowance_bytes=peak+120*(min(N,4096)-len(ids))+128*1024**2),
|
|
wall_seconds=time.perf_counter()-started,new_scientific_outputs_saved=False,new_distance_queries=0,new_representation=False,
|
|
limitations=['64 deterministic stratified points, not a randomized timing sample or confidence interval.','Per-stratum fast/slow scenarios span the two sampled query timings, not guaranteed bounds.','Per-neighbor average includes per-query overhead; separate NNLS intercept/slope is diagnostic only.','First/cold query, allocator, node order, machine load and source locality affect full-run timing.','Only existing E6 integrator and output allocation modeled; future O2-rep, edge contractions, file serialization and training excluded.','128 MiB extra memory reserve is a planning allowance, not a proof of maximum RSS.'])
|
|
(out/(case+'.calibration.json')).write_text(json.dumps(result,indent=2)+'\n')
|
|
print(json.dumps({k:v for k,v in result.items() if k not in ['queries','strata']},indent=2))
|
|
|
|
|
|
if __name__=='__main__':main()
|