"""Approved <=64-query/part timing-only calibration of the frozen integrator.""" import argparse import hashlib import json from pathlib import Path import resource import time import numpy as np from scipy.optimize import nnls if __package__: from . import run_e6_real_kernel as e6 else: import run_e6_real_kernel as e6 def sha(p):return hashlib.sha256(Path(p).read_bytes()).hexdigest() def choose(counts,positive,old_ids): rows=[] for name,mask in [('boundary',~positive),('interior',positive)]: ids=np.flatnonzero(mask);ids=ids[np.argsort(counts[ids],kind='stable')] for j,chunk in enumerate(np.array_split(ids,16)): first=int(chunk[len(chunk)//4]);second=int(chunk[3*len(chunk)//4]) overlap=chunk[np.isin(chunk,old_ids)] if len(overlap):first=int(overlap[len(overlap)//2]) if j==15:second=int(chunk[-1]) if first==second: first=int(chunk[0] if chunk[0]!=second else chunk[1]) for node in [first,second]: rows.append(dict(node=node,group=name,stratum=j,population=len(chunk),population_neighbors=int(counts[chunk].sum()),neighbors=int(counts[node]))) assert len(rows)<=64 and len({r['node'] for r in rows})==len(rows) assert max(r['neighbors'] for r in rows)==int(counts.max()) return rows def main(): p=argparse.ArgumentParser();p.add_argument('output',type=Path);p.add_argument('phase',choices=['plan','run']);p.add_argument('--case-index',type=int);a=p.parse_args();out=a.output contract=json.loads((out/'calibration_contract.json').read_text()) for path,h in contract['input_sha256'].items(): if sha(path)!=h:raise ValueError('Changed source '+path) if a.phase=='plan': cases=[] for spec in contract['cases']: cost=np.load(out/(spec['case']+'_counts.npz'));old=np.load(Path(contract['prior_directory'])/(spec['case']+'_tetra_four_point_anchors.npz')) rows=choose(cost['neighbor_counts'],cost['positive_distance_mask'],old['unique_node_ids']) cases.append(dict(case=spec['case'],queries=rows,neighbor_visits=sum(r['neighbors'] for r in rows))) total=sum(r['neighbor_visits'] for r in cases) if total>contract['total_visit_cap']:raise ValueError('Calibration visit cap exceeded') result=dict(contract_sha256=sha(out/'calibration_contract.json'),cases=cases,total_neighbor_visits=total) with (out/'calibration_plan.json').open('x') as f:json.dump(result,f,indent=2) print(json.dumps(dict(total_visits=total,parts=[dict(case=c['case'],queries=len(c['queries']),visits=c['neighbor_visits']) for c in cases]),indent=2));return plan=json.loads((out/'calibration_plan.json').read_text());assert plan['contract_sha256']==sha(out/'calibration_contract.json') assert plan['total_neighbor_visits']<=contract['total_visit_cap'] spec=contract['cases'][a.case_index];case=spec['case'];queries=plan['cases'][a.case_index]['queries'];assert plan['cases'][a.case_index]['case']==case with (out/(case+'.calibration.started.json')).open('x') as f:json.dump(dict(contract_sha256=sha(out/'calibration_contract.json')),f) started=time.perf_counter() with np.load(spec['sample']) as data:x=data['points_raw'];tet=data['tetra'];mass=data['node_volume_raw'] d=np.load(spec['distance']);loaded=time.perf_counter() raw,_=e6.raw_field(x,tet,d);raw_done=time.perf_counter() sx,sm,st=e6.samples(x,tet,mass,'tetra_four_point',raw);sources_done=time.perf_counter() count_result=json.loads((out/(case+'.json')).read_text());R=count_result['radius'] ids=np.array([q['node'] for q in queries]);record=dict(tree_seconds=None,query_seconds=[],neighbor_counts=[],tree_calls=0) original_tree=e6.cKDTree class TimedTree: def __init__(self,*args,**kwargs): t=time.perf_counter();self.tree=original_tree(*args,**kwargs);record['tree_seconds']=time.perf_counter()-t;record['tree_calls']+=1;record['last_start']=None def query_ball_point(self,*args,**kwargs): now=time.perf_counter() if record['last_start'] is not None:record['query_seconds'].append(now-record['last_start']) record['last_start']=now result=self.tree.query_ball_point(*args,**kwargs);record['neighbor_counts'].append(len(result));return result e6.cKDTree=TimedTree try: tick=time.perf_counter();values=e6.integrate(x[ids],sx,sm,st,R);integrated=time.perf_counter() record['query_seconds'].append(integrated-record['last_start']) finally:e6.cKDTree=original_tree assert record['tree_calls']==1 and len(record['query_seconds'])==len(queries) np.testing.assert_array_equal(record['neighbor_counts'],[q['neighbors'] for q in queries]) # Reusing frozen code plus an exact overlap check avoids inventing a timing surrogate. old=np.load(Path(contract['prior_directory'])/(case+'_tetra_four_point_fields.npz'));old_ids=old['unique_node_ids'];overlap=np.isin(ids,old_ids) assert overlap.any() for key in values: np.testing.assert_allclose(values[key][overlap],old['original_'+key][np.searchsorted(old_ids,ids[overlap])],rtol=1e-11,atol=1e-12) for q,t in zip(queries,record['query_seconds']):q['seconds']=t;q['amortized_seconds_per_neighbor']=t/q['neighbors'] strata=[] for group in ['boundary','interior']: for j in range(16): v=[q for q in queries if q['group']==group and q['stratum']==j];assert len(v)==2 times=[q['seconds'] for q in v];n=v[0]['population'] strata.append(dict(group=group,stratum=j,nodes=n,neighbor_visits=v[0]['population_neighbors'],sample_nodes=[q['node'] for q in v], sample_query_seconds=times,estimated_seconds=n*np.mean(times),fast_scenario_seconds=n*min(times),slow_scenario_seconds=n*max(times))) fixed=dict(load_seconds=loaded-started,raw_field_seconds=raw_done-loaded,source_samples_seconds=sources_done-raw_done,tree_seconds=record['tree_seconds']) groups={} for name,allowed in [('all_nodes',['boundary','interior']),('positive_distance_nodes',['interior']),('boundary_nodes',['boundary'])]: select=[s for s in strata if s['group'] in allowed] groups[name]=dict(integration_seconds=sum(s['estimated_seconds'] for s in select),fast_scenario_seconds=sum(s['fast_scenario_seconds'] for s in select),slow_scenario_seconds=sum(s['slow_scenario_seconds'] for s in select)) groups[name]['total_seconds_with_fixed_setup']=sum(fixed.values())+groups[name]['integration_seconds'] counts=np.array([q['neighbors'] for q in queries]);times=np.array(record['query_seconds']) coef,residual=nnls(np.c_[np.ones(len(counts)),counts/counts.max()],times) beta=coef[1]/counts.max();fit={name:sum(fixed.values())+coef[0]*v['nodes']+beta*v['neighbor_sum'] for name,v in count_result['groups'].items()} peak=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss*1024;N=len(x) result=dict(case=case,contract_sha256=sha(out/'calibration_contract.json'),queries=queries,strata=strata, query_count=len(ids),neighbor_visits=int(counts.sum()),fixed_setup=fixed,measured_integrate_seconds=integrated-tick, measured_query_loop_seconds=float(times.sum()),amortized_seconds_per_neighbor=float(times.sum()/counts.sum()), old_field_overlap_checked=int(overlap.sum()),estimates=groups, nonnegative_two_term_diagnostic_fit=dict(seconds_per_query_intercept=float(coef[0]),seconds_per_neighbor_slope=float(beta),residual_l2_seconds=float(residual),estimated_total_seconds=fit), memory=dict(measured_calibration_peak_bytes=peak,full_E6_output_addition_bytes=120*(N-len(ids)), full_E6_peak_allowance_bytes=peak+120*(N-len(ids))+128*1024**2, block4096_E6_peak_allowance_bytes=peak+120*(min(N,4096)-len(ids))+128*1024**2), wall_seconds=time.perf_counter()-started,new_scientific_outputs_saved=False,new_distance_queries=0,new_representation=False, limitations=['64 deterministic stratified points, not a randomized timing sample or confidence interval.','Per-stratum fast/slow scenarios span the two sampled query timings, not guaranteed bounds.','Per-neighbor average includes per-query overhead; separate NNLS intercept/slope is diagnostic only.','First/cold query, allocator, node order, machine load and source locality affect full-run timing.','Only existing E6 integrator and output allocation modeled; future O2-rep, edge contractions, file serialization and training excluded.','128 MiB extra memory reserve is a planning allowance, not a proof of maximum RSS.']) (out/(case+'.calibration.json')).write_text(json.dumps(result,indent=2)+'\n') print(json.dumps({k:v for k,v in result.items() if k not in ['queries','strata']},indent=2)) if __name__=='__main__':main()