77 lines
5.4 KiB
Python
77 lines
5.4 KiB
Python
"""Freeze two-route k=3 candidates; no registration or automatic geometry labels."""
|
|
from pathlib import Path
|
|
from collections import defaultdict
|
|
from itertools import combinations
|
|
import hashlib,json,time,resource
|
|
import numpy as np
|
|
from g1_candidate_fingerprints import ROOT,check_contract
|
|
from s2_e1_supervisor import save
|
|
|
|
def route_ranks(vectors, keys, developer_indices):
|
|
x=np.asarray(vectors,dtype=np.float64);groups=defaultdict(list)
|
|
for i in developer_indices:groups[keys[i]].append(x[i])
|
|
anchors=np.array([np.median(groups[k],axis=0) for k in sorted(groups)])
|
|
center=np.median(anchors,axis=0);q=np.quantile(anchors,[.25,.75],axis=0,method='linear');iqr=q[1]-q[0]
|
|
keep=iqr>0
|
|
if not keep.any():raise ValueError('All descriptor columns have zero IQR')
|
|
z=(x[:,keep]-center[keep])/iqr[keep]
|
|
# Squared Euclidean distances preserve ordering and exact ties without sqrt.
|
|
d=np.sum((z[:,None]-z[None,:])**2,axis=2)
|
|
ranks=np.full(d.shape,np.iinfo(np.int32).max,dtype=np.int32)
|
|
for i in range(len(x)):
|
|
ids=np.array([j for j in range(len(x)) if keys[j]!=keys[i]],dtype=int)
|
|
values=np.sort(d[i,ids]);ranks[i,ids]=1+np.searchsorted(values,d[i,ids],side='left')
|
|
return ranks,d,{'median':center.tolist(),'iqr':iqr.tolist(),'kept_columns':np.flatnonzero(keep).tolist(),
|
|
'zero_iqr_columns':np.flatnonzero(~keep).tolist(),'development_source_keys':len(groups),
|
|
'weighting':'within-source-key median over calibration parts, then source-key-equal median/IQR'}
|
|
|
|
def main():
|
|
start=time.monotonic();c=check_contract(ROOT)
|
|
batch=json.loads((ROOT/'batch-complete.json').read_text());assert batch['status']=='ALL_FINGERPRINTS_COMPLETE' and batch['failed_jobs']==0
|
|
plan=json.loads((ROOT/'custodian/plan.json').read_text());jobs=plan['jobs'];cases=[j['case'] for j in jobs];keys=[j['source_key'] for j in jobs]
|
|
assert len(cases)==len(set(cases))==624 and cases==sorted(cases)
|
|
rows=[json.loads((ROOT/'custodian/fingerprints'/case/'fingerprint.json').read_text()) for case in cases]
|
|
dev=set(plan['development_cases']);developer_indices=[i for i,x in enumerate(cases) if x in dev];assert len(developer_indices)==24
|
|
matrices={};scalers={};distances={}
|
|
for route,field in [('physical','physical_vector'),('scale_free','shape_vector')]:
|
|
matrices[route],distances[route],scalers[route]=route_ranks([r[field] for r in rows],keys,developer_indices)
|
|
proposals=defaultdict(set);index={case:i for i,case in enumerate(cases)};route_counts={}
|
|
for route,ranks in matrices.items():
|
|
selected={(min(i,j),max(i,j)) for i,j in zip(*np.where(ranks<=3))}
|
|
route_counts[route]=len(selected)
|
|
for pair in selected:proposals[pair].add(route+'_k3')
|
|
exact=defaultdict(list)
|
|
for i,r in enumerate(rows):exact[r['exact_boundary_mesh_sha256']].append(i)
|
|
exact_pairs=set()
|
|
for group in exact.values():
|
|
for i,j in combinations(group,2):
|
|
if keys[i]!=keys[j]:exact_pairs.add((i,j));proposals[(i,j)].add('exact_boundary_mesh')
|
|
old=json.loads(Path(c['historical_pairs']).read_text());legacy_pairs=set();legacy_same_key=[]
|
|
for p in old:
|
|
i,j=sorted((index[p['a']],index[p['b']]))
|
|
if keys[i]==keys[j]:legacy_same_key.append([p['a'],p['b']]);continue
|
|
legacy_pairs.add((i,j));proposals[(i,j)].add('historical_exposed_candidate')
|
|
candidates=[]
|
|
for (i,j),reasons in proposals.items():
|
|
ranks={route:[int(m[i,j]),int(m[j,i])] for route,m in matrices.items()}
|
|
candidates.append({'a':cases[i],'b':cases[j],'source_key_a':keys[i],'source_key_b':keys[j],
|
|
'routes_direction_ranks':ranks,'min_rank':min(v for x in ranks.values() for v in x),
|
|
'squared_distances':{route:float(d[i,j]) for route,d in distances.items()},
|
|
'reasons':sorted(reasons),'relation':'UNREVIEWED','registration_status':'NOT_STARTED'})
|
|
candidates.sort(key=lambda r:(r['min_rank'],r['a'],r['b']))
|
|
path=ROOT/'custodian/candidate-pairs.json'
|
|
if path.exists():raise FileExistsError('Candidate set already frozen')
|
|
save(path,{'k':3,'pairs':candidates,'scalers':scalers,'same_key_legacy_pairs_excluded':legacy_same_key,
|
|
'calibration_thresholds_sha256':c['thresholds_sha256'],'ranking':'competition rank=1+#strictly closer cross-key items; literal float64 ties all included'})
|
|
path.chmod(0o600)
|
|
report={'status':'CANDIDATES_COUNTED_NOT_REGISTERED','parts':624,'source_keys':len(set(keys)),'k_per_route':3,
|
|
'route_unordered_counts':route_counts,'exact_cross_key_pairs':len(exact_pairs),'historical_cross_key_pairs':len(legacy_pairs),
|
|
'historical_same_key_excluded':len(legacy_same_key),'union_pairs':len(candidates),'above_4000_report_before_registration':len(candidates)>4000,
|
|
'candidate_sha256':hashlib.sha256(path.read_bytes()).hexdigest(),'calibration_thresholds_sha256':c['thresholds_sha256'],
|
|
'scalers':scalers,'max_min_rank':max(r['min_rank'] for r in candidates),'all_relations_unreviewed':True,
|
|
'worker_cpu_seconds_fingerprints':batch['worker_cpu_seconds'],'count_cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
|
'count_wall_seconds':time.monotonic()-start,'cross_key_recall_certified':False,'registration_started':False}
|
|
save(ROOT/'public/candidate-count.json',report);print(json.dumps(report,indent=2))
|
|
|
|
if __name__=='__main__':main()
|