Freeze completed G1 calibration and start note16 fingerprint candidate count
This commit is contained in:
1 parent
2ba7800d39
commit
ff439e81fc
76 files changed
+170909
No files matched your search
@@ -0,0 +1,76 @@
|
||||
"""Freeze two-route k=3 candidates; no registration or automatic geometry labels."""
|
||||
from pathlib import Path
|
||||
from collections import defaultdict
|
||||
from itertools import combinations
|
||||
import hashlib,json,time,resource
|
||||
import numpy as np
|
||||
from g1_candidate_fingerprints import ROOT,check_contract
|
||||
from s2_e1_supervisor import save
|
||||
|
||||
def route_ranks(vectors, keys, developer_indices):
|
||||
x=np.asarray(vectors,dtype=np.float64);groups=defaultdict(list)
|
||||
for i in developer_indices:groups[keys[i]].append(x[i])
|
||||
anchors=np.array([np.median(groups[k],axis=0) for k in sorted(groups)])
|
||||
center=np.median(anchors,axis=0);q=np.quantile(anchors,[.25,.75],axis=0,method='linear');iqr=q[1]-q[0]
|
||||
keep=iqr>0
|
||||
if not keep.any():raise ValueError('All descriptor columns have zero IQR')
|
||||
z=(x[:,keep]-center[keep])/iqr[keep]
|
||||
# Squared Euclidean distances preserve ordering and exact ties without sqrt.
|
||||
d=np.sum((z[:,None]-z[None,:])**2,axis=2)
|
||||
ranks=np.full(d.shape,np.iinfo(np.int32).max,dtype=np.int32)
|
||||
for i in range(len(x)):
|
||||
ids=np.array([j for j in range(len(x)) if keys[j]!=keys[i]],dtype=int)
|
||||
values=np.sort(d[i,ids]);ranks[i,ids]=1+np.searchsorted(values,d[i,ids],side='left')
|
||||
return ranks,d,{'median':center.tolist(),'iqr':iqr.tolist(),'kept_columns':np.flatnonzero(keep).tolist(),
|
||||
'zero_iqr_columns':np.flatnonzero(~keep).tolist(),'development_source_keys':len(groups),
|
||||
'weighting':'within-source-key median over calibration parts, then source-key-equal median/IQR'}
|
||||
|
||||
def main():
|
||||
start=time.monotonic();c=check_contract(ROOT)
|
||||
batch=json.loads((ROOT/'batch-complete.json').read_text());assert batch['status']=='ALL_FINGERPRINTS_COMPLETE' and batch['failed_jobs']==0
|
||||
plan=json.loads((ROOT/'custodian/plan.json').read_text());jobs=plan['jobs'];cases=[j['case'] for j in jobs];keys=[j['source_key'] for j in jobs]
|
||||
assert len(cases)==len(set(cases))==624 and cases==sorted(cases)
|
||||
rows=[json.loads((ROOT/'custodian/fingerprints'/case/'fingerprint.json').read_text()) for case in cases]
|
||||
dev=set(plan['development_cases']);developer_indices=[i for i,x in enumerate(cases) if x in dev];assert len(developer_indices)==24
|
||||
matrices={};scalers={};distances={}
|
||||
for route,field in [('physical','physical_vector'),('scale_free','shape_vector')]:
|
||||
matrices[route],distances[route],scalers[route]=route_ranks([r[field] for r in rows],keys,developer_indices)
|
||||
proposals=defaultdict(set);index={case:i for i,case in enumerate(cases)};route_counts={}
|
||||
for route,ranks in matrices.items():
|
||||
selected={(min(i,j),max(i,j)) for i,j in zip(*np.where(ranks<=3))}
|
||||
route_counts[route]=len(selected)
|
||||
for pair in selected:proposals[pair].add(route+'_k3')
|
||||
exact=defaultdict(list)
|
||||
for i,r in enumerate(rows):exact[r['exact_boundary_mesh_sha256']].append(i)
|
||||
exact_pairs=set()
|
||||
for group in exact.values():
|
||||
for i,j in combinations(group,2):
|
||||
if keys[i]!=keys[j]:exact_pairs.add((i,j));proposals[(i,j)].add('exact_boundary_mesh')
|
||||
old=json.loads(Path(c['historical_pairs']).read_text());legacy_pairs=set();legacy_same_key=[]
|
||||
for p in old:
|
||||
i,j=sorted((index[p['a']],index[p['b']]))
|
||||
if keys[i]==keys[j]:legacy_same_key.append([p['a'],p['b']]);continue
|
||||
legacy_pairs.add((i,j));proposals[(i,j)].add('historical_exposed_candidate')
|
||||
candidates=[]
|
||||
for (i,j),reasons in proposals.items():
|
||||
ranks={route:[int(m[i,j]),int(m[j,i])] for route,m in matrices.items()}
|
||||
candidates.append({'a':cases[i],'b':cases[j],'source_key_a':keys[i],'source_key_b':keys[j],
|
||||
'routes_direction_ranks':ranks,'min_rank':min(v for x in ranks.values() for v in x),
|
||||
'squared_distances':{route:float(d[i,j]) for route,d in distances.items()},
|
||||
'reasons':sorted(reasons),'relation':'UNREVIEWED','registration_status':'NOT_STARTED'})
|
||||
candidates.sort(key=lambda r:(r['min_rank'],r['a'],r['b']))
|
||||
path=ROOT/'custodian/candidate-pairs.json'
|
||||
if path.exists():raise FileExistsError('Candidate set already frozen')
|
||||
save(path,{'k':3,'pairs':candidates,'scalers':scalers,'same_key_legacy_pairs_excluded':legacy_same_key,
|
||||
'calibration_thresholds_sha256':c['thresholds_sha256'],'ranking':'competition rank=1+#strictly closer cross-key items; literal float64 ties all included'})
|
||||
path.chmod(0o600)
|
||||
report={'status':'CANDIDATES_COUNTED_NOT_REGISTERED','parts':624,'source_keys':len(set(keys)),'k_per_route':3,
|
||||
'route_unordered_counts':route_counts,'exact_cross_key_pairs':len(exact_pairs),'historical_cross_key_pairs':len(legacy_pairs),
|
||||
'historical_same_key_excluded':len(legacy_same_key),'union_pairs':len(candidates),'above_4000_report_before_registration':len(candidates)>4000,
|
||||
'candidate_sha256':hashlib.sha256(path.read_bytes()).hexdigest(),'calibration_thresholds_sha256':c['thresholds_sha256'],
|
||||
'scalers':scalers,'max_min_rank':max(r['min_rank'] for r in candidates),'all_relations_unreviewed':True,
|
||||
'worker_cpu_seconds_fingerprints':batch['worker_cpu_seconds'],'count_cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
||||
'count_wall_seconds':time.monotonic()-start,'cross_key_recall_certified':False,'registration_started':False}
|
||||
save(ROOT/'public/candidate-count.json',report);print(json.dumps(report,indent=2))
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Tet-only volume/area fingerprints for the fixed two-route candidate count."""
|
||||
from pathlib import Path
|
||||
from xml.etree import ElementTree as ET
|
||||
from zipfile import ZipFile
|
||||
import argparse, hashlib, json, os, resource, time
|
||||
import numpy as np
|
||||
from audit_samples import decode_array
|
||||
from g1_registration_cost import sample_surface
|
||||
from s2_e1_supervisor import save
|
||||
|
||||
ROOT=Path(os.environ.get('MELTMIND_G1_FINGERPRINT_ROOT','artifacts/s2-g1-candidates-2026-11-14'))
|
||||
|
||||
def read_tets(stream):
|
||||
points=None;cells={};stack=[];allowed={'Points','connectivity','offsets','types'}
|
||||
for event,e in ET.iterparse(stream,events=('start','end')):
|
||||
if event=='start':
|
||||
stack.append(e.tag)
|
||||
if e.tag in ('PointData','CellData'):raise ValueError('Field association before geometry complete')
|
||||
if e.tag=='VTKFile':
|
||||
if e.get('byte_order')!='LittleEndian' or e.get('compressor')!='vtkZLibDataCompressor':raise ValueError('Unsupported geometry encoding')
|
||||
header=e.get('header_type','UInt32')
|
||||
else:
|
||||
if e.tag=='DataArray':
|
||||
name=e.get('Name')
|
||||
if name not in allowed or stack[-2] not in ('Points','Cells'):raise ValueError('Non-geometry array')
|
||||
v=decode_array(e,header).copy()
|
||||
if name=='Points':points=v
|
||||
else:cells[name]=v
|
||||
e.clear()
|
||||
if points is not None and len(cells)==3:break
|
||||
stack.pop()
|
||||
if points is None or len(cells)!=3:raise ValueError('Incomplete geometry')
|
||||
offsets=np.r_[0,cells['offsets']];ids=np.flatnonzero(cells['types']==10)
|
||||
if not len(ids) or np.any(offsets[ids+1]-offsets[ids]!=4):raise ValueError('Invalid tetrahedra')
|
||||
return points,cells['connectivity'][offsets[ids,None]+np.arange(4)]
|
||||
|
||||
def fingerprint(points,tetra,prefix):
|
||||
if not np.isfinite(points).all() or tetra.min()<0 or tetra.max()>=len(points):raise ValueError('Invalid geometry')
|
||||
used=np.unique(tetra);x=np.asarray(points[used],dtype=np.float64);tet=np.searchsorted(used,tetra)
|
||||
q=x[tet];volume=np.abs(np.einsum('ij,ij->i',np.cross(q[:,1]-q[:,0],q[:,2]-q[:,0]),q[:,3]-q[:,0]))/6
|
||||
if np.any(volume<=0):raise ValueError('Zero-volume tetrahedron')
|
||||
v=float(volume.sum());center=np.einsum('ni,n->i',q.mean(1),volume)/v
|
||||
centered=q-center;tot=centered.sum(1)
|
||||
# Exact degree-2 integration in each linear tetrahedron (Dirichlet moments).
|
||||
covariance=(np.einsum('nki,nkj,n->ij',centered,centered,volume)+np.einsum('ni,nj,n->ij',tot,tot,volume))/(20*v)
|
||||
eigenvalues=np.linalg.eigvalsh(covariance)
|
||||
if eigenvalues[0]<=0:raise ValueError('Invalid volume covariance')
|
||||
# Preserve the registration Rg convention: tetra volume/4 nodal quadrature.
|
||||
rg=float(np.sqrt(np.einsum('nki,nki,n->',centered,centered,volume)/(4*v)))
|
||||
del q,centered,tot
|
||||
f=np.sort(tet[:,np.array([[0,1,2],[0,1,3],[0,2,3],[1,2,3]])].reshape(-1,3),axis=1)
|
||||
unique,counts=np.unique(f,axis=0,return_counts=True)
|
||||
if np.any(counts>2):raise ValueError('Nonmanifold tetra faces')
|
||||
boundary=unique[counts==1];tri=x[boundary]
|
||||
a=float(np.linalg.norm(np.cross(tri[:,1]-tri[:,0],tri[:,2]-tri[:,0]),axis=1).sum()/2)
|
||||
g={'points':x,'boundary':boundary,'center':center,'rg':np.array(rg)}
|
||||
sampled,_=sample_surface(g,prefix)
|
||||
radial=np.quantile(np.linalg.norm(sampled-center,axis=1)/rg,[.1,.5,.9],method='inverted_cdf')
|
||||
ratios=eigenvalues/eigenvalues.sum()
|
||||
shape=np.r_[np.log(a/rg**2),np.log(v/rg**3),ratios[:2],radial]
|
||||
# Exact boundary-mesh equality candidate, independent of vertex/face order.
|
||||
surf_ids=np.unique(boundary);sx=x[surf_ids].copy();sx[sx==0]=0
|
||||
sf=np.searchsorted(surf_ids,boundary);order=np.lexsort((sx[:,2],sx[:,1],sx[:,0]));inverse=np.empty_like(order);inverse[order]=np.arange(len(order))
|
||||
canon=np.sort(inverse[sf],axis=1);canon=canon[np.lexsort((canon[:,2],canon[:,1],canon[:,0]))]
|
||||
h=hashlib.sha256();h.update(np.asarray(sx[order],dtype='<f8').tobytes());h.update(np.asarray(canon,dtype='<i8').tobytes())
|
||||
return g,{'volume':v,'area':a,'rg_registration_quadrature':rg,'volume_center':center.tolist(),
|
||||
'exact_volume_covariance_eigenvalues':eigenvalues.tolist(),'shape_vector':shape.tolist(),
|
||||
'physical_vector':np.r_[shape,np.log(rg)].tolist(),'surface_radial_quantiles':radial.tolist(),
|
||||
'exact_boundary_mesh_sha256':h.hexdigest(),'tetrahedra':len(tet),'surface_triangles':len(boundary)}
|
||||
|
||||
def check_contract(root):
|
||||
c=json.loads((root/'contract.json').read_text())
|
||||
for path,h in {**c['code_sha256'],**c['input_sha256']}.items():
|
||||
if hashlib.sha256(Path(path).read_bytes()).hexdigest()!=h:raise ValueError('Frozen input/source changed: '+path)
|
||||
return c
|
||||
|
||||
def main():
|
||||
ap=argparse.ArgumentParser();ap.add_argument('index',type=int);args=ap.parse_args();c=check_contract(ROOT)
|
||||
job=json.loads((ROOT/'custodian/plan.json').read_text())['jobs'][args.index];case=job['case'];start=time.monotonic()
|
||||
dest=ROOT/'custodian/fingerprints'/case;dest.mkdir(parents=True,exist_ok=False)
|
||||
if job['tet_cache']:
|
||||
if hashlib.sha256(Path(job['tet_cache']).read_bytes()).hexdigest()!=job['tet_cache_sha256']:raise ValueError('Tet cache changed')
|
||||
with np.load(job['tet_cache'],allow_pickle=False) as z:
|
||||
if 'fill_time_raw' in z.files:raise ValueError('Mixed label cache forbidden')
|
||||
points=z['points_raw'];tetra=z['tetra']
|
||||
mode='existing_label_free_tet_cache'
|
||||
else:
|
||||
verification=json.loads(Path(c['archive_verification']).read_text());st=Path(c['archive']).stat()
|
||||
if (st.st_size,st.st_mtime_ns,st.st_ino)!=(verification['bytes'],verification['mtime_ns'],verification['inode']):raise ValueError('Archive changed')
|
||||
with ZipFile(c['archive']) as z,z.open(case+'_sim.vtu') as stream:points,tetra=read_tets(stream)
|
||||
mode='geometry_prefix_for_missing_volume_fingerprint'
|
||||
g,result=fingerprint(points,tetra,int(case.split('_')[0]));np.savez_compressed(dest/'geometry.npz',**g)
|
||||
save(dest/'fingerprint.json',{'case':case,'source_key':job['source_key'],**result,'source_mode':mode,
|
||||
'simulation_fields_read':False,'cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
||||
'wall_seconds':time.monotonic()-start,'peak_rss_bytes':resource.getrusage(resource.RUSAGE_SELF).ru_maxrss*1024})
|
||||
print(json.dumps({'index':args.index,'status':'FINGERPRINT_COMPLETE'}),flush=True)
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Freeze missing note16 fingerprints and k=3 counting, with no threshold refit."""
|
||||
from pathlib import Path
|
||||
from zipfile import ZipFile
|
||||
import json,hashlib
|
||||
from g1_candidate_fingerprints import ROOT
|
||||
from s2_e1_supervisor import save
|
||||
|
||||
def sha(path):return hashlib.sha256(Path(path).read_bytes()).hexdigest()
|
||||
def main():
|
||||
ROOT.mkdir(exist_ok=False);private=ROOT/'custodian';private.mkdir(mode=0o700);(ROOT/'public').mkdir()
|
||||
inventory=Path('artifacts/s2-g1-inventory-2026-11-14/custodian/parts.json');parts=json.loads(inventory.read_text())
|
||||
calibration=Path('artifacts/s2-g1-copy-calibration-2026-11-14/public/calibration-thresholds.json');t=json.loads(calibration.read_text())
|
||||
assert t['status']=='CALIBRATION_THRESHOLDS_FROZEN'
|
||||
development=json.loads(Path('artifacts/s2-g1-copy-calibration-2026-11-14/custodian/plan.json').read_text())
|
||||
dev=sorted({case for j in development['jobs'] for case in j['cases']});assert len(dev)==24
|
||||
jobs=[]
|
||||
for r in sorted(parts,key=lambda r:r['case']):
|
||||
cache=None
|
||||
for item in r['surface_caches']:
|
||||
if item['contains_field_labels']:continue
|
||||
with ZipFile(item['path']) as z:keys={n.removesuffix('.npy') for n in z.namelist()}
|
||||
if {'points_raw','tetra'}<=keys:cache=item['path'];break
|
||||
jobs.append({'case':r['case'],'source_key':r['source_key'],'tet_cache':cache,
|
||||
'tet_cache_sha256':sha(cache) if cache else None,'prior_exposure':r['exposure_state']})
|
||||
assert len(jobs)==624
|
||||
save(private/'plan.json',{'jobs':jobs,'development_cases':dev,'purpose':'missing tet-volume note16 fingerprints, not repeated calibration'})
|
||||
(private/'plan.json').chmod(0o600)
|
||||
files=['prepare_g1_candidate_fingerprints','g1_candidate_fingerprints','run_g1_candidate_fingerprints','count_g1_candidates',
|
||||
's2_e1_supervisor','g1_registration_cost','audit_samples','boundary_distance']
|
||||
contract={'authorizing_rulings':['1794661944.308269','1794662380.787319','1794664166.242149'],
|
||||
'code_sha256':{f'tools/{name}.py':sha(f'tools/{name}.py') for name in files},
|
||||
'input_sha256':{str(p):sha(p) for p in [inventory,private/'plan.json',calibration]},'thresholds_sha256':sha(calibration),
|
||||
'historical_pairs':'artifacts/s2-g1-inventory-2026-11-14/custodian/old_exposed_pairs.json',
|
||||
'archive':'data.zip','archive_verification':'artifacts/intake-2026-11-13/archive_verification.json',
|
||||
'shape_columns':['log(A/Rg^2)','log(V/Rg^3)','exact_volume_cov_eigenvalue_ratio_ascending_0','exact_volume_cov_eigenvalue_ratio_ascending_1','area_PPS_radial_over_Rg_P10','area_PPS_radial_over_Rg_P50','area_PPS_radial_over_Rg_P90'],
|
||||
'physical_extra_column':'log(Rg / one_original_numerical_length_unit)',
|
||||
'integrals':'V and volume center exact linear tetrahedra; covariance exact degree-2 tetra integration; A exact triangles',
|
||||
'rg':'unchanged registration convention: tetra-volume/4 nodal quadrature of centered squared radius; not replaced by sqrt(trace(exact covariance))',
|
||||
'radial':'2000 area PPS points; same numeric-prefix seed convention as fixed registration; P10/P50/P90 inverted_cdf',
|
||||
'scaling':'within-key median over 24 calibration source parts then equal-key median and linear IQR; drop exactly zero IQR columns',
|
||||
'ranking':'float64 squared Euclidean; cross-key only; competition rank=1+strictly closer count; all literal ties at k=3 included; union either route/direction',
|
||||
'extras':'exact canonical boundary-coordinate+triangle hash matches and original 30 exposed candidates; same-key pairs excluded because already same side',
|
||||
'candidate_order':'minimum four direction ranks then stable a,b IDs; count above 4000 reports before any registration',
|
||||
'workers':8,'per_pair_wall_seconds':1800,'per_worker_rss_bytes':1342177280,'aggregate_rss_bytes':20*1024**3,
|
||||
'worker_cpu_budget_seconds':8*3600,'wall_seconds':8*3600,'disk_bytes':8*1024**3,
|
||||
'stage_scope':'fingerprints and candidate counting only; no pair registration, no field labels, no training/GPU',
|
||||
'resources':'preparation capped at 8 CPU-h/8 GiB, separate from the 100 worker CPU-h pair-registration budget; prior E1 and calibration usage retained',
|
||||
'failure_rate':'greater than 2% at every 100 terminal jobs and final partial batch; no replacement/retry; incomplete corpus cannot freeze candidates'}
|
||||
save(ROOT/'contract.json',contract)
|
||||
save(ROOT/'public/preparation.json',{'status':'MISSING_FINGERPRINT_CONTRACT_FROZEN','parts':624,
|
||||
'reusable_tet_caches':sum(j['tet_cache'] is not None for j in jobs),'missing_tet_prefix_reads':sum(j['tet_cache'] is None for j in jobs),
|
||||
'development_parts':24,'k_per_route':3,'contract_sha256':sha(ROOT/'contract.json'),'plan_sha256':sha(private/'plan.json'),
|
||||
'thresholds_sha256':sha(calibration),'legacy_point_fingerprints_not_substituted':True,'registration_started':False})
|
||||
print((ROOT/'public/preparation.json').read_text())
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Guarded missing fingerprint extraction; no retries; count follows a complete corpus."""
|
||||
from pathlib import Path
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
import resource
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import psutil
|
||||
from g1_candidate_fingerprints import ROOT, check_contract
|
||||
from s2_e1_supervisor import supervise, save, disk_usage, stop_group
|
||||
|
||||
def guard(index):
|
||||
c=check_contract(ROOT);plan=json.loads((ROOT/'custodian/plan.json').read_text());n=1
|
||||
job=ROOT/'jobs'/f'job-{index:02d}';job.mkdir(parents=True,exist_ok=True)
|
||||
limit=n*c['per_pair_wall_seconds']
|
||||
r=supervise([sys.executable,'tools/g1_candidate_fingerprints.py',str(index)],job,'worker',
|
||||
dict(total_cpu_seconds=limit+1,job_cpu_seconds=limit,job_wall_seconds=limit,
|
||||
tree_rss_bytes=c['per_worker_rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=2),poll=.25)
|
||||
save(job/'guard-result.json',{'worker':r,'supervisor_cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2])})
|
||||
return 0 if r['status']=='PASS' else 1
|
||||
|
||||
def terminate_workers(active):
|
||||
# Guards stay alive to reap killed workers and write failure receipts.
|
||||
for p,_ in active.values():
|
||||
try:
|
||||
for child in psutil.Process(p.pid).children():stop_group(child.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
|
||||
def run():
|
||||
c=check_contract(ROOT);plan=json.loads((ROOT/'custodian/plan.json').read_text())
|
||||
lock=(ROOT/'batch.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (ROOT/'batch-started.json').exists():raise FileExistsError('Batch already attempted; no automatic restart')
|
||||
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
||||
save(ROOT/'batch-started.json',{'pid':os.getpid(),'started_unix':time.time(),'jobs':len(plan['jobs']),'parts':len(plan['jobs']),'workers':c['workers'],'contract_sha256':__import__('hashlib').sha256((ROOT/'contract.json').read_bytes()).hexdigest()})
|
||||
started=time.monotonic();active={};done=[];next_index=0;reason=None;peak_rss=0
|
||||
def interrupted(*_):
|
||||
nonlocal reason
|
||||
reason='coordinator_interrupted'
|
||||
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
|
||||
try:
|
||||
while active or (next_index<len(plan['jobs']) and reason is None):
|
||||
while reason is None and len(active)<c['workers'] and next_index<len(plan['jobs']):
|
||||
i=next_index;next_index+=1;path=ROOT/'jobs'/f'job-{i:02d}';path.mkdir(parents=True,exist_ok=True)
|
||||
log=(path/'guard.log').open('x')
|
||||
p=subprocess.Popen([sys.executable,__file__,'--guard',str(i)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
||||
active[i]=(p,log)
|
||||
print(json.dumps({'event':'STARTED','job':i,'guard_pid':p.pid}),flush=True)
|
||||
for i,(p,log) in list(active.items()):
|
||||
code=p.poll()
|
||||
if code is not None:
|
||||
log.close();path=ROOT/'jobs'/f'job-{i:02d}'/'guard-result.json'
|
||||
result=json.loads(path.read_text()) if path.exists() else {'worker':{'status':'GUARD_FAILED','cpu_seconds':0},'supervisor_cpu_seconds':0}
|
||||
done.append({'index':i,'returncode':code,**result});del active[i]
|
||||
if len(done)%100==0 and sum(r['worker']['status']!='PASS' for r in done)/len(done)>.02:
|
||||
reason='failure_rate_over_2_percent'
|
||||
print(json.dumps({'event':'FINISHED','job':i,'status':result['worker']['status'],'completed':len(done)}),flush=True)
|
||||
# Evaluate failures at every 100 terminal jobs and at the final partial batch.
|
||||
if done and len(done)==len(plan['jobs']):
|
||||
if sum(r['worker']['status']!='PASS' for r in done)/len(done)>.02:reason='failure_rate_over_2_percent'
|
||||
if time.monotonic()-started>=c['wall_seconds']:reason='global_wall_limit'
|
||||
if disk_usage(ROOT)>=c['disk_bytes']:reason='global_disk_limit'
|
||||
live_cpu=live_rss=0
|
||||
for p,_ in active.values():
|
||||
try:
|
||||
for q in [psutil.Process(p.pid)]+psutil.Process(p.pid).children(recursive=True):
|
||||
try:
|
||||
t=q.cpu_times();live_cpu+=t.user+t.system+t.children_user+t.children_system;live_rss+=q.memory_info().rss
|
||||
except psutil.NoSuchProcess:pass
|
||||
except psutil.NoSuchProcess:pass
|
||||
peak_rss=max(peak_rss,live_rss)
|
||||
completed_cpu=sum(r['worker']['cpu_seconds']+r['supervisor_cpu_seconds'] for r in done)
|
||||
if completed_cpu+live_cpu>=c['worker_cpu_budget_seconds']:reason='global_cpu_limit'
|
||||
if live_rss>=c['aggregate_rss_bytes']:reason='aggregate_rss_limit'
|
||||
if reason:terminate_workers(active)
|
||||
save(ROOT/'progress.json',{'completed_jobs':len(done),'active_jobs':list(active),'next_job':next_index,
|
||||
'completed_worker_cpu_seconds':sum(r['worker']['cpu_seconds'] for r in done),
|
||||
'live_tree_cpu_seconds':live_cpu,'active_tree_rss_bytes':live_rss,'stop_reason':reason,'elapsed_seconds':time.monotonic()-started})
|
||||
if active:time.sleep(.5)
|
||||
finally:
|
||||
terminate_workers(active)
|
||||
for p,log in active.values():
|
||||
try:p.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:stop_group(p.pid);p.wait()
|
||||
log.close()
|
||||
failures=sum(r['worker']['status']!='PASS' for r in done)
|
||||
passed=reason is None and len(done)==len(plan['jobs']) and failures==0
|
||||
summary={'status':'ALL_FINGERPRINTS_COMPLETE' if passed else 'FINGERPRINTS_INCOMPLETE',
|
||||
'jobs':done,'failed_jobs':failures,'failure_rate_at_final_checkpoint':failures/len(done) if done else None,
|
||||
'stop_reason':reason,'worker_cpu_seconds':sum(r['worker']['cpu_seconds'] for r in done),
|
||||
'guard_cpu_seconds':sum(r['supervisor_cpu_seconds'] for r in done),
|
||||
'coordinator_cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
||||
'wall_seconds':time.monotonic()-started,'peak_aggregate_tree_rss_bytes':peak_rss,
|
||||
'disk_bytes':disk_usage(ROOT),'thresholds_frozen_before_extraction':True,'candidate_generation':False,'no_retries':True}
|
||||
save(ROOT/'batch-complete.json' if passed else ROOT/'batch-incomplete.json',summary)
|
||||
print(json.dumps({k:v for k,v in summary.items() if k!='jobs'}),flush=True)
|
||||
return 0 if passed else 1
|
||||
|
||||
if __name__=='__main__':
|
||||
ap=argparse.ArgumentParser();ap.add_argument('--guard',type=int);a=ap.parse_args()
|
||||
sys.exit(guard(a.guard) if a.guard is not None else run())
|
||||
@@ -0,0 +1,33 @@
|
||||
import unittest
|
||||
import numpy as np
|
||||
from scipy.spatial.transform import Rotation
|
||||
from g1_candidate_fingerprints import fingerprint
|
||||
from count_g1_candidates import route_ranks
|
||||
|
||||
class FingerprintChecks(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.x=np.array([[0.,0,0],[3.,0,0],[0,2.,0],[0,0,1.]]);self.t=np.array([[0,1,2,3]])
|
||||
def test_exact_tetra_moments_and_registration_rg(self):
|
||||
g,r=fingerprint(self.x,self.t,123)
|
||||
cov=np.array([[27,-6,-3],[-6,12,-2],[-3,-2,3]])/80
|
||||
self.assertAlmostEqual(r['volume'],1)
|
||||
np.testing.assert_allclose(r['volume_center'],[.75,.5,.25])
|
||||
np.testing.assert_allclose(r['exact_volume_covariance_eigenvalues'],np.linalg.eigvalsh(cov))
|
||||
self.assertAlmostEqual(float(g['rg'])**2,2.625)
|
||||
def test_scale_rotation_mirror_invariance(self):
|
||||
_,a=fingerprint(self.x,self.t,123);r=Rotation.from_rotvec([.3,.5,-.7]).as_matrix()
|
||||
for parity in [-1,1]:
|
||||
y=2*self.x@(r@np.diag([parity,1,1])).T+[7,-3,2]
|
||||
_,b=fingerprint(y,self.t,123)
|
||||
np.testing.assert_allclose(a['shape_vector'],b['shape_vector'],atol=1e-12,rtol=1e-12)
|
||||
self.assertAlmostEqual(b['physical_vector'][-1]-a['physical_vector'][-1],np.log(2))
|
||||
def test_exact_hash_independent_of_vertex_numbering(self):
|
||||
_,a=fingerprint(self.x,self.t,123);order=np.array([2,0,3,1]);inv=np.argsort(order)
|
||||
_,b=fingerprint(self.x[order],inv[self.t[:,::-1]],123)
|
||||
self.assertEqual(a['exact_boundary_mesh_sha256'],b['exact_boundary_mesh_sha256'])
|
||||
def test_competition_ties_and_same_key_exclusion(self):
|
||||
r,_,s=route_ranks([[0,8],[1,8],[-1,8],[0,8]],['a','b','c','a'],[0,1,2,3])
|
||||
self.assertEqual(r[0,1],1);self.assertEqual(r[0,2],1);self.assertGreater(r[0,3],3)
|
||||
self.assertEqual(s['zero_iqr_columns'],[1])
|
||||
|
||||
if __name__=='__main__':unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user