Count 2012 frozen candidates and start bounded registration with first-100 checkpoint

This commit is contained in:
Frank14f committed 2026-11-14 22:07:12 +08:00
1 parent ff439e81fc
commit d4e26d48d0
17 files changed
+16816 -3

No files matched your search

+51
View File
@@ -0,0 +1,51 @@
"""Register frozen candidates and apply the fixed engineering rule, no field labels."""
from pathlib import Path
import argparse,hashlib,json,os,resource,time
import numpy as np
from g1_registration_cost import register
from g1_candidate_fingerprints import check_contract
from s2_e1_supervisor import save
ROOT=Path(os.environ.get('MELTMIND_G1_REGISTRATION_ROOT','artifacts/s2-g1-registration-2026-11-14'))
def engineering_rule(routes,rga,rgb,thresholds):
ratio=max(rga,rgb)/min(rga,rgb);mean=(rga+rgb)/2;report={};scores=[]
for name in ['physical','scale_free']:
rows=[r for r in routes if r['route']==name]
if len(rows)!=2 or {r['parity'] for r in rows}!={-1,1}:raise ValueError('Missing det branch')
branches={str(r['parity']):r['residual_max_two_way_area_pps_p95']/(mean if name=='physical' else 1.) for r in rows}
best=min(branches,key=lambda k:(branches[k],k));other='-1' if best=='1' else '1';value=branches[best]
if not np.isfinite(value) or value<0 or thresholds[name]<=0:raise ValueError('Invalid residual/threshold')
eligible=name=='scale_free' or ratio<=1.05;score=value/thresholds[name]
label=('det'+('+' if best=='1' else '-')+' preferred') if eligible and value<thresholds[name] and branches[other]>2*value else 'branch-ambiguous (near-symmetric)'
report[name]={'best_residual':value,'size_gate_pass':eligible,'threshold':thresholds[name],'normalized_score':score,
'branch_residuals':branches,'branch_label':label}
if eligible:scores.append(score)
score=min(scores)
return {'routes':report,'rg_ratio':ratio,'combined_score':score,
'engineering_status':'LINK_UNDER_FIXED_RULE' if score<=1 else 'BAND_REVIEW_REQUIRED' if score<=2 else 'OUTSIDE_FIXED_RULE',
'not_a_relatedness_or_recall_certificate':True}
def main():
ap=argparse.ArgumentParser();ap.add_argument('index',type=int);args=ap.parse_args();c=check_contract(ROOT)
plan=json.loads((ROOT/'custodian/plan.json').read_text());pair=plan['jobs'][args.index];start=time.monotonic();g=[]
for case in [pair['a'],pair['b']]:
cache=plan['geometry_cache'][case];p=Path(cache['path'])
if hashlib.sha256(p.read_bytes()).hexdigest()!=cache['sha256']:raise ValueError('Frozen geometry cache changed')
with np.load(p,allow_pickle=False) as z:
if set(z.files)!={'points','boundary','center','rg'}:raise ValueError('Unexpected non-geometry cache member')
g.append({k:z[k] for k in z.files})
loaded=time.monotonic()-start;registration_start=time.monotonic()
routes=register(g[0],g[1],int(pair['a'].split('_')[0]),int(pair['b'].split('_')[0]))
decision=engineering_rule(routes,float(g[0]['rg']),float(g[1]['rg']),c['thresholds'])
dest=ROOT/'custodian/results'/f'pair-{args.index:04d}.json'
if dest.exists():raise FileExistsError('Result already exists; no retry')
save(dest,{'index':args.index,'pair':pair,'routes':routes,'engineering_rule':decision,
'cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),'wall_seconds':time.monotonic()-start,
'geometry_load_wall_seconds':loaded,'registration_wall_seconds':time.monotonic()-registration_start,
'max_starts_per_branch':max(len(r['starts']) for r in routes),
'peak_rss_bytes':resource.getrusage(resource.RUSAGE_SELF).ru_maxrss*1024,'simulation_fields_read':False,
'geometric_relationship':'UNREVIEWED','components_frozen':False})
print(json.dumps({'index':args.index,'status':'REGISTRATION_COMPLETE','engineering_status':decision['engineering_status']}),flush=True)
if __name__=='__main__':main()
+26
View File
@@ -0,0 +1,26 @@
"""First fixed-rank block cost projection; no fitting or changes to k/thresholds."""
from statistics import mean
def first_block_projection(records,total_pairs):
if len(records)!=100:raise ValueError('First checkpoint requires exactly 100 terminal pairs')
cpu=[r['cpu_seconds'] for r in records];good=[r for r in records if r['status']=='PASS']
failed=len(records)-len(good);starts=sum(r.get('max_starts_per_branch',14)>=14 for r in records)
central=mean(cpu)*total_pairs/3600
successes=mean(r['cpu_seconds'] for r in good)*total_pairs/3600 if good else central
loads=[r.get('geometry_load_wall_seconds',0) for r in good]
load_delta=max(loads)-mean(loads) if loads else 0
sensitivity=max(central,successes)+load_delta*total_pairs/3600
by_cohort={str(heavy):[r['cpu_seconds'] for r in good if (r.get('max_starts_per_branch',0)>=14)==heavy] for heavy in [False,True]}
reasons=[]
if failed/100>.02:reasons.append('failure_rate_over_2_percent')
if starts/100>.85:reasons.append('fourteen_or_more_start_share_over_85_percent')
if max(central,sensitivity)>100:reasons.append('projected_worker_cpu_over_100_hours')
return {'pairs':100,'total_queue_pairs':total_pairs,'mean_worker_cpu_seconds':mean(cpu),
'projected_worker_cpu_hours':central,'successful_pair_mean_projection_hours':successes,
'high_load_as_cpu_sensitivity_hours':sensitivity,'fourteen_or_more_start_pairs':starts,
'fourteen_or_more_start_share':starts/100,'failed_pairs':failed,
'worker_cpu_by_start_cohort':{k:mean(v) if v else None for k,v in by_cohort.items()},
'geometry_load_wall_seconds_mean':mean(loads) if loads else None,'geometry_load_wall_seconds_max':max(loads) if loads else None,
'cache_note':'OS cache not flushed; fingerprint preparation warms it. Measured NPZ/hash load included in worker CPU. High-load sensitivity charges max-minus-mean observed load wall time as extra CPU, not a measured cold-cache guarantee.',
'sampling_note':'First 100 in fixed min-rank order; not a random cost sample or certified upper bound',
'decision':'STOP_AND_REPORT' if reasons else 'CONTINUE','stop_reasons':reasons}
@@ -0,0 +1,48 @@
"""Freeze the counted <=4000 candidate queue under the already granted G1 budget."""
from pathlib import Path
import hashlib,json
from g1_register_candidates import ROOT
from s2_e1_supervisor import save
def sha(p):return hashlib.sha256(Path(p).read_bytes()).hexdigest()
def main():
source=Path('artifacts/s2-g1-candidates-2026-11-14');count=source/'public/candidate-count.json'
summary=json.loads(count.read_text());candidates=source/'custodian/candidate-pairs.json'
assert summary['status']=='CANDIDATES_COUNTED_NOT_REGISTERED' and summary['union_pairs']<=4000
assert sha(candidates)==summary['candidate_sha256']
pairs=json.loads(candidates.read_text())['pairs'];assert len(pairs)==summary['union_pairs']
assert pairs==sorted(pairs,key=lambda r:(r['min_rank'],r['a'],r['b']))
thresholds=Path('artifacts/s2-g1-copy-calibration-2026-11-14/public/calibration-thresholds.json')
assert sha(thresholds)==summary['calibration_thresholds_sha256']
t=json.loads(thresholds.read_text());assert t['status']=='CALIBRATION_THRESHOLDS_FROZEN'
ROOT.mkdir(exist_ok=False);private=ROOT/'custodian';private.mkdir(mode=0o700);(private/'results').mkdir();(ROOT/'public').mkdir()
cases=sorted({r[k] for r in pairs for k in ['a','b']});cache={}
for case in cases:
path=source/'custodian/fingerprints'/case/'geometry.npz';cache[case]={'path':str(path),'sha256':sha(path)}
save(private/'plan.json',{'jobs':pairs,'geometry_cache':cache,'candidate_sha256':sha(candidates),'ordering':'frozen min_rank, a, b'})
(private/'plan.json').chmod(0o600)
code=['prepare_g1_candidate_registration','g1_register_candidates','run_g1_registered_candidates','g1_registration_checkpoint',
'g1_registration_cost','g1_candidate_fingerprints','s2_e1_supervisor','audit_samples','boundary_distance']
c={'authorizing_rulings':['1794661944.308269','1794662380.787319','1794664166.242149','1794664685.158429','1794664857.650469'],
'count_report_before_registration':'1794664837.008399','pairs':len(pairs),'thresholds':t['thresholds'],
'code_sha256':{f'tools/{x}.py':sha(f'tools/{x}.py') for x in code},
'input_sha256':{str(p):sha(p) for p in [private/'plan.json',count,candidates,thresholds]},
'workers':16,'per_pair_wall_seconds':1800,'per_worker_rss_bytes':1342177280,
'aggregate_rss_bytes':20*1024**3,'worker_cpu_budget_seconds':100*3600,'wall_seconds':8*3600,'disk_bytes':20*1024**3,
'protocol':'Unchanged g1_registration_cost.register; both det branches, both routes, fixed 2000 area PPS, full triangle two-way P95',
'engineering_rule':'physical raw/mean Rg with ratio<=1.05, scale_free unchanged; best eligible route/threshold <=1 link; (1,2] band review; >2 outside rule, not certified unrelated',
'failure_policy':'retain failed attempts; check >2% at every 100 terminal jobs and final partial batch; no retries/replacements',
'first_100_checkpoint':'launch only first 100 fixed-rank pairs until all terminal; report mean CPU and >=14-start share and load-cache sensitivity; stop if projection>100 CPU-h or >=14-start share>85%',
'branch_label':'name better det branch only if residual strictly below route tau and other branch >2 times larger; otherwise branch-ambiguous (near-symmetric)',
'budget_accounting':'worker and supervision recorded separately; aggregate stop conservatively also includes supervision',
'simulation_fields_training_gpu':False,'components_frozen':False,
'pending_before_component_freeze':['anonymous threshold-band review; unresolved merges per ruling'],
'calibration_mirror_flags_closed_by':'1794664685.158429'}
save(ROOT/'contract.json',c)
save(ROOT/'public/preparation.json',{'status':'REGISTRATION_QUEUE_FROZEN_NOT_STARTED','pairs':len(pairs),'unique_parts':len(cases),
'contract_sha256':sha(ROOT/'contract.json'),'plan_sha256':sha(private/'plan.json'),'candidate_sha256':sha(candidates),
'thresholds_sha256':sha(thresholds),'workers':16,'worker_cpu_hours':100,'per_pair_minutes':30,
'count_previously_reported':True,'no_new_geometry_extraction':True,'no_new_approval_gate':True})
print((ROOT/'public/preparation.json').read_text())
if __name__=='__main__':main()
+117
View File
@@ -0,0 +1,117 @@
"""Frozen min-rank candidate registration; bounded workers, CPU, disk and failure checkpoints."""
from pathlib import Path
import argparse
import fcntl
import json
import os
import resource
import signal
import subprocess
import sys
import time
import psutil
from g1_register_candidates import ROOT, check_contract
from s2_e1_supervisor import supervise, save, disk_usage, stop_group
from g1_registration_checkpoint import first_block_projection
def guard(index):
c=check_contract(ROOT);plan=json.loads((ROOT/'custodian/plan.json').read_text());n=1
job=ROOT/'jobs'/f'job-{index:02d}';job.mkdir(parents=True,exist_ok=True)
limit=n*c['per_pair_wall_seconds']
r=supervise([sys.executable,'tools/g1_register_candidates.py',str(index)],job,'worker',
dict(total_cpu_seconds=limit+1,job_cpu_seconds=limit,job_wall_seconds=limit,
tree_rss_bytes=c['per_worker_rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=2),poll=.25)
save(job/'guard-result.json',{'worker':r,'supervisor_cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2])})
return 0 if r['status']=='PASS' else 1
def terminate_workers(active):
# Guards stay alive to reap killed workers and write failure receipts.
for p,_ in active.values():
try:
for child in psutil.Process(p.pid).children():stop_group(child.pid)
except psutil.NoSuchProcess:pass
def run():
c=check_contract(ROOT);plan=json.loads((ROOT/'custodian/plan.json').read_text())
lock=(ROOT/'batch.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (ROOT/'batch-started.json').exists():raise FileExistsError('Batch already attempted; no automatic restart')
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
save(ROOT/'batch-started.json',{'pid':os.getpid(),'started_unix':time.time(),'jobs':len(plan['jobs']),'pairs':len(plan['jobs']),'workers':c['workers'],'contract_sha256':__import__('hashlib').sha256((ROOT/'contract.json').read_bytes()).hexdigest()})
started=time.monotonic();active={};done=[];next_index=0;reason=None;peak_rss=0;first_checkpoint_done=False
def interrupted(*_):
nonlocal reason
reason='coordinator_interrupted'
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
try:
while active or (next_index<len(plan['jobs']) and reason is None):
while reason is None and len(active)<c['workers'] and next_index<len(plan['jobs']) and (first_checkpoint_done or next_index<100):
i=next_index;next_index+=1;path=ROOT/'jobs'/f'job-{i:02d}';path.mkdir(parents=True,exist_ok=True)
log=(path/'guard.log').open('x')
p=subprocess.Popen([sys.executable,__file__,'--guard',str(i)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
active[i]=(p,log)
print(json.dumps({'event':'STARTED','job':i,'guard_pid':p.pid}),flush=True)
for i,(p,log) in list(active.items()):
code=p.poll()
if code is not None:
log.close();path=ROOT/'jobs'/f'job-{i:02d}'/'guard-result.json'
result=json.loads(path.read_text()) if path.exists() else {'worker':{'status':'GUARD_FAILED','cpu_seconds':0},'supervisor_cpu_seconds':0}
done.append({'index':i,'returncode':code,**result});del active[i]
if len(done)%100==0 and sum(r['worker']['status']!='PASS' for r in done)/len(done)>.02:
reason='failure_rate_over_2_percent'
print(json.dumps({'event':'FINISHED','job':i,'status':result['worker']['status'],'completed':len(done)}),flush=True)
if not first_checkpoint_done and len(done)==100:
checkpoint_rows=[]
for record in sorted(done,key=lambda r:r['index']):
row={'status':record['worker']['status'],'cpu_seconds':record['worker']['cpu_seconds']}
path=ROOT/'custodian/results'/f"pair-{record['index']:04d}.json"
if path.exists():
data=json.loads(path.read_text());row.update(max_starts_per_branch=data['max_starts_per_branch'],geometry_load_wall_seconds=data['geometry_load_wall_seconds'])
checkpoint_rows.append(row)
checkpoint=first_block_projection(checkpoint_rows,len(plan['jobs']))
save(ROOT/'public/first-100-checkpoint.json',checkpoint);first_checkpoint_done=True
if checkpoint['decision']=='STOP_AND_REPORT':reason='first_100_projection_stop'
print(json.dumps({'event':'FIRST_100_CHECKPOINT',**checkpoint}),flush=True)
# Evaluate failures at every 100 terminal jobs and at the final partial batch.
if done and len(done)==len(plan['jobs']):
if sum(r['worker']['status']!='PASS' for r in done)/len(done)>.02:reason='failure_rate_over_2_percent'
if time.monotonic()-started>=c['wall_seconds']:reason='global_wall_limit'
if disk_usage(ROOT)>=c['disk_bytes']:reason='global_disk_limit'
live_cpu=live_rss=0
for p,_ in active.values():
try:
for q in [psutil.Process(p.pid)]+psutil.Process(p.pid).children(recursive=True):
try:
t=q.cpu_times();live_cpu+=t.user+t.system+t.children_user+t.children_system;live_rss+=q.memory_info().rss
except psutil.NoSuchProcess:pass
except psutil.NoSuchProcess:pass
peak_rss=max(peak_rss,live_rss)
completed_cpu=sum(r['worker']['cpu_seconds']+r['supervisor_cpu_seconds'] for r in done)
if completed_cpu+live_cpu>=c['worker_cpu_budget_seconds']:reason='global_cpu_limit'
if live_rss>=c['aggregate_rss_bytes']:reason='aggregate_rss_limit'
if reason:terminate_workers(active)
save(ROOT/'progress.json',{'completed_jobs':len(done),'active_jobs':list(active),'next_job':next_index,
'completed_worker_cpu_seconds':sum(r['worker']['cpu_seconds'] for r in done),
'live_tree_cpu_seconds':live_cpu,'active_tree_rss_bytes':live_rss,'stop_reason':reason,'elapsed_seconds':time.monotonic()-started})
if active:time.sleep(.5)
finally:
terminate_workers(active)
for p,log in active.values():
try:p.wait(timeout=10)
except subprocess.TimeoutExpired:stop_group(p.pid);p.wait()
log.close()
failures=sum(r['worker']['status']!='PASS' for r in done)
passed=reason is None and len(done)==len(plan['jobs']) and failures==0
summary={'status':'ALL_REGISTRATIONS_COMPLETE' if passed else 'REGISTRATION_INCOMPLETE',
'jobs':done,'failed_jobs':failures,'failure_rate_at_final_checkpoint':failures/len(done) if done else None,
'stop_reason':reason,'worker_cpu_seconds':sum(r['worker']['cpu_seconds'] for r in done),
'guard_cpu_seconds':sum(r['supervisor_cpu_seconds'] for r in done),
'coordinator_cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
'wall_seconds':time.monotonic()-started,'peak_aggregate_tree_rss_bytes':peak_rss,
'disk_bytes':disk_usage(ROOT),'thresholds_frozen_before_registration':True,'candidate_generation_completed':True,'no_retries':True}
save(ROOT/'batch-complete.json' if passed else ROOT/'batch-incomplete.json',summary)
print(json.dumps({k:v for k,v in summary.items() if k!='jobs'}),flush=True)
return 0 if passed else 1
if __name__=='__main__':
ap=argparse.ArgumentParser();ap.add_argument('--guard',type=int);a=ap.parse_args()
sys.exit(guard(a.guard) if a.guard is not None else run())
+34
View File
@@ -0,0 +1,34 @@
import unittest
from g1_register_candidates import engineering_rule
from g1_registration_checkpoint import first_block_projection
def routes(p1,p2,s1,s2):
return [{'route':route,'parity':parity,'residual_max_two_way_area_pps_p95':value}
for route,parity,value in [('physical',1,p1),('physical',-1,p2),('scale_free',1,s1),('scale_free',-1,s2)]]
class RuleChecks(unittest.TestCase):
def test_physical_mean_rg_and_best_branch(self):
r=engineering_rule(routes(1,.1,1,2),10,10,{'physical':.02,'scale_free':.02})
self.assertEqual(r['engineering_status'],'LINK_UNDER_FIXED_RULE');self.assertAlmostEqual(r['combined_score'],.5)
def test_scale_gate_excludes_even_perfect_physical_fit(self):
r=engineering_rule(routes(0,0,.03,.04),1,2,{'physical':.02,'scale_free':.02})
self.assertFalse(r['routes']['physical']['size_gate_pass']);self.assertEqual(r['engineering_status'],'BAND_REVIEW_REQUIRED')
def test_threshold_and_band_endpoints(self):
for x,status in [(1,'LINK_UNDER_FIXED_RULE'),(2,'BAND_REVIEW_REQUIRED'),(2.01,'OUTSIDE_FIXED_RULE')]:
r=engineering_rule(routes(x,x,x,x),1,1,{'physical':1,'scale_free':1});self.assertEqual(r['engineering_status'],status)
def test_branch_label_requires_strict_threshold_and_twofold_gap(self):
r=engineering_rule(routes(.1,.15,.1,.15),1,1,{'physical':1,'scale_free':1})
self.assertEqual(r['routes']['physical']['branch_label'],'branch-ambiguous (near-symmetric)')
r=engineering_rule(routes(.1,.21,.1,.21),1,1,{'physical':1,'scale_free':1})
self.assertEqual(r['routes']['physical']['branch_label'],'det+ preferred')
def test_first_100_cost_and_start_stop_boundaries(self):
rows=[{'status':'PASS','cpu_seconds':90,'max_starts_per_branch':4,'geometry_load_wall_seconds':.2} for _ in range(100)]
self.assertEqual(first_block_projection(rows,2012)['decision'],'CONTINUE')
for r in rows[:85]:r['max_starts_per_branch']=14
self.assertEqual(first_block_projection(rows,2012)['decision'],'CONTINUE')
rows[85]['max_starts_per_branch']=14
self.assertIn('fourteen_or_more_start_share_over_85_percent',first_block_projection(rows,2012)['stop_reasons'])
for r in rows:r['cpu_seconds']=200
self.assertIn('projected_worker_cpu_over_100_hours',first_block_projection(rows,2012)['stop_reasons'])
if __name__=='__main__':unittest.main(verbosity=2)