Freeze independent exact-reference and count-cost gate continuation
This commit is contained in:
1 parent
48748f59f6
commit
8a7965d202
16 files changed
+510
No files matched your search
@@ -0,0 +1,75 @@
|
||||
"""Wait for the sole ten terminal, then supervised reference and cost gates."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
import psutil
|
||||
from s2_recovery_supervisor import save,supervise,stop_group
|
||||
from s3b_runtime import read,sha
|
||||
from s3b_v04_gates import verify_gate_config
|
||||
|
||||
|
||||
def run(root):
|
||||
lock=(root/'execution.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
c=verify_gate_config(root);ten=Path(c['ten_root']);external=Path(c['stage_guard_root'])
|
||||
if (root/'started.json').exists():raise FileExistsError('Never restart gate chain')
|
||||
os.sched_setaffinity(0,[30,31])
|
||||
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),code_freeze=read(root/'code-freeze.json')))
|
||||
stopped=False
|
||||
def stop(*_):
|
||||
nonlocal stopped
|
||||
stopped=True
|
||||
for p in psutil.Process(os.getpid()).children():stop_group(p.pid)
|
||||
signal.signal(signal.SIGTERM,stop);signal.signal(signal.SIGINT,stop)
|
||||
guards=[];reason=None;terminal=None;active=None
|
||||
extra=c['additional_synthetic_CPU_s']
|
||||
def cumulative():return terminal['cumulative_worker_CPU_s']+extra+sum(g['cpu_seconds'] for g in guards)
|
||||
try:
|
||||
while not (ten/'complete.json').exists() or not (external/'complete.json').exists():
|
||||
if stopped:raise RuntimeError('Gate chain interrupted')
|
||||
if (external/'failure.json').exists():raise RuntimeError('External stage guard failed')
|
||||
save(root/'progress.json',dict(stage='WAITING_ORIGINAL_TEN',unix=time.time(),new_scientific_workers=0))
|
||||
time.sleep(5)
|
||||
terminal=read(ten/'complete.json');ext=read(external/'complete.json')
|
||||
if terminal['status']!='TEN_PASS_AWAITING_REFERENCE_AND_COST' or ext['status']!='OBSERVED_QUEUE_TERMINAL':
|
||||
reason='TEN_OR_STAGE_CAP_GATE_FAILED';return 1
|
||||
for kind in ['reference','census']:
|
||||
verify_gate_config(root)
|
||||
if stopped:reason='GATE_CHAIN_INTERRUPTED';return 1
|
||||
if cumulative()+1800+7200>72000:reason='GATE_FULL_JOB_RESERVATION';return 1
|
||||
dest=root/kind;dest.mkdir(exist_ok=False);active=kind
|
||||
save(root/'progress.json',dict(stage=kind.upper(),cumulative_settled_CPU_s=cumulative()))
|
||||
w=supervise(['taskset','-c','16',sys.executable,'tools/s3b_v04_gates.py',str(root),kind],dest,'worker',
|
||||
dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=1800,
|
||||
tree_rss_bytes=4*2**30,disk_bytes=2**30,max_processes=1),poll=.2)
|
||||
guards.append(w);active=None
|
||||
if w['status']!='PASS' or w['monitor_errors']:reason=kind.upper()+'_GUARD_FAILED';return 1
|
||||
result=read(dest/'output/result.json')
|
||||
if kind=='reference' and result['status']!='PASS':reason='EXACT_REFERENCE_GATE_FAILED';return 1
|
||||
if kind=='census':
|
||||
terms=result['projection_terms']
|
||||
projected=terms['variable_projection_CPU_s']+terms['fixed_projection_CPU_s']+cumulative()+7200
|
||||
gate=dict(status='PASS' if projected<=72000 else 'FAIL',projected_total_worker_CPU_s=projected,
|
||||
cap_CPU_s=72000,postfit_reserve_CPU_s=7200,cumulative_actual_CPU_s=cumulative(),
|
||||
projection_terms=terms,reference_gate_sha256=sha(root/'reference/output/result.json'),
|
||||
census_sha256=sha(dest/'output/result.json'),full617_launched=False)
|
||||
save(root/'cost-gate.json',gate)
|
||||
if gate['status']!='PASS':reason='COST_PROJECTION_GATE_FAILED';return 1
|
||||
return 0
|
||||
except Exception as exc:
|
||||
reason='GATE_CHAIN_EXCEPTION'
|
||||
save(root/'failure.json',dict(type=type(exc).__name__,reason=str(exc),active=active))
|
||||
raise
|
||||
finally:
|
||||
save(root/'complete.json',dict(status='GATES_PASS_READY_FOR_FULL617' if reason is None else 'STOPPED',
|
||||
reason=reason,guards=guards,ten_complete_sha256=sha(ten/'complete.json') if (ten/'complete.json').exists() else None,
|
||||
additional_synthetic_CPU_s=extra,cumulative_worker_CPU_s=cumulative() if terminal else None,
|
||||
account_complete=active is None,full617_launched=False,no_solver_switch=True,labels_read=False,
|
||||
chain_CPU_s=time.process_time()))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();raise SystemExit(run(a.root))
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Independent exact-reference and count-only cost gates; no label access."""
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import time
|
||||
import traceback
|
||||
import numpy as np
|
||||
from scipy.stats import kendalltau
|
||||
from s3b_runtime import read,sha,checked,restrict_prediction_io
|
||||
from naive_ordering_math import cdf_interval
|
||||
from s2_recovery_supervisor import save
|
||||
|
||||
|
||||
def compare_fields(exact,batch):
|
||||
for key in ['raw_ids','mass']:
|
||||
if not np.array_equal(exact[key],batch[key]):raise ValueError('Full tet support mismatch: '+key)
|
||||
n=len(exact['raw_ids'])
|
||||
for field in [exact,batch]:
|
||||
if (field['arrival'].shape!=(n,) or field['remaining'].shape!=(n,) or
|
||||
not np.isfinite(field['arrival']).all() or np.any(field['remaining']!=0)):
|
||||
raise ValueError('Reference comparison needs all finite full-tet arrivals and zero remaining')
|
||||
tau=float(kendalltau(exact['arrival'],batch['arrival'],variant='b',nan_policy='raise').statistic)
|
||||
left_exact,_=cdf_interval(exact['arrival'],exact['mass'])
|
||||
left_batch,_=cdf_interval(batch['arrival'],batch['mass'])
|
||||
return dict(status='PASS' if np.isfinite(tau) and tau>=.999 else 'FAIL',nodes=n,
|
||||
tau_b=tau if np.isfinite(tau) else None,threshold=.999,
|
||||
max_strict_volume_CDF_delta=float(np.max(abs(left_exact-left_batch))),
|
||||
exact_unique_arrivals=len(np.unique(exact['arrival'])),batch_unique_arrivals=len(np.unique(batch['arrival'])),
|
||||
tau_measure='one vote per full tet node',CDF_measure='strict less-than, original full tet mass',
|
||||
CDF_delta_gate=False,no_solver_switch=True)
|
||||
|
||||
|
||||
def query_count(edges,faces,dw):
|
||||
n=len(dw)
|
||||
if not np.isfinite(dw).all() or np.any(dw<0):raise ValueError('Invalid Dw')
|
||||
e=np.asarray(edges);f=np.asarray(faces)
|
||||
wall=np.sort(np.concatenate([f[:,[0,1]],f[:,[0,2]],f[:,[1,2]]]),axis=1)
|
||||
keys=e[:,0].astype(np.int64)*n+e[:,1]
|
||||
wallkeys=np.unique(wall[:,0].astype(np.int64)*n+wall[:,1])
|
||||
is_wall=np.isin(keys,wallkeys);both=(dw[e]==0).all(axis=1)
|
||||
if not both[is_wall].all():raise ValueError('Surface edge does not have zero-Dw endpoints')
|
||||
return int(np.count_nonzero(both&~is_wall))
|
||||
|
||||
|
||||
def projection_terms(measurements,total_nodes,total_queries,parts):
|
||||
if len(measurements)!=10 or any(r['status']!='COMPLETE' for r in measurements):
|
||||
raise ValueError('Ten complete measurements required')
|
||||
midpoint_rates=[r['costs']['midpoint_CPU_s']/r['exact_midpoint_queries'] for r in measurements if r['exact_midpoint_queries']>0]
|
||||
if total_queries>0 and not midpoint_rates:raise ValueError('No ten-part positive-query rate evidence')
|
||||
event_rate=max(r['costs']['events_CPU_s']/r['nodes'] for r in measurements)
|
||||
midpoint_rate=max(midpoint_rates,default=0.)
|
||||
# Fixed import/verification/geometry/output overhead is conservatively
|
||||
# bounded by the largest measured non-event/non-query overhead per part.
|
||||
overhead=max(max(0.,r['guard_CPU_s']-r['costs']['events_CPU_s']-
|
||||
(r['costs']['midpoint_CPU_s'] if r['exact_midpoint_queries']>0 else 0.)) for r in measurements)
|
||||
return dict(max_event_CPU_per_node=event_rate,max_midpoint_CPU_per_query=midpoint_rate,
|
||||
positive_query_measurements=len(midpoint_rates),max_fixed_overhead_CPU_per_part=overhead,
|
||||
total_nodes=total_nodes,total_queries=total_queries,parts=parts,
|
||||
variable_projection_CPU_s=1.25*(event_rate*total_nodes+midpoint_rate*total_queries),
|
||||
fixed_projection_CPU_s=overhead*parts,margin=1.25,
|
||||
note='Frozen full617 max-rate screening; includes ten in conservative projection, but winning ten are reused, never rerun.')
|
||||
|
||||
|
||||
def verify_gate_config(root):
|
||||
c=read(root/'contract.json');f=read(root/'code-freeze.json')
|
||||
if f['contract_sha256']!=sha(root/'contract.json') or not f['git_commit']:raise ValueError('Gates not Git-frozen')
|
||||
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():checked(p,h)
|
||||
return c
|
||||
|
||||
|
||||
def measure(root,kind):
|
||||
c=verify_gate_config(root);ten=Path(c['ten_root']);tc=read(ten/'contract.json')
|
||||
terminal=read(ten/'complete.json');chosen=read(ten/'selected-solver.json')
|
||||
if terminal['status']!='TEN_PASS_AWAITING_REFERENCE_AND_COST' or terminal['selected_solver']!=chosen['method']:
|
||||
raise ValueError('Ten not fully passed and frozen')
|
||||
if (Path(c['stage_guard_root'])/'trigger.json').exists():raise ValueError('External stage-cap failure')
|
||||
method=chosen['method'];dest=root/kind/'output';dest.mkdir(exist_ok=False)
|
||||
start=time.process_time();allowed=[tc['roster']]
|
||||
fields=[];results=[];measurements=[]
|
||||
for i in range(10):
|
||||
d=ten/'candidates'/method/'parts'/f'part-{i:02d}'
|
||||
receipt=read(d/'guard-result.json')
|
||||
if receipt['classification']['category']!='PASS':raise ValueError('Ten receipt not PASS')
|
||||
p=d/'output/result.json';checked(p,receipt['files']['result']['sha256'])
|
||||
result=read(p);result['guard_CPU_s']=receipt['worker']['cpu_seconds'];measurements.append(result)
|
||||
if i<3:
|
||||
f=d/'output/event-field.npz';fields.append(f)
|
||||
allowed.append(str(f));results.append(result)
|
||||
if kind=='reference':
|
||||
allowed += [r['field'] for r in tc['exact_references']]
|
||||
restrict_prediction_io(allowed,dest)
|
||||
rows=[]
|
||||
for i,ref in enumerate(tc['exact_references']):
|
||||
row=dict(index=i,case=ref['case'],reference_sha256=ref['field_sha256'],
|
||||
batch_sha256=results[i]['output_sha256']['event-field.npz'])
|
||||
try:
|
||||
checked(ref['field'],ref['field_sha256']);checked(fields[i],row['batch_sha256'])
|
||||
with np.load(ref['field'],allow_pickle=False) as a,np.load(fields[i],allow_pickle=False) as b:
|
||||
row.update(compare_fields(a,b))
|
||||
except Exception as exc:row.update(status='FAIL',type=type(exc).__name__,reason=str(exc))
|
||||
rows.append(row)
|
||||
result=dict(status='PASS' if len(rows)==3 and all(r['status']=='PASS' for r in rows) else 'FAIL',
|
||||
solver=method,parts=rows,large_part_accuracy_verified=False,no_solver_switch=True,
|
||||
labels_read=False,worker_CPU_s=time.process_time()-start)
|
||||
save(dest/'result.json',result)
|
||||
return 0
|
||||
if kind!='census':raise ValueError('Unknown gate stage')
|
||||
# This only counts original edges; it does not query distances or solve.
|
||||
checked(tc['roster'],tc['roster_sha256']);roster=read(tc['roster'])
|
||||
included=sorted([r for r in roster['parts'] if r['included']],key=lambda r:r['case'])
|
||||
if [r['case'] for r in included]!=tc['all_cases'] or len(included)!=617:raise ValueError('Roster mismatch')
|
||||
for r in included:
|
||||
source=Path(r['preparation']['preparation_root']);allowed += [str(source/'geometry.npz'),str(source/'boundary_distance_raw.npy')]
|
||||
restrict_prediction_io(allowed,dest)
|
||||
counts=[]
|
||||
try:
|
||||
for row in included:
|
||||
p=row['preparation'];source=Path(p['preparation_root'])
|
||||
for name in ['geometry.npz','boundary_distance_raw.npy']:checked(source/name,p['output_sha256'][name])
|
||||
dw=np.load(source/'boundary_distance_raw.npy',allow_pickle=False)
|
||||
with np.load(source/'geometry.npz',allow_pickle=False) as z:
|
||||
edges=z['edges'];faces=z['boundary_triangles'];raw=z['raw_node_ids']
|
||||
if len(raw)!=row['nodes'] or len(dw)!=len(raw):raise ValueError('Node count differs')
|
||||
count=query_count(edges,faces,dw)
|
||||
counts.append(dict(case=row['case'],nodes=row['nodes'],exact_midpoint_queries=count,
|
||||
geometry_sha256=p['output_sha256']['geometry.npz'],dw_sha256=p['output_sha256']['boundary_distance_raw.npy']))
|
||||
save(dest/'progress.json',dict(parts=len(counts),worker_CPU_s=time.process_time()-start))
|
||||
save(dest/'counts.json',dict(parts=counts,labels_read=False,distances_queried=0))
|
||||
bycase={r['case']:r for r in counts}
|
||||
for r in measurements:
|
||||
if bycase[r['case']]['exact_midpoint_queries']!=r['exact_midpoint_queries']:raise ValueError('Census/ten query count differs')
|
||||
terms=projection_terms(measurements,sum(r['nodes'] for r in counts),sum(r['exact_midpoint_queries'] for r in counts),len(counts))
|
||||
save(dest/'result.json',dict(status='COMPLETE',projection_terms=terms,counts_sha256=sha(dest/'counts.json'),
|
||||
labels_read=False,distances_queried=0,worker_CPU_s=time.process_time()-start))
|
||||
return 0
|
||||
except Exception as exc:
|
||||
save(dest/'failure.json',dict(type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),
|
||||
complete_parts=counts,worker_CPU_s=time.process_time()-start))
|
||||
raise
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('kind',choices=['reference','census'])
|
||||
a=p.parse_args();raise SystemExit(measure(a.root,a.kind))
|
||||
@@ -0,0 +1,91 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import numpy as np
|
||||
from s3b_v04_gates import compare_fields,query_count,projection_terms
|
||||
from run_s3b_v04_gates import run
|
||||
from s2_recovery_supervisor import save
|
||||
from s3b_runtime import read
|
||||
|
||||
|
||||
class GateTests(unittest.TestCase):
|
||||
def field(self,values):
|
||||
return dict(raw_ids=np.arange(4),arrival=np.array(values,float),remaining=np.zeros(4),mass=np.array([1.,2.,3.,4.]))
|
||||
|
||||
def test_tied_order_scale_invariant_strict_mass_CDF(self):
|
||||
r=compare_fields(self.field([0,1,1,2]),self.field([0,7,7,14]))
|
||||
self.assertEqual(r['status'],'PASS');self.assertAlmostEqual(r['tau_b'],1)
|
||||
self.assertEqual(r['max_strict_volume_CDF_delta'],0)
|
||||
|
||||
def test_actual_order_failure_and_undefined_tau_fail(self):
|
||||
r=compare_fields(self.field([0,1,2,3]),self.field([0,2,1,3]))
|
||||
self.assertEqual(r['status'],'FAIL');self.assertAlmostEqual(r['max_strict_volume_CDF_delta'],.3)
|
||||
r=compare_fields(self.field([0,0,0,0]),self.field([0,0,0,0]))
|
||||
self.assertEqual(r['status'],'FAIL');self.assertIsNone(r['tau_b'])
|
||||
|
||||
def test_partial_or_changed_mass_never_comparable(self):
|
||||
for key,value in [('arrival',np.array([0,1,np.nan,3.])),('remaining',np.array([0,0,0,1.])),('mass',np.ones(4)),('raw_ids',np.array([1,2,3,4]))]:
|
||||
b=self.field([0,1,2,3]);b[key]=value
|
||||
with self.assertRaises(ValueError):compare_fields(self.field([0,1,2,3]),b)
|
||||
|
||||
def test_count_only_nonwall_double_zero_edges(self):
|
||||
edges=np.array([[0,1],[0,2],[0,3],[1,2],[1,3],[2,3]])
|
||||
faces=np.array([[0,1,2]])
|
||||
self.assertEqual(query_count(edges,faces,np.zeros(4)),3)
|
||||
self.assertEqual(query_count(edges,faces,np.array([0.,0,0,1.])),0)
|
||||
with self.assertRaises(ValueError):query_count(edges,faces,np.ones(4))
|
||||
|
||||
def test_max_rates_margin_and_zero_denominator(self):
|
||||
rows=[dict(status='COMPLETE',nodes=100,exact_midpoint_queries=2,
|
||||
costs=dict(events_CPU_s=20.,midpoint_CPU_s=3.),guard_CPU_s=30.) for _ in range(10)]
|
||||
r=projection_terms(rows,1000,100,617)
|
||||
self.assertEqual(r['variable_projection_CPU_s'],437.5)
|
||||
self.assertEqual(r['fixed_projection_CPU_s'],7*617)
|
||||
for row in rows:row['exact_midpoint_queries']=0
|
||||
with self.assertRaises(ValueError):projection_terms(rows,1000,1,617)
|
||||
r=projection_terms(rows,1000,0,617)
|
||||
self.assertEqual(r['max_midpoint_CPU_per_query'],0)
|
||||
self.assertEqual(r['max_fixed_overhead_CPU_per_part'],10)
|
||||
with self.assertRaises(ValueError):projection_terms(rows[:9],1000,0,617)
|
||||
|
||||
def chain(self,reference_status='PASS',projection=100.):
|
||||
calls=[]
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root=Path(temp);ten=root/'ten';ten.mkdir();guard=root/'guard';guard.mkdir();gate=root/'gates';gate.mkdir()
|
||||
save(ten/'complete.json',dict(status='TEN_PASS_AWAITING_REFERENCE_AND_COST',cumulative_worker_CPU_s=5000.))
|
||||
save(guard/'complete.json',dict(status='OBSERVED_QUEUE_TERMINAL'))
|
||||
save(gate/'code-freeze.json',{'git_commit':'synthetic'})
|
||||
c=dict(ten_root=str(ten),stage_guard_root=str(guard),additional_synthetic_CPU_s=2.)
|
||||
def fake(command,dest,name,limits,**kw):
|
||||
kind=command[-1];calls.append(kind);(dest/'output').mkdir()
|
||||
data=dict(status=reference_status) if kind=='reference' else dict(status='COMPLETE',
|
||||
projection_terms=dict(variable_projection_CPU_s=projection,fixed_projection_CPU_s=10.))
|
||||
save(dest/'output/result.json',data)
|
||||
return dict(status='PASS',monitor_errors=0,cpu_seconds=3.)
|
||||
with patch('run_s3b_v04_gates.verify_gate_config',return_value=c),patch('run_s3b_v04_gates.supervise',side_effect=fake),patch('run_s3b_v04_gates.os.sched_setaffinity'),patch('run_s3b_v04_gates.signal.signal'):
|
||||
code=run(gate)
|
||||
result=read(gate/'complete.json')
|
||||
cost=read(gate/'cost-gate.json') if (gate/'cost-gate.json').exists() else None
|
||||
return code,calls,result,cost
|
||||
|
||||
def test_reference_failure_never_runs_census(self):
|
||||
code,calls,result,cost=self.chain(reference_status='FAIL')
|
||||
self.assertEqual(code,1);self.assertEqual(calls,['reference']);self.assertIsNone(cost)
|
||||
self.assertEqual(result['reason'],'EXACT_REFERENCE_GATE_FAILED')
|
||||
self.assertEqual(result['cumulative_worker_CPU_s'],5005.)
|
||||
|
||||
def test_cost_uses_final_guard_CPU_and_reserve(self):
|
||||
code,calls,result,cost=self.chain()
|
||||
self.assertEqual(code,0);self.assertEqual(calls,['reference','census'])
|
||||
self.assertEqual(cost['projected_total_worker_CPU_s'],5000+2+6+100+10+7200)
|
||||
self.assertFalse(result['full617_launched'])
|
||||
|
||||
def test_cost_failure_stops_without_retrial(self):
|
||||
code,calls,result,cost=self.chain(projection=70000.)
|
||||
self.assertEqual(code,1);self.assertEqual(calls,['reference','census'])
|
||||
self.assertEqual(result['reason'],'COST_PROJECTION_GATE_FAILED')
|
||||
self.assertEqual(cost['status'],'FAIL')
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user