Freeze corrected full-volume ordering diagnostic and one-time custody contract

This commit is contained in:
Frank14f committed 2026-11-15 17:22:20 +08:00
1 parent 6d907acc7c
commit d2adb6bcce
14 files changed
+2599

No files matched your search

+21
View File
@@ -0,0 +1,21 @@
"""Independent bounded batch input verification; never touches S3-A sources."""
from pathlib import Path
from freeze_g1_components import sha
from s2_custody_common import read_json, checked
def contract(root):
root = Path(root)
c = read_json(root/'contract.json')
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():
checked(p,h)
frozen = read_json(root/'code-freeze.json')
if not frozen.get('git_commit') or frozen['contract_sha256'] != sha(root/'contract.json'):
raise ValueError('Naive-ordering contract/code must be fixed before any execution')
if c['latest_ruling'] != '1794733619.697459' or c['limits']['worker_cpu_seconds'] != 36000:
raise ValueError('Wrong custody authorization or budget')
return c
def rows(c):
return sorted((r for r in read_json(checked(c['roster'],c['roster_sha256']))['parts'] if r['included']),key=lambda r:r['case'])
+84
View File
@@ -0,0 +1,84 @@
"""Array-only ordering diagnostics; no learning, archives or test selection."""
from collections import defaultdict
import numpy as np
from scipy.sparse import coo_matrix
from scipy.sparse.csgraph import dijkstra
def cdf_interval(scores, weights):
s = np.asarray(scores, dtype=np.float64)
w = np.asarray(weights, dtype=np.float64)
if s.ndim != 1 or s.shape != w.shape or not len(s) or not np.isfinite(s).all() or not np.isfinite(w).all() or np.any(w <= 0):
raise ValueError('CDF requires finite scores and positive weights')
order = np.argsort(s, kind='stable')
prefix = np.r_[0., np.cumsum(w[order])]
total = w.sum()
return prefix[np.searchsorted(s[order], s, side='left')]/total, prefix[np.searchsorted(s[order], s, side='right')]/total
def pps_cdf(scores, multiplicity):
counts = np.asarray(multiplicity, dtype=np.float64)
if not np.isfinite(counts).all() or np.any(counts != np.floor(counts)):
raise ValueError('PPS multiplicity must be an integer count, not node volume')
return cdf_interval(scores, counts)
def training_clock(rows, fold, pool, end_times):
"""Component/part-equal mean of full VTU T_end / pre-solve t_set."""
groups = defaultdict(list)
access = []
for row in rows:
a = row['assignments'][fold]
if not row['included'] or a['role'] != 'learning' or str(pool) not in a['learning_pools']:
continue
# Do not substitute a query maximum or an available validation/test case.
end = end_times[row['case']]
ratio = end/row['preparation']['t_set']
if not np.isfinite(ratio) or ratio <= 0:
raise ValueError('Invalid learning full-node end time')
groups[row['component_id']].append(float(ratio))
access.append(dict(case=row['case'], role='learning', full_end_over_t_set=float(ratio)))
if not groups:
raise ValueError('Empty original learning pool')
return float(np.mean([np.mean(x) for x in groups.values()])), access
def squared_wall_distance(points, edges, dw, gate):
x = np.asarray(points, dtype=np.float64)
e = np.asarray(edges)
d = np.asarray(dw, dtype=np.float64)
g = np.asarray(gate, dtype=np.float64)
if x.shape != (len(d), 3) or e.ndim != 2 or e.shape[1] != 2 or e.dtype.kind not in 'iu' or g.shape != (3,):
raise ValueError('Geometry shape')
if not len(d) or not len(e) or e.min() < 0 or e.max() >= len(d) or not np.isfinite(x).all() or not np.isfinite(d).all() or np.any(d < 0):
raise ValueError('Invalid geometry/distance')
source = int(np.argmin(np.linalg.norm(x-g, axis=1)))
# Piecewise-linear D_w at edge midpoint, then square; NOT mean(D_w**2).
midpoint = d[e].mean(axis=1)
keep = midpoint > 0
selected = e[keep]
cost = np.linalg.norm(x[selected[:,1]]-x[selected[:,0]], axis=1)/midpoint[keep]**2
if not np.isfinite(cost).all() or np.any(cost <= 0):
raise ValueError('Invalid positive-edge costs; no epsilon floor')
graph = coo_matrix((np.tile(cost, 2), (np.r_[selected[:,0],selected[:,1]], np.r_[selected[:,1],selected[:,0]])), shape=(len(x),len(x))).tocsr()
out = dijkstra(graph, directed=False, indices=source)
if not np.isfinite(out).all():
raise ValueError('Unreachable full tet node; keep failure, do not add zero-wall edges')
return out, source, int(np.count_nonzero(~keep))
def volume_clock_curve(truth, mass, end_full, t_set):
t = np.asarray(truth, dtype=np.float64)
m = np.asarray(mass, dtype=np.float64)
left, right = cdf_interval(t, m)
if not np.isfinite([end_full,t_set]).all() or min(end_full,t_set) <= 0:
raise ValueError('Positive seconds scales required')
grid = np.linspace(0,1,101)
order = np.argsort(t,kind='stable')
cumulative = np.cumsum(m[order])/m.sum()
indices = np.minimum(np.searchsorted(cumulative,grid,side='left'),len(t)-1)
times = t[order[indices]]
gap = np.maximum(np.maximum(left-t/end_full,t/end_full-right),0.)
return dict(volume_quantiles=grid.tolist(),seconds=times.tolist(),over_t_set=(times/t_set).tolist(),
over_full_end=(times/end_full).tolist(),max_distance_from_linear_time_to_CDF_tie_interval=float(gap.max()),
max_tie_volume=float(np.max(right-left)),tet_end=float(t.max()),full_VTU_end=float(end_full))
+62
View File
@@ -0,0 +1,62 @@
"""Freeze the corrected, explicitly authorized full-node diagnostic before execution."""
import json
from pathlib import Path
import numpy as np
import scipy
from freeze_g1_components import sha
from s2_custody_common import checked,read_json
from s2_recovery_supervisor import save
ROOT=Path('artifacts/naive-ordering-2026-11-15')
def main():
if ROOT.exists():raise FileExistsError('Do not overwrite a naive-ordering batch')
base=Path('artifacts/s2-custody-evaluation-2026-11-15');s2=read_json(base/'contract.json')
for p,h in s2['code_sha256'].items():checked(p,h)
roster=read_json(checked(s2['roster'],s2['roster_sha256']))
cases=sorted(r['case'] for r in roster['parts'] if r['included'])
assert len(cases)==617
paths=set(s2['code_sha256'])|{str(p) for p in Path('tools').glob('*naive_ordering*.py')}
sources=[s2['roster'],s2['exposure_membership'],s2['models'],str(base/'contract.json'),
str(base/'custody/label-index.json'),str(base/'custody/part-scores.json'),
str(base/'prediction-index.json'),str(base/'predictions-complete.json'),
'docs/naive-ordering-contract-2026-11-15.md','docs/mold-route-a-review-2026-11-15.md']
c=dict(version='naive-ordering-corrected-v1',latest_ruling='1794733619.697459',
custody_ruling='1794733395.244139',direction_ruling='1794733255.090409',
main_review_complete=True,launch_authorized_after_freeze=True,
S2_root=str(base),S2_contract=str(base/'contract.json'),S2_contract_sha256=sha(base/'contract.json'),
roster=s2['roster'],roster_sha256=s2['roster_sha256'],cases=cases,
archive=s2['archive'],archive_stat=s2['archive_stat'],
label_index=str(base/'custody/label-index.json'),label_index_sha256=sha(base/'custody/label-index.json'),
prediction_index=str(base/'prediction-index.json'),prediction_index_sha256=sha(base/'prediction-index.json'),
original_part_scores=str(base/'custody/part-scores.json'),original_part_scores_sha256=sha(base/'custody/part-scores.json'),
exposure_membership=s2['exposure_membership'],exposure_membership_sha256=s2['exposure_membership_sha256'],
worker_cpus=[16,18,20,22],coordinator_cpus=[28,29],
limits=dict(worker_cpu_seconds=36000,preparation_cpu_seconds=28800,post_cpu_seconds=7200,
workers=4,part_cpu_seconds=180,part_wall_seconds=600,part_rss_bytes=4*2**30,
part_disk_bytes=2*2**30,disk_bytes=20*2**30,retries=0,GPU_seconds=0),
numpy_version=np.__version__,scipy_version=scipy.__version__,
full_decodes_authorized=617,full_decode_policy='Exactly one whole-VTU Fill_time array per part; immutable attempt; preserve S2 cache and validate all common IDs',
time_scale='Per fold and nested learning pool component/part-equal mean(full_VTU_T_end/t_set); shared all arms',
CDF='Strict< used for all new predictions; save[F<,F<=] per tie group',
full_volume='all retained tet node lumped volume including d=0',
conditional_volume='frozen PPS multiplicity once; no second node-volume weight',
alpha='estimate (M+/Mall)*K/2000; plugin intervals are not certified whole-domain bounds',
old_Dw_distance='Reuse original Df/Vf; verify CDF equivalence; no re-solve',
new_distance='Dw^2 only, edge midpoint interpolation then square; exclude zero-speed edges, no epsilon, retain every node/mass',
eikonal_vs_Dijkstra_tested=False,original_models_refitted=False,
training_clocks_frozen_before_maps=True,complete_prediction_manifest_before_score=True,
scopes=['full-volume true-order oracles','conditional reorders R2/N/E/B0','reused Dw clock-only','new Dw2 approximation','full-volume timing curves'],
statistics=dict(replicates=2000,seed=2026111408,margin=.1,pools=[25,50,100],six_region_unchanged=True,
baseline='Original R2 part scores reused',naive_close_band=[-.1,.1],oracle_rows_separate=True,confirmatory=False),
S3A_scope_unchanged=True,interpretation='Exploration after S2 results were known; not a fresh confirmatory test',
source_sha256={p:sha(p) for p in sources},code_sha256={p:sha(p) for p in sorted(paths)})
ROOT.mkdir();save(ROOT/'contract.json',c)
save(ROOT/'prelaunch.json',dict(status='AUTHORIZED_FIXED_NOT_STARTED',contract_sha256=sha(ROOT/'contract.json'),
custody_decodes=0,new_distance_solves=0,new_fits=0,original_Dw_reused=True,
latest_ruling=c['latest_ruling'],full_decode_ruling=c['custody_ruling']))
print(json.dumps(read_json(ROOT/'prelaunch.json')))
if __name__=='__main__':main()
+262
View File
@@ -0,0 +1,262 @@
"""Frozen clocks, mapped predictions and diagnostic scoring; no model fitting."""
import argparse
from collections import defaultdict
import json
import os
from pathlib import Path
import resource
import sys
import threading
import time
import numpy as np
import psutil
from freeze_g1_components import sha
from s2_custody_common import checked, read_json, prediction_inputs, expected_models, prediction_path
from s2_custody_statistics import part_metrics, component_values, contrast
from s2_recovery_supervisor import save, supervise, stop_group
from s2_recovery_monitoring import disk_usage
from naive_ordering_common import contract, rows
from naive_ordering_math import pps_cdf, training_clock
ROUTES = {'R2': 'R2', 'ORACLE_FULL_TRAIN': 'R2', 'ORACLE_FULL_END': 'R2',
'ORACLE_MASK_TRAIN': 'R2', 'B0_DW1_CLOCK': 'R2', 'B0_DW2_CLOCK': 'R2',
'REORDER_R2': 'R2', 'REORDER_N': 'N', 'REORDER_E': 'E', 'REORDER_B0': 'R2'}
def validate_manifest(c,index):
original=read_json(checked(c['S2_contract'],c['S2_contract_sha256']))
expected=set()
for row in rows(c):
fold=next(a['fold'] for a in row['assignments'] if a['role']=='test')
for pool in (25,50,100):
for arm in ('ORACLE_FULL_TRAIN','ORACLE_FULL_END','ORACLE_MASK_TRAIN','B0_DW1_CLOCK','B0_DW2_CLOCK'):
expected.add((row['case'],fold,pool,arm,None))
for model in expected_models(original,fold):
if model['arm']=='B0' or model['pool']==pool:
expected.add((row['case'],fold,pool,'REORDER_'+model['arm'],model['seed']))
actual=[(v['case'],v['fold'],v['pool'],v['arm'],v['seed']) for v in index.values()]
if len(actual)!=len(set(actual)) or set(actual)!=expected:
raise ValueError('Missing/extra/duplicate mapped predictions against frozen original model roster')
def prepared(root,c):
batch=read_json(root/'batch-complete.json')
if batch['status']!='ALL_PARTS_TERMINAL' or batch['passed']!=617 or batch['failed'] or not batch['accounting_complete']:
raise ValueError('All617 custody/geometry attempts must PASS; no silent calibration roster change')
if batch['contract_sha256']!=sha(root/'contract.json'):raise ValueError('Wrong preparation contract')
result={}
for job in batch['jobs']:
p=checked(job['result_path'],job['result_sha256']);r=read_json(p)
for name,digest in r['output_sha256'].items():checked(p.parent/name,digest)
result[r['case']] = (r,p.parent)
if set(result)!=set(c['cases']):raise ValueError('Missing/extra prepared part')
return result
def make_maps(root):
c=contract(root);parts=rows(c);prep=prepared(root,c)
if (root/'mapping-started.json').exists():raise FileExistsError('No repeat map phase')
save(root/'mapping-started.json',dict(unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json')))
start=time.process_time()
original=read_json(checked(c['S2_contract'],c['S2_contract_sha256']))
original_index=read_json(checked(c['prediction_index'],c['prediction_index_sha256']))
ends={case:r['full_VTU_T_end'] for case,(r,_) in prep.items()}
clocks=[]
for fold in range(5):
for pool in (25,50,100):
value,access=training_clock(parts,fold,pool,ends)
clocks.append(dict(fold=fold,pool=pool,c=value,learning_access=access,
definition='mean across components of within-component mean(full_VTU_T_end/t_set)',
validation_or_test_used_for_scale=False))
save(root/'training-clocks.json',clocks)
clock={(r['fold'],r['pool']):r['c'] for r in clocks}
save(root/'clock-freeze.json',dict(contract_sha256=sha(root/'contract.json'),
clocks_sha256=sha(root/'training-clocks.json'),batch_sha256=sha(root/'batch-complete.json'),
before_new_test_mapping=True))
index={};counter=0;curves=[]
for row in parts:
case=row['case'];fold=next(a['fold'] for a in row['assignments'] if a['role']=='test')
meta,directory=prep[case];d=prediction_inputs(row)
with np.load(directory/'full-domain-maps.npz',allow_pickle=False) as z:
if not np.array_equal(z['raw_ids'][d['ids']],d['raw_ids']):raise ValueError('Full/query map order')
full={key:z[key][d['ids']].copy() for key in z.files if key!='raw_ids'}
with np.load(directory/'full-node-labels.npz',allow_pickle=False) as z:
truth=z['seconds'][d['raw_ids']].copy()
masked_left,masked_right=pps_cdf(truth,d['counts'])
def output(route,pool,seed,left,right,seconds,source=None):
nonlocal counter
tag=route+('' if seed is None else f'-seed{seed}')
path=root/'predictions'/str(fold)/str(pool)/tag/(case+'.npz')
path.parent.mkdir(parents=True,exist_ok=True)
if path.exists():raise FileExistsError('Duplicate mapped prediction')
extra={}
if route.startswith('REORDER_') or route=='ORACLE_MASK_TRAIN':
# These are plug-in estimates, not certified whole-domain bounds.
alpha=meta['estimated_alpha']
extra=dict(estimated_domain_lower=alpha*left,estimated_domain_upper=alpha*right+1-alpha)
np.savez_compressed(path,seconds=seconds,raw_ids=d['raw_ids'],counts=d['counts'],F_left=left,F_right=right,**extra)
index[str(path)]=dict(sha256=sha(path),case=case,component_id=row['component_id'],fold=fold,pool=pool,
arm=route,seed=seed,source_prediction=source,oracle=route.startswith('ORACLE_'),
domain='conditional_PPS' if extra else 'full_tet_lumped_volume')
counter+=1
for pool in (25,50,100):
scale=d['t_set']*clock[fold,pool]
output('ORACLE_FULL_TRAIN',pool,None,full['truth_left'],full['truth_right'],scale*full['truth_left'])
output('ORACLE_FULL_END',pool,None,full['truth_left'],full['truth_right'],meta['full_VTU_T_end']*full['truth_left'])
output('ORACLE_MASK_TRAIN',pool,None,masked_left,masked_right,scale*masked_left)
output('B0_DW1_CLOCK',pool,None,full['df1_left'],full['df1_right'],scale*full['df1_left'])
output('B0_DW2_CLOCK',pool,None,full['df2_left'],full['df2_right'],scale*full['df2_left'])
for model in expected_models(original,fold):
if model['arm']!='B0' and model['pool']!=pool:continue
p=prediction_path(Path(c['S2_root']),model,case)
with np.load(checked(p,original_index[str(p)]),allow_pickle=False) as z:
if not np.array_equal(z['raw_ids'],d['raw_ids']) or not np.array_equal(z['counts'],d['counts']):raise ValueError('Source prediction domain')
score=z['seconds'].copy()
left,right=pps_cdf(score,d['counts']);mapped=scale*left
old_last=d['raw_ids'][score==score.max()].min();new_last=d['raw_ids'][mapped==mapped.max()].min()
if old_last!=new_last:raise ValueError('Order-preserving map changed final location')
if model['arm']=='B0' and not np.allclose(score,d['t_set']*full['df1_left'],rtol=1e-12,atol=1e-12):
raise ValueError('Unscaled Dw fallback differs from frozen B0')
output('REORDER_'+model['arm'],pool,model['seed'],left,right,mapped,str(p))
curves.append(dict(case=case,component_id=row['component_id'],**read_json(directory/'volume-clock-curve.json')))
save(root/'mapping-progress.json',dict(completed_parts=len(curves),prediction_files=counter,CPU_s=time.process_time()-start))
validate_manifest(c,index)
save(root/'prediction-index.json',index)
save(root/'volume-clock-curves.json',curves)
distribution={}
for field in ('seconds','over_t_set','over_full_end'):
values=np.array([r[field] for r in curves])
distribution[field]=dict(median=np.median(values,axis=0).tolist(),p10=np.percentile(values,10,axis=0,method='linear').tolist(),
p90=np.percentile(values,90,axis=0,method='linear').tolist())
save(root/'curve-summary.json',dict(part_count=len(curves),aggregation='part-equal descriptive median/p10/p90; no independent-sample inference',
volume_quantiles=curves[0]['volume_quantiles'],**distribution))
save(root/'predictions-complete.json',dict(status='ALL_NAIVE_MAPS_FROZEN',contract_sha256=sha(root/'contract.json'),
code_freeze=read_json(root/'code-freeze.json'),prediction_index_sha256=sha(root/'prediction-index.json'),
clock_freeze_sha256=sha(root/'clock-freeze.json'),prediction_files=counter,CPU_s=time.process_time()-start,
full_decode_count=617,original_predictions_unchanged=True,new_model_fits=0))
return 0
def reports(scores,roster,members):
tables=[];contrasts=[];components=[]
for subset in ('all','not_known_exposed'):
for pool in (25,50,100):
primary={}
for route,alias in ROUTES.items():
data=[dict(r,arm=alias) for r in scores if r['arm']==route]
for metric in ('relative_l2','near_wall','kendall_tau_b','last_location'):
values,excluded,details=component_values(data,roster,alias,pool,metric,subset!='all',members)
worst=max(values,key=values.get) if values and metric=='relative_l2' else None
tables.append(dict(subset=subset,pool=pool,arm=route,metric=metric,
mean=float(np.mean(list(values.values()))) if values else None,components=len(values),
parts=sum(details[k]['parts'] for k in values),nodes=sum(details[k]['nodes'] for k in values),
excluded_components=excluded,worst_component=dict(id=worst,value=values[worst]) if worst else None))
components.extend(dict(subset=subset,pool=pool,arm=route,metric=metric,component_id=k,value=v,**details[k]) for k,v in values.items())
if metric=='relative_l2':primary[route]=(values,details)
ref,details=primary['R2']
for route,(candidate,cd) in primary.items():
if route=='R2':continue
pair=set(ref)&set(candidate);excluded=sorted((set(details)|set(cd))-pair)
result=contrast(ref,candidate)
low,high=result.get('low'),result.get('high')
contrasts.append(dict(subset=subset,pool=pool,reference='R2',candidate=route,
oracle=route.startswith('ORACLE_'),exploratory=True,primary_confirmatory=False,
within_descriptive_10percent_band=low is not None and low>=-.1 and high<=.1,
included_folds=sorted({details[k]['fold'] for k in pair}),excluded_components=excluded,
excluded_parts=sum(details[k]['parts'] for k in excluded),excluded_nodes=sum(details[k]['nodes'] for k in excluded),**result))
return tables,contrasts,components
def score_maps(root):
c=contract(root);roster=read_json(c['roster'])['parts'];by_case={r['case']:r for r in roster}
prepared(root,c)
proof=read_json(root/'predictions-complete.json')
if proof['status']!='ALL_NAIVE_MAPS_FROZEN' or proof['contract_sha256']!=sha(root/'contract.json'):raise ValueError('Mapping not frozen')
clock_proof=read_json(checked(root/'clock-freeze.json',proof['clock_freeze_sha256']))
checked(root/'training-clocks.json',clock_proof['clocks_sha256'])
index=read_json(checked(root/'prediction-index.json',proof['prediction_index_sha256']))
if len(index)!=proof['prediction_files']:raise ValueError('Manifest size')
validate_manifest(c,index)
for path,row in index.items():checked(path,row['sha256'])
dest=root/'scores';dest.mkdir(exist_ok=False)
start=time.process_time();save(dest/'started.json',dict(unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json')))
originals=read_json(checked(c['original_part_scores'],c['original_part_scores_sha256']))
original_lookup={(r['case'],r['arm'],r['pool'],r['seed']):r for r in originals}
scores=[r for r in originals if r['arm']=='R2']
labels={r['case']:r for r in read_json(checked(c['label_index'],c['label_index_sha256']))}
grouped=defaultdict(list)
for path,item in index.items():grouped[item['case']].append((path,item))
unchanged_last=0
for case,items in sorted(grouped.items()):
d=prediction_inputs(by_case[case]);entry=labels[case]
with np.load(checked(entry['path'],entry['sha256']),allow_pickle=False) as z:
if not np.array_equal(z['raw_ids'],d['raw_ids']) or not np.array_equal(z['counts'],d['counts']):raise ValueError('Scoring cache domain')
truth=z['seconds'].copy()
for path,item in items:
with np.load(path,allow_pickle=False) as z:
if not np.array_equal(z['raw_ids'],d['raw_ids']) or not np.array_equal(z['counts'],d['counts']):raise ValueError('Mapped domain')
metrics=part_metrics(z['seconds'],truth,d['counts'],d['near_wall'],d['raw_ids'],d['coordinates_over_Rg'],case.split('_')[0])
if item['arm'].startswith('REORDER_'):
source=item['arm'].removeprefix('REORDER_');pool=None if source=='B0' else item['pool']
old=original_lookup[case,source,pool,item['seed']]
if metrics['last_location']!=old['last_location'] or metrics['kendall_tau_b']!=old['kendall_tau_b']:
raise ValueError('Reorder altered invariant last location or sampled tau')
unchanged_last+=1
scores.append({k:item[k] for k in ('case','component_id','fold','pool','arm','seed')}|metrics)
members={p for f in read_json(checked(c['exposure_membership'],c['exposure_membership_sha256'])) for p in f['parts']}
tables,contrasts,components=reports(scores,roster,members)
save(dest/'part-scores.json',scores);save(dest/'tables.json',tables);save(dest/'contrasts.json',contrasts);save(dest/'component-scores.json',components)
save(dest/'result.json',dict(status='NAIVE_ORDERING_COMPLETE',headline='Exploration after S2 results were known; not a fresh confirmatory test',
no_model_training=True,new_full_custody_decodes=617,reorder_invariant_rows=unchanged_last,
eikonal_vs_Dijkstra_tested=False,Dw_distance_reused=True,only_new_distance_exponent=2,
CPU_s=time.process_time()-start,prediction_rows=len(index),
limitations=['Full discrete tet lumped-volume measure, not exact continuous cavity integration',
'T_end_full is maximum over original VTU Fill_time; raw/tet/query maxima separately reported',
'PPS conditional CDF and estimated alpha do not certify unobserved domain',
'Oracle success does not establish usable learned ordering; poor masked oracle is not a standalone refutation of constant flow',
'Reordering preserves final location and ties; no gate-ranking validation',
'All comparisons exploratory; no new material/process generalization or O1 test'],
output_sha256={p.name:sha(p) for p in dest.glob('*.json')}))
return 0
def guarded(root,phase):
c=contract(root);lim=c['limits'];batch=read_json(root/'batch-complete.json')
prepared(root,c)
spent=batch['worker_CPU_s']
if phase=='score':
previous=read_json(root/'map-guard-result.json')
if previous['worker']['status']!='PASS':raise ValueError('Map phase did not PASS')
spent+=previous['worker']['cpu_seconds']
phase_budget=min(5400 if phase=='map' else 7200,lim['worker_cpu_seconds']-spent,
lim['post_cpu_seconds']-(spent-batch['worker_CPU_s']))
if phase_budget<=0:raise ValueError('No postprocess budget')
command=[sys.executable,__file__,str(root),phase,'--worker']
done=threading.Event();reasons=[]
def watch():
while not done.is_set():
try:
if disk_usage(root)>lim['disk_bytes']:reasons.append('GLOBAL_DISK_LIMIT')
except Exception as exc:reasons.append('DISK_MONITOR_ERROR:'+str(exc))
if reasons:
for child in psutil.Process().children(recursive=True):
try:
if child.cmdline()==command:stop_group(child.pid)
except psutil.NoSuchProcess:pass
done.wait(1)
os.sched_setaffinity(0,[c['worker_cpus'][0]])
watcher=threading.Thread(target=watch,daemon=True);watcher.start()
try:
w=supervise(command,root/(phase+'-guard'),'worker',dict(total_cpu_seconds=phase_budget,job_cpu_seconds=phase_budget,
job_wall_seconds=10800,tree_rss_bytes=lim['part_rss_bytes'],disk_bytes=lim['disk_bytes'],max_processes=1),poll=.2)
finally:done.set();watcher.join(timeout=3)
if reasons:w.update(status='STOPPED_OR_FAILED',stop_reason=reasons[0])
save(root/(phase+'-guard-result.json'),dict(worker=w,prior_worker_CPU_s=spent,cumulative_worker_CPU_s=spent+w['cpu_seconds'],
total_cap_CPU_s=lim['worker_cpu_seconds'],postphase_cap_CPU_s=lim['post_cpu_seconds'],
guard_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2])))
return 0 if w['status']=='PASS' else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['map','score']);p.add_argument('--worker',action='store_true');a=p.parse_args()
raise SystemExit((make_maps(a.root) if a.phase=='map' else score_maps(a.root)) if a.worker else guarded(a.root,a.phase))
+108
View File
@@ -0,0 +1,108 @@
"""One authorized full-node custody decode and D_w^2 graph approximation."""
import argparse
import json
import os
from pathlib import Path
import time
import traceback
from zipfile import ZipFile
from xml.etree import ElementTree as ET
import numpy as np
from audit_samples import decode_array
from freeze_g1_components import sha
from s2_custody_common import checked, read_json
from s2_recovery_supervisor import save
from naive_ordering_common import contract, rows
from naive_ordering_math import cdf_interval, squared_wall_distance, volume_clock_curve
def decode_full(archive, case):
stack=[];count=None;header=None
with ZipFile(archive) as z,z.open(case+'_sim.vtu') as stream:
for event,e in ET.iterparse(stream,events=('start','end')):
if event=='start':
stack.append(e.tag)
if e.tag=='VTKFile':
if e.get('byte_order')!='LittleEndian' or e.get('compressor')!='vtkZLibDataCompressor':
raise ValueError('Unsupported source encoding')
header=e.get('header_type','UInt32')
elif e.tag=='Piece':
if count is not None:raise ValueError('Multiple pieces')
count=int(e.get('NumberOfPoints'))
else:
if e.tag=='DataArray':
if stack[-2]=='PointData' and e.get('Name')=='Fill_time':
values=decode_array(e,header)
if values.shape!=(count,):raise ValueError('Full-node label length')
return values.copy()
e.clear()
stack.pop()
raise ValueError('No Fill_time array')
def run(root, index):
c=contract(root);row=rows(c)[index];prep=row['preparation'];source=Path(prep['preparation_root'])
dest=root/'parts'/f'part-{index:03d}'/'output'
dest.mkdir(exist_ok=False)
start=time.monotonic();cpu=time.process_time()
save(dest/'started.json',dict(pid=os.getpid(),case=row['case'],index=index,unix=time.time(),
contract_sha256=sha(root/'contract.json'),custody_authorization=c['latest_ruling']))
try:
for name in ['geometry.npz','r2.npz','queries.npz']:checked(source/name,prep['output_sha256'][name])
metadata=read_json(checked(source/'result.json',prep['result_sha256']))
if metadata['r2']['columns'][8]!='Dw':raise ValueError('Expected original wall-distance column')
with np.load(source/'geometry.npz',allow_pickle=False) as z:
x=z['points_raw'];edges=z['edges'];mass=z['node_volume_raw'];raw=z['raw_node_ids'];gate=z['gate_location_raw']
with np.load(source/'r2.npz',allow_pickle=False) as z:
dw=z['features'][:,8];df1=z['Df'];vf=z['Vf']
with np.load(source/'queries.npz',allow_pickle=False) as z:
mask=z['eligible_s2'];ids=z['anchor_ids'][mask];query_raw=z['raw_node_ids'][mask];counts=z['multiplicity'][mask]
if not np.array_equal(raw[ids],query_raw):raise ValueError('Query/raw-ID mismatch')
st=Path(c['archive']).stat();a=c['archive_stat']
if (st.st_size,st.st_mtime_ns,st.st_ino)!=(a['bytes'],a['mtime_ns'],a['inode']):raise ValueError('Archive identity changed')
# Exactly once per immutable attempt; save raw bytes even if validation later fails.
full=decode_full(c['archive'],row['case'])
np.savez_compressed(dest/'full-node-labels.npz',seconds=full,retained_tet_raw_ids=raw)
save(dest/'decode-receipt.json',dict(case=row['case'],decoded_arrays=['Fill_time'],decode_count=1,
full_VTU_nodes=len(full),retained_tet_nodes=len(raw),label_sha256=sha(dest/'full-node-labels.npz'),
original_S2_query_decode_preserved=True,custody_authorization=c['latest_ruling']))
if not np.isfinite(full).all() or np.any(full<0) or full.max()<=0:raise ValueError('Invalid full-node Fill_time; preserve raw cache, no retry')
if raw.ndim!=1 or np.any(np.diff(raw)<=0) or raw[0]<0 or raw[-1]>=len(full):raise ValueError('Full-node/tet indexing')
old={v['case']:v for v in read_json(checked(c['label_index'],c['label_index_sha256']))}[row['case']]
with np.load(checked(old['path'],old['sha256']),allow_pickle=False) as z:
if not np.array_equal(z['raw_ids'],query_raw) or not np.array_equal(z['counts'],counts) or not np.array_equal(z['seconds'],full[query_raw]):
raise ValueError('Full decode differs from immutable S2 query cache')
left1,right1=cdf_interval(df1,mass)
difference=float(np.max(np.abs(left1-vf)))
if difference>1e-12:raise ValueError('Existing B0 is not the expected full-volume strict D_w Dijkstra CDF')
save(dest/'progress.json',dict(stage='Dw_squared_graph',cpu_seconds=time.process_time()-cpu))
df2,gate_node,zero_edges=squared_wall_distance(x,edges,dw,gate)
left2,right2=cdf_interval(df2,mass)
leftT,rightT=cdf_interval(full[raw],mass)
np.savez_compressed(dest/'full-domain-maps.npz',raw_ids=raw,truth_left=leftT,truth_right=rightT,
df1_left=left1,df1_right=right1,df2_left=left2,df2_right=right2)
end=float(full.max());tet_end=float(full[raw].max());query_end=float(full[query_raw].max())
positive=float(mass[dw>0].sum()/mass.sum());K=int(counts.sum());alpha=positive*K/metadata['draw_slots']
curve=volume_clock_curve(full[raw],mass,end,prep['t_set'])
save(dest/'volume-clock-curve.json',curve)
save(dest/'result.json',dict(status='COMPLETE',case=row['case'],index=index,component_id=row['component_id'],
t_set=prep['t_set'],full_VTU_T_end=end,full_tet_T_end=tet_end,query_T_max=query_end,
raw_nodes=len(full),tet_nodes=len(raw),zero_mass_excluded_VTU_nodes=len(full)-len(raw),
full_vs_tet_end_difference=end-tet_end,eligible_slots=K,draw_slots=metadata['draw_slots'],
positive_mass_fraction=positive,estimated_alpha=alpha,alpha_is_estimate=True,
DKW_halfwidth_if_fixed_score_independent_PPS=float(np.sqrt(np.log(40)/(2*K))),
baseline_Dw_Vf_max_abs_difference=difference,Dw_solve_reused=True,Dw2_solver='tet-edge Dijkstra approximation',
zero_midpoint_edges_excluded=zero_edges,gate_node=gate_node,
cpu_seconds=time.process_time()-cpu,wall_seconds=time.monotonic()-start,
output_sha256={p.name:sha(p) for p in dest.glob('*') if p.is_file() and p.name not in ['progress.json']},
no_training=True,no_GPU=True,decode_count=1))
return 0
except Exception as exc:
save(dest/'failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),
cpu_seconds=time.process_time()-cpu,wall_seconds=time.monotonic()-start,retry=False))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args()
raise SystemExit(run(a.root,a.index))
+128
View File
@@ -0,0 +1,128 @@
"""Separate CPU-only full-node custody queue, bounded independently of S3-A."""
import argparse
import fcntl
import json
import math
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
import traceback
from freeze_g1_components import sha
from s2_custody_common import read_json, checked
from s2_recovery_supervisor import save, supervise, tree_usage
from s2_recovery_monitoring import disk_usage, Monitor, budget_reason
from run_s2_pilot_preparation import terminate
from naive_ordering_common import contract
def guard(root,index,slot):
c=contract(root);lim=c['limits'];os.sched_setaffinity(0,[c['worker_cpus'][slot]])
dest=root/'parts'/f'part-{index:03d}';dest.mkdir(parents=True,exist_ok=True)
w=supervise([sys.executable,'tools/run_naive_ordering_part.py',str(root),str(index)],dest,'worker',
dict(total_cpu_seconds=lim['part_cpu_seconds']+1,job_cpu_seconds=lim['part_cpu_seconds'],
job_wall_seconds=lim['part_wall_seconds'],tree_rss_bytes=lim['part_rss_bytes'],disk_bytes=lim['part_disk_bytes'],max_processes=1),poll=.2)
result=dest/'output/result.json'
if w['status']=='PASS' and (not result.exists() or read_json(result)['status']!='COMPLETE'):
w.update(status='STOPPED_OR_FAILED',stop_reason='MISSING_COMPLETE_RESULT')
save(dest/'guard-result.json',dict(index=index,case=c['cases'][index],worker=w,
guard_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
result_path=str(result) if result.exists() else None,result_sha256=sha(result) if result.exists() else None))
return 0 if w['status']=='PASS' else 1
def run(root):
with (root/'batch.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
return run_locked(root)
def run_locked(root):
c=contract(root);lim=c['limits'];total=len(c['cases'])
if (root/'batch-started.json').exists():raise FileExistsError('Never repeat a custody batch/part')
os.sched_setaffinity(0,c['coordinator_cpus'])
start=time.monotonic();save(root/'batch-started.json',dict(pid=os.getpid(),unix=time.time(),
contract_sha256=sha(root/'contract.json'),code_freeze=read_json(root/'code-freeze.json')))
monitor=Monitor(root/'monitor-errors.jsonl');active={};done=[];cursor=0;reason=None;checked_first=False;accounting=True
live=mem=disk=0
env=dict(os.environ,OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
def interrupted(*_):
nonlocal reason
reason='NAIVE_COORDINATOR_INTERRUPTED'
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
def stop():terminate({k:(v['proc'],v['log']) for k,v in active.items()})
def collect():
nonlocal reason,accounting
for slot,item in list(active.items()):
code=item['proc'].poll()
if code is None:continue
p=root/'parts'/f"part-{item['index']:03d}"/'guard-result.json'
if not p.exists():
reason='MISSING_GUARD_RECEIPT';accounting=False
save(root/'infrastructure-stop.json',dict(index=item['index'],returncode=code,reason=reason,accounting_complete=False))
else:
receipt=read_json(p)
if receipt['index']!=item['index'] or not math.isfinite(receipt['worker']['cpu_seconds']) or receipt['worker']['cpu_seconds']<0:raise ValueError('Receipt mismatch')
if receipt.get('result_path'):checked(receipt['result_path'],receipt['result_sha256'])
done.append(receipt)
print(json.dumps(dict(event='NAIVE_PART_TERMINAL',index=item['index'],status=receipt['worker']['status'],completed=len(done))),flush=True)
item['log'].close();del active[slot]
try:
while active or (cursor<total and reason is None):
collect()
def usage():
samples=[]
for item in active.values():
x=tree_usage(item['proc'].pid);item['cpu_peak']=max(item['cpu_peak'],x[0]);samples.append((item['cpu_peak'],x[1]))
return sum(v[0] for v in samples),sum(v[1] for v in samples)
(live,mem),fresh=monitor.sample('live',usage,(live,mem));disk,ok=monitor.sample('disk',lambda:disk_usage(root),disk);fresh &= ok
if not fresh:reason='GLOBAL_MONITOR_ACCOUNTING_ERROR'
spent=sum(v['worker']['cpu_seconds'] for v in done)
over=budget_reason(0,spent,live,disk,cpu_limit=lim['preparation_cpu_seconds'],disk_limit=lim['disk_bytes'])
if over:reason=over
if not checked_first and len(done)==1 and not active and reason is None:
first=done[0]['worker'];forecast=first['cpu_seconds']*(1+1.25*(total-1))
approved=first['status']=='PASS' and forecast<=lim['preparation_cpu_seconds']
save(root/'first-part-cost-check.json',dict(continue_within_cap=approved,first_worker_CPU_s=first['cpu_seconds'],
forecast_CPU_s=forecast,method='First-part total worker CPU scaled to617 with25percent reserve; heuristic'))
checked_first=True
if not approved:reason='FIRST_PART_FAILURE_OR_COST_OVER_CAP'
if reason:stop()
ceiling=total if checked_first else 1;parallel=lim['workers'] if checked_first else 1
while fresh and reason is None and len(active)<parallel and cursor<ceiling:
if spent+live+(len(active)+1)*lim['part_cpu_seconds']>lim['preparation_cpu_seconds']:
if not active:reason='INSUFFICIENT_REMAINING_PART_CAP'
break
slot=next(k for k in range(lim['workers']) if k not in active)
dest=root/'parts'/f'part-{cursor:03d}';dest.mkdir(parents=True,exist_ok=True);log=(dest/'guard.log').open('x')
p=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(cursor),'--slot',str(slot)],env=env,
stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
active[slot]=dict(proc=p,log=log,index=cursor,cpu_peak=0.);cursor+=1
save(root/'progress.json',dict(completed=len(done),active={str(k):v['index'] for k,v in active.items()},next_index=cursor,
worker_CPU_s=spent,live_guard_and_worker_CPU_s=live,RSS=mem,disk_bytes=disk,first_checked=checked_first,
stop_reason=reason,monitor_errors=monitor.errors,fresh_accounting=fresh,elapsed_seconds=time.monotonic()-start))
if active or not fresh:time.sleep(.5)
except Exception as exc:
reason='COORDINATOR_EXCEPTION'
save(root/'coordinator-failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc()))
finally:
stop()
for item in active.values():item['proc'].wait()
collect()
complete=reason is None and len(done)==total and accounting
passed=sum(r['worker']['status']=='PASS' for r in done)
save(root/'batch-complete.json',dict(status='ALL_PARTS_TERMINAL' if complete else 'NAIVE_PREPARATION_INCOMPLETE',
stop_reason=reason,accounting_complete=accounting,jobs=done,passed=passed,failed=len(done)-passed,
not_run_indices=[i for i in range(total) if i not in {r['index'] for r in done}],
worker_CPU_s=sum(v['worker']['cpu_seconds'] for v in done),guard_CPU_s=sum(v['guard_CPU_s'] for v in done),
coordinator_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-start,
monitor_errors=monitor.errors,contract_sha256=sha(root/'contract.json'),retry=False))
return 0 if complete and passed==total else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);p.add_argument('--slot',type=int);a=p.parse_args()
raise SystemExit(guard(a.root,a.guard,a.slot) if a.guard is not None else run(a.root))
+190
View File
@@ -0,0 +1,190 @@
"""Synthetic mathematical/custody/queue checks; no real dataset decode or solve."""
import io
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from zipfile import ZipFile
import numpy as np
from freeze_g1_components import sha
from s2_custody_common import read_json
from naive_ordering_math import cdf_interval,pps_cdf,training_clock,squared_wall_distance,volume_clock_curve
import run_naive_ordering_part as part
import run_naive_ordering_queue as queue
import run_naive_ordering_analysis as analysis
from s2_models.features import R2_COLUMNS
class Checks(unittest.TestCase):
def test_CDF_strict_and_right_ties_mass_counted_once(self):
left,right=cdf_interval([3,1,3],[2,1,1])
np.testing.assert_array_equal(left,[.25,0,.25]);np.testing.assert_array_equal(right,[1,.25,1])
a,b=pps_cdf([3,1,3],[2,1,1]);np.testing.assert_array_equal(a,left)
expanded=np.repeat([3,1,3],[2,1,1]);el,_=pps_cdf(expanded,np.ones(4))
np.testing.assert_array_equal(el,[.25,.25,0,.25])
def test_invalid_PPS_or_empty_CDF_rejected(self):
for s,w in [([],[]),([1],[0]),([np.nan],[1]),([1],[np.inf])]:
with self.assertRaises(ValueError):cdf_interval(s,w)
with self.assertRaises(ValueError):pps_cdf([1,2],[.2,.8])
def test_conditional_CDF_differs_under_full_constant_flow(self):
full,_=cdf_interval([1,2,3,4],[1,1,1,1]);masked,_=pps_cdf([1,2],[1,1])
np.testing.assert_array_equal(full[:2],[0,.25]);np.testing.assert_array_equal(masked,[0,.5])
alpha=.5
self.assertTrue(np.all((full[:2]>=alpha*masked)&(full[:2]<=alpha*masked+1-alpha)))
# K/2000 alone is not total-cavity eligible mass when M+ is only0.4.
self.assertEqual(.4*1000/2000,.2)
def test_c_is_component_part_equal_learning_only(self):
def row(k,g,role):return dict(case=k,component_id=g,included=True,assignments=[dict(role=role,learning_pools=['100'])],preparation=dict(t_set=2.))
rows=[row('a','A','learning'),row('b','A','learning'),row('c','C','learning'),row('v','V','validation'),row('t','T','test')]
c,access=training_clock(rows,0,100,dict(a=2.,b=6.,c=10.))
self.assertEqual(c,3.5);self.assertEqual({r['case'] for r in access},set('abc'))
with self.assertRaises(KeyError):training_clock(rows,0,100,dict(a=2.,b=6.))
def test_Dw2_is_square_of_midpoint_not_mean_of_squares(self):
out,source,excluded=squared_wall_distance([[0,0,0],[1,0,0],[2,0,0]],[[0,1],[1,2]],[0,2,4],[0,0,0])
np.testing.assert_allclose(out,[0,1,1+1/9]);self.assertEqual(source,0);self.assertEqual(excluded,0)
with self.assertRaisesRegex(ValueError,'Unreachable'):
squared_wall_distance([[0,0,0],[1,0,0],[2,0,0]],[[0,1],[1,2]],[0,0,2],[0,0,0])
def test_order_and_final_ties_preserved(self):
s=np.array([-3.,1.,1.,9.]);ids=np.array([4,2,1,7]);left,_=pps_cdf(s,[1,2,3,1]);mapped=3.7*left
self.assertEqual(ids[s==s.max()].min(),ids[mapped==mapped.max()].min())
np.testing.assert_array_equal(np.sign(s[:,None]-s),np.sign(mapped[:,None]-mapped))
def test_curve_records_full_end_distinct_from_tet_max(self):
d=volume_clock_curve([0,.4,.8],[1,1,1],1.2,2.)
self.assertEqual(d['tet_end'],.8);self.assertEqual(d['full_VTU_end'],1.2)
self.assertEqual(d['seconds'][-1],.8);self.assertAlmostEqual(d['over_full_end'][-1],2/3)
def test_one_full_decode_cache_identity_and_no_duplicate_attempt(self):
with tempfile.TemporaryDirectory() as temp:
root=Path(temp);source=root/'source';source.mkdir();(root/'parts/part-000').mkdir(parents=True)
x=np.array([[0,0,0],[1,0,0],[0,1,0],[0,0,1.]])
edges=np.array([[0,1],[0,2],[0,3],[1,2],[1,3],[2,3]])
np.savez(source/'geometry.npz',points_raw=x,edges=edges,node_volume_raw=np.ones(4),raw_node_ids=np.arange(4),gate_location_raw=x[0])
features=np.zeros((4,20));features[:,8]=[0,1,1,1]
np.savez(source/'r2.npz',features=features,Df=[0,2,2,2],Vf=[0,.25,.25,.25])
np.savez(source/'queries.npz',eligible_s2=[True]*3,anchor_ids=[1,2,3],raw_node_ids=[1,2,3],multiplicity=[1,2,1])
(source/'result.json').write_text(json.dumps(dict(r2=dict(columns=list(R2_COLUMNS)),draw_slots=4)))
full=np.array([0.,.2,.4,.8,1.2]);old=root/'old.npz';np.savez(old,seconds=full[[1,2,3]],raw_ids=[1,2,3],counts=[1,2,1])
labels=root/'labels.json';labels.write_text(json.dumps([dict(case='case',path=str(old),sha256=sha(old))]))
archive=root/'archive.zip'
with ZipFile(archive,'w') as z:z.writestr('case_sim.vtu','<VTKFile byte_order="LittleEndian" compressor="vtkZLibDataCompressor"><UnstructuredGrid><Piece NumberOfPoints="5"><PointData><DataArray Name="Fill_time">x</DataArray></PointData></Piece></UnstructuredGrid></VTKFile>')
st=archive.stat();(root/'contract.json').write_text('{}')
c=dict(archive=str(archive),archive_stat=dict(bytes=st.st_size,mtime_ns=st.st_mtime_ns,inode=st.st_ino),label_index=str(labels),label_index_sha256=sha(labels),latest_ruling='synthetic')
prep=dict(preparation_root=str(source),t_set=1.,output_sha256={n:sha(source/n) for n in ['geometry.npz','r2.npz','queries.npz']},result_sha256=sha(source/'result.json'))
row=dict(case='case',component_id='A',preparation=prep)
with patch.object(part,'contract',return_value=c),patch.object(part,'rows',return_value=[row]),patch.object(part,'decode_array',return_value=full) as decode:
self.assertEqual(part.run(root,0),0)
with self.assertRaises(FileExistsError):part.run(root,0)
self.assertEqual(decode.call_count,1)
r=read_json(root/'parts/part-000/output/result.json')
self.assertEqual(r['full_VTU_T_end'],1.2);self.assertEqual(r['full_tet_T_end'],.8);self.assertEqual(r['query_T_max'],.8)
self.assertEqual(r['baseline_Dw_Vf_max_abs_difference'],0.)
self.assertEqual(r['raw_nodes'],5);self.assertEqual(r['tet_nodes'],4)
def test_neural_route_keeps_three_seeds_and_original_missing_mask(self):
roster=[dict(case='a',component_id='A',included=True,nodes=10,exposure=dict(development_exposed=False),assignments=[dict(fold=0,role='test')])]
def row(arm,seed):return dict(case='a',arm=arm,pool=100,seed=seed,**{k:dict(value=.2,reason=None) for k in ['relative_l2','near_wall','kendall_tau_b','last_location']})
data=[row('R2',None),row('REORDER_N',0),row('REORDER_N',1)]
tables,contrasts,_=analysis.reports(data,roster,{'a'})
r=next(x for x in contrasts if x['subset']=='all' and x['pool']==100 and x['candidate']=='REORDER_N')
self.assertEqual(r['components'],0)
data.append(row('REORDER_N',2));_,contrasts,_=analysis.reports(data,roster,{'a'})
r=next(x for x in contrasts if x['subset']=='all' and x['pool']==100 and x['candidate']=='REORDER_N')
self.assertEqual(r['components'],1)
def test_synthetic_mapping_scoring_and_exact_manifest_end_to_end(self):
from s2_custody_common import prediction_path
from s2_custody_statistics import part_metrics
with tempfile.TemporaryDirectory() as temp:
root=Path(temp);old=root/'old';old.mkdir();case='00000001_case'
(root/'contract.json').write_text('{}');(root/'code-freeze.json').write_text('{}');(root/'batch-complete.json').write_text('{}')
s2=old/'contract.json';s2.write_text('{}')
row=dict(case=case,component_id='A',included=True,nodes=3,exposure=dict(development_exposed=False),
assignments=[dict(fold=f,role='test' if f==0 else 'learning',learning_pools=['25','50','100']) for f in range(5)])
roster=root/'roster.json';roster.write_text(json.dumps(dict(parts=[row])))
truth=np.array([1.,2.,4.]);raw=np.arange(3);counts=np.array([1,2,1]);left=np.array([0.,1/3,2/3]);right=left+1/3
d=dict(ids=raw,raw_ids=raw,counts=counts,t_set=4.,near_wall=np.array([True,False,True]),coordinates_over_Rg=np.array([[0,0,0],[1,0,0],[2,0,0]]))
prepared=root/'part';prepared.mkdir()
np.savez(prepared/'full-domain-maps.npz',raw_ids=raw,truth_left=left,truth_right=right,df1_left=left,df1_right=right,df2_left=left,df2_right=right)
np.savez(prepared/'full-node-labels.npz',seconds=truth)
(prepared/'volume-clock-curve.json').write_text(json.dumps(volume_clock_curve(truth,np.ones(3),4.,4.)))
models=[]
for pool in [25,50,100]:
for arm in ['R2','N','E']:
for seed in ([None] if arm=='R2' else [0,1,2]):
models.append(dict(fold=0,pool=pool,arm=arm,seed=seed,name=f'{arm}-{pool}-{seed}'))
models.append(dict(fold=0,pool=None,arm='B0',seed=None,name='B0'))
index={};scores=[]
for m in models:
pred=4*left if m['arm']=='B0' else np.array([.9,2.2,3.7])
p=prediction_path(old,m,case);p.parent.mkdir(parents=True,exist_ok=True);np.savez(p,seconds=pred,raw_ids=raw,counts=counts)
index[str(p)]=sha(p)
scores.append(dict(case=case,component_id='A',fold=0,pool=m['pool'],arm=m['arm'],seed=m['seed'],
**part_metrics(pred,truth,counts,d['near_wall'],raw,d['coordinates_over_Rg'],1)))
pi=old/'index.json';pi.write_text(json.dumps(index));sp=old/'scores.json';sp.write_text(json.dumps(scores))
lp=old/'labels.npz';np.savez(lp,seconds=truth,raw_ids=raw,counts=counts)
li=old/'labels.json';li.write_text(json.dumps([dict(case=case,path=str(lp),sha256=sha(lp))]))
ex=root/'exposure.json';ex.write_text(json.dumps([dict(parts=[case])]))
c=dict(roster=str(roster),S2_contract=str(s2),S2_contract_sha256=sha(s2),prediction_index=str(pi),prediction_index_sha256=sha(pi),S2_root=str(old),
original_part_scores=str(sp),original_part_scores_sha256=sha(sp),label_index=str(li),label_index_sha256=sha(li),exposure_membership=str(ex),exposure_membership_sha256=sha(ex))
prep={case:(dict(full_VTU_T_end=4.,estimated_alpha=.6),prepared)}
with patch.object(analysis,'contract',return_value=c),patch.object(analysis,'prepared',return_value=prep),patch.object(analysis,'rows',return_value=[row]),patch.object(analysis,'prediction_inputs',return_value=d),patch.object(analysis,'expected_models',return_value=models),patch.object(analysis,'training_clock',return_value=(1.1,[])):
self.assertEqual(analysis.make_maps(root),0)
self.assertEqual(analysis.score_maps(root),0)
result=read_json(root/'scores/result.json');self.assertEqual(result['prediction_rows'],39);self.assertEqual(result['reorder_invariant_rows'],24)
mapped=read_json(root/'prediction-index.json');mapped.pop(next(iter(mapped)))
with self.assertRaisesRegex(ValueError,'Missing/extra'):analysis.validate_manifest(c,mapped)
def simulated_queue(self,root,fail_first=False,missing=False):
c=dict(cases=['a','b','c'],coordinator_cpus=[28,29],limits=dict(workers=4,part_cpu_seconds=180,preparation_cpu_seconds=28800,disk_bytes=2**30))
(root/'contract.json').write_text('{}');(root/'code-freeze.json').write_text('{}');spawned=[]
class Process:
def __init__(self,index):self.index=index;self.pid=800000+index;self.polls=0
def poll(self):
self.polls+=1
if self.polls<2:return None
if missing:return 1
failed=fail_first and self.index==0;dest=root/'parts'/f'part-{self.index:03d}'
(dest/'guard-result.json').write_text(json.dumps(dict(index=self.index,case=c['cases'][self.index],worker=dict(status='STOPPED_OR_FAILED' if failed else 'PASS',cpu_seconds=.1),guard_CPU_s=.01,result_path=None,result_sha256=None)))
return 1 if failed else 0
def wait(self):self.polls=2;return self.poll()
def spawn(command,**kwargs):
i=int(command[-3]);p=Process(i);spawned.append(p);return p
with patch.object(queue,'contract',return_value=c),patch.object(queue.os,'sched_setaffinity'),patch.object(queue.subprocess,'Popen',spawn),patch.object(queue,'tree_usage',return_value=(0,1,1)),patch.object(queue.time,'sleep'),patch.object(queue,'terminate'),patch('builtins.print'):
result=queue.run(root)
return result,spawned
def test_independent_part_queue_once_only_and_no_redecode_restart(self):
with tempfile.TemporaryDirectory() as temp:
root=Path(temp);code,spawned=self.simulated_queue(root)
self.assertEqual(code,0);self.assertEqual(len(spawned),3)
self.assertEqual(read_json(root/'batch-complete.json')['passed'],3)
self.assertTrue(read_json(root/'first-part-cost-check.json')['continue_within_cap'])
with patch.object(queue,'contract',return_value=dict(cases=[],limits={})),patch.object(queue.subprocess,'Popen') as spawn:
with self.assertRaises(FileExistsError):queue.run(root)
spawn.assert_not_called()
def test_failed_first_part_and_infrastructure_receipt_stop(self):
for missing in [False,True]:
with tempfile.TemporaryDirectory() as temp:
root=Path(temp);code,spawned=self.simulated_queue(root,fail_first=not missing,missing=missing)
self.assertEqual(code,1);self.assertEqual(len(spawned),1)
r=read_json(root/'batch-complete.json');self.assertEqual(r['status'],'NAIVE_PREPARATION_INCOMPLETE')
if missing:self.assertFalse(r['accounting_complete'])
def test_global_monitor_failure_prevents_any_dispatch(self):
with tempfile.TemporaryDirectory() as temp:
root=Path(temp)
with patch.object(queue.Monitor,'sample',side_effect=lambda op,fn,previous:(previous,False)):
code,spawned=self.simulated_queue(root)
self.assertEqual(code,1);self.assertEqual(len(spawned),0)
self.assertEqual(read_json(root/'batch-complete.json')['stop_reason'],'GLOBAL_MONITOR_ACCOUNTING_ERROR')
if __name__=='__main__':unittest.main(verbosity=2)