Freeze S3-A execution contract and cache-only scoring for main review
This commit is contained in:
1 parent
296c7fb741
commit
b0390231f0
14 files changed
+19188
No files matched your search
@@ -42,3 +42,5 @@
|
||||
- Latest custody COMPLETE (2026-11-15,终态核验): prediction42606 and custody87444 exited0, allguardsPASS.13449predictions frozen+codelocked c47dd6a before one decode of617parts; results1dea89d/report docs/s2-custody-results-2026-11-15.md. Main100pct209components/617parts: Delta1-E/N=-0.01292367747,95CI[-0.02122332192,-0.00563340237],harmful;R2bestprimaryerror,N usefulness vsR2 fails. Allnewmetricsvalid,undefinedbootstrap0. Prediction410.070561GPU-s,custody72.962534CPU-s. Do not restart/redecode/retrain/change frozen stats; all original NA preserved. Next team stage review ledClaude; no O1/organisation inference from this result. Read newest handoff, older running entries historical.
|
||||
|
||||
- Latest posthoc D1-D4 complete (2026-11-15): S2 acceptedClaude1794731320, stage47259fee/correctiond2d69cac. Authorized read-only diagnostics fixeddaed8e873afa7bf56fd3a58f6701f78ce566c5b0; report docs/s2-posthoc-diagnostics-2026-11-15.md; completion.json is latest. D1242.573114CPU-s,D317.133535CPU-s,13449D3rows,0undefined,2lowpoolE signed-oracle reversals retained; no newlabels/model/training/prediction change. S3-A/B not launched; require stage discussion and explicit execution contract. Do not replay earlier queues or treat oracle as deployable calibration.
|
||||
|
||||
- Latest S3-A preparation (2026-11-15): Claude1794732043 approved direction, requires one main review before running. Contract263b0b42b832d958ccf5843215403d629cf9d037084951d10f2af40bb89348d8 at artifacts/s3a-planning-2026-11-15;17syntheticchecksPASS. NewR2_E4/R2_G2,360HPO+30final planned, old15R2fits/predictions reused. CPU20h including18hfit+2hpostfit,0retry,noGPU,newdecode0. Scoring/data/budget helpers frozen; actual queue implementation and launch pending main review. Mold1794732176 final D3 read accepted, no recomputation; H not pure ordering, D2 cannot decide longer training success/failure. S3-B not launched/O1open. Do not restart S2.
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"status": "AWAITING_ONE_MAIN_REVIEW_NOT_STARTED",
|
||||
"contract_sha256": "263b0b42b832d958ccf5843215403d629cf9d037084951d10f2af40bb89348d8",
|
||||
"HPO_trials": 360,
|
||||
"new_final_fits": 30,
|
||||
"baseline_refits": 0,
|
||||
"labels_decoded": false,
|
||||
"fits_started": false,
|
||||
"predictions_started": false,
|
||||
"scientific_computation_started": false
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"status": "PREPARED_FOR_ONE_MAIN_REVIEW",
|
||||
"contract_sha256": "263b0b42b832d958ccf5843215403d629cf9d037084951d10f2af40bb89348d8",
|
||||
"original_S2_code_hashes_verified": true,
|
||||
"original_R2_models_verified": 15,
|
||||
"existing_custody_cache_hashes_verified": 617,
|
||||
"real_label_arrays_read": 0,
|
||||
"new_fits": 0,
|
||||
"new_predictions": 0,
|
||||
"tests_passed": 17,
|
||||
"queue_implementation": "pending after contract review; data, budget helpers, statistics and scoring code frozen",
|
||||
"direction_slack": "1794732043.086139",
|
||||
"Mold_final_read_slack": "1794732176.169409",
|
||||
"estimated_worker_CPU_h": 9.452898426319438
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"tests": 17,
|
||||
"failures": 0,
|
||||
"errors": 0,
|
||||
"cpu_seconds": 0.105099742,
|
||||
"wall_seconds": 0.12929833395173773,
|
||||
"real_models_fitted": 0,
|
||||
"real_labels_read": 0,
|
||||
"output": "test_budget_projection_uses_both_arms_and_stops_failed_first (test_s3a.S3AChecks) ... ok\ntest_cache_hash_ids_multiplicity_and_seconds_target (test_s3a.S3AChecks) ... ok\ntest_component_equal_single_fit_and_direction (test_s3a.S3AChecks) ... ok\ntest_dispatch_reserves_active_and_next_caps_before_launch (test_s3a.S3AChecks) ... ok\ntest_duplicate_scores_and_seed_replication_rejected (test_s3a.S3AChecks) ... ok\ntest_exact_feature_order_and_two_separate_arms (test_s3a.S3AChecks) ... ok\ntest_exposure_filters_parts_and_keeps_main_invalid_component_excluded (test_s3a.S3AChecks) ... ok\ntest_failure_invalidates_whole_component_not_other_arm (test_s3a.S3AChecks) ... ok\ntest_final_loader_reads_learning_only_from_cache (test_s3a.S3AChecks) ... ok\ntest_multiplicity_once_and_negative_prediction_not_clipped (test_s3a.S3AChecks) ... ok\ntest_original_failure_and_tie_policy (test_s3a.S3AChecks) ... ok\ntest_plan_is360_hpo30final_no_original_R2_refit (test_s3a.S3AChecks) ... ok\ntest_prediction_gate_rejects_missing_extra_before_array_IO (test_s3a.S3AChecks) ... ok\ntest_review_gate_blocks_missing_or_wrong_contract (test_s3a.S3AChecks) ... ok\ntest_terminal_manifest_requires_all30_cells_and_model_hash (test_s3a.S3AChecks) ... ok\ntest_test_role_rejected_before_any_feature_or_cache_access (test_s3a.S3AChecks) ... ok\ntest_undefined_bootstrap_and_six_region_boundaries_unchanged (test_s3a.S3AChecks) ... ok\n\n----------------------------------------------------------------------\nRan 17 tests in 0.129s\n\nOK\n",
|
||||
"code_sha256": {
|
||||
"tools/prepare_s3a.py": "7824b5e5c84578bc410be87dfb3b0a337ff61e41b2759182396adec680d8159a",
|
||||
"tools/run_s3a_score.py": "21dead7fc11ead75811819b229cf53d5e24231c27de3ed3da0d5f13b2d452e5f",
|
||||
"tools/s3a_budget.py": "65b847203c59ebf7338dd4118a4cb1d28ddcb72a13f05f60a96e23e66b736ac5",
|
||||
"tools/s3a_data.py": "b609692f70ffb8330f4279d8deac19918ce4995ea8164546fdb583f5181fbd01",
|
||||
"tools/s3a_statistics.py": "4b190ca8c9cba50b029d094e716d820dfe0f6abbbbd5f70f91048b5a91a1127b",
|
||||
"tools/test_s3a.py": "6715ae895f129f9a5c24fc747cffe839b1d6ce8f32b7cc72f6190690120fcea3"
|
||||
},
|
||||
"contract_sha256": "263b0b42b832d958ccf5843215403d629cf9d037084951d10f2af40bb89348d8"
|
||||
}
|
||||
@@ -35,3 +35,7 @@ Yanqi 经 TRON 在本会话与 Slack `1794661756.347349` 同步:
|
||||
Grok `1794661806.338179` 已正式见证 PASS。Claude `1794661802.418469` 与纠正后的 `1794661944.308269` 给出后续顺序:托管关联 → 无场标签合成复制定标 → 双路 k=3 候选计数 → 固定 det 分支配准(≤4,000 对、≤100 worker CPU-h、≤16 workers;超过 4,000 先报告)→ 阈值带小盲审 → 冻结组件,再衔接十件预处理、R2-G24/N/E 与 pilot。此为持续推进下 Claude 的阶段裁量;前述早期 E1 范围记录保留历史含义,不应再解释为后续每批次都要用户审批。实际校准、候选、配准和 pilot 各自在对应记录中陈述执行状态,不把裁量当作已经完成。
|
||||
|
||||
Claude 已撤回“约 6% 亲缘率上界”和“跨键更少”的外推;本批分层同键对仅确认 2 个视觉 rigid 对,27 个未决非负例,三类留验不足,跨键召回未认证。保守并组是工程取舍,并非统计保证。
|
||||
|
||||
## S2结果回顾后:S3-A准备,待一次主审(2026-11-15)
|
||||
|
||||
已汇总Claude1794732043的方向裁量、Mold1794732176对D3实际材料/源码的最终核读和既有Grok1794731529对S2结果的定点核对。研究目标、新强基线、已检验/未检验命题与D2/D3解释边界收于`docs/s3a-execution-contract-2026-11-15.md`。S3-A仅两条R2新增特征臂;复用原R2,不再解码标签。17项合成验收通过,合同/评分冻结,实际队列尚未实现或启动。Claude要求一次合同主审;这不是再次请求用户分享许可。S3-B未启动,O1仍独立开放,不重复计算已完成的D1-D4。
|
||||
@@ -1,3 +1,7 @@
|
||||
# Latest handoff: S3-A contract prepared, not launched
|
||||
|
||||
Claude1794732043.086139 and Mold1794732176.169409 were read directly. S2 and D1-D4 are complete; all earlier running entries below are historical. New review-only package: `artifacts/s3a-planning-2026-11-15`, contract `263b0b42b832d958ccf5843215403d629cf9d037084951d10f2af40bb89348d8`, document `docs/s3a-execution-contract-2026-11-15.md`. 17 synthetic checks PASS, old source hashes/15 R2 model hashes/617 cache file hashes verified without decoding targets. Fit/prediction/scientific execution not started. Data and frozen score implementation ready; queue implementation is next after the required single main review. CPU-only total20h;360HPO+30newfinal maximum; reuse original R2. Original S2 sources/results untouched.
|
||||
|
||||
# S2 当前执行接续记录
|
||||
|
||||
## S2已接受,D1–D4事后诊断也已交付(2026-11-15)
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
# S3-A 短执行合同:在 R2 内检验 O2 读数
|
||||
|
||||
状态:**方向批准、待一次主审;本提交仅准备合同、缓存读取和冻结评分代码及合成验收。未启动真实拟合、预测或新评分。** 主审来源为 Claude [1794732043.086139](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794732043086139)。主审通过后按本合同实现运行队列,记录实现固定版本、实际进程及预算;不可把本准备记录当实际启动。S3-B 未启动,O1 保持开放。
|
||||
|
||||
## 问题、范围与比较
|
||||
|
||||
研究目标仍是面向新件、实用且有样本效率的填充代理。S2 的 E/N 不利结果只适用于已训练骨干与共同查询域,并未否定独立的拓扑/类型分块或真实连通路线。当前强对照是 R2-G24 的四面体边 Dijkstra / Gre24 风格近似,不能称原论文 eikonal 复现。
|
||||
|
||||
沿用固定617件、209组件、5折、25/50/100%嵌套组件池、原共同有效query ID及multiplicity。**既有15个R2 final模型、1851份R2测试预测和逐件评分原样复用,绝不重拟合R2。** 新增两臂各15格、每格原12配置HPO加一次learning-only final,最多360 HPO +30 final:
|
||||
|
||||
- 主比较:R2_E4 = 原20列R2 + `mean_q1,max_q1,mean_q2,max_q2`,保持原query缓存的列顺序;100%池 Δ=1−mean(relativeL2_R2_E4)/mean(relativeL2_R2)。
|
||||
- 描述性比较:R2_G2 = 原20列R2 + `g12,g23`,不同时加E4。其余池及近壁、Kendall τ、最后填充位置、最差组件均描述性。
|
||||
|
||||
sklearn1.7.2,原12配置与顺序不变:lr .05/.1 ×leaves31/63 ×iter200/500/1000;min_samples_leaf20、random_state0、early_stopping=False。目标T/t_set;无新增标准化、插补、裁剪或后处理。sample_weight沿原multiplicity归一、件/组件等权,总和1、不放大。验证以组件等权relativeL2选最优成功配置,平手按固定网格次序;失败配置保留NA。12个全失败或final失败使该格NA,不补跑、不补组件、不得因新臂失败修改617名单。final只读同learning池,不并入validation。
|
||||
|
||||
全部标签来自已存在的617件S2 custody缓存;不再次解码原始数据。训练缓存读取必须先校验当前fold角色,拒绝test;final不读validation。S2 test已经公开,此为结果知情的后续探索,不能称新盲测或独立确认性证据;not-known-exposed一行也不恢复未见性质。
|
||||
|
||||
## 预算、停止与冻结
|
||||
|
||||
CPU-only,**总计≤20 worker CPU-h**:拟合上限18h,预留2h给测试预测与评分;最多4个单线程worker。沿原单fit≤1800 CPU-s、7200 wall-s、16GiB树RSS、5GiB目录;总新目录≤100GiB。预测/评分单任务wall≤3h,合计在预留7200 CPU-s内。加载、失败、推理和评分都计入worker账;guard/coordinator另报,活跃树保守计费。分配下一作业前需预留全部在途作业剩余上限;采样超限即停并如实报告超出量,零重试。
|
||||
|
||||
原R2全部195任务实际2.5923125 worker CPU-h;按两新臂列数24/20、22/20缩放,再加25%余量,拟合启发式约7.45h,另预留2h,合计约9.45h。不是保证。首次E4/G2 HPO(固定job0/job12)依次单独执行,按各臂learning query×iterations×leaves及加载成本外推,合计含25%余量须≤18h才继续4worker;任一首试失败或超预算即停止报告,不自动重试/扩大预算。
|
||||
|
||||
所有360 HPO与30 final终态及checkpoint哈希先冻结,再产生测试预测;全体成功模型在各自test fold的预测按准确名单hash冻结,缺失/额外文件拒绝评分。`terminal-freeze.json`记录30个PASS/NA,`predictions-complete.json`绑定合同、terminal-freeze、prediction-index与Git提交。此处为必需证据次序,不是逐批重复用户审批。主审记录一次写入`main-review.json`并绑定合同SHA和固定Git提交。
|
||||
|
||||
## 冻结统计
|
||||
|
||||
`tools/s3a_statistics.py`直接复用不改动的`tools/s2_custody_statistics.py`:原relativeL2、multiplicity只一次、件/组件等权;每臂单fit,不能复制为3seed。每个比较仅纳入两臂都有所有件的完整组件;不沿用S2神经50%池的缺失掩码。报告每臂和每个contrast的件/节点/组件/折覆盖与所有NA。
|
||||
|
||||
组件合并五折、ID排序,PCG64(SeedSequence(2026111408))、2000次同索引配对bootstrap、linear百分位95%区间,六区表m=0.10沿原未舍入边界。零分母replicate保留undefined及索引、不补抽。not-known-exposed采用原冻结522成员∩617名单∩各比较主分析有效组件,组件内按件先过滤,混合组件不整组删除。原R2逐件分数不重算;新臂同一缓存真值、query IDs、multiplicity计分。主分析定义为一个E4/100%比较;G2或小池有利不能改选主分析。
|
||||
|
||||
## 阶段回顾与解释限制
|
||||
|
||||
Mold [1794732176.169409](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794732176169409)已实读`daed8e87`报告、分解行/汇总与源码,确认13,449行恒等式及100%四臂汇总;本阶段不重复计算。两个负a是同一件的两行,并非两件。
|
||||
|
||||
N的oracle H=0.013707仍高于原R2 mean(e²)=0.007320,因此单一幅值修正无法填平**这项均方诊断**。12.4148%=S_N/e²_N与81.9279%=(H_N−H_R2)/(e²_N−e²_R2)分母不同;都不能改写为主mean(relativeL2)改善。H含相对幅度/局部时序间距/空间误差,非纯排序。D2晚期小幅下降不能外推延长训练必胜或必败,合同不采用“更长训练无效”的断言。D4几何分箱只描述,节点/覆盖量不是性能。
|
||||
|
||||
Claude主审及下一阶段的决定以原串明确记录为准。用户对受控Drive、Git、Slack的完整共享授权已在`docs/sharing-authorization-2026-11-14.md`保存;合同、原始失败、审查范围和解释限制一并共享,不重复请求分享许可。
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Freeze a review package only: no fitting, inference, labels or geometry calculation."""
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
from freeze_g1_components import sha
|
||||
from prepare_s2_r2 import build_jobs as old_jobs
|
||||
from s2_custody_common import checked, read_json
|
||||
from s2_models.features import histgbrt_grid
|
||||
from s2_recovery_supervisor import save
|
||||
from s3a_data import ARMS, columns
|
||||
|
||||
ROOT = Path('artifacts/s3a-planning-2026-11-15')
|
||||
|
||||
|
||||
def build_jobs(roster):
|
||||
jobs, cells = old_jobs(roster)
|
||||
new_jobs, new_cells = [], []
|
||||
for cell in cells:
|
||||
for arm in ARMS:
|
||||
item = copy.deepcopy(cell)
|
||||
item.update(index=len(new_cells), arm=arm, hpo_indices=[])
|
||||
for i in cell['hpo_indices']:
|
||||
job = copy.deepcopy(jobs[i])
|
||||
job.update(index=len(new_jobs), cell=item['index'], arm=arm)
|
||||
item['hpo_indices'].append(job['index'])
|
||||
new_jobs.append(job)
|
||||
new_cells.append(item)
|
||||
for cell in new_cells:
|
||||
original = next(j for j in jobs if j['kind'] == 'FINAL' and j['cell'] == cell['index']//2)
|
||||
job = copy.deepcopy(original)
|
||||
job.update(index=len(new_jobs), cell=cell['index'], arm=cell['arm'])
|
||||
new_jobs.append(job)
|
||||
return new_jobs, new_cells
|
||||
|
||||
|
||||
def main():
|
||||
if ROOT.exists():
|
||||
raise FileExistsError('Never overwrite a frozen S3-A plan')
|
||||
s2 = Path('artifacts/s2-custody-evaluation-2026-11-15')
|
||||
previous = read_json(s2/'contract.json')
|
||||
for p, h in previous['code_sha256'].items():
|
||||
checked(p, h)
|
||||
r2 = Path('artifacts/s2-r2-main-2026-11-15')
|
||||
baseline = read_json(r2/'batch-complete.json')
|
||||
finals = baseline['final_fits']
|
||||
if len(finals) != 15 or any(f['status'] != 'PASS' for f in finals):
|
||||
raise ValueError('Expected all 15 original R2 final fits')
|
||||
for f in finals:
|
||||
checked(f['model_path'], f['model_sha256'])
|
||||
roster = read_json(checked(previous['roster'], previous['roster_sha256']))
|
||||
jobs, cells = build_jobs(roster)
|
||||
labels = s2/'custody/label-index.json'
|
||||
entries = read_json(labels)
|
||||
included = {r['case'] for r in roster['parts'] if r['included']}
|
||||
if len(entries) != 617 or {r['case'] for r in entries} != included:
|
||||
raise ValueError('617 cached label manifest required')
|
||||
# Byte hashing is provenance verification, not target decoding or scoring.
|
||||
for e in entries:
|
||||
checked(e['path'], e['sha256'])
|
||||
paths = set(previous['code_sha256'])
|
||||
paths.update(str(p) for p in Path('tools').glob('*s3a*.py'))
|
||||
sources = [previous['roster'], previous['exposure_membership'], str(s2/'contract.json'),
|
||||
str(s2/'custody/part-scores.json'), str(s2/'custody/result.json'), str(labels),
|
||||
str(s2/'prediction-index.json'), str(s2/'predictions-complete.json'), str(r2/'batch-complete.json'),
|
||||
str(r2/'contract.json'), 'docs/s3a-execution-contract-2026-11-15.md',
|
||||
'artifacts/s2-posthoc-diagnostics-2026-11-15/completion.json']
|
||||
historical = baseline['worker_cpu_seconds']
|
||||
forecast = historical*(24/20+22/20)*1.25
|
||||
c = dict(version='S3-A-review-v1', status='AWAITING_ONE_MAIN_REVIEW_NOT_STARTED',
|
||||
direction_ruling='1794732043.086139', execution_approved=False,
|
||||
root='artifacts/s3a-execution-2026-11-15',
|
||||
roster=previous['roster'], roster_sha256=previous['roster_sha256'],
|
||||
exposure_membership=previous['exposure_membership'], exposure_membership_sha256=previous['exposure_membership_sha256'],
|
||||
label_index=str(labels), label_index_sha256=sha(labels),
|
||||
baseline_scores=str(s2/'custody/part-scores.json'), baseline_scores_sha256=sha(s2/'custody/part-scores.json'),
|
||||
baseline_prediction_index=str(s2/'prediction-index.json'), baseline_prediction_index_sha256=sha(s2/'prediction-index.json'),
|
||||
baseline_fits=finals, baseline_refit=False, baseline_predictions_reused=True,
|
||||
jobs=jobs, cells=cells, grid=histgbrt_grid(), sklearn_version='1.7.2',
|
||||
feature_columns={a: columns(a) for a in ARMS}, no_scaler_or_imputation=True, target='T/t_set',
|
||||
final_refit='same_learning_pool_only', selection_policy='best_finished_config_fixed_grid_tie_order',
|
||||
weighting='Original multiplicity / part total / parts in component / component count; sum1, no rescaling',
|
||||
worker_cpus=[8, 10, 12, 14], coordinator_cpus=[30, 31],
|
||||
limits=dict(workers=4, worker_cpu_seconds=72000, fitting_worker_cpu_seconds=64800,
|
||||
postfit_reserved_cpu_seconds=7200, per_fit_cpu_seconds=1800, per_fit_wall_seconds=7200,
|
||||
per_fit_tree_rss_bytes=16*2**30, per_fit_disk_bytes=5*2**30, disk_bytes=100*2**30,
|
||||
prediction_scoring_wall_seconds=10800, retries=0, gpu_seconds=0),
|
||||
budget_estimate=dict(historical_R2_worker_cpu_seconds=historical,
|
||||
fit_projection_cpu_seconds=forecast, postfit_reserve_cpu_seconds=7200,
|
||||
projected_total_cpu_hours=(forecast+7200)/3600,
|
||||
method='Two feature-count-scaled original195-job costs plus25percent reserve; heuristic not guaranteed'),
|
||||
first_fit_checkpoint='First E4 and G2 HPO trials sequential, frozen job0/job12, then per-arm query/iteration/leaf projection with25percent reserve; continue4workers only if combined<=18CPUh. Failure or over-cap stops without retry.',
|
||||
budget_accounting='Charge all worker loading/fits/failures/inference/scoring; separately report guards/coordinator. Live tree accounting conservative; reserve remaining running caps before dispatch. Sampled overshoot fails and is disclosed.',
|
||||
failure_policy='All12 HPO terminal per cell, choose best finite finished validation score, grid-index ties; all12fail or finalfail=>cellNA. No new geometry exclusion or retry. Terminal and checkpoint freeze precede any test prediction.',
|
||||
statistics=dict(primary='1-mean_L2(R2_E4)/mean_L2(R2)', primary_pool=100,
|
||||
descriptive_arms=['R2_G2'], replicates=2000, seed=2026111408, margin=.1,
|
||||
aggregation='Equal parts then components; deterministic single fit, no seed triplication',
|
||||
paired_validity='Every part in both arms within a component; no inherited S2 neural NA mask',
|
||||
exposure='Original522 membership intersect617 roster intersect each contrast main-valid components; filter parts first',
|
||||
inherited_code='tools/s2_custody_statistics.py unchanged; percentile linear; six-region; undefined replicates retained'),
|
||||
target_policy='Existing617 custody cache only; reject currentfold test before training IO; final learningonly; no archive decoder',
|
||||
scoring_gate='One main-review approval references frozen contract/code Git commit; all30 new final cells terminal (PASS/NA), model hashes frozen; all successful predictions then hash-frozen with exact expected manifest before scoring',
|
||||
interpretation='Adaptive post-S2 follow-up on already disclosed test outcomes; not fresh confirmatory holdout. Not-known-exposed remains descriptive.',
|
||||
code_sha256={p: sha(p) for p in sorted(paths)}, source_sha256={p: sha(p) for p in sources})
|
||||
ROOT.mkdir()
|
||||
save(ROOT/'contract.json', c)
|
||||
save(ROOT/'prelaunch.json', dict(status=c['status'], contract_sha256=sha(ROOT/'contract.json'),
|
||||
HPO_trials=360, new_final_fits=30, baseline_refits=0, labels_decoded=False,
|
||||
fits_started=False, predictions_started=False, scientific_computation_started=False))
|
||||
print(json.dumps(read_json(ROOT/'prelaunch.json')))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Score only hash-frozen new predictions against the existing S2 custody cache."""
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import time
|
||||
import numpy as np
|
||||
from freeze_g1_components import sha
|
||||
from s2_custody_common import checked, read_json, prediction_inputs
|
||||
from s2_recovery_supervisor import save
|
||||
from s3a_data import ARMS, cached_truth
|
||||
from s3a_statistics import part_metrics, reports
|
||||
|
||||
|
||||
def review_gate(plan):
|
||||
c = read_json(plan/'contract.json')
|
||||
for path, digest in {**c['code_sha256'], **c['source_sha256']}.items():
|
||||
checked(path, digest)
|
||||
approval = read_json(plan/'main-review.json')
|
||||
if (approval.get('approved') is not True or not approval.get('slack_ts') or not approval.get('git_commit')
|
||||
or approval.get('contract_sha256') != sha(plan/'contract.json')):
|
||||
raise PermissionError('S3-A main review missing or mismatched')
|
||||
return c
|
||||
|
||||
|
||||
def expected_predictions(root, c, terminals, roster):
|
||||
by_cell = {r['cell']: r for r in terminals}
|
||||
if len(terminals) != 30 or len(by_cell) != 30 or set(by_cell) != set(range(30)):
|
||||
raise ValueError('Exactly 30 unique new final terminal cells required')
|
||||
expected = {}
|
||||
for cell in c['cells']:
|
||||
terminal = by_cell[cell['index']]
|
||||
if (terminal['arm'], terminal['fold'], terminal['pool']) != (cell['arm'], cell['fold'], cell['pool']):
|
||||
raise ValueError('Final cell identity mismatch')
|
||||
if terminal['status'] == 'NA':
|
||||
if not terminal.get('reason'):
|
||||
raise ValueError('NA requires original reason')
|
||||
continue
|
||||
if terminal['status'] != 'PASS':
|
||||
raise ValueError('Nonterminal final cell')
|
||||
checked(terminal['model_path'], terminal['model_sha256'])
|
||||
for row in roster:
|
||||
if row['included'] and row['assignments'][cell['fold']]['role'] == 'test':
|
||||
path = root/'predictions'/f"cell-{cell['index']:02d}"/(row['case']+'.npz')
|
||||
expected[str(path)] = (cell, row)
|
||||
return expected
|
||||
|
||||
|
||||
def prediction_gate(plan, root, c):
|
||||
proof = read_json(root/'predictions-complete.json')
|
||||
if (proof['status'] != 'ALL_S3A_PREDICTIONS_FROZEN' or not proof.get('git_commit')
|
||||
or proof['contract_sha256'] != sha(plan/'contract.json')):
|
||||
raise ValueError('New predictions must be frozen against approved contract')
|
||||
frozen = read_json(checked(root/'terminal-freeze.json', proof['terminal_freeze_sha256']))
|
||||
if (frozen['status'] != 'ALL_S3A_FITS_TERMINAL' or frozen['contract_sha256'] != sha(plan/'contract.json')
|
||||
or sorted(frozen['hpo_terminal_indices']) != list(range(360))):
|
||||
raise ValueError('All 360 HPO and 30 final terminal records required before test predictions')
|
||||
checked(frozen['batch_receipt'], frozen['batch_receipt_sha256'])
|
||||
roster = read_json(checked(c['roster'], c['roster_sha256']))['parts']
|
||||
expected = expected_predictions(root, c, frozen['final_fits'], roster)
|
||||
index = read_json(checked(root/'prediction-index.json', proof['prediction_index_sha256']))
|
||||
if set(index) != set(expected):
|
||||
raise ValueError('Missing or extra new predictions; cannot silently drop part')
|
||||
for p, h in index.items():
|
||||
checked(p, h)
|
||||
return roster, expected, index, frozen
|
||||
|
||||
|
||||
def baseline_rows(c, roster):
|
||||
rows = [r for r in read_json(checked(c['baseline_scores'], c['baseline_scores_sha256'])) if r['arm'] == 'R2']
|
||||
expected = {(r['case'], p) for r in roster if r['included'] for p in (25, 50, 100)}
|
||||
if len(rows) != len(expected) or {(r['case'], r['pool']) for r in rows} != expected:
|
||||
raise ValueError('Original R2 part scores incomplete or duplicated')
|
||||
if any(r['seed'] is not None for r in rows):
|
||||
raise ValueError('R2 is single fit, not three copies')
|
||||
index = read_json(checked(c['baseline_prediction_index'], c['baseline_prediction_index_sha256']))
|
||||
paths = {p: h for p, h in index.items() if Path(p).parent.name.endswith('-R2')}
|
||||
actual = {(Path(p).stem, int(Path(p).parent.name.split('-pool')[1].split('-')[0])) for p in paths}
|
||||
if len(paths) != len(expected) or actual != expected:
|
||||
raise ValueError('Original R2 prediction manifest differs')
|
||||
for p, h in paths.items():
|
||||
checked(p, h)
|
||||
for model in c['baseline_fits']:
|
||||
checked(model['model_path'], model['model_sha256'])
|
||||
return rows
|
||||
|
||||
|
||||
def run(plan, root):
|
||||
c = review_gate(plan)
|
||||
roster, expected, index, frozen = prediction_gate(plan, root, c)
|
||||
dest = root/'scores'
|
||||
dest.mkdir(exist_ok=False)
|
||||
start = time.monotonic()
|
||||
cpu = time.process_time()
|
||||
save(dest/'started.json', dict(contract_sha256=sha(plan/'contract.json'), unix=time.time(),
|
||||
predictions_complete_sha256=sha(root/'predictions-complete.json')))
|
||||
scores = baseline_rows(c, roster)
|
||||
entries = read_json(checked(c['label_index'], c['label_index_sha256']))
|
||||
labels = {r['case']: r for r in entries}
|
||||
members = {case for f in read_json(checked(c['exposure_membership'], c['exposure_membership_sha256'])) for case in f['parts']}
|
||||
# Load one part at a time, use identical IDs for all new arms/pools, retain finite negative predictions.
|
||||
by_case = {}
|
||||
for p, (cell, row) in expected.items():
|
||||
by_case.setdefault(row['case'], []).append((p, cell, row))
|
||||
for case, items in sorted(by_case.items()):
|
||||
d = prediction_inputs(items[0][2])
|
||||
truth = cached_truth(labels[case], d)
|
||||
for path, cell, row in items:
|
||||
with np.load(checked(path, index[path]), allow_pickle=False) as z:
|
||||
if not np.array_equal(z['raw_ids'], d['raw_ids']) or not np.array_equal(z['counts'], d['counts']):
|
||||
raise ValueError('Prediction/query IDs or multiplicity mismatch')
|
||||
metrics = part_metrics(z['seconds'], truth, d['counts'], d['near_wall'],
|
||||
d['raw_ids'], d['coordinates_over_Rg'], case.split('_')[0])
|
||||
scores.append(dict(case=case, component_id=row['component_id'], fold=cell['fold'],
|
||||
pool=cell['pool'], arm=cell['arm'], seed=None, **metrics))
|
||||
tables, contrasts, components = reports(scores, roster, members)
|
||||
save(dest/'part-scores.json', scores)
|
||||
save(dest/'tables.json', tables)
|
||||
save(dest/'contrasts.json', contrasts)
|
||||
save(dest/'component-scores.json', components)
|
||||
save(dest/'learning-pool-counts.json', read_json(c['roster'])['fold_pool_counts'])
|
||||
save(dest/'result.json', dict(status='S3A_SCORED', new_prediction_rows=len(expected),
|
||||
reused_R2_score_rows=len(scores)-len(expected), primary=next(r for r in contrasts if r['primary']),
|
||||
labels_newly_decoded=0, original_R2_refits=0, cpu_seconds=time.process_time()-cpu,
|
||||
wall_seconds=time.monotonic()-start, final_NA=[r for r in frozen['final_fits'] if r['status']=='NA'],
|
||||
interpretation=c['interpretation'], output_sha256={p.name: sha(p) for p in dest.glob('*.json')}))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('plan', type=Path)
|
||||
parser.add_argument('root', type=Path)
|
||||
args = parser.parse_args()
|
||||
run(args.plan, args.root)
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Pure budget decisions for the planned S3-A queue; no task launch."""
|
||||
import math
|
||||
|
||||
|
||||
def first_fit_projection(c, first_by_arm):
|
||||
estimates = {}
|
||||
for arm in ('R2_E4', 'R2_G2'):
|
||||
first = first_by_arm[arm]
|
||||
if first['status'] != 'PASS':
|
||||
return dict(continue_within_cap=False, reason='FIRST_TRIAL_FAILED_NO_RETRY', arm=arm)
|
||||
jobs = [j for j in c['jobs'] if j['arm'] == arm]
|
||||
initial = jobs[0]
|
||||
cpu, fit = first['worker_cpu_seconds'], first['fit_cpu_seconds']
|
||||
if not math.isfinite(cpu) or not math.isfinite(fit) or not 0 <= fit <= cpu:
|
||||
raise ValueError('Invalid first fit CPU accounting')
|
||||
estimates[arm] = cpu + 1.25 * (fit / initial['cost_units'] * sum(j['cost_units'] for j in jobs[1:])
|
||||
+ (cpu-fit) * sum(j['cost_queries']/initial['cost_queries'] for j in jobs[1:]))
|
||||
total = sum(estimates.values())
|
||||
return dict(continue_within_cap=total <= c['limits']['fitting_worker_cpu_seconds'],
|
||||
projected_fitting_worker_cpu_seconds=total, by_arm=estimates,
|
||||
method='Per-arm actual first trial; cost/query scaling,25percent reserve; heuristic')
|
||||
|
||||
|
||||
def can_dispatch(c, settled_cpu, live_charged_cpu, active_remaining_caps, next_cap):
|
||||
values = [settled_cpu, live_charged_cpu, next_cap, *active_remaining_caps]
|
||||
if any(not math.isfinite(v) or v < 0 for v in values) or next_cap <= 0:
|
||||
raise ValueError('Unknown/invalid CPU accounting blocks dispatch')
|
||||
return sum(values) <= c['limits']['fitting_worker_cpu_seconds']
|
||||
@@ -0,0 +1,64 @@
|
||||
"""S3-A fixed feature extensions and cache-only targets; no archive decoder."""
|
||||
import numpy as np
|
||||
from s2_custody_common import checked, read_json, prediction_inputs
|
||||
from s2_main_data import select_cell
|
||||
from s2_pilot_data import permitted_role
|
||||
from s2_models.features import R2_COLUMNS
|
||||
from s2_r2_data import learning_arrays, validation_score
|
||||
|
||||
ARMS = ('R2_E4', 'R2_G2')
|
||||
EXTRA_COLUMNS = {'R2_E4': ('mean_q1', 'max_q1', 'mean_q2', 'max_q2'),
|
||||
'R2_G2': ('g12', 'g23')}
|
||||
|
||||
|
||||
def columns(arm):
|
||||
if arm not in ARMS:
|
||||
raise ValueError('Only the two new arms may be fitted; never refit R2')
|
||||
return list(R2_COLUMNS + EXTRA_COLUMNS[arm])
|
||||
|
||||
|
||||
def features(d, arm):
|
||||
expected = columns(arm)
|
||||
x = np.asarray(d['x'])
|
||||
extra = np.asarray(d['edge' if arm == 'R2_E4' else 'gaps'])
|
||||
if x.ndim != 2 or x.shape[1] != 20 or extra.shape != (len(x), len(expected)-20):
|
||||
raise ValueError('Frozen feature order/shape mismatch')
|
||||
if not len(x) or not np.isfinite(x).all() or not np.isfinite(extra).all():
|
||||
raise ValueError('Nonfinite or empty common query features')
|
||||
return np.column_stack((x, extra))
|
||||
|
||||
|
||||
def cached_truth(entry, d):
|
||||
with np.load(checked(entry['path'], entry['sha256']), allow_pickle=False) as z:
|
||||
if not np.array_equal(z['raw_ids'], d['raw_ids']) or not np.array_equal(z['counts'], d['counts']):
|
||||
raise ValueError('Custody cache/query IDs or multiplicity mismatch')
|
||||
truth = z['seconds'].copy()
|
||||
if truth.shape != np.asarray(d['raw_ids']).shape or not np.isfinite(truth).all() or np.any(truth < 0):
|
||||
raise ValueError('Invalid cached target')
|
||||
return truth
|
||||
|
||||
|
||||
def read_part(row, roles, label_index, arm):
|
||||
role = permitted_role(roles, row['case']) # reject current test before any IO
|
||||
columns(arm)
|
||||
d = prediction_inputs(row)
|
||||
entry = label_index[row['case']]
|
||||
truth = cached_truth(entry, d)
|
||||
return dict(case=row['case'], component_id=row['component_id'], role=role,
|
||||
x=features(d, arm), y=truth/d['t_set'], seconds=truth, counts=d['counts'],
|
||||
t_set=d['t_set'], query_ids=d['ids'], label_sha256=entry['sha256'],
|
||||
label_mode='S2_custody_cache_role_checked_no_decode')
|
||||
|
||||
|
||||
def load_cell(root, c, job):
|
||||
columns(job['arm'])
|
||||
rows, roles = select_cell(read_json(checked(c['roster'], c['roster_sha256'])), job['fold'], job['pool'])
|
||||
entries = read_json(checked(c['label_index'], c['label_index_sha256']))
|
||||
labels = {r['case']: r for r in entries}
|
||||
if len(labels) != len(entries):
|
||||
raise ValueError('Duplicate cached label case')
|
||||
cases = roles['learning_parts'] + (roles['validation_parts'] if job['kind'] == 'HPO' else [])
|
||||
if job['kind'] not in ('HPO', 'FINAL'):
|
||||
raise ValueError('Unknown job kind')
|
||||
parts = [read_part(rows[k], roles, labels, job['arm']) for k in sorted(cases)]
|
||||
return [p for p in parts if p['role'] == 'learning'], [p for p in parts if p['role'] == 'validation'], roles
|
||||
@@ -0,0 +1,45 @@
|
||||
"""S3-A reporting using unchanged S2 metrics, aggregation and bootstrap."""
|
||||
import numpy as np
|
||||
from s2_custody_statistics import component_values, contrast, part_metrics
|
||||
from s3a_data import ARMS
|
||||
|
||||
|
||||
def reports(rows, roster, frozen_members):
|
||||
keys = [(r['case'], r['arm'], r['pool'], r['seed']) for r in rows]
|
||||
if len(keys) != len(set(keys)):
|
||||
raise ValueError('Duplicate part/arm/pool score')
|
||||
if any(r['arm'] not in ('R2', *ARMS) or r['pool'] not in (25, 50, 100) or r['seed'] is not None for r in rows):
|
||||
raise ValueError('Unexpected arm/pool/seed')
|
||||
tables, contrasts, component_rows = [], [], []
|
||||
for subset in ('all', 'not_known_exposed'):
|
||||
for pool in (25, 50, 100):
|
||||
primary = {}
|
||||
for metric in ('relative_l2', 'near_wall', 'kendall_tau_b', 'last_location'):
|
||||
for arm in ('R2', *ARMS):
|
||||
values, excluded, details = component_values(rows, roster, arm, pool, metric,
|
||||
subset != 'all', frozen_members)
|
||||
worst = max(values, key=values.get) if values and metric == 'relative_l2' else None
|
||||
tables.append(dict(subset=subset, pool=pool, arm=arm, metric=metric,
|
||||
mean=float(np.mean(list(values.values()))) if values else None,
|
||||
components=len(values), included_parts=sum(details[k]['parts'] for k in values),
|
||||
included_nodes=sum(details[k]['nodes'] for k in values), excluded_components=excluded,
|
||||
fold_means={str(f): float(np.mean(v)) if v else None for f in range(5)
|
||||
for v in [[value for k, value in values.items() if details[k]['fold'] == f]]},
|
||||
worst_component=dict(component_id=worst, value=values[worst], **details[worst]) if worst else None))
|
||||
component_rows.extend(dict(subset=subset, pool=pool, arm=arm, metric=metric,
|
||||
component_id=k, value=v, **details[k]) for k, v in values.items())
|
||||
if metric == 'relative_l2':
|
||||
primary[arm] = (values, details)
|
||||
reference, details = primary['R2']
|
||||
for arm in ARMS:
|
||||
candidate, other_details = primary[arm]
|
||||
pair = set(reference) & set(candidate)
|
||||
excluded = sorted((set(details) | set(other_details)) - pair)
|
||||
contrasts.append(dict(subset=subset, pool=pool, reference='R2', candidate=arm,
|
||||
primary=subset == 'all' and pool == 100 and arm == 'R2_E4',
|
||||
interpretation='post-S2 adaptive follow-up; interval conditional on fixed models/folds',
|
||||
included_folds=sorted({details[k]['fold'] for k in pair}),
|
||||
excluded_components=excluded, excluded_parts=sum(details[k]['parts'] for k in excluded),
|
||||
excluded_nodes=sum(details[k]['nodes'] for k in excluded),
|
||||
**contrast(reference, candidate, replicates=2000, seed=2026111408)))
|
||||
return tables, contrasts, component_rows
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Synthetic S3-A contract tests; never access actual target arrays or fit a model."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import numpy as np
|
||||
import s3a_data as data
|
||||
import s3a_statistics as stats
|
||||
import run_s3a_score as scorer
|
||||
from freeze_g1_components import sha
|
||||
from s3a_budget import first_fit_projection, can_dispatch
|
||||
from prepare_s3a import build_jobs
|
||||
from run_s2_r2 import select_finished
|
||||
from s2_custody_statistics import contrast, six_region, part_metrics
|
||||
|
||||
|
||||
def roster_part(case, component, fold=0):
|
||||
return dict(case=case, component_id=component, included=True, nodes=10,
|
||||
exposure=dict(development_exposed=False),
|
||||
assignments=[dict(fold=f, role='test' if f == fold else 'learning', learning_pools=['25','50','100']) for f in range(5)])
|
||||
|
||||
|
||||
def score(case, arm, value, pool=100):
|
||||
return dict(case=case, arm=arm, pool=pool, seed=None,
|
||||
**{k: dict(value=value, reason=None) for k in ('relative_l2','near_wall','kendall_tau_b','last_location')})
|
||||
|
||||
|
||||
class S3AChecks(unittest.TestCase):
|
||||
def test_exact_feature_order_and_two_separate_arms(self):
|
||||
d=dict(x=np.arange(40).reshape(2,20),edge=np.array([[41,42,43,44],[51,52,53,54]]),gaps=np.array([[61,62],[71,72]]))
|
||||
e=data.features(d,'R2_E4');g=data.features(d,'R2_G2')
|
||||
self.assertEqual(e.shape,(2,24));self.assertEqual(g.shape,(2,22))
|
||||
np.testing.assert_array_equal(e[:,:20],d['x']);np.testing.assert_array_equal(e[:,20:],d['edge'])
|
||||
np.testing.assert_array_equal(g[:,20:],d['gaps'])
|
||||
self.assertEqual(data.columns('R2_E4')[-4:],['mean_q1','max_q1','mean_q2','max_q2'])
|
||||
with self.assertRaises(ValueError):data.features(d,'R2')
|
||||
d['edge'][0,0]=0;d['edge']=d['edge'].astype(float);d['edge'][0,0]=np.nan
|
||||
with self.assertRaises(ValueError):data.features(d,'R2_E4')
|
||||
|
||||
def test_test_role_rejected_before_any_feature_or_cache_access(self):
|
||||
roles=dict(learning_parts=['l'],validation_parts=['v'],sealed_test_parts=['t'])
|
||||
with patch.object(data,'prediction_inputs',side_effect=AssertionError('feature IO')) as io:
|
||||
with self.assertRaises(PermissionError):data.read_part(dict(case='t'),roles,{},'R2_E4')
|
||||
io.assert_not_called()
|
||||
|
||||
def test_cache_hash_ids_multiplicity_and_seconds_target(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
p=Path(t)/'label.npz';np.savez(p,seconds=[2.,4.],raw_ids=[3,7],counts=[1,5])
|
||||
entry=dict(path=str(p),sha256=sha(p))
|
||||
d=dict(raw_ids=np.array([3,7]),counts=np.array([1,5]))
|
||||
np.testing.assert_array_equal(data.cached_truth(entry,d),[2,4])
|
||||
d['counts']=np.array([5,1])
|
||||
with self.assertRaises(ValueError):data.cached_truth(entry,d)
|
||||
entry['sha256']='wrong'
|
||||
with self.assertRaises(ValueError):data.cached_truth(entry,d)
|
||||
|
||||
def test_final_loader_reads_learning_only_from_cache(self):
|
||||
roles=dict(learning_parts=['l'],validation_parts=['v'],sealed_test_parts=['t'])
|
||||
rows={k:dict(case=k) for k in ['l','v']};seen=[]
|
||||
def part(row,*args):
|
||||
seen.append(row['case']);return dict(role='learning' if row['case']=='l' else 'validation')
|
||||
c=dict(roster='r',roster_sha256='h',label_index='l',label_index_sha256='h')
|
||||
with patch.object(data,'checked',side_effect=lambda p,h:p),patch.object(data,'read_json',side_effect=[{},[]]),patch.object(data,'select_cell',return_value=(rows,roles)),patch.object(data,'read_part',part):
|
||||
learning,validation,_=data.load_cell(None,c,dict(arm='R2_G2',fold=0,pool=25,kind='FINAL'))
|
||||
self.assertEqual(seen,['l']);self.assertEqual(validation,[])
|
||||
|
||||
def test_plan_is360_hpo30final_no_original_R2_refit(self):
|
||||
# Real roster metadata only: no query arrays or labels are loaded.
|
||||
roster=json.loads(Path('artifacts/s2-main-preparation-2026-11-15/final-roster.json').read_text())
|
||||
jobs,cells=build_jobs(roster)
|
||||
self.assertEqual((len(jobs),len(cells)),(390,30))
|
||||
self.assertEqual(sum(j['kind']=='HPO' for j in jobs),360)
|
||||
self.assertTrue(all(j['kind']=='FINAL' for j in jobs[360:]))
|
||||
self.assertEqual(jobs[0]['arm'],'R2_E4');self.assertEqual(jobs[12]['arm'],'R2_G2')
|
||||
self.assertEqual({j['arm'] for j in jobs},set(data.ARMS))
|
||||
self.assertEqual(len({(c['arm'],c['fold'],c['pool']) for c in cells}),30)
|
||||
for c in cells:self.assertEqual([jobs[i]['config_index'] for i in c['hpo_indices']],list(range(12)))
|
||||
|
||||
def test_original_failure_and_tie_policy(self):
|
||||
trials=[dict(config_index=i,status='NA',score=None) for i in range(12)]
|
||||
self.assertEqual(select_finished(0,trials)['status'],'NA_ALL_HPO_FAILED')
|
||||
trials[3].update(status='PASS',score=.1);trials[1].update(status='PASS',score=.1)
|
||||
r=select_finished(0,trials);self.assertEqual(r['config_index'],1)
|
||||
self.assertEqual(r['finished_configurations'],2)
|
||||
|
||||
def test_component_equal_single_fit_and_direction(self):
|
||||
roster=[roster_part('a','A'),roster_part('b','A'),roster_part('c','C',1)]
|
||||
rows=[score(k,arm,v) for arm,vs in [('R2',[1,3,5]),('R2_E4',[.5,1.5,2.5]),('R2_G2',[2,6,10])] for k,v in zip('abc',vs)]
|
||||
tables,contrasts,_=stats.reports(rows,roster,set('abc'))
|
||||
primary=next(c for c in contrasts if c['primary'])
|
||||
self.assertEqual(primary['reference_mean'],3.5);self.assertEqual(primary['candidate_mean'],1.75)
|
||||
self.assertEqual(primary['value'],.5);self.assertEqual(primary['region'],'meaningful gain supported')
|
||||
self.assertEqual(sum(c['primary'] for c in contrasts),1)
|
||||
r=next(t for t in tables if t['arm']=='R2' and t['pool']==100 and t['metric']=='relative_l2' and t['subset']=='all')
|
||||
self.assertEqual(r['included_parts'],3)
|
||||
|
||||
def test_failure_invalidates_whole_component_not_other_arm(self):
|
||||
roster=[roster_part('a','A'),roster_part('b','A'),roster_part('c','C',1)]
|
||||
rows=[score(k,arm,1) for k in 'abc' for arm in ['R2','R2_G2']]+[score('a','R2_E4',.5),score('c','R2_E4',.5)]
|
||||
_,contrasts,_=stats.reports(rows,roster,set('abc'))
|
||||
e=next(c for c in contrasts if c['primary'])
|
||||
g=next(c for c in contrasts if c['subset']=='all' and c['pool']==100 and c['candidate']=='R2_G2')
|
||||
self.assertEqual(e['component_ids'],['C']);self.assertEqual(e['excluded_parts'],2)
|
||||
self.assertEqual(g['components'],2)
|
||||
|
||||
def test_exposure_filters_parts_and_keeps_main_invalid_component_excluded(self):
|
||||
roster=[roster_part('a','A'),roster_part('b','A'),roster_part('c','C',1)]
|
||||
rows=[score(k,a,1) for k in 'abc' for a in ['R2','R2_E4','R2_G2']]
|
||||
tables,_,_=stats.reports(rows,roster,{'a','c'})
|
||||
r=next(t for t in tables if t['subset']=='not_known_exposed' and t['pool']==100 and t['arm']=='R2_E4' and t['metric']=='relative_l2')
|
||||
self.assertEqual((r['components'],r['included_parts']),(2,2))
|
||||
rows=[r for r in rows if not(r['case']=='b' and r['arm']=='R2_E4')]
|
||||
_,contrasts,_=stats.reports(rows,roster,{'a','c'})
|
||||
r=next(c for c in contrasts if c['subset']=='not_known_exposed' and c['pool']==100 and c['candidate']=='R2_E4')
|
||||
self.assertEqual(r['component_ids'],['C'])
|
||||
|
||||
def test_undefined_bootstrap_and_six_region_boundaries_unchanged(self):
|
||||
r=contrast({'a':0.,'b':1.},{'a':0.,'b':.5})
|
||||
self.assertGreater(r['undefined_replicates'],0);self.assertIsNone(r['low'])
|
||||
self.assertEqual(six_region(0,.1),'insufficient evidence')
|
||||
self.assertEqual(six_region(.1,.2),'positive, size undetermined')
|
||||
self.assertEqual(six_region(-.2,-.01),'harmful')
|
||||
|
||||
def test_multiplicity_once_and_negative_prediction_not_clipped(self):
|
||||
r=part_metrics(np.array([-1.,2.]),np.array([1.,2.]),np.array([3,1]),np.array([True,False]),np.array([3,7]),np.zeros((2,3)),1)
|
||||
self.assertAlmostEqual(r['relative_l2']['value'],np.sqrt(12/7))
|
||||
self.assertEqual(r['near_wall']['value'],2.)
|
||||
|
||||
def test_duplicate_scores_and_seed_replication_rejected(self):
|
||||
r=score('a','R2',1);roster=[roster_part('a','A')]
|
||||
with self.assertRaises(ValueError):stats.reports([r,r],roster,{'a'})
|
||||
r['seed']=0
|
||||
with self.assertRaises(ValueError):stats.reports([r],roster,{'a'})
|
||||
|
||||
def test_review_gate_blocks_missing_or_wrong_contract(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);(root/'contract.json').write_text(json.dumps(dict(code_sha256={},source_sha256={})))
|
||||
with self.assertRaises(FileNotFoundError):scorer.review_gate(root)
|
||||
(root/'main-review.json').write_text(json.dumps(dict(approved=True,slack_ts='synthetic',git_commit='synthetic',contract_sha256='wrong')))
|
||||
with self.assertRaises(PermissionError):scorer.review_gate(root)
|
||||
|
||||
def test_terminal_manifest_requires_all30_cells_and_model_hash(self):
|
||||
cells=[dict(index=i,arm=data.ARMS[i%2],fold=i//6,pool=[25,50,100][i//2%3]) for i in range(30)]
|
||||
terminals=[dict(cell=c['index'],arm=c['arm'],fold=c['fold'],pool=c['pool'],status='NA',reason='synthetic_failed') for c in cells]
|
||||
self.assertEqual(scorer.expected_predictions(Path('/none'),dict(cells=cells),terminals,[]),{})
|
||||
with self.assertRaises(ValueError):scorer.expected_predictions(Path('/none'),dict(cells=cells),terminals[:-1],[])
|
||||
terminals[0].update(status='PASS',model_path='missing',model_sha256='wrong')
|
||||
with self.assertRaises(FileNotFoundError):scorer.expected_predictions(Path('/none'),dict(cells=cells),terminals,[])
|
||||
|
||||
def test_budget_projection_uses_both_arms_and_stops_failed_first(self):
|
||||
jobs=[dict(arm=a,cost_units=1,cost_queries=1) for a in data.ARMS for _ in range(195)]
|
||||
c=dict(jobs=jobs,limits=dict(fitting_worker_cpu_seconds=64800))
|
||||
first={a:dict(status='PASS',worker_cpu_seconds=100,fit_cpu_seconds=80) for a in data.ARMS}
|
||||
self.assertTrue(first_fit_projection(c,first)['continue_within_cap'])
|
||||
first['R2_G2']['worker_cpu_seconds']=200
|
||||
self.assertFalse(first_fit_projection(c,first)['continue_within_cap'])
|
||||
first['R2_E4']['status']='NA'
|
||||
self.assertEqual(first_fit_projection(c,first)['reason'],'FIRST_TRIAL_FAILED_NO_RETRY')
|
||||
|
||||
def test_dispatch_reserves_active_and_next_caps_before_launch(self):
|
||||
c=dict(limits=dict(fitting_worker_cpu_seconds=64800))
|
||||
self.assertTrue(can_dispatch(c,60000,1000,[1000],1800))
|
||||
self.assertFalse(can_dispatch(c,60001,1000,[2000],1800))
|
||||
with self.assertRaises(ValueError):can_dispatch(c,0,float('nan'),[],1800)
|
||||
|
||||
def test_prediction_gate_rejects_missing_extra_before_array_IO(self):
|
||||
proof=dict(status='ALL_S3A_PREDICTIONS_FROZEN',git_commit='fixed',contract_sha256='hash',terminal_freeze_sha256='hash',prediction_index_sha256='hash')
|
||||
frozen=dict(status='ALL_S3A_FITS_TERMINAL',contract_sha256='hash',hpo_terminal_indices=list(range(360)),batch_receipt='b',batch_receipt_sha256='hash',final_fits=[])
|
||||
c=dict(roster='roster',roster_sha256='hash')
|
||||
with patch.object(scorer,'sha',return_value='hash'),patch.object(scorer,'checked',side_effect=lambda p,h:p),patch.object(scorer,'read_json',side_effect=[proof,frozen,{'parts':[]},{'extra':'hash'}]),patch.object(scorer,'expected_predictions',return_value={'expected':None}):
|
||||
with self.assertRaisesRegex(ValueError,'Missing or extra'):scorer.prediction_gate(Path('plan'),Path('root'),c)
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user