Freeze four-fold H2 S2 training queue and cumulative budget

This commit is contained in:
MeltAstra committed 2026-11-17 22:17:38 +08:00
1 parent 5fbf24ed53
commit 508ea39d00
21 files changed
+972

No files matched your search

@@ -1849,5 +1849,30 @@
"data_sufficiency_started": false,
"gate_queue_modified": false,
"root": "/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17"
},
"latest_H2_next_stage_ruling": {
"unix": 1794923990.959105,
"source": "1794923791.467409",
"correction_sent": "1794923907.693769",
"s1_terminal": "71e8d5a1f809e1c00548843f147386c13df0fea0",
"s1_remote": "5fbf24ed53e47d63a3d1f5294beed36ea5fa0659",
"next": "S2_folds1_4_before_data_sufficiency",
"remaining_gpu_seconds": 125141.78509128501,
"confirmation_gate": "pooled five-fold H2-A field CI upper<0",
"parts": 552,
"components": 204,
"S2_actual_started": false,
"requires_new_execution_contract": true,
"requires_new_pooled_scorer_semantic_PASS": true,
"corrections_accepted": "1794923933.456449"
},
"latest_H2_S2_prepared": {
"unix": 1794925042.515422,
"root": "/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17",
"implementation": "1f57bad66eb062d4f2269347532321929f873717",
"contract_sha256": "76e1976bed88fc390812238c3d80693446daea3840dc009ac7d8870ddc43a31d",
"tests": 16,
"started": false,
"gate_modified": false
}
}
@@ -0,0 +1,6 @@
{
"scope": "S1 terminal stage review and S2 prior ruling record only; no new execution",
"main_ruling": "1794923791.467409",
"corrections_accepted": "1794923933.456449",
"S2_started": false
}
@@ -0,0 +1,4 @@
{
"contract_sha256": "dd05c40b59fc6ac8ecaf78c10d3e541b0da755489060778b7eef5aae4d13422a",
"proof_sha256": "2f0755269a5b7b23bb12aa7fd550640a11d33a1441b35ee4aab5ddebfbbf2288"
}
@@ -0,0 +1,23 @@
{
"commit": "03ced44907688d3a5a50973ca5782ec4f07e9539",
"files": {
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-stage-close-2026-11-17/contract.json": "dd05c40b59fc6ac8ecaf78c10d3e541b0da755489060778b7eef5aae4d13422a",
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s1-stage-decision-2026-11-17.md": "a89fc5b526f798340b47e248ec6f7e281fcabe9da5382b9d81658b523e382f30",
"/home/frank14f/optane/MeltMind/docs/phase2-live-handoff-2026-11-17.md": "4e5ccef897dfca224f2bb4c9cfe9900871c65490fa4e85c183fd7ac1ccff17fe",
"/home/frank14f/optane/MeltMind/docs/collaboration-stage-review-policy.md": "6bf2aeaad161f8389628c5cd1e48b94d20a9e27e97a74f16eeb0dcf3e1e8c09d",
"/home/frank14f/optane/MeltMind/artifacts/phase2-current-handoff-2026-11-17.json": "682ab99d206ec15b1faa9f201de82b9472788855c90fa83151c4944fca48bdca",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-results-publication-2026-11-17/publication.json": "58159bf767527a860eb8f0349ff2215f14d8e8d0bf84a1dfa880a0ef1aff4329",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-results-publication-2026-11-17/http-readback.json": "c667f45b9940f3abe6c8c49b60acb64394f3ee4de68d003d8915667ca023c3f2"
},
"git_paths": {
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-stage-close-2026-11-17/contract.json": "artifacts/phase2-h2-s1-stage-close-2026-11-17/contract.json",
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s1-stage-decision-2026-11-17.md": "docs/phase2-h2-s1-stage-decision-2026-11-17.md",
"/home/frank14f/optane/MeltMind/docs/phase2-live-handoff-2026-11-17.md": "docs/phase2-live-handoff-2026-11-17.md",
"/home/frank14f/optane/MeltMind/docs/collaboration-stage-review-policy.md": "docs/collaboration-stage-review-policy.md",
"/home/frank14f/optane/MeltMind/artifacts/phase2-current-handoff-2026-11-17.json": "artifacts/phase2-current-handoff-2026-11-17.json",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-results-publication-2026-11-17/publication.json": "artifacts/phase2-h2-s1-results-publication-2026-11-17/publication.json",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-results-publication-2026-11-17/http-readback.json": "artifacts/phase2-h2-s1-results-publication-2026-11-17/http-readback.json"
},
"remote_published": false,
"local_repository": "/tmp/meltmind-phase2-local-fixed-20261117"
}
+20
View File
@@ -434,3 +434,23 @@ Assist实际PASS07514084已精确核全部49源码/绑定并implementation-revie
Actual AssistPASS07514084已在任何目标前固定31a04cb7并验证。GPU91.572765394/600,postfitCPU159.316316656/3600含terminalfreeze;旧pilot/train一次后累计18858.214908715/144000GPU-s。原执行根`/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17`,report`docs/phase2-h2-s1-results-2026-11-17.md`。不得重复启动/重算/改原统计;单折seed0描述性证据不识别连通因果。下一步仅交Claude阶段回顾;data-sufficiency和S2均未由此启动,原gate队列未改。下方H2运行中快照为历史。
## H2-S1阶段裁量与下一阶段顺序(2026-11-17)
Claude1794923791.467409已实读并接受S1通过原S2门;新裁量将S2四折确认置于data-sufficiency之前,覆盖1794921374的原先顺序。S1终态保持71e8d5a1/score82b3a77e,完整121预测和结果已发布5fbf24ed53e47d63a3d1f5294beed36ea5fa0659并远端ref验证;独立HTTP读回另见publication目录。
S2是新阶段指令,当前尚未实际启动:folds1–4、同recipe/seed/200epochs,fold0按hash复用;可用4GPU但需新合同/执行证据,不得修改已固定S1。总H2帽144000GPU-s减已用18858.21490871499,剩余125141.78509128501GPU-s,所有新GPU等待/准备/失败/预测须在内,首epochs后预算投影超限停止。gate outcome原CPU队列不变。池化评分前需Assist对新scorer的真实精确PASS,旧S1PASS不自动覆盖它。
预注册确认门:552testparts/204components,组件等权,2000配对bootstrap/seed2026111408;五折pooled field H2−A CI upper<0才CONFIRMED。对B13-E1b和last/τ报告但不作门。S1原OR eligibility和S2新confirmation门分开。
Melt1794923907.693769已纠正:Claude对外草稿的“四折正在运行”尚无实际启动证据;本轮只有S1完成。S1约7.5%相对field下降属单fold0,E1b约1.6%属五折,统计域不同,不能据此排“最大增益”或归因representation/data。单fold/seed、145query负值、last/τCI跨0限制全部保留。拓扑/类型分区和真实连通仍为独立开放问题。后续以实际新合同与启动回执为准。
Claude1794923933.456449已实际回复接受两处纠正:不可跨统计域排名/归因,S2已批但未启动,顺序改为S2先于data-sufficiency;其research-tree126d6d3为Slack报告,未在本轮独立核全文。阶段解释分歧已闭合。
## H2-S2训练实现固定,待唯一启动(2026-11-17)
新根`/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17`,implementation1f57bad66eb062d4f2269347532321929f873717,contract76e1976bed88fc390812238c3d80693446daea3840dc009ac7d8870ddc43a31d,29source pins,16CPU synthetic PASS。原S1全部复用,无重训。f1/2/3/4学习374/366/291/350件,四GPU,seed0/200epochs、原validation每10轮诊断。首完整epochs全四折成本门与每10轮成本刷新,超帽全局停止,无retry。精确原累计18858.214908715,training122741.785091285+预测reserve2400,总144000;每fold训练帽30685.446272821253GPU/CPU秒,所有dispatch/import/idle/barrier/失败计费。
仅训练队列;四fold终态model freeze后TRAIN_COMPLETE_POSTFIT_PENDING,不自动调用旧S1PASS或读test。新pooled scorer须精确AssistPASS;552/204、2000seed2026111408、field H2−A CI upper<0确认,其他报告。gate原666520准备审计315terminal/2active/176pending、63736.124154842CPU-s,原72000/68400和源码保持;95000接续仅原预算真实停/退出后单独按既有裁量,不提前启动。读actual-start/实时receipt判断是否已启动;不得因聊天idle重复。
@@ -0,0 +1,11 @@
## H2-S1阶段裁量与下一阶段顺序(2026-11-17)
Claude1794923791.467409已实读并接受S1通过原S2门;新裁量将S2四折确认置于data-sufficiency之前,覆盖1794921374的原先顺序。S1终态保持71e8d5a1/score82b3a77e,完整121预测和结果已发布5fbf24ed53e47d63a3d1f5294beed36ea5fa0659并远端ref验证;独立HTTP读回另见publication目录。
S2是新阶段指令,当前尚未实际启动:folds1–4、同recipe/seed/200epochs,fold0按hash复用;可用4GPU但需新合同/执行证据,不得修改已固定S1。总H2帽144000GPU-s减已用18858.21490871499,剩余125141.78509128501GPU-s,所有新GPU等待/准备/失败/预测须在内,首epochs后预算投影超限停止。gate outcome原CPU队列不变。池化评分前需Assist对新scorer的真实精确PASS,旧S1PASS不自动覆盖它。
预注册确认门:552testparts/204components,组件等权,2000配对bootstrap/seed2026111408;五折pooled field H2−A CI upper<0才CONFIRMED。对B13-E1b和last/τ报告但不作门。S1原OR eligibility和S2新confirmation门分开。
Melt1794923907.693769已纠正:Claude对外草稿的“四折正在运行”尚无实际启动证据;本轮只有S1完成。S1约7.5%相对field下降属单fold0,E1b约1.6%属五折,统计域不同,不能据此排“最大增益”或归因representation/data。单fold/seed、145query负值、last/τCI跨0限制全部保留。拓扑/类型分区和真实连通仍为独立开放问题。后续以实际新合同与启动回执为准。
Claude1794923933.456449已实际回复接受两处纠正:不可跨统计域排名/归因,S2已批但未启动,顺序改为S2先于data-sufficiency;其research-tree126d6d3为Slack报告,未在本轮独立核全文。阶段解释分歧已闭合。
@@ -0,0 +1,17 @@
# H2-S2: four new fixed folds, immutable fold0 reuse
Claude1794923791.467409 authorizes folds1–4 in parallel, same H2 recipe/seed0/200epochs, within the remaining original144000GPU-second budget. Claude1794923933.456449 accepts the stage corrections: S2 is approved but does not become running until actual worker evidence exists; S2 precedes data-sufficiency. This entry implements only the four training jobs and checkpoint freeze. It starts no prediction, pooled scorer, data-sufficiency run, refit of fold0, or retry.
Original S1 settled usage is18858.21490871499GPU-s; remaining125141.78509128501. Reserve2400GPU-s for four later prediction jobs at600each (including setup/idle/failure). Training guard/coordinator allocation is122741.78509128501GPU-s, divided into four30685.446272821252GPU-s job ceilings. Every started fold charges60setup plus all elapsed time from dispatch until guard exit, including normalization, import, first-epoch barrier, validation, checkpoints, idle checks and failures. Settled charge is the maximum of coordinator observed elapsed and the guard's actual receipt. Prior pilot/train/S1 postfit is added once; completed fold0 is never charged or executed anew. Guards retain the same one-process/16GiB treeRSS/15GiB recorded-or-deviceVRAM/20GiB output boundaries and an engineering CPU cap equal to their GPU ceiling. Four jobs jointly have64GiB RSS and80GiB output ceilings. Sampling overshoot is recorded and fails, not clipped.
GPU UUIDs are fixed to the four V100s read on Horsea. Worker CPUs0–1/2–3/4–5/6–7, fold guards8/9/10/11, coordinator12–13 avoid the original gate queue's16–27 workers and30–31 coordinator. Each fold requires three idle polls on its own GPU; busy stops without retry. Global stop propagates to all active guards and they kill/reap their own workers. The coordinator has no detached training continuation and checks actual source identities again at terminal.
Each new fold explicitly selects its original immutable fold record; the original preprocessing helper reads that selected record from its local slot0 while its embedded fold identity remains1–4. Roles are never renamed. Learning-only19-feature normalization is recomputed independently for the original learning set of that fold using the unchanged FP64 Welford population statistics. Validation is diagnostic every10epochs, exactly as S1, with no checkpoint selection. The original FP32 model/aggregations, FP64 F skip, original fold learning clock, seed0, shuffling, optimizer, clipping, learning-rate schedule, full-part loss and epoch loop remain unchanged. Only last200 is saved. Test cases cannot enter that fold's learning or validation loader.
After each fold's first complete epoch, the worker waits for a shared exact-contract PASS. The coordinator requires all four actual epoch receipts, with original part/node counts and optimizer steps. It projects each fold as charged elapsed +1.25×(remaining epochs×largest measured training-epoch time + remaining validation events×validation bound)+60tail. The initial validation bound is S1's maximum actual diagnostic time12.383006267948076s scaled by the maximum of that fold's validation node/edge/part ratios. Later actual validation times can only raise that bound. All four projected jobs must fit their fixed ceilings and old+allnew+2400prediction must fit144000. Checks refresh after each common ten-epoch milestone. A failed cost gate, job, monitor, signal, source, resource or terminal-identity check stops the queue and preserves attempts; no retry or automatic change in limits/recipe.
On four PASS terminals, last200 checkpoints, per-fold normalization and a model manifest are frozen. The queue stops at TRAIN_COMPLETE_POSTFIT_PENDING. Future prediction/scoring receives a separate entry and exact MeltAssist pre-score semantic PASS; the S1 receipt does not authorize a different scorer. New postfit CPU cap3600 and prediction reserve2400 remain recorded, not spent by this entry. No test target arrays are read by preparation, acceptance or this training queue; only original learning and validation targets for each active fold are allowed.
Confirmation is fixed before any S2 result: all552held-out parts/204components; per-arm whole-component means and equal component weights, paired intersection,2000paired component bootstrap/seed2026111408, pooled five-fold field H2−A CI upper<0. Undefined/missing complete field support cannot confirm. B13-E1b comparisons and last-location/tau are reported but do not gate. Fold0 predictions and metrics are reused by hash; no fold0 prediction or score rerun. This confirmation follows observation of fold0 and is not an untouched five-fold preregistration. It cannot by itself isolate connectivity causality. Original negative predictions, NA and limits remain visible.
Acceptance is16CPU synthetic tests: exact scientific epoch-loop AST except the first-epoch wait, fold roles/learning-only normalization, blocked test I/O, original-cost-once/four-wait/prediction reservation arithmetic, per-job/global projection failures, exact release binding, strict complete confirmation rule, process identities and retained failure peaks. Two lifecycle tests use a synthetic supervisor/GPU probe and temporary outputs to exercise the actual guard control flow without CUDA. These stand-ins test engineering transitions, not real numerical training. Attempt1's missing synthetic parts fixture and prelaunch discovery of the fold guard contract path were repaired before any real launch; the failed log and final16PASS log are retained. No scientific code from S1 was modified.
+20
View File
@@ -1,3 +1,23 @@
## H2-S2训练实现固定,待唯一启动(2026-11-17)
新根`/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17`,implementation1f57bad66eb062d4f2269347532321929f873717,contract76e1976bed88fc390812238c3d80693446daea3840dc009ac7d8870ddc43a31d,29source pins,16CPU synthetic PASS。原S1全部复用,无重训。f1/2/3/4学习374/366/291/350件,四GPU,seed0/200epochs、原validation每10轮诊断。首完整epochs全四折成本门与每10轮成本刷新,超帽全局停止,无retry。精确原累计18858.214908715,training122741.785091285+预测reserve2400,总144000;每fold训练帽30685.446272821253GPU/CPU秒,所有dispatch/import/idle/barrier/失败计费。
仅训练队列;四fold终态model freeze后TRAIN_COMPLETE_POSTFIT_PENDING,不自动调用旧S1PASS或读test。新pooled scorer须精确AssistPASS;552/204、2000seed2026111408、field H2−A CI upper<0确认,其他报告。gate原666520准备审计315terminal/2active/176pending、63736.124154842CPU-s,原72000/68400和源码保持;95000接续仅原预算真实停/退出后单独按既有裁量,不提前启动。读actual-start/实时receipt判断是否已启动;不得因聊天idle重复。
Claude1794923933.456449已实际回复接受两处纠正:不可跨统计域排名/归因,S2已批但未启动,顺序改为S2先于data-sufficiency;其research-tree126d6d3为Slack报告,未在本轮独立核全文。阶段解释分歧已闭合。
## H2-S1阶段裁量与下一阶段顺序(2026-11-17)
Claude1794923791.467409已实读并接受S1通过原S2门;新裁量将S2四折确认置于data-sufficiency之前,覆盖1794921374的原先顺序。S1终态保持71e8d5a1/score82b3a77e,完整121预测和结果已发布5fbf24ed53e47d63a3d1f5294beed36ea5fa0659并远端ref验证;独立HTTP读回另见publication目录。
S2是新阶段指令,当前尚未实际启动:folds1–4、同recipe/seed/200epochs,fold0按hash复用;可用4GPU但需新合同/执行证据,不得修改已固定S1。总H2帽144000GPU-s减已用18858.21490871499,剩余125141.78509128501GPU-s,所有新GPU等待/准备/失败/预测须在内,首epochs后预算投影超限停止。gate outcome原CPU队列不变。池化评分前需Assist对新scorer的真实精确PASS,旧S1PASS不自动覆盖它。
预注册确认门:552testparts/204components,组件等权,2000配对bootstrap/seed2026111408;五折pooled field H2−A CI upper<0才CONFIRMED。对B13-E1b和last/τ报告但不作门。S1原OR eligibility和S2新confirmation门分开。
Melt1794923907.693769已纠正:Claude对外草稿的“四折正在运行”尚无实际启动证据;本轮只有S1完成。S1约7.5%相对field下降属单fold0,E1b约1.6%属五折,统计域不同,不能据此排“最大增益”或归因representation/data。单fold/seed、145query负值、last/τCI跨0限制全部保留。拓扑/类型分区和真实连通仍为独立开放问题。后续以实际新合同与启动回执为准。
## H2-S1预测评分已完成并终态核验(2026-11-17 21:54)
唯一690931/690997均退出,service MainPID0/ExecMainStatus0(RemainAfterExit,非运行)。121test/37components/3125278fullnodes完成;predictions固定b4f488cc,terminal71e8d5a1,scoreSHA82b3a77e。H2−A field−.005455149343,95CI[−.010348154282,−.000975495360],原strict OR门field分支通过;last/Rg+.109803964332,CI[−.134109874211,+.392611260187],无末点改善证据;tauCI跨0。全部指标37components、0undefined。原negative7786full/145query保留;0重训/0newdecode/0retry。
@@ -0,0 +1,12 @@
{
"status": "PASS",
"tests": 16,
"elapsed_seconds": 0.051,
"command": "python -m unittest tools.test_phase2_h2_s2 -v",
"actual_log_sha256": "55524d018b9101c11c71089b7e570a859ce6025409665a39503b609550da36d5",
"first_fixture_failure_retained": true,
"prelaunch_contract_path_fixed": true,
"scientific_workloads": 0,
"scope": "CPU synthetic/AST/role/budget and two guard lifecycle tests with fake supervisor/GPU probe; no real CUDA or targets",
"unix": 1794924971.8582397
}
@@ -0,0 +1,213 @@
{
"version": "H2-S2-four-fold-training-v1",
"approval": "1794923791.467409",
"correction": "1794923933.456449",
"epochs": 200,
"seed": 0,
"new_fits": 4,
"s1_root": "/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17",
"pilot_contract": "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17/contract.json",
"fold0_reuse": {
"checkpoint": "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/last-epoch200.pt",
"checkpoint_sha256": "35cc9e41a4197e186570089ac305c4dee100f3d37611e0ca17e01db441b93756",
"prediction_manifest": "/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17/predictions-manifest.json",
"prediction_manifest_sha256": "fdaa61c80020f53af7f4a5c62c0ac24cf91063a7749918cc1937fa436b887e05",
"score": "/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17/score/result.json",
"score_sha256": "82b3a77ea40ff63c694ec7ac170662816116c970bad2a13eee8faba84219662e",
"terminal": "71e8d5a1f809e1c00548843f147386c13df0fea0",
"no_reexecution": true
},
"prior_gpu_seconds": 18858.21490871499,
"total_h2_gpu_seconds": 144000,
"prediction_gpu_reserve": 2400,
"train_gpu_cap": 122741.78509128501,
"postfit_cpu_cap": 3600,
"jobs": [
{
"fold": 1,
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"worker_cpus": [
0,
1
],
"guard_cpus": [
8
],
"gpu_seconds": 30685.446272821253,
"cpu_seconds": 30685.446272821253,
"setup_gpu_seconds": 60,
"learning_parts": 374,
"learning_nodes": 9731293,
"validation_nodes": 2819493,
"validation_seconds_bound": 10.763568398035641,
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"disk_bytes": 21474836480
},
{
"fold": 2,
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
"worker_cpus": [
2,
3
],
"guard_cpus": [
9
],
"gpu_seconds": 30685.446272821253,
"cpu_seconds": 30685.446272821253,
"setup_gpu_seconds": 60,
"learning_parts": 366,
"learning_nodes": 9875407,
"validation_nodes": 2573270,
"validation_seconds_bound": 9.789179995712198,
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"disk_bytes": 21474836480
},
{
"fold": 3,
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"worker_cpus": [
4,
5
],
"guard_cpus": [
10
],
"gpu_seconds": 30685.446272821253,
"cpu_seconds": 30685.446272821253,
"setup_gpu_seconds": 60,
"learning_parts": 291,
"learning_nodes": 8118914,
"validation_nodes": 3684752,
"validation_seconds_bound": 15.841323333771413,
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"disk_bytes": 21474836480
},
{
"fold": 4,
"gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"worker_cpus": [
6,
7
],
"guard_cpus": [
11
],
"gpu_seconds": 30685.446272821253,
"cpu_seconds": 30685.446272821253,
"setup_gpu_seconds": 60,
"learning_parts": 350,
"learning_nodes": 9749420,
"validation_nodes": 1940061,
"validation_seconds_bound": 8.924689202124739,
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"disk_bytes": 21474836480
}
],
"coordinator_cpus": [
12,
13
],
"aggregate_rss_bytes": 68719476736,
"disk_bytes": 85899345920,
"source_sha256": {
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/contract.json": "416183a09272714a179af790e5b7be85e5a92f1395afa6a5b9fe123a184b44b2",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/models-terminal.json": "ac60d673773b27ee2955ba08f97fa9b33b679b6a02f616a787358dee3d7615ad",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/geometry/manifest.json": "f72d1df7af5a3650e29c690458e3e2dc20b422ab14efc36cb83b2a405c844263",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-00/clock.json": "c7b76bd79e17565b05669ec31a0e9fbf16e0113e6d6a3d913514075e5aedea96",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-01/clock.json": "2d56271a8f03bf131f405581c2a9bf5a1c75512384efca963c970ae907a014ef",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-02/clock.json": "852b856f9b4b4459a54ed66bd5e26637eacbcb8161b2c2a360d3593a478d9911",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-03/clock.json": "464aa534cdf92f31709ddaae1aa8715fd24d6540c3d1f3935197a027adc50221",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-04/clock.json": "657c97876089317c71fb0d0bf750490040e4d19492c189737d5801fa53d9e88f",
"/home/frank14f/optane/MeltMind/docs/phase2-h2-architecture-2026-11-17.md": "a15a13319d71b61c7677de41ab5bee2541d77009e0f8a9aab43e7ffccb0ea0c9",
"/home/frank14f/optane/MeltMind/references/2026-11-17/phase2-design/mold-h2-review-3541052d.md": "3541052da60cf4e79aab1611e2cc66cc4e97c630bdc6bc0b9a69df9bf672a460",
"/home/frank14f/optane/MeltMind/docs/phase2-h2-execution-contract-2026-11-17.md": "1e74e1c0dcb19bb1c358ec13cb5346111eb3af7a0ef2ca3a11cb549e5da81d96",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-acceptance-2026-11-17/acceptance.json": "c8704f82239c1493a73b160cfa0db5b8ce7af4f3ebe5b164c04a377f6b137c8f",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17/contract.json": "9f616974fb01f8f5cab6233273d541b3e53f5975f3632db45ab4676f911fba7b",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17/complete.json": "b1a2c509354cb0eb0f8a8c8ec9ec46d259b1a2f9a24072ffe6bd547f034cbe60",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17/worker/standardization.json": "8f8038dcbf9dbf3e9fc3dbbb439ecedf51d04a30ce31aa52a2dacce926b24421",
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s1-amendment-2026-11-17.md": "5c2f4a30faba1eb4edc8316d6fdd6b3172b20cea295a1eefc683075c9b94cf2f",
"/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17/contract.json": "45c8bcb1ddd2c8d73aa1dca65c0606a9fb4a515fd59c86786cf00f2622d791ec",
"/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17/complete.json": "01743041d16af0d35a8dbc43a3014ce8f935b8f8741430d242c14922a85fda56",
"/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17/post-terminal-cost.json": "d3d9d50db1dd37328f28c9801da5de11a2d8bc2873c71aaedb12d54bb7f60ffb",
"/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17/score/result.json": "82b3a77ea40ff63c694ec7ac170662816116c970bad2a13eee8faba84219662e",
"/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17/predictions-manifest.json": "fdaa61c80020f53af7f4a5c62c0ac24cf91063a7749918cc1937fa436b887e05",
"/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17/terminal-git-freeze.json": "40e998b70208684855f7387d3e521d9a73f7337dda241679d9c1f43453454585",
"/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17/terminal-authorization.json": "462efc91386f22e4614267383d73ac3c0c206ad49ffe6c68dfae0d73049a787f",
"/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17/predictions-git-freeze.json": "19d5cdf551d85f0b68e397a0defa904ef98c6bb1b42bbc16d1916a5832b5e609",
"/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17/predictions-authorization.json": "aeab9e49e80ccf5d86140a89466090275fae8d3edd720d58fcf2d2ea15e1766b",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/last-epoch200.pt": "35cc9e41a4197e186570089ac305c4dee100f3d37611e0ca17e01db441b93756",
"/home/frank14f/optane/MeltMind/artifacts/phase2-b13-e1b-2026-11-17/contract.json": "de4196e1c2fc242efd88e1218f3ec2166ab58f1c4d194e6ff1155c40d57d319c",
"/home/frank14f/optane/MeltMind/artifacts/phase2-b13-e1b-2026-11-17/predictions-manifest.json": "66af9f9a9c4774358544ce04cf16ef7fb7f63db45335bb64378df664d7fbe51d",
"/home/frank14f/optane/MeltMind/artifacts/phase2-b13-e1b-2026-11-17/score/result.json": "89ecf6d471cfd244377d7c069ea576a2817ab6188796276a00cf85709860508d",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/010.json": "9633a5ee7dbbb3f7d23ab9d695dc1701d926b904ab43106f29a356a5b18778ec",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/020.json": "58f0f65710b72a64be8cb69267ea7ea61d8933b5fcebb6705f2f85cf488a4fa5",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/030.json": "c0e7ac20ba42b37771fb512d8e361d0faec7c9d149dda107c700d9cfc00a198a",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/040.json": "6b9af61690ecc2f723dec91c84295a476a4f3e98424cf530de2e8d6637284805",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/050.json": "830ed3deee3037751f8997fb32b42e3157aceed8c9bfa7b315afc921264a585b",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/060.json": "5e77dcb1ad08a2ec94e8f9487b78ec5abf810b7fc6d41e04cfbbc5cfb22af230",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/070.json": "d72b7a9427efec8a804a1ce9e8160a01fe5956cb54612f4d42595d758e36ad90",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/080.json": "a3153d3f8a7a5302c38c0c09b197940d1eb79ce760cbd63dd0300aff60a9a657",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/090.json": "2ac2fde814b5d167d8312753d4827db3a824496c66e2c32fc236812ff603c44e",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/100.json": "7a9eaef91cf57cc592784a025561a24c85800842f75373e986f05afd09bee1c9",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/110.json": "132ae8718e286f6d736e8743c819749ad019bf22ef9f16911b801c95d205fad1",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/120.json": "8b37b513bc5b4d27329da2a1c31df47daf8cbf994e13b3893eba79e57e88a2dd",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/130.json": "16836051ea9c17c35fade1372c7d6d27f9a6d094450e0e6a586e9b07f7272569",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/140.json": "b2a48eb48652d13781cd138cfdac7d27baa7363be99fddb1d9be842489720cd2",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/150.json": "57fdbfce102a5bd08f37d8ce36188d1d7d2e911991b6286dca00762ef8c50f74",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/160.json": "d0eb813a7206f3f4cd12325d054ff297130232dd45a463150b7a7d92f4cc07f2",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/170.json": "685ed97210349304571117cc4b2fffb67a86181d356cf6f2b14b9e612efad144",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/180.json": "17e3d70a147de465a0273c799fc84366780c5cb5507b34ef315d7e8ab8ff05ca",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/190.json": "fda3852e9d771c67a9a41b73064fdd6a318b8d38d4ab5db3cbaabea9e3b2e402",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/history/200.json": "b6aa4d4e046a7440073074f027e7107de6fdccfeb8f1755d2d6b1bcce6a59128"
},
"code_sha256": {
"/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805",
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062",
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5",
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073"
},
"scoring": {
"parts": 552,
"components": 204,
"bootstrap_replicates": 2000,
"bootstrap_seed": 2026111408,
"confirmation": "pooled field H2-A CI upper<0 with complete552/204",
"B13_and_last_and_tau": "report_only",
"new_exact_semantic_PASS_required": true,
"S1_PASS_reuse": false,
"fold0_score_hash_reuse": true
},
"stop_after": "TRAIN_COMPLETE_POSTFIT_PENDING",
"no_retry": true,
"data_sufficiency_started": false
}
@@ -0,0 +1,4 @@
{
"contract_sha256": "76e1976bed88fc390812238c3d80693446daea3840dc009ac7d8870ddc43a31d",
"proof_sha256": "58efa635aa04e1b258447935375a914b36fcb471cf17c1572dc5918682e35d1b"
}
@@ -0,0 +1,77 @@
{
"commit": "1f57bad66eb062d4f2269347532321929f873717",
"files": {
"/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/contract.json": "76e1976bed88fc390812238c3d80693446daea3840dc009ac7d8870ddc43a31d",
"/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/acceptance.json": "1a746680eae6490172d5e3220b7fe1fd1d5f58599c98120cfc2bad374a983234",
"/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/preparation-audit.json": "00e65b0512771d83a48ce06461a4f6474cb0c4d21bc3f790254d25be40674d26",
"/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/synthetic-attempt1.log": "dad348e2aa80da3fd7c8d6879a54971da876cdb58be0cca8ccb318d684c51bba",
"/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/synthetic-attempt2.log": "55524d018b9101c11c71089b7e570a859ce6025409665a39503b609550da36d5",
"/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805",
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062",
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5",
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073"
},
"git_paths": {
"/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/contract.json": "home-runs/phase2-h2-s2-2026-11-17/contract.json",
"/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/acceptance.json": "home-runs/phase2-h2-s2-2026-11-17/acceptance.json",
"/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/preparation-audit.json": "home-runs/phase2-h2-s2-2026-11-17/preparation-audit.json",
"/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/synthetic-attempt1.log": "home-runs/phase2-h2-s2-2026-11-17/synthetic-attempt1.log",
"/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17/synthetic-attempt2.log": "home-runs/phase2-h2-s2-2026-11-17/synthetic-attempt2.log",
"/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "tools/test_phase2_h2_s2.py",
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "tools/s2_main_memory_guard.py",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "tools/s2_recovery_gpu_supervisor.py",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "tools/s2_recovery_monitoring.py",
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "tools/s2_precision_continuation_guard.py",
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "tools/s2_gpu_idle_gate.py",
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "tools/s2_pilot_train.py",
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "tools/s2_models/network.py",
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "tools/freeze_g1_components.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "tools/phase2_h1_worker_v2.py",
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "tools/phase2_residual_math.py",
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "tools/gre24_models.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "tools/phase2_h2_s2_guard.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "tools/phase2_h2_s2_common.py",
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "tools/phase2_home_freeze_io.py",
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "tools/phase2_local_freeze_io.py",
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "tools/phase2_fixed_io.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "tools/phase2_h2_s2_worker.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "tools/phase2_h2_pilot_worker.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "tools/phase2_h2_model.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "tools/phase2_h2_data.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "tools/phase2_h2_s1_common.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "tools/phase2_h2_s1_worker.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "tools/phase2_h2_s2_prepare.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "tools/phase2_h2_prepare.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "tools/phase2_h2_s2_queue.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "tools/phase2_h2_s1_launch.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "tools/phase2_h2_s1_guard.py",
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "docs/phase2-h2-s2-execution-contract-2026-11-17.md"
},
"remote_published": false,
"local_repository": "/tmp/meltmind-phase2-local-fixed-20261117"
}
@@ -0,0 +1,34 @@
{
"status": "PREPARED_NOT_LAUNCHED",
"unix": 1794924971.8866704,
"h2_active_processes": [],
"original_fold0_predictions_hash_checked": 121,
"original_A_roles_models_checked": 5,
"test_partition552_204": true,
"test_target_values_read": false,
"validation_timing_source_max": 12.383006267948076,
"gate_progress": {
"stage": "GATE_DEPENDENT_FEATURES_ONLY",
"pending": 176,
"active": [
{
"slot": 0,
"pid": 700779,
"index": 315
},
{
"slot": 1,
"pid": 700809,
"index": 316
}
],
"terminal": 315,
"cumulative_cpu_seconds": 63736.12415484199,
"unix": 1794924971.790554
},
"gate_original_total_cpu": 72000,
"gate_geometry_cpu": 68400,
"gate_pins_verified": true,
"gate_95000_ruling": "1794919865.091919 /1794920747.468449; only after original budget stop and exit; all terminal hashes/old CPU once, no retries, new postfit identity review",
"gate_modified": false
}
@@ -0,0 +1,29 @@
test_actual_validation_cost_can_raise_projection (tools.test_phase2_h2_s2.BudgetAndGate) ... ok
test_budget_and_per_job_limits_stop (tools.test_phase2_h2_s2.BudgetAndGate) ... ok
test_confirmation_is_strict_and_complete_five_fold_only (tools.test_phase2_h2_s2.BudgetAndGate) ... ok
test_epoch10_and_final_remaining_validation_count (tools.test_phase2_h2_s2.BudgetAndGate) ... ok
test_exact_process_identity_excludes_preparation (tools.test_phase2_h2_s2.BudgetAndGate) ... ok
test_failure_peak_remains_even_after_lower_final_peak (tools.test_phase2_h2_s2.BudgetAndGate) ... ok
test_first_epoch_barrier_requires_exact_pass (tools.test_phase2_h2_s2.BudgetAndGate) ... ok
test_missing_epoch_invalid_timing_rejected (tools.test_phase2_h2_s2.BudgetAndGate) ... ok
test_old_cost_once_plus_all_four_waits_and_prediction_reserve (tools.test_phase2_h2_s2.BudgetAndGate) ... ok
test_all_new_folds_map_exact_original_roles_without_mutation (tools.test_phase2_h2_s2.Roles) ... ok
test_fold0_and_role_overlap_rejected (tools.test_phase2_h2_s2.Roles) ... ok
test_normalization_uses_only_actual_learning (tools.test_phase2_h2_s2.Roles) ... ERROR
test_scientific_epoch_loop_AST_unchanged_except_cost_wait (tools.test_phase2_h2_s2.Roles) ... ok
test_test_members_rejected_before_geometry_or_target_io (tools.test_phase2_h2_s2.Roles) ... ok
======================================================================
ERROR: test_normalization_uses_only_actual_learning (tools.test_phase2_h2_s2.Roles)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py", line 35, in test_normalization_uses_only_actual_learning
v=standardization(fold_view(self.pilot,f),Path(p));self.assertEqual(v['parts'],['learn'+str(f)]);self.assertEqual(v['mean'],[1.]*19);self.assertEqual(v['scale'],[1.]*19);self.assertEqual(seen,['learn'+str(f)])
File "/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py", line 31, in standardization
if x.shape!=(c['parts'][case]['nodes'],19) or not np.isfinite(x).all():raise ValueError('Learning features invalid')
KeyError: 'parts'
----------------------------------------------------------------------
Ran 14 tests in 0.032s
FAILED (errors=1)
@@ -0,0 +1,21 @@
test_actual_validation_cost_can_raise_projection (tools.test_phase2_h2_s2.BudgetAndGate) ... ok
test_budget_and_per_job_limits_stop (tools.test_phase2_h2_s2.BudgetAndGate) ... ok
test_confirmation_is_strict_and_complete_five_fold_only (tools.test_phase2_h2_s2.BudgetAndGate) ... ok
test_epoch10_and_final_remaining_validation_count (tools.test_phase2_h2_s2.BudgetAndGate) ... ok
test_exact_process_identity_excludes_preparation (tools.test_phase2_h2_s2.BudgetAndGate) ... ok
test_failure_peak_remains_even_after_lower_final_peak (tools.test_phase2_h2_s2.BudgetAndGate) ... ok
test_first_epoch_barrier_requires_exact_pass (tools.test_phase2_h2_s2.BudgetAndGate) ... ok
test_missing_epoch_invalid_timing_rejected (tools.test_phase2_h2_s2.BudgetAndGate) ... ok
test_old_cost_once_plus_all_four_waits_and_prediction_reserve (tools.test_phase2_h2_s2.BudgetAndGate) ... ok
test_queue_contract_identity_and_complete_guard (tools.test_phase2_h2_s2.GuardLifecycle) ... ok
test_recorded_failure_peak_cannot_be_ignored (tools.test_phase2_h2_s2.GuardLifecycle) ... ok
test_all_new_folds_map_exact_original_roles_without_mutation (tools.test_phase2_h2_s2.Roles) ... ok
test_fold0_and_role_overlap_rejected (tools.test_phase2_h2_s2.Roles) ... ok
test_normalization_uses_only_actual_learning (tools.test_phase2_h2_s2.Roles) ... ok
test_scientific_epoch_loop_AST_unchanged_except_cost_wait (tools.test_phase2_h2_s2.Roles) ... ok
test_test_members_rejected_before_geometry_or_target_io (tools.test_phase2_h2_s2.Roles) ... ok
----------------------------------------------------------------------
Ran 16 tests in 0.051s
OK
+55
View File
@@ -0,0 +1,55 @@
"""S2 identity/roles and cumulative GPU arithmetic; no fitting or target I/O."""
import json,math
from pathlib import Path
from phase2_fixed_io import BASE
from phase2_h1_worker_v2 import checked,sha
from phase2_home_freeze_io import verify
def read(p):return json.loads(Path(p).read_text())
def execution_command(args):
return any(Path(x).name in {'phase2_h2_s2_queue.py','phase2_h2_s2_guard.py','phase2_h2_s2_worker.py','phase2_h2_s1_guard.py','phase2_h2_s1_worker.py','phase2_h2_s1_postfit.py'} for x in args)
def fold_view(pilot,fold):
if fold not in (1,2,3,4):raise PermissionError('Only new folds1..4; fold0 is immutable reuse')
f=pilot['folds'][fold]
if f['fold']!=fold:raise ValueError('Original fold identity')
roles=[set(f[k]) for k in ('learning','validation','test')]
if any(roles[i]&roles[j] for i in range(3) for j in range(i)):raise ValueError('Role overlap')
return dict(pilot,folds=[f]) # Original preprocessing reads slot0; embedded fold identity remains actual fold.
def config(root):
c=read(root/'contract.json');verify(root,'implementation')
if c['approval']!='1794923791.467409' or c['total_h2_gpu_seconds']!=144000 or c['epochs']!=200 or c['seed']!=0 or [j['fold'] for j in c['jobs']]!=[1,2,3,4]:raise PermissionError('Fixed S2 scope')
if c['prior_gpu_seconds']!=18858.21490871499 or c['prediction_gpu_reserve']!=2400 or c['postfit_cpu_cap']!=3600 or c['new_fits']!=4:raise ValueError('Fixed prior ledger and bounded postfit reserve')
if len({j['gpu_uuid'] for j in c['jobs']})!=4:raise ValueError('Four distinct GPUs')
if c['train_gpu_cap']!=144000-c['prior_gpu_seconds']-c['prediction_gpu_reserve']:raise ValueError('GPU ledger identity')
for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h)
pilot=read(c['pilot_contract']);prior=read(Path(c['s1_root'])/'post-terminal-cost.json')
import torch,numpy as np
if torch.__version__!=pilot['versions']['torch'] or np.__version__!=pilot['versions']['numpy']:raise ValueError('Original scientific library versions required')
if prior['cumulative_h2_gpu_seconds']!=c['prior_gpu_seconds'] or not prior['within_cap']:raise ValueError('Original settled S1 cost')
for j in c['jobs']:
v=fold_view(pilot,j['fold']);f=v['folds'][0]
if j['learning_parts']!=len(f['learning']) or j['learning_nodes']!=f['learning_nodes'] or j['validation_nodes']!=f['validation_nodes']:raise ValueError('Original fold roster changed')
return c,pilot
def projection(c,observations):
if set(observations)!={1,2,3,4}:raise ValueError('All four first epochs required')
rows=[]
for j in c['jobs']:
v=observations[j['fold']];epoch=v['epoch'];charged=v['charged_gpu_seconds'];train=v['max_train_epoch_seconds'];validation=max(v.get('max_validation_seconds',0),j['validation_seconds_bound'])
if not isinstance(epoch,int) or not 1<=epoch<=200 or not all(math.isfinite(x) and x>=0 for x in [charged,train,validation]) or train<=0:raise ValueError('Invalid timing evidence')
future=1.25*((200-epoch)*train+(20-epoch//10)*validation)+60
estimate=charged+future;rows.append(dict(fold=j['fold'],completed_epochs=epoch,charged_gpu_seconds=charged,max_train_epoch_seconds=train,validation_seconds_bound=validation,projected_new_gpu_seconds=estimate,job_cap=j['gpu_seconds'],within_job_cap=estimate<=j['gpu_seconds']))
total=c['prior_gpu_seconds']+sum(r['projected_new_gpu_seconds'] for r in rows)+c['prediction_gpu_reserve']
passed=total<=c['total_h2_gpu_seconds'] and all(r['within_job_cap'] for r in rows)
return dict(status='PASS' if passed else 'STOP_BUDGET_PROJECTION',rows=rows,prior_gpu_once=c['prior_gpu_seconds'],prediction_gpu_reserve=c['prediction_gpu_reserve'],projected_total_gpu_seconds=total,total_cap=c['total_h2_gpu_seconds'],safety_factor=1.25,no_retry=True)
def first_epoch_release(root,contract_sha):
if (root/'global-stop.json').exists():raise RuntimeError('Global S2 stop before next epoch')
p=root/'first-epoch-projection.json'
if not p.exists():return False
d=read(p)
if d.get('contract_sha256')!=contract_sha or d.get('status')!='PASS':raise RuntimeError('First-epoch budget gate rejected')
return True
def confirmed(high,parts,components):
return bool(parts==552 and components==204 and high is not None and math.isfinite(high) and high<0)
+86
View File
@@ -0,0 +1,86 @@
"""Independent bounded S2 fold guard; shared global stop and immutable S1."""
import fcntl
import json
import os
from pathlib import Path
import signal
import sys
import threading
import time
import psutil
from phase2_h2_s2_common import config
from phase2_h1_worker_v2 import save,sha
from s2_main_memory_guard import PeakRecords
from s2_precision_continuation_guard import device_usage
from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage
def run(queue,fold_id):
c,_=config(queue);job=c['jobs'][fold_id-1];c=dict(c,**job);root=queue/'folds'/str(fold_id);os.sched_setaffinity(0,c['guard_cpus'])
with (root/'fold.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('Single S2 fold launch, no retry')
start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(queue/'contract.json'),gpu_uuid=c['gpu_uuid'],setup_gpu_seconds=c['setup_gpu_seconds']))
def elapsed():return c['setup_gpu_seconds']+time.monotonic()-start
def reason(x):
if x not in reasons:reasons.append(x)
command=[sys.executable,'tools/phase2_h2_s2_worker.py',str(queue),str(fold_id)]
def kill_worker():
for p in psutil.Process().children(recursive=True):
try:
if p.cmdline()==command:stop_group(p.pid)
except psutil.NoSuchProcess:pass
old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGINT,signal.SIGTERM]}
def scan():
if tracker.scan(root/'worker'):reason('RECORDED_VRAM_LIMIT')
def watch():
while not done.is_set():
try:
scan();mem,util=device_usage(c['gpu_uuid']);_,rss,_=tree_usage(os.getpid())
if mem>c['vram_bytes']:reason('DEVICE_VRAM_LIMIT')
if rss>c['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT')
if disk_usage(root)>c['disk_bytes']:reason('OUTPUT_DISK_LIMIT')
if elapsed()>=c['gpu_seconds']-5:reason('S2_FOLD_TIME_LIMIT')
if (queue/'global-stop.json').exists():reason('S2_GLOBAL_STOP')
for x in requested:reason(x)
save(root/'live-progress.json',dict(stage='S2_TRAINING',fold=fold_id,elapsed_gpu_seconds=elapsed(),device_bytes=mem,device_utilization=util,tree_rss_bytes=rss,recorded_peaks=tracker.maxima,reasons=reasons,unix=time.time()))
except Exception as exc:reason('MONITOR_ERROR');errors.append(dict(type=type(exc).__name__,reason=str(exc)))
if reasons:kill_worker()
done.wait(.5)
thread=None
try:
for i in range(3):
mem,util=device_usage(c['gpu_uuid']);samples.append(dict(unix=time.time(),memory_bytes=mem,utilization=util))
if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY')
if i<2:time.sleep(2)
save(root/'idle-polls.json',samples)
thread=threading.Thread(target=watch,daemon=True);thread.start()
remaining=c['gpu_seconds']-elapsed()-10
receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=c['cpu_seconds']-60,job_cpu_seconds=c['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=c['gpu_uuid'])
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill_worker()
finally:
done.set()
if thread:thread.join(timeout=4)
if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT')
try:scan()
except Exception as exc:reason('FINAL_PEAK_SCAN_ERROR');errors.append(dict(reason=str(exc)))
for s,handler in old.items():signal.signal(s,handler)
value=None;p=root/'worker/result.json'
try:
if p.exists():value=json.loads(p.read_text())
except Exception as exc:reason('RESULT_READ_ERROR');errors.append(dict(reason=str(exc)))
if not receipt or receipt['status']!='PASS':reason('WORKER_NOT_PASS')
if receipt and receipt.get('monitor_errors'):reason('WORKER_MONITOR_ERRORS')
if not value or value.get('status')!='COMPLETE':reason('NO_COMPLETE_TRAIN_RESULT')
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_CUDA_PEAK_RECORDS')
totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu
if elapsed()>c['gpu_seconds'] or totalcpu>c['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
try:config(queue)
except Exception as exc:reason('FINAL_SOURCE_IDENTITY: '+str(exc))
if value and (value.get('completed_epochs')!=200 or value.get('optimizer_steps')!=200*c['learning_parts'] or value.get('fold')!=fold_id):reason('INCOMPLETE_EPOCH_OR_FOLD_COUNT')
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',decision='TRAIN_COMPLETE_CHECKPOINT_FREEZE_PENDING' if not reasons else 'S2_FOLD_INCOMPLETE',reasons=reasons,monitor_errors=errors,
worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=elapsed(),fold=fold_id,cpu_seconds_charged=totalcpu,
worker_result_sha256=sha(p) if p.exists() else None,prediction_started=False,scoring_started=False,s2_started=True,no_retry=True))
return 0 if not reasons else 1
if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2])))
+51
View File
@@ -0,0 +1,51 @@
"""Metadata and hash preparation; no target arrays, GPU workload or launch."""
import json,time,shutil
from pathlib import Path
import psutil
from phase2_fixed_io import BASE
from phase2_h1_worker_v2 import save,sha,checked
from phase2_home_freeze_io import freeze,verify
from phase2_h2_prepare import closure
from phase2_h2_s2_common import config,execution_command
def read(p):return json.loads(Path(p).read_text())
def main():
root=Path('/home/frank14f/MeltMind-runs/phase2-h2-s2-2026-11-17')
if root.exists():raise FileExistsError('Existing S2 root; inspect, do not duplicate')
active=[p.info for p in psutil.process_iter(['pid','cmdline']) if execution_command(p.info['cmdline'] or [])]
if active:raise RuntimeError('Existing H2 task')
s1=Path('/home/frank14f/MeltMind-runs/phase2-h2-s1-postfit-2026-11-17');verify(s1,'terminal');verify(s1,'predictions');old=read(s1/'contract.json');cost=read(s1/'post-terminal-cost.json');done=read(s1/'complete.json');pilotpath=BASE/'artifacts/phase2-h2-pilot-2026-11-17/contract.json';pilot=read(pilotpath);aroot=BASE/'artifacts/phase2-b13-e1b-2026-11-17';a=read(aroot/'contract.json');h=read(BASE/'artifacts/phase2-h1-execution-2026-11-17/contract.json')
if done['status']!='COMPLETE' or not done['s2_eligible_by_original_gate'] or not cost['within_cap'] or cost['cumulative_h2_gpu_seconds']!=18858.21490871499:raise ValueError('Original S1 gate or ledger')
pred=read(s1/'predictions-manifest.json')['entries']
if len(pred)!=121:raise ValueError('Original121 predictions')
for e in pred:checked(e['path'],e['sha256'])
test=[case for f in pilot['folds'] for case in f['test']];by={r['case']:r for r in a['parts']}
if len(test)!=552 or len(set(test))!=552 or len({by[k]['component_id'] for k in test})!=204:raise ValueError('Fixed five-fold552/204')
for f in pilot['folds']:
af=a['folds'][f['fold']];hj=next(j for j in h['jobs'] if j['protocol']=='grouped' and j['pool']==100 and j['fold']==f['fold'])
if {r['case'] for r in af['rows']['learning']}!=set(f['learning']) or {r['case'] for r in af['rows']['test']}!=set(f['test']) or hj['baseline_model']!=af['A_baseline']:raise ValueError('Original H2/A model or fold role mismatch')
groups={role:{by[k]['component_id'] for k in f[role]} for role in ['learning','validation','test']}
if groups['test']&(groups['learning']|groups['validation']) or groups['learning']&groups['validation']:raise ValueError('Component leakage')
originaltrain=BASE/'artifacts/phase2-h2-s1-2026-11-17';history=[originaltrain/'worker/history'/f'{e:03d}.json' for e in range(10,201,10)];valmax=max(read(p)['validation_seconds'] for p in history)
uuids=['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c','GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36','GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76','GPU-8b71b032-1b75-2891-935c-ce106a53034a'];traincap=144000-cost['cumulative_h2_gpu_seconds']-2400;jobs=[];f0=pilot['folds'][0]
for fold in range(1,5):
f=pilot['folds'][fold];ratio=max(f['validation_nodes']/f0['validation_nodes'],f['validation_edges']/f0['validation_edges'],len(f['validation'])/len(f0['validation']))
jobs.append(dict(fold=fold,gpu_uuid=uuids[fold-1],worker_cpus=[2*(fold-1),2*(fold-1)+1],guard_cpus=[fold+7],gpu_seconds=traincap/4,cpu_seconds=traincap/4,setup_gpu_seconds=60,learning_parts=len(f['learning']),learning_nodes=f['learning_nodes'],validation_nodes=f['validation_nodes'],validation_seconds_bound=valmax*ratio,rss_bytes=16*2**30,vram_bytes=15*2**30,disk_bytes=20*2**30))
sources={str(BASE/p):h for p,h in read(originaltrain/'contract.json')['source_sha256'].items()}
for p in [s1/'contract.json',s1/'complete.json',s1/'post-terminal-cost.json',s1/'score/result.json',s1/'predictions-manifest.json',s1/'terminal-git-freeze.json',s1/'terminal-authorization.json',s1/'predictions-git-freeze.json',s1/'predictions-authorization.json',Path(old['checkpoint']),pilotpath,aroot/'contract.json',aroot/'predictions-manifest.json',aroot/'score/result.json',*history]:sources[str(p)]=sha(p)
for f in pilot['folds']:sources[f['clock_path']]=f['clock_sha256']
for p,digest in sources.items():checked(p,digest)
codes={str(BASE/p):h for p,h in closure(['phase2_h2_s2_common','phase2_h2_s2_worker','phase2_h2_s2_guard','phase2_h2_s2_queue','phase2_h2_s2_prepare','test_phase2_h2_s2']).items()}
for p,h in read(originaltrain/'contract.json')['code_sha256'].items():checked(BASE/p,h);codes[str(BASE/p)]=h
doc=BASE/'docs/phase2-h2-s2-execution-contract-2026-11-17.md';codes[str(doc)]=sha(doc)
root.mkdir();fold0=dict(checkpoint=old['checkpoint'],checkpoint_sha256=old['checkpoint_sha256'],prediction_manifest=str(s1/'predictions-manifest.json'),prediction_manifest_sha256=sha(s1/'predictions-manifest.json'),score=str(s1/'score/result.json'),score_sha256=sha(s1/'score/result.json'),terminal=verify(s1,'terminal')['commit'],no_reexecution=True)
c=dict(version='H2-S2-four-fold-training-v1',approval='1794923791.467409',correction='1794923933.456449',epochs=200,seed=0,new_fits=4,s1_root=str(s1),pilot_contract=str(pilotpath),fold0_reuse=fold0,prior_gpu_seconds=cost['cumulative_h2_gpu_seconds'],total_h2_gpu_seconds=144000,prediction_gpu_reserve=2400,train_gpu_cap=traincap,postfit_cpu_cap=3600,jobs=jobs,coordinator_cpus=[12,13],aggregate_rss_bytes=64*2**30,disk_bytes=80*2**30,source_sha256=sources,code_sha256=codes,scoring=dict(parts=552,components=204,bootstrap_replicates=2000,bootstrap_seed=2026111408,confirmation='pooled field H2-A CI upper<0 with complete552/204',B13_and_last_and_tau='report_only',new_exact_semantic_PASS_required=True,S1_PASS_reuse=False,fold0_score_hash_reuse=True),stop_after='TRAIN_COMPLETE_POSTFIT_PENDING',no_retry=True,data_sufficiency_started=False)
save(root/'contract.json',c)
for i in [1,2]:shutil.copyfile('/tmp/h2-s2-synthetic-attempt'+str(i)+'.log',root/('synthetic-attempt'+str(i)+'.log'))
if not (root/'synthetic-attempt2.log').read_text().rstrip().endswith('OK'):raise ValueError('Synthetic acceptance absent')
save(root/'acceptance.json',dict(status='PASS',tests=16,elapsed_seconds=.051,command='python -m unittest tools.test_phase2_h2_s2 -v',actual_log_sha256=sha(root/'synthetic-attempt2.log'),first_fixture_failure_retained=True,prelaunch_contract_path_fixed=True,scientific_workloads=0,scope='CPU synthetic/AST/role/budget and two guard lifecycle tests with fake supervisor/GPU probe; no real CUDA or targets',unix=time.time()))
gate=Path('/home/frank14f/MeltMind-runs/phase2-gate-outcome-calibration-v2-2026-11-17');g=read(gate/'contract.json');gprog=read(gate/'live-progress.json');gatepins={p:sha(BASE/p)==h for p,h in g['code_sha256'].items()};assert all(gatepins.values())
save(root/'preparation-audit.json',dict(status='PREPARED_NOT_LAUNCHED',unix=time.time(),h2_active_processes=active,original_fold0_predictions_hash_checked=121,original_A_roles_models_checked=5,test_partition552_204=True,test_target_values_read=False,validation_timing_source_max=valmax,gate_progress=gprog,gate_original_total_cpu=72000,gate_geometry_cpu=68400,gate_pins_verified=all(gatepins.values()),gate_95000_ruling='1794919865.091919 /1794920747.468449; only after original budget stop and exit; all terminal hashes/old CPU once, no retries, new postfit identity review',gate_modified=False))
freeze(root,'implementation',[root/'contract.json',root/'acceptance.json',root/'preparation-audit.json',root/'synthetic-attempt1.log',root/'synthetic-attempt2.log',*[Path(p) for p in codes]]);config(root)
print(json.dumps(dict(root=str(root),implementation=verify(root,'implementation')['commit'],contract_sha256=sha(root/'contract.json'),code_files=len(codes),jobs=[dict(fold=j['fold'],learning_parts=j['learning_parts'],gpu_seconds=j['gpu_seconds']) for j in jobs],launched=False)))
if __name__=='__main__':main()
+100
View File
@@ -0,0 +1,100 @@
"""One four-fold S2 queue, global GPU budget, first-epoch gate; stops before postfit."""
import fcntl,json,os,signal,subprocess,sys,time
from pathlib import Path
import psutil
from phase2_h2_s2_common import config,read,projection,execution_command
from phase2_h1_worker_v2 import save,sha,checked
from phase2_home_freeze_io import freeze,verify
from s2_recovery_gpu_supervisor import tree_usage,disk_usage
def run(root):
c,pilot=config(root);lock=(root/'queue.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('Single S2 queue, no retry')
for p in psutil.process_iter(['pid','cmdline']):
if p.pid!=os.getpid() and execution_command(p.info['cmdline'] or []):raise RuntimeError('Existing H2 execution process')
os.sched_setaffinity(0,c['coordinator_cpus']);os.environ['PYTORCH_CUDA_ALLOC_CONF']='expandable_segments:True';start=time.monotonic();cpu=time.process_time();active={};terminal={};requested=[];errors=[];timings={};projection_round=None
def stop(reason):
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time(),no_retry=True))
def charged():
return sum(v['gpu_seconds_charged'] for v in terminal.values())+sum(60+time.monotonic()-v['start'] for v in active.values())
def terminate(v):
if v.get('stop_sent') is None:
v['process'].send_signal(signal.SIGTERM);v['stop_sent']=time.monotonic()
elif time.monotonic()-v['stop_sent']>20:
try:
parent=psutil.Process(v['process'].pid);children=parent.children(recursive=True)
for p in reversed(children):
try:p.kill()
except psutil.NoSuchProcess:pass
parent.kill()
except psutil.NoSuchProcess:pass
def collect(fold,v):
p=v['process'];rc=p.poll()
if rc is None:return False
elapsed=60+time.monotonic()-v['start'];receipt_path=root/'folds'/str(fold)/'complete.json';d=read(receipt_path) if receipt_path.exists() else None
cost=max(elapsed,d['gpu_seconds_charged'] if d else 0);ok=rc==0 and d is not None and d['status']=='PASS'
row=dict(fold=fold,status='PASS' if ok else 'STOPPED_OR_FAILED',returncode=rc,gpu_seconds_charged=cost,coordinator_observed_seconds=elapsed,guard_gpu_seconds=d['gpu_seconds_charged'] if d else None,complete_sha256=sha(receipt_path) if d else None,receipt_present=d is not None)
terminal[fold]=row;v['log'].close();del active[fold];save(root/'terminal-ledger.json',dict(entries=list(terminal.values()),settled_gpu_seconds=sum(x['gpu_seconds_charged'] for x in terminal.values()),prior_gpu_once=c['prior_gpu_seconds']))
if not ok:stop('FOLD_'+str(fold)+'_FAILED_OR_MISSING_TERMINAL')
return True
def observe(fold):
history=root/'folds'/str(fold)/'worker/history';rows=[]
for p in sorted(history.glob('*.json')):
d=read(p)
if d['epoch_parts']!=c['jobs'][fold-1]['learning_parts'] or d['epoch_nodes']!=c['jobs'][fold-1]['learning_nodes'] or d['optimizer_steps']!=d['epoch']*c['jobs'][fold-1]['learning_parts']:raise ValueError('Incomplete timing epoch evidence')
rows.append(d)
if not rows:return None
cost=terminal[fold]['gpu_seconds_charged'] if fold in terminal else 60+time.monotonic()-active[fold]['start']
return dict(epoch=rows[-1]['epoch'],charged_gpu_seconds=cost,max_train_epoch_seconds=max(r['train_epoch_seconds'] for r in rows),max_validation_seconds=max(r.get('validation_seconds',0) for r in rows))
old={s:signal.signal(s,lambda n,f:requested.append('COORDINATOR_SIGNAL_'+str(n))) for s in [signal.SIGTERM,signal.SIGINT]}
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),folds=[1,2,3,4],prior_gpu_once=c['prior_gpu_seconds'],prediction_gpu_reserve=c['prediction_gpu_reserve'],postfit_started=False))
try:
for job in c['jobs']:
fold=job['fold'];d=root/'folds'/str(fold);d.mkdir(parents=True,exist_ok=False);log=(d/'entry.log').open('x');begin=time.monotonic()
command=[sys.executable,'tools/phase2_h2_s2_guard.py',str(root),str(fold)];p=subprocess.Popen(command,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
active[fold]=dict(process=p,log=log,start=begin,stop_sent=None);save(d/'dispatch.json',dict(fold=fold,guard_pid=p.pid,command=command,unix=time.time(),gpu_uuid=job['gpu_uuid'],setup_gpu_seconds=60))
while active:
for fold,v in list(active.items()):collect(fold,v)
for reason in requested:stop(reason)
amount=charged();_,rss,_=tree_usage(os.getpid());used=disk_usage(root)
if amount>=c['train_gpu_cap']-80:stop('CUMULATIVE_GPU_RESERVE_BOUNDARY')
if rss>c['aggregate_rss_bytes']:stop('AGGREGATE_RSS_LIMIT')
if used>c['disk_bytes']:stop('TOTAL_OUTPUT_DISK_LIMIT')
if not (root/'global-stop.json').exists():
timings={f:observe(f) for f in range(1,5)}
if all(v is not None for v in timings.values()):
current_round=min(v['epoch'] for v in timings.values())//10
if projection_round is None or current_round>projection_round:
d=projection(c,timings);d.update(contract_sha256=sha(root/'contract.json'),unix=time.time());save(root/('first-epoch-projection.json' if projection_round is None else 'latest-cost-projection.json'),d)
projection_round=current_round
if d['status']!='PASS':stop('PROJECTED_GPU_BUDGET_EXCEEDED')
if (root/'global-stop.json').exists():
for v in active.values():terminate(v)
save(root/'live-progress.json',dict(stage='STOPPING' if (root/'global-stop.json').exists() else 'S2_TRAINING',unix=time.time(),active=[dict(fold=f,guard_pid=v['process'].pid) for f,v in active.items()],terminal=list(terminal.values()),new_gpu_seconds_charged=amount,cumulative_h2_gpu_seconds=c['prior_gpu_seconds']+amount,prediction_gpu_reserve=c['prediction_gpu_reserve'],train_gpu_cap=c['train_gpu_cap'],epoch_counts={str(f):(x['epoch'] if x else 0) for f,x in timings.items()},rss_bytes=rss,disk_bytes=used))
if active:time.sleep(1)
if (root/'global-stop.json').exists() or len(terminal)!=4 or any(r['status']!='PASS' for r in terminal.values()):raise RuntimeError('S2 training incomplete; originals retained, no retries')
config(root);models=[]
for j in c['jobs']:
fold=j['fold'];w=root/'folds'/str(fold)/'worker';r=read(w/'result.json');checkpoint=checked(r['checkpoint'],r['checkpoint_sha256']);stats=read(checked(w/'standardization.json',r['standardization_sha256']))
if r['fold']!=fold or r['completed_epochs']!=200 or r['optimizer_steps']!=200*j['learning_parts'] or r['test_targets_read'] is not False or stats['parts']!=pilot['folds'][fold]['learning']:raise ValueError('Final checkpoint/normalization role identity')
models.append(dict(fold=fold,checkpoint=str(checkpoint),checkpoint_sha256=r['checkpoint_sha256'],standardization=str(w/'standardization.json'),standardization_sha256=r['standardization_sha256'],worker_result_sha256=sha(w/'result.json')))
amount=charged()
if amount>c['train_gpu_cap'] or c['prior_gpu_seconds']+amount+c['prediction_gpu_reserve']>144000:raise RuntimeError('Final aggregate GPU limit')
save(root/'models-terminal.json',dict(models=models,fold0_reused_by_hash=c['fold0_reuse'],new_fits=4,test_targets_read=False,postfit_started=False));freeze(root,'models',[root/'contract.json',root/'models-terminal.json',root/'terminal-ledger.json',*[Path(m['checkpoint']) for m in models],*[Path(m['standardization']) for m in models]])
save(root/'complete.json',dict(status='TRAIN_COMPLETE_POSTFIT_PENDING',terminal=list(terminal.values()),new_gpu_seconds_charged=amount,prior_gpu_once=c['prior_gpu_seconds'],cumulative_h2_gpu_seconds=c['prior_gpu_seconds']+amount,prediction_gpu_reserve=c['prediction_gpu_reserve'],coordinator_cpu_seconds=time.process_time()-cpu,wall_seconds=time.monotonic()-start,new_fits=4,postfit_started=False,no_retry=True));freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'models-terminal.json'])
except BaseException as exc:
stop(type(exc).__name__+': '+str(exc));errors.append(str(exc))
while active:
for f,v in list(active.items()):
try:
if not collect(f,v):terminate(v)
except Exception as e:
errors.append(str(e));terminate(v)
if v['process'].poll() is not None:
terminal[f]=dict(fold=f,status='MALFORMED_TERMINAL',gpu_seconds_charged=60+time.monotonic()-v['start'],returncode=v['process'].returncode);v['log'].close();del active[f]
if active:time.sleep(.5)
amount=charged();save(root/'complete.json',dict(status='STOPPED',reason=str(exc),errors=errors,terminal=list(terminal.values()),new_gpu_seconds_charged=amount,prior_gpu_once=c['prior_gpu_seconds'],cumulative_h2_gpu_seconds=c['prior_gpu_seconds']+amount,coordinator_cpu_seconds=time.process_time()-cpu,postfit_started=False,no_retry=True));return 1
finally:
for s,h in old.items():signal.signal(s,h)
return 0
if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1])))
+67
View File
@@ -0,0 +1,67 @@
"""One new S2 fold, same frozen seed0 recipe/200epochs; no test prediction or target reads."""
import json,os,sys,time,traceback
from pathlib import Path
import numpy as np
import torch
from phase2_h2_s1_common import selected_checkpoint
from phase2_h2_s2_common import config,fold_view,first_epoch_release
from phase2_h2_data import standardization
from phase2_h2_data import load_part,to_device,arguments,cached
from phase2_h2_model import H2Geo,optimizer,lr_epoch
from phase2_h2_pilot_worker import seed,peaks,train_step
from phase2_residual_math import residual_target
from phase2_h1_worker_v2 import save,sha
def run(root,fold_id):
c,pilot=config(root);old=fold_view(pilot,fold_id);job=c['jobs'][fold_id-1];out=root/'folds'/str(fold_id)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir()
os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0
def record(item,epoch_file=False):
item=dict(fold=fold_id,stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks())
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
save(out/'progress.json',item)
if epoch_file:save(out/'history'/f'{epoch:03d}.json',item)
return item
try:
if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity')
stage='LEARNING_ONLY_NORMALIZATION';normal=out/'normalization';normal.mkdir();stats=standardization(old,normal);save(out/'standardization.json',stats);save(out/'fold-identity.json',dict(actual_fold=fold_id,learning=old['folds'][0]['learning'],validation=old['folds'][0]['validation'],test=old['folds'][0]['test'],clock_sha256=old['folds'][0]['clock_sha256']))
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed()
model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes']
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=job['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters())))
for epoch in range(1,201):
stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0.
if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch')
order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts)
for index in order:
case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda')
result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss']
record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip']))
del data
if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch')
metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch))
if epoch%10==0:
stage='VALIDATION_DIAGNOSTIC';model.eval();val_loss=0.;val_nodes=0;vstart=time.monotonic()
with torch.no_grad():
for case in fold['validation']:
data=to_device(load_part(old,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False)
if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output')
with (out/'validation-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,epoch=epoch,role='validation_diagnostic_only',target_sha256=old['parts'][case]['output_sha256']['target.npy']))+'\n')
target=residual_target(cached(old,case,'target'),data['F'],fold['c'],fit_dtype='float32')
delta=p.cpu().numpy().astype(np.float64)-target;val_loss+=float(delta@delta);val_nodes+=data['nodes'];del data,p
torch.cuda.synchronize()
if val_nodes!=fold['validation_nodes']:raise ValueError('Validation coverage')
metrics.update(validation_mse=val_loss/val_nodes,validation_nodes=val_nodes,validation_seconds=time.monotonic()-vstart,validation_used_for_selection=False)
stage='EPOCH_COMPLETE';record(metrics,epoch_file=True)
if epoch==1:
stage='WAIT_ALL_FIRST_EPOCH_COST_GATE';record(dict(epoch_parts=seen,epoch_nodes=nodes))
while not first_epoch_release(root,sha(root/'contract.json')):time.sleep(.5)
if epoch<200:
for group in opt.param_groups:group['lr']=lr_epoch(epoch+1)
stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch)
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=fold_id,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name)
save(out/'result.json',dict(status='COMPLETE',fold=fold_id,completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks()))
except BaseException as exc:
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
if __name__=='__main__':run(Path(sys.argv[1]),int(sys.argv[2]))
+97
View File
@@ -0,0 +1,97 @@
"""S2 CPU synthetic boundaries; no real learning/validation/test arrays or GPU."""
import ast,copy,inspect,json,tempfile,unittest
from pathlib import Path
from unittest.mock import patch
import numpy as np
from phase2_h2_s2_common import fold_view,projection,first_epoch_release,confirmed,execution_command
from phase2_h2_data import load_part,standardization
import phase2_h2_s1_worker as oldworker
import phase2_h2_s2_worker as newworker
import phase2_h2_s2_guard as guard
from phase2_h1_worker_v2 import save,sha
from s2_main_memory_guard import PeakRecords
class Roles(unittest.TestCase):
def setUp(self):
self.pilot=dict(folds=[dict(fold=i,learning=['learn'+str(i)],validation=['val'+str(i)],test=['test'+str(i)],learning_nodes=2) for i in range(5)],parts={'learn'+str(i):dict(nodes=2) for i in range(5)})
def test_all_new_folds_map_exact_original_roles_without_mutation(self):
before=copy.deepcopy(self.pilot)
for f in range(1,5):
v=fold_view(self.pilot,f);self.assertEqual(v['folds'][0],self.pilot['folds'][f]);self.assertEqual(v['folds'][0]['fold'],f)
self.assertEqual(self.pilot,before)
def test_fold0_and_role_overlap_rejected(self):
for f in [0,5,-1]:
with self.assertRaises(PermissionError):fold_view(self.pilot,f)
self.pilot['folds'][2]['test']=['learn2']
with self.assertRaises(ValueError):fold_view(self.pilot,2)
def test_test_members_rejected_before_geometry_or_target_io(self):
for f in range(1,5):
for role in ['learning','validation']:
with self.assertRaises(PermissionError):load_part(fold_view(self.pilot,f),'test'+str(f),{},role,Path('/unused'))
def test_normalization_uses_only_actual_learning(self):
for f in range(1,5):
seen=[]
def features(c,case,name):
self.assertEqual(case,'learn'+str(f));self.assertEqual(name,'features');seen.append(case);return np.vstack([np.zeros(19),np.full(19,2.)])
with tempfile.TemporaryDirectory() as p,patch('phase2_h2_data.cached',features):
v=standardization(fold_view(self.pilot,f),Path(p));self.assertEqual(v['parts'],['learn'+str(f)]);self.assertEqual(v['mean'],[1.]*19);self.assertEqual(v['scale'],[1.]*19);self.assertEqual(seen,['learn'+str(f)])
def test_scientific_epoch_loop_AST_unchanged_except_cost_wait(self):
def loop(fn):
t=ast.parse(inspect.getsource(fn));return next(n for n in ast.walk(t) if isinstance(n,ast.For) and isinstance(n.target,ast.Name) and n.target.id=='epoch')
old=loop(oldworker.run);new=loop(newworker.run);new.body=[n for n in new.body if not(isinstance(n,ast.If) and ast.unparse(n.test)=='epoch == 1')]
self.assertEqual(ast.dump(old),ast.dump(new))
class BudgetAndGate(unittest.TestCase):
def setUp(self):
self.c=dict(prior_gpu_seconds=18858.21490871499,total_h2_gpu_seconds=144000,prediction_gpu_reserve=2400,jobs=[dict(fold=f,gpu_seconds=(144000-18858.21490871499-2400)/4,validation_seconds_bound=12.) for f in range(1,5)])
self.obs={f:dict(epoch=1,charged_gpu_seconds=200.,max_train_epoch_seconds=90.,max_validation_seconds=0.) for f in range(1,5)}
def test_old_cost_once_plus_all_four_waits_and_prediction_reserve(self):
d=projection(self.c,self.obs);expected=18858.21490871499+4*(200+1.25*(199*90+20*12)+60)+2400;self.assertEqual(d['projected_total_gpu_seconds'],expected);self.assertEqual(d['status'],'PASS')
def test_budget_and_per_job_limits_stop(self):
o=copy.deepcopy(self.obs);o[1]['max_train_epoch_seconds']=150;self.assertEqual(projection(self.c,o)['status'],'STOP_BUDGET_PROJECTION')
for v in o.values():v['max_train_epoch_seconds']=200
self.assertGreater(projection(self.c,o)['projected_total_gpu_seconds'],144000)
def test_missing_epoch_invalid_timing_rejected(self):
with self.assertRaises(ValueError):projection(self.c,{1:self.obs[1]})
for field,value in [('epoch',0),('max_train_epoch_seconds',float('nan')),('charged_gpu_seconds',-1)]:
o=copy.deepcopy(self.obs);o[2][field]=value
with self.assertRaises(ValueError):projection(self.c,o)
def test_actual_validation_cost_can_raise_projection(self):
a=projection(self.c,self.obs);o=copy.deepcopy(self.obs);o[1]['max_validation_seconds']=30;self.assertGreater(projection(self.c,o)['projected_total_gpu_seconds'],a['projected_total_gpu_seconds'])
def test_epoch10_and_final_remaining_validation_count(self):
o=copy.deepcopy(self.obs)
for v in o.values():v['epoch']=200
self.assertAlmostEqual(projection(self.c,o)['projected_total_gpu_seconds'],18858.21490871499+4*(200+60)+2400)
def test_first_epoch_barrier_requires_exact_pass(self):
with tempfile.TemporaryDirectory() as d:
p=Path(d);self.assertFalse(first_epoch_release(p,'fixed'))
for result in [dict(status='STOP_BUDGET_PROJECTION',contract_sha256='fixed'),dict(status='PASS',contract_sha256='other')]:
(p/'first-epoch-projection.json').write_text(json.dumps(result))
with self.assertRaises(RuntimeError):first_epoch_release(p,'fixed')
(p/'first-epoch-projection.json').write_text(json.dumps(dict(status='PASS',contract_sha256='fixed')));self.assertTrue(first_epoch_release(p,'fixed'))
(p/'global-stop.json').write_text('{}')
with self.assertRaises(RuntimeError):first_epoch_release(p,'fixed')
def test_confirmation_is_strict_and_complete_five_fold_only(self):
self.assertTrue(confirmed(-.001,552,204))
for h,n,g in [(0,552,204),(.001,552,204),(None,552,204),(float('nan'),552,204),(-.001,121,37),(-.001,551,204),(-.001,552,203)]:self.assertFalse(confirmed(h,n,g))
def test_exact_process_identity_excludes_preparation(self):
self.assertFalse(execution_command(['python','tools/prepare_phase2_h2_s2.py']));self.assertTrue(execution_command(['python','tools/phase2_h2_s2_queue.py']))
def test_failure_peak_remains_even_after_lower_final_peak(self):
with tempfile.TemporaryDirectory() as d:
p=Path(d);t=PeakRecords();(p/'failure.json').write_text(json.dumps(dict(peak_allocated_bytes=100,peak_reserved_bytes=16*2**30)));self.assertTrue(t.scan(p));(p/'failure.json').unlink();(p/'result.json').write_text(json.dumps(dict(peak_allocated_bytes=100,peak_reserved_bytes=200)));self.assertTrue(t.scan(p));self.assertEqual(t.maxima['peak_reserved_bytes'],16*2**30)
class GuardLifecycle(unittest.TestCase):
def exercise(self,peak):
with tempfile.TemporaryDirectory() as d:
q=Path(d);save(q/'contract.json',dict(synthetic=True));r=q/'folds/1';r.mkdir(parents=True)
job=dict(fold=1,gpu_uuid='synthetic',guard_cpus=[0],gpu_seconds=600,cpu_seconds=600,setup_gpu_seconds=60,learning_parts=2,rss_bytes=16*2**30,vram_bytes=15*2**30,disk_bytes=20*2**30)
calls=[]
def worker(command,*args,**kwargs):
calls.append(command);self.assertEqual(command[-2:],[str(q),'1']);w=r/'worker';w.mkdir();save(w/'result.json',dict(status='COMPLETE',fold=1,completed_epochs=200,optimizer_steps=400,peak_allocated_bytes=100,peak_reserved_bytes=peak));return dict(status='PASS',cpu_seconds=.01,monitor_errors=[])
with patch.object(guard,'config',return_value=(dict(jobs=[job]),{})),patch.object(guard.os,'sched_setaffinity'),patch.object(guard,'device_usage',return_value=(0,0)),patch.object(guard,'supervise',worker),patch.object(guard.time,'sleep'):
rc=guard.run(q,1)
self.assertEqual(len(calls),1);self.assertEqual(json.loads((r/'started.json').read_text())['contract_sha256'],sha(q/'contract.json'));return rc,json.loads((r/'complete.json').read_text())
def test_queue_contract_identity_and_complete_guard(self):
rc,d=self.exercise(100);self.assertEqual(rc,0);self.assertEqual(d['status'],'PASS')
def test_recorded_failure_peak_cannot_be_ignored(self):
rc,d=self.exercise(16*2**30);self.assertEqual(rc,1);self.assertIn('RECORDED_VRAM_LIMIT',d['reasons'])
if __name__=='__main__':unittest.main()