Publish fixed item c training entry, shared capacity and synthetic acceptance for Assist review

This commit is contained in:
MeltAstra committed 2026-11-18 21:38:44 +08:00
1 parent 5b6a953fc5
commit 706608dfe0
28 files changed
+13373

No files matched your search

@@ -0,0 +1,15 @@
{
"status": "PASS",
"returncode": 0,
"wall_seconds": 3.8560638920171186,
"cpu_seconds": 3.8524149999999997,
"synthetic_only": true,
"scientific_arrays_decoded": false,
"gpu_workload": false,
"prior_attempt_logs_preserved": [
"attempt1.log",
"attempt2.log"
],
"prior_process_cpu_not_measured": true,
"all_preparation_charge_reserved": 600
}
@@ -0,0 +1,34 @@
test_actual_runaway_stops (test_phase3_item_c.BudgetTests) ... ok
test_failed_worker_late_memory_peak (test_phase3_item_c.BudgetTests) ... ok
test_incomplete_epoch_history (test_phase3_item_c.BudgetTests) ... ok
test_initial_inclusive_cap (test_phase3_item_c.BudgetTests) ... ok
test_negative_nan_cost (test_phase3_item_c.BudgetTests) ... ok
test_pending_wait_not_free (test_phase3_item_c.BudgetTests) ... ok
test_projected_fold_above_runaway_not_alone_stop (test_phase3_item_c.BudgetTests) ... ok
test_cpu_headroom (test_phase3_item_c.CapacityTests) ... ok
test_foreign_processes_count (test_phase3_item_c.CapacityTests) ... ok
test_gpu0_old_pending (test_phase3_item_c.CapacityTests) ... ok
test_gpu3_single (test_phase3_item_c.CapacityTests) ... ok
test_host_ram_reserve (test_phase3_item_c.CapacityTests) ... ok
test_memory_reserves_future_peaks (test_phase3_item_c.CapacityTests) ... ok
test_phase_reservation_once (test_phase3_item_c.CapacityTests) ... ok
test_two_admissions_atomic (test_phase3_item_c.CapacityTests) ... ok
test_unseen_reservations_block_fourth (test_phase3_item_c.CapacityTests) ... ok
test_visible_reserved_process_counted_once (test_phase3_item_c.CapacityTests) ... ok
test_bad_duplicate_fails (test_phase3_item_c.ObjectiveTests) ... ok
test_constant_shift (test_phase3_item_c.ObjectiveTests) ... ok
test_degenerate_gradients_zero (test_phase3_item_c.ObjectiveTests) ... ok
test_duplicate_order_does_not_vote (test_phase3_item_c.ObjectiveTests) ... ok
test_exact_old_peak_tie (test_phase3_item_c.ObjectiveTests) ... ok
test_fp64_prior_preserved (test_phase3_item_c.ObjectiveTests) ... ok
test_gradient_formula (test_phase3_item_c.ObjectiveTests) ... ok
test_missing_id_fails (test_phase3_item_c.ObjectiveTests) ... ok
test_nan_truth_not_undefined (test_phase3_item_c.ObjectiveTests) ... ok
test_no_validation_or_test_role (test_phase3_item_c.ObjectiveTests) ... ok
test_original_weight_plus_one_location (test_phase3_item_c.ObjectiveTests) ... ok
test_worker_only_loads_learning (test_phase3_item_c.ObjectiveTests) ... ok
----------------------------------------------------------------------
Ran 29 tests in 0.190s
OK
@@ -0,0 +1,36 @@
test_actual_runaway_stops (test_phase3_item_c.BudgetTests) ... ok
test_failed_worker_late_memory_peak (test_phase3_item_c.BudgetTests) ... ok
test_incomplete_epoch_history (test_phase3_item_c.BudgetTests) ... ok
test_initial_inclusive_cap (test_phase3_item_c.BudgetTests) ... ok
test_negative_nan_cost (test_phase3_item_c.BudgetTests) ... ok
test_pending_wait_not_free (test_phase3_item_c.BudgetTests) ... ok
test_projected_fold_above_runaway_not_alone_stop (test_phase3_item_c.BudgetTests) ... ok
test_cpu_headroom (test_phase3_item_c.CapacityTests) ... ok
test_foreign_processes_count (test_phase3_item_c.CapacityTests) ... ok
test_gpu0_old_pending (test_phase3_item_c.CapacityTests) ... ok
test_gpu3_single (test_phase3_item_c.CapacityTests) ... ok
test_host_ram_reserve (test_phase3_item_c.CapacityTests) ... ok
test_memory_reserves_future_peaks (test_phase3_item_c.CapacityTests) ... ok
test_phase_reservation_once (test_phase3_item_c.CapacityTests) ... ok
test_two_admissions_atomic (test_phase3_item_c.CapacityTests) ... ok
test_unseen_reservations_block_fourth (test_phase3_item_c.CapacityTests) ... ok
test_visible_reserved_process_counted_once (test_phase3_item_c.CapacityTests) ... ok
test_bad_duplicate_fails (test_phase3_item_c.ObjectiveTests) ... ok
test_constant_shift (test_phase3_item_c.ObjectiveTests) ... ok
test_degenerate_gradients_zero (test_phase3_item_c.ObjectiveTests) ... ok
test_duplicate_order_does_not_vote (test_phase3_item_c.ObjectiveTests) ... ok
test_exact_old_peak_tie (test_phase3_item_c.ObjectiveTests) ... ok
test_fp64_prior_preserved (test_phase3_item_c.ObjectiveTests) ... ok
test_gradient_formula (test_phase3_item_c.ObjectiveTests) ... ok
test_missing_id_fails (test_phase3_item_c.ObjectiveTests) ... ok
test_nan_truth_not_undefined (test_phase3_item_c.ObjectiveTests) ... ok
test_no_validation_or_test_role (test_phase3_item_c.ObjectiveTests) ... ok
test_original_weight_plus_one_location (test_phase3_item_c.ObjectiveTests) ... ok
test_worker_only_loads_learning (test_phase3_item_c.ObjectiveTests) ... ok
test_all_five_terminate_before_model_freeze (test_phase3_item_c.QueueSyntheticTests) ... ok
test_failure_stops_without_retry (test_phase3_item_c.QueueSyntheticTests) ... ok
----------------------------------------------------------------------
Ran 31 tests in 0.213s
OK
@@ -0,0 +1,39 @@
test_actual_runaway_stops (test_phase3_item_c.BudgetTests) ... ok
test_failed_worker_late_memory_peak (test_phase3_item_c.BudgetTests) ... ok
test_incomplete_epoch_history (test_phase3_item_c.BudgetTests) ... ok
test_initial_inclusive_cap (test_phase3_item_c.BudgetTests) ... ok
test_negative_nan_cost (test_phase3_item_c.BudgetTests) ... ok
test_pending_wait_not_free (test_phase3_item_c.BudgetTests) ... ok
test_projected_fold_above_runaway_not_alone_stop (test_phase3_item_c.BudgetTests) ... ok
test_cpu_headroom (test_phase3_item_c.CapacityTests) ... ok
test_foreign_processes_count (test_phase3_item_c.CapacityTests) ... ok
test_gpu0_old_pending (test_phase3_item_c.CapacityTests) ... ok
test_gpu3_single (test_phase3_item_c.CapacityTests) ... ok
test_host_ram_reserve (test_phase3_item_c.CapacityTests) ... ok
test_memory_reserves_future_peaks (test_phase3_item_c.CapacityTests) ... ok
test_phase_reservation_once (test_phase3_item_c.CapacityTests) ... ok
test_stale_lease_blocks_without_reclaim (test_phase3_item_c.CapacityTests) ... ok
test_two_admissions_atomic (test_phase3_item_c.CapacityTests) ... ok
test_unseen_reservations_block_fourth (test_phase3_item_c.CapacityTests) ... ok
test_visible_reserved_process_counted_once (test_phase3_item_c.CapacityTests) ... ok
test_bad_duplicate_fails (test_phase3_item_c.ObjectiveTests) ... ok
test_constant_shift (test_phase3_item_c.ObjectiveTests) ... ok
test_degenerate_gradients_zero (test_phase3_item_c.ObjectiveTests) ... ok
test_duplicate_order_does_not_vote (test_phase3_item_c.ObjectiveTests) ... ok
test_exact_old_peak_tie (test_phase3_item_c.ObjectiveTests) ... ok
test_fp64_prior_preserved (test_phase3_item_c.ObjectiveTests) ... ok
test_gradient_formula (test_phase3_item_c.ObjectiveTests) ... ok
test_missing_id_fails (test_phase3_item_c.ObjectiveTests) ... ok
test_nan_truth_not_undefined (test_phase3_item_c.ObjectiveTests) ... ok
test_no_validation_or_test_role (test_phase3_item_c.ObjectiveTests) ... ok
test_original_weight_plus_one_location (test_phase3_item_c.ObjectiveTests) ... ok
test_worker_only_loads_learning (test_phase3_item_c.ObjectiveTests) ... ok
test_all_five_terminate_before_model_freeze (test_phase3_item_c.QueueSyntheticTests) ... ok
test_failure_stops_without_retry (test_phase3_item_c.QueueSyntheticTests) ... ok
test_third_failed_admission_stops (test_phase3_item_c.QueueSyntheticTests) ... ok
test_two_failed_admission_samples_then_success (test_phase3_item_c.QueueSyntheticTests) ... ok
----------------------------------------------------------------------
Ran 34 tests in 0.260s
OK
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,13 @@
{
"status": "PREPARED_PENDING_ASSIST_REVIEW",
"contract_sha256": "837b869e9d42d7402c8cdb4f3f9236462e9ec5bf2a80bcb561e5a4c566361fb7",
"code_files": 46,
"source_files": 86,
"metadata_cpu_seconds": 0.44506590199999996,
"arrays_decoded": false,
"model_weights_loaded": false,
"labels_decoded": false,
"new_fits_started": false,
"phase_reserve_total": 711581.3159993868,
"preparation_gpu_charge_reserved": 600
}
@@ -0,0 +1,14 @@
{
"status": "READY_FOR_ASSIST_REVIEW",
"implementation_commit": "a2540974ae646f341beca248ce8a5c4f638fe2d8",
"contract_sha256": "76fa0975b61d86363d5b9bf110a58ae56ca0a1462a3154744e1df54285d01233",
"binding_sha256": "20bf6ca532f42fdba10e480a0405a4f7333a946c951e45a1eac83c582024625b",
"code_files": 52,
"source_files": 87,
"old_seed1_code_pins": "PASS",
"old_track_a_recovery_code_pins": "PASS",
"snapshot_unix": 1795009065,
"item_c_started": false,
"actual_peer_review": false,
"shared_capacity_registry_created": false
}
@@ -737,3 +737,12 @@ source8a8ebd1f/contract4b21dedd/proof8baec54f/comparison828530e4/27pins不变,
Melt只做25.5685s六次只读采样并写过两份未完成草稿;草稿已移出tools、逐字归档withdrawn-throughput-draft,从未运行benchmark/测试/冻结/读标签,无probe root/process。原seed1 PID1476593和三worker、Track A recovery PID1650006和fold4worker1650207均核在,78code pins全匹配;两队列无STOP/complete。资源及撤回证据artifacts/gpu-concurrency-assessment-2026-11-18;说明docs/independent-research-concurrency-2026-11-18.md。新共享capacity调度仅待新入口实现/审阅,没有已执行收益或新研究结果。
## Item c 入口固定待 Assist 审阅(2026-11-18)
Claude1795007580/7058原条已实读。固定implementation a2540974ae646f341beca248ce8a5c4f638fe2d8,contract76fa0975b61d86363d5b9bf110a58ae56ca0a1462a3154744e1df54285d01233,proof54af93dca18b7a9eb416a1230a265be74287af690e493f2d96521ae09f256b04,comparisona9dac0a4aff9168f6c98beef46333e5a68c1fc455d818a1afc7d21b12a11be72,完整52code map,binding20bf6ca532f42fdba10e480a0405a4f7333a946c951e45a1eac83c582024625b。34 CPU合成PASS,含双进程原子抢槽及退出时序/失败不重试模拟;最新验收进程CPU3.852415s,早两轮未单独测量进程CPU,原日志保留、准备保守预收600s。
主规格9974ae27/次终点05344bac、19列/seed0/200epoch不变;只加固定lambda=.001/theta=.03位置损失,学习query固定原ID/顺序/multiplicity/尺度;位置项唯一ID等底权、精确seconds真峰最小ID并列。无validation/test标签读取,省略旧仅诊断validation loop。新共享准入真实整机进程+lease,GPU0–2最多3/GPU3最多1、c只GPU1/2、GPU0旧pending保留。60000秒为实际失控限,1.5只用于剩余训练总投影。c205200/post7200GPU+3600CPU,phase3预留711581.315999/720000,seed1旧失败已含其167200帽只计一次,Track A独立。
唯一root /home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18;config及绑定核验PASS不是同行PASS。尚无review.json/started/共享registry目录,没有真实训练/预测/评分。旧seed1与Track A共78代码pins实核保持;仅读取最新回执,两原队列当时仍在训练/等待。下一步为实际Assist入口审阅,不重开已撤回benchmark、不重复启动旧队列。全五终态/模型冻结/退出后仍需独立postfit入口审阅及post-terminal-cost最终账检查。详见 docs/phase3-item-c-entry-2026-11-18.md。
+10
View File
@@ -1,3 +1,13 @@
## Item c 入口固定待 Assist 审阅(2026-11-18)
Claude1795007580/7058原条已实读。固定implementation a2540974ae646f341beca248ce8a5c4f638fe2d8,contract76fa0975b61d86363d5b9bf110a58ae56ca0a1462a3154744e1df54285d01233,proof54af93dca18b7a9eb416a1230a265be74287af690e493f2d96521ae09f256b04,comparisona9dac0a4aff9168f6c98beef46333e5a68c1fc455d818a1afc7d21b12a11be72,完整52code map,binding20bf6ca532f42fdba10e480a0405a4f7333a946c951e45a1eac83c582024625b。34 CPU合成PASS,含双进程原子抢槽及退出时序/失败不重试模拟;最新验收进程CPU3.852415s,早两轮未单独测量进程CPU,原日志保留、准备保守预收600s。
主规格9974ae27/次终点05344bac、19列/seed0/200epoch不变;只加固定lambda=.001/theta=.03位置损失,学习query固定原ID/顺序/multiplicity/尺度;位置项唯一ID等底权、精确seconds真峰最小ID并列。无validation/test标签读取,省略旧仅诊断validation loop。新共享准入真实整机进程+lease,GPU0–2最多3/GPU3最多1、c只GPU1/2、GPU0旧pending保留。60000秒为实际失控限,1.5只用于剩余训练总投影。c205200/post7200GPU+3600CPU,phase3预留711581.315999/720000,seed1旧失败已含其167200帽只计一次,Track A独立。
唯一root /home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18;config及绑定核验PASS不是同行PASS。尚无review.json/started/共享registry目录,没有真实训练/预测/评分。旧seed1与Track A共78代码pins实核保持;仅读取最新回执,两原队列当时仍在训练/等待。下一步为实际Assist入口审阅,不重开已撤回benchmark、不重复启动旧队列。全五终态/模型冻结/退出后仍需独立postfit入口审阅及post-terminal-cost最终账检查。详见 docs/phase3-item-c-entry-2026-11-18.md。
---
## 最新用户纠正:研究事项并行,吞吐benchmark撤回且未启动(2026-11-18)
用户通过TRON明确要不同研究工作的并行,不是同一训练1/2/3副本吞吐或算子优化;以后意见先由Claude领导裁量。Claude1795006714/6766已实读:撤回6592的benchmark,入口按c→1b→b→Track B构建,各自审后可并行;seed1只约束效果/噪声判决,不一律阻塞开工。每项训练入口一次审阅,test前独立审阅保持。GPU0–2最多3任务、GPU3严格1,当前frozen训练和postfit不改。现有pending时保留一张0–2不给新入口,不能停现有worker制造空卡或重派旧fold。
+39
View File
@@ -0,0 +1,39 @@
# Item c:固定末填位置损失训练入口,等待 Assist 审阅
本轮完成入口和合成验收,不运行真实训练、预测或评分。唯一计划运行目录为 `/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18`;其 `expected-review-binding.json` 是实际审阅身份,发布提交不能替代 implementation commit。
## 裁量与研究目标
Claude [7058](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1795007058477789) 要求先建设 c,再交 Assist。已实读 [7580](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1795007580895539):c 总帽205200 GPU秒,含准备、等待、失败、后处理,postfit预留7200 GPU秒/3600 CPU秒;phase3总帽720000,Track A单独记账,ensemble另议。
沿用 Mold 主规格9974ae2780fcc9754b305448f3eec51f5902d9fa3770a2807eaf3ed9f497b537、次终点05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521。目标是改善原有限 query 上的硬末点定位,同时守住全场 relative L2 的预定退化门;这不是全模腔末填或拓扑分支结论。
## 科学实现
原19列、H2图/网络、归一化、学习时钟、五组折552件/204组件、seed0、200epochs、AdamW/LR、全节点回归权重、梯度裁剪及末轮模型保留规则不变。
每学习件调用原无标签 `prediction_inputs` 抽取 query,保留原ID/顺序/multiplicity/坐标/Rg/near_wall/anchor ID。位置项只使用唯一ID、每候选底权相同。真峰用原 `seconds.npy` 的精确并列、最小原ID;不从FP32残差重建真峰。学习件 `seconds` 读取前记录角色/哈希,距离和query身份写成逐折逐件缓存并固定manifest,跨epoch复用。
固定 `s=cF+residual_hat`,cF与位置项保持FP64;theta=.03、lambda=.001。总损失为原weighted MSE加一次每件位置期望距离;无新头、区域列、调参、集成、裁剪时场或软坐标输出。空/非有限/不一致query属于错误;原评分器在有限非空输入上没有“末点undefined”返回,因此不能自行把异常转成零损失。单候选/相等距离可自然得到零梯度。
新worker不读验证或测试标签;旧训练的每10轮验证只是诊断、不参与优化或选模,此入口省略该诊断循环,明确作为接口差异提交审阅。训练仍为200完整轮,无早停、best epoch或宽度选择。
## 并发与预算
新共享锁和registry放在 `/home/frank14f/MeltMind-runs/phase3-shared-capacity-2026-11-18`,只在真实入口获审后调用时创建。按整机GPU计算进程与未显现的跨入口lease联合计槽,去重已显现worker;GPU0–2最多3、GPU3最多1,当前 c 只选GPU1/2。GPU0旧队列pending保留规则也写入通用准入。不会改动旧worker、源码、亲和性或预算。
新任务预留4GiB GPU(原H2实测约1.4–1.52GB allocator峰的保守工程包络为2GiB,再加2GiB余量),实际allocator记录超过2GiB或自身NVML进程超过4GiB即失败;这些是新入口待审工程限制,并非已测 c 峰。主机RSS每job4GiB,GPU/主机RAM保留20%,另为原队列未来折保留8GiB RAM;CPU准入需当前负载不超过80%,新worker独占预留16–25中的一个核,guard用26/27。磁盘每job8GiB,启动要求剩余100GiB。未登记的真实GPU进程也占槽并保守补足4GiB峰预留。
lease在spawn前持锁预留,转移给guard后才允许worker启动;退出后由原owner收集、确认worker不再存活才释放。正常退出时序差或失联lease仅阻止新准入,不自动清除、不杀其它任务。进程身份含创建时间;guard/worker核对父进程身份并设置父进程退出信号。观察到外部任务导致超槽时只停止本入口,不抢占外部任务。无法强制不参与共享锁的外部进程遵守准入;这种外部竞争仍可能让本入口停止。
各折从队列开始到自身终态计费,包括未派发等待、setup和失败,另各预收60秒setup;离线准备/合成保守预收600秒。单折60000秒只用于实际失控保护,不作为单折投影门。完整epoch实测后,以最大完整轮耗时对剩余轮乘1.5,附120秒收尾;未测折按原均分、已测中位数和实际等待+收尾三者最大值保留。只按总投影/真实累计停;首轮通过该总门再释放后续训练。任一真实失败停止剩余队列、无重试。
总预留:item1已完成72421.31599938683、seed1整个累计帽167200(旧失败不再额外加)、c205200、1b88560、b178200,共711581.3159993868≤720000。后两项只是预留,未创建入口或启动。Track A和ensemble不混入本账。现有入口原冻结帽不改。训练终态模型冻结后,`post-terminal-cost.json` 另计冻结尾段一次;后处理必须检查该最终账以及完整五折/模型冻结/退出,不能仅凭训练PASS。
## 验收与后续门
CPU合成检查覆盖旧评分并列等价、重复ID不加票、梯度解析式、平移不变性、退化零梯度、FP64 prior、原回归权重、非学习角色拒绝、GPU槽位/未显现预留/真实外部进程、双进程竞争、旧pending保留、主机余量、总投影和失败记录峰。还模拟完整五折终态及单折失败不重试,不使用真实数据或GPU。日志和测量见 `artifacts/phase3-item-c-acceptance-2026-11-18`;早期两轮完整进程CPU未单独测量,保留原日志,不能用unittest内部时长冒充进程CPU实测。
只有本次完整绑定的实际Assist PASS、审阅原件采用并冻结后,才具备训练入口的审阅条件;本次不代造PASS、不启动。所有五模型终态/哈希/退出后仍需独立postfit实现及实际事前审阅,才可预测和读取test。当前次终点只绑定规格,未实现或执行评分。
旧库门保持mean ΔD≤−.010、ΔD上界<0、ΔrelativeL2上界<+.001,组件等权成对2000bootstrap、seed2026111408,另受同终点seed1噪声比较约束;seed1不阻塞建入口或获审后的训练,却约束最终收益结论。全新独立家族确认没有数据群与执行授权,保持待定。当前没有新科学结果,也未恢复已撤回的吞吐benchmark。
@@ -0,0 +1,9 @@
{
"status": "PREPARED_BINDING_CONFIG_PASS",
"binding_sha256": "20bf6ca532f42fdba10e480a0405a4f7333a946c951e45a1eac83c582024625b",
"code_files": 52,
"source_files": 87,
"review_present": false,
"started": false,
"actual_peer_review": false
}
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,60 @@
{
"reviewed_commit": "a2540974ae646f341beca248ce8a5c4f638fe2d8",
"contract_sha256": "76fa0975b61d86363d5b9bf110a58ae56ca0a1462a3154744e1df54285d01233",
"implementation_proof_sha256": "54af93dca18b7a9eb416a1230a265be74287af690e493f2d96521ae09f256b04",
"comparison_contract_sha256": "a9dac0a4aff9168f6c98beef46333e5a68c1fc455d818a1afc7d21b12a11be72",
"code_sha256": {
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_common.py": "4e85f5d211c4d3e2188422aa4c0cefa40dcfa14ef32e90894ed3412de1fd1d12",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_data.py": "fe3aa3e023f22f4fe55cd47e7ffa677b58f52a041867ac05620cbd329d913cb6",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_guard.py": "7144fa77ffe61a2a3c92c050234e370ef4dfeb0e31fafb419f752f969737ff1a",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_loss.py": "79e291866bda300ecdc5577c62903d34abeec14f5cc36db05e346f9ffd64c24b",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_queue.py": "a04879cc0169ecb523a88360cddca1e234d91b3adbc20b3ec31d02461ed767eb",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_worker.py": "0f8f35e1422021b2a4f097c2e62267fb935fe4658fd0ee8ccd251609a4062ea7",
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d",
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "968c4c7daeace0efa48713a93e38b97fa8a809502bf4dda3b55f523489123e9a",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_c.py": "782c415b1f536eb2b85aa2c9c4a9a0dc278f9b7134e8a83e2a6af8199396c53a",
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "588c646101be0d50f505f9a12a1c98361aa3437c8c594eecb7321831f717a741",
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "188565801f6901d0192f79aec0889e632501e672a4149d76d1835efbb9b45447",
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"/home/frank14f/optane/MeltMind/tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609",
"/home/frank14f/optane/MeltMind/tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61",
"/home/frank14f/optane/MeltMind/tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
"/home/frank14f/optane/MeltMind/tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0",
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_c.py": "17f9a6b3a146a9326b7975948ad6647612f5a208f5985a8dda8225254ff066bb",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
"/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062",
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5",
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073"
}
}
@@ -0,0 +1,4 @@
{
"contract_sha256": "76fa0975b61d86363d5b9bf110a58ae56ca0a1462a3154744e1df54285d01233",
"proof_sha256": "54af93dca18b7a9eb416a1230a265be74287af690e493f2d96521ae09f256b04"
}
@@ -0,0 +1,139 @@
{
"commit": "a2540974ae646f341beca248ce8a5c4f638fe2d8",
"files": {
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_common.py": "4e85f5d211c4d3e2188422aa4c0cefa40dcfa14ef32e90894ed3412de1fd1d12",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_data.py": "fe3aa3e023f22f4fe55cd47e7ffa677b58f52a041867ac05620cbd329d913cb6",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_guard.py": "7144fa77ffe61a2a3c92c050234e370ef4dfeb0e31fafb419f752f969737ff1a",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_loss.py": "79e291866bda300ecdc5577c62903d34abeec14f5cc36db05e346f9ffd64c24b",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_queue.py": "a04879cc0169ecb523a88360cddca1e234d91b3adbc20b3ec31d02461ed767eb",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_worker.py": "0f8f35e1422021b2a4f097c2e62267fb935fe4658fd0ee8ccd251609a4062ea7",
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d",
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "968c4c7daeace0efa48713a93e38b97fa8a809502bf4dda3b55f523489123e9a",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_c.py": "782c415b1f536eb2b85aa2c9c4a9a0dc278f9b7134e8a83e2a6af8199396c53a",
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "588c646101be0d50f505f9a12a1c98361aa3437c8c594eecb7321831f717a741",
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "188565801f6901d0192f79aec0889e632501e672a4149d76d1835efbb9b45447",
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"/home/frank14f/optane/MeltMind/tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609",
"/home/frank14f/optane/MeltMind/tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61",
"/home/frank14f/optane/MeltMind/tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
"/home/frank14f/optane/MeltMind/tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0",
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_c.py": "17f9a6b3a146a9326b7975948ad6647612f5a208f5985a8dda8225254ff066bb",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
"/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062",
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5",
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073",
"/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/contract.json": "76fa0975b61d86363d5b9bf110a58ae56ca0a1462a3154744e1df54285d01233",
"/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/comparison-contract.json": "a9dac0a4aff9168f6c98beef46333e5a68c1fc455d818a1afc7d21b12a11be72",
"/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/preparation-audit.json": "c0cf684f9d403d319151bd467c0aff571a7157eba069bc7430da1db3e753f6d8",
"/home/frank14f/optane/MeltMind/docs/phase3-item-c-entry-2026-11-18.md": "e239980a0db53826f774539b56ccf29d8deee3fcab15907da51e4f93b0a2ea42",
"/home/frank14f/optane/MeltMind/references/2026-11-18/phase3-item-c/mold-tail-volume-peak-stability-spec.md": "05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521",
"/home/frank14f/optane/MeltMind/references/2026-11-18/phase3-item-c/mold-last-fill-objective-spec.md": "9974ae2780fcc9754b305448f3eec51f5902d9fa3770a2807eaf3ed9f497b537",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/acceptance.json": "44a77dd619ef99118d23f24902f757a74fa10c3b37934cb702a227fbb594d2a3",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/attempt1.log": "0442ffe4c529ebed72c6be3cce5feb6f9f24d1bad3dc65301d1edb928105af1e",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/attempt2.log": "c9b75fd6f0d3e8777ce06aff726fc6c1f4c75985c86a9aeb1fd11c1bb1f9552f",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/attempt3.log": "318f7c124fadf04823a3a9ee870c1dbe111410831e4a7fc42d83786c8f2376e6",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/initial-prepared-comparison-contract.json": "9b75d33ac08f6a437f6755668f9efe52493872673ae9aa65bbc88e3a667eedec",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/initial-prepared-contract.json": "837b869e9d42d7402c8cdb4f3f9236462e9ec5bf2a80bcb561e5a4c566361fb7",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/initial-prepared-preparation-audit.json": "3a48b1d2b119707e38b0470029cf28019fd40c61afbfad3b21ca9cf345bcf5cf"
},
"git_paths": {
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "tools/freeze_g1_components.py",
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "tools/gre24_models.py",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "tools/phase2_gate_outcome_review.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "tools/phase2_h1_worker_v2.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "tools/phase2_h2_data.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "tools/phase2_h2_model.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "tools/phase2_h2_pilot_worker.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "tools/phase2_h2_s1_common.py",
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "tools/phase2_home_freeze_io.py",
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "tools/phase2_local_freeze_io.py",
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "tools/phase2_residual_math.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_common.py": "tools/phase3_item_c_common.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_data.py": "tools/phase3_item_c_data.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_guard.py": "tools/phase3_item_c_guard.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_loss.py": "tools/phase3_item_c_loss.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_queue.py": "tools/phase3_item_c_queue.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_worker.py": "tools/phase3_item_c_worker.py",
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "tools/phase3_seed1_v2_common.py",
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "tools/phase3_shared_capacity.py",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_c.py": "tools/prepare_phase3_item_c.py",
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "tools/s2_custody_common.py",
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "tools/s2_custody_statistics.py",
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "tools/s2_gpu_idle_gate.py",
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "tools/s2_main_memory_guard.py",
"/home/frank14f/optane/MeltMind/tools/s2_models/__init__.py": "tools/s2_models/__init__.py",
"/home/frank14f/optane/MeltMind/tools/s2_models/attention.py": "tools/s2_models/attention.py",
"/home/frank14f/optane/MeltMind/tools/s2_models/blocks.py": "tools/s2_models/blocks.py",
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "tools/s2_models/features.py",
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "tools/s2_models/network.py",
"/home/frank14f/optane/MeltMind/tools/s2_models/normalization.py": "tools/s2_models/normalization.py",
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "tools/s2_models/objectives.py",
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "tools/s2_pilot_train.py",
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "tools/s2_precision_continuation_guard.py",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "tools/s2_recovery_gpu_supervisor.py",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "tools/s2_recovery_monitoring.py",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_c.py": "tools/test_phase3_item_c.py",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "tools/track_a_h2_common.py",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "tools/track_a_h2_prepare.py",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "tools/track_a_recovery_monitor.py",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "tools/track_a_recovery_supervisor.py",
"/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "tools/test_phase2_h2_s2.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "tools/phase2_h2_s2_guard.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "tools/phase2_h2_s2_common.py",
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "tools/phase2_fixed_io.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "tools/phase2_h2_s2_worker.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "tools/phase2_h2_s1_worker.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "tools/phase2_h2_s2_prepare.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "tools/phase2_h2_prepare.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "tools/phase2_h2_s2_queue.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "tools/phase2_h2_s1_launch.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "tools/phase2_h2_s1_guard.py",
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "docs/phase2-h2-s2-execution-contract-2026-11-17.md",
"/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/contract.json": "home-runs/phase3-item-c-2026-11-18/contract.json",
"/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/comparison-contract.json": "home-runs/phase3-item-c-2026-11-18/comparison-contract.json",
"/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/preparation-audit.json": "home-runs/phase3-item-c-2026-11-18/preparation-audit.json",
"/home/frank14f/optane/MeltMind/docs/phase3-item-c-entry-2026-11-18.md": "docs/phase3-item-c-entry-2026-11-18.md",
"/home/frank14f/optane/MeltMind/references/2026-11-18/phase3-item-c/mold-tail-volume-peak-stability-spec.md": "references/2026-11-18/phase3-item-c/mold-tail-volume-peak-stability-spec.md",
"/home/frank14f/optane/MeltMind/references/2026-11-18/phase3-item-c/mold-last-fill-objective-spec.md": "references/2026-11-18/phase3-item-c/mold-last-fill-objective-spec.md",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/acceptance.json": "artifacts/phase3-item-c-acceptance-2026-11-18/acceptance.json",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/attempt1.log": "artifacts/phase3-item-c-acceptance-2026-11-18/attempt1.log",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/attempt2.log": "artifacts/phase3-item-c-acceptance-2026-11-18/attempt2.log",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/attempt3.log": "artifacts/phase3-item-c-acceptance-2026-11-18/attempt3.log",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/initial-prepared-comparison-contract.json": "artifacts/phase3-item-c-acceptance-2026-11-18/initial-prepared-comparison-contract.json",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/initial-prepared-contract.json": "artifacts/phase3-item-c-acceptance-2026-11-18/initial-prepared-contract.json",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/initial-prepared-preparation-audit.json": "artifacts/phase3-item-c-acceptance-2026-11-18/initial-prepared-preparation-audit.json"
},
"remote_published": false,
"local_repository": "/tmp/meltmind-phase2-local-fixed-20261117"
}
@@ -0,0 +1,15 @@
{
"status": "PREPARED_PENDING_ASSIST_REVIEW",
"contract_sha256": "76fa0975b61d86363d5b9bf110a58ae56ca0a1462a3154744e1df54285d01233",
"code_files": 52,
"source_files": 87,
"metadata_cpu_seconds": 0.44506590199999996,
"arrays_decoded": false,
"model_weights_loaded": false,
"labels_decoded": false,
"new_fits_started": false,
"phase_reserve_total": 711581.3159993868,
"preparation_gpu_charge_reserved": 600,
"synthetic_checks": 34,
"acceptance_sha256": "44a77dd619ef99118d23f24902f757a74fa10c3b37934cb702a227fbb594d2a3"
}
@@ -0,0 +1,29 @@
# H2 后续 C|用候选末点的空间误差监督到达场
MoldAstra · 2026-11-17 · 事前规格。次序 A 种子、B 区域、C 末填;由 Claude 在 H2-S2 确认后裁量。与原 H2 单模型 seed 0 配对。仍用 19 列和原网格 H2;不叠加八区特征、集成、终端预测头或额外标签来源。B17 是树模型上最后 10% 体积加权与路径列的组合实验,两项分别判决。[B17 规格](https://drive.google.com/file/d/196qqDkaP2fVlwj27Jc-zctm9kqV35ZZM/view)
## 对准什么误差
两个相距很远的分支末端,真实归一化时间为 1.00、0.99,预测为 0.99、1.00。所有其他点完全正确时,全场误差可以很小,最后一点却落到了另一支。简单提高晚填节点的回归权重仍没有给这次空间错位明确的代价。另外,先把两个候选位置加权平均,可能产生一个模腔之外的“位置”。我们在**每个实际候选点**算定位误差,然后平均这些误差。Li 等在视觉关键点定位中研究过这种期望定位误差;这里可以在冻结的有限查询集上精确求和,避免采样。论文没有验证注塑末填。[Li 等,NeurIPS 2021](https://proceedings.neurips.cc/paper_files/paper/2021/hash/e4a6222cdb5b34375400904f03d8e6a5-Abstract.html)
## 固定目标与接口
每次训练仍处理一个完整学习件 p。依既有评分 query 的无标签抽取规则,在学习件上固定同口径的 query 原 ID、顺序与 multiplicity,跨 epoch 复用;不取任何验证/测试件标签。Q 取这些记录中的唯一节点 ID。定义 d_i 为“若预测在 i 处取唯一最大,冻结末点评分器对该件真值给出的 distance/Rg”。这样真值的并列、参考位置、重复 ID 和几何尺度均由原评分器决定,不能挑离预测最近的真峰。实现可先按原规则求一次真末点,再一次向量化算各候选到它的距离;应与逐点唯一最大定义在合成并列例上相等。训练时每个唯一候选在 softmax 的底权相同,防止重复抽到同一 ID 就获得额外票数;正式评估仍用全部原始评分记录。旧全节点回归的权重保持原状。
设 s_i=cF_i+r̂_i,是原节点无量纲预测,固定温度 θ=0.03(t_set 的 3%),π_i=exp((s_i−max s)/θ)/Σexp((s_j−max s)/θ)。逐件训练损失为
L_p = L_H2,p + 0.001 × Σ_{i∈Q} π_i d_i .
L_H2,p 精确沿用旧节点 MSE 和件大小权重。位置项每件一次,不随网格节点数放大。常数 0.001 与温度一次锁定,是研究取值,未经工业容差校准;不搜索温度、尾段宽度或系数。软权重使用带梯度的无量纲时间;FP64 的 cF 保持原精度。对任意节点 k,位置项未乘系数前的导数为 π_k(d_k−Σπd)/θ。因此训练会压低离真末点较远而当前又可能胜出的节点,也会提高近端候选。偏移所有预测同一常数不改变该项;单一候选、距离全相等时梯度为零。过冷会在一个错误峰上梯度饱和,过热会分散到早填区,放大预测时间差也可能使该项改善;故保留场监督并在判决中检验场退化。
仅当学习件的原评分明确返回无法定义的真末点时,该件仍用 L_H2,p,位置项置零,登记原因/分母。预测时直接输出原时场,使用原**硬**末点、原排序、原并列、原尺度评分;不做截断、软坐标或后处理。这个假设针对有限查询集,查询集漏掉真实三维末填区时不能据此推断全模腔末点。
## 事前判决与封存
- 旧库筛选固定 552 件/204 亲缘组件原五折,五个 seed 0 候选与原五个 H2 同身份测试预测配对。令 ΔD 为逐件硬末点误差候选减 H2,ΔE 为逐件 relative L2 差。组件内件均值、组件等权,同批组件索引成对 bootstrap 2,000 次、seed 2026111408、双侧 95% CI。仅当平均 ΔD≤−0.010 Rg、ΔD 上界 <0、且 ΔE 上界 <+0.001,才称“入围”。预定场容许退化为 0.10 个百分点。未达标停止该固定配方。全名单共同评分不全时仅作不完整诊断;并列规模、各折、τ 和两臂原值一起报告。
- 要主张对新几何家族的末点收益,在任何新标签打开前固定分组、单位、query 和比较模型,旧库与确认主群在近重复/来源图上无连接,至少 25 个可配对新组件。对每个新组件,把其冻结 UTF-8 标识的 SHA-256 解释为无符号大端整数,模 5 选择同一外折的已冻结候选与对照,整组同行,两臂每件均只推理一次。此确认评估已训练五对单模型的固定分派策略;不需用确认件训练或选模型。仅当新群 ΔD 的 95% 上界 <−0.010、且 ΔE 上界 <+0.001,确认有预设幅度的末点收益并满足场容差。两条件同时成立,等号不通过。门槛是研究取值。另报组件数、两臂输入覆盖、各折分派数量/件数,覆盖不足只作描述。
- 本项不占用 B17 的一次封存或已解封数据。缺少可独立使用的全新主群时可完成旧库筛选,确认保持待定。不能从旧库或不同候选反复试出结论,也不能把不同工程末填缺陷与本 query 尺度的末点误差等同。
## 预计代价
旧库五次新增训练,按 H2 fold 0 累计 5.24 GPU-h 加 25% 余量约 32.75 GPU-h;若位置项增加 5–15% 时间,规划约 34–38 GPU-h,增幅未经测量。每件每步新增 O(|Q|) 的 softmax、加权距离和梯度;固定学习件的距离与 ID 可预存。部署不增加网络推理次数。新群对照另需每件两次前向及 CPU 评分,不需重训。以上是成本情景;若实际分块反传或查询准备超过估计,按实账呈现。三份规格都不修改 S2 的训练或评分,也没有启动新试验。[H2-S2 既有裁量](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794923791467409)
+108
View File
@@ -0,0 +1,108 @@
"""Item c immutable scope, assignments and conservative total-cost projection."""
import json
import math
import statistics
from pathlib import Path
from phase2_h1_worker_v2 import checked, sha
from phase2_home_freeze_io import verify
from phase3_seed1_v2_common import fold_view
ROOT = Path('/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18')
def read(p):
return json.loads(Path(p).read_text())
def config(root):
if root.resolve() != ROOT:
raise PermissionError('Sole item c root')
c = read(root/'contract.json')
verify(root, 'implementation')
expected = dict(approval='1795007580.895539', seed=0, epochs=200, new_fits=5,
gpu_cap=205200, postfit_gpu_reserve=7200, postfit_cpu_cap=3600,
phase3_gpu_cap=720000, theta=.03, coefficient=.001, no_retry=True)
if any(c.get(k) != v for k,v in expected.items()):
raise PermissionError('Fixed c scope/budgets')
for p,h in {**c['source_sha256'], **c['code_sha256']}.items():
checked(p,h)
pilot = read(c['pilot_contract'])
import numpy as np
import torch
if np.__version__ != pilot['versions']['numpy'] or torch.__version__ != pilot['versions']['torch']:
raise ValueError('Original environment versions')
if [j['id'] for j in c['jobs']] != list(range(5)):
raise ValueError('Exactly five folds')
tests = []
for job in c['jobs']:
f = fold_view(pilot, job['id'])['folds'][0]
tests.extend(f['test'])
if job['target_steps'] != 200*len(f['learning']):
raise ValueError('Full fixed epochs')
for k in ('learning','validation','test','c','clock_sha256','learning_nodes'):
if job[k] != f[k]:
raise ValueError('Original fold identity changed')
if len(tests) != 552 or len(set(tests)) != 552:
raise ValueError('Original552 roster')
rows=read(c['query_roster_contract'])['rows']
if len(rows)!=552 or {r['case'] for r in rows}!=set(tests) or len({r['component_id'] for r in rows})!=204:
raise ValueError('552 parts /204 components query roster')
for r in rows:
p=pilot['parts'][r['case']]
if r['v5_cache']['output_sha256']!=p['output_sha256'] or r['nodes']!=p['nodes'] or r['component_id']!=p['clock_group']:
raise ValueError('Learning query/geometry/cache identity')
checked(c['comparison_contract'], c['comparison_contract_sha256'])
return c,pilot
def assignment(root,c,index,digest):
a = read(checked(root/'jobs'/str(index)/'assignment.json',digest))
if a['id'] != index or a['contract_sha256'] != sha(root/'contract.json'):
raise PermissionError('Assignment identity')
return dict(c['jobs'][index], **{k:a[k] for k in ('gpu_uuid','worker_cpus','token','queue_identity')})
def first_epoch_release(root,index):
if (root/'global-stop.json').exists():
raise RuntimeError('Global stop')
p = root/'jobs'/str(index)/'first-epoch-projection.json'
if not p.exists():
return False
d = read(p)
if d['status'] != 'PASS' or d['contract_sha256'] != sha(root/'contract.json'):
raise RuntimeError('First-epoch gate not passed')
return True
def project(c, charges, histories, terminal):
if set(charges) != set(range(5)) or set(histories)-set(charges) or set(terminal)-set(charges):
raise ValueError('Cost ledger fold identity')
values = {}
for i,v in charges.items():
if not math.isfinite(v) or v < 0:
raise ValueError('Invalid actual cost')
if i in terminal:
values[i] = v
elif histories.get(i):
rows = histories[i]
e = max(r['epoch'] for r in rows)
if sorted(r['epoch'] for r in rows) != list(range(1,e+1)) or e > 200:
raise ValueError('Incomplete epoch history')
t = max(r['train_epoch_seconds'] for r in rows)
if not math.isfinite(t) or t <= 0:
raise ValueError('Invalid measured epoch')
# Factor1.5 applies to remaining training, never to already charged cost.
values[i] = v + 1.5*(200-e)*t + 120
initial = (c['gpu_cap']-c['postfit_gpu_reserve']-c['preparation_charge_gpu_seconds'])/5
median = statistics.median(values.values()) if values else initial
for i in charges:
if i not in values:
values[i] = max(charges[i]+120, initial, median)
actual = c['preparation_charge_gpu_seconds']+sum(charges.values())
projected = c['preparation_charge_gpu_seconds']+sum(values.values())+c['postfit_gpu_reserve']
ok = projected <= c['gpu_cap'] and actual+c['postfit_gpu_reserve'] <= c['gpu_cap']
ok &= all(charges[i] < c['runaway_gpu_seconds'] for i in charges if i not in terminal)
return dict(status='PASS' if ok else 'STOP_BUDGET', projected_gpu_seconds=projected,
actual_training_and_preparation_gpu_seconds=actual,
folds={str(i):dict(charged=charges[i], projected=values[i]) for i in charges},
postfit_gpu_reserve=c['postfit_gpu_reserve'], safety_factor=1.5)
+57
View File
@@ -0,0 +1,57 @@
"""Learning-only query cache. No validation/test labels or fitted selection."""
import json
from pathlib import Path
import numpy as np
from phase2_h1_worker_v2 import checked, sha, save
from phase2_h2_data import cached
from s2_custody_common import prediction_inputs
from phase3_item_c_loss import candidates
def require_learning(view, case):
f = view['folds'][0]
if case not in f['learning'] or case in f['validation'] or case in f['test']:
raise PermissionError('Location truth restricted to learning role')
def prepare_queries(view, rows, output):
output = Path(output)
output.mkdir(exist_ok=False)
manifest = {}
by_case = {r['case']: r for r in rows}
if len(by_case) != len(rows):
raise ValueError('Repeated roster case')
for case in view['folds'][0]['learning']:
require_learning(view, case) # before ANY case array access
row = by_case[case]
q = prediction_inputs(row) # existing label-free query recipe
full = cached(view, case, 'raw_ids')
idx = np.searchsorted(full, q['raw_ids'])
if np.any(idx >= len(full)) or not np.array_equal(full[idx], q['raw_ids']):
raise ValueError('Original query/full-node identity')
if q['t_set'] != view['parts'][case]['t_set']:
raise ValueError('Original process-time identity')
# The original scorer uses physical seconds with exact equality ties.
# FP32 residual reconstruction is not an acceptable true-peak source.
digest = view['parts'][case]['output_sha256']['seconds.npy']
with (output/'learning-location-target-access.jsonl').open('a') as f:
f.write(json.dumps(dict(case=case, role='learning', seconds_sha256=digest))+'\n')
seconds = cached(view, case, 'seconds')
d = candidates(full, q['raw_ids'], q['counts'], q['coordinates_over_Rg'], seconds[idx])
p = output/(case+'.npz')
np.savez(p, **d, original_raw_ids=q['raw_ids'], original_counts=q['counts'],
original_anchor_ids=q['ids'], original_near_wall=q['near_wall'],
original_coordinates_over_Rg=q['coordinates_over_Rg'])
manifest[case] = dict(path=str(p), sha256=sha(p), seconds_sha256=digest,
queries_sha256=row['preparation']['output_sha256']['queries.npz'],
unique_candidates=len(d['indices']), true_peak_raw_id=d['true_peak_raw_id'],
true_peak_ties=d['true_peak_ties'], undefined=False)
save(output/'manifest.json', manifest)
return manifest
def query_data(view, case, manifest):
require_learning(view, case)
r = manifest[case]
with np.load(checked(r['path'], r['sha256']), allow_pickle=False) as z:
return {k:z[k].copy() for k in ('indices', 'distances')}
+136
View File
@@ -0,0 +1,136 @@
"""One item c attempt, scoped process memory plus failed-worker peak checks."""
import ctypes
import json
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import threading
import time
import psutil
from phase2_h1_worker_v2 import save,sha
from phase3_item_c_common import config,assignment,read
from phase3_shared_capacity import validate_lease,alive
from track_a_recovery_monitor import Monitor
from track_a_recovery_supervisor import supervise,tree_usage,stop_group,disk_usage
from s2_main_memory_guard import PeakRecords
def run(queue,index,digest):
parent=os.getppid()
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGTERM,0,0,0)!=0:
raise OSError('PDEATHSIG')
requested=[]
for sig in (signal.SIGTERM,signal.SIGINT):
signal.signal(sig,lambda n,f:requested.append('SIGNAL_'+str(n)))
if os.getppid()!=parent:
raise RuntimeError('Queue lost before setup')
c,_=config(queue);job=assignment(queue,c,index,digest)
root=queue/'jobs'/str(index);start=time.monotonic();cpu=time.process_time()
original=read(root/'assignment.json');wait=original['pre_dispatch_wait_seconds']
remaining=c['runaway_gpu_seconds']-wait-60
if remaining<90:
raise RuntimeError('No cleanup window before worker')
deadline=start+remaining
# Queue writes ready only after atomically transferring reservation ownership.
while not (root/'lease-ready.json').exists():
if requested or not alive(job['queue_identity']) or time.monotonic()-start>30:
raise RuntimeError('Lease transfer timeout')
time.sleep(.1)
validate_lease(job['token'],queue,index,sha(queue/'contract.json'))
if os.getppid()!=job['queue_identity']['pid'] or not alive(job['queue_identity']):
raise RuntimeError('Pinned queue lost before guard import completed')
os.sched_setaffinity(0,[26,27])
tracker=PeakRecords();done=threading.Event();monitor=Monitor(root/'monitor-errors.jsonl')
reasons=[];receipt=None
command=[sys.executable,'tools/phase3_item_c_worker.py',str(queue),str(index),digest]
def reason(s):
if s not in reasons:reasons.append(s)
def workers():
result=[]
for p in psutil.Process().children(recursive=True):
try:
if p.cmdline()==command:result.append(p.pid)
except psutil.NoSuchProcess:pass
return result
def kill():
for pid in workers():stop_group(pid)
def scan():
tracker.scan(root/'worker')
if max(tracker.maxima.values())>job['recorded_vram_bytes']:reason('RECORDED_VRAM_LIMIT')
def local_watch():
while not done.is_set():
monitor.sample('recorded_peaks',scan)
usage,ok=monitor.sample('tree',lambda:tree_usage(os.getpid()))
if ok and (usage[0]>job['cpu_seconds']-30 or usage[1]>job['rss_bytes']):reason('GUARD_TREE_LIMIT')
disk,ok=monitor.sample('disk',lambda:disk_usage(root))
if ok and disk>job['disk_bytes']:reason('OUTPUT_LIMIT')
if time.monotonic()>deadline-30:reason('OWN_ACTUAL_TIME_LIMIT')
queue_live,ok=monitor.sample('queue_identity',lambda:alive(job['queue_identity']))
if ok and not queue_live:reason('QUEUE_LOST')
if (queue/'global-stop.json').exists():reason('GLOBAL_STOP')
for s in requested:reason(s)
if monitor.failed:reason('MONITOR_FAILED')
if reasons:kill()
done.wait(.5)
def gpu_watch():
def sample():
text=subprocess.check_output(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,used_gpu_memory','--format=csv,noheader,nounits'],text=True,timeout=10)
rows=[]
for line in text.strip().splitlines():
if not line:continue
u,p,m=[x.strip() for x in line.split(',')]
if u==job['gpu_uuid']:rows.append(dict(pid=int(p),bytes=int(m)*2**20))
if len(rows)>3:reason('HOST_GPU_SLOT_LIMIT')
own=set(workers())
if sum(r['bytes'] for r in rows if r['pid'] in own)>job['process_vram_bytes']:reason('OWN_GPU_PROCESS_MEMORY_LIMIT')
save(root/'gpu-sample.json',dict(unix=time.time(),rows=rows,own_pids=sorted(own)))
while not done.is_set():
monitor.sample('gpu_processes',sample)
if monitor.failed:reason('MONITOR_FAILED')
if reasons:kill()
done.wait(1)
threads=[threading.Thread(target=f,daemon=True) for f in (local_watch,gpu_watch)]
try:
save(root/'started.json',dict(unix=time.time(),assignment_sha256=digest))
for t in threads:t.start()
if reasons or requested or time.monotonic()>deadline-60:raise RuntimeError('Prelaunch guard rejected')
receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=job['cpu_seconds']-60,
job_cpu_seconds=job['cpu_seconds']-60,job_wall_seconds=deadline-time.monotonic()-40,
tree_rss_bytes=job['rss_bytes'],disk_bytes=job['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=job['gpu_uuid'])
except BaseException as exc:
reason(type(exc).__name__+': '+str(exc));kill()
finally:
done.set()
for t in threads:
if t.ident is not None:t.join(timeout=12)
if any(t.is_alive() for t in threads):reason('MONITOR_DID_NOT_EXIT')
try:monitor.retry('final_peaks',scan,.1)
except Exception as exc:reason('FINAL_PEAK_SCAN: '+str(exc))
for s in requested:reason(s)
if monitor.failed:reason('MONITOR_FAILED')
p=root/'worker/result.json';result=read(p) if p.exists() else None
if not receipt or receipt['status']!='PASS' or receipt['returncode']!=0:reason('WORKER_NOT_PASS')
if not result or result.get('status')!='COMPLETE' or result.get('fold')!=index or result.get('completed_epochs')!=200 or result.get('optimizer_steps')!=job['target_steps']:reason('NO_COMPLETE_IDENTITY')
if result:
from phase2_h1_worker_v2 import checked
checked(result['checkpoint'],result['checkpoint_sha256'])
if len(tracker.witnesses)!=2:reason('MISSING_RECORDED_PEAKS')
elapsed=time.monotonic()-start
child=resource.getrusage(resource.RUSAGE_CHILDREN)
charged_cpu=time.process_time()-cpu+child.ru_utime+child.ru_stime
if wait+60+elapsed>c['runaway_gpu_seconds'] or charged_cpu>job['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
try:config(queue)
except Exception as exc:reason('FINAL_PINS: '+str(exc))
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',reasons=reasons,
worker=receipt,worker_result_sha256=sha(p) if p.exists() else None,
cpu_seconds_charged=charged_cpu,pre_dispatch_wait_seconds=wait,guard_wall_seconds=elapsed,
recorded_memory=dict(maxima=tracker.maxima,witnesses=tracker.witnesses,observations_count=len(tracker.observations)),
monitor_policy=monitor.policy(),monitor_errors=monitor.records,no_retry=True))
return 0 if not reasons else 1
if __name__=='__main__':
raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3]))
+59
View File
@@ -0,0 +1,59 @@
"""Fixed Mold 9974ae27 objective; original regression plus unique-query E[d]."""
import numpy as np
import torch
from phase2_h2_model import weighted_loss
THETA = .03
COEFFICIENT = .001
def candidates(full_ids, query_ids, counts, coordinates, truth_seconds):
"""Exact original hard-peak reference; multiplicities do not vote in softmax.
Keep original query records separately. Repeated IDs must have identical
coordinates/truth; malformed data is an error, never an undefined waiver.
"""
full = np.asarray(full_ids)
ids = np.asarray(query_ids)
counts = np.asarray(counts)
xyz = np.asarray(coordinates, dtype=np.float64)
truth = np.asarray(truth_seconds, dtype=np.float64)
if (full.ndim != 1 or ids.ndim != 1 or not len(ids) or
full.dtype.kind not in 'iu' or ids.dtype.kind not in 'iu' or
np.any(np.diff(full) <= 0) or counts.shape != ids.shape or
counts.dtype.kind not in 'iu' or np.any(counts <= 0) or
xyz.shape != (len(ids), 3) or truth.shape != ids.shape or
not np.isfinite(xyz).all() or not np.isfinite(truth).all()):
raise ValueError('Invalid frozen query identity or truth')
unique, first, inverse = np.unique(ids, return_index=True, return_inverse=True)
if not np.array_equal(xyz, xyz[first][inverse]) or not np.array_equal(truth, truth[first][inverse]):
raise ValueError('Repeated raw ID has inconsistent coordinate/truth')
index = np.searchsorted(full, unique)
if np.any(index >= len(full)) or not np.array_equal(full[index], unique):
raise ValueError('Query ID absent from full-node geometry')
# np.unique orders raw IDs; the first true maximum implements the old tie.
t = truth[first]
peak = int(np.flatnonzero(t == t.max())[0])
distances = np.linalg.norm(xyz[first] - xyz[first][peak], axis=1)
return dict(indices=index.astype(np.int64), distances=distances,
unique_raw_ids=unique, true_peak_raw_id=int(unique[peak]),
true_peak_ties=int(np.sum(t == t.max())))
def location_loss(residual, prior64, indices, distances):
if prior64.dtype != torch.float64 or distances.dtype != torch.float64:
raise ValueError('Location prior and distances must stay FP64')
if residual.ndim != 1 or prior64.shape != residual.shape or indices.ndim != 1 or not indices.numel():
raise ValueError('Invalid location shapes')
if indices.dtype != torch.int64 or distances.shape != indices.shape or torch.unique(indices).numel() != indices.numel():
raise ValueError('Exactly one equal-weight vote per candidate')
s = prior64[indices] + residual[indices].to(torch.float64)
if not bool(torch.isfinite(s).all()) or not bool(torch.isfinite(distances).all()) or bool((distances < 0).any()):
raise ValueError('Nonfinite location data')
return torch.sum(torch.softmax((s - s.max()) / THETA, dim=0) * distances)
def objective(prediction, target, parts, nodes, total, prior64, indices, distances):
regression = weighted_loss(prediction, target, parts, nodes, total)
location = location_loss(prediction, prior64, indices, distances)
return regression + COEFFICIENT * location, regression, location
+144
View File
@@ -0,0 +1,144 @@
"""Review-gated five-fold c training. Stops at frozen models, never postfit."""
import fcntl
import json
import os
from pathlib import Path
import signal
import subprocess
import sys
import time
import psutil
from phase2_h1_worker_v2 import save,sha
from phase2_home_freeze_io import freeze
from phase2_gate_outcome_review import validate_review
from phase3_item_c_common import ROOT,config,read,project
from phase3_shared_capacity import register_entry,acquire,transfer,release,process_identity,alive
from track_a_recovery_monitor import Monitor
def run(root):
c,_=config(root);validate_review(root)
with (root/'queue.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('Sole attempt, no resume')
if os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':
raise ValueError('Explicit allocator required')
if psutil.disk_usage(root).free<100*2**30:raise RuntimeError('Less than100GiB free')
register_entry(root,sha(root/'contract.json'))
start=time.monotonic();identity=process_identity(os.getpid())
save(root/'started.json',dict(unix=time.time(),identity=identity,contract_sha256=sha(root/'contract.json')))
(root/'jobs').mkdir(exist_ok=False)
active={};terminal={};pending=list(range(5));histories={};stops=[];failure=None
monitor=Monitor(root/'admission-errors.jsonl')
for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,lambda n,f:stops.append('SIGNAL_'+str(n)))
def charge(i):
return terminal[i]['gpu_seconds_charged'] if i in terminal else 60+time.monotonic()-start
def stop(reason):
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time()))
try:
while active or pending:
if stops:raise RuntimeError(stops[0])
if (root/'global-stop.json').exists():raise RuntimeError('EXTERNAL_GLOBAL_STOP')
# Collect exits before granting any further admission.
for i,(p,lease,log,owner) in list(active.items()):
if p.poll() is None:continue
log.close();jr=root/'jobs'/str(i);cp=jr/'complete.json'
result=read(cp) if cp.exists() else dict(status='MISSING_GUARD_RECEIPT')
worker_identity=jr/'worker/worker-identity.json'
if worker_identity.exists() and alive(read(worker_identity)):
raise RuntimeError('WORKER_OUTLIVED_GUARD_LEASE_HELD')
terminal[i]=dict(id=i,status=result['status'],returncode=p.returncode,
gpu_seconds_charged=charge(i),guard_sha256=sha(cp) if cp.exists() else None)
release(lease['token'],owner);del active[i]
if result['status']!='PASS' or p.returncode!=0:raise RuntimeError('FOLD_FAILED_'+str(i))
for i in active:
folder=root/'jobs'/str(i)/'worker/history'
histories[i]=[read(p) for p in sorted(folder.glob('*.json'))]
charges={i:charge(i) for i in range(5)}
projection=project(c,charges,histories,terminal)
save(root/'latest-cost-projection.json',projection)
if projection['status']!='PASS':raise RuntimeError('TOTAL_PROJECTION_OR_ACTUAL_CAP')
for i in active:
gate=root/'jobs'/str(i)/'first-epoch-projection.json'
if histories.get(i) and not gate.exists():
save(gate,dict(status='PASS',contract_sha256=sha(root/'contract.json'),projection=projection))
save(root/'live-progress.json',dict(unix=time.time(),stage='ITEM_C_TRAINING',active=list(active),pending=pending,
terminal=list(terminal.values()),charged_training_gpu_seconds=projection['actual_training_and_preparation_gpu_seconds'],postfit_started=False))
if pending:
i=pending[0]
decision,ok=monitor.sample('admission',lambda:acquire(root,i,sha(root/'contract.json')))
if monitor.failed:raise RuntimeError('THREE_ADMISSION_FAILURES')
if not ok:
time.sleep(1)
continue
lease,observation=decision
save(root/'latest-admission.json',observation)
# Acquisition includes bounded GPU queries; recheck cost before Popen.
fresh=project(c,{j:charge(j) for j in range(5)},histories,terminal)
if fresh['status']!='PASS':
if lease:release(lease['token'])
raise RuntimeError('CAP_DURING_ADMISSION')
if lease:
jr=root/'jobs'/str(i);jr.mkdir(exist_ok=False)
save(jr/'assignment.json',dict(id=i,contract_sha256=sha(root/'contract.json'),
gpu_uuid=lease['gpu_uuid'],worker_cpus=lease['worker_cpus'],token=lease['token'],
queue_identity=identity,pre_dispatch_wait_seconds=time.monotonic()-start))
log=(jr/'guard.log').open('x')
env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1')
p=subprocess.Popen([sys.executable,'tools/phase3_item_c_guard.py',str(root),str(i),sha(jr/'assignment.json')],
cwd=str(Path(__file__).resolve().parents[1]),env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
owner=process_identity(p.pid)
active[i]=(p,lease,log,owner);pending.remove(i)
transfer(lease['token'],owner)
save(jr/'lease-ready.json',dict(token=lease['token'],owner=owner))
time.sleep(1)
except BaseException as exc:
failure=type(exc).__name__+': '+str(exc);stop(failure)
finally:
if active:
stop(failure or 'QUEUE_CLEANUP')
for p,lease,log,owner in active.values():
if p.poll() is None:p.terminate()
deadline=time.monotonic()+45
while time.monotonic()<deadline and any(p.poll() is None for p,_,_,_ in active.values()):time.sleep(.2)
for i,(p,lease,log,owner) in active.items():
if p.poll() is None:p.kill()
p.wait();log.close();jr=root/'jobs'/str(i);cp=jr/'complete.json'
result=read(cp) if cp.exists() else dict(status='MISSING_GUARD_RECEIPT')
terminal[i]=dict(id=i,status=result['status'],returncode=p.returncode,gpu_seconds_charged=charge(i),
guard_sha256=sha(cp) if cp.exists() else None)
wp=jr/'worker/worker-identity.json'
try:
if wp.exists() and alive(read(wp)):raise RuntimeError('Live worker; lease retained')
release(lease['token'],owner)
except Exception as exc:
terminal[i]['lease_release_failure']=str(exc);failure=failure or str(exc)
final_charges={i:charge(i) for i in range(5)}
final_cost=c['preparation_charge_gpu_seconds']+sum(final_charges.values())
if final_cost+c['postfit_gpu_reserve']>c['gpu_cap']:failure=failure or 'FINAL_CUMULATIVE_CAP'
complete=not failure and len(terminal)==5 and all(r['status']=='PASS' and r['returncode']==0 for r in terminal.values())
files=[root/'contract.json']
if complete:
models=[]
for i in range(5):
r=read(root/'jobs'/str(i)/'worker/result.json')
models.append(dict(r));files.append(Path(r['checkpoint']))
files.extend([root/'jobs'/str(i)/'complete.json',root/'jobs'/str(i)/'worker/result.json',root/'jobs'/str(i)/'worker/standardization.json'])
save(root/'models.json',models);files.append(root/'models.json')
save(root/'complete.json',dict(status='TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING' if complete else 'STOPPED_INCOMPLETE',
failure=failure,entries=list(terminal.values()),never_attempted=pending,
never_attempted_wait_charges={str(i):final_charges[i] for i in pending},
cumulative_gpu_seconds=final_cost,postfit_gpu_reserve=c['postfit_gpu_reserve'],
no_retry=True,test_targets_read=False,prediction_started=False,scoring_started=False))
files.append(root/'complete.json')
freeze_start=time.monotonic()
freeze(root,'training-terminal' if complete else 'training-stop',files)
final_cost+=time.monotonic()-freeze_start
within=final_cost+c['postfit_gpu_reserve']<=c['gpu_cap']
save(root/'post-terminal-cost.json',dict(cumulative_gpu_seconds=final_cost,within_cap=within,
freeze_tail_charged_once=True,postfit_gpu_reserve=c['postfit_gpu_reserve'],postfit_started=False,
training_complete_sha256=sha(root/'complete.json'),required_before_any_postfit=True))
return 0 if complete and within else 1
if __name__=='__main__':raise SystemExit(run(ROOT))
+82
View File
@@ -0,0 +1,82 @@
"""One new S2 fold, fixed item c loss with seed0/200epochs; no test prediction or target reads."""
import json,os,sys,time,traceback,random
from pathlib import Path
import numpy as np
import torch
from phase2_h2_s1_common import selected_checkpoint
from phase3_item_c_common import config,fold_view,first_epoch_release,assignment as assigned_job,read
from phase3_item_c_data import prepare_queries,query_data
from phase3_item_c_loss import objective
from phase3_shared_capacity import process_identity
from phase2_h2_data import standardization
from phase2_h2_data import load_part,to_device,arguments,cached
from phase2_h2_model import H2Geo,optimizer,lr_epoch
from phase2_h2_pilot_worker import peaks
from phase2_residual_math import residual_target
from phase2_h1_worker_v2 import save,sha
def train_step(model,opt,data,parts,total,query,c):
model.train();opt.zero_grad(set_to_none=True)
prediction=model(*arguments(data))
prior=torch.as_tensor(c*data['F'],dtype=torch.float64,device=prediction.device)
indices=torch.as_tensor(query['indices'],dtype=torch.int64,device=prediction.device)
distances=torch.as_tensor(query['distances'],dtype=torch.float64,device=prediction.device)
loss,regression,location=objective(prediction,data['target'],parts,data['nodes'],total,prior,indices,distances)
if not bool(torch.isfinite(loss)):raise ValueError('Nonfinite loss')
loss.backward();norm=torch.nn.utils.clip_grad_norm_(model.parameters(),1.,error_if_nonfinite=True);opt.step()
if any(not bool(torch.isfinite(p).all()) for p in model.parameters()):raise ValueError('Nonfinite parameter')
torch.cuda.synchronize()
return dict(loss=float(loss.detach()),regression_loss=float(regression.detach()),location_loss=float(location.detach()),gradient_norm_before_clip=float(norm),**peaks())
def run(root,fold_id,assignment_sha):
c,pilot=config(root);old=fold_view(pilot,fold_id);job=assigned_job(root,c,fold_id,assignment_sha);out=root/'jobs'/str(fold_id)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir()
os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0
save(out/'worker-identity.json',process_identity(os.getpid()))
def record(item,epoch_file=False):
item=dict(fold=fold_id,stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks())
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
save(out/'progress.json',item)
if epoch_file:save(out/'history'/f'{epoch:03d}.json',item)
return item
try:
if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity')
stage='LEARNING_ONLY_NORMALIZATION';normal=out/'normalization';normal.mkdir();stats=standardization(old,normal);save(out/'standardization.json',stats);save(out/'fold-identity.json',dict(actual_fold=fold_id,learning=old['folds'][0]['learning'],validation=old['folds'][0]['validation'],test=old['folds'][0]['test'],clock_sha256=old['folds'][0]['clock_sha256']))
stage='LEARNING_QUERY_PREPARATION';queries=prepare_queries(old,read(c['query_roster_contract'])['rows'],out/'queries')
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();random.seed(0);np.random.seed(0);torch.manual_seed(0);torch.cuda.manual_seed_all(0)
model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes']
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=job['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters())))
for epoch in range(1,201):
stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0.
if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch')
order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts)
for index in order:
case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda')
query=query_data(old,case,queries);result=train_step(model,opt,data,parts,total,query,fold['c']);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss']
record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],regression_loss=result['regression_loss'],location_loss=result['location_loss'],gradient_norm_before_clip=result['gradient_norm_before_clip']))
del data
if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch')
metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch))
stage='EPOCH_COMPLETE';record(metrics,epoch_file=True)
if epoch==1:
stage='WAIT_THIS_FOLD_FIRST_EPOCH_COST_GATE';record(dict(epoch_parts=seen,epoch_nodes=nodes))
while not first_epoch_release(root,fold_id):time.sleep(.5)
if epoch<200:
for group in opt.param_groups:group['lr']=lr_epoch(epoch+1)
stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch)
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=fold_id,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name)
save(out/'result.json',dict(status='COMPLETE',fold=fold_id,completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,validation_targets_read=False,query_manifest_sha256=sha(out/'queries/manifest.json'),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks()))
except BaseException as exc:
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
if __name__=='__main__':
import ctypes,signal
from phase3_shared_capacity import alive
expected=read(Path(sys.argv[1])/'jobs'/sys.argv[2]/'lease-ready.json')['owner']
parent=os.getppid()
if parent!=expected['pid'] or not alive(expected):raise RuntimeError('Pinned guard lost before worker import completed')
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGKILL,0,0,0)!=0:raise OSError('PDEATHSIG')
if os.getppid()!=parent:raise RuntimeError('Guard exited before worker setup')
run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3])
+197
View File
@@ -0,0 +1,197 @@
"""New-entry admission only; never changes or kills another entry's worker.
All participating future entries must use this same registry/lock. Unknown
processes consume real slots and a conservative reservation; stale leases fail
closed. Old fixed queues do not join this registry and remain untouched.
"""
from contextlib import contextmanager
import fcntl
import json
import math
import os
from pathlib import Path
import subprocess
import time
import uuid
import psutil
from phase2_h1_worker_v2 import save
GIB = 2**30
GPUS = ['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c', 'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76', 'GPU-8b71b032-1b75-2891-935c-ce106a53034a']
SHARED = Path('/home/frank14f/MeltMind-runs/phase3-shared-capacity-2026-11-18')
OLD = [Path('/home/frank14f/MeltMind-runs')/s for s in
['phase3-seed1-v2-2026-11-18', 'track-a-recovery-2026-11-18']]
PHASE_RESERVES = dict(item1=72421.31599938683, seed1=167200., c=205200., item1b=88560., b=178200.)
def process_identity(pid):
p = psutil.Process(pid)
return dict(pid=pid, created=p.create_time())
def alive(identity):
try:
p = psutil.Process(identity['pid'])
return p.create_time() == identity['created'] and p.status() != psutil.STATUS_ZOMBIE
except psutil.NoSuchProcess:
return False
def descendants(identity):
if not alive(identity):
raise RuntimeError('Stale lease: explicit reconciliation required')
return {identity['pid']} | {p.pid for p in psutil.Process(identity['pid']).children(recursive=True)}
def old_pending():
for root in OLD:
p = root/'complete.json'
if p.exists():
json.loads(p.read_text()) # corrupt terminal fails closed
continue
d = json.loads((root/'live-progress.json').read_text())
if time.time()-d['unix'] > 120:
raise RuntimeError('Stale old queue snapshot')
if d['pending']:
return True
return False
def host_snapshot():
def query(arg):
return subprocess.check_output(['nvidia-smi', arg, '--format=csv,noheader,nounits'], text=True, timeout=10)
devices = {}
for line in query('--query-gpu=uuid,memory.total,memory.used').strip().splitlines():
u, total, used = [s.strip() for s in line.split(',')]
devices[u] = dict(total=int(total)*2**20, used=int(used)*2**20)
processes = []
for line in query('--query-compute-apps=gpu_uuid,pid,used_gpu_memory').strip().splitlines():
if not line.strip():
continue
u, pid, mem = [s.strip() for s in line.split(',')]
processes.append(dict(gpu_uuid=u, pid=int(pid), vram=int(mem)*2**20))
if set(devices) != set(GPUS) or len({(p['gpu_uuid'],p['pid']) for p in processes}) != len(processes):
raise ValueError('Unknown GPU topology or duplicate compute PID')
vm = psutil.virtual_memory()
return dict(devices=devices, processes=processes, ram_total=vm.total, ram_available=vm.available,
cpu_percent=psutil.cpu_percent(interval=.1), old_pending=old_pending(), unix=time.time())
def admission(snapshot, leases, gpu, *, vram_reserve=4*GIB, rss_reserve=4*GIB):
"""Pure admission rule. leases include observed descendant PIDs and RSS."""
if gpu not in GPUS:
raise ValueError('Unknown GPU')
if gpu == GPUS[0] and snapshot['old_pending']:
return False, 'GPU0_RESERVED_FOR_OLD_PENDING'
processes = snapshot['processes']
card = [p for p in processes if p['gpu_uuid'] == gpu]
same = [l for l in leases if l['gpu_uuid'] == gpu]
real = {p['pid'] for p in card}
occupied = len(real) + sum(not (real & set(l['observed_pids'])) for l in same)
if occupied >= (1 if gpu == GPUS[3] else 3):
return False, 'GPU_SLOT_LIMIT'
owned = set().union(*(set(l['observed_pids']) for l in same)) if same else set()
# Charge the unseen part of each reserved peak; do not double-count visible memory.
missing = sum(max(0, l['vram_reserve']-sum(p['vram'] for p in card if p['pid'] in l['observed_pids'])) for l in same)
missing += sum(max(0, 4*GIB-p['vram']) for p in card if p['pid'] not in owned)
d = snapshot['devices'][gpu]
if d['used'] + missing + vram_reserve > .8*d['total']:
return False, 'GPU_HEADROOM'
ram_missing = sum(max(0,l['rss_reserve']-l['observed_rss']) for l in leases)
# Extra8GiB preserves future original folds; visible old RSS already in available.
if snapshot['ram_available']-ram_missing-rss_reserve-8*GIB < .2*snapshot['ram_total']:
return False, 'HOST_RAM_HEADROOM'
if snapshot['cpu_percent'] > 80:
return False, 'HOST_CPU_HEADROOM'
return True, 'ADMITTED'
@contextmanager
def registry():
SHARED.mkdir(exist_ok=True)
with (SHARED/'capacity.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
p = SHARED/'registry.json'
d = json.loads(p.read_text()) if p.exists() else dict(version=1, entries={}, leases={})
if d['version'] != 1:
raise ValueError('Shared registry version')
yield d
save(p, d)
def register_entry(root, digest):
with registry() as d:
if 'c' in d['entries']:
raise FileExistsError('Sole item c entry already registered; no resume')
if sum(PHASE_RESERVES.values()) > 720000:
raise RuntimeError('Whole phase3 reserved cap')
d['entries']['c'] = dict(root=str(root), contract_sha256=digest, owner=process_identity(os.getpid()),
cap=205200, phase_reservations=PHASE_RESERVES, started_unix=time.time())
def acquire(root, fold, digest):
with registry() as d:
entry = d['entries']['c']
if entry['root'] != str(root) or entry['contract_sha256'] != digest or not alive(entry['owner']):
raise PermissionError('Entry owner/identity')
leases = []
for l in d['leases'].values():
# Normal guard exit can race this observation. Block admission until
# its owning coordinator collects it; never reclaim or kill peers.
if not alive(l['owner']):
return None, dict(reason='STALE_LEASE_AWAIT_OWNER_COLLECTION',token=l['token'])
try:
ids = descendants(l['owner'])
except (psutil.NoSuchProcess,RuntimeError):
return None, dict(reason='LEASE_EXIT_RACE_AWAIT_OWNER_COLLECTION',token=l['token'])
rss=0
for pid in ids:
try:rss+=psutil.Process(pid).memory_info().rss
except psutil.NoSuchProcess:pass
leases.append(dict(l, observed_pids=list(ids), observed_rss=rss))
s = host_snapshot() # fresh inside inter-entry lock, before making reservation
used_cpu = {x for l in leases for x in l['worker_cpus']}
cpus = [x for x in range(16,26) if x not in used_cpu and x in os.sched_getaffinity(0)]
if not cpus:
return None, dict(reason='CPU_RESERVATIONS', snapshot=s)
for gpu in GPUS[1:3]:
ok, reason = admission(s, leases, gpu)
if not ok:
continue
token = uuid.uuid4().hex
l = dict(token=token, root=str(root), fold=fold, contract_sha256=digest, gpu_uuid=gpu,
worker_cpus=[cpus[0]], owner=process_identity(os.getpid()),
vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time())
d['leases'][token] = l
return l, dict(reason='ADMITTED', snapshot=s)
return None, dict(reason='NO_CAPACITY', snapshot=s)
def transfer(token, owner):
with registry() as d:
l = d['leases'][token]
if l['owner']['pid'] != os.getpid() or not alive(owner):
raise PermissionError('Only queue transfers its own lease to live guard')
l['owner'] = owner
def release(token, guard_identity=None):
with registry() as d:
l = d['leases'][token]
if guard_identity is None:
if l['owner']['pid'] != os.getpid():
raise PermissionError('Only unused own lease')
elif l['owner'] != guard_identity or alive(guard_identity):
raise PermissionError('Guard must be reaped before release')
del d['leases'][token]
def validate_lease(token, root, fold, digest):
with registry() as d:
l = d['leases'][token]
if (l['root'], l['fold'], l['contract_sha256']) != (str(root), fold, digest):
raise PermissionError('Lease identity')
if l['owner'] != process_identity(os.getpid()):
raise PermissionError('Lease not transferred to this guard')
return dict(l)
+74
View File
@@ -0,0 +1,74 @@
"""Prepare metadata-only c contract. Does not freeze, train or adopt a review."""
from pathlib import Path
import time
from phase2_h1_worker_v2 import checked,sha,save
from phase3_item_c_common import ROOT,read
from track_a_h2_prepare import local_code_closure
from phase3_shared_capacity import PHASE_RESERVES
B=Path(__file__).resolve().parents[1]
H=Path('/home/frank14f/MeltMind-runs')
def main():
start=time.process_time()
if ROOT.exists():raise FileExistsError('Sole item c root exists')
old=H/'phase2-h2-s2-2026-11-17';post=H/'phase2-h2-s2-postfit-2026-11-17'
oc=read(old/'contract.json');pilot=read(oc['pilot_contract'])
source=dict(oc['source_sha256']);source.update(oc['code_sha256'])
specs={'mold-last-fill-objective-spec.md':'9974ae2780fcc9754b305448f3eec51f5902d9fa3770a2807eaf3ed9f497b537',
'mold-tail-volume-peak-stability-spec.md':'05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521'}
for name,h in specs.items():
p=B/'references/2026-11-18/phase3-item-c'/name;checked(p,h);source[str(p)]=h
for p in [old/'contract.json',old/'complete.json',post/'contract.json',post/'complete.json',post/'predictions-manifest.json',post/'score/result.json',
Path(oc['pilot_contract']),H/'phase3-item1-postfit-recovery-2026-11-18/post-terminal-cost.json']:
source[str(p)]=sha(p)
if read(post/'complete.json')['decision']!='CONFIRMED':raise ValueError('Original H2 stage')
itemcost=read(H/'phase3-item1-postfit-recovery-2026-11-18/post-terminal-cost.json')['cumulative_item_gpu_seconds']
if abs(itemcost-PHASE_RESERVES['item1'])>1e-6:raise ValueError('Original item1 ledger')
files=sorted(B.glob('tools/phase3_item_c_*.py'))+[B/'tools/phase3_shared_capacity.py',Path(__file__).resolve(),B/'tools/test_phase3_item_c.py']
codes=local_code_closure(files+sorted((B/'tools/s2_models').glob('*.py')))
# Include scientific package modules that the existing local AST resolver skips.
for p,h in oc['code_sha256'].items():codes.setdefault(p,h)
for p,h in source.items():checked(p,h)
comparison=dict(version='item-c-training-v1',main_spec_sha256=specs['mold-last-fill-objective-spec.md'],secondary_spec_sha256=specs['mold-tail-volume-peak-stability-spec.md'],
folds=pilot['folds'],roster_parts=552,components=204,seed=0,epochs=200,theta=.03,coefficient=.001,
regression='original H2 weighted_loss unchanged',location='unique original query IDs equal softmax base weight; original exact-seconds true max tied by minimum raw ID',
validation='no validation labels read; original diagnostic-only loop omitted, no optimizer or selection change',
selection='last_epoch_only',inputs=19,query_scoring='original hard maximum and exact tie rule',
old_corpus_gate=dict(mean_delta_D_at_most=-.010,upper_delta_D_below=0,upper_delta_relativeL2_below=.001,
component_equal_weight=True,bootstrap=2000,seed=2026111408,paired_draws=True,
seed1_same_endpoint_noise_comparison_required=True,
additional_gate='absolute candidate effect must exceed absolute seed1-minus-seed0 point difference at the same endpoint'),
secondary='frozen full retained-tet final1pct volume overlap endpoint05344bac; separate postfit entry required',
fresh_family_confirmation='DEFERRED_NOT_AUTHORIZED_NO_COHORT',ensemble=False,regional_features=False,
postfit='not implemented or launched in training entry; all5 models frozen and separate actual review before predictions/test reads',
baseline_predictions_sha256=sha(post/'predictions-manifest.json'))
ROOT.mkdir();save(ROOT/'comparison-contract.json',comparison)
jobs=[]
for f in pilot['folds']:
jobs.append({k:f[k] for k in ('learning','validation','test','c','clock_sha256','learning_nodes')}|
dict(id=f['fold'],target_steps=200*len(f['learning']),cpu_seconds=60000,
rss_bytes=4*2**30,recorded_vram_bytes=2*2**30,process_vram_bytes=4*2**30,disk_bytes=8*2**30))
c=dict(version='phase3-item-c-v1',approval='1795007580.895539',entry_approval='1795007058.477789',
seed=0,epochs=200,new_fits=5,theta=.03,coefficient=.001,no_retry=True,
gpu_cap=205200,postfit_gpu_reserve=7200,postfit_cpu_cap=3600,phase3_gpu_cap=720000,
preparation_charge_gpu_seconds=600,preparation_cpu_cap=600,runaway_gpu_seconds=60000,
phase_reservations=PHASE_RESERVES,track_a_ledger='separate',
waiting_charge='all five folds from queue start through own terminal or stop, including never-dispatched pending;60s setup per fold',
projection='charged +1.5 remaining epochs at largest observed full epoch,120s cleanup; pending=max(initial share,started median,actual+120); total stop only',
capacity='global lock, real GPU PIDs plus unseen reservations,3/3/3/1 slots,c onlyGPU1/2,20pct headroom,4GiB reservation per new task',
learning_query_scope='original query extractor on learning parts only; seconds target read logged first; immutable local per-fold manifest',
validation_targets_read=False,test_targets_read=False,postfit_started=False,
jobs=jobs,pilot_contract=oc['pilot_contract'],query_roster_contract=str(post/'contract.json'),
source_sha256=source,code_sha256=codes,comparison_contract=str(ROOT/'comparison-contract.json'),
comparison_contract_sha256=sha(ROOT/'comparison-contract.json'))
save(ROOT/'contract.json',c)
save(ROOT/'preparation-audit.json',dict(status='PREPARED_PENDING_ASSIST_REVIEW',contract_sha256=sha(ROOT/'contract.json'),
code_files=len(codes),source_files=len(source),metadata_cpu_seconds=time.process_time()-start,
arrays_decoded=False,model_weights_loaded=False,labels_decoded=False,new_fits_started=False,
phase_reserve_total=sum(PHASE_RESERVES.values()),preparation_gpu_charge_reserved=600))
print(read(ROOT/'preparation-audit.json'))
if __name__=='__main__':main()
+223
View File
@@ -0,0 +1,223 @@
"""CPU synthetic acceptance only: no original arrays, labels, CUDA or fits."""
import ast
import copy
import json
import multiprocessing
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
import numpy as np
import torch
from phase3_item_c_loss import candidates,location_loss,objective,THETA
from phase3_item_c_data import require_learning
from phase3_item_c_common import project
from phase3_shared_capacity import admission,GPUS,GIB,PHASE_RESERVES
from s2_custody_statistics import last_location
from s2_main_memory_guard import PeakRecords
def snapshot():
return dict(devices={u:dict(total=16*GIB,used=2*GIB) for u in GPUS},
processes=[dict(gpu_uuid=GPUS[1],pid=101,vram=2*GIB)],
ram_total=128*GIB,ram_available=96*GIB,cpu_percent=20,old_pending=False)
def lease(pid=102):
return dict(gpu_uuid=GPUS[1],observed_pids=[pid],observed_rss=GIB,vram_reserve=4*GIB,rss_reserve=4*GIB)
def concurrent_admit(shared, barrier, results):
import phase3_shared_capacity as m
m.SHARED=Path(shared)
s=snapshot()
# two actual tasks leave one slot onGPU1; GPU2 already has three.
s['processes'].append(dict(gpu_uuid=GPUS[1],pid=103,vram=GIB))
s['processes'] += [dict(gpu_uuid=GPUS[2],pid=p,vram=GIB) for p in (201,202,203)]
m.host_snapshot=lambda:s
try:
l,_=m.acquire(Path('/synthetic'),0,'abc')
results.put(bool(l))
barrier.wait(timeout=10) # keep lease owners alive through both admissions
except BaseException as exc:
results.put(type(exc).__name__+': '+str(exc))
class ObjectiveTests(unittest.TestCase):
def test_exact_old_peak_tie(self):
raw=np.array([7,19,42]);xyz=np.array([[0.,0,0],[2.,0,0],[0.,3,0]])
truth=np.array([5.,5.,1.]);q=candidates(raw,raw,np.ones(3,dtype=int),xyz,truth)
self.assertEqual(q['true_peak_raw_id'],7)
for i in range(3):
p=np.zeros(3);p[i]=1
self.assertEqual(q['distances'][i],last_location(truth,p,raw,xyz))
def test_duplicate_order_does_not_vote(self):
full=np.array([1,2,3]);ids=np.array([3,1,3,2]);xyz=np.array([[2.,0,0],[0,0,0],[2,0,0],[1,0,0]])
q=candidates(full,ids,np.array([50,1,99,2]),xyz,np.array([9.,10,9,8]))
np.testing.assert_array_equal(q['indices'],[0,1,2]);np.testing.assert_array_equal(q['distances'],[0,1,2])
def test_bad_duplicate_fails(self):
with self.assertRaises(ValueError):candidates(np.array([1]),np.array([1,1]),np.array([1,1]),np.zeros((2,3)),np.array([1.,2.]))
def test_missing_id_fails(self):
with self.assertRaises(ValueError):candidates(np.array([1]),np.array([2]),np.array([1]),np.zeros((1,3)),np.array([1.]))
def test_nan_truth_not_undefined(self):
with self.assertRaises(ValueError):candidates(np.array([1]),np.array([1]),np.array([1]),np.zeros((1,3)),np.array([np.nan]))
def test_gradient_formula(self):
r=torch.tensor([.01,-.02,.03],dtype=torch.float64,requires_grad=True);prior=torch.tensor([.7,.8,.9],dtype=torch.float64)
d=torch.tensor([2.,1.,0.],dtype=torch.float64);idx=torch.arange(3)
loss=location_loss(r,prior,idx,d);loss.backward();p=torch.softmax((r.detach()+prior)/THETA,0)
torch.testing.assert_close(r.grad,p*(d-(p*d).sum())/THETA)
def test_constant_shift(self):
r=torch.tensor([.1,.2],dtype=torch.float64);p=torch.tensor([.7,.8],dtype=torch.float64);d=torch.tensor([2.,0.],dtype=torch.float64)
torch.testing.assert_close(location_loss(r,p,torch.arange(2),d),location_loss(r+12,p,torch.arange(2),d))
def test_degenerate_gradients_zero(self):
for d in (torch.tensor([2.],dtype=torch.float64),torch.tensor([2.,2.,2.],dtype=torch.float64)):
r=torch.arange(len(d),dtype=torch.float64,requires_grad=True);location_loss(r,torch.zeros_like(r),torch.arange(len(d)),d).backward()
torch.testing.assert_close(r.grad,torch.zeros_like(r))
def test_fp64_prior_preserved(self):
r=torch.zeros(2,dtype=torch.float32,requires_grad=True);p=torch.tensor([1.,1.+1e-8],dtype=torch.float64);d=torch.tensor([0.,1.],dtype=torch.float64)
loss=location_loss(r,p,torch.arange(2),d);self.assertGreater(float(loss),.5);loss.backward();self.assertTrue(torch.isfinite(r.grad).all())
with self.assertRaises(ValueError):location_loss(r,p.float(),torch.arange(2),d)
def test_original_weight_plus_one_location(self):
p=torch.tensor([.2,.4]);t=torch.tensor([.1,.7]);prior=torch.tensor([.7,.6],dtype=torch.float64);d=torch.tensor([0.,3.],dtype=torch.float64)
total,reg,loc=objective(p,t,7,2,19,prior,torch.arange(2),d)
self.assertEqual(float(reg),float(((p-t)**2).mean()*(14/19)))
torch.testing.assert_close(total-reg,.001*loc)
def test_no_validation_or_test_role(self):
view=dict(folds=[dict(learning=['a'],validation=['b'],test=['c'])])
require_learning(view,'a')
for case in ('b','c','missing'):
with self.assertRaises(PermissionError):require_learning(view,case)
def test_worker_only_loads_learning(self):
tree=ast.parse((Path(__file__).parent/'phase3_item_c_worker.py').read_text())
calls=[n for n in ast.walk(tree) if isinstance(n,ast.Call) and isinstance(n.func,ast.Name) and n.func.id=='load_part']
self.assertEqual(len(calls),1);self.assertEqual(calls[0].args[3].value,'learning')
class CapacityTests(unittest.TestCase):
def test_visible_reserved_process_counted_once(self):
self.assertTrue(admission(snapshot(),[lease(101)],GPUS[1])[0])
def test_unseen_reservations_block_fourth(self):
self.assertFalse(admission(snapshot(),[lease(102),lease(103)],GPUS[1])[0])
def test_foreign_processes_count(self):
s=snapshot();s['processes'] += [dict(gpu_uuid=GPUS[1],pid=i,vram=GIB) for i in (102,103)]
self.assertEqual(admission(s,[],GPUS[1])[1],'GPU_SLOT_LIMIT')
def test_gpu3_single(self):
s=snapshot();s['processes']=[dict(gpu_uuid=GPUS[3],pid=1,vram=1)]
self.assertFalse(admission(s,[],GPUS[3])[0])
def test_gpu0_old_pending(self):
s=snapshot();s['old_pending']=True;self.assertEqual(admission(s,[],GPUS[0])[1],'GPU0_RESERVED_FOR_OLD_PENDING')
def test_memory_reserves_future_peaks(self):
s=snapshot();s['devices'][GPUS[1]]['used']=9*GIB
self.assertEqual(admission(s,[lease(101)],GPUS[1])[1],'GPU_HEADROOM')
def test_host_ram_reserve(self):
s=snapshot();s['ram_available']=30*GIB;self.assertEqual(admission(s,[],GPUS[1])[1],'HOST_RAM_HEADROOM')
def test_cpu_headroom(self):
s=snapshot();s['cpu_percent']=81;self.assertEqual(admission(s,[],GPUS[1])[1],'HOST_CPU_HEADROOM')
def test_phase_reservation_once(self):
self.assertEqual(PHASE_RESERVES['seed1'],167200);self.assertLess(sum(PHASE_RESERVES.values()),720000)
def test_stale_lease_blocks_without_reclaim(self):
import phase3_shared_capacity as m
with tempfile.TemporaryDirectory() as tmp,patch.object(m,'SHARED',Path(tmp)):
owner=m.process_identity(__import__('os').getpid())
stale=dict(token='stale',owner=dict(pid=99999999,created=0))
Path(tmp,'registry.json').write_text(json.dumps(dict(version=1,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=owner)},leases={'stale':stale})))
decision,reason=m.acquire(Path('/synthetic'),0,'abc')
self.assertIsNone(decision);self.assertIn('STALE_LEASE',reason['reason'])
self.assertIn('stale',json.loads(Path(tmp,'registry.json').read_text())['leases'])
def test_two_admissions_atomic(self):
import phase3_shared_capacity as m
with tempfile.TemporaryDirectory() as tmp:
parent=m.process_identity(__import__('os').getpid())
Path(tmp,'registry.json').write_text(json.dumps(dict(version=1,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=parent)},leases={})))
ctx=multiprocessing.get_context('fork');barrier=ctx.Barrier(2);results=ctx.Queue()
ps=[ctx.Process(target=concurrent_admit,args=(tmp,barrier,results)) for _ in range(2)]
for p in ps:p.start()
for p in ps:p.join(15)
try:
self.assertTrue(all(p.exitcode==0 for p in ps))
self.assertEqual(sorted([results.get(timeout=1),results.get(timeout=1)]),[False,True])
finally:
for p in ps:
if p.is_alive():p.kill();p.join()
class BudgetTests(unittest.TestCase):
def setUp(self):self.c=dict(gpu_cap=205200,postfit_gpu_reserve=7200,preparation_charge_gpu_seconds=600,runaway_gpu_seconds=60000)
def test_initial_inclusive_cap(self):
p=project(self.c,{i:60 for i in range(5)},{},{})
self.assertEqual(p['projected_gpu_seconds'],205200);self.assertEqual(p['status'],'PASS')
def test_pending_wait_not_free(self):
p=project(self.c,{i:50000 for i in range(5)},{},{})
self.assertEqual(p['status'],'STOP_BUDGET')
def test_projected_fold_above_runaway_not_alone_stop(self):
h={0:[dict(epoch=1,train_epoch_seconds=220)]}
p=project(self.c,{i:1000 for i in range(5)},h,{1:{},2:{},3:{}})
self.assertGreater(p['folds']['0']['projected'],60000);self.assertEqual(p['status'],'PASS')
def test_actual_runaway_stops(self):
self.assertEqual(project(self.c,{i:60000 if i==0 else 1 for i in range(5)},{},{})['status'],'STOP_BUDGET')
def test_negative_nan_cost(self):
for v in (-1,float('nan')):
with self.assertRaises(ValueError):project(self.c,{i:v for i in range(5)},{},{})
def test_incomplete_epoch_history(self):
with self.assertRaises(ValueError):project(self.c,{i:60 for i in range(5)},{0:[dict(epoch=2,train_epoch_seconds=10)]},{})
def test_failed_worker_late_memory_peak(self):
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp)/'failure.json';p.write_text(json.dumps(dict(peak_reserved_bytes=3*GIB,peak_allocated_bytes=GIB)))
tracker=PeakRecords();tracker.scan(tmp)
self.assertGreater(max(tracker.maxima.values()),2*GIB)
p.write_text(json.dumps(dict(peak_reserved_bytes=1)));tracker.scan(tmp)
self.assertEqual(tracker.maxima['peak_reserved_bytes'],3*GIB)
class QueueSyntheticTests(unittest.TestCase):
def exercise(self,failed=False,admission_errors=0):
import phase3_item_c_queue as q
from contextlib import ExitStack
from phase2_h1_worker_v2 import save
with tempfile.TemporaryDirectory() as tmp, ExitStack() as stack:
root=Path(tmp);save(root/'contract.json',{'synthetic':True})
c=dict(gpu_cap=205200,postfit_gpu_reserve=7200,preparation_charge_gpu_seconds=600,runaway_gpu_seconds=60000)
class FakeProcess:
def __init__(self,cmd,**kw):
self.pid=999991;self.returncode=1 if failed else 0
i=int(cmd[3]);jr=root/'jobs'/str(i);(jr/'worker').mkdir()
save(jr/'complete.json',dict(status='STOPPED_OR_FAILED' if failed else 'PASS'))
checkpoint=jr/'worker/final.pt';checkpoint.write_bytes(b'synthetic')
save(jr/'worker/result.json',dict(fold=i,checkpoint=str(checkpoint),checkpoint_sha256=q.sha(checkpoint)))
save(jr/'worker/standardization.json',dict(synthetic=True))
def poll(self):return self.returncode
def wait(self):return self.returncode
def terminate(self):pass
def kill(self):pass
attempts=[]
def acquire(root,i,digest):
attempts.append(i)
if len(attempts)<=admission_errors:raise TimeoutError('synthetic admission query')
return dict(token=str(i),gpu_uuid=GPUS[1],worker_cpus=[16]),dict(synthetic=True)
patches=dict(config=lambda r:(c,{}),validate_review=lambda r:None,register_entry=lambda *a:None,
acquire=acquire,transfer=lambda *a:None,release=lambda *a:None,
process_identity=lambda pid:dict(pid=pid,created=0),freeze=lambda *a:None)
for k,v in patches.items():stack.enter_context(patch.object(q,k,v))
stack.enter_context(patch.object(q.subprocess,'Popen',FakeProcess))
stack.enter_context(patch.object(q.time,'sleep',lambda t:None))
stack.enter_context(patch.object(q.signal,'signal',lambda *a:None))
stack.enter_context(patch.object(q.psutil,'disk_usage',return_value=type('D',(),{'free':200*GIB})()))
stack.enter_context(patch.dict(q.os.environ,PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True'))
result=q.run(root);done=q.read(root/'complete.json')
if admission_errors>=3:
self.assertEqual(result,1);self.assertEqual(done['never_attempted'],[0,1,2,3,4])
self.assertEqual(len(attempts),3)
elif failed:
self.assertEqual(result,1);self.assertEqual(done['never_attempted'],[1,2,3,4]);self.assertFalse((root/'models.json').exists())
self.assertEqual(set(done['never_attempted_wait_charges']),{'1','2','3','4'})
else:
self.assertEqual(result,0);self.assertEqual(len(q.read(root/'models.json')),5)
self.assertEqual(done['status'],'TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING')
self.assertFalse(done['test_targets_read']);self.assertFalse(done['prediction_started'])
def test_all_five_terminate_before_model_freeze(self):self.exercise()
def test_failure_stops_without_retry(self):self.exercise(failed=True)
def test_two_failed_admission_samples_then_success(self):self.exercise(admission_errors=2)
def test_third_failed_admission_stops(self):self.exercise(admission_errors=3)
if __name__=='__main__':unittest.main(verbosity=2)