Publish fixed item c training entry, shared capacity and synthetic acceptance for Assist review
This commit is contained in:
1 parent
5b6a953fc5
commit
706608dfe0
28 files changed
+13373
No files matched your search
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"returncode": 0,
|
||||
"wall_seconds": 3.8560638920171186,
|
||||
"cpu_seconds": 3.8524149999999997,
|
||||
"synthetic_only": true,
|
||||
"scientific_arrays_decoded": false,
|
||||
"gpu_workload": false,
|
||||
"prior_attempt_logs_preserved": [
|
||||
"attempt1.log",
|
||||
"attempt2.log"
|
||||
],
|
||||
"prior_process_cpu_not_measured": true,
|
||||
"all_preparation_charge_reserved": 600
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
test_actual_runaway_stops (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_failed_worker_late_memory_peak (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_incomplete_epoch_history (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_initial_inclusive_cap (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_negative_nan_cost (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_pending_wait_not_free (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_projected_fold_above_runaway_not_alone_stop (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_cpu_headroom (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_foreign_processes_count (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_gpu0_old_pending (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_gpu3_single (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_host_ram_reserve (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_memory_reserves_future_peaks (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_phase_reservation_once (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_two_admissions_atomic (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_unseen_reservations_block_fourth (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_visible_reserved_process_counted_once (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_bad_duplicate_fails (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_constant_shift (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_degenerate_gradients_zero (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_duplicate_order_does_not_vote (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_exact_old_peak_tie (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_fp64_prior_preserved (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_gradient_formula (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_missing_id_fails (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_nan_truth_not_undefined (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_no_validation_or_test_role (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_original_weight_plus_one_location (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_worker_only_loads_learning (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 29 tests in 0.190s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,36 @@
|
||||
test_actual_runaway_stops (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_failed_worker_late_memory_peak (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_incomplete_epoch_history (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_initial_inclusive_cap (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_negative_nan_cost (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_pending_wait_not_free (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_projected_fold_above_runaway_not_alone_stop (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_cpu_headroom (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_foreign_processes_count (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_gpu0_old_pending (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_gpu3_single (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_host_ram_reserve (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_memory_reserves_future_peaks (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_phase_reservation_once (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_two_admissions_atomic (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_unseen_reservations_block_fourth (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_visible_reserved_process_counted_once (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_bad_duplicate_fails (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_constant_shift (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_degenerate_gradients_zero (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_duplicate_order_does_not_vote (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_exact_old_peak_tie (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_fp64_prior_preserved (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_gradient_formula (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_missing_id_fails (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_nan_truth_not_undefined (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_no_validation_or_test_role (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_original_weight_plus_one_location (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_worker_only_loads_learning (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_all_five_terminate_before_model_freeze (test_phase3_item_c.QueueSyntheticTests) ... ok
|
||||
test_failure_stops_without_retry (test_phase3_item_c.QueueSyntheticTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 31 tests in 0.213s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,39 @@
|
||||
test_actual_runaway_stops (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_failed_worker_late_memory_peak (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_incomplete_epoch_history (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_initial_inclusive_cap (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_negative_nan_cost (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_pending_wait_not_free (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_projected_fold_above_runaway_not_alone_stop (test_phase3_item_c.BudgetTests) ... ok
|
||||
test_cpu_headroom (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_foreign_processes_count (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_gpu0_old_pending (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_gpu3_single (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_host_ram_reserve (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_memory_reserves_future_peaks (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_phase_reservation_once (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_stale_lease_blocks_without_reclaim (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_two_admissions_atomic (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_unseen_reservations_block_fourth (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_visible_reserved_process_counted_once (test_phase3_item_c.CapacityTests) ... ok
|
||||
test_bad_duplicate_fails (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_constant_shift (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_degenerate_gradients_zero (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_duplicate_order_does_not_vote (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_exact_old_peak_tie (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_fp64_prior_preserved (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_gradient_formula (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_missing_id_fails (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_nan_truth_not_undefined (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_no_validation_or_test_role (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_original_weight_plus_one_location (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_worker_only_loads_learning (test_phase3_item_c.ObjectiveTests) ... ok
|
||||
test_all_five_terminate_before_model_freeze (test_phase3_item_c.QueueSyntheticTests) ... ok
|
||||
test_failure_stops_without_retry (test_phase3_item_c.QueueSyntheticTests) ... ok
|
||||
test_third_failed_admission_stops (test_phase3_item_c.QueueSyntheticTests) ... ok
|
||||
test_two_failed_admission_samples_then_success (test_phase3_item_c.QueueSyntheticTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 34 tests in 0.260s
|
||||
|
||||
OK
|
||||
+2885
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"status": "PREPARED_PENDING_ASSIST_REVIEW",
|
||||
"contract_sha256": "837b869e9d42d7402c8cdb4f3f9236462e9ec5bf2a80bcb561e5a4c566361fb7",
|
||||
"code_files": 46,
|
||||
"source_files": 86,
|
||||
"metadata_cpu_seconds": 0.44506590199999996,
|
||||
"arrays_decoded": false,
|
||||
"model_weights_loaded": false,
|
||||
"labels_decoded": false,
|
||||
"new_fits_started": false,
|
||||
"phase_reserve_total": 711581.3159993868,
|
||||
"preparation_gpu_charge_reserved": 600
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"status": "READY_FOR_ASSIST_REVIEW",
|
||||
"implementation_commit": "a2540974ae646f341beca248ce8a5c4f638fe2d8",
|
||||
"contract_sha256": "76fa0975b61d86363d5b9bf110a58ae56ca0a1462a3154744e1df54285d01233",
|
||||
"binding_sha256": "20bf6ca532f42fdba10e480a0405a4f7333a946c951e45a1eac83c582024625b",
|
||||
"code_files": 52,
|
||||
"source_files": 87,
|
||||
"old_seed1_code_pins": "PASS",
|
||||
"old_track_a_recovery_code_pins": "PASS",
|
||||
"snapshot_unix": 1795009065,
|
||||
"item_c_started": false,
|
||||
"actual_peer_review": false,
|
||||
"shared_capacity_registry_created": false
|
||||
}
|
||||
@@ -737,3 +737,12 @@ source8a8ebd1f/contract4b21dedd/proof8baec54f/comparison828530e4/27pins不变,
|
||||
|
||||
Melt只做25.5685s六次只读采样并写过两份未完成草稿;草稿已移出tools、逐字归档withdrawn-throughput-draft,从未运行benchmark/测试/冻结/读标签,无probe root/process。原seed1 PID1476593和三worker、Track A recovery PID1650006和fold4worker1650207均核在,78code pins全匹配;两队列无STOP/complete。资源及撤回证据artifacts/gpu-concurrency-assessment-2026-11-18;说明docs/independent-research-concurrency-2026-11-18.md。新共享capacity调度仅待新入口实现/审阅,没有已执行收益或新研究结果。
|
||||
|
||||
|
||||
|
||||
## Item c 入口固定待 Assist 审阅(2026-11-18)
|
||||
|
||||
Claude1795007580/7058原条已实读。固定implementation a2540974ae646f341beca248ce8a5c4f638fe2d8,contract76fa0975b61d86363d5b9bf110a58ae56ca0a1462a3154744e1df54285d01233,proof54af93dca18b7a9eb416a1230a265be74287af690e493f2d96521ae09f256b04,comparisona9dac0a4aff9168f6c98beef46333e5a68c1fc455d818a1afc7d21b12a11be72,完整52code map,binding20bf6ca532f42fdba10e480a0405a4f7333a946c951e45a1eac83c582024625b。34 CPU合成PASS,含双进程原子抢槽及退出时序/失败不重试模拟;最新验收进程CPU3.852415s,早两轮未单独测量进程CPU,原日志保留、准备保守预收600s。
|
||||
|
||||
主规格9974ae27/次终点05344bac、19列/seed0/200epoch不变;只加固定lambda=.001/theta=.03位置损失,学习query固定原ID/顺序/multiplicity/尺度;位置项唯一ID等底权、精确seconds真峰最小ID并列。无validation/test标签读取,省略旧仅诊断validation loop。新共享准入真实整机进程+lease,GPU0–2最多3/GPU3最多1、c只GPU1/2、GPU0旧pending保留。60000秒为实际失控限,1.5只用于剩余训练总投影。c205200/post7200GPU+3600CPU,phase3预留711581.315999/720000,seed1旧失败已含其167200帽只计一次,Track A独立。
|
||||
|
||||
唯一root /home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18;config及绑定核验PASS不是同行PASS。尚无review.json/started/共享registry目录,没有真实训练/预测/评分。旧seed1与Track A共78代码pins实核保持;仅读取最新回执,两原队列当时仍在训练/等待。下一步为实际Assist入口审阅,不重开已撤回benchmark、不重复启动旧队列。全五终态/模型冻结/退出后仍需独立postfit入口审阅及post-terminal-cost最终账检查。详见 docs/phase3-item-c-entry-2026-11-18.md。
|
||||
@@ -1,3 +1,13 @@
|
||||
## Item c 入口固定待 Assist 审阅(2026-11-18)
|
||||
|
||||
Claude1795007580/7058原条已实读。固定implementation a2540974ae646f341beca248ce8a5c4f638fe2d8,contract76fa0975b61d86363d5b9bf110a58ae56ca0a1462a3154744e1df54285d01233,proof54af93dca18b7a9eb416a1230a265be74287af690e493f2d96521ae09f256b04,comparisona9dac0a4aff9168f6c98beef46333e5a68c1fc455d818a1afc7d21b12a11be72,完整52code map,binding20bf6ca532f42fdba10e480a0405a4f7333a946c951e45a1eac83c582024625b。34 CPU合成PASS,含双进程原子抢槽及退出时序/失败不重试模拟;最新验收进程CPU3.852415s,早两轮未单独测量进程CPU,原日志保留、准备保守预收600s。
|
||||
|
||||
主规格9974ae27/次终点05344bac、19列/seed0/200epoch不变;只加固定lambda=.001/theta=.03位置损失,学习query固定原ID/顺序/multiplicity/尺度;位置项唯一ID等底权、精确seconds真峰最小ID并列。无validation/test标签读取,省略旧仅诊断validation loop。新共享准入真实整机进程+lease,GPU0–2最多3/GPU3最多1、c只GPU1/2、GPU0旧pending保留。60000秒为实际失控限,1.5只用于剩余训练总投影。c205200/post7200GPU+3600CPU,phase3预留711581.315999/720000,seed1旧失败已含其167200帽只计一次,Track A独立。
|
||||
|
||||
唯一root /home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18;config及绑定核验PASS不是同行PASS。尚无review.json/started/共享registry目录,没有真实训练/预测/评分。旧seed1与Track A共78代码pins实核保持;仅读取最新回执,两原队列当时仍在训练/等待。下一步为实际Assist入口审阅,不重开已撤回benchmark、不重复启动旧队列。全五终态/模型冻结/退出后仍需独立postfit入口审阅及post-terminal-cost最终账检查。详见 docs/phase3-item-c-entry-2026-11-18.md。
|
||||
|
||||
---
|
||||
|
||||
## 最新用户纠正:研究事项并行,吞吐benchmark撤回且未启动(2026-11-18)
|
||||
|
||||
用户通过TRON明确要不同研究工作的并行,不是同一训练1/2/3副本吞吐或算子优化;以后意见先由Claude领导裁量。Claude1795006714/6766已实读:撤回6592的benchmark,入口按c→1b→b→Track B构建,各自审后可并行;seed1只约束效果/噪声判决,不一律阻塞开工。每项训练入口一次审阅,test前独立审阅保持。GPU0–2最多3任务、GPU3严格1,当前frozen训练和postfit不改。现有pending时保留一张0–2不给新入口,不能停现有worker制造空卡或重派旧fold。
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
# Item c:固定末填位置损失训练入口,等待 Assist 审阅
|
||||
|
||||
本轮完成入口和合成验收,不运行真实训练、预测或评分。唯一计划运行目录为 `/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18`;其 `expected-review-binding.json` 是实际审阅身份,发布提交不能替代 implementation commit。
|
||||
|
||||
## 裁量与研究目标
|
||||
|
||||
Claude [7058](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1795007058477789) 要求先建设 c,再交 Assist。已实读 [7580](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1795007580895539):c 总帽205200 GPU秒,含准备、等待、失败、后处理,postfit预留7200 GPU秒/3600 CPU秒;phase3总帽720000,Track A单独记账,ensemble另议。
|
||||
|
||||
沿用 Mold 主规格9974ae2780fcc9754b305448f3eec51f5902d9fa3770a2807eaf3ed9f497b537、次终点05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521。目标是改善原有限 query 上的硬末点定位,同时守住全场 relative L2 的预定退化门;这不是全模腔末填或拓扑分支结论。
|
||||
|
||||
## 科学实现
|
||||
|
||||
原19列、H2图/网络、归一化、学习时钟、五组折552件/204组件、seed0、200epochs、AdamW/LR、全节点回归权重、梯度裁剪及末轮模型保留规则不变。
|
||||
|
||||
每学习件调用原无标签 `prediction_inputs` 抽取 query,保留原ID/顺序/multiplicity/坐标/Rg/near_wall/anchor ID。位置项只使用唯一ID、每候选底权相同。真峰用原 `seconds.npy` 的精确并列、最小原ID;不从FP32残差重建真峰。学习件 `seconds` 读取前记录角色/哈希,距离和query身份写成逐折逐件缓存并固定manifest,跨epoch复用。
|
||||
|
||||
固定 `s=cF+residual_hat`,cF与位置项保持FP64;theta=.03、lambda=.001。总损失为原weighted MSE加一次每件位置期望距离;无新头、区域列、调参、集成、裁剪时场或软坐标输出。空/非有限/不一致query属于错误;原评分器在有限非空输入上没有“末点undefined”返回,因此不能自行把异常转成零损失。单候选/相等距离可自然得到零梯度。
|
||||
|
||||
新worker不读验证或测试标签;旧训练的每10轮验证只是诊断、不参与优化或选模,此入口省略该诊断循环,明确作为接口差异提交审阅。训练仍为200完整轮,无早停、best epoch或宽度选择。
|
||||
|
||||
## 并发与预算
|
||||
|
||||
新共享锁和registry放在 `/home/frank14f/MeltMind-runs/phase3-shared-capacity-2026-11-18`,只在真实入口获审后调用时创建。按整机GPU计算进程与未显现的跨入口lease联合计槽,去重已显现worker;GPU0–2最多3、GPU3最多1,当前 c 只选GPU1/2。GPU0旧队列pending保留规则也写入通用准入。不会改动旧worker、源码、亲和性或预算。
|
||||
|
||||
新任务预留4GiB GPU(原H2实测约1.4–1.52GB allocator峰的保守工程包络为2GiB,再加2GiB余量),实际allocator记录超过2GiB或自身NVML进程超过4GiB即失败;这些是新入口待审工程限制,并非已测 c 峰。主机RSS每job4GiB,GPU/主机RAM保留20%,另为原队列未来折保留8GiB RAM;CPU准入需当前负载不超过80%,新worker独占预留16–25中的一个核,guard用26/27。磁盘每job8GiB,启动要求剩余100GiB。未登记的真实GPU进程也占槽并保守补足4GiB峰预留。
|
||||
|
||||
lease在spawn前持锁预留,转移给guard后才允许worker启动;退出后由原owner收集、确认worker不再存活才释放。正常退出时序差或失联lease仅阻止新准入,不自动清除、不杀其它任务。进程身份含创建时间;guard/worker核对父进程身份并设置父进程退出信号。观察到外部任务导致超槽时只停止本入口,不抢占外部任务。无法强制不参与共享锁的外部进程遵守准入;这种外部竞争仍可能让本入口停止。
|
||||
|
||||
各折从队列开始到自身终态计费,包括未派发等待、setup和失败,另各预收60秒setup;离线准备/合成保守预收600秒。单折60000秒只用于实际失控保护,不作为单折投影门。完整epoch实测后,以最大完整轮耗时对剩余轮乘1.5,附120秒收尾;未测折按原均分、已测中位数和实际等待+收尾三者最大值保留。只按总投影/真实累计停;首轮通过该总门再释放后续训练。任一真实失败停止剩余队列、无重试。
|
||||
|
||||
总预留:item1已完成72421.31599938683、seed1整个累计帽167200(旧失败不再额外加)、c205200、1b88560、b178200,共711581.3159993868≤720000。后两项只是预留,未创建入口或启动。Track A和ensemble不混入本账。现有入口原冻结帽不改。训练终态模型冻结后,`post-terminal-cost.json` 另计冻结尾段一次;后处理必须检查该最终账以及完整五折/模型冻结/退出,不能仅凭训练PASS。
|
||||
|
||||
## 验收与后续门
|
||||
|
||||
CPU合成检查覆盖旧评分并列等价、重复ID不加票、梯度解析式、平移不变性、退化零梯度、FP64 prior、原回归权重、非学习角色拒绝、GPU槽位/未显现预留/真实外部进程、双进程竞争、旧pending保留、主机余量、总投影和失败记录峰。还模拟完整五折终态及单折失败不重试,不使用真实数据或GPU。日志和测量见 `artifacts/phase3-item-c-acceptance-2026-11-18`;早期两轮完整进程CPU未单独测量,保留原日志,不能用unittest内部时长冒充进程CPU实测。
|
||||
|
||||
只有本次完整绑定的实际Assist PASS、审阅原件采用并冻结后,才具备训练入口的审阅条件;本次不代造PASS、不启动。所有五模型终态/哈希/退出后仍需独立postfit实现及实际事前审阅,才可预测和读取test。当前次终点只绑定规格,未实现或执行评分。
|
||||
|
||||
旧库门保持mean ΔD≤−.010、ΔD上界<0、ΔrelativeL2上界<+.001,组件等权成对2000bootstrap、seed2026111408,另受同终点seed1噪声比较约束;seed1不阻塞建入口或获审后的训练,却约束最终收益结论。全新独立家族确认没有数据群与执行授权,保持待定。当前没有新科学结果,也未恢复已撤回的吞吐benchmark。
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"status": "PREPARED_BINDING_CONFIG_PASS",
|
||||
"binding_sha256": "20bf6ca532f42fdba10e480a0405a4f7333a946c951e45a1eac83c582024625b",
|
||||
"code_files": 52,
|
||||
"source_files": 87,
|
||||
"review_present": false,
|
||||
"started": false,
|
||||
"actual_peer_review": false
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"reviewed_commit": "a2540974ae646f341beca248ce8a5c4f638fe2d8",
|
||||
"contract_sha256": "76fa0975b61d86363d5b9bf110a58ae56ca0a1462a3154744e1df54285d01233",
|
||||
"implementation_proof_sha256": "54af93dca18b7a9eb416a1230a265be74287af690e493f2d96521ae09f256b04",
|
||||
"comparison_contract_sha256": "a9dac0a4aff9168f6c98beef46333e5a68c1fc455d818a1afc7d21b12a11be72",
|
||||
"code_sha256": {
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_common.py": "4e85f5d211c4d3e2188422aa4c0cefa40dcfa14ef32e90894ed3412de1fd1d12",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_data.py": "fe3aa3e023f22f4fe55cd47e7ffa677b58f52a041867ac05620cbd329d913cb6",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_guard.py": "7144fa77ffe61a2a3c92c050234e370ef4dfeb0e31fafb419f752f969737ff1a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_loss.py": "79e291866bda300ecdc5577c62903d34abeec14f5cc36db05e346f9ffd64c24b",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_queue.py": "a04879cc0169ecb523a88360cddca1e234d91b3adbc20b3ec31d02461ed767eb",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_worker.py": "0f8f35e1422021b2a4f097c2e62267fb935fe4658fd0ee8ccd251609a4062ea7",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "968c4c7daeace0efa48713a93e38b97fa8a809502bf4dda3b55f523489123e9a",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_c.py": "782c415b1f536eb2b85aa2c9c4a9a0dc278f9b7134e8a83e2a6af8199396c53a",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "588c646101be0d50f505f9a12a1c98361aa3437c8c594eecb7321831f717a741",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "188565801f6901d0192f79aec0889e632501e672a4149d76d1835efbb9b45447",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_c.py": "17f9a6b3a146a9326b7975948ad6647612f5a208f5985a8dda8225254ff066bb",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5",
|
||||
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"contract_sha256": "76fa0975b61d86363d5b9bf110a58ae56ca0a1462a3154744e1df54285d01233",
|
||||
"proof_sha256": "54af93dca18b7a9eb416a1230a265be74287af690e493f2d96521ae09f256b04"
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
{
|
||||
"commit": "a2540974ae646f341beca248ce8a5c4f638fe2d8",
|
||||
"files": {
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_common.py": "4e85f5d211c4d3e2188422aa4c0cefa40dcfa14ef32e90894ed3412de1fd1d12",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_data.py": "fe3aa3e023f22f4fe55cd47e7ffa677b58f52a041867ac05620cbd329d913cb6",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_guard.py": "7144fa77ffe61a2a3c92c050234e370ef4dfeb0e31fafb419f752f969737ff1a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_loss.py": "79e291866bda300ecdc5577c62903d34abeec14f5cc36db05e346f9ffd64c24b",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_queue.py": "a04879cc0169ecb523a88360cddca1e234d91b3adbc20b3ec31d02461ed767eb",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_worker.py": "0f8f35e1422021b2a4f097c2e62267fb935fe4658fd0ee8ccd251609a4062ea7",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "968c4c7daeace0efa48713a93e38b97fa8a809502bf4dda3b55f523489123e9a",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_c.py": "782c415b1f536eb2b85aa2c9c4a9a0dc278f9b7134e8a83e2a6af8199396c53a",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "588c646101be0d50f505f9a12a1c98361aa3437c8c594eecb7321831f717a741",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "188565801f6901d0192f79aec0889e632501e672a4149d76d1835efbb9b45447",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_c.py": "17f9a6b3a146a9326b7975948ad6647612f5a208f5985a8dda8225254ff066bb",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5",
|
||||
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/contract.json": "76fa0975b61d86363d5b9bf110a58ae56ca0a1462a3154744e1df54285d01233",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/comparison-contract.json": "a9dac0a4aff9168f6c98beef46333e5a68c1fc455d818a1afc7d21b12a11be72",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/preparation-audit.json": "c0cf684f9d403d319151bd467c0aff571a7157eba069bc7430da1db3e753f6d8",
|
||||
"/home/frank14f/optane/MeltMind/docs/phase3-item-c-entry-2026-11-18.md": "e239980a0db53826f774539b56ccf29d8deee3fcab15907da51e4f93b0a2ea42",
|
||||
"/home/frank14f/optane/MeltMind/references/2026-11-18/phase3-item-c/mold-tail-volume-peak-stability-spec.md": "05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521",
|
||||
"/home/frank14f/optane/MeltMind/references/2026-11-18/phase3-item-c/mold-last-fill-objective-spec.md": "9974ae2780fcc9754b305448f3eec51f5902d9fa3770a2807eaf3ed9f497b537",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/acceptance.json": "44a77dd619ef99118d23f24902f757a74fa10c3b37934cb702a227fbb594d2a3",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/attempt1.log": "0442ffe4c529ebed72c6be3cce5feb6f9f24d1bad3dc65301d1edb928105af1e",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/attempt2.log": "c9b75fd6f0d3e8777ce06aff726fc6c1f4c75985c86a9aeb1fd11c1bb1f9552f",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/attempt3.log": "318f7c124fadf04823a3a9ee870c1dbe111410831e4a7fc42d83786c8f2376e6",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/initial-prepared-comparison-contract.json": "9b75d33ac08f6a437f6755668f9efe52493872673ae9aa65bbc88e3a667eedec",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/initial-prepared-contract.json": "837b869e9d42d7402c8cdb4f3f9236462e9ec5bf2a80bcb561e5a4c566361fb7",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/initial-prepared-preparation-audit.json": "3a48b1d2b119707e38b0470029cf28019fd40c61afbfad3b21ca9cf345bcf5cf"
|
||||
},
|
||||
"git_paths": {
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "tools/freeze_g1_components.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "tools/gre24_models.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "tools/phase2_gate_outcome_review.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "tools/phase2_h1_worker_v2.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "tools/phase2_h2_data.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "tools/phase2_h2_model.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "tools/phase2_h2_pilot_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "tools/phase2_h2_s1_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "tools/phase2_home_freeze_io.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "tools/phase2_local_freeze_io.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "tools/phase2_residual_math.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_common.py": "tools/phase3_item_c_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_data.py": "tools/phase3_item_c_data.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_guard.py": "tools/phase3_item_c_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_loss.py": "tools/phase3_item_c_loss.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_queue.py": "tools/phase3_item_c_queue.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_worker.py": "tools/phase3_item_c_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "tools/phase3_seed1_v2_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "tools/phase3_shared_capacity.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_c.py": "tools/prepare_phase3_item_c.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "tools/s2_custody_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "tools/s2_custody_statistics.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "tools/s2_gpu_idle_gate.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "tools/s2_main_memory_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/__init__.py": "tools/s2_models/__init__.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/attention.py": "tools/s2_models/attention.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/blocks.py": "tools/s2_models/blocks.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "tools/s2_models/features.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "tools/s2_models/network.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/normalization.py": "tools/s2_models/normalization.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "tools/s2_models/objectives.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "tools/s2_pilot_train.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "tools/s2_precision_continuation_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "tools/s2_recovery_gpu_supervisor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "tools/s2_recovery_monitoring.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_c.py": "tools/test_phase3_item_c.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "tools/track_a_h2_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "tools/track_a_h2_prepare.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "tools/track_a_recovery_monitor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "tools/track_a_recovery_supervisor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "tools/test_phase2_h2_s2.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "tools/phase2_h2_s2_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "tools/phase2_h2_s2_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "tools/phase2_fixed_io.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "tools/phase2_h2_s2_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "tools/phase2_h2_s1_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "tools/phase2_h2_s2_prepare.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "tools/phase2_h2_prepare.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "tools/phase2_h2_s2_queue.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "tools/phase2_h2_s1_launch.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "tools/phase2_h2_s1_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "docs/phase2-h2-s2-execution-contract-2026-11-17.md",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/contract.json": "home-runs/phase3-item-c-2026-11-18/contract.json",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/comparison-contract.json": "home-runs/phase3-item-c-2026-11-18/comparison-contract.json",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/preparation-audit.json": "home-runs/phase3-item-c-2026-11-18/preparation-audit.json",
|
||||
"/home/frank14f/optane/MeltMind/docs/phase3-item-c-entry-2026-11-18.md": "docs/phase3-item-c-entry-2026-11-18.md",
|
||||
"/home/frank14f/optane/MeltMind/references/2026-11-18/phase3-item-c/mold-tail-volume-peak-stability-spec.md": "references/2026-11-18/phase3-item-c/mold-tail-volume-peak-stability-spec.md",
|
||||
"/home/frank14f/optane/MeltMind/references/2026-11-18/phase3-item-c/mold-last-fill-objective-spec.md": "references/2026-11-18/phase3-item-c/mold-last-fill-objective-spec.md",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/acceptance.json": "artifacts/phase3-item-c-acceptance-2026-11-18/acceptance.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/attempt1.log": "artifacts/phase3-item-c-acceptance-2026-11-18/attempt1.log",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/attempt2.log": "artifacts/phase3-item-c-acceptance-2026-11-18/attempt2.log",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/attempt3.log": "artifacts/phase3-item-c-acceptance-2026-11-18/attempt3.log",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/initial-prepared-comparison-contract.json": "artifacts/phase3-item-c-acceptance-2026-11-18/initial-prepared-comparison-contract.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/initial-prepared-contract.json": "artifacts/phase3-item-c-acceptance-2026-11-18/initial-prepared-contract.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-acceptance-2026-11-18/initial-prepared-preparation-audit.json": "artifacts/phase3-item-c-acceptance-2026-11-18/initial-prepared-preparation-audit.json"
|
||||
},
|
||||
"remote_published": false,
|
||||
"local_repository": "/tmp/meltmind-phase2-local-fixed-20261117"
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"status": "PREPARED_PENDING_ASSIST_REVIEW",
|
||||
"contract_sha256": "76fa0975b61d86363d5b9bf110a58ae56ca0a1462a3154744e1df54285d01233",
|
||||
"code_files": 52,
|
||||
"source_files": 87,
|
||||
"metadata_cpu_seconds": 0.44506590199999996,
|
||||
"arrays_decoded": false,
|
||||
"model_weights_loaded": false,
|
||||
"labels_decoded": false,
|
||||
"new_fits_started": false,
|
||||
"phase_reserve_total": 711581.3159993868,
|
||||
"preparation_gpu_charge_reserved": 600,
|
||||
"synthetic_checks": 34,
|
||||
"acceptance_sha256": "44a77dd619ef99118d23f24902f757a74fa10c3b37934cb702a227fbb594d2a3"
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
# H2 后续 C|用候选末点的空间误差监督到达场
|
||||
|
||||
MoldAstra · 2026-11-17 · 事前规格。次序 A 种子、B 区域、C 末填;由 Claude 在 H2-S2 确认后裁量。与原 H2 单模型 seed 0 配对。仍用 19 列和原网格 H2;不叠加八区特征、集成、终端预测头或额外标签来源。B17 是树模型上最后 10% 体积加权与路径列的组合实验,两项分别判决。[B17 规格](https://drive.google.com/file/d/196qqDkaP2fVlwj27Jc-zctm9kqV35ZZM/view)
|
||||
|
||||
## 对准什么误差
|
||||
|
||||
两个相距很远的分支末端,真实归一化时间为 1.00、0.99,预测为 0.99、1.00。所有其他点完全正确时,全场误差可以很小,最后一点却落到了另一支。简单提高晚填节点的回归权重仍没有给这次空间错位明确的代价。另外,先把两个候选位置加权平均,可能产生一个模腔之外的“位置”。我们在**每个实际候选点**算定位误差,然后平均这些误差。Li 等在视觉关键点定位中研究过这种期望定位误差;这里可以在冻结的有限查询集上精确求和,避免采样。论文没有验证注塑末填。[Li 等,NeurIPS 2021](https://proceedings.neurips.cc/paper_files/paper/2021/hash/e4a6222cdb5b34375400904f03d8e6a5-Abstract.html)
|
||||
|
||||
## 固定目标与接口
|
||||
|
||||
每次训练仍处理一个完整学习件 p。依既有评分 query 的无标签抽取规则,在学习件上固定同口径的 query 原 ID、顺序与 multiplicity,跨 epoch 复用;不取任何验证/测试件标签。Q 取这些记录中的唯一节点 ID。定义 d_i 为“若预测在 i 处取唯一最大,冻结末点评分器对该件真值给出的 distance/Rg”。这样真值的并列、参考位置、重复 ID 和几何尺度均由原评分器决定,不能挑离预测最近的真峰。实现可先按原规则求一次真末点,再一次向量化算各候选到它的距离;应与逐点唯一最大定义在合成并列例上相等。训练时每个唯一候选在 softmax 的底权相同,防止重复抽到同一 ID 就获得额外票数;正式评估仍用全部原始评分记录。旧全节点回归的权重保持原状。
|
||||
|
||||
设 s_i=cF_i+r̂_i,是原节点无量纲预测,固定温度 θ=0.03(t_set 的 3%),π_i=exp((s_i−max s)/θ)/Σexp((s_j−max s)/θ)。逐件训练损失为
|
||||
|
||||
L_p = L_H2,p + 0.001 × Σ_{i∈Q} π_i d_i .
|
||||
|
||||
L_H2,p 精确沿用旧节点 MSE 和件大小权重。位置项每件一次,不随网格节点数放大。常数 0.001 与温度一次锁定,是研究取值,未经工业容差校准;不搜索温度、尾段宽度或系数。软权重使用带梯度的无量纲时间;FP64 的 cF 保持原精度。对任意节点 k,位置项未乘系数前的导数为 π_k(d_k−Σπd)/θ。因此训练会压低离真末点较远而当前又可能胜出的节点,也会提高近端候选。偏移所有预测同一常数不改变该项;单一候选、距离全相等时梯度为零。过冷会在一个错误峰上梯度饱和,过热会分散到早填区,放大预测时间差也可能使该项改善;故保留场监督并在判决中检验场退化。
|
||||
|
||||
仅当学习件的原评分明确返回无法定义的真末点时,该件仍用 L_H2,p,位置项置零,登记原因/分母。预测时直接输出原时场,使用原**硬**末点、原排序、原并列、原尺度评分;不做截断、软坐标或后处理。这个假设针对有限查询集,查询集漏掉真实三维末填区时不能据此推断全模腔末点。
|
||||
|
||||
## 事前判决与封存
|
||||
|
||||
- 旧库筛选固定 552 件/204 亲缘组件原五折,五个 seed 0 候选与原五个 H2 同身份测试预测配对。令 ΔD 为逐件硬末点误差候选减 H2,ΔE 为逐件 relative L2 差。组件内件均值、组件等权,同批组件索引成对 bootstrap 2,000 次、seed 2026111408、双侧 95% CI。仅当平均 ΔD≤−0.010 Rg、ΔD 上界 <0、且 ΔE 上界 <+0.001,才称“入围”。预定场容许退化为 0.10 个百分点。未达标停止该固定配方。全名单共同评分不全时仅作不完整诊断;并列规模、各折、τ 和两臂原值一起报告。
|
||||
- 要主张对新几何家族的末点收益,在任何新标签打开前固定分组、单位、query 和比较模型,旧库与确认主群在近重复/来源图上无连接,至少 25 个可配对新组件。对每个新组件,把其冻结 UTF-8 标识的 SHA-256 解释为无符号大端整数,模 5 选择同一外折的已冻结候选与对照,整组同行,两臂每件均只推理一次。此确认评估已训练五对单模型的固定分派策略;不需用确认件训练或选模型。仅当新群 ΔD 的 95% 上界 <−0.010、且 ΔE 上界 <+0.001,确认有预设幅度的末点收益并满足场容差。两条件同时成立,等号不通过。门槛是研究取值。另报组件数、两臂输入覆盖、各折分派数量/件数,覆盖不足只作描述。
|
||||
- 本项不占用 B17 的一次封存或已解封数据。缺少可独立使用的全新主群时可完成旧库筛选,确认保持待定。不能从旧库或不同候选反复试出结论,也不能把不同工程末填缺陷与本 query 尺度的末点误差等同。
|
||||
|
||||
## 预计代价
|
||||
|
||||
旧库五次新增训练,按 H2 fold 0 累计 5.24 GPU-h 加 25% 余量约 32.75 GPU-h;若位置项增加 5–15% 时间,规划约 34–38 GPU-h,增幅未经测量。每件每步新增 O(|Q|) 的 softmax、加权距离和梯度;固定学习件的距离与 ID 可预存。部署不增加网络推理次数。新群对照另需每件两次前向及 CPU 评分,不需重训。以上是成本情景;若实际分块反传或查询准备超过估计,按实账呈现。三份规格都不修改 S2 的训练或评分,也没有启动新试验。[H2-S2 既有裁量](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794923791467409)
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Item c immutable scope, assignments and conservative total-cost projection."""
|
||||
import json
|
||||
import math
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
from phase2_h1_worker_v2 import checked, sha
|
||||
from phase2_home_freeze_io import verify
|
||||
from phase3_seed1_v2_common import fold_view
|
||||
|
||||
ROOT = Path('/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18')
|
||||
|
||||
|
||||
def read(p):
|
||||
return json.loads(Path(p).read_text())
|
||||
|
||||
|
||||
def config(root):
|
||||
if root.resolve() != ROOT:
|
||||
raise PermissionError('Sole item c root')
|
||||
c = read(root/'contract.json')
|
||||
verify(root, 'implementation')
|
||||
expected = dict(approval='1795007580.895539', seed=0, epochs=200, new_fits=5,
|
||||
gpu_cap=205200, postfit_gpu_reserve=7200, postfit_cpu_cap=3600,
|
||||
phase3_gpu_cap=720000, theta=.03, coefficient=.001, no_retry=True)
|
||||
if any(c.get(k) != v for k,v in expected.items()):
|
||||
raise PermissionError('Fixed c scope/budgets')
|
||||
for p,h in {**c['source_sha256'], **c['code_sha256']}.items():
|
||||
checked(p,h)
|
||||
pilot = read(c['pilot_contract'])
|
||||
import numpy as np
|
||||
import torch
|
||||
if np.__version__ != pilot['versions']['numpy'] or torch.__version__ != pilot['versions']['torch']:
|
||||
raise ValueError('Original environment versions')
|
||||
if [j['id'] for j in c['jobs']] != list(range(5)):
|
||||
raise ValueError('Exactly five folds')
|
||||
tests = []
|
||||
for job in c['jobs']:
|
||||
f = fold_view(pilot, job['id'])['folds'][0]
|
||||
tests.extend(f['test'])
|
||||
if job['target_steps'] != 200*len(f['learning']):
|
||||
raise ValueError('Full fixed epochs')
|
||||
for k in ('learning','validation','test','c','clock_sha256','learning_nodes'):
|
||||
if job[k] != f[k]:
|
||||
raise ValueError('Original fold identity changed')
|
||||
if len(tests) != 552 or len(set(tests)) != 552:
|
||||
raise ValueError('Original552 roster')
|
||||
rows=read(c['query_roster_contract'])['rows']
|
||||
if len(rows)!=552 or {r['case'] for r in rows}!=set(tests) or len({r['component_id'] for r in rows})!=204:
|
||||
raise ValueError('552 parts /204 components query roster')
|
||||
for r in rows:
|
||||
p=pilot['parts'][r['case']]
|
||||
if r['v5_cache']['output_sha256']!=p['output_sha256'] or r['nodes']!=p['nodes'] or r['component_id']!=p['clock_group']:
|
||||
raise ValueError('Learning query/geometry/cache identity')
|
||||
checked(c['comparison_contract'], c['comparison_contract_sha256'])
|
||||
return c,pilot
|
||||
|
||||
|
||||
def assignment(root,c,index,digest):
|
||||
a = read(checked(root/'jobs'/str(index)/'assignment.json',digest))
|
||||
if a['id'] != index or a['contract_sha256'] != sha(root/'contract.json'):
|
||||
raise PermissionError('Assignment identity')
|
||||
return dict(c['jobs'][index], **{k:a[k] for k in ('gpu_uuid','worker_cpus','token','queue_identity')})
|
||||
|
||||
|
||||
def first_epoch_release(root,index):
|
||||
if (root/'global-stop.json').exists():
|
||||
raise RuntimeError('Global stop')
|
||||
p = root/'jobs'/str(index)/'first-epoch-projection.json'
|
||||
if not p.exists():
|
||||
return False
|
||||
d = read(p)
|
||||
if d['status'] != 'PASS' or d['contract_sha256'] != sha(root/'contract.json'):
|
||||
raise RuntimeError('First-epoch gate not passed')
|
||||
return True
|
||||
|
||||
|
||||
def project(c, charges, histories, terminal):
|
||||
if set(charges) != set(range(5)) or set(histories)-set(charges) or set(terminal)-set(charges):
|
||||
raise ValueError('Cost ledger fold identity')
|
||||
values = {}
|
||||
for i,v in charges.items():
|
||||
if not math.isfinite(v) or v < 0:
|
||||
raise ValueError('Invalid actual cost')
|
||||
if i in terminal:
|
||||
values[i] = v
|
||||
elif histories.get(i):
|
||||
rows = histories[i]
|
||||
e = max(r['epoch'] for r in rows)
|
||||
if sorted(r['epoch'] for r in rows) != list(range(1,e+1)) or e > 200:
|
||||
raise ValueError('Incomplete epoch history')
|
||||
t = max(r['train_epoch_seconds'] for r in rows)
|
||||
if not math.isfinite(t) or t <= 0:
|
||||
raise ValueError('Invalid measured epoch')
|
||||
# Factor1.5 applies to remaining training, never to already charged cost.
|
||||
values[i] = v + 1.5*(200-e)*t + 120
|
||||
initial = (c['gpu_cap']-c['postfit_gpu_reserve']-c['preparation_charge_gpu_seconds'])/5
|
||||
median = statistics.median(values.values()) if values else initial
|
||||
for i in charges:
|
||||
if i not in values:
|
||||
values[i] = max(charges[i]+120, initial, median)
|
||||
actual = c['preparation_charge_gpu_seconds']+sum(charges.values())
|
||||
projected = c['preparation_charge_gpu_seconds']+sum(values.values())+c['postfit_gpu_reserve']
|
||||
ok = projected <= c['gpu_cap'] and actual+c['postfit_gpu_reserve'] <= c['gpu_cap']
|
||||
ok &= all(charges[i] < c['runaway_gpu_seconds'] for i in charges if i not in terminal)
|
||||
return dict(status='PASS' if ok else 'STOP_BUDGET', projected_gpu_seconds=projected,
|
||||
actual_training_and_preparation_gpu_seconds=actual,
|
||||
folds={str(i):dict(charged=charges[i], projected=values[i]) for i in charges},
|
||||
postfit_gpu_reserve=c['postfit_gpu_reserve'], safety_factor=1.5)
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Learning-only query cache. No validation/test labels or fitted selection."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
from phase2_h1_worker_v2 import checked, sha, save
|
||||
from phase2_h2_data import cached
|
||||
from s2_custody_common import prediction_inputs
|
||||
from phase3_item_c_loss import candidates
|
||||
|
||||
|
||||
def require_learning(view, case):
|
||||
f = view['folds'][0]
|
||||
if case not in f['learning'] or case in f['validation'] or case in f['test']:
|
||||
raise PermissionError('Location truth restricted to learning role')
|
||||
|
||||
|
||||
def prepare_queries(view, rows, output):
|
||||
output = Path(output)
|
||||
output.mkdir(exist_ok=False)
|
||||
manifest = {}
|
||||
by_case = {r['case']: r for r in rows}
|
||||
if len(by_case) != len(rows):
|
||||
raise ValueError('Repeated roster case')
|
||||
for case in view['folds'][0]['learning']:
|
||||
require_learning(view, case) # before ANY case array access
|
||||
row = by_case[case]
|
||||
q = prediction_inputs(row) # existing label-free query recipe
|
||||
full = cached(view, case, 'raw_ids')
|
||||
idx = np.searchsorted(full, q['raw_ids'])
|
||||
if np.any(idx >= len(full)) or not np.array_equal(full[idx], q['raw_ids']):
|
||||
raise ValueError('Original query/full-node identity')
|
||||
if q['t_set'] != view['parts'][case]['t_set']:
|
||||
raise ValueError('Original process-time identity')
|
||||
# The original scorer uses physical seconds with exact equality ties.
|
||||
# FP32 residual reconstruction is not an acceptable true-peak source.
|
||||
digest = view['parts'][case]['output_sha256']['seconds.npy']
|
||||
with (output/'learning-location-target-access.jsonl').open('a') as f:
|
||||
f.write(json.dumps(dict(case=case, role='learning', seconds_sha256=digest))+'\n')
|
||||
seconds = cached(view, case, 'seconds')
|
||||
d = candidates(full, q['raw_ids'], q['counts'], q['coordinates_over_Rg'], seconds[idx])
|
||||
p = output/(case+'.npz')
|
||||
np.savez(p, **d, original_raw_ids=q['raw_ids'], original_counts=q['counts'],
|
||||
original_anchor_ids=q['ids'], original_near_wall=q['near_wall'],
|
||||
original_coordinates_over_Rg=q['coordinates_over_Rg'])
|
||||
manifest[case] = dict(path=str(p), sha256=sha(p), seconds_sha256=digest,
|
||||
queries_sha256=row['preparation']['output_sha256']['queries.npz'],
|
||||
unique_candidates=len(d['indices']), true_peak_raw_id=d['true_peak_raw_id'],
|
||||
true_peak_ties=d['true_peak_ties'], undefined=False)
|
||||
save(output/'manifest.json', manifest)
|
||||
return manifest
|
||||
|
||||
|
||||
def query_data(view, case, manifest):
|
||||
require_learning(view, case)
|
||||
r = manifest[case]
|
||||
with np.load(checked(r['path'], r['sha256']), allow_pickle=False) as z:
|
||||
return {k:z[k].copy() for k in ('indices', 'distances')}
|
||||
@@ -0,0 +1,136 @@
|
||||
"""One item c attempt, scoped process memory plus failed-worker peak checks."""
|
||||
import ctypes
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from phase3_item_c_common import config,assignment,read
|
||||
from phase3_shared_capacity import validate_lease,alive
|
||||
from track_a_recovery_monitor import Monitor
|
||||
from track_a_recovery_supervisor import supervise,tree_usage,stop_group,disk_usage
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
|
||||
|
||||
def run(queue,index,digest):
|
||||
parent=os.getppid()
|
||||
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGTERM,0,0,0)!=0:
|
||||
raise OSError('PDEATHSIG')
|
||||
requested=[]
|
||||
for sig in (signal.SIGTERM,signal.SIGINT):
|
||||
signal.signal(sig,lambda n,f:requested.append('SIGNAL_'+str(n)))
|
||||
if os.getppid()!=parent:
|
||||
raise RuntimeError('Queue lost before setup')
|
||||
c,_=config(queue);job=assignment(queue,c,index,digest)
|
||||
root=queue/'jobs'/str(index);start=time.monotonic();cpu=time.process_time()
|
||||
original=read(root/'assignment.json');wait=original['pre_dispatch_wait_seconds']
|
||||
remaining=c['runaway_gpu_seconds']-wait-60
|
||||
if remaining<90:
|
||||
raise RuntimeError('No cleanup window before worker')
|
||||
deadline=start+remaining
|
||||
# Queue writes ready only after atomically transferring reservation ownership.
|
||||
while not (root/'lease-ready.json').exists():
|
||||
if requested or not alive(job['queue_identity']) or time.monotonic()-start>30:
|
||||
raise RuntimeError('Lease transfer timeout')
|
||||
time.sleep(.1)
|
||||
validate_lease(job['token'],queue,index,sha(queue/'contract.json'))
|
||||
if os.getppid()!=job['queue_identity']['pid'] or not alive(job['queue_identity']):
|
||||
raise RuntimeError('Pinned queue lost before guard import completed')
|
||||
os.sched_setaffinity(0,[26,27])
|
||||
tracker=PeakRecords();done=threading.Event();monitor=Monitor(root/'monitor-errors.jsonl')
|
||||
reasons=[];receipt=None
|
||||
command=[sys.executable,'tools/phase3_item_c_worker.py',str(queue),str(index),digest]
|
||||
def reason(s):
|
||||
if s not in reasons:reasons.append(s)
|
||||
def workers():
|
||||
result=[]
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:result.append(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
return result
|
||||
def kill():
|
||||
for pid in workers():stop_group(pid)
|
||||
def scan():
|
||||
tracker.scan(root/'worker')
|
||||
if max(tracker.maxima.values())>job['recorded_vram_bytes']:reason('RECORDED_VRAM_LIMIT')
|
||||
def local_watch():
|
||||
while not done.is_set():
|
||||
monitor.sample('recorded_peaks',scan)
|
||||
usage,ok=monitor.sample('tree',lambda:tree_usage(os.getpid()))
|
||||
if ok and (usage[0]>job['cpu_seconds']-30 or usage[1]>job['rss_bytes']):reason('GUARD_TREE_LIMIT')
|
||||
disk,ok=monitor.sample('disk',lambda:disk_usage(root))
|
||||
if ok and disk>job['disk_bytes']:reason('OUTPUT_LIMIT')
|
||||
if time.monotonic()>deadline-30:reason('OWN_ACTUAL_TIME_LIMIT')
|
||||
queue_live,ok=monitor.sample('queue_identity',lambda:alive(job['queue_identity']))
|
||||
if ok and not queue_live:reason('QUEUE_LOST')
|
||||
if (queue/'global-stop.json').exists():reason('GLOBAL_STOP')
|
||||
for s in requested:reason(s)
|
||||
if monitor.failed:reason('MONITOR_FAILED')
|
||||
if reasons:kill()
|
||||
done.wait(.5)
|
||||
def gpu_watch():
|
||||
def sample():
|
||||
text=subprocess.check_output(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,used_gpu_memory','--format=csv,noheader,nounits'],text=True,timeout=10)
|
||||
rows=[]
|
||||
for line in text.strip().splitlines():
|
||||
if not line:continue
|
||||
u,p,m=[x.strip() for x in line.split(',')]
|
||||
if u==job['gpu_uuid']:rows.append(dict(pid=int(p),bytes=int(m)*2**20))
|
||||
if len(rows)>3:reason('HOST_GPU_SLOT_LIMIT')
|
||||
own=set(workers())
|
||||
if sum(r['bytes'] for r in rows if r['pid'] in own)>job['process_vram_bytes']:reason('OWN_GPU_PROCESS_MEMORY_LIMIT')
|
||||
save(root/'gpu-sample.json',dict(unix=time.time(),rows=rows,own_pids=sorted(own)))
|
||||
while not done.is_set():
|
||||
monitor.sample('gpu_processes',sample)
|
||||
if monitor.failed:reason('MONITOR_FAILED')
|
||||
if reasons:kill()
|
||||
done.wait(1)
|
||||
threads=[threading.Thread(target=f,daemon=True) for f in (local_watch,gpu_watch)]
|
||||
try:
|
||||
save(root/'started.json',dict(unix=time.time(),assignment_sha256=digest))
|
||||
for t in threads:t.start()
|
||||
if reasons or requested or time.monotonic()>deadline-60:raise RuntimeError('Prelaunch guard rejected')
|
||||
receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=job['cpu_seconds']-60,
|
||||
job_cpu_seconds=job['cpu_seconds']-60,job_wall_seconds=deadline-time.monotonic()-40,
|
||||
tree_rss_bytes=job['rss_bytes'],disk_bytes=job['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=job['gpu_uuid'])
|
||||
except BaseException as exc:
|
||||
reason(type(exc).__name__+': '+str(exc));kill()
|
||||
finally:
|
||||
done.set()
|
||||
for t in threads:
|
||||
if t.ident is not None:t.join(timeout=12)
|
||||
if any(t.is_alive() for t in threads):reason('MONITOR_DID_NOT_EXIT')
|
||||
try:monitor.retry('final_peaks',scan,.1)
|
||||
except Exception as exc:reason('FINAL_PEAK_SCAN: '+str(exc))
|
||||
for s in requested:reason(s)
|
||||
if monitor.failed:reason('MONITOR_FAILED')
|
||||
p=root/'worker/result.json';result=read(p) if p.exists() else None
|
||||
if not receipt or receipt['status']!='PASS' or receipt['returncode']!=0:reason('WORKER_NOT_PASS')
|
||||
if not result or result.get('status')!='COMPLETE' or result.get('fold')!=index or result.get('completed_epochs')!=200 or result.get('optimizer_steps')!=job['target_steps']:reason('NO_COMPLETE_IDENTITY')
|
||||
if result:
|
||||
from phase2_h1_worker_v2 import checked
|
||||
checked(result['checkpoint'],result['checkpoint_sha256'])
|
||||
if len(tracker.witnesses)!=2:reason('MISSING_RECORDED_PEAKS')
|
||||
elapsed=time.monotonic()-start
|
||||
child=resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
charged_cpu=time.process_time()-cpu+child.ru_utime+child.ru_stime
|
||||
if wait+60+elapsed>c['runaway_gpu_seconds'] or charged_cpu>job['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
|
||||
try:config(queue)
|
||||
except Exception as exc:reason('FINAL_PINS: '+str(exc))
|
||||
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',reasons=reasons,
|
||||
worker=receipt,worker_result_sha256=sha(p) if p.exists() else None,
|
||||
cpu_seconds_charged=charged_cpu,pre_dispatch_wait_seconds=wait,guard_wall_seconds=elapsed,
|
||||
recorded_memory=dict(maxima=tracker.maxima,witnesses=tracker.witnesses,observations_count=len(tracker.observations)),
|
||||
monitor_policy=monitor.policy(),monitor_errors=monitor.records,no_retry=True))
|
||||
return 0 if not reasons else 1
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3]))
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Fixed Mold 9974ae27 objective; original regression plus unique-query E[d]."""
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase2_h2_model import weighted_loss
|
||||
|
||||
THETA = .03
|
||||
COEFFICIENT = .001
|
||||
|
||||
|
||||
def candidates(full_ids, query_ids, counts, coordinates, truth_seconds):
|
||||
"""Exact original hard-peak reference; multiplicities do not vote in softmax.
|
||||
|
||||
Keep original query records separately. Repeated IDs must have identical
|
||||
coordinates/truth; malformed data is an error, never an undefined waiver.
|
||||
"""
|
||||
full = np.asarray(full_ids)
|
||||
ids = np.asarray(query_ids)
|
||||
counts = np.asarray(counts)
|
||||
xyz = np.asarray(coordinates, dtype=np.float64)
|
||||
truth = np.asarray(truth_seconds, dtype=np.float64)
|
||||
if (full.ndim != 1 or ids.ndim != 1 or not len(ids) or
|
||||
full.dtype.kind not in 'iu' or ids.dtype.kind not in 'iu' or
|
||||
np.any(np.diff(full) <= 0) or counts.shape != ids.shape or
|
||||
counts.dtype.kind not in 'iu' or np.any(counts <= 0) or
|
||||
xyz.shape != (len(ids), 3) or truth.shape != ids.shape or
|
||||
not np.isfinite(xyz).all() or not np.isfinite(truth).all()):
|
||||
raise ValueError('Invalid frozen query identity or truth')
|
||||
unique, first, inverse = np.unique(ids, return_index=True, return_inverse=True)
|
||||
if not np.array_equal(xyz, xyz[first][inverse]) or not np.array_equal(truth, truth[first][inverse]):
|
||||
raise ValueError('Repeated raw ID has inconsistent coordinate/truth')
|
||||
index = np.searchsorted(full, unique)
|
||||
if np.any(index >= len(full)) or not np.array_equal(full[index], unique):
|
||||
raise ValueError('Query ID absent from full-node geometry')
|
||||
# np.unique orders raw IDs; the first true maximum implements the old tie.
|
||||
t = truth[first]
|
||||
peak = int(np.flatnonzero(t == t.max())[0])
|
||||
distances = np.linalg.norm(xyz[first] - xyz[first][peak], axis=1)
|
||||
return dict(indices=index.astype(np.int64), distances=distances,
|
||||
unique_raw_ids=unique, true_peak_raw_id=int(unique[peak]),
|
||||
true_peak_ties=int(np.sum(t == t.max())))
|
||||
|
||||
|
||||
def location_loss(residual, prior64, indices, distances):
|
||||
if prior64.dtype != torch.float64 or distances.dtype != torch.float64:
|
||||
raise ValueError('Location prior and distances must stay FP64')
|
||||
if residual.ndim != 1 or prior64.shape != residual.shape or indices.ndim != 1 or not indices.numel():
|
||||
raise ValueError('Invalid location shapes')
|
||||
if indices.dtype != torch.int64 or distances.shape != indices.shape or torch.unique(indices).numel() != indices.numel():
|
||||
raise ValueError('Exactly one equal-weight vote per candidate')
|
||||
s = prior64[indices] + residual[indices].to(torch.float64)
|
||||
if not bool(torch.isfinite(s).all()) or not bool(torch.isfinite(distances).all()) or bool((distances < 0).any()):
|
||||
raise ValueError('Nonfinite location data')
|
||||
return torch.sum(torch.softmax((s - s.max()) / THETA, dim=0) * distances)
|
||||
|
||||
|
||||
def objective(prediction, target, parts, nodes, total, prior64, indices, distances):
|
||||
regression = weighted_loss(prediction, target, parts, nodes, total)
|
||||
location = location_loss(prediction, prior64, indices, distances)
|
||||
return regression + COEFFICIENT * location, regression, location
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Review-gated five-fold c training. Stops at frozen models, never postfit."""
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import psutil
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from phase2_home_freeze_io import freeze
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase3_item_c_common import ROOT,config,read,project
|
||||
from phase3_shared_capacity import register_entry,acquire,transfer,release,process_identity,alive
|
||||
from track_a_recovery_monitor import Monitor
|
||||
|
||||
|
||||
def run(root):
|
||||
c,_=config(root);validate_review(root)
|
||||
with (root/'queue.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():raise FileExistsError('Sole attempt, no resume')
|
||||
if os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':
|
||||
raise ValueError('Explicit allocator required')
|
||||
if psutil.disk_usage(root).free<100*2**30:raise RuntimeError('Less than100GiB free')
|
||||
register_entry(root,sha(root/'contract.json'))
|
||||
start=time.monotonic();identity=process_identity(os.getpid())
|
||||
save(root/'started.json',dict(unix=time.time(),identity=identity,contract_sha256=sha(root/'contract.json')))
|
||||
(root/'jobs').mkdir(exist_ok=False)
|
||||
active={};terminal={};pending=list(range(5));histories={};stops=[];failure=None
|
||||
monitor=Monitor(root/'admission-errors.jsonl')
|
||||
for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,lambda n,f:stops.append('SIGNAL_'+str(n)))
|
||||
def charge(i):
|
||||
return terminal[i]['gpu_seconds_charged'] if i in terminal else 60+time.monotonic()-start
|
||||
def stop(reason):
|
||||
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time()))
|
||||
try:
|
||||
while active or pending:
|
||||
if stops:raise RuntimeError(stops[0])
|
||||
if (root/'global-stop.json').exists():raise RuntimeError('EXTERNAL_GLOBAL_STOP')
|
||||
# Collect exits before granting any further admission.
|
||||
for i,(p,lease,log,owner) in list(active.items()):
|
||||
if p.poll() is None:continue
|
||||
log.close();jr=root/'jobs'/str(i);cp=jr/'complete.json'
|
||||
result=read(cp) if cp.exists() else dict(status='MISSING_GUARD_RECEIPT')
|
||||
worker_identity=jr/'worker/worker-identity.json'
|
||||
if worker_identity.exists() and alive(read(worker_identity)):
|
||||
raise RuntimeError('WORKER_OUTLIVED_GUARD_LEASE_HELD')
|
||||
terminal[i]=dict(id=i,status=result['status'],returncode=p.returncode,
|
||||
gpu_seconds_charged=charge(i),guard_sha256=sha(cp) if cp.exists() else None)
|
||||
release(lease['token'],owner);del active[i]
|
||||
if result['status']!='PASS' or p.returncode!=0:raise RuntimeError('FOLD_FAILED_'+str(i))
|
||||
for i in active:
|
||||
folder=root/'jobs'/str(i)/'worker/history'
|
||||
histories[i]=[read(p) for p in sorted(folder.glob('*.json'))]
|
||||
charges={i:charge(i) for i in range(5)}
|
||||
projection=project(c,charges,histories,terminal)
|
||||
save(root/'latest-cost-projection.json',projection)
|
||||
if projection['status']!='PASS':raise RuntimeError('TOTAL_PROJECTION_OR_ACTUAL_CAP')
|
||||
for i in active:
|
||||
gate=root/'jobs'/str(i)/'first-epoch-projection.json'
|
||||
if histories.get(i) and not gate.exists():
|
||||
save(gate,dict(status='PASS',contract_sha256=sha(root/'contract.json'),projection=projection))
|
||||
save(root/'live-progress.json',dict(unix=time.time(),stage='ITEM_C_TRAINING',active=list(active),pending=pending,
|
||||
terminal=list(terminal.values()),charged_training_gpu_seconds=projection['actual_training_and_preparation_gpu_seconds'],postfit_started=False))
|
||||
if pending:
|
||||
i=pending[0]
|
||||
decision,ok=monitor.sample('admission',lambda:acquire(root,i,sha(root/'contract.json')))
|
||||
if monitor.failed:raise RuntimeError('THREE_ADMISSION_FAILURES')
|
||||
if not ok:
|
||||
time.sleep(1)
|
||||
continue
|
||||
lease,observation=decision
|
||||
save(root/'latest-admission.json',observation)
|
||||
# Acquisition includes bounded GPU queries; recheck cost before Popen.
|
||||
fresh=project(c,{j:charge(j) for j in range(5)},histories,terminal)
|
||||
if fresh['status']!='PASS':
|
||||
if lease:release(lease['token'])
|
||||
raise RuntimeError('CAP_DURING_ADMISSION')
|
||||
if lease:
|
||||
jr=root/'jobs'/str(i);jr.mkdir(exist_ok=False)
|
||||
save(jr/'assignment.json',dict(id=i,contract_sha256=sha(root/'contract.json'),
|
||||
gpu_uuid=lease['gpu_uuid'],worker_cpus=lease['worker_cpus'],token=lease['token'],
|
||||
queue_identity=identity,pre_dispatch_wait_seconds=time.monotonic()-start))
|
||||
log=(jr/'guard.log').open('x')
|
||||
env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1')
|
||||
p=subprocess.Popen([sys.executable,'tools/phase3_item_c_guard.py',str(root),str(i),sha(jr/'assignment.json')],
|
||||
cwd=str(Path(__file__).resolve().parents[1]),env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
||||
owner=process_identity(p.pid)
|
||||
active[i]=(p,lease,log,owner);pending.remove(i)
|
||||
transfer(lease['token'],owner)
|
||||
save(jr/'lease-ready.json',dict(token=lease['token'],owner=owner))
|
||||
time.sleep(1)
|
||||
except BaseException as exc:
|
||||
failure=type(exc).__name__+': '+str(exc);stop(failure)
|
||||
finally:
|
||||
if active:
|
||||
stop(failure or 'QUEUE_CLEANUP')
|
||||
for p,lease,log,owner in active.values():
|
||||
if p.poll() is None:p.terminate()
|
||||
deadline=time.monotonic()+45
|
||||
while time.monotonic()<deadline and any(p.poll() is None for p,_,_,_ in active.values()):time.sleep(.2)
|
||||
for i,(p,lease,log,owner) in active.items():
|
||||
if p.poll() is None:p.kill()
|
||||
p.wait();log.close();jr=root/'jobs'/str(i);cp=jr/'complete.json'
|
||||
result=read(cp) if cp.exists() else dict(status='MISSING_GUARD_RECEIPT')
|
||||
terminal[i]=dict(id=i,status=result['status'],returncode=p.returncode,gpu_seconds_charged=charge(i),
|
||||
guard_sha256=sha(cp) if cp.exists() else None)
|
||||
wp=jr/'worker/worker-identity.json'
|
||||
try:
|
||||
if wp.exists() and alive(read(wp)):raise RuntimeError('Live worker; lease retained')
|
||||
release(lease['token'],owner)
|
||||
except Exception as exc:
|
||||
terminal[i]['lease_release_failure']=str(exc);failure=failure or str(exc)
|
||||
final_charges={i:charge(i) for i in range(5)}
|
||||
final_cost=c['preparation_charge_gpu_seconds']+sum(final_charges.values())
|
||||
if final_cost+c['postfit_gpu_reserve']>c['gpu_cap']:failure=failure or 'FINAL_CUMULATIVE_CAP'
|
||||
complete=not failure and len(terminal)==5 and all(r['status']=='PASS' and r['returncode']==0 for r in terminal.values())
|
||||
files=[root/'contract.json']
|
||||
if complete:
|
||||
models=[]
|
||||
for i in range(5):
|
||||
r=read(root/'jobs'/str(i)/'worker/result.json')
|
||||
models.append(dict(r));files.append(Path(r['checkpoint']))
|
||||
files.extend([root/'jobs'/str(i)/'complete.json',root/'jobs'/str(i)/'worker/result.json',root/'jobs'/str(i)/'worker/standardization.json'])
|
||||
save(root/'models.json',models);files.append(root/'models.json')
|
||||
save(root/'complete.json',dict(status='TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING' if complete else 'STOPPED_INCOMPLETE',
|
||||
failure=failure,entries=list(terminal.values()),never_attempted=pending,
|
||||
never_attempted_wait_charges={str(i):final_charges[i] for i in pending},
|
||||
cumulative_gpu_seconds=final_cost,postfit_gpu_reserve=c['postfit_gpu_reserve'],
|
||||
no_retry=True,test_targets_read=False,prediction_started=False,scoring_started=False))
|
||||
files.append(root/'complete.json')
|
||||
freeze_start=time.monotonic()
|
||||
freeze(root,'training-terminal' if complete else 'training-stop',files)
|
||||
final_cost+=time.monotonic()-freeze_start
|
||||
within=final_cost+c['postfit_gpu_reserve']<=c['gpu_cap']
|
||||
save(root/'post-terminal-cost.json',dict(cumulative_gpu_seconds=final_cost,within_cap=within,
|
||||
freeze_tail_charged_once=True,postfit_gpu_reserve=c['postfit_gpu_reserve'],postfit_started=False,
|
||||
training_complete_sha256=sha(root/'complete.json'),required_before_any_postfit=True))
|
||||
return 0 if complete and within else 1
|
||||
|
||||
|
||||
if __name__=='__main__':raise SystemExit(run(ROOT))
|
||||
@@ -0,0 +1,82 @@
|
||||
"""One new S2 fold, fixed item c loss with seed0/200epochs; no test prediction or target reads."""
|
||||
import json,os,sys,time,traceback,random
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase2_h2_s1_common import selected_checkpoint
|
||||
from phase3_item_c_common import config,fold_view,first_epoch_release,assignment as assigned_job,read
|
||||
from phase3_item_c_data import prepare_queries,query_data
|
||||
from phase3_item_c_loss import objective
|
||||
from phase3_shared_capacity import process_identity
|
||||
from phase2_h2_data import standardization
|
||||
from phase2_h2_data import load_part,to_device,arguments,cached
|
||||
from phase2_h2_model import H2Geo,optimizer,lr_epoch
|
||||
from phase2_h2_pilot_worker import peaks
|
||||
from phase2_residual_math import residual_target
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
|
||||
def train_step(model,opt,data,parts,total,query,c):
|
||||
model.train();opt.zero_grad(set_to_none=True)
|
||||
prediction=model(*arguments(data))
|
||||
prior=torch.as_tensor(c*data['F'],dtype=torch.float64,device=prediction.device)
|
||||
indices=torch.as_tensor(query['indices'],dtype=torch.int64,device=prediction.device)
|
||||
distances=torch.as_tensor(query['distances'],dtype=torch.float64,device=prediction.device)
|
||||
loss,regression,location=objective(prediction,data['target'],parts,data['nodes'],total,prior,indices,distances)
|
||||
if not bool(torch.isfinite(loss)):raise ValueError('Nonfinite loss')
|
||||
loss.backward();norm=torch.nn.utils.clip_grad_norm_(model.parameters(),1.,error_if_nonfinite=True);opt.step()
|
||||
if any(not bool(torch.isfinite(p).all()) for p in model.parameters()):raise ValueError('Nonfinite parameter')
|
||||
torch.cuda.synchronize()
|
||||
return dict(loss=float(loss.detach()),regression_loss=float(regression.detach()),location_loss=float(location.detach()),gradient_norm_before_clip=float(norm),**peaks())
|
||||
|
||||
|
||||
def run(root,fold_id,assignment_sha):
|
||||
c,pilot=config(root);old=fold_view(pilot,fold_id);job=assigned_job(root,c,fold_id,assignment_sha);out=root/'jobs'/str(fold_id)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir()
|
||||
os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
|
||||
start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0
|
||||
save(out/'worker-identity.json',process_identity(os.getpid()))
|
||||
def record(item,epoch_file=False):
|
||||
item=dict(fold=fold_id,stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks())
|
||||
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
|
||||
save(out/'progress.json',item)
|
||||
if epoch_file:save(out/'history'/f'{epoch:03d}.json',item)
|
||||
return item
|
||||
try:
|
||||
if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity')
|
||||
stage='LEARNING_ONLY_NORMALIZATION';normal=out/'normalization';normal.mkdir();stats=standardization(old,normal);save(out/'standardization.json',stats);save(out/'fold-identity.json',dict(actual_fold=fold_id,learning=old['folds'][0]['learning'],validation=old['folds'][0]['validation'],test=old['folds'][0]['test'],clock_sha256=old['folds'][0]['clock_sha256']))
|
||||
stage='LEARNING_QUERY_PREPARATION';queries=prepare_queries(old,read(c['query_roster_contract'])['rows'],out/'queries')
|
||||
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
|
||||
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();random.seed(0);np.random.seed(0);torch.manual_seed(0);torch.cuda.manual_seed_all(0)
|
||||
model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes']
|
||||
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=job['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters())))
|
||||
for epoch in range(1,201):
|
||||
stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0.
|
||||
if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch')
|
||||
order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts)
|
||||
for index in order:
|
||||
case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda')
|
||||
query=query_data(old,case,queries);result=train_step(model,opt,data,parts,total,query,fold['c']);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss']
|
||||
record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],regression_loss=result['regression_loss'],location_loss=result['location_loss'],gradient_norm_before_clip=result['gradient_norm_before_clip']))
|
||||
del data
|
||||
if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch')
|
||||
metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch))
|
||||
stage='EPOCH_COMPLETE';record(metrics,epoch_file=True)
|
||||
if epoch==1:
|
||||
stage='WAIT_THIS_FOLD_FIRST_EPOCH_COST_GATE';record(dict(epoch_parts=seen,epoch_nodes=nodes))
|
||||
while not first_epoch_release(root,fold_id):time.sleep(.5)
|
||||
if epoch<200:
|
||||
for group in opt.param_groups:group['lr']=lr_epoch(epoch+1)
|
||||
stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch)
|
||||
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=fold_id,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name)
|
||||
save(out/'result.json',dict(status='COMPLETE',fold=fold_id,completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,validation_targets_read=False,query_manifest_sha256=sha(out/'queries/manifest.json'),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks()))
|
||||
except BaseException as exc:
|
||||
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
|
||||
save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
|
||||
if __name__=='__main__':
|
||||
import ctypes,signal
|
||||
from phase3_shared_capacity import alive
|
||||
expected=read(Path(sys.argv[1])/'jobs'/sys.argv[2]/'lease-ready.json')['owner']
|
||||
parent=os.getppid()
|
||||
if parent!=expected['pid'] or not alive(expected):raise RuntimeError('Pinned guard lost before worker import completed')
|
||||
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGKILL,0,0,0)!=0:raise OSError('PDEATHSIG')
|
||||
if os.getppid()!=parent:raise RuntimeError('Guard exited before worker setup')
|
||||
run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3])
|
||||
@@ -0,0 +1,197 @@
|
||||
"""New-entry admission only; never changes or kills another entry's worker.
|
||||
|
||||
All participating future entries must use this same registry/lock. Unknown
|
||||
processes consume real slots and a conservative reservation; stale leases fail
|
||||
closed. Old fixed queues do not join this registry and remain untouched.
|
||||
"""
|
||||
from contextlib import contextmanager
|
||||
import fcntl
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
import psutil
|
||||
from phase2_h1_worker_v2 import save
|
||||
|
||||
GIB = 2**30
|
||||
GPUS = ['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c', 'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
|
||||
'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76', 'GPU-8b71b032-1b75-2891-935c-ce106a53034a']
|
||||
SHARED = Path('/home/frank14f/MeltMind-runs/phase3-shared-capacity-2026-11-18')
|
||||
OLD = [Path('/home/frank14f/MeltMind-runs')/s for s in
|
||||
['phase3-seed1-v2-2026-11-18', 'track-a-recovery-2026-11-18']]
|
||||
PHASE_RESERVES = dict(item1=72421.31599938683, seed1=167200., c=205200., item1b=88560., b=178200.)
|
||||
|
||||
|
||||
def process_identity(pid):
|
||||
p = psutil.Process(pid)
|
||||
return dict(pid=pid, created=p.create_time())
|
||||
|
||||
|
||||
def alive(identity):
|
||||
try:
|
||||
p = psutil.Process(identity['pid'])
|
||||
return p.create_time() == identity['created'] and p.status() != psutil.STATUS_ZOMBIE
|
||||
except psutil.NoSuchProcess:
|
||||
return False
|
||||
|
||||
|
||||
def descendants(identity):
|
||||
if not alive(identity):
|
||||
raise RuntimeError('Stale lease: explicit reconciliation required')
|
||||
return {identity['pid']} | {p.pid for p in psutil.Process(identity['pid']).children(recursive=True)}
|
||||
|
||||
|
||||
def old_pending():
|
||||
for root in OLD:
|
||||
p = root/'complete.json'
|
||||
if p.exists():
|
||||
json.loads(p.read_text()) # corrupt terminal fails closed
|
||||
continue
|
||||
d = json.loads((root/'live-progress.json').read_text())
|
||||
if time.time()-d['unix'] > 120:
|
||||
raise RuntimeError('Stale old queue snapshot')
|
||||
if d['pending']:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def host_snapshot():
|
||||
def query(arg):
|
||||
return subprocess.check_output(['nvidia-smi', arg, '--format=csv,noheader,nounits'], text=True, timeout=10)
|
||||
devices = {}
|
||||
for line in query('--query-gpu=uuid,memory.total,memory.used').strip().splitlines():
|
||||
u, total, used = [s.strip() for s in line.split(',')]
|
||||
devices[u] = dict(total=int(total)*2**20, used=int(used)*2**20)
|
||||
processes = []
|
||||
for line in query('--query-compute-apps=gpu_uuid,pid,used_gpu_memory').strip().splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
u, pid, mem = [s.strip() for s in line.split(',')]
|
||||
processes.append(dict(gpu_uuid=u, pid=int(pid), vram=int(mem)*2**20))
|
||||
if set(devices) != set(GPUS) or len({(p['gpu_uuid'],p['pid']) for p in processes}) != len(processes):
|
||||
raise ValueError('Unknown GPU topology or duplicate compute PID')
|
||||
vm = psutil.virtual_memory()
|
||||
return dict(devices=devices, processes=processes, ram_total=vm.total, ram_available=vm.available,
|
||||
cpu_percent=psutil.cpu_percent(interval=.1), old_pending=old_pending(), unix=time.time())
|
||||
|
||||
|
||||
def admission(snapshot, leases, gpu, *, vram_reserve=4*GIB, rss_reserve=4*GIB):
|
||||
"""Pure admission rule. leases include observed descendant PIDs and RSS."""
|
||||
if gpu not in GPUS:
|
||||
raise ValueError('Unknown GPU')
|
||||
if gpu == GPUS[0] and snapshot['old_pending']:
|
||||
return False, 'GPU0_RESERVED_FOR_OLD_PENDING'
|
||||
processes = snapshot['processes']
|
||||
card = [p for p in processes if p['gpu_uuid'] == gpu]
|
||||
same = [l for l in leases if l['gpu_uuid'] == gpu]
|
||||
real = {p['pid'] for p in card}
|
||||
occupied = len(real) + sum(not (real & set(l['observed_pids'])) for l in same)
|
||||
if occupied >= (1 if gpu == GPUS[3] else 3):
|
||||
return False, 'GPU_SLOT_LIMIT'
|
||||
owned = set().union(*(set(l['observed_pids']) for l in same)) if same else set()
|
||||
# Charge the unseen part of each reserved peak; do not double-count visible memory.
|
||||
missing = sum(max(0, l['vram_reserve']-sum(p['vram'] for p in card if p['pid'] in l['observed_pids'])) for l in same)
|
||||
missing += sum(max(0, 4*GIB-p['vram']) for p in card if p['pid'] not in owned)
|
||||
d = snapshot['devices'][gpu]
|
||||
if d['used'] + missing + vram_reserve > .8*d['total']:
|
||||
return False, 'GPU_HEADROOM'
|
||||
ram_missing = sum(max(0,l['rss_reserve']-l['observed_rss']) for l in leases)
|
||||
# Extra8GiB preserves future original folds; visible old RSS already in available.
|
||||
if snapshot['ram_available']-ram_missing-rss_reserve-8*GIB < .2*snapshot['ram_total']:
|
||||
return False, 'HOST_RAM_HEADROOM'
|
||||
if snapshot['cpu_percent'] > 80:
|
||||
return False, 'HOST_CPU_HEADROOM'
|
||||
return True, 'ADMITTED'
|
||||
|
||||
|
||||
@contextmanager
|
||||
def registry():
|
||||
SHARED.mkdir(exist_ok=True)
|
||||
with (SHARED/'capacity.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
p = SHARED/'registry.json'
|
||||
d = json.loads(p.read_text()) if p.exists() else dict(version=1, entries={}, leases={})
|
||||
if d['version'] != 1:
|
||||
raise ValueError('Shared registry version')
|
||||
yield d
|
||||
save(p, d)
|
||||
|
||||
|
||||
def register_entry(root, digest):
|
||||
with registry() as d:
|
||||
if 'c' in d['entries']:
|
||||
raise FileExistsError('Sole item c entry already registered; no resume')
|
||||
if sum(PHASE_RESERVES.values()) > 720000:
|
||||
raise RuntimeError('Whole phase3 reserved cap')
|
||||
d['entries']['c'] = dict(root=str(root), contract_sha256=digest, owner=process_identity(os.getpid()),
|
||||
cap=205200, phase_reservations=PHASE_RESERVES, started_unix=time.time())
|
||||
|
||||
|
||||
def acquire(root, fold, digest):
|
||||
with registry() as d:
|
||||
entry = d['entries']['c']
|
||||
if entry['root'] != str(root) or entry['contract_sha256'] != digest or not alive(entry['owner']):
|
||||
raise PermissionError('Entry owner/identity')
|
||||
leases = []
|
||||
for l in d['leases'].values():
|
||||
# Normal guard exit can race this observation. Block admission until
|
||||
# its owning coordinator collects it; never reclaim or kill peers.
|
||||
if not alive(l['owner']):
|
||||
return None, dict(reason='STALE_LEASE_AWAIT_OWNER_COLLECTION',token=l['token'])
|
||||
try:
|
||||
ids = descendants(l['owner'])
|
||||
except (psutil.NoSuchProcess,RuntimeError):
|
||||
return None, dict(reason='LEASE_EXIT_RACE_AWAIT_OWNER_COLLECTION',token=l['token'])
|
||||
rss=0
|
||||
for pid in ids:
|
||||
try:rss+=psutil.Process(pid).memory_info().rss
|
||||
except psutil.NoSuchProcess:pass
|
||||
leases.append(dict(l, observed_pids=list(ids), observed_rss=rss))
|
||||
s = host_snapshot() # fresh inside inter-entry lock, before making reservation
|
||||
used_cpu = {x for l in leases for x in l['worker_cpus']}
|
||||
cpus = [x for x in range(16,26) if x not in used_cpu and x in os.sched_getaffinity(0)]
|
||||
if not cpus:
|
||||
return None, dict(reason='CPU_RESERVATIONS', snapshot=s)
|
||||
for gpu in GPUS[1:3]:
|
||||
ok, reason = admission(s, leases, gpu)
|
||||
if not ok:
|
||||
continue
|
||||
token = uuid.uuid4().hex
|
||||
l = dict(token=token, root=str(root), fold=fold, contract_sha256=digest, gpu_uuid=gpu,
|
||||
worker_cpus=[cpus[0]], owner=process_identity(os.getpid()),
|
||||
vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time())
|
||||
d['leases'][token] = l
|
||||
return l, dict(reason='ADMITTED', snapshot=s)
|
||||
return None, dict(reason='NO_CAPACITY', snapshot=s)
|
||||
|
||||
|
||||
def transfer(token, owner):
|
||||
with registry() as d:
|
||||
l = d['leases'][token]
|
||||
if l['owner']['pid'] != os.getpid() or not alive(owner):
|
||||
raise PermissionError('Only queue transfers its own lease to live guard')
|
||||
l['owner'] = owner
|
||||
|
||||
|
||||
def release(token, guard_identity=None):
|
||||
with registry() as d:
|
||||
l = d['leases'][token]
|
||||
if guard_identity is None:
|
||||
if l['owner']['pid'] != os.getpid():
|
||||
raise PermissionError('Only unused own lease')
|
||||
elif l['owner'] != guard_identity or alive(guard_identity):
|
||||
raise PermissionError('Guard must be reaped before release')
|
||||
del d['leases'][token]
|
||||
|
||||
|
||||
def validate_lease(token, root, fold, digest):
|
||||
with registry() as d:
|
||||
l = d['leases'][token]
|
||||
if (l['root'], l['fold'], l['contract_sha256']) != (str(root), fold, digest):
|
||||
raise PermissionError('Lease identity')
|
||||
if l['owner'] != process_identity(os.getpid()):
|
||||
raise PermissionError('Lease not transferred to this guard')
|
||||
return dict(l)
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Prepare metadata-only c contract. Does not freeze, train or adopt a review."""
|
||||
from pathlib import Path
|
||||
import time
|
||||
from phase2_h1_worker_v2 import checked,sha,save
|
||||
from phase3_item_c_common import ROOT,read
|
||||
from track_a_h2_prepare import local_code_closure
|
||||
from phase3_shared_capacity import PHASE_RESERVES
|
||||
|
||||
B=Path(__file__).resolve().parents[1]
|
||||
H=Path('/home/frank14f/MeltMind-runs')
|
||||
|
||||
|
||||
def main():
|
||||
start=time.process_time()
|
||||
if ROOT.exists():raise FileExistsError('Sole item c root exists')
|
||||
old=H/'phase2-h2-s2-2026-11-17';post=H/'phase2-h2-s2-postfit-2026-11-17'
|
||||
oc=read(old/'contract.json');pilot=read(oc['pilot_contract'])
|
||||
source=dict(oc['source_sha256']);source.update(oc['code_sha256'])
|
||||
specs={'mold-last-fill-objective-spec.md':'9974ae2780fcc9754b305448f3eec51f5902d9fa3770a2807eaf3ed9f497b537',
|
||||
'mold-tail-volume-peak-stability-spec.md':'05344bacd877de87cbde9eb18139ad124e6cd4a15659e3c469b655b3f5a3f521'}
|
||||
for name,h in specs.items():
|
||||
p=B/'references/2026-11-18/phase3-item-c'/name;checked(p,h);source[str(p)]=h
|
||||
for p in [old/'contract.json',old/'complete.json',post/'contract.json',post/'complete.json',post/'predictions-manifest.json',post/'score/result.json',
|
||||
Path(oc['pilot_contract']),H/'phase3-item1-postfit-recovery-2026-11-18/post-terminal-cost.json']:
|
||||
source[str(p)]=sha(p)
|
||||
if read(post/'complete.json')['decision']!='CONFIRMED':raise ValueError('Original H2 stage')
|
||||
itemcost=read(H/'phase3-item1-postfit-recovery-2026-11-18/post-terminal-cost.json')['cumulative_item_gpu_seconds']
|
||||
if abs(itemcost-PHASE_RESERVES['item1'])>1e-6:raise ValueError('Original item1 ledger')
|
||||
files=sorted(B.glob('tools/phase3_item_c_*.py'))+[B/'tools/phase3_shared_capacity.py',Path(__file__).resolve(),B/'tools/test_phase3_item_c.py']
|
||||
codes=local_code_closure(files+sorted((B/'tools/s2_models').glob('*.py')))
|
||||
# Include scientific package modules that the existing local AST resolver skips.
|
||||
for p,h in oc['code_sha256'].items():codes.setdefault(p,h)
|
||||
for p,h in source.items():checked(p,h)
|
||||
comparison=dict(version='item-c-training-v1',main_spec_sha256=specs['mold-last-fill-objective-spec.md'],secondary_spec_sha256=specs['mold-tail-volume-peak-stability-spec.md'],
|
||||
folds=pilot['folds'],roster_parts=552,components=204,seed=0,epochs=200,theta=.03,coefficient=.001,
|
||||
regression='original H2 weighted_loss unchanged',location='unique original query IDs equal softmax base weight; original exact-seconds true max tied by minimum raw ID',
|
||||
validation='no validation labels read; original diagnostic-only loop omitted, no optimizer or selection change',
|
||||
selection='last_epoch_only',inputs=19,query_scoring='original hard maximum and exact tie rule',
|
||||
old_corpus_gate=dict(mean_delta_D_at_most=-.010,upper_delta_D_below=0,upper_delta_relativeL2_below=.001,
|
||||
component_equal_weight=True,bootstrap=2000,seed=2026111408,paired_draws=True,
|
||||
seed1_same_endpoint_noise_comparison_required=True,
|
||||
additional_gate='absolute candidate effect must exceed absolute seed1-minus-seed0 point difference at the same endpoint'),
|
||||
secondary='frozen full retained-tet final1pct volume overlap endpoint05344bac; separate postfit entry required',
|
||||
fresh_family_confirmation='DEFERRED_NOT_AUTHORIZED_NO_COHORT',ensemble=False,regional_features=False,
|
||||
postfit='not implemented or launched in training entry; all5 models frozen and separate actual review before predictions/test reads',
|
||||
baseline_predictions_sha256=sha(post/'predictions-manifest.json'))
|
||||
ROOT.mkdir();save(ROOT/'comparison-contract.json',comparison)
|
||||
jobs=[]
|
||||
for f in pilot['folds']:
|
||||
jobs.append({k:f[k] for k in ('learning','validation','test','c','clock_sha256','learning_nodes')}|
|
||||
dict(id=f['fold'],target_steps=200*len(f['learning']),cpu_seconds=60000,
|
||||
rss_bytes=4*2**30,recorded_vram_bytes=2*2**30,process_vram_bytes=4*2**30,disk_bytes=8*2**30))
|
||||
c=dict(version='phase3-item-c-v1',approval='1795007580.895539',entry_approval='1795007058.477789',
|
||||
seed=0,epochs=200,new_fits=5,theta=.03,coefficient=.001,no_retry=True,
|
||||
gpu_cap=205200,postfit_gpu_reserve=7200,postfit_cpu_cap=3600,phase3_gpu_cap=720000,
|
||||
preparation_charge_gpu_seconds=600,preparation_cpu_cap=600,runaway_gpu_seconds=60000,
|
||||
phase_reservations=PHASE_RESERVES,track_a_ledger='separate',
|
||||
waiting_charge='all five folds from queue start through own terminal or stop, including never-dispatched pending;60s setup per fold',
|
||||
projection='charged +1.5 remaining epochs at largest observed full epoch,120s cleanup; pending=max(initial share,started median,actual+120); total stop only',
|
||||
capacity='global lock, real GPU PIDs plus unseen reservations,3/3/3/1 slots,c onlyGPU1/2,20pct headroom,4GiB reservation per new task',
|
||||
learning_query_scope='original query extractor on learning parts only; seconds target read logged first; immutable local per-fold manifest',
|
||||
validation_targets_read=False,test_targets_read=False,postfit_started=False,
|
||||
jobs=jobs,pilot_contract=oc['pilot_contract'],query_roster_contract=str(post/'contract.json'),
|
||||
source_sha256=source,code_sha256=codes,comparison_contract=str(ROOT/'comparison-contract.json'),
|
||||
comparison_contract_sha256=sha(ROOT/'comparison-contract.json'))
|
||||
save(ROOT/'contract.json',c)
|
||||
save(ROOT/'preparation-audit.json',dict(status='PREPARED_PENDING_ASSIST_REVIEW',contract_sha256=sha(ROOT/'contract.json'),
|
||||
code_files=len(codes),source_files=len(source),metadata_cpu_seconds=time.process_time()-start,
|
||||
arrays_decoded=False,model_weights_loaded=False,labels_decoded=False,new_fits_started=False,
|
||||
phase_reserve_total=sum(PHASE_RESERVES.values()),preparation_gpu_charge_reserved=600))
|
||||
print(read(ROOT/'preparation-audit.json'))
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,223 @@
|
||||
"""CPU synthetic acceptance only: no original arrays, labels, CUDA or fits."""
|
||||
import ast
|
||||
import copy
|
||||
import json
|
||||
import multiprocessing
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase3_item_c_loss import candidates,location_loss,objective,THETA
|
||||
from phase3_item_c_data import require_learning
|
||||
from phase3_item_c_common import project
|
||||
from phase3_shared_capacity import admission,GPUS,GIB,PHASE_RESERVES
|
||||
from s2_custody_statistics import last_location
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
|
||||
|
||||
def snapshot():
|
||||
return dict(devices={u:dict(total=16*GIB,used=2*GIB) for u in GPUS},
|
||||
processes=[dict(gpu_uuid=GPUS[1],pid=101,vram=2*GIB)],
|
||||
ram_total=128*GIB,ram_available=96*GIB,cpu_percent=20,old_pending=False)
|
||||
|
||||
|
||||
def lease(pid=102):
|
||||
return dict(gpu_uuid=GPUS[1],observed_pids=[pid],observed_rss=GIB,vram_reserve=4*GIB,rss_reserve=4*GIB)
|
||||
|
||||
|
||||
def concurrent_admit(shared, barrier, results):
|
||||
import phase3_shared_capacity as m
|
||||
m.SHARED=Path(shared)
|
||||
s=snapshot()
|
||||
# two actual tasks leave one slot onGPU1; GPU2 already has three.
|
||||
s['processes'].append(dict(gpu_uuid=GPUS[1],pid=103,vram=GIB))
|
||||
s['processes'] += [dict(gpu_uuid=GPUS[2],pid=p,vram=GIB) for p in (201,202,203)]
|
||||
m.host_snapshot=lambda:s
|
||||
try:
|
||||
l,_=m.acquire(Path('/synthetic'),0,'abc')
|
||||
results.put(bool(l))
|
||||
barrier.wait(timeout=10) # keep lease owners alive through both admissions
|
||||
except BaseException as exc:
|
||||
results.put(type(exc).__name__+': '+str(exc))
|
||||
|
||||
|
||||
class ObjectiveTests(unittest.TestCase):
|
||||
def test_exact_old_peak_tie(self):
|
||||
raw=np.array([7,19,42]);xyz=np.array([[0.,0,0],[2.,0,0],[0.,3,0]])
|
||||
truth=np.array([5.,5.,1.]);q=candidates(raw,raw,np.ones(3,dtype=int),xyz,truth)
|
||||
self.assertEqual(q['true_peak_raw_id'],7)
|
||||
for i in range(3):
|
||||
p=np.zeros(3);p[i]=1
|
||||
self.assertEqual(q['distances'][i],last_location(truth,p,raw,xyz))
|
||||
def test_duplicate_order_does_not_vote(self):
|
||||
full=np.array([1,2,3]);ids=np.array([3,1,3,2]);xyz=np.array([[2.,0,0],[0,0,0],[2,0,0],[1,0,0]])
|
||||
q=candidates(full,ids,np.array([50,1,99,2]),xyz,np.array([9.,10,9,8]))
|
||||
np.testing.assert_array_equal(q['indices'],[0,1,2]);np.testing.assert_array_equal(q['distances'],[0,1,2])
|
||||
def test_bad_duplicate_fails(self):
|
||||
with self.assertRaises(ValueError):candidates(np.array([1]),np.array([1,1]),np.array([1,1]),np.zeros((2,3)),np.array([1.,2.]))
|
||||
def test_missing_id_fails(self):
|
||||
with self.assertRaises(ValueError):candidates(np.array([1]),np.array([2]),np.array([1]),np.zeros((1,3)),np.array([1.]))
|
||||
def test_nan_truth_not_undefined(self):
|
||||
with self.assertRaises(ValueError):candidates(np.array([1]),np.array([1]),np.array([1]),np.zeros((1,3)),np.array([np.nan]))
|
||||
def test_gradient_formula(self):
|
||||
r=torch.tensor([.01,-.02,.03],dtype=torch.float64,requires_grad=True);prior=torch.tensor([.7,.8,.9],dtype=torch.float64)
|
||||
d=torch.tensor([2.,1.,0.],dtype=torch.float64);idx=torch.arange(3)
|
||||
loss=location_loss(r,prior,idx,d);loss.backward();p=torch.softmax((r.detach()+prior)/THETA,0)
|
||||
torch.testing.assert_close(r.grad,p*(d-(p*d).sum())/THETA)
|
||||
def test_constant_shift(self):
|
||||
r=torch.tensor([.1,.2],dtype=torch.float64);p=torch.tensor([.7,.8],dtype=torch.float64);d=torch.tensor([2.,0.],dtype=torch.float64)
|
||||
torch.testing.assert_close(location_loss(r,p,torch.arange(2),d),location_loss(r+12,p,torch.arange(2),d))
|
||||
def test_degenerate_gradients_zero(self):
|
||||
for d in (torch.tensor([2.],dtype=torch.float64),torch.tensor([2.,2.,2.],dtype=torch.float64)):
|
||||
r=torch.arange(len(d),dtype=torch.float64,requires_grad=True);location_loss(r,torch.zeros_like(r),torch.arange(len(d)),d).backward()
|
||||
torch.testing.assert_close(r.grad,torch.zeros_like(r))
|
||||
def test_fp64_prior_preserved(self):
|
||||
r=torch.zeros(2,dtype=torch.float32,requires_grad=True);p=torch.tensor([1.,1.+1e-8],dtype=torch.float64);d=torch.tensor([0.,1.],dtype=torch.float64)
|
||||
loss=location_loss(r,p,torch.arange(2),d);self.assertGreater(float(loss),.5);loss.backward();self.assertTrue(torch.isfinite(r.grad).all())
|
||||
with self.assertRaises(ValueError):location_loss(r,p.float(),torch.arange(2),d)
|
||||
def test_original_weight_plus_one_location(self):
|
||||
p=torch.tensor([.2,.4]);t=torch.tensor([.1,.7]);prior=torch.tensor([.7,.6],dtype=torch.float64);d=torch.tensor([0.,3.],dtype=torch.float64)
|
||||
total,reg,loc=objective(p,t,7,2,19,prior,torch.arange(2),d)
|
||||
self.assertEqual(float(reg),float(((p-t)**2).mean()*(14/19)))
|
||||
torch.testing.assert_close(total-reg,.001*loc)
|
||||
def test_no_validation_or_test_role(self):
|
||||
view=dict(folds=[dict(learning=['a'],validation=['b'],test=['c'])])
|
||||
require_learning(view,'a')
|
||||
for case in ('b','c','missing'):
|
||||
with self.assertRaises(PermissionError):require_learning(view,case)
|
||||
def test_worker_only_loads_learning(self):
|
||||
tree=ast.parse((Path(__file__).parent/'phase3_item_c_worker.py').read_text())
|
||||
calls=[n for n in ast.walk(tree) if isinstance(n,ast.Call) and isinstance(n.func,ast.Name) and n.func.id=='load_part']
|
||||
self.assertEqual(len(calls),1);self.assertEqual(calls[0].args[3].value,'learning')
|
||||
|
||||
|
||||
class CapacityTests(unittest.TestCase):
|
||||
def test_visible_reserved_process_counted_once(self):
|
||||
self.assertTrue(admission(snapshot(),[lease(101)],GPUS[1])[0])
|
||||
def test_unseen_reservations_block_fourth(self):
|
||||
self.assertFalse(admission(snapshot(),[lease(102),lease(103)],GPUS[1])[0])
|
||||
def test_foreign_processes_count(self):
|
||||
s=snapshot();s['processes'] += [dict(gpu_uuid=GPUS[1],pid=i,vram=GIB) for i in (102,103)]
|
||||
self.assertEqual(admission(s,[],GPUS[1])[1],'GPU_SLOT_LIMIT')
|
||||
def test_gpu3_single(self):
|
||||
s=snapshot();s['processes']=[dict(gpu_uuid=GPUS[3],pid=1,vram=1)]
|
||||
self.assertFalse(admission(s,[],GPUS[3])[0])
|
||||
def test_gpu0_old_pending(self):
|
||||
s=snapshot();s['old_pending']=True;self.assertEqual(admission(s,[],GPUS[0])[1],'GPU0_RESERVED_FOR_OLD_PENDING')
|
||||
def test_memory_reserves_future_peaks(self):
|
||||
s=snapshot();s['devices'][GPUS[1]]['used']=9*GIB
|
||||
self.assertEqual(admission(s,[lease(101)],GPUS[1])[1],'GPU_HEADROOM')
|
||||
def test_host_ram_reserve(self):
|
||||
s=snapshot();s['ram_available']=30*GIB;self.assertEqual(admission(s,[],GPUS[1])[1],'HOST_RAM_HEADROOM')
|
||||
def test_cpu_headroom(self):
|
||||
s=snapshot();s['cpu_percent']=81;self.assertEqual(admission(s,[],GPUS[1])[1],'HOST_CPU_HEADROOM')
|
||||
def test_phase_reservation_once(self):
|
||||
self.assertEqual(PHASE_RESERVES['seed1'],167200);self.assertLess(sum(PHASE_RESERVES.values()),720000)
|
||||
def test_stale_lease_blocks_without_reclaim(self):
|
||||
import phase3_shared_capacity as m
|
||||
with tempfile.TemporaryDirectory() as tmp,patch.object(m,'SHARED',Path(tmp)):
|
||||
owner=m.process_identity(__import__('os').getpid())
|
||||
stale=dict(token='stale',owner=dict(pid=99999999,created=0))
|
||||
Path(tmp,'registry.json').write_text(json.dumps(dict(version=1,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=owner)},leases={'stale':stale})))
|
||||
decision,reason=m.acquire(Path('/synthetic'),0,'abc')
|
||||
self.assertIsNone(decision);self.assertIn('STALE_LEASE',reason['reason'])
|
||||
self.assertIn('stale',json.loads(Path(tmp,'registry.json').read_text())['leases'])
|
||||
def test_two_admissions_atomic(self):
|
||||
import phase3_shared_capacity as m
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
parent=m.process_identity(__import__('os').getpid())
|
||||
Path(tmp,'registry.json').write_text(json.dumps(dict(version=1,entries={'c':dict(root='/synthetic',contract_sha256='abc',owner=parent)},leases={})))
|
||||
ctx=multiprocessing.get_context('fork');barrier=ctx.Barrier(2);results=ctx.Queue()
|
||||
ps=[ctx.Process(target=concurrent_admit,args=(tmp,barrier,results)) for _ in range(2)]
|
||||
for p in ps:p.start()
|
||||
for p in ps:p.join(15)
|
||||
try:
|
||||
self.assertTrue(all(p.exitcode==0 for p in ps))
|
||||
self.assertEqual(sorted([results.get(timeout=1),results.get(timeout=1)]),[False,True])
|
||||
finally:
|
||||
for p in ps:
|
||||
if p.is_alive():p.kill();p.join()
|
||||
|
||||
|
||||
class BudgetTests(unittest.TestCase):
|
||||
def setUp(self):self.c=dict(gpu_cap=205200,postfit_gpu_reserve=7200,preparation_charge_gpu_seconds=600,runaway_gpu_seconds=60000)
|
||||
def test_initial_inclusive_cap(self):
|
||||
p=project(self.c,{i:60 for i in range(5)},{},{})
|
||||
self.assertEqual(p['projected_gpu_seconds'],205200);self.assertEqual(p['status'],'PASS')
|
||||
def test_pending_wait_not_free(self):
|
||||
p=project(self.c,{i:50000 for i in range(5)},{},{})
|
||||
self.assertEqual(p['status'],'STOP_BUDGET')
|
||||
def test_projected_fold_above_runaway_not_alone_stop(self):
|
||||
h={0:[dict(epoch=1,train_epoch_seconds=220)]}
|
||||
p=project(self.c,{i:1000 for i in range(5)},h,{1:{},2:{},3:{}})
|
||||
self.assertGreater(p['folds']['0']['projected'],60000);self.assertEqual(p['status'],'PASS')
|
||||
def test_actual_runaway_stops(self):
|
||||
self.assertEqual(project(self.c,{i:60000 if i==0 else 1 for i in range(5)},{},{})['status'],'STOP_BUDGET')
|
||||
def test_negative_nan_cost(self):
|
||||
for v in (-1,float('nan')):
|
||||
with self.assertRaises(ValueError):project(self.c,{i:v for i in range(5)},{},{})
|
||||
def test_incomplete_epoch_history(self):
|
||||
with self.assertRaises(ValueError):project(self.c,{i:60 for i in range(5)},{0:[dict(epoch=2,train_epoch_seconds=10)]},{})
|
||||
def test_failed_worker_late_memory_peak(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
p=Path(tmp)/'failure.json';p.write_text(json.dumps(dict(peak_reserved_bytes=3*GIB,peak_allocated_bytes=GIB)))
|
||||
tracker=PeakRecords();tracker.scan(tmp)
|
||||
self.assertGreater(max(tracker.maxima.values()),2*GIB)
|
||||
p.write_text(json.dumps(dict(peak_reserved_bytes=1)));tracker.scan(tmp)
|
||||
self.assertEqual(tracker.maxima['peak_reserved_bytes'],3*GIB)
|
||||
|
||||
|
||||
class QueueSyntheticTests(unittest.TestCase):
|
||||
def exercise(self,failed=False,admission_errors=0):
|
||||
import phase3_item_c_queue as q
|
||||
from contextlib import ExitStack
|
||||
from phase2_h1_worker_v2 import save
|
||||
with tempfile.TemporaryDirectory() as tmp, ExitStack() as stack:
|
||||
root=Path(tmp);save(root/'contract.json',{'synthetic':True})
|
||||
c=dict(gpu_cap=205200,postfit_gpu_reserve=7200,preparation_charge_gpu_seconds=600,runaway_gpu_seconds=60000)
|
||||
class FakeProcess:
|
||||
def __init__(self,cmd,**kw):
|
||||
self.pid=999991;self.returncode=1 if failed else 0
|
||||
i=int(cmd[3]);jr=root/'jobs'/str(i);(jr/'worker').mkdir()
|
||||
save(jr/'complete.json',dict(status='STOPPED_OR_FAILED' if failed else 'PASS'))
|
||||
checkpoint=jr/'worker/final.pt';checkpoint.write_bytes(b'synthetic')
|
||||
save(jr/'worker/result.json',dict(fold=i,checkpoint=str(checkpoint),checkpoint_sha256=q.sha(checkpoint)))
|
||||
save(jr/'worker/standardization.json',dict(synthetic=True))
|
||||
def poll(self):return self.returncode
|
||||
def wait(self):return self.returncode
|
||||
def terminate(self):pass
|
||||
def kill(self):pass
|
||||
attempts=[]
|
||||
def acquire(root,i,digest):
|
||||
attempts.append(i)
|
||||
if len(attempts)<=admission_errors:raise TimeoutError('synthetic admission query')
|
||||
return dict(token=str(i),gpu_uuid=GPUS[1],worker_cpus=[16]),dict(synthetic=True)
|
||||
patches=dict(config=lambda r:(c,{}),validate_review=lambda r:None,register_entry=lambda *a:None,
|
||||
acquire=acquire,transfer=lambda *a:None,release=lambda *a:None,
|
||||
process_identity=lambda pid:dict(pid=pid,created=0),freeze=lambda *a:None)
|
||||
for k,v in patches.items():stack.enter_context(patch.object(q,k,v))
|
||||
stack.enter_context(patch.object(q.subprocess,'Popen',FakeProcess))
|
||||
stack.enter_context(patch.object(q.time,'sleep',lambda t:None))
|
||||
stack.enter_context(patch.object(q.signal,'signal',lambda *a:None))
|
||||
stack.enter_context(patch.object(q.psutil,'disk_usage',return_value=type('D',(),{'free':200*GIB})()))
|
||||
stack.enter_context(patch.dict(q.os.environ,PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True'))
|
||||
result=q.run(root);done=q.read(root/'complete.json')
|
||||
if admission_errors>=3:
|
||||
self.assertEqual(result,1);self.assertEqual(done['never_attempted'],[0,1,2,3,4])
|
||||
self.assertEqual(len(attempts),3)
|
||||
elif failed:
|
||||
self.assertEqual(result,1);self.assertEqual(done['never_attempted'],[1,2,3,4]);self.assertFalse((root/'models.json').exists())
|
||||
self.assertEqual(set(done['never_attempted_wait_charges']),{'1','2','3','4'})
|
||||
else:
|
||||
self.assertEqual(result,0);self.assertEqual(len(q.read(root/'models.json')),5)
|
||||
self.assertEqual(done['status'],'TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING')
|
||||
self.assertFalse(done['test_targets_read']);self.assertFalse(done['prediction_started'])
|
||||
def test_all_five_terminate_before_model_freeze(self):self.exercise()
|
||||
def test_failure_stops_without_retry(self):self.exercise(failed=True)
|
||||
def test_two_failed_admission_samples_then_success(self):self.exercise(admission_errors=2)
|
||||
def test_third_failed_admission_stops(self):self.exercise(admission_errors=3)
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user