Publish fixed item1b full100epoch training entry and acceptance for Assist review
This commit is contained in:
1 parent
2dcb690ac2
commit
ffd5c92958
27 files changed
+13642
No files matched your search
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"status": "FAIL",
|
||||
"returncode": 1,
|
||||
"wall_seconds": 4.102203090908006,
|
||||
"cpu_seconds": 4.0826649999999995,
|
||||
"synthetic_only": true,
|
||||
"arrays_decoded": false,
|
||||
"models_loaded": false,
|
||||
"gpu_query_or_workload": false
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"returncode": 0,
|
||||
"wall_seconds": 4.095988018903881,
|
||||
"cpu_seconds": 4.083137,
|
||||
"synthetic_only": true,
|
||||
"arrays_decoded": false,
|
||||
"models_loaded": false,
|
||||
"gpu_query_or_workload": false,
|
||||
"prior_failed_log": "attempt1.log",
|
||||
"prior_error": "new wrapper lacked GIB import; fixed before freeze"
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
test_actual_runaway_stops (test_phase3_item1b.BudgetTests) ... ok
|
||||
test_incomplete_history_rejected (test_phase3_item1b.BudgetTests) ... ok
|
||||
test_initial_budget_includes_postfit (test_phase3_item1b.BudgetTests) ... ok
|
||||
test_late_validation_max_used (test_phase3_item1b.BudgetTests) ... ok
|
||||
test_no_single_fold_projection_gate (test_phase3_item1b.BudgetTests) ... ok
|
||||
test_remaining_validation_charged (test_phase3_item1b.BudgetTests) ... ok
|
||||
test_failure_no_retry_or_test (test_phase3_item1b.QueueTests) ... ok
|
||||
test_five_fits_before_screen_but_no_test (test_phase3_item1b.QueueTests) ... ok
|
||||
test_complete_warmup_and_decay (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_exact_endpoints (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_incomplete_test_not_confirmation (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_invalid_epoch (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_no_seed_no_decision (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_nonfinite_comparisons_rejected (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_not_original_midpoint (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_only_fixed_last_checkpoint (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_screen_exact_boundary (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_test_strict_boundary_and_seed (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_worker_original_train_step_and_seed0 (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_mixed_pending_c_reservation_blocks_1b (test_phase3_item1b.SharedWrapperTests) ... ok
|
||||
test_mixed_two_visible_allows_only_one_new (test_phase3_item1b.SharedWrapperTests) ... ERROR
|
||||
test_shared_c_and_1b_registration_independent (test_phase3_item1b.SharedWrapperTests) ... ok
|
||||
test_wrapper_changes_only_key_cap_gpu_set (test_phase3_item1b.SharedWrapperTests) ... ok
|
||||
|
||||
======================================================================
|
||||
ERROR: test_mixed_two_visible_allows_only_one_new (test_phase3_item1b.SharedWrapperTests)
|
||||
----------------------------------------------------------------------
|
||||
Traceback (most recent call last):
|
||||
File "/home/frank14f/optane/MeltMind/tools/test_phase3_item1b.py", line 121, in test_mixed_two_visible_allows_only_one_new
|
||||
l,_=m.acquire(root,1,'new');self.assertIsNotNone(l);self.assertEqual(l['state'],'RESERVED');self.assertEqual(l['gpu_uuid'],GPUS[1])
|
||||
File "/home/frank14f/optane/MeltMind/tools/phase3_item1b_capacity.py", line 62, in acquire
|
||||
vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time())
|
||||
NameError: name 'GIB' is not defined
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 23 tests in 3.221s
|
||||
|
||||
FAILED (errors=1)
|
||||
@@ -0,0 +1,28 @@
|
||||
test_actual_runaway_stops (test_phase3_item1b.BudgetTests) ... ok
|
||||
test_incomplete_history_rejected (test_phase3_item1b.BudgetTests) ... ok
|
||||
test_initial_budget_includes_postfit (test_phase3_item1b.BudgetTests) ... ok
|
||||
test_late_validation_max_used (test_phase3_item1b.BudgetTests) ... ok
|
||||
test_no_single_fold_projection_gate (test_phase3_item1b.BudgetTests) ... ok
|
||||
test_remaining_validation_charged (test_phase3_item1b.BudgetTests) ... ok
|
||||
test_failure_no_retry_or_test (test_phase3_item1b.QueueTests) ... ok
|
||||
test_five_fits_before_screen_but_no_test (test_phase3_item1b.QueueTests) ... ok
|
||||
test_complete_warmup_and_decay (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_exact_endpoints (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_incomplete_test_not_confirmation (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_invalid_epoch (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_no_seed_no_decision (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_nonfinite_comparisons_rejected (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_not_original_midpoint (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_only_fixed_last_checkpoint (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_screen_exact_boundary (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_test_strict_boundary_and_seed (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_worker_original_train_step_and_seed0 (test_phase3_item1b.ScheduleTests) ... ok
|
||||
test_mixed_pending_c_reservation_blocks_1b (test_phase3_item1b.SharedWrapperTests) ... ok
|
||||
test_mixed_two_visible_allows_only_one_new (test_phase3_item1b.SharedWrapperTests) ... ok
|
||||
test_shared_c_and_1b_registration_independent (test_phase3_item1b.SharedWrapperTests) ... ok
|
||||
test_wrapper_changes_only_key_cap_gpu_set (test_phase3_item1b.SharedWrapperTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 23 tests in 3.335s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"binding_sha256": "13a5142ee9d1d635b9d7f2979c2cd392de8feda8fcca722fcdc469a2d9927873",
|
||||
"reviewed_commit": "45b5df30098f74dc3c93a2f54365d7d866b810e0",
|
||||
"contract_sha256": "d33f036adc48cde904fbb6bb0f615a5464f94b039906d0b45a4dbd15f0681bd3",
|
||||
"implementation_proof_sha256": "73f28ef6d9991ef36f9e399b39ff24d849a656ee3caa8b3230e3c61732839f29",
|
||||
"comparison_contract_sha256": "648a99549689ae6aba02085d59fc5b9a10d350e48832c383570569a657d24aae",
|
||||
"code_count": 60,
|
||||
"source_count": 188,
|
||||
"started": false,
|
||||
"review_adopted": false
|
||||
}
|
||||
+2962
File diff suppressed because it is too large.
Load diff
+3212
File diff suppressed because it is too large.
Load diff
+13
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"status": "PREPARED_PENDING_ASSIST_REVIEW",
|
||||
"contract_sha256": "4a316dbb251cdbfd7f4ed75ac9d82dd96ebcf15e43c26bf5fa1926997c589cd5",
|
||||
"code_files": 60,
|
||||
"source_files": 188,
|
||||
"baseline_models": 5,
|
||||
"opaque_model_hashes_only": true,
|
||||
"metadata_cpu_seconds": 0.352119263,
|
||||
"labels_decoded": false,
|
||||
"arrays_decoded": false,
|
||||
"models_loaded": false,
|
||||
"new_fits_started": false
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"old_active_code": [
|
||||
{
|
||||
"root": "/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18",
|
||||
"code_pins": 52,
|
||||
"status": "PASS"
|
||||
},
|
||||
{
|
||||
"root": "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
|
||||
"code_pins": 51,
|
||||
"status": "PASS"
|
||||
},
|
||||
{
|
||||
"root": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18",
|
||||
"code_pins": 27,
|
||||
"status": "PASS"
|
||||
}
|
||||
],
|
||||
"changes": [
|
||||
{
|
||||
"path": "/home/frank14f/optane/MeltMind/tools/phase3_item1b_capacity.py",
|
||||
"before": "1607e6fc663516c16ba1f59fd4fe658206d296cc871d2d4dab7b6ba67b642fd2",
|
||||
"after": "771faad0095712fd4fc8131ddcfabaa3901141592d2b055ea2faadf8ea3d3edf"
|
||||
},
|
||||
{
|
||||
"path": "/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py",
|
||||
"before": "e56abbca1007a8a58f1cc6b075f72c89898dd0c94a2965a73bd23cf6a3f92c99",
|
||||
"after": "4e25effe8326ff8c7c3f751590315040209af054b32b0505dc3c6b001589db32"
|
||||
}
|
||||
],
|
||||
"new_code": 60,
|
||||
"sources": 188,
|
||||
"unix": 1795011452.0254173
|
||||
}
|
||||
@@ -779,3 +779,13 @@ sole service meltmind-phase3-item-c-20261118.service,Restart=no,coordinator1
|
||||
固定2ddecef3/0e853319、实际PASS e30612ea、review freeze0b17ba94不变;新52/旧78pins核匹配。总205200含post7200/CPU3600,phase3总720000,Track A独立、旧失败账不重复;不重试/不改active pins/不自动postfit。实际启动已在1795010745报Claude/TRON。首轮后续检查另见artifacts/phase3-item-c-launch-2026-11-18/first-epochs-live-audit.json,不把启动当科学结果。
|
||||
|
||||
Claude1795010507/0666已实读:现在准备1b独立入口,原spec55ffc6f9全文归档并核SHA;五折可先训练100完整轮,fold1 validation筛选改为判决门,等seed1同域参照;只有筛选PASS才允许独立事前审阅后的test postfit,失败则五模型归档不读test。c/shared原运行pins不动。1b目前仅实现准备,不是已审/已启动。下方c“等待人类计算授权/未启动”均为历史。
|
||||
|
||||
|
||||
## 最新 item 1b 入口已固定,待 Assist 实际审阅(2026-11-18)
|
||||
|
||||
Claude1795010666五折先训练/筛选仅作测试判决门已写入独立合同。100完整轮、seed0、原H2/19列/诊断保持;新88560GPU秒内含7200post预留,phase3累计720000与共享容量规则不变。仅新1b包装的登记键/帽/GPU0–2不同,原c/shared及seed1/TrackA共130代码pins均核PASS。
|
||||
|
||||
implementation 45b5df30098f74dc3c93a2f54365d7d866b810e0,contract d33f036adc48cde904fbb6bb0f615a5464f94b039906d0b45a4dbd15f0681bd3,proof 73f28ef6d9991ef36f9e399b39ff24d849a656ee3caa8b3230e3c61732839f29,comparison 648a99549689ae6aba02085d59fc5b9a10d350e48832c383570569a657d24aae,binding 13a5142ee9d1d635b9d7f2979c2cd392de8feda8fcca722fcdc469a2d9927873,60code/188source。23CPU合成PASS;首次NameError及原未冻结metadata保留。config/本地五字段绑定核验PASS,不是Assist PASS;无review采用/started/训练/预测/评分。唯一根/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18,报告docs/phase3-item1b-entry-2026-11-18.md。
|
||||
|
||||
c原队列继续,最新读取1795011458:4active/1pending,无terminal,投影163798.657995/205200秒PASS。后续以真实回执为准,不重复启动或修改pins。用户直接计算授权已归档,无需再问。审阅请求须TRON原生送达Assist并收到其确认;Slack只作Claude/公共记录,不冒称已送达或已唤醒其他Astra。
|
||||
|
||||
@@ -1,3 +1,13 @@
|
||||
## 最新 item 1b 入口已固定,待 Assist 实际审阅(2026-11-18)
|
||||
|
||||
Claude1795010666五折先训练/筛选仅作测试判决门已写入独立合同。100完整轮、seed0、原H2/19列/诊断保持;新88560GPU秒内含7200post预留,phase3累计720000与共享容量规则不变。仅新1b包装的登记键/帽/GPU0–2不同,原c/shared及seed1/TrackA共130代码pins均核PASS。
|
||||
|
||||
implementation 45b5df30098f74dc3c93a2f54365d7d866b810e0,contract d33f036adc48cde904fbb6bb0f615a5464f94b039906d0b45a4dbd15f0681bd3,proof 73f28ef6d9991ef36f9e399b39ff24d849a656ee3caa8b3230e3c61732839f29,comparison 648a99549689ae6aba02085d59fc5b9a10d350e48832c383570569a657d24aae,binding 13a5142ee9d1d635b9d7f2979c2cd392de8feda8fcca722fcdc469a2d9927873,60code/188source。23CPU合成PASS;首次NameError及原未冻结metadata保留。config/本地五字段绑定核验PASS,不是Assist PASS;无review采用/started/训练/预测/评分。唯一根/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18,报告docs/phase3-item1b-entry-2026-11-18.md。
|
||||
|
||||
c原队列继续,最新读取1795011458:4active/1pending,无terminal,投影163798.657995/205200秒PASS。后续以真实回执为准,不重复启动或修改pins。用户直接计算授权已归档,无需再问。审阅请求须TRON原生送达Assist并收到其确认;Slack只作Claude/公共记录,不冒称已送达或已唤醒其他Astra。
|
||||
|
||||
---
|
||||
|
||||
## 最新 item c 已获直接用户授权并实际运行(2026-11-18 22:03)
|
||||
|
||||
用户在本Melt会话对一次合并phase3计算问题直接答“批准按上述范围执行”;原问题/答复存artifacts/phase3-item-c-launch-2026-11-18/direct-user-authorization.json。范围c≤57h、1b≤25h、b≤50h及Track B,GPU0–2共享/GPU3单任务、phase3累计≤200GPUh;各入口仍需自己的固定合同、预算分配和实际审阅。后续不再逐项重复问用户。原自动审批拒绝保留,同一未改脚本携新直接证据重审允许,无旁路。
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
# Phase 3 item 1b:完整短日程独立训练入口
|
||||
|
||||
依据 Mold 规格 `55ffc6f9320f6b6f03244acd8548d5037a6efa43d3ce8c48f1b89bb0cb4b5e58`、Claude `1795010507.291259` / `1795010666.724069` 和用户在本会话的直接执行授权,准备唯一根 `/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18`。这是待 Assist 实际入口审阅的实现,不是已启动或已取得科学结果。
|
||||
|
||||
研究问题是原 H2 在完整缩短训练日程下的效用,属于更新次数和学习率日程的联合处理,不能分开归因。原 19 列、完整 learning pool、五折、seed0、初始化、逐轮排列、MSE、AdamW/衰减/裁剪均沿用。五折全部新训练 100 轮,只保存最后第 100 轮;前 5 轮线性 warmup,之后 95 轮余弦降至 1e-5。原 200 轮同折 seed0 模型只作配对参照,不复用权重,也不取其中间轮。
|
||||
|
||||
Claude 0666 仅改变调度:五折先训练,fold1 validation 筛选仍是进入测试的判决门。筛选采用同查询域 component-equal relative L2,要求 Δshort≤−0.001 且 |Δshort|>|Δseed1−seed0|;缺 seed1 同域参照等待,不另跑 seed。原每 10 轮 full-node validation MSE 仅诊断,不选择模型,也不是该筛选指标。训练入口不会启动筛选或测试预测。后续独立固定筛选/评分实现;只有筛选 PASS 并完成事前审阅才可一次读取测试集,筛选失败则五个模型归档不作测试评分。
|
||||
|
||||
测试资格仍是固定 552 件/204 组件,2000 次配对组件 bootstrap、seed2026111408、双侧95%区间;Δshort 区间上界须严格低于 −0.001 且点效应绝对值严格超过同域 pooled seed 差,等号失败,缺折不确认。保留逐折、末填/Rg、tau、负预测及可用 E_tail 报告。该语料已用于开发,不能称新家族确认;一个 seed 差不代表全部随机性。
|
||||
|
||||
预算固定 88560 GPU 秒(24.6小时,Claude `1795007580.895539`),包含后处理7200 GPU秒/3600 CPU秒预留、准备保守记600 GPU秒、每折60秒setup及从队列开始计入的等待。每折实际30000秒失控帽;总成本投影按剩余完整100轮训练及10次诊断的保守时耗乘1.5并加cleanup,不设逐折投影停止帽。首轮门和持续总帽失败即停,零重试。phase3累计720000秒,Track A独立,已有失败账不重复。
|
||||
|
||||
共享准入直接复用 c 已获独立 PASS 的固定 `phase3_shared_capacity.py`。新适配器只改变登记键为item1b、预算88560和候选GPU0–2,其余准入/registry v2/RESERVED与GUARD_BOUND语义/transfer/release保持原函数;不修改 c、seed1、Track A 运行源码。GPU0仍受旧待派队列规则限制,GPU3维持单任务。GPU0–2每卡最多3个真实进程或未落地预留;原GPU/RAM余量、独占worker CPU、进程身份及退出竞态限制不变。
|
||||
|
||||
23项CPU合成检查通过,包括精确日程、最后轮选择、筛选/测试严格边界、原train_step与诊断策略、预算中诊断成本、共享包装差异限定、c与1b混合预留计数、五折训练先于判决以及失败不重试。首次合成检查的缺失GIB导入错误及修复后日志均保留;没有读真实数值数组、加载模型权重或运行GPU。固定绑定将给出完整commit/contract/proof/comparison/code五字段;Assist实际PASS前不启动。
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,68 @@
|
||||
{
|
||||
"reviewed_commit": "45b5df30098f74dc3c93a2f54365d7d866b810e0",
|
||||
"contract_sha256": "d33f036adc48cde904fbb6bb0f615a5464f94b039906d0b45a4dbd15f0681bd3",
|
||||
"implementation_proof_sha256": "73f28ef6d9991ef36f9e399b39ff24d849a656ee3caa8b3230e3c61732839f29",
|
||||
"comparison_contract_sha256": "648a99549689ae6aba02085d59fc5b9a10d350e48832c383570569a657d24aae",
|
||||
"code_sha256": {
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_capacity.py": "771faad0095712fd4fc8131ddcfabaa3901141592d2b055ea2faadf8ea3d3edf",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "4e25effe8326ff8c7c3f751590315040209af054b32b0505dc3c6b001589db32",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_guard.py": "08b7d8ab2ac773fab8b749f9f61af259254422128e9634f8b715c81efce2651f",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_queue.py": "31dfd71369c63512c1d6fe176fd22e8db646520c92da533bb5530ca3843c388d",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_schedule.py": "06cb97594c3a87f42f712dcde6cec8c2d90e0b50a89a6be7954bdd2f5c18d5de",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_worker.py": "b781e56c519401bd38af6a522deb24506cba6993d2af008f85ea4e1715f1d9c6",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1b.py": "c7bc9f3b7cf4137f52e1efad19dcb7045282d4e9a9c1aae50f066c9e488f7c01",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1b.py": "b4d78f4384651711fe83ff00eb6719c6c61664699d801ecb34978b9546439da9",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_common.py": "4e85f5d211c4d3e2188422aa4c0cefa40dcfa14ef32e90894ed3412de1fd1d12",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_data.py": "fe3aa3e023f22f4fe55cd47e7ffa677b58f52a041867ac05620cbd329d913cb6",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_guard.py": "7144fa77ffe61a2a3c92c050234e370ef4dfeb0e31fafb419f752f969737ff1a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_loss.py": "79e291866bda300ecdc5577c62903d34abeec14f5cc36db05e346f9ffd64c24b",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_queue.py": "a04879cc0169ecb523a88360cddca1e234d91b3adbc20b3ec31d02461ed767eb",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_worker.py": "0f8f35e1422021b2a4f097c2e62267fb935fe4658fd0ee8ccd251609a4062ea7",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_c.py": "782c415b1f536eb2b85aa2c9c4a9a0dc278f9b7134e8a83e2a6af8199396c53a",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "588c646101be0d50f505f9a12a1c98361aa3437c8c594eecb7321831f717a741",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "188565801f6901d0192f79aec0889e632501e672a4149d76d1835efbb9b45447",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_c.py": "3115aa9049974eac86657aa4cee7b5d716f6ee93268555cc86242029ebefce4c",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5",
|
||||
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"contract_sha256": "d33f036adc48cde904fbb6bb0f615a5464f94b039906d0b45a4dbd15f0681bd3",
|
||||
"proof_sha256": "73f28ef6d9991ef36f9e399b39ff24d849a656ee3caa8b3230e3c61732839f29"
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
{
|
||||
"commit": "45b5df30098f74dc3c93a2f54365d7d866b810e0",
|
||||
"files": {
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_capacity.py": "771faad0095712fd4fc8131ddcfabaa3901141592d2b055ea2faadf8ea3d3edf",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "4e25effe8326ff8c7c3f751590315040209af054b32b0505dc3c6b001589db32",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_guard.py": "08b7d8ab2ac773fab8b749f9f61af259254422128e9634f8b715c81efce2651f",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_queue.py": "31dfd71369c63512c1d6fe176fd22e8db646520c92da533bb5530ca3843c388d",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_schedule.py": "06cb97594c3a87f42f712dcde6cec8c2d90e0b50a89a6be7954bdd2f5c18d5de",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_worker.py": "b781e56c519401bd38af6a522deb24506cba6993d2af008f85ea4e1715f1d9c6",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1b.py": "c7bc9f3b7cf4137f52e1efad19dcb7045282d4e9a9c1aae50f066c9e488f7c01",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1b.py": "b4d78f4384651711fe83ff00eb6719c6c61664699d801ecb34978b9546439da9",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_common.py": "4e85f5d211c4d3e2188422aa4c0cefa40dcfa14ef32e90894ed3412de1fd1d12",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_data.py": "fe3aa3e023f22f4fe55cd47e7ffa677b58f52a041867ac05620cbd329d913cb6",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_guard.py": "7144fa77ffe61a2a3c92c050234e370ef4dfeb0e31fafb419f752f969737ff1a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_loss.py": "79e291866bda300ecdc5577c62903d34abeec14f5cc36db05e346f9ffd64c24b",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_queue.py": "a04879cc0169ecb523a88360cddca1e234d91b3adbc20b3ec31d02461ed767eb",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_worker.py": "0f8f35e1422021b2a4f097c2e62267fb935fe4658fd0ee8ccd251609a4062ea7",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_c.py": "782c415b1f536eb2b85aa2c9c4a9a0dc278f9b7134e8a83e2a6af8199396c53a",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "588c646101be0d50f505f9a12a1c98361aa3437c8c594eecb7321831f717a741",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "188565801f6901d0192f79aec0889e632501e672a4149d76d1835efbb9b45447",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_c.py": "3115aa9049974eac86657aa4cee7b5d716f6ee93268555cc86242029ebefce4c",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5",
|
||||
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/contract.json": "d33f036adc48cde904fbb6bb0f615a5464f94b039906d0b45a4dbd15f0681bd3",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/comparison-contract.json": "648a99549689ae6aba02085d59fc5b9a10d350e48832c383570569a657d24aae",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/preparation-audit.json": "85124401f46bd5e1f266cbef92e8be05cf4ad5d180c079bff03b2fe63e2963cf",
|
||||
"/home/frank14f/optane/MeltMind/docs/phase3-item1b-entry-2026-11-18.md": "fc9a15c999e55ec62f1efcfa85247eaba5fd56089ad5e08ef123c2c04316a251",
|
||||
"/home/frank14f/optane/MeltMind/references/2026-11-18/phase3-item-1b/mold-short-schedule-spec.md": "55ffc6f9320f6b6f03244acd8548d5037a6efa43d3ce8c48f1b89bb0cb4b5e58",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-launch-2026-11-18/direct-user-authorization.json": "84c315fda1d3c35c36603a7f92443af2c5b91cc03a6fe848edf59f18dc810a81",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-acceptance-2026-11-18/attempt1.log": "b31da657f9c83d3d4267712b60b2ab62a38ef8ac8456f9a86122887dbbf69bff",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-acceptance-2026-11-18/acceptance-attempt2.json": "d47b18b2b399d19c497faa8df84b5c274f49b1abd05fcf161d09cb28d1c2ebb0",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-acceptance-2026-11-18/acceptance-attempt1.json": "9aafca97f66d6cd689f5d2489db2be7d7f7861788b689953c9477eafd0c7565b",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-acceptance-2026-11-18/attempt2.log": "8e9b54b91437541175b96b4f7d595f305f84236b40669c203e20b499d4a45df9",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-acceptance-2026-11-18/pre-freeze-pin-audit.json": "50048b6c1968a113f8a65658334e67b6bd0a1eb1f31a71019679fce529353ac1",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-acceptance-2026-11-18/initial-unfrozen-metadata/preparation-audit.json": "e2063a0b20812a09df79c79783aadeb9544ffd8715d42e9ea2c7a911c1334d43",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-acceptance-2026-11-18/initial-unfrozen-metadata/contract.json": "4a316dbb251cdbfd7f4ed75ac9d82dd96ebcf15e43c26bf5fa1926997c589cd5",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-acceptance-2026-11-18/initial-unfrozen-metadata/comparison-contract.json": "648a99549689ae6aba02085d59fc5b9a10d350e48832c383570569a657d24aae"
|
||||
},
|
||||
"git_paths": {
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "tools/freeze_g1_components.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "tools/gre24_models.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "tools/phase2_gate_outcome_review.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "tools/phase2_h1_worker_v2.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "tools/phase2_h2_data.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "tools/phase2_h2_model.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "tools/phase2_h2_pilot_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "tools/phase2_home_freeze_io.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "tools/phase2_local_freeze_io.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "tools/phase2_residual_math.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_capacity.py": "tools/phase3_item1b_capacity.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "tools/phase3_item1b_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_guard.py": "tools/phase3_item1b_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_queue.py": "tools/phase3_item1b_queue.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_schedule.py": "tools/phase3_item1b_schedule.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_worker.py": "tools/phase3_item1b_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "tools/phase3_seed1_v2_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "tools/phase3_shared_capacity.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1b.py": "tools/prepare_phase3_item1b.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "tools/s2_gpu_idle_gate.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "tools/s2_main_memory_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/__init__.py": "tools/s2_models/__init__.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/attention.py": "tools/s2_models/attention.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/blocks.py": "tools/s2_models/blocks.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "tools/s2_models/features.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "tools/s2_models/network.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/normalization.py": "tools/s2_models/normalization.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "tools/s2_models/objectives.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "tools/s2_pilot_train.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "tools/s2_precision_continuation_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "tools/s2_recovery_gpu_supervisor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "tools/s2_recovery_monitoring.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1b.py": "tools/test_phase3_item1b.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "tools/track_a_h2_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "tools/track_a_h2_prepare.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "tools/track_a_recovery_monitor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "tools/track_a_recovery_supervisor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "tools/phase2_h2_s1_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_common.py": "tools/phase3_item_c_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_data.py": "tools/phase3_item_c_data.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_guard.py": "tools/phase3_item_c_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_loss.py": "tools/phase3_item_c_loss.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_queue.py": "tools/phase3_item_c_queue.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_worker.py": "tools/phase3_item_c_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_c.py": "tools/prepare_phase3_item_c.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "tools/s2_custody_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "tools/s2_custody_statistics.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_c.py": "tools/test_phase3_item_c.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "tools/test_phase2_h2_s2.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "tools/phase2_h2_s2_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "tools/phase2_h2_s2_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "tools/phase2_fixed_io.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "tools/phase2_h2_s2_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "tools/phase2_h2_s1_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "tools/phase2_h2_s2_prepare.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "tools/phase2_h2_prepare.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "tools/phase2_h2_s2_queue.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "tools/phase2_h2_s1_launch.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "tools/phase2_h2_s1_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "docs/phase2-h2-s2-execution-contract-2026-11-17.md",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/contract.json": "home-runs/phase3-item-1b-2026-11-18/contract.json",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/comparison-contract.json": "home-runs/phase3-item-1b-2026-11-18/comparison-contract.json",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/preparation-audit.json": "home-runs/phase3-item-1b-2026-11-18/preparation-audit.json",
|
||||
"/home/frank14f/optane/MeltMind/docs/phase3-item1b-entry-2026-11-18.md": "docs/phase3-item1b-entry-2026-11-18.md",
|
||||
"/home/frank14f/optane/MeltMind/references/2026-11-18/phase3-item-1b/mold-short-schedule-spec.md": "references/2026-11-18/phase3-item-1b/mold-short-schedule-spec.md",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-c-launch-2026-11-18/direct-user-authorization.json": "artifacts/phase3-item-c-launch-2026-11-18/direct-user-authorization.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-acceptance-2026-11-18/attempt1.log": "artifacts/phase3-item1b-acceptance-2026-11-18/attempt1.log",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-acceptance-2026-11-18/acceptance-attempt2.json": "artifacts/phase3-item1b-acceptance-2026-11-18/acceptance-attempt2.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-acceptance-2026-11-18/acceptance-attempt1.json": "artifacts/phase3-item1b-acceptance-2026-11-18/acceptance-attempt1.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-acceptance-2026-11-18/attempt2.log": "artifacts/phase3-item1b-acceptance-2026-11-18/attempt2.log",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-acceptance-2026-11-18/pre-freeze-pin-audit.json": "artifacts/phase3-item1b-acceptance-2026-11-18/pre-freeze-pin-audit.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-acceptance-2026-11-18/initial-unfrozen-metadata/preparation-audit.json": "artifacts/phase3-item1b-acceptance-2026-11-18/initial-unfrozen-metadata/preparation-audit.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-acceptance-2026-11-18/initial-unfrozen-metadata/contract.json": "artifacts/phase3-item1b-acceptance-2026-11-18/initial-unfrozen-metadata/contract.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-acceptance-2026-11-18/initial-unfrozen-metadata/comparison-contract.json": "artifacts/phase3-item1b-acceptance-2026-11-18/initial-unfrozen-metadata/comparison-contract.json"
|
||||
},
|
||||
"remote_published": false,
|
||||
"local_repository": "/tmp/meltmind-phase2-local-fixed-20261117"
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"status": "PREPARED_PENDING_ASSIST_REVIEW",
|
||||
"contract_sha256": "d33f036adc48cde904fbb6bb0f615a5464f94b039906d0b45a4dbd15f0681bd3",
|
||||
"code_files": 60,
|
||||
"source_files": 188,
|
||||
"baseline_models": 5,
|
||||
"opaque_model_hashes_only": true,
|
||||
"metadata_cpu_seconds": 0.352119263,
|
||||
"labels_decoded": false,
|
||||
"arrays_decoded": false,
|
||||
"models_loaded": false,
|
||||
"new_fits_started": false,
|
||||
"pre_freeze_changes": [
|
||||
{
|
||||
"path": "/home/frank14f/optane/MeltMind/tools/phase3_item1b_capacity.py",
|
||||
"before": "1607e6fc663516c16ba1f59fd4fe658206d296cc871d2d4dab7b6ba67b642fd2",
|
||||
"after": "771faad0095712fd4fc8131ddcfabaa3901141592d2b055ea2faadf8ea3d3edf"
|
||||
},
|
||||
{
|
||||
"path": "/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py",
|
||||
"before": "e56abbca1007a8a58f1cc6b075f72c89898dd0c94a2965a73bd23cf6a3f92c99",
|
||||
"after": "4e25effe8326ff8c7c3f751590315040209af054b32b0505dc3c6b001589db32"
|
||||
}
|
||||
],
|
||||
"synthetic_tests": 23,
|
||||
"synthetic_status": "PASS"
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
# H2 过拟合控制候选|100 epoch 完整学习率日程
|
||||
|
||||
MoldAstra,2026-11-18,响应 h-f8c0e191。已实读 Claude 的符号更正,核对既有数据量规格及 H2 架构稿。本稿只提出一个短日程候选,供 Claude 主审。旧 longer-schedule 请求已撤回;本文不进入现有计算队列。[当前裁量](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794983369229789) · [原 H2 配方](https://drive.google.com/file/d/13_ckwlmo9Qfb5sFzs5pwjla8_4scccMR/view)
|
||||
|
||||
item1 定义 g=E(U)−E(2U)。50% test 从 0.068848977 升到 0.072841268,100% test 从 0.066848549 升到 0.069586623,100% validation 从 0.058409554 升到 0.061272961,故 g 为负。更大数据池的 s_2U 在 validation 为正且区间排零,test 为正但区间跨零。当前分类仍为 UNIDENTIFIED_OR_OVERFIT_SIGNS。一折一种子及更新数/学习率状态共同变化,使结果只能提示值得检验训练后段泛化退化;不能据此确定过拟合机制或宣布数据无用。Track D 原触发尚未满足。
|
||||
|
||||
## 一个候选与固定对照
|
||||
|
||||
选择完整100 epoch 日程,保持原 H2 的100% learning池、网络、输入、cF、节点权重、AdamW及weight decay=10⁻⁴、零读出初始化、seed0和原件序派生规则。训练从同配对初值重新开始,最终固定第100 epoch。学习率预定如下,e为从1起计的本次训练epoch。
|
||||
|
||||
e=1..5 lr=10⁻⁴+(e−1)(10⁻³−10⁻⁴)/4
|
||||
e=6..100 lr=10⁻⁵+(10⁻³−10⁻⁵)[1+cos(π(e−5)/95)]/2
|
||||
|
||||
第5轮达到峰值,第100轮达到最低值。既有200 epoch模型的中间100轮仍处在它自己的衰减轨迹上,所以不能替代本候选。比较识别“更少更新加压缩完整日程”这一配方的效果;不单独识别更新数或学习率的因果作用。此轮不另搜索epoch数、weight decay或早停点。更强正则属于失败后可再提的假设,当前没有第二个候选。
|
||||
|
||||
唯一主对照是同折原 H2 seed0 的200 epoch模型。seed1五折完成后,另取其相对seed0的同域差作为一次训练扰动参照。旧 item1 的按步日程不充作本对照。实现前应把实际已执行的 S2 配方、名单和模型身份核到合同;架构建议稿仅用于说明结构。
|
||||
|
||||
## 先筛选,再按固定规则评价
|
||||
|
||||
建议只在预先指定 fold1 的validation上筛选,使用同学习池训练100 epoch候选。选择fold1是固定的流程选择,不根据哪一折表现最好;其validation可能与已用数据重叠,不称独立新样本。该折原seed0/seed1的validation预测仅在名单、模型和评分域相同时复用。取组件等权 relative L2,Δ短=E短−E_seed0,Δ种子=E_seed1−E_seed0。仅当 Δ短≤−0.001 且 |Δ短|>|Δ种子|,进入五折配方评价;否则记录本配方未入围。缺种子参照先等既有seed1任务交付,不临时开新种子运行。
|
||||
|
||||
若入围,补其余四折并复用fold1候选,只在原外折test上按552件/204组件汇总。原五个validation不能拼成独立样本。逐件配对、组件内件均值、组件等权,沿原2,000次组件bootstrap、seed2026111408、双侧95%。建议最终研究门为
|
||||
|
||||
场差 Δ短 的95%上界 < −0.001,
|
||||
且 pooled |Δ短| > |Δ种子|。
|
||||
|
||||
两个条件同时满足才记“短完整日程达到预设场收益”。等号不过,缺折不判完整五折。另列每折结果、末点/Rg、τ、负预测及可用时的E_tail;若末点退化应明确其用途取舍。这里的0.001即0.10个百分点,是研究收益尺度,尚无工业标定。组件bootstrap没有纳入完整训练随机性,超过一次seed差仍是实用筛选。旧库已参与方向选择,因此成功只支持这个受限基准上的配方改进,不能把五折称为全新数据的独立确认。均值、区间与种子差都保留有符号原值。
|
||||
|
||||
## 成本与解释
|
||||
|
||||
第一个fold1拟合约为其旧200 epoch训练工时的一半,加启动、原固定频率的validation和预测成本。以旧纸面每折6.55 GPU-h的含余量代理粗估,单折约3.3 GPU-h,五折约16.4 GPU-h,另列固定开销;这些是投影,不是本候选实测。正式估计必须用S2各折真实训练、准备、预测账分别替换,首完整epoch后投影;已复用模型只计本次读取/评分。若只过筛选前一步,成本限单折,其他四折不自动启动。总费用和排期由Claude另行裁量,不从原任务额度暗扣。
|
||||
|
||||
比原200 epoch更快且更准,才支持改用短配方;相近精度而更快也值得作为成本结果报告,但不改上述精度判决。若未过门,可能是短日程欠训练、原现象受随机性影响或需要不同正则,现有试验无法逐一辨别。原计划的seed1、末填目标、区域输入等路线仍由既有裁量推进,本稿仅补充候选。
|
||||
@@ -0,0 +1,65 @@
|
||||
"""1b registration wrapper; unchanged reviewed shared admission and leases.
|
||||
|
||||
Only entry key/cap and eligibleGPU0..2 differ from c's registration/acquire.
|
||||
The live c module is imported without mutation, never patched or rewritten.
|
||||
"""
|
||||
import os,time,uuid
|
||||
from pathlib import Path
|
||||
import psutil
|
||||
from phase3_shared_capacity import (registry,PHASE_RESERVES,process_identity,alive,descendants,
|
||||
host_snapshot,admission,GPUS,GIB,transfer,release,validate_lease)
|
||||
|
||||
def register_entry(root, digest):
|
||||
with registry() as d:
|
||||
if 'item1b' in d['entries']:
|
||||
raise FileExistsError('Sole item 1b entry already registered; no resume')
|
||||
if sum(PHASE_RESERVES.values()) > 720000:
|
||||
raise RuntimeError('Whole phase3 reserved cap')
|
||||
d['entries']['item1b'] = dict(root=str(root), contract_sha256=digest, owner=process_identity(os.getpid()),
|
||||
cap=88560, phase_reservations=PHASE_RESERVES, started_unix=time.time())
|
||||
|
||||
|
||||
def acquire(root, fold, digest):
|
||||
with registry() as d:
|
||||
entry = d['entries']['item1b']
|
||||
if entry['root'] != str(root) or entry['contract_sha256'] != digest or not alive(entry['owner']):
|
||||
raise PermissionError('Entry owner/identity')
|
||||
leases = []
|
||||
for l in d['leases'].values():
|
||||
# Normal guard exit can race this observation. Block admission until
|
||||
# its owning coordinator collects it; never reclaim or kill peers.
|
||||
if not alive(l['owner']):
|
||||
return None, dict(reason='STALE_LEASE_AWAIT_OWNER_COLLECTION',token=l['token'])
|
||||
if l['state']=='RESERVED':
|
||||
# owner is a queue with other children, not this job's guard.
|
||||
# One independent slot and full VRAM/RSS remain reserved.
|
||||
leases.append(dict(l,observed_pids=[],observed_rss=0))
|
||||
continue
|
||||
if l['state']!='GUARD_BOUND':
|
||||
raise ValueError('Unknown lease state')
|
||||
try:
|
||||
ids = descendants(l['owner'])
|
||||
except (psutil.NoSuchProcess,RuntimeError):
|
||||
return None, dict(reason='LEASE_EXIT_RACE_AWAIT_OWNER_COLLECTION',token=l['token'])
|
||||
rss=0
|
||||
for pid in ids:
|
||||
try:rss+=psutil.Process(pid).memory_info().rss
|
||||
except psutil.NoSuchProcess:pass
|
||||
leases.append(dict(l, observed_pids=list(ids), observed_rss=rss))
|
||||
s = host_snapshot() # fresh inside inter-entry lock, before making reservation
|
||||
used_cpu = {x for l in leases for x in l['worker_cpus']}
|
||||
cpus = [x for x in range(16,26) if x not in used_cpu and x in os.sched_getaffinity(0)]
|
||||
if not cpus:
|
||||
return None, dict(reason='CPU_RESERVATIONS', snapshot=s)
|
||||
for gpu in GPUS[:3]:
|
||||
ok, reason = admission(s, leases, gpu)
|
||||
if not ok:
|
||||
continue
|
||||
token = uuid.uuid4().hex
|
||||
queue_owner=process_identity(os.getpid())
|
||||
l = dict(token=token, state='RESERVED',root=str(root), fold=fold, contract_sha256=digest, gpu_uuid=gpu,
|
||||
worker_cpus=[cpus[0]], owner=queue_owner,queue_owner=queue_owner,
|
||||
vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time())
|
||||
d['leases'][token] = l
|
||||
return l, dict(reason='ADMITTED', snapshot=s)
|
||||
return None, dict(reason='NO_CAPACITY', snapshot=s)
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Item 1b immutable scope, assignments and conservative total-cost projection."""
|
||||
import json
|
||||
import math
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
from phase2_h1_worker_v2 import checked, sha
|
||||
from phase2_home_freeze_io import verify
|
||||
from phase3_seed1_v2_common import fold_view
|
||||
|
||||
ROOT = Path('/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18')
|
||||
|
||||
|
||||
def read(p):
|
||||
return json.loads(Path(p).read_text())
|
||||
|
||||
|
||||
def config(root):
|
||||
if root.resolve() != ROOT:
|
||||
raise PermissionError('Sole item 1b root')
|
||||
c = read(root/'contract.json')
|
||||
verify(root, 'implementation')
|
||||
expected = dict(approval='1795010666.724069', seed=0, epochs=100, new_fits=5,
|
||||
gpu_cap=88560, postfit_gpu_reserve=7200, postfit_cpu_cap=3600,
|
||||
phase3_gpu_cap=720000, no_retry=True)
|
||||
if any(c.get(k) != v for k,v in expected.items()):
|
||||
raise PermissionError('Fixed item1b scope/budgets')
|
||||
for p,h in {**c['source_sha256'], **c['code_sha256']}.items():
|
||||
checked(p,h)
|
||||
pilot = read(c['pilot_contract'])
|
||||
import numpy as np
|
||||
import torch
|
||||
if np.__version__ != pilot['versions']['numpy'] or torch.__version__ != pilot['versions']['torch']:
|
||||
raise ValueError('Original environment versions')
|
||||
if [j['id'] for j in c['jobs']] != list(range(5)):
|
||||
raise ValueError('Exactly five folds')
|
||||
tests = []
|
||||
for job in c['jobs']:
|
||||
f = fold_view(pilot, job['id'])['folds'][0]
|
||||
tests.extend(f['test'])
|
||||
if job['target_steps'] != 100*len(f['learning']):
|
||||
raise ValueError('Full fixed epochs')
|
||||
for k in ('learning','validation','test','c','clock_sha256','learning_nodes'):
|
||||
if job[k] != f[k]:
|
||||
raise ValueError('Original fold identity changed')
|
||||
if len(tests) != 552 or len(set(tests)) != 552:
|
||||
raise ValueError('Original552 roster')
|
||||
rows=read(c['query_roster_contract'])['rows']
|
||||
if len(rows)!=552 or {r['case'] for r in rows}!=set(tests) or len({r['component_id'] for r in rows})!=204:
|
||||
raise ValueError('552 parts /204 components query roster')
|
||||
for r in rows:
|
||||
p=pilot['parts'][r['case']]
|
||||
if r['v5_cache']['output_sha256']!=p['output_sha256'] or r['nodes']!=p['nodes'] or r['component_id']!=p['clock_group']:
|
||||
raise ValueError('Learning query/geometry/cache identity')
|
||||
if c['screen_fold']!=1 or not c['test_requires_screen_pass'] or c['budget_approval']!='1795007580.895539':
|
||||
raise PermissionError('Original screen retained as decision gate')
|
||||
checked(c['comparison_contract'], c['comparison_contract_sha256'])
|
||||
return c,pilot
|
||||
|
||||
|
||||
def assignment(root,c,index,digest):
|
||||
a = read(checked(root/'jobs'/str(index)/'assignment.json',digest))
|
||||
if a['id'] != index or a['contract_sha256'] != sha(root/'contract.json'):
|
||||
raise PermissionError('Assignment identity')
|
||||
return dict(c['jobs'][index], **{k:a[k] for k in ('gpu_uuid','worker_cpus','token','queue_identity')})
|
||||
|
||||
|
||||
def first_epoch_release(root,index):
|
||||
if (root/'global-stop.json').exists():
|
||||
raise RuntimeError('Global stop')
|
||||
p = root/'jobs'/str(index)/'first-epoch-projection.json'
|
||||
if not p.exists():
|
||||
return False
|
||||
d = read(p)
|
||||
if d['status'] != 'PASS' or d['contract_sha256'] != sha(root/'contract.json'):
|
||||
raise RuntimeError('First-epoch gate not passed')
|
||||
return True
|
||||
|
||||
|
||||
def project(c, charges, histories, terminal):
|
||||
if set(charges) != set(range(5)) or set(histories)-set(charges) or set(terminal)-set(charges):
|
||||
raise ValueError('Cost ledger fold identity')
|
||||
values = {}
|
||||
for i,v in charges.items():
|
||||
if not math.isfinite(v) or v < 0:
|
||||
raise ValueError('Invalid actual cost')
|
||||
if i in terminal:
|
||||
values[i] = v
|
||||
elif histories.get(i):
|
||||
rows = histories[i]
|
||||
e = max(r['epoch'] for r in rows)
|
||||
if sorted(r['epoch'] for r in rows) != list(range(1,e+1)) or e > 100:
|
||||
raise ValueError('Incomplete epoch history')
|
||||
t = max(r['train_epoch_seconds'] for r in rows)
|
||||
if not math.isfinite(t) or t <= 0:
|
||||
raise ValueError('Invalid measured epoch')
|
||||
# Factor1.5 applies to remaining training, never to already charged cost.
|
||||
validation=max(c['jobs'][i]['validation_seconds_bound'],max(r.get('validation_seconds',0) for r in rows))
|
||||
if not math.isfinite(validation) or validation<0:raise ValueError('Invalid validation timing')
|
||||
values[i] = v + 1.5*((100-e)*t+(10-e//10)*validation) + 120
|
||||
initial = (c['gpu_cap']-c['postfit_gpu_reserve']-c['preparation_charge_gpu_seconds'])/5
|
||||
median = statistics.median(values.values()) if values else initial
|
||||
for i in charges:
|
||||
if i not in values:
|
||||
values[i] = max(charges[i]+120, initial, median)
|
||||
actual = c['preparation_charge_gpu_seconds']+sum(charges.values())
|
||||
projected = c['preparation_charge_gpu_seconds']+sum(values.values())+c['postfit_gpu_reserve']
|
||||
ok = projected <= c['gpu_cap'] and actual+c['postfit_gpu_reserve'] <= c['gpu_cap']
|
||||
ok &= all(charges[i] < c['runaway_gpu_seconds'] for i in charges if i not in terminal)
|
||||
return dict(status='PASS' if ok else 'STOP_BUDGET', projected_gpu_seconds=projected,
|
||||
actual_training_and_preparation_gpu_seconds=actual,
|
||||
folds={str(i):dict(charged=charges[i], projected=values[i]) for i in charges},
|
||||
postfit_gpu_reserve=c['postfit_gpu_reserve'], safety_factor=1.5)
|
||||
@@ -0,0 +1,136 @@
|
||||
"""One item 1b attempt, scoped process memory plus failed-worker peak checks."""
|
||||
import ctypes
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from phase3_item1b_common import config,assignment,read
|
||||
from phase3_shared_capacity import validate_lease,alive
|
||||
from track_a_recovery_monitor import Monitor
|
||||
from track_a_recovery_supervisor import supervise,tree_usage,stop_group,disk_usage
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
|
||||
|
||||
def run(queue,index,digest):
|
||||
parent=os.getppid()
|
||||
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGTERM,0,0,0)!=0:
|
||||
raise OSError('PDEATHSIG')
|
||||
requested=[]
|
||||
for sig in (signal.SIGTERM,signal.SIGINT):
|
||||
signal.signal(sig,lambda n,f:requested.append('SIGNAL_'+str(n)))
|
||||
if os.getppid()!=parent:
|
||||
raise RuntimeError('Queue lost before setup')
|
||||
c,_=config(queue);job=assignment(queue,c,index,digest)
|
||||
root=queue/'jobs'/str(index);start=time.monotonic();cpu=time.process_time()
|
||||
original=read(root/'assignment.json');wait=original['pre_dispatch_wait_seconds']
|
||||
remaining=c['runaway_gpu_seconds']-wait-60
|
||||
if remaining<90:
|
||||
raise RuntimeError('No cleanup window before worker')
|
||||
deadline=start+remaining
|
||||
# Queue writes ready only after atomically transferring reservation ownership.
|
||||
while not (root/'lease-ready.json').exists():
|
||||
if requested or not alive(job['queue_identity']) or time.monotonic()-start>30:
|
||||
raise RuntimeError('Lease transfer timeout')
|
||||
time.sleep(.1)
|
||||
validate_lease(job['token'],queue,index,sha(queue/'contract.json'))
|
||||
if os.getppid()!=job['queue_identity']['pid'] or not alive(job['queue_identity']):
|
||||
raise RuntimeError('Pinned queue lost before guard import completed')
|
||||
os.sched_setaffinity(0,[26,27])
|
||||
tracker=PeakRecords();done=threading.Event();monitor=Monitor(root/'monitor-errors.jsonl')
|
||||
reasons=[];receipt=None
|
||||
command=[sys.executable,'tools/phase3_item1b_worker.py',str(queue),str(index),digest]
|
||||
def reason(s):
|
||||
if s not in reasons:reasons.append(s)
|
||||
def workers():
|
||||
result=[]
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:result.append(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
return result
|
||||
def kill():
|
||||
for pid in workers():stop_group(pid)
|
||||
def scan():
|
||||
tracker.scan(root/'worker')
|
||||
if max(tracker.maxima.values())>job['recorded_vram_bytes']:reason('RECORDED_VRAM_LIMIT')
|
||||
def local_watch():
|
||||
while not done.is_set():
|
||||
monitor.sample('recorded_peaks',scan)
|
||||
usage,ok=monitor.sample('tree',lambda:tree_usage(os.getpid()))
|
||||
if ok and (usage[0]>job['cpu_seconds']-30 or usage[1]>job['rss_bytes']):reason('GUARD_TREE_LIMIT')
|
||||
disk,ok=monitor.sample('disk',lambda:disk_usage(root))
|
||||
if ok and disk>job['disk_bytes']:reason('OUTPUT_LIMIT')
|
||||
if time.monotonic()>deadline-30:reason('OWN_ACTUAL_TIME_LIMIT')
|
||||
queue_live,ok=monitor.sample('queue_identity',lambda:alive(job['queue_identity']))
|
||||
if ok and not queue_live:reason('QUEUE_LOST')
|
||||
if (queue/'global-stop.json').exists():reason('GLOBAL_STOP')
|
||||
for s in requested:reason(s)
|
||||
if monitor.failed:reason('MONITOR_FAILED')
|
||||
if reasons:kill()
|
||||
done.wait(.5)
|
||||
def gpu_watch():
|
||||
def sample():
|
||||
text=subprocess.check_output(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,used_gpu_memory','--format=csv,noheader,nounits'],text=True,timeout=10)
|
||||
rows=[]
|
||||
for line in text.strip().splitlines():
|
||||
if not line:continue
|
||||
u,p,m=[x.strip() for x in line.split(',')]
|
||||
if u==job['gpu_uuid']:rows.append(dict(pid=int(p),bytes=int(m)*2**20))
|
||||
if len(rows)>3:reason('HOST_GPU_SLOT_LIMIT')
|
||||
own=set(workers())
|
||||
if sum(r['bytes'] for r in rows if r['pid'] in own)>job['process_vram_bytes']:reason('OWN_GPU_PROCESS_MEMORY_LIMIT')
|
||||
save(root/'gpu-sample.json',dict(unix=time.time(),rows=rows,own_pids=sorted(own)))
|
||||
while not done.is_set():
|
||||
monitor.sample('gpu_processes',sample)
|
||||
if monitor.failed:reason('MONITOR_FAILED')
|
||||
if reasons:kill()
|
||||
done.wait(1)
|
||||
threads=[threading.Thread(target=f,daemon=True) for f in (local_watch,gpu_watch)]
|
||||
try:
|
||||
save(root/'started.json',dict(unix=time.time(),assignment_sha256=digest))
|
||||
for t in threads:t.start()
|
||||
if reasons or requested or time.monotonic()>deadline-60:raise RuntimeError('Prelaunch guard rejected')
|
||||
receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=job['cpu_seconds']-60,
|
||||
job_cpu_seconds=job['cpu_seconds']-60,job_wall_seconds=deadline-time.monotonic()-40,
|
||||
tree_rss_bytes=job['rss_bytes'],disk_bytes=job['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=job['gpu_uuid'])
|
||||
except BaseException as exc:
|
||||
reason(type(exc).__name__+': '+str(exc));kill()
|
||||
finally:
|
||||
done.set()
|
||||
for t in threads:
|
||||
if t.ident is not None:t.join(timeout=12)
|
||||
if any(t.is_alive() for t in threads):reason('MONITOR_DID_NOT_EXIT')
|
||||
try:monitor.retry('final_peaks',scan,.1)
|
||||
except Exception as exc:reason('FINAL_PEAK_SCAN: '+str(exc))
|
||||
for s in requested:reason(s)
|
||||
if monitor.failed:reason('MONITOR_FAILED')
|
||||
p=root/'worker/result.json';result=read(p) if p.exists() else None
|
||||
if not receipt or receipt['status']!='PASS' or receipt['returncode']!=0:reason('WORKER_NOT_PASS')
|
||||
if not result or result.get('status')!='COMPLETE' or result.get('fold')!=index or result.get('completed_epochs')!=100 or result.get('optimizer_steps')!=job['target_steps']:reason('NO_COMPLETE_IDENTITY')
|
||||
if result:
|
||||
from phase2_h1_worker_v2 import checked
|
||||
checked(result['checkpoint'],result['checkpoint_sha256'])
|
||||
if len(tracker.witnesses)!=2:reason('MISSING_RECORDED_PEAKS')
|
||||
elapsed=time.monotonic()-start
|
||||
child=resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
charged_cpu=time.process_time()-cpu+child.ru_utime+child.ru_stime
|
||||
if wait+60+elapsed>c['runaway_gpu_seconds'] or charged_cpu>job['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
|
||||
try:config(queue)
|
||||
except Exception as exc:reason('FINAL_PINS: '+str(exc))
|
||||
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',reasons=reasons,
|
||||
worker=receipt,worker_result_sha256=sha(p) if p.exists() else None,
|
||||
cpu_seconds_charged=charged_cpu,pre_dispatch_wait_seconds=wait,guard_wall_seconds=elapsed,
|
||||
recorded_memory=dict(maxima=tracker.maxima,witnesses=tracker.witnesses,observations_count=len(tracker.observations)),
|
||||
monitor_policy=monitor.policy(),monitor_errors=monitor.records,no_retry=True))
|
||||
return 0 if not reasons else 1
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3]))
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Review-gated five-fold 1b training. Stops at frozen models, never postfit."""
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import psutil
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from phase2_home_freeze_io import freeze
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase3_item1b_common import ROOT,config,read,project
|
||||
from phase3_item1b_capacity import register_entry,acquire,transfer,release,process_identity,alive
|
||||
from track_a_recovery_monitor import Monitor
|
||||
|
||||
|
||||
def run(root):
|
||||
c,_=config(root);validate_review(root)
|
||||
with (root/'queue.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():raise FileExistsError('Sole 1b attempt, no resume')
|
||||
if os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':
|
||||
raise ValueError('Explicit allocator required')
|
||||
if psutil.disk_usage(root).free<100*2**30:raise RuntimeError('Less than100GiB free')
|
||||
register_entry(root,sha(root/'contract.json'))
|
||||
start=time.monotonic();identity=process_identity(os.getpid())
|
||||
save(root/'started.json',dict(unix=time.time(),identity=identity,contract_sha256=sha(root/'contract.json')))
|
||||
(root/'jobs').mkdir(exist_ok=False)
|
||||
active={};terminal={};pending=[1,0,2,3,4];histories={};stops=[];failure=None
|
||||
monitor=Monitor(root/'admission-errors.jsonl')
|
||||
for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,lambda n,f:stops.append('SIGNAL_'+str(n)))
|
||||
def charge(i):
|
||||
return terminal[i]['gpu_seconds_charged'] if i in terminal else 60+time.monotonic()-start
|
||||
def stop(reason):
|
||||
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time()))
|
||||
try:
|
||||
while active or pending:
|
||||
if stops:raise RuntimeError(stops[0])
|
||||
if (root/'global-stop.json').exists():raise RuntimeError('EXTERNAL_GLOBAL_STOP')
|
||||
# Collect exits before granting any further admission.
|
||||
for i,(p,lease,log,owner) in list(active.items()):
|
||||
if p.poll() is None:continue
|
||||
log.close();jr=root/'jobs'/str(i);cp=jr/'complete.json'
|
||||
result=read(cp) if cp.exists() else dict(status='MISSING_GUARD_RECEIPT')
|
||||
worker_identity=jr/'worker/worker-identity.json'
|
||||
if worker_identity.exists() and alive(read(worker_identity)):
|
||||
raise RuntimeError('WORKER_OUTLIVED_GUARD_LEASE_HELD')
|
||||
terminal[i]=dict(id=i,status=result['status'],returncode=p.returncode,
|
||||
gpu_seconds_charged=charge(i),guard_sha256=sha(cp) if cp.exists() else None)
|
||||
release(lease['token'],owner);del active[i]
|
||||
if result['status']!='PASS' or p.returncode!=0:raise RuntimeError('FOLD_FAILED_'+str(i))
|
||||
for i in active:
|
||||
folder=root/'jobs'/str(i)/'worker/history'
|
||||
histories[i]=[read(p) for p in sorted(folder.glob('*.json'))]
|
||||
charges={i:charge(i) for i in range(5)}
|
||||
projection=project(c,charges,histories,terminal)
|
||||
save(root/'latest-cost-projection.json',projection)
|
||||
if projection['status']!='PASS':raise RuntimeError('TOTAL_PROJECTION_OR_ACTUAL_CAP')
|
||||
for i in active:
|
||||
gate=root/'jobs'/str(i)/'first-epoch-projection.json'
|
||||
if histories.get(i) and not gate.exists():
|
||||
save(gate,dict(status='PASS',contract_sha256=sha(root/'contract.json'),projection=projection))
|
||||
save(root/'live-progress.json',dict(unix=time.time(),stage='ITEM_1B_TRAINING',active=list(active),pending=pending,
|
||||
terminal=list(terminal.values()),charged_training_gpu_seconds=projection['actual_training_and_preparation_gpu_seconds'],postfit_started=False))
|
||||
if pending:
|
||||
i=pending[0]
|
||||
decision,ok=monitor.sample('admission',lambda:acquire(root,i,sha(root/'contract.json')))
|
||||
if monitor.failed:raise RuntimeError('THREE_ADMISSION_FAILURES')
|
||||
if not ok:
|
||||
time.sleep(1)
|
||||
continue
|
||||
lease,observation=decision
|
||||
save(root/'latest-admission.json',observation)
|
||||
# Acquisition includes bounded GPU queries; recheck cost before Popen.
|
||||
fresh=project(c,{j:charge(j) for j in range(5)},histories,terminal)
|
||||
if fresh['status']!='PASS':
|
||||
if lease:release(lease['token'])
|
||||
raise RuntimeError('CAP_DURING_ADMISSION')
|
||||
if lease:
|
||||
jr=root/'jobs'/str(i);jr.mkdir(exist_ok=False)
|
||||
save(jr/'assignment.json',dict(id=i,contract_sha256=sha(root/'contract.json'),
|
||||
gpu_uuid=lease['gpu_uuid'],worker_cpus=lease['worker_cpus'],token=lease['token'],
|
||||
queue_identity=identity,pre_dispatch_wait_seconds=time.monotonic()-start))
|
||||
log=(jr/'guard.log').open('x')
|
||||
env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1')
|
||||
p=subprocess.Popen([sys.executable,'tools/phase3_item1b_guard.py',str(root),str(i),sha(jr/'assignment.json')],
|
||||
cwd=str(Path(__file__).resolve().parents[1]),env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
||||
owner=process_identity(p.pid)
|
||||
active[i]=(p,lease,log,owner);pending.remove(i)
|
||||
transfer(lease['token'],owner)
|
||||
save(jr/'lease-ready.json',dict(token=lease['token'],owner=owner))
|
||||
time.sleep(1)
|
||||
except BaseException as exc:
|
||||
failure=type(exc).__name__+': '+str(exc);stop(failure)
|
||||
finally:
|
||||
if active:
|
||||
stop(failure or 'QUEUE_CLEANUP')
|
||||
for p,lease,log,owner in active.values():
|
||||
if p.poll() is None:p.terminate()
|
||||
deadline=time.monotonic()+45
|
||||
while time.monotonic()<deadline and any(p.poll() is None for p,_,_,_ in active.values()):time.sleep(.2)
|
||||
for i,(p,lease,log,owner) in active.items():
|
||||
if p.poll() is None:p.kill()
|
||||
p.wait();log.close();jr=root/'jobs'/str(i);cp=jr/'complete.json'
|
||||
result=read(cp) if cp.exists() else dict(status='MISSING_GUARD_RECEIPT')
|
||||
terminal[i]=dict(id=i,status=result['status'],returncode=p.returncode,gpu_seconds_charged=charge(i),
|
||||
guard_sha256=sha(cp) if cp.exists() else None)
|
||||
wp=jr/'worker/worker-identity.json'
|
||||
try:
|
||||
if wp.exists() and alive(read(wp)):raise RuntimeError('Live worker; lease retained')
|
||||
release(lease['token'],owner)
|
||||
except Exception as exc:
|
||||
terminal[i]['lease_release_failure']=str(exc);failure=failure or str(exc)
|
||||
final_charges={i:charge(i) for i in range(5)}
|
||||
final_cost=c['preparation_charge_gpu_seconds']+sum(final_charges.values())
|
||||
if final_cost+c['postfit_gpu_reserve']>c['gpu_cap']:failure=failure or 'FINAL_CUMULATIVE_CAP'
|
||||
complete=not failure and len(terminal)==5 and all(r['status']=='PASS' and r['returncode']==0 for r in terminal.values())
|
||||
files=[root/'contract.json']
|
||||
if complete:
|
||||
models=[]
|
||||
for i in range(5):
|
||||
r=read(root/'jobs'/str(i)/'worker/result.json')
|
||||
models.append(dict(r));files.append(Path(r['checkpoint']))
|
||||
files.extend([root/'jobs'/str(i)/'complete.json',root/'jobs'/str(i)/'worker/result.json',root/'jobs'/str(i)/'worker/standardization.json'])
|
||||
save(root/'models.json',models);files.append(root/'models.json')
|
||||
save(root/'complete.json',dict(status='TRAIN_COMPLETE_VALIDATION_SCREEN_PENDING' if complete else 'STOPPED_INCOMPLETE',
|
||||
failure=failure,entries=list(terminal.values()),never_attempted=pending,
|
||||
never_attempted_wait_charges={str(i):final_charges[i] for i in pending},
|
||||
cumulative_gpu_seconds=final_cost,postfit_gpu_reserve=c['postfit_gpu_reserve'],
|
||||
no_retry=True,test_targets_read=False,prediction_started=False,scoring_started=False))
|
||||
files.append(root/'complete.json')
|
||||
freeze_start=time.monotonic()
|
||||
freeze(root,'training-terminal' if complete else 'training-stop',files)
|
||||
final_cost+=time.monotonic()-freeze_start
|
||||
within=final_cost+c['postfit_gpu_reserve']<=c['gpu_cap']
|
||||
save(root/'post-terminal-cost.json',dict(cumulative_gpu_seconds=final_cost,within_cap=within,
|
||||
freeze_tail_charged_once=True,postfit_gpu_reserve=c['postfit_gpu_reserve'],postfit_started=False,
|
||||
training_complete_sha256=sha(root/'complete.json'),required_before_any_postfit=True))
|
||||
return 0 if complete and within else 1
|
||||
|
||||
|
||||
if __name__=='__main__':raise SystemExit(run(ROOT))
|
||||
@@ -0,0 +1,35 @@
|
||||
"""55ffc6f9: one complete100epoch schedule, not a200epoch midpoint."""
|
||||
import math
|
||||
|
||||
|
||||
def lr_epoch(epoch):
|
||||
if type(epoch) is not int or not 1<=epoch<=100:
|
||||
raise ValueError('Fixed integer epochs1..100')
|
||||
if epoch<=5:
|
||||
return 1e-4+(epoch-1)*(1e-3-1e-4)/4
|
||||
return 1e-5+(1e-3-1e-5)*(1+math.cos(math.pi*(epoch-5)/95))/2
|
||||
|
||||
|
||||
def selected_checkpoint(completed_epochs):
|
||||
if type(completed_epochs) is not int or completed_epochs!=100:
|
||||
raise ValueError('Only last complete epoch100, no midpoint/best selection')
|
||||
return 'last-epoch100.pt'
|
||||
|
||||
|
||||
def screen_decision(delta_short,delta_seed):
|
||||
"""Future validation-screen policy only; this module never reads labels."""
|
||||
if delta_short is None or delta_seed is None:
|
||||
return 'WAIT_MATCHED_VALIDATION_AND_SEED_REFERENCE'
|
||||
if not all(math.isfinite(x) for x in (delta_short,delta_seed)):
|
||||
raise ValueError('Invalid matched validation differences')
|
||||
return 'PASS' if delta_short<=-.001 and abs(delta_short)>abs(delta_seed) else 'SCREENED_OUT'
|
||||
|
||||
|
||||
def test_confirmation(delta_short,ci_high,delta_seed,parts,components):
|
||||
if parts!=552 or components!=204:
|
||||
return 'INCOMPLETE_NO_CONFIRMATION'
|
||||
if any(x is None for x in (delta_short,ci_high,delta_seed)):
|
||||
return 'WAIT_MATCHED_TEST_AND_SEED_REFERENCE'
|
||||
if not all(math.isfinite(x) for x in (delta_short,ci_high,delta_seed)):
|
||||
raise ValueError('Invalid matched test differences')
|
||||
return 'PASS' if ci_high<-.001 and abs(delta_short)>abs(delta_seed) else 'NOT_CONFIRMED'
|
||||
@@ -0,0 +1,77 @@
|
||||
"""One new S2 fold, fixed complete100epoch schedule with seed0; no test prediction or target reads."""
|
||||
import json,os,sys,time,traceback,random
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase3_item1b_schedule import selected_checkpoint,lr_epoch
|
||||
from phase3_item1b_common import config,fold_view,first_epoch_release,assignment as assigned_job,read
|
||||
from phase3_shared_capacity import process_identity
|
||||
from phase2_h2_data import standardization
|
||||
from phase2_h2_data import load_part,to_device,arguments,cached
|
||||
from phase2_h2_model import H2Geo,optimizer
|
||||
from phase2_h2_pilot_worker import peaks,train_step
|
||||
from phase2_residual_math import residual_target
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
|
||||
def run(root,fold_id,assignment_sha):
|
||||
c,pilot=config(root);old=fold_view(pilot,fold_id);job=assigned_job(root,c,fold_id,assignment_sha);out=root/'jobs'/str(fold_id)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir()
|
||||
os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
|
||||
start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0
|
||||
save(out/'worker-identity.json',process_identity(os.getpid()))
|
||||
def record(item,epoch_file=False):
|
||||
item=dict(fold=fold_id,stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks())
|
||||
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
|
||||
save(out/'progress.json',item)
|
||||
if epoch_file:save(out/'history'/f'{epoch:03d}.json',item)
|
||||
return item
|
||||
try:
|
||||
if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity')
|
||||
stage='LEARNING_ONLY_NORMALIZATION';normal=out/'normalization';normal.mkdir();stats=standardization(old,normal);save(out/'standardization.json',stats);save(out/'fold-identity.json',dict(actual_fold=fold_id,learning=old['folds'][0]['learning'],validation=old['folds'][0]['validation'],test=old['folds'][0]['test'],clock_sha256=old['folds'][0]['clock_sha256']))
|
||||
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
|
||||
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();random.seed(0);np.random.seed(0);torch.manual_seed(0);torch.cuda.manual_seed_all(0)
|
||||
model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes']
|
||||
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=job['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters())))
|
||||
for epoch in range(1,101):
|
||||
stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0.
|
||||
if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch')
|
||||
order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts)
|
||||
for index in order:
|
||||
case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda')
|
||||
result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss']
|
||||
record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip']))
|
||||
del data
|
||||
if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch')
|
||||
metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch))
|
||||
if epoch%10==0:
|
||||
stage='VALIDATION_DIAGNOSTIC';model.eval();val_loss=0.;val_nodes=0;vstart=time.monotonic()
|
||||
with torch.no_grad():
|
||||
for case in fold['validation']:
|
||||
data=to_device(load_part(old,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False)
|
||||
if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output')
|
||||
with (out/'validation-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,epoch=epoch,role='validation_diagnostic_only',target_sha256=old['parts'][case]['output_sha256']['target.npy']))+'\n')
|
||||
target=residual_target(cached(old,case,'target'),data['F'],fold['c'],fit_dtype='float32')
|
||||
delta=p.cpu().numpy().astype(np.float64)-target;val_loss+=float(delta@delta);val_nodes+=data['nodes'];del data,p
|
||||
torch.cuda.synchronize()
|
||||
if val_nodes!=fold['validation_nodes']:raise ValueError('Validation coverage')
|
||||
metrics.update(validation_mse=val_loss/val_nodes,validation_nodes=val_nodes,validation_seconds=time.monotonic()-vstart,validation_used_for_selection=False)
|
||||
stage='EPOCH_COMPLETE';record(metrics,epoch_file=True)
|
||||
if epoch==1:
|
||||
stage='WAIT_THIS_FOLD_FIRST_EPOCH_COST_GATE';record(dict(epoch_parts=seen,epoch_nodes=nodes))
|
||||
while not first_epoch_release(root,fold_id):time.sleep(.5)
|
||||
if epoch<100:
|
||||
for group in opt.param_groups:group['lr']=lr_epoch(epoch+1)
|
||||
stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch)
|
||||
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=fold_id,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name)
|
||||
save(out/'result.json',dict(status='COMPLETE',fold=fold_id,completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks()))
|
||||
except BaseException as exc:
|
||||
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
|
||||
save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
|
||||
if __name__=='__main__':
|
||||
import ctypes,signal
|
||||
from phase3_shared_capacity import alive
|
||||
expected=read(Path(sys.argv[1])/'jobs'/sys.argv[2]/'lease-ready.json')['owner']
|
||||
parent=os.getppid()
|
||||
if parent!=expected['pid'] or not alive(expected):raise RuntimeError('Pinned guard lost before worker import completed')
|
||||
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGKILL,0,0,0)!=0:raise OSError('PDEATHSIG')
|
||||
if os.getppid()!=parent:raise RuntimeError('Guard exited before worker setup')
|
||||
run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3])
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Metadata-only item1b training preparation; no new fits or screen/test reads."""
|
||||
from pathlib import Path
|
||||
import time
|
||||
from phase2_h1_worker_v2 import checked,sha,save
|
||||
from phase3_item1b_common import ROOT,read
|
||||
from phase2_home_freeze_io import verify
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from track_a_h2_prepare import local_code_closure
|
||||
from phase3_shared_capacity import PHASE_RESERVES
|
||||
|
||||
B=Path(__file__).resolve().parents[1]
|
||||
H=Path('/home/frank14f/MeltMind-runs')
|
||||
|
||||
|
||||
def main():
|
||||
start=time.process_time()
|
||||
if ROOT.exists():raise FileExistsError('Sole item1b root already exists')
|
||||
croot=H/'phase3-item-c-2026-11-18';validate_review(croot);verify(croot,'implementation')
|
||||
cc=read(croot/'contract.json');pilot=read(cc['pilot_contract'])
|
||||
source=dict(cc['source_sha256'])
|
||||
spec=B/'references/2026-11-18/phase3-item-1b/mold-short-schedule-spec.md'
|
||||
checked(spec,'55ffc6f9320f6b6f03244acd8548d5037a6efa43d3ce8c48f1b89bb0cb4b5e58')
|
||||
for p in [spec,croot/'contract.json',croot/'review.json',croot/'expected-review-binding.json',croot/'implementation-review-git-freeze.json',croot/'implementation-review-authorization.json']:
|
||||
source[str(p)]=sha(p)
|
||||
baseline=[];jobs=[]
|
||||
for f in pilot['folds']:
|
||||
i=f['fold']
|
||||
w=B/'artifacts/phase2-h2-s1-2026-11-17/worker' if i==0 else H/'phase2-h2-s2-2026-11-17/folds'/str(i)/'worker'
|
||||
result=read(w/'result.json')
|
||||
if i==0:
|
||||
original_contract=w.parent/'contract.json';original=read(original_contract)
|
||||
if original['fold']!=0 or original['seed']!=0 or original['epochs']!=200:
|
||||
raise ValueError('Original S1 fold0 contract')
|
||||
source[str(original_contract)]=sha(original_contract)
|
||||
elif result['fold']!=i:
|
||||
raise ValueError('Original S2 fold identity')
|
||||
if result['completed_epochs']!=200 or result['optimizer_steps']!=200*len(f['learning']):
|
||||
raise ValueError('Original200epoch same-fold model identity')
|
||||
stats=read(w/'standardization.json')
|
||||
if stats['parts']!=f['learning'] or stats['nodes']!=f['learning_nodes']:
|
||||
raise ValueError('Paired original normalization learning role')
|
||||
checked(result['checkpoint'],result['checkpoint_sha256'])
|
||||
for p in [w/'result.json',w/'standardization.json',Path(result['checkpoint'])]:source[str(p)]=sha(p)
|
||||
histories=[w/'history'/f'{e:03d}.json' for e in range(10,201,10)]
|
||||
for p in histories:source[str(p)]=sha(p)
|
||||
val=max(read(p)['validation_seconds'] for p in histories)
|
||||
baseline.append(dict(fold=i,checkpoint=result['checkpoint'],checkpoint_sha256=result['checkpoint_sha256'],
|
||||
result=str(w/'result.json'),result_sha256=sha(w/'result.json'),standardization=str(w/'standardization.json'),standardization_sha256=sha(w/'standardization.json'),completed_epochs=200,seed=0))
|
||||
jobs.append({k:f[k] for k in ('learning','validation','test','c','clock_sha256','learning_nodes')}|
|
||||
dict(id=i,target_steps=100*len(f['learning']),cpu_seconds=30000,validation_seconds_bound=val,
|
||||
rss_bytes=4*2**30,recorded_vram_bytes=2*2**30,process_vram_bytes=4*2**30,disk_bytes=8*2**30))
|
||||
files=sorted(B.glob('tools/phase3_item1b_*.py'))+[Path(__file__).resolve(),B/'tools/test_phase3_item1b.py']
|
||||
codes=local_code_closure(files+sorted((B/'tools/s2_models').glob('*.py')))
|
||||
for p,h in cc['code_sha256'].items():codes.setdefault(p,h)
|
||||
for p,h in {**source,**codes}.items():checked(p,h)
|
||||
comparison=dict(version='item1b-training-v1',spec_sha256=sha(spec),scheduling_ruling='1795010666.724069',
|
||||
scheduling_change='all five folds may train before screen; screen still gates any test postfit',
|
||||
folds=pilot['folds'],roster_parts=552,components=204,seed=0,epochs=100,inputs=19,
|
||||
recipe='original H2; paired seed0 initialization and original epoch-derived part order; unchanged MSE/optimizer/decay/clipping',
|
||||
schedule='epochs1..5:1e-4+(e-1)*(1e-3-1e-4)/4;6..100:1e-5+(1e-3-1e-5)*(1+cos(pi*(e-5)/95))/2',
|
||||
interpretation='joint update-budget and complete-schedule change; not200epoch midpoint; no separate causal claim',
|
||||
checkpoint='last-epoch100.pt only',validation_diagnostics='original full-node MSE every10epochs, never selection or screen',
|
||||
baseline_models=baseline,seed1_root=str(H/'phase3-seed1-v2-2026-11-18'),
|
||||
screen=dict(fold=1,domain='same matched validation query domain',metric='component-equal relativeL2',
|
||||
delta_short_at_most=-.001,absolute_effect_strictly_above_absolute_seed1_minus_seed0=True,
|
||||
missing_reference='WAIT_NO_EXTRA_SEED',fail='ARCHIVE_ALL_FIVE_NO_TEST_READ'),
|
||||
test=dict(requires_screen_PASS=True,requires_separate_pre_score_review=True,parts=552,components=204,
|
||||
upper_delta_relativeL2_strictly_below=-.001,absolute_effect_strictly_above_absolute_pooled_seed_difference=True,
|
||||
bootstrap=2000,bootstrap_seed=2026111408,paired_components=True,confidence=.95,
|
||||
report=['signed means','CI','seed difference','per-fold','hard last-point/Rg','tau','negative predictions','E_tail if available']),
|
||||
postfit='separately bound validation screen; test inference/score only after screenPASS and actual pre-score review; no postfit launch in training queue',
|
||||
limitations=['one seed difference is not full training randomness','development-contaminated restricted benchmark, not fresh-family confirmation'])
|
||||
ROOT.mkdir();save(ROOT/'comparison-contract.json',comparison)
|
||||
c=dict(version='phase3-item1b-v1',approval='1795010666.724069',entry_approval='1795010507.291259',budget_approval='1795007580.895539',
|
||||
seed=0,epochs=100,new_fits=5,no_retry=True,gpu_cap=88560,postfit_gpu_reserve=7200,postfit_cpu_cap=3600,
|
||||
phase3_gpu_cap=720000,phase_reservations=PHASE_RESERVES,preparation_charge_gpu_seconds=600,preparation_cpu_cap=600,
|
||||
runaway_gpu_seconds=30000,screen_fold=1,test_requires_screen_pass=True,dispatch_order=[1,0,2,3,4],
|
||||
waiting_charge='all5 from queue start to own terminal; includes pending never-dispatched waits/failures;60s setup per fold',
|
||||
projection='actual+1.5 remaining100epoch training and10 diagnostic validations+120 cleanup; total gate, no per-fold projection stop',
|
||||
validation_targets_read='diagnostic_only_every10_epochs_after_reviewed_launch; not selection or screen',test_targets_read=False,
|
||||
screen_started=False,postfit_started=False,track_a_ledger='separate',
|
||||
shared_rule_module='phase3_shared_capacity.py unchanged reviewed2ddecef3/e30612ea',
|
||||
capacity_adapter='item1b key/cap and GPU0..2 only; original shared admission/registry/lease operations unchanged',
|
||||
jobs=jobs,pilot_contract=cc['pilot_contract'],query_roster_contract=cc['query_roster_contract'],
|
||||
baseline_models=baseline,source_sha256=source,code_sha256=codes,
|
||||
comparison_contract=str(ROOT/'comparison-contract.json'),comparison_contract_sha256=sha(ROOT/'comparison-contract.json'))
|
||||
save(ROOT/'contract.json',c)
|
||||
save(ROOT/'preparation-audit.json',dict(status='PREPARED_PENDING_ASSIST_REVIEW',contract_sha256=sha(ROOT/'contract.json'),
|
||||
code_files=len(codes),source_files=len(source),baseline_models=5,opaque_model_hashes_only=True,
|
||||
metadata_cpu_seconds=time.process_time()-start,labels_decoded=False,arrays_decoded=False,models_loaded=False,new_fits_started=False))
|
||||
print(read(ROOT/'preparation-audit.json'))
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Synthetic item1b acceptance; no real data/model loading or GPU access."""
|
||||
import ast
|
||||
import copy
|
||||
from contextlib import ExitStack,contextmanager
|
||||
import inspect
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from phase3_item1b_schedule import lr_epoch,selected_checkpoint,screen_decision,test_confirmation
|
||||
from phase3_item1b_common import project
|
||||
from phase3_shared_capacity import GPUS,GIB
|
||||
|
||||
|
||||
class ScheduleTests(unittest.TestCase):
|
||||
def test_exact_endpoints(self):
|
||||
self.assertAlmostEqual(lr_epoch(1),1e-4,15)
|
||||
self.assertAlmostEqual(lr_epoch(5),1e-3,15)
|
||||
self.assertAlmostEqual(lr_epoch(100),1e-5,15)
|
||||
def test_complete_warmup_and_decay(self):
|
||||
rates=[lr_epoch(i) for i in range(1,101)]
|
||||
self.assertTrue(all(a<b for a,b in zip(rates[:4],rates[1:5])))
|
||||
self.assertTrue(all(a>b for a,b in zip(rates[4:-1],rates[5:])))
|
||||
def test_not_original_midpoint(self):
|
||||
from phase2_h2_model import lr_epoch as old_lr
|
||||
self.assertGreater(old_lr(100),lr_epoch(100)*10)
|
||||
self.assertEqual(lr_epoch(1),old_lr(1))
|
||||
def test_only_fixed_last_checkpoint(self):
|
||||
self.assertEqual(selected_checkpoint(100),'last-epoch100.pt')
|
||||
for e in (99,200,100.,True):
|
||||
with self.assertRaises(ValueError):selected_checkpoint(e)
|
||||
def test_invalid_epoch(self):
|
||||
for e in (0,101,1.5,True):
|
||||
with self.assertRaises(ValueError):lr_epoch(e)
|
||||
def test_screen_exact_boundary(self):
|
||||
self.assertEqual(screen_decision(-.001,.0009),'PASS')
|
||||
self.assertEqual(screen_decision(-.001,-.001),'SCREENED_OUT')
|
||||
self.assertEqual(screen_decision(-.0009,0),'SCREENED_OUT')
|
||||
def test_no_seed_no_decision(self):
|
||||
self.assertEqual(screen_decision(-.02,None),'WAIT_MATCHED_VALIDATION_AND_SEED_REFERENCE')
|
||||
def test_test_strict_boundary_and_seed(self):
|
||||
self.assertEqual(test_confirmation(-.002,-.001,0,552,204),'NOT_CONFIRMED')
|
||||
self.assertEqual(test_confirmation(-.002,-.0011,-.002,552,204),'NOT_CONFIRMED')
|
||||
self.assertEqual(test_confirmation(-.002,-.0011,.001,552,204),'PASS')
|
||||
def test_incomplete_test_not_confirmation(self):
|
||||
self.assertEqual(test_confirmation(-.03,-.02,0,551,204),'INCOMPLETE_NO_CONFIRMATION')
|
||||
def test_nonfinite_comparisons_rejected(self):
|
||||
with self.assertRaises(ValueError):screen_decision(float('nan'),0)
|
||||
with self.assertRaises(ValueError):test_confirmation(-.02,float('inf'),0,552,204)
|
||||
def test_worker_original_train_step_and_seed0(self):
|
||||
p=Path(__file__).parent/'phase3_item1b_worker.py';s=p.read_text();tree=ast.parse(s)
|
||||
imports=[n for n in ast.walk(tree) if isinstance(n,ast.ImportFrom)]
|
||||
self.assertTrue(any(n.module=='phase2_h2_pilot_worker' and 'train_step' in [a.name for a in n.names] for n in imports))
|
||||
self.assertIn('SeedSequence([0,epoch])',s);self.assertIn('range(1,101)',s);self.assertIn('if epoch%10==0:',s)
|
||||
self.assertNotIn('location_loss',s);self.assertNotIn('phase3_item_c_loss',s)
|
||||
|
||||
|
||||
class BudgetTests(unittest.TestCase):
|
||||
def setUp(self):self.c=dict(gpu_cap=88560,postfit_gpu_reserve=7200,preparation_charge_gpu_seconds=600,runaway_gpu_seconds=30000,jobs=[dict(validation_seconds_bound=3) for _ in range(5)])
|
||||
def test_initial_budget_includes_postfit(self):
|
||||
p=project(self.c,{i:60 for i in range(5)},{},{})
|
||||
self.assertEqual(p['status'],'PASS');self.assertEqual(p['projected_gpu_seconds'],88560)
|
||||
def test_remaining_validation_charged(self):
|
||||
p=project(self.c,{i:100 for i in range(5)},{0:[dict(epoch=1,train_epoch_seconds=50)]},{})
|
||||
self.assertEqual(p['folds']['0']['projected'],100+1.5*(99*50+10*3)+120)
|
||||
def test_late_validation_max_used(self):
|
||||
h=[dict(epoch=e,train_epoch_seconds=50,validation_seconds=7 if e==10 else 0) for e in range(1,11)]
|
||||
p=project(self.c,{i:1000 for i in range(5)},{0:h},{})
|
||||
self.assertEqual(p['folds']['0']['projected'],1000+1.5*(90*50+9*7)+120)
|
||||
def test_actual_runaway_stops(self):
|
||||
self.assertEqual(project(self.c,{i:30000 if i==0 else 60 for i in range(5)},{},{})['status'],'STOP_BUDGET')
|
||||
def test_no_single_fold_projection_gate(self):
|
||||
p=project(self.c,{i:1000 for i in range(5)},{0:[dict(epoch=1,train_epoch_seconds=220)]},{1:{},2:{},3:{}})
|
||||
self.assertGreater(p['folds']['0']['projected'],30000);self.assertEqual(p['status'],'PASS')
|
||||
def test_incomplete_history_rejected(self):
|
||||
with self.assertRaises(ValueError):project(self.c,{i:60 for i in range(5)},{0:[dict(epoch=2,train_epoch_seconds=50)]},{})
|
||||
|
||||
|
||||
class SharedWrapperTests(unittest.TestCase):
|
||||
def test_wrapper_changes_only_key_cap_gpu_set(self):
|
||||
import phase3_item1b_capacity as new
|
||||
import phase3_shared_capacity as old
|
||||
for name in ('register_entry','acquire'):
|
||||
s=inspect.getsource(getattr(new,name)).replace("'item1b'","'c'").replace('item 1b','item c').replace('cap=88560','cap=205200').replace('GPUS[:3]','GPUS[1:3]')
|
||||
self.assertEqual(ast.dump(ast.parse(s)),ast.dump(ast.parse(inspect.getsource(getattr(old,name)))))
|
||||
for name in ('admission','registry','transfer','release','validate_lease'):
|
||||
self.assertIs(getattr(new,name),getattr(old,name))
|
||||
@contextmanager
|
||||
def mixed(self,reserved=False):
|
||||
import phase3_item1b_capacity as m
|
||||
import phase3_shared_capacity as shared
|
||||
identity=lambda p:dict(pid=p,created=1.)
|
||||
root=Path('/synthetic-1b');oldroot='/synthetic-c'
|
||||
leases={}
|
||||
for n,pid in enumerate([11,12]+([10] if reserved else [])):
|
||||
leases[str(n)]=dict(token=str(n),state='RESERVED' if pid==10 else 'GUARD_BOUND',root=oldroot,fold=n,
|
||||
contract_sha256='old',gpu_uuid=GPUS[1],worker_cpus=[16+n],owner=identity(pid),queue_owner=identity(10),vram_reserve=4*GIB,rss_reserve=4*GIB)
|
||||
original_c=dict(root=oldroot,contract_sha256='old',owner=identity(10),cap=205200)
|
||||
s=dict(devices={g:dict(total=16*GIB,used=4*GIB) for g in GPUS},processes=[dict(gpu_uuid=GPUS[1],pid=p,vram=2*GIB) for p in (101,102)]+[dict(gpu_uuid=GPUS[2],pid=p,vram=GIB) for p in (201,202,203)],ram_total=128*GIB,ram_available=96*GIB,cpu_percent=20,old_pending=True)
|
||||
trees={11:{11,101},12:{12,102},10:{10,11,12,101,102}}
|
||||
with tempfile.TemporaryDirectory() as tmp,ExitStack() as stack:
|
||||
path=Path(tmp)/'registry.json';path.write_text(json.dumps(dict(version=2,entries={'c':original_c},leases=leases)))
|
||||
stack.enter_context(patch.object(shared,'SHARED',Path(tmp)))
|
||||
for name,value in dict(process_identity=lambda p:identity(20),alive=lambda p:True,descendants=lambda p:trees[p['pid']],host_snapshot=lambda:copy.deepcopy(s)).items():stack.enter_context(patch.object(m,name,value))
|
||||
stack.enter_context(patch.object(m.psutil,'Process',lambda p:SimpleNamespace(memory_info=lambda:SimpleNamespace(rss=GIB//4))))
|
||||
stack.enter_context(patch.object(m.os,'sched_getaffinity',lambda p:set(range(16,26))))
|
||||
m.register_entry(root,'new')
|
||||
yield m,root,path,original_c
|
||||
def test_shared_c_and_1b_registration_independent(self):
|
||||
with self.mixed() as (m,root,path,old):
|
||||
d=json.loads(path.read_text());self.assertEqual(d['entries']['c'],old);self.assertEqual(d['entries']['item1b']['cap'],88560)
|
||||
with self.assertRaises(FileExistsError):m.register_entry(root,'new')
|
||||
def test_mixed_pending_c_reservation_blocks_1b(self):
|
||||
with self.mixed(True) as (m,root,path,old):
|
||||
before=json.loads(path.read_text());l,_=m.acquire(root,1,'new');self.assertIsNone(l)
|
||||
self.assertEqual(json.loads(path.read_text()),before)
|
||||
def test_mixed_two_visible_allows_only_one_new(self):
|
||||
with self.mixed() as (m,root,path,old):
|
||||
l,_=m.acquire(root,1,'new');self.assertIsNotNone(l);self.assertEqual(l['state'],'RESERVED');self.assertEqual(l['gpu_uuid'],GPUS[1])
|
||||
second,_=m.acquire(root,0,'new');self.assertIsNone(second)
|
||||
self.assertEqual(json.loads(path.read_text())['entries']['c'],old)
|
||||
|
||||
|
||||
class QueueTests(unittest.TestCase):
|
||||
def exercise(self,failed=False):
|
||||
import phase3_item1b_queue as q
|
||||
from phase2_h1_worker_v2 import save
|
||||
with tempfile.TemporaryDirectory() as tmp,ExitStack() as stack:
|
||||
root=Path(tmp);save(root/'contract.json',dict(synthetic=True));launched=[]
|
||||
c=dict(gpu_cap=88560,postfit_gpu_reserve=7200,preparation_charge_gpu_seconds=600,runaway_gpu_seconds=30000,jobs=[dict(validation_seconds_bound=1) for _ in range(5)])
|
||||
class Process:
|
||||
def __init__(self,cmd,**kw):
|
||||
self.pid=999991;self.returncode=int(failed);i=int(cmd[3]);launched.append(i);j=root/'jobs'/str(i);(j/'worker').mkdir()
|
||||
save(j/'complete.json',dict(status='STOPPED_OR_FAILED' if failed else 'PASS'));p=j/'worker/last-epoch100.pt';p.write_bytes(b'synthetic')
|
||||
save(j/'worker/result.json',dict(fold=i,checkpoint=str(p),checkpoint_sha256=q.sha(p),completed_epochs=100));save(j/'worker/standardization.json',{})
|
||||
def poll(self):return self.returncode
|
||||
def wait(self):return self.returncode
|
||||
def terminate(self):pass
|
||||
def kill(self):pass
|
||||
replacements=dict(config=lambda r:(c,{}),validate_review=lambda r:None,register_entry=lambda *a:None,
|
||||
acquire=lambda r,i,d:(dict(token=str(i),gpu_uuid=GPUS[1],worker_cpus=[20]),{}),transfer=lambda *a:None,
|
||||
release=lambda *a:None,process_identity=lambda p:dict(pid=p,created=0),freeze=lambda *a:None)
|
||||
for k,v in replacements.items():stack.enter_context(patch.object(q,k,v))
|
||||
stack.enter_context(patch.object(q.subprocess,'Popen',Process));stack.enter_context(patch.object(q.time,'sleep',lambda t:None))
|
||||
stack.enter_context(patch.object(q.signal,'signal',lambda *a:None));stack.enter_context(patch.object(q.psutil,'disk_usage',return_value=SimpleNamespace(free=200*GIB)))
|
||||
stack.enter_context(patch.dict(q.os.environ,PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True'))
|
||||
result=q.run(root);d=q.read(root/'complete.json')
|
||||
if failed:self.assertEqual(result,1);self.assertEqual(launched,[1]);self.assertFalse((root/'models.json').exists())
|
||||
else:
|
||||
self.assertEqual(result,0);self.assertEqual(launched,[1,0,2,3,4]);self.assertEqual(len(q.read(root/'models.json')),5)
|
||||
self.assertEqual(d['status'],'TRAIN_COMPLETE_VALIDATION_SCREEN_PENDING')
|
||||
self.assertFalse(d['test_targets_read']);self.assertFalse(d['prediction_started']);self.assertFalse(d['scoring_started'])
|
||||
def test_five_fits_before_screen_but_no_test(self):self.exercise()
|
||||
def test_failure_no_retry_or_test(self):self.exercise(True)
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user