Prepare reviewed-scope Track A H2 training entry and synthetic acceptance
This commit is contained in:
1 parent
1eae27ade8
commit
2cf0a0307d
18 files changed
+43660
No files matched your search
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"tests": 20,
|
||||
"failures": 0,
|
||||
"errors": 0,
|
||||
"cpu_seconds": 0.38168451700000006,
|
||||
"wall_seconds": 0.38341774605214596,
|
||||
"real_scientific_arrays_read": false,
|
||||
"GPU_used": false,
|
||||
"synthetic_worker_epochs": 200,
|
||||
"attempt01_tests": 18,
|
||||
"attempt01_pass": true
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
test_01_exact_original_roles (test_track_a_h2_entry.EntryTests) ... ok
|
||||
test_02_test_to_train_rejected (test_track_a_h2_entry.EntryTests) ... ok
|
||||
test_03_extra_validation_rejected (test_track_a_h2_entry.EntryTests) ... ok
|
||||
test_04_test_loader_rejects_before_io (test_track_a_h2_entry.EntryTests) ... ok
|
||||
test_05_clock_cannot_be_changed (test_track_a_h2_entry.EntryTests) ... ok
|
||||
test_06_two_slots_do_not_require_five_first_epochs (test_track_a_h2_entry.EntryTests) ... ok
|
||||
test_07_item1_gpu_protected (test_track_a_h2_entry.EntryTests) ... ok
|
||||
test_08_existing_item1_is_not_duplicate_track_a (test_track_a_h2_entry.EntryTests) ... ok
|
||||
test_09_pending_caps_reserved (test_track_a_h2_entry.EntryTests) ... ok
|
||||
test_10_terminal_replaces_estimate_once (test_track_a_h2_entry.EntryTests) ... ok
|
||||
test_11_active_waits_cannot_disappear (test_track_a_h2_entry.EntryTests) ... ok
|
||||
test_12_slow_first_epoch_stops (test_track_a_h2_entry.EntryTests) ... ok
|
||||
test_13_invalid_projection_rejected (test_track_a_h2_entry.EntryTests) ... ok
|
||||
test_14_first_epoch_global_stop (test_track_a_h2_entry.EntryTests) ... ok
|
||||
test_15_guard_success_path (test_track_a_h2_entry.EntryTests) ... ok
|
||||
test_16_worker_monitor_error_overrides_pass (test_track_a_h2_entry.EntryTests) ... ok
|
||||
test_17_failure_peak_remains_enforced (test_track_a_h2_entry.EntryTests) ... ok
|
||||
test_18_foreign_ledger_job_rejected (test_track_a_h2_entry.EntryTests) ... ok
|
||||
test_19_geometry_cannot_mix_versions (test_track_a_h2_entry.EntryTests) ... ok
|
||||
test_20_worker_all_epochs_no_validation_or_test (test_track_a_h2_entry.EntryTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 20 tests in 0.383s
|
||||
|
||||
OK
|
||||
@@ -625,3 +625,6 @@ Yanqi经TRON补充有学生许可、曾用Fusion with Moldflow Adviser、愿考
|
||||
|
||||
最新Track A复用/覆盖实核(Claude1794973608):旧v4 A五模型和1537预测hash、split534b2a25、19列/实际名单匹配,复用不重拟合。原H2 grouped552虽缺77件,但原H1已有完整615件v4_random图,615图hash与冻结链已核;选v4原生特征/IDs/图后输入候选train311/304/307/306/310、test304/311/308/309/305,原14失败保留。7件v4/v5 rawID差异已由几何只读核查定位,不混用编号。证据/输入入口artifacts/track-a-existing-input-audit-2026-11-18,报告docs/track-a-existing-input-audit-2026-11-18.md。没有新准备/拟合/预测/评分或训练启动;H2训练入口与其独立审阅尚待完成,当前item1不改。最新3297/3352已允许Track A经审阅在item1不需要的闲置GPU并行,替代旧2672中必须等待item1全结束的约束;45GPUh和评分前审阅保留。
|
||||
|
||||
|
||||
|
||||
Track A执行配方明确(Claude1794975093.432709):采用A同5split全实际训练名单、无另划validation、seed0/200epochs/原优化器调度/last-epoch,复用H1 v4 learning-only时钟,各split重新标准化;GPU2/3,item1的0/1保留。独立训练入口已实现并20CPU合成PASS,本地固定6701e4d6/contractbe4ac1f6。尚未入口审阅或训练,不将输入审计1eae27ad扩大为训练PASS;45GPUh含setup/waits/predict,首epoch成本门和test前Assist审阅保持。报告docs/track-a-h2-entry-2026-11-18.md。
|
||||
@@ -1,3 +1,9 @@
|
||||
## 最新 Track A H2 独立入口已实现、待审(2026-11-18)
|
||||
|
||||
Claude1794975093批准原5split全实际train、无validation、seed0/200epoch、原H1 v4时钟、GPU2/3。新根/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18,合同be4ac1f6,implementation6701e4d6,23源码固定、20CPU合成PASS。尚无review.json/started.json,无新训练/test访问。报告docs/track-a-h2-entry-2026-11-18.md;待实际独立入口审阅,再按既有45GPUh/首epoch门执行。输入1eae27ad不作入口PASS;旧A及615图审计不重做,14失败保留,无63准备/映射;item1代码和队列不变,保留其GPU0/1。
|
||||
|
||||
最新Claude1794973932.596749已实读:正式采用615件v4原生H1图+v4特征/rawIDs;取消3838的63件准备和坐标映射步骤,均无需运行。A不重拟合,旧14失败保留;共同test计数仅候选,待H2实际预测终态。输入审计包1eae27ade8d51644b428347651528a15a91a76e5已远端固定;45GPUh/闲置GPU不抢item1/评分前审阅保持。
|
||||
|
||||
最新Track A复用/覆盖实核(Claude1794973608):旧v4 A五模型和1537预测hash、split534b2a25、19列/实际名单匹配,复用不重拟合。原H2 grouped552虽缺77件,但原H1已有完整615件v4_random图,615图hash与冻结链已核;选v4原生特征/IDs/图后输入候选train311/304/307/306/310、test304/311/308/309/305,原14失败保留。7件v4/v5 rawID差异已由几何只读核查定位,不混用编号。证据/输入入口artifacts/track-a-existing-input-audit-2026-11-18,报告docs/track-a-existing-input-audit-2026-11-18.md。没有新准备/拟合/预测/评分或训练启动;H2训练入口与其独立审阅尚待完成,当前item1不改。最新3297/3352已允许Track A经审阅在item1不需要的闲置GPU并行,替代旧2672中必须等待item1全结束的约束;45GPUh和评分前审阅保留。
|
||||
|
||||
最新优先级补充(2026-11-18,Yanqi经TRON):先把尚未探索完的研究路线做好,不急于许可/造数深挖;后者作为后续准备,避免挤占主线。C/D继续原文归档。TRON报告11:41资源快照为DanteSindy约6 CPU workers、4张V100中item1用1卡、available54GiB/Optane52GiB/home224GiB;这是转达的当时快照,Melt本轮未独立核验,不视作当前可分配保证。Claude据有价值问题和既有范围裁量并行,不自行改item1或启动重复任务。
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# Track A H2: five original v4 random splits
|
||||
|
||||
Status: implemented, 20 CPU synthetic checks PASS, local implementation frozen; independent entry review pending. No new training, prediction or scoring launched. Input reuse audit 1eae27ad is not training acceptance.
|
||||
|
||||
Research question: compare the unchanged H2 recipe with the existing Greif-style A baseline under the same five original random 314/315 splits. This is protocol-level re-creation, not the unpublished author split identity. Family leakage remains possible in random-part splits. The metric remains per-part RMSE/max(full truth), min/median/max over each test set and mean over five splits; old A median0.0396 is context, not the new paired result. No topology/local-type hypothesis is closed by this comparison.
|
||||
|
||||
Claude3608/3932 require A reuse without refit, 615 frozen v4-native H1 graphs/features/raw IDs and preservation of14 old failures. Claude1794975093.432709 approves full actual train311/304/307/306/310, seed0,200epochs, original optimizer/schedule, last-epoch weights, no validation and original H1 v4 learning-only clocks. No63-part queue or cross-version ID mapping. Candidate test304/311/308/309/305 remain provisional until H2 predictions exist.
|
||||
|
||||
The independent entry is tools/track_a_h2_{prepare,common,worker,guard,queue}.py. Shared H2 scientific modules are unchanged and source-pinned. Every epoch uses all actual train parts and their full nodes/true graph edges. Feature normalization is computed per split on train only with the original FP64 population statistics; the network and optimizer retain the original FP32 recipe, separate FP64 Vf skip, no AMP/TF32. The five clocks are reused by hash with exact train membership and equal-part grouping; no clock targets reopened. Model seed0 is distinct from split seeds2026111601..1605.
|
||||
|
||||
Run root: /home/frank14f/MeltMind-runs/track-a-h2-2026-11-18. Final contract SHA256 be4ac1f6d5a5d6c0183aa50d2e046b22d7e5fa77c7714fef7eab6d616e68bee7. Local implementation commit6701e4d67f2f7ee6c70d22a3512f40c4ad9180f5; proof SHA f740a9c4654c67911d344e4b98c19adf3776179ba31dcaf859b5c44669420700. expected-review-binding.json specifies actual exact review identity. Initial unreviewed f0def4cd contract is retained in before-final-checks, superseded only before any launch.
|
||||
|
||||
GPU2/3 only, at most two workers; GPU1 remains item1 training and GPU0 reserved for its frozen postfit. CPU feeders8/10, guards28/29, coordinator30/31. Each dispatch takes a GPU lock and requires three idle polls. No other queue is stopped or modified. Total162000 GPU-s includes setup/waits/prediction:154800training plus7200postfit reserve; five job caps30960 each, setup60 charged once per job. Per-job CPU61920s/15GiB VRAM/16GiB RSS/8GiB output, aggregate40GiB RSS/48GiB output. These are engineering stop ceilings, not projected use.
|
||||
|
||||
Each job pauses after its own first epoch. Pending jobs reserve their entire caps, active elapsed time cannot disappear, terminal costs replace forecasts once. Cost uses maximum observed full-epoch duration with1.25future margin and120s final overhead. It is updated every epoch. No five-job barrier on two slots. Over-budget projection, worker/monitor failure, recorded or live VRAM breach, or global stop ends without retry. All setup and active waiting remain charged. Whole-job guards inspect underlying monitor_errors even if the supervisor reports PASS, and scan peaks in failed-worker records.
|
||||
|
||||
All five terminal PASS checkpoints and normalizations are Git-frozen before this training-only queue completes. It contains no postfit launch. A separate actual MeltAssist review is required before any test access; old review PASS cannot substitute. Prediction/scoring implementation is outside this training entry and must preserve the7200s reserve and total45h.
|
||||
|
||||
Validation:20 CPU-only synthetic checks passed (0.381684517CPU-s test body;0.383417746wall-s; imports excluded). Coverage includes exact roles, test-to-train rejection, no new holdout, test loader refusal before IO, clock and native-graph identity, two-slot dispatch, reserved item1 GPUs, pending/active/terminal accounting, slow first-epoch stop, global-stop gate, raw monitor-error rejection and failed peak enforcement. A mocked worker traverses all200epochs/400synthetic updates with no validation/test reads; it is not a real fit or GPU benchmark. First18-check pass is retained as prior test history; expanded20-check pass is final. Existing A/615graph audit was not rerun.
|
||||
|
||||
Next decision: independent review of these actual files, then authorized idle-GPU execution if PASS; first-epoch evidence decides cost feasibility. The original scientific task and item1 remain unchanged.
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"status": "ENTRY_PREPARED_NOT_EXECUTED",
|
||||
"contract_sha256": "f0def4cdb9918de0f14236e28aab645010faff1f9b8372d8ee42b8c956ffb775",
|
||||
"code_files": 22,
|
||||
"clock_values_recomputed": false,
|
||||
"scientific_arrays_read": false,
|
||||
"new_fits_started": false,
|
||||
"prior_A_audit_rerun": false,
|
||||
"learning_counts": [
|
||||
311,
|
||||
304,
|
||||
307,
|
||||
306,
|
||||
310
|
||||
],
|
||||
"cpu_seconds": 0.177887447,
|
||||
"protocol_ruling": "1794975093.432709"
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"reviewed_commit": "6701e4d67f2f7ee6c70d22a3512f40c4ad9180f5",
|
||||
"contract_sha256": "be4ac1f6d5a5d6c0183aa50d2e046b22d7e5fa77c7714fef7eab6d616e68bee7",
|
||||
"implementation_proof_sha256": "f740a9c4654c67911d344e4b98c19adf3776179ba31dcaf859b5c44669420700",
|
||||
"comparison_contract_sha256": "8f0f3c1088d899b85aa3f578b7552f04cb409fd7a5dcfca3e17eaa3d7e25d0d6",
|
||||
"code_sha256": {
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_h2_entry.py": "6fffc5dea817d972be2a0330b0c2ec46f7906ff3f7bf27b516603bbbad0f4006",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_guard.py": "9ce791a5e81f23b03d0c6381ab45d9a6d61e862ab3b2a154f6a8d569d0ee3362",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_queue.py": "d81e002fb0e66c3254745808b908ce28ed6b6e78b9a6853fe0d0132bf548dd22",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_worker.py": "3710044d610eb2d0887500b820b9c10452d170624850d7a85a5fbb88192ab676"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"contract_sha256": "be4ac1f6d5a5d6c0183aa50d2e046b22d7e5fa77c7714fef7eab6d616e68bee7",
|
||||
"proof_sha256": "f740a9c4654c67911d344e4b98c19adf3776179ba31dcaf859b5c44669420700"
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
{
|
||||
"commit": "6701e4d67f2f7ee6c70d22a3512f40c4ad9180f5",
|
||||
"files": {
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/contract.json": "be4ac1f6d5a5d6c0183aa50d2e046b22d7e5fa77c7714fef7eab6d616e68bee7",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/preparation-receipt.json": "9ba67dcf2ae77cbc4e2f94a0aa0df0067f21677b4e3cf54abc3372af399e5ce9",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-h2-entry-acceptance-2026-11-18/acceptance.json": "ca7acd7d9389744625c3ca64c3b3efabc6ecd872a1d4d0b83035b06fd3b34f9e",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-h2-entry-acceptance-2026-11-18/attempt-02.log": "9093142af036fca7dbda6d4ce2c7882a74c1e9304c940159a2cb16a8f1130c6c",
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_h2_entry.py": "6fffc5dea817d972be2a0330b0c2ec46f7906ff3f7bf27b516603bbbad0f4006",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_guard.py": "9ce791a5e81f23b03d0c6381ab45d9a6d61e862ab3b2a154f6a8d569d0ee3362",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_queue.py": "d81e002fb0e66c3254745808b908ce28ed6b6e78b9a6853fe0d0132bf548dd22",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_worker.py": "3710044d610eb2d0887500b820b9c10452d170624850d7a85a5fbb88192ab676",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-existing-input-audit-2026-11-18/reuse-entry.json": "8f0f3c1088d899b85aa3f578b7552f04cb409fd7a5dcfca3e17eaa3d7e25d0d6",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol/models-freeze.json": "34af9f16272dc0352fd710c338fc23e245ed521ce840072c90d2308b3ead8fda",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol/predictions-freeze.json": "b3b25f021413d197c9bdd00e78ebe4de5852b1132e075e467618b63199c63e86",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol/splits.json": "534b2a25f92dbac2335b6282e1dfe0bb8ecdbc8db55bdfc24c7cffa83f9e931b",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol/preparation-freeze.json": "ac7add88b54f0cd75b667cacbb813aac40a23b9a7f47557e635efd7dbbc2d3d0",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/geometry/manifest.json": "f72d1df7af5a3650e29c690458e3e2dc20b422ab14efc36cb83b2a405c844263",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17/contract.json": "9f616974fb01f8f5cab6233273d541b3e53f5975f3632db45ab4676f911fba7b",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-labelled-recovery-2026-11-17/models-terminal.json": "ac60d673773b27ee2955ba08f97fa9b33b679b6a02f616a787358dee3d7615ad",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-15/clock.json": "d523928bcad46b21ec2c6db3522d1937c5a3d601676f3b362ecb78daa6f84e8e",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-16/clock.json": "64979bc5ca002e4d3aa2dbc5946df3e06e6c62f05f6a20d91deddeae75c81dec",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-17/clock.json": "7105a0165e27e5c0e29b091776f73ad01103d5c2278adaa58b92d026dafd921c",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-18/clock.json": "ebeede2a8dd450864801232a555c470ec1136da5060a72da0364f29778fef468",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-19/clock.json": "de33b97428e0d77992b41924533241818a05244a932521af7a9546572671e6c7"
|
||||
},
|
||||
"git_paths": {
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/contract.json": "home-runs/track-a-h2-2026-11-18/contract.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/preparation-receipt.json": "home-runs/track-a-h2-2026-11-18/preparation-receipt.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-h2-entry-acceptance-2026-11-18/acceptance.json": "artifacts/track-a-h2-entry-acceptance-2026-11-18/acceptance.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-h2-entry-acceptance-2026-11-18/attempt-02.log": "artifacts/track-a-h2-entry-acceptance-2026-11-18/attempt-02.log",
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "tools/freeze_g1_components.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "tools/gre24_models.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "tools/phase2_gate_outcome_review.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "tools/phase2_h1_worker_v2.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "tools/phase2_h2_data.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "tools/phase2_h2_model.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "tools/phase2_h2_pilot_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "tools/phase2_h2_s1_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "tools/phase2_home_freeze_io.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "tools/phase2_local_freeze_io.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "tools/phase2_residual_math.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "tools/s2_gpu_idle_gate.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "tools/s2_main_memory_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "tools/s2_pilot_train.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "tools/s2_precision_continuation_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "tools/s2_recovery_gpu_supervisor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "tools/s2_recovery_monitoring.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_h2_entry.py": "tools/test_track_a_h2_entry.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "tools/track_a_h2_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_guard.py": "tools/track_a_h2_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "tools/track_a_h2_prepare.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_queue.py": "tools/track_a_h2_queue.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_worker.py": "tools/track_a_h2_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-existing-input-audit-2026-11-18/reuse-entry.json": "artifacts/track-a-existing-input-audit-2026-11-18/reuse-entry.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol/models-freeze.json": "artifacts/gre24-v4-reproduction-2026-11-16/protocol/models-freeze.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol/predictions-freeze.json": "artifacts/gre24-v4-reproduction-2026-11-16/protocol/predictions-freeze.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol/splits.json": "artifacts/gre24-v4-reproduction-2026-11-16/protocol/splits.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol/preparation-freeze.json": "artifacts/gre24-v4-reproduction-2026-11-16/protocol/preparation-freeze.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/geometry/manifest.json": "artifacts/phase2-h1-execution-2026-11-17/geometry/manifest.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17/contract.json": "artifacts/phase2-h2-pilot-2026-11-17/contract.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-labelled-recovery-2026-11-17/models-terminal.json": "artifacts/phase2-h1-labelled-recovery-2026-11-17/models-terminal.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-15/clock.json": "artifacts/phase2-h1-execution-2026-11-17/fits/job-15/clock.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-16/clock.json": "artifacts/phase2-h1-execution-2026-11-17/fits/job-16/clock.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-17/clock.json": "artifacts/phase2-h1-execution-2026-11-17/fits/job-17/clock.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-18/clock.json": "artifacts/phase2-h1-execution-2026-11-17/fits/job-18/clock.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase2-h1-execution-2026-11-17/fits/job-19/clock.json": "artifacts/phase2-h1-execution-2026-11-17/fits/job-19/clock.json"
|
||||
},
|
||||
"remote_published": false,
|
||||
"local_repository": "/tmp/meltmind-phase2-local-fixed-20261117"
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"status": "ENTRY_PREPARED_NOT_EXECUTED",
|
||||
"contract_sha256": "be4ac1f6d5a5d6c0183aa50d2e046b22d7e5fa77c7714fef7eab6d616e68bee7",
|
||||
"code_files": 23,
|
||||
"scientific_arrays_read": false,
|
||||
"new_fits_started": false,
|
||||
"prior_A_audit_rerun": false,
|
||||
"protocol_ruling": "1794975093.432709",
|
||||
"acceptance_sha256": "ca7acd7d9389744625c3ca64c3b3efabc6ecd872a1d4d0b83035b06fd3b34f9e",
|
||||
"original_metadata_cpu_seconds": 0.177887447
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
"""CPU synthetic regression checks. No real feature/target arrays, CUDA or fits."""
|
||||
import copy
|
||||
from contextlib import ExitStack
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from phase2_h1_worker_v2 import save
|
||||
from phase2_h2_data import load_part
|
||||
from track_a_h2_common import (dispatchable, execution_command, first_epoch_release,
|
||||
project_cell, project_total, read, validate_roles)
|
||||
import track_a_h2_guard as guard
|
||||
|
||||
ROOT = Path('/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18')
|
||||
|
||||
|
||||
class EntryTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.c = read(ROOT / 'contract.json')
|
||||
cls.entry = read(cls.c['reuse_entry'])
|
||||
|
||||
def test_01_exact_original_roles(self):
|
||||
validate_roles(self.c, self.entry)
|
||||
self.assertEqual([len(j['learning']) for j in self.c['jobs']], [311, 304, 307, 306, 310])
|
||||
|
||||
def test_02_test_to_train_rejected(self):
|
||||
c = copy.deepcopy(self.c)
|
||||
c['jobs'][0]['learning'][0] = c['jobs'][0]['test'][0]
|
||||
with self.assertRaises(ValueError):
|
||||
validate_roles(c, self.entry)
|
||||
|
||||
def test_03_extra_validation_rejected(self):
|
||||
c = copy.deepcopy(self.c)
|
||||
c['jobs'][0]['validation'] = [c['jobs'][0]['learning'][0]]
|
||||
with self.assertRaises(ValueError):
|
||||
validate_roles(c, self.entry)
|
||||
|
||||
def test_04_test_loader_rejects_before_io(self):
|
||||
with patch('phase2_h2_data.np.load', side_effect=AssertionError('Unexpected array read')):
|
||||
with self.assertRaises(PermissionError):
|
||||
load_part(dict(folds=[dict(learning=['a'], validation=[])]), 'b', {}, 'test', Path('/tmp'))
|
||||
|
||||
def test_05_clock_cannot_be_changed(self):
|
||||
c = copy.deepcopy(self.c)
|
||||
c['jobs'][0]['c'] += .01
|
||||
with self.assertRaises(ValueError):
|
||||
validate_roles(c, self.entry)
|
||||
|
||||
def test_06_two_slots_do_not_require_five_first_epochs(self):
|
||||
jobs = self.c['jobs']
|
||||
self.assertEqual([j['id'] for j in dispatchable(jobs, [])], [0, 1])
|
||||
self.assertEqual([j['id'] for j in dispatchable(jobs[2:], [jobs[1]])], [2])
|
||||
|
||||
def test_07_item1_gpu_protected(self):
|
||||
j = copy.deepcopy(self.c['jobs'][0])
|
||||
j['gpu_uuid'] = 'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c'
|
||||
with self.assertRaises(ValueError):
|
||||
dispatchable([j], [])
|
||||
|
||||
def test_08_existing_item1_is_not_duplicate_track_a(self):
|
||||
self.assertFalse(execution_command(['python', 'tools/phase3_data_sufficiency_v2_worker.py']))
|
||||
self.assertTrue(execution_command(['python', 'tools/track_a_h2_queue.py']))
|
||||
|
||||
def test_09_pending_caps_reserved(self):
|
||||
p = project_total(self.c, {}, {})
|
||||
self.assertEqual(p['projected_gpu_seconds'], 162000)
|
||||
self.assertEqual(p['status'], 'PASS')
|
||||
|
||||
def test_10_terminal_replaces_estimate_once(self):
|
||||
p = project_total(self.c, {0: dict(projected_gpu_seconds=30000)},
|
||||
{0: dict(gpu_seconds_charged=10000)})
|
||||
self.assertEqual(p['projected_gpu_seconds'], 162000 - 30960 + 10000)
|
||||
|
||||
def test_11_active_waits_cannot_disappear(self):
|
||||
p = project_total(self.c, {0: dict(projected_gpu_seconds=100)}, {}, {0: 31000})
|
||||
self.assertEqual(p['status'], 'STOP_BUDGET_PROJECTION')
|
||||
|
||||
def test_12_slow_first_epoch_stops(self):
|
||||
j = self.c['jobs'][0]
|
||||
self.assertFalse(project_cell(j, dict(epoch=1, max_epoch_seconds=200, charged=300))['within_cap'])
|
||||
self.assertTrue(project_cell(j, dict(epoch=1, max_epoch_seconds=50, charged=100))['within_cap'])
|
||||
|
||||
def test_13_invalid_projection_rejected(self):
|
||||
for d in [float('nan'), -1, 0, float('inf')]:
|
||||
with self.assertRaises(ValueError):
|
||||
project_cell(self.c['jobs'][0], dict(epoch=1, max_epoch_seconds=d, charged=100))
|
||||
|
||||
def test_14_first_epoch_global_stop(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
self.assertFalse(first_epoch_release(root, 0))
|
||||
save(root / 'global-stop.json', dict(reason='synthetic'))
|
||||
with self.assertRaises(RuntimeError):
|
||||
first_epoch_release(root, 0)
|
||||
|
||||
def guard_case(self, *, monitor_errors=None, peak=1024, worker_status='PASS'):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / 'jobs/0').mkdir(parents=True)
|
||||
c = copy.deepcopy(self.c)
|
||||
# No GPU query/process is used; preserve real guard decision paths.
|
||||
save(root / 'contract.json', c)
|
||||
def fake_supervise(*args, **kwargs):
|
||||
w = root / 'jobs/0/worker'
|
||||
w.mkdir()
|
||||
save(w / 'result.json', dict(status='COMPLETE', job=0,
|
||||
optimizer_steps=c['jobs'][0]['target_steps'], completed_epochs=200,
|
||||
peak_allocated_bytes=1024, peak_reserved_bytes=1024))
|
||||
if peak > 1024:
|
||||
save(w / 'failure.json', dict(peak_reserved_bytes=peak))
|
||||
return dict(status=worker_status, cpu_seconds=0,
|
||||
monitor_errors=monitor_errors or [])
|
||||
with patch.object(guard, 'config', return_value=(c, c)), \
|
||||
patch.object(guard, 'device_usage', return_value=(0, 0)), \
|
||||
patch.object(guard, 'supervise', side_effect=fake_supervise), \
|
||||
patch.object(guard.os, 'sched_setaffinity'), patch.object(guard.time, 'sleep'):
|
||||
rc = guard.run(root, 0)
|
||||
return rc, read(root / 'jobs/0/complete.json')
|
||||
|
||||
def test_15_guard_success_path(self):
|
||||
rc, r = self.guard_case()
|
||||
self.assertEqual(rc, 0)
|
||||
self.assertEqual(r['status'], 'PASS')
|
||||
|
||||
def test_16_worker_monitor_error_overrides_pass(self):
|
||||
rc, r = self.guard_case(monitor_errors=[dict(reason='synthetic monitor error')])
|
||||
self.assertEqual(rc, 1)
|
||||
self.assertIn('WORKER_MONITOR_ERRORS', r['reasons'])
|
||||
|
||||
def test_17_failure_peak_remains_enforced(self):
|
||||
rc, r = self.guard_case(peak=16 * 2**30, worker_status='FAILED')
|
||||
self.assertEqual(rc, 1)
|
||||
self.assertIn('RECORDED_VRAM_LIMIT', r['reasons'])
|
||||
|
||||
def test_18_foreign_ledger_job_rejected(self):
|
||||
with self.assertRaises(ValueError):
|
||||
project_total(self.c, {5: dict(projected_gpu_seconds=1)}, {})
|
||||
|
||||
def test_19_geometry_cannot_mix_versions(self):
|
||||
c = copy.deepcopy(self.c)
|
||||
case = next(iter(c['geometry']))
|
||||
c['geometry'][case]['protocol'] = 'grouped'
|
||||
with self.assertRaises(ValueError):
|
||||
validate_roles(c, self.entry)
|
||||
|
||||
def test_20_worker_all_epochs_no_validation_or_test(self):
|
||||
import track_a_h2_worker as worker
|
||||
import os
|
||||
class Model:
|
||||
def cuda(self): return self
|
||||
def parameters(self): return []
|
||||
def state_dict(self): return {}
|
||||
class Optimizer:
|
||||
param_groups = [{'lr': worker.lr_epoch(1)}]
|
||||
def state_dict(self): return {}
|
||||
job = dict(id=0, learning=['a', 'b'], validation=[], test=['sealed'], c=1.,
|
||||
learning_nodes=5, clock_sha256='synthetic', worker_cpus=[0], gpu_uuid='synthetic')
|
||||
c = dict(jobs=[job])
|
||||
accesses = []
|
||||
def load(view, case, stats, role, out):
|
||||
accesses.append((case, role))
|
||||
if case not in ['a', 'b'] or role != 'learning':
|
||||
raise AssertionError('Test/validation accessed')
|
||||
return dict(nodes=2 if case == 'a' else 3)
|
||||
with tempfile.TemporaryDirectory() as tmp, ExitStack() as stack:
|
||||
root = Path(tmp)
|
||||
(root / 'jobs/0').mkdir(parents=True)
|
||||
save(root / 'contract.json', c)
|
||||
stack.enter_context(patch.dict(os.environ, CUDA_VISIBLE_DEVICES='synthetic',
|
||||
PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True'))
|
||||
replacements = dict(config=lambda _: (c, c), standardization=lambda view, out: {},
|
||||
H2Geo=Model, optimizer=lambda _: Optimizer(), seed=lambda: None,
|
||||
peaks=lambda: dict(peak_allocated_bytes=0, peak_reserved_bytes=0),
|
||||
load_part=load, to_device=lambda data, device: data,
|
||||
train_step=lambda *a: dict(loss=0., gradient_norm_before_clip=0.),
|
||||
first_epoch_release=lambda *a: True)
|
||||
for name, value in replacements.items():
|
||||
stack.enter_context(patch.object(worker, name, value))
|
||||
for name in ['set_device', 'init', 'reset_peak_memory_stats']:
|
||||
stack.enter_context(patch.object(worker.torch.cuda, name))
|
||||
stack.enter_context(patch.object(worker.torch.cuda.memory, '_snapshot', return_value={}))
|
||||
stack.enter_context(patch.object(worker.os, 'sched_setaffinity'))
|
||||
stack.enter_context(patch.object(worker.torch, 'set_num_threads'))
|
||||
stack.enter_context(patch.object(worker.torch, 'set_num_interop_threads'))
|
||||
stack.enter_context(patch.object(worker.torch, 'save', side_effect=lambda v, p: p.write_bytes(b'synthetic')))
|
||||
worker.run(root, 0)
|
||||
result = read(root / 'jobs/0/worker/result.json')
|
||||
self.assertEqual(result['completed_epochs'], 200)
|
||||
self.assertEqual(result['optimizer_steps'], 400)
|
||||
self.assertEqual(accesses.count(('a', 'learning')), 200)
|
||||
self.assertEqual(accesses.count(('b', 'learning')), 200)
|
||||
self.assertFalse(result['test_targets_read'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Track A random-split identity and cost gates; no array or target reads."""
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
from phase2_h1_worker_v2 import checked, sha
|
||||
from phase2_home_freeze_io import verify
|
||||
|
||||
GPU_SECONDS = 162000
|
||||
PROTECTED_GPUS = {
|
||||
'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c',
|
||||
'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
|
||||
}
|
||||
|
||||
|
||||
def read(path):
|
||||
return json.loads(Path(path).read_text())
|
||||
|
||||
|
||||
def validate_roles(c, entry):
|
||||
if len(c['parts']) != 615 or len(c['jobs']) != 5:
|
||||
raise ValueError('Five original splits and 615 v4 parts required')
|
||||
if c['original_failed'] != entry['original_failed']:
|
||||
raise ValueError('Original failures changed')
|
||||
for case, original in entry['H2_inputs'].items():
|
||||
f = original['features_and_targets']
|
||||
expected = dict(case=case, output=f['output'], nodes=f['nodes'],
|
||||
t_set=f['t_scale_seconds'], output_sha256=f['output_sha256'])
|
||||
if c['parts'].get(case) != expected or c['geometry'].get(case) != original['geometry']:
|
||||
raise ValueError('Frozen native v4 input identity differs')
|
||||
for j, original in zip(c['jobs'], entry['jobs']):
|
||||
if j['id'] != original['split'] or j['split_seed'] != original['split_seed']:
|
||||
raise ValueError('Original split identity changed')
|
||||
for role in ('learning', 'test'):
|
||||
if j[role] != original[role] or len(set(j[role])) != len(j[role]):
|
||||
raise ValueError('Original membership/order changed: ' + role)
|
||||
if j['validation'] or set(j['learning']) & set(j['test']):
|
||||
raise ValueError('No new holdout or test leakage')
|
||||
clock = read(checked(j['clock_path'], j['clock_sha256']))
|
||||
if set(clock['parts']) != set(j['learning']) or len(clock['parts']) != len(j['learning']):
|
||||
raise ValueError('Clock learning membership differs')
|
||||
if clock['groups'] != ['part:' + x for x in clock['parts']]:
|
||||
raise ValueError('Original v4 equal-part clock groups differ')
|
||||
if j['c'] != clock['c'] or not math.isfinite(j['c']) or j['c'] <= 0:
|
||||
raise ValueError('Clock value differs')
|
||||
if j['learning_nodes'] != sum(c['parts'][x]['nodes'] for x in j['learning']):
|
||||
raise ValueError('Learning node count differs')
|
||||
if j['gpu_uuid'] in PROTECTED_GPUS or j['target_steps'] != 200 * len(j['learning']):
|
||||
raise ValueError('Protected GPU or epoch count')
|
||||
|
||||
|
||||
def config(root):
|
||||
root = Path(root)
|
||||
c = read(root / 'contract.json')
|
||||
verify(root, 'implementation')
|
||||
if (c['approval'] != '1794973932.596749' or c['gpu_cap'] != GPU_SECONDS
|
||||
or c['new_fits'] != 5 or c['seed'] != 0 or c['epochs'] != 200
|
||||
or not c['no_retry'] or c['train_gpu_cap'] + c['postfit_gpu_reserve'] != GPU_SECONDS):
|
||||
raise PermissionError('Frozen Track A scope differs')
|
||||
if c['protocol_ruling'] != '1794975093.432709':
|
||||
raise PermissionError('Actual protocol ruling differs')
|
||||
for p, h in {**c['source_sha256'], **c['code_sha256']}.items():
|
||||
checked(p, h)
|
||||
if sum(j['gpu_seconds'] for j in c['jobs']) != c['train_gpu_cap']:
|
||||
raise ValueError('Five job budgets must partition training cap')
|
||||
entry = read(c['reuse_entry'])
|
||||
validate_roles(c, entry)
|
||||
import numpy as np
|
||||
import torch
|
||||
if np.__version__ != c['versions']['numpy'] or torch.__version__ != c['versions']['torch']:
|
||||
raise ValueError('Original library versions differ')
|
||||
return c, c
|
||||
|
||||
|
||||
def view_for(c, job):
|
||||
# The shared loader rejects every case outside this split's learning role.
|
||||
return dict(c, folds=[job])
|
||||
|
||||
|
||||
def project_cell(job, observation):
|
||||
epoch = observation['epoch']
|
||||
duration = observation['max_epoch_seconds']
|
||||
charged = observation['charged']
|
||||
if (not isinstance(epoch, int) or not 1 <= epoch <= 200
|
||||
or not math.isfinite(duration) or duration <= 0
|
||||
or not math.isfinite(charged) or charged < 0):
|
||||
raise ValueError('Invalid full-epoch cost evidence')
|
||||
projected = charged + 1.25 * (200 - epoch) * duration + 120
|
||||
return dict(id=job['id'], projected_gpu_seconds=projected,
|
||||
within_cap=projected <= job['gpu_seconds'])
|
||||
|
||||
|
||||
def project_total(c, estimates, terminal, active_charges=None):
|
||||
"""Pending jobs retain full reserved caps; terminal costs replace estimates."""
|
||||
active_charges = active_charges or {}
|
||||
valid = {j['id'] for j in c['jobs']}
|
||||
if (set(estimates) | set(terminal) | set(active_charges)) - valid:
|
||||
raise ValueError('Unknown job in ledger')
|
||||
total = c['postfit_gpu_reserve']
|
||||
rows = []
|
||||
for j in c['jobs']:
|
||||
i = j['id']
|
||||
value = (terminal[i]['gpu_seconds_charged'] if i in terminal else
|
||||
max(active_charges.get(i, 0), estimates.get(i, {}).get('projected_gpu_seconds', j['gpu_seconds'])))
|
||||
if not math.isfinite(value) or value < 0:
|
||||
raise ValueError('Invalid ledger value')
|
||||
total += value
|
||||
rows.append(dict(id=i, projected_gpu_seconds=value, within_cap=value <= j['gpu_seconds']))
|
||||
return dict(status='PASS' if total <= c['gpu_cap'] and all(r['within_cap'] for r in rows)
|
||||
else 'STOP_BUDGET_PROJECTION', rows=rows, projected_gpu_seconds=total,
|
||||
postfit_reserve=c['postfit_gpu_reserve'])
|
||||
|
||||
|
||||
def first_epoch_release(root, jobid):
|
||||
if (root / 'global-stop.json').exists():
|
||||
raise RuntimeError('Track A global stop')
|
||||
p = root / 'jobs' / str(jobid) / 'first-epoch-projection.json'
|
||||
if not p.exists():
|
||||
return False
|
||||
r = read(p)
|
||||
if r['status'] != 'PASS' or r['contract_sha256'] != sha(root / 'contract.json'):
|
||||
raise RuntimeError('First-epoch cost gate rejected')
|
||||
return True
|
||||
|
||||
|
||||
def dispatchable(pending, active):
|
||||
occupied = {j['gpu_uuid'] for j in active}
|
||||
if len(occupied) != len(active) or len(active) > 2:
|
||||
raise ValueError('Active GPU collision')
|
||||
selected = []
|
||||
for j in pending:
|
||||
if len(active) + len(selected) == 2:
|
||||
break
|
||||
if j['gpu_uuid'] in PROTECTED_GPUS:
|
||||
raise ValueError('Item1 GPU reservation')
|
||||
if j['gpu_uuid'] not in occupied:
|
||||
selected.append(j)
|
||||
occupied.add(j['gpu_uuid'])
|
||||
return selected
|
||||
|
||||
|
||||
def execution_command(args):
|
||||
return any(Path(a).name in {'track_a_h2_queue.py', 'track_a_h2_guard.py',
|
||||
'track_a_h2_worker.py'} for a in args)
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Independent bounded TRACK_A fold guard; shared global stop and immutable S1."""
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
from track_a_h2_common import config
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
from s2_precision_continuation_guard import device_usage
|
||||
from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage
|
||||
|
||||
def run(queue,fold_id):
|
||||
c,_=config(queue);job=c['jobs'][fold_id];c=dict(c,**job);root=queue/'jobs'/str(fold_id);os.sched_setaffinity(0,c['guard_cpus'])
|
||||
with (root/'fold.lock').open('a') as lock, Path('/tmp/meltmind-track-a-'+c['gpu_uuid']+'.lock').open('a') as gpu_lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():raise FileExistsError('Single TRACK_A fold launch, no retry')
|
||||
start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None
|
||||
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(queue/'contract.json'),gpu_uuid=c['gpu_uuid'],setup_gpu_seconds=c['setup_gpu_seconds']))
|
||||
def elapsed():return c['setup_gpu_seconds']+time.monotonic()-start
|
||||
def reason(x):
|
||||
if x not in reasons:reasons.append(x)
|
||||
command=[sys.executable,'tools/track_a_h2_worker.py',str(queue),str(fold_id)]
|
||||
def kill_worker():
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:stop_group(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGINT,signal.SIGTERM]}
|
||||
def scan():
|
||||
if tracker.scan(root/'worker'):reason('RECORDED_VRAM_LIMIT')
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
try:
|
||||
scan();mem,util=device_usage(c['gpu_uuid']);_,rss,_=tree_usage(os.getpid())
|
||||
if mem>c['vram_bytes']:reason('DEVICE_VRAM_LIMIT')
|
||||
if rss>c['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT')
|
||||
if disk_usage(root)>c['disk_bytes']:reason('OUTPUT_DISK_LIMIT')
|
||||
if elapsed()>=c['gpu_seconds']-5:reason('TRACK_A_FOLD_TIME_LIMIT')
|
||||
if (queue/'global-stop.json').exists():reason('TRACK_A_GLOBAL_STOP')
|
||||
for x in requested:reason(x)
|
||||
save(root/'live-progress.json',dict(stage='TRACK_A_TRAINING',fold=fold_id,elapsed_gpu_seconds=elapsed(),device_bytes=mem,device_utilization=util,tree_rss_bytes=rss,recorded_peaks=tracker.maxima,reasons=reasons,unix=time.time()))
|
||||
except Exception as exc:reason('MONITOR_ERROR');errors.append(dict(type=type(exc).__name__,reason=str(exc)))
|
||||
if reasons:kill_worker()
|
||||
done.wait(.5)
|
||||
thread=None
|
||||
try:
|
||||
for i in range(3):
|
||||
mem,util=device_usage(c['gpu_uuid']);samples.append(dict(unix=time.time(),memory_bytes=mem,utilization=util))
|
||||
if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY')
|
||||
if i<2:time.sleep(2)
|
||||
save(root/'idle-polls.json',samples)
|
||||
if requested or (queue/'global-stop.json').exists():raise RuntimeError('STOP_BEFORE_WORKER_START')
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
remaining=c['gpu_seconds']-elapsed()-10
|
||||
receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=c['cpu_seconds']-60,job_cpu_seconds=c['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=c['gpu_uuid'])
|
||||
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill_worker()
|
||||
finally:
|
||||
done.set()
|
||||
if thread:thread.join(timeout=4)
|
||||
if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT')
|
||||
try:scan()
|
||||
except Exception as exc:reason('FINAL_PEAK_SCAN_ERROR');errors.append(dict(reason=str(exc)))
|
||||
for s,handler in old.items():signal.signal(s,handler)
|
||||
value=None;p=root/'worker/result.json'
|
||||
try:
|
||||
if p.exists():value=json.loads(p.read_text())
|
||||
except Exception as exc:reason('RESULT_READ_ERROR');errors.append(dict(reason=str(exc)))
|
||||
if not receipt or receipt['status']!='PASS':reason('WORKER_NOT_PASS')
|
||||
if receipt and receipt.get('monitor_errors'):reason('WORKER_MONITOR_ERRORS')
|
||||
if not value or value.get('status')!='COMPLETE':reason('NO_COMPLETE_TRAIN_RESULT')
|
||||
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_CUDA_PEAK_RECORDS')
|
||||
totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu
|
||||
if elapsed()>c['gpu_seconds'] or totalcpu>c['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
|
||||
try:config(queue)
|
||||
except Exception as exc:reason('FINAL_SOURCE_IDENTITY: '+str(exc))
|
||||
if value and (value.get('optimizer_steps')!=c['target_steps'] or value.get('job')!=fold_id or value.get('completed_epochs')!=200):reason('INCOMPLETE_EPOCH_OR_FOLD_COUNT')
|
||||
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',decision='TRAIN_COMPLETE_CHECKPOINT_FREEZE_PENDING' if not reasons else 'TRACK_A_FOLD_INCOMPLETE',reasons=reasons,monitor_errors=errors,
|
||||
worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=elapsed(),fold=fold_id,cpu_seconds_charged=totalcpu,
|
||||
worker_result_sha256=sha(p) if p.exists() else None,prediction_started=False,scoring_started=False,track_a_started=True,no_retry=True))
|
||||
return 0 if not reasons else 1
|
||||
|
||||
if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2])))
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Build the independent Track A contract using JSON metadata only. Never launch."""
|
||||
import ast
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from track_a_h2_common import read, validate_roles
|
||||
|
||||
BASE = Path(__file__).resolve().parents[1]
|
||||
ENTRY = BASE / 'artifacts/track-a-existing-input-audit-2026-11-18/reuse-entry.json'
|
||||
ENTRY_SHA = '8f0f3c1088d899b85aa3f578b7552f04cb409fd7a5dcfca3e17eaa3d7e25d0d6'
|
||||
RULING = '1794975093.432709'
|
||||
|
||||
|
||||
def local_code_closure(paths):
|
||||
found = set()
|
||||
pending = list(paths)
|
||||
while pending:
|
||||
p = Path(pending.pop()).resolve()
|
||||
if p in found:
|
||||
continue
|
||||
found.add(p)
|
||||
for node in ast.walk(ast.parse(p.read_text())):
|
||||
names = ([node.module] if isinstance(node, ast.ImportFrom) and node.module else
|
||||
[a.name for a in node.names] if isinstance(node, ast.Import) else [])
|
||||
for name in names:
|
||||
q = BASE / 'tools' / (name.split('.')[0] + '.py')
|
||||
if q.exists():
|
||||
pending.append(q)
|
||||
return {str(p): sha(p) for p in sorted(found)}
|
||||
|
||||
|
||||
def prepare(root):
|
||||
cpu = time.process_time()
|
||||
entry = read(checked(ENTRY, ENTRY_SHA))
|
||||
sources = {str(ENTRY): ENTRY_SHA}
|
||||
for key in ('A_models_freeze', 'A_predictions_freeze', 'split_manifest',
|
||||
'preparation_manifest', 'geometry_manifest', 'source_h2_recipe'):
|
||||
ref = entry[key]
|
||||
checked(ref['path'], ref['sha256'])
|
||||
sources[ref['path']] = ref['sha256']
|
||||
pilot = read(entry['source_h2_recipe']['path'])
|
||||
h1root = BASE / 'artifacts/phase2-h1-labelled-recovery-2026-11-17'
|
||||
h1models = read(h1root / 'models-terminal.json')['models']
|
||||
sources[str(h1root / 'models-terminal.json')] = sha(h1root / 'models-terminal.json')
|
||||
parts = {}
|
||||
geometry = {}
|
||||
for case, row in entry['H2_inputs'].items():
|
||||
f = row['features_and_targets']
|
||||
parts[case] = dict(case=case, output=f['output'], nodes=f['nodes'],
|
||||
t_set=f['t_scale_seconds'], output_sha256=f['output_sha256'])
|
||||
geometry[case] = row['geometry']
|
||||
if geometry[case]['protocol'] != 'v4_random' or geometry[case]['nodes'] != f['nodes']:
|
||||
raise ValueError('Native v4 geometry metadata mismatch')
|
||||
jobs = []
|
||||
gpus = ['GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76', 'GPU-8b71b032-1b75-2891-935c-ce106a53034a']
|
||||
for original in entry['jobs']:
|
||||
i = original['split']
|
||||
model = next(m for m in h1models if m['index'] == original['existing_H1_v4_job'])
|
||||
if model['status'] != 'PASS':
|
||||
raise ValueError('Original H1 clock unavailable')
|
||||
clock = read(checked(model['clock_path'], model['clock_sha256']))
|
||||
sources[model['clock_path']] = model['clock_sha256']
|
||||
jobs.append(dict(id=i, split_seed=original['split_seed'], fold=i,
|
||||
learning=original['learning'], test=original['test'], validation=[],
|
||||
missing_train=original['missing_train'], missing_test=original['missing_test'],
|
||||
clock_path=model['clock_path'], clock_sha256=model['clock_sha256'], c=clock['c'],
|
||||
learning_nodes=sum(parts[x]['nodes'] for x in original['learning']),
|
||||
learning_parts=len(original['learning']), validation_nodes=0,
|
||||
target_steps=200 * len(original['learning']), gpu_uuid=gpus[i % 2],
|
||||
worker_cpus=[8 + 2 * (i % 2)], guard_cpus=[28 + (i % 2)],
|
||||
gpu_seconds=30960, setup_gpu_seconds=60, cpu_seconds=61920,
|
||||
vram_bytes=15 * 2**30, rss_bytes=16 * 2**30, disk_bytes=8 * 2**30))
|
||||
code = local_code_closure([*[BASE / 'tools' / ('track_a_h2_' + name + '.py')
|
||||
for name in ('common', 'worker', 'guard', 'queue', 'prepare')],
|
||||
BASE / 'tools/test_track_a_h2_entry.py'])
|
||||
# Pin original scientific functions against the already frozen H2 recipe.
|
||||
for name in ('phase2_h2_data.py', 'phase2_h2_model.py', 'phase2_h2_pilot_worker.py', 'phase2_residual_math.py'):
|
||||
p = BASE / 'tools' / name
|
||||
pin = next(h for path, h in pilot['code_sha256'].items() if Path(path).name == name)
|
||||
checked(p, pin)
|
||||
c = dict(version='track-a-h2-v1', approval='1794973932.596749', protocol_ruling=RULING,
|
||||
reuse_entry=str(ENTRY), comparison_contract=str(ENTRY), comparison_contract_sha256=ENTRY_SHA,
|
||||
new_fits=5, A_refit=False, seed=0, epochs=200, no_retry=True, new_decodes=0,
|
||||
model=pilot['model'], versions=pilot['versions'], parts=parts, geometry=geometry, jobs=jobs,
|
||||
original_failed=entry['original_failed'], source_sha256=sources, code_sha256=code,
|
||||
gpu_cap=162000, train_gpu_cap=154800, postfit_gpu_reserve=7200,
|
||||
coordinator_cpus=[30, 31], aggregate_rss_bytes=40 * 2**30, aggregate_disk_bytes=48 * 2**30,
|
||||
normalization='Per-split learning-only original 19 columns, FP64 population mean/std; separate FP64 Vf skip',
|
||||
validation='None; no checkpoint selection; no test access in this training entry',
|
||||
selection='Last epoch 200 only', clock_policy='Reuse frozen v4 H1 equal successful train-part clock',
|
||||
postfit='Separate exact Assist review required before any test access; training never starts postfit',
|
||||
metric=entry['metric'], expected_pairing_counts_provisional=[304, 311, 308, 309, 305])
|
||||
validate_roles(c, entry)
|
||||
root.mkdir(parents=True, exist_ok=False)
|
||||
save(root / 'contract.json', c)
|
||||
save(root / 'preparation-receipt.json', dict(status='ENTRY_PREPARED_NOT_EXECUTED',
|
||||
contract_sha256=sha(root / 'contract.json'), code_files=len(code),
|
||||
clock_values_recomputed=False, scientific_arrays_read=False, new_fits_started=False,
|
||||
prior_A_audit_rerun=False, learning_counts=[len(j['learning']) for j in jobs],
|
||||
cpu_seconds=time.process_time() - cpu, protocol_ruling=RULING))
|
||||
print(json.dumps(read(root / 'preparation-receipt.json')))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
prepare(Path(sys.argv[1]).resolve())
|
||||
@@ -0,0 +1,205 @@
|
||||
"""Five independent fits on two idle GPUs; train and freeze only, never score."""
|
||||
import fcntl
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import psutil
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from phase2_home_freeze_io import freeze
|
||||
from s2_recovery_gpu_supervisor import disk_usage, tree_usage
|
||||
from track_a_h2_common import config, dispatchable, execution_command, project_cell, project_total, read
|
||||
|
||||
|
||||
def run(root):
|
||||
c, _ = config(root)
|
||||
validate_review(root)
|
||||
lock = (root / 'queue.lock').open('a')
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if (root / 'started.json').exists():
|
||||
raise FileExistsError('One Track A attempt; no retry')
|
||||
for p in psutil.process_iter(['pid', 'cmdline']):
|
||||
if p.pid != os.getpid() and execution_command(p.info['cmdline'] or []):
|
||||
raise RuntimeError('Existing Track A execution')
|
||||
os.sched_setaffinity(0, c['coordinator_cpus'])
|
||||
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
|
||||
start = time.monotonic()
|
||||
cpu = time.process_time()
|
||||
active, terminal, estimates, last_epoch = {}, {}, {}, {}
|
||||
pending = list(c['jobs'])
|
||||
requested = []
|
||||
|
||||
def stop(reason):
|
||||
if not (root / 'global-stop.json').exists():
|
||||
save(root / 'global-stop.json', dict(reason=reason, unix=time.time(), no_retry=True))
|
||||
|
||||
def active_charges():
|
||||
return {i: c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start']
|
||||
for i, v in active.items()}
|
||||
|
||||
def charged():
|
||||
return sum(v['gpu_seconds_charged'] for v in terminal.values()) + sum(active_charges().values())
|
||||
|
||||
def terminate(v):
|
||||
if v.get('stop_sent') is None:
|
||||
if v['process'].poll() is None:
|
||||
v['process'].send_signal(signal.SIGTERM)
|
||||
v['stop_sent'] = time.monotonic()
|
||||
elif time.monotonic() - v['stop_sent'] > 20:
|
||||
try:
|
||||
parent = psutil.Process(v['process'].pid)
|
||||
for p in reversed(parent.children(recursive=True)):
|
||||
try:
|
||||
p.kill()
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
parent.kill()
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
|
||||
def collect(i, v):
|
||||
rc = v['process'].poll()
|
||||
if rc is None:
|
||||
return False
|
||||
p = root / 'jobs' / str(i) / 'complete.json'
|
||||
r = read(p) if p.exists() else None
|
||||
observed = c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start']
|
||||
ok = (rc == 0 and r is not None and r['status'] == 'PASS'
|
||||
and not r.get('monitor_errors') and r.get('worker') is not None
|
||||
and not r['worker'].get('monitor_errors'))
|
||||
terminal[i] = dict(id=i, status='PASS' if ok else 'STOPPED_OR_FAILED', returncode=rc,
|
||||
gpu_seconds_charged=max(observed, r['gpu_seconds_charged'] if r else 0),
|
||||
guard_complete_sha256=sha(p) if r else None)
|
||||
v['log'].close()
|
||||
del active[i]
|
||||
save(root / 'terminal-ledger.json', dict(entries=list(terminal.values()),
|
||||
settled_gpu_seconds=sum(x['gpu_seconds_charged'] for x in terminal.values())))
|
||||
if not ok:
|
||||
stop('JOB_' + str(i) + '_FAILED')
|
||||
return True
|
||||
|
||||
def dispatch():
|
||||
for j in dispatchable(pending, [c['jobs'][i] for i in active]):
|
||||
i = j['id']
|
||||
d = root / 'jobs' / str(i)
|
||||
d.mkdir(parents=True, exist_ok=False)
|
||||
log = (d / 'entry.log').open('x')
|
||||
begin = time.monotonic()
|
||||
cmd = [sys.executable, 'tools/track_a_h2_guard.py', str(root), str(i)]
|
||||
try:
|
||||
p = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
except BaseException:
|
||||
log.close()
|
||||
raise
|
||||
active[i] = dict(process=p, log=log, start=begin, stop_sent=None)
|
||||
pending.remove(j)
|
||||
save(d / 'dispatch.json', dict(pid=p.pid, command=cmd, unix=time.time(), gpu_uuid=j['gpu_uuid']))
|
||||
|
||||
handlers = {s: signal.signal(s, lambda n, f: requested.append('SIGNAL_' + str(n)))
|
||||
for s in (signal.SIGTERM, signal.SIGINT)}
|
||||
save(root / 'started.json', dict(pid=os.getpid(), unix=time.time(), new_fits=5,
|
||||
contract_sha256=sha(root / 'contract.json'), review_sha256=sha(root / 'review.json'),
|
||||
gpu_cap=c['gpu_cap'], A_refit=False, test_targets_read=False))
|
||||
try:
|
||||
while active or pending:
|
||||
for i, v in list(active.items()):
|
||||
collect(i, v)
|
||||
for reason in requested:
|
||||
stop(reason)
|
||||
amount = charged()
|
||||
_, rss, _ = tree_usage(os.getpid())
|
||||
if amount >= c['train_gpu_cap'] - 80:
|
||||
stop('CUMULATIVE_TRAIN_GPU_BOUNDARY')
|
||||
if rss > c['aggregate_rss_bytes'] or disk_usage(root) > c['aggregate_disk_bytes']:
|
||||
stop('AGGREGATE_MEMORY_DISK_LIMIT')
|
||||
if not (root / 'global-stop.json').exists():
|
||||
for i in active:
|
||||
files = sorted((root / 'jobs' / str(i) / 'worker/history').glob('*.json'))
|
||||
rows = [read(p) for p in files]
|
||||
if not rows:
|
||||
continue
|
||||
epoch = rows[-1]['epoch']
|
||||
# Recalculate every epoch: waiting, setup and observed cost stay charged.
|
||||
if epoch > last_epoch.get(i, 0):
|
||||
observation = dict(epoch=epoch, charged=active_charges()[i],
|
||||
max_epoch_seconds=max(r['train_epoch_seconds'] for r in rows))
|
||||
cell = project_cell(c['jobs'][i], observation)
|
||||
estimates[i] = cell
|
||||
projection = dict(**project_total(c, estimates, terminal, active_charges()),
|
||||
cell=cell, contract_sha256=sha(root / 'contract.json'), unix=time.time())
|
||||
name = 'first-epoch-projection.json' if i not in last_epoch else 'latest-cost-projection.json'
|
||||
save(root / 'jobs' / str(i) / name, projection)
|
||||
save(root / 'latest-cost-projection.json', projection)
|
||||
last_epoch[i] = epoch
|
||||
if projection['status'] != 'PASS':
|
||||
stop('PROJECTED_TRACK_A_BUDGET')
|
||||
if not (root / 'global-stop.json').exists():
|
||||
dispatch()
|
||||
if (root / 'global-stop.json').exists():
|
||||
for v in active.values():
|
||||
terminate(v)
|
||||
save(root / 'live-progress.json', dict(stage='STOPPING' if (root / 'global-stop.json').exists()
|
||||
else 'TRACK_A_TRAINING', unix=time.time(), active=[dict(id=i, guard_pid=v['process'].pid)
|
||||
for i, v in active.items()], terminal=list(terminal.values()), pending=[j['id'] for j in pending],
|
||||
charged_training_gpu_seconds=charged(), postfit_gpu_reserve=c['postfit_gpu_reserve']))
|
||||
if (root / 'global-stop.json').exists() and not active:
|
||||
break
|
||||
if active:
|
||||
time.sleep(1)
|
||||
if (root / 'global-stop.json').exists() or len(terminal) != 5 or any(v['status'] != 'PASS' for v in terminal.values()):
|
||||
raise RuntimeError('Track A training incomplete')
|
||||
config(root)
|
||||
models = []
|
||||
for j in c['jobs']:
|
||||
w = root / 'jobs' / str(j['id']) / 'worker'
|
||||
r = read(w / 'result.json')
|
||||
stats = read(checked(w / 'standardization.json', r['standardization_sha256']))
|
||||
if (r['optimizer_steps'] != j['target_steps'] or r['completed_epochs'] != 200
|
||||
or stats['parts'] != j['learning'] or r['test_targets_read']):
|
||||
raise ValueError('Terminal model identity')
|
||||
checked(r['checkpoint'], r['checkpoint_sha256'])
|
||||
models.append(dict(job=j['id'], path=r['checkpoint'], sha256=r['checkpoint_sha256'],
|
||||
standardization=str(w / 'standardization.json'),
|
||||
standardization_sha256=r['standardization_sha256']))
|
||||
if charged() > c['train_gpu_cap']:
|
||||
raise ValueError('Terminal cumulative cost')
|
||||
save(root / 'models-terminal.json', dict(models=models, new_fits=5, A_refit=False, test_prediction_started=False))
|
||||
freeze(root, 'models', [root / 'contract.json', root / 'models-terminal.json', root / 'terminal-ledger.json',
|
||||
*[Path(m['path']) for m in models], *[Path(m['standardization']) for m in models]])
|
||||
save(root / 'complete.json', dict(status='TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING', entries=list(terminal.values()),
|
||||
training_gpu_seconds=charged(), postfit_gpu_reserve=c['postfit_gpu_reserve'], gpu_cap=c['gpu_cap'],
|
||||
coordinator_cpu_seconds=time.process_time() - cpu, wall_seconds=time.monotonic() - start,
|
||||
no_retry=True, test_targets_read=False, postfit_started=False))
|
||||
freeze(root, 'terminal', [root / 'contract.json', root / 'complete.json', root / 'models-terminal.json'])
|
||||
return 0
|
||||
except BaseException as exc:
|
||||
stop(type(exc).__name__ + ': ' + str(exc))
|
||||
while active:
|
||||
for i, v in list(active.items()):
|
||||
try:
|
||||
if not collect(i, v):
|
||||
terminate(v)
|
||||
except Exception:
|
||||
terminate(v)
|
||||
if v['process'].poll() is not None:
|
||||
terminal[i] = dict(id=i, status='MALFORMED_TERMINAL',
|
||||
gpu_seconds_charged=c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start'])
|
||||
v['log'].close()
|
||||
del active[i]
|
||||
if active:
|
||||
time.sleep(.5)
|
||||
save(root / 'complete.json', dict(status='STOPPED', reason=str(exc), entries=list(terminal.values()),
|
||||
never_attempted=[j['id'] for j in pending], training_gpu_seconds=charged(),
|
||||
postfit_started=False, no_retry=True))
|
||||
return 1
|
||||
finally:
|
||||
for s, handler in handlers.items():
|
||||
signal.signal(s, handler)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(run(Path(sys.argv[1])))
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Track A v4 random split; same H2 seed0/200epochs, learning-only, no test access."""
|
||||
import json,os,sys,time,traceback
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase2_h2_s1_common import selected_checkpoint
|
||||
from track_a_h2_common import config,view_for,first_epoch_release
|
||||
from phase2_h2_data import standardization
|
||||
from phase2_h2_data import load_part,to_device,arguments,cached
|
||||
from phase2_h2_model import H2Geo,optimizer,lr_epoch
|
||||
from phase2_h2_pilot_worker import seed,peaks,train_step
|
||||
from phase2_residual_math import residual_target
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
|
||||
def run(root,fold_id):
|
||||
c,_=config(root);job=c['jobs'][fold_id];old=view_for(c,job);out=root/'jobs'/str(fold_id)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir()
|
||||
os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
|
||||
start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0
|
||||
def record(item,epoch_file=False):
|
||||
item=dict(fold=fold_id,stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks())
|
||||
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
|
||||
save(out/'progress.json',item)
|
||||
if epoch_file:save(out/'history'/f'{epoch:03d}.json',item)
|
||||
return item
|
||||
try:
|
||||
if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity')
|
||||
stage='LEARNING_ONLY_NORMALIZATION';normal=out/'normalization';normal.mkdir();stats=standardization(old,normal);save(out/'standardization.json',stats);save(out/'fold-identity.json',dict(actual_fold=fold_id,learning=old['folds'][0]['learning'],validation=old['folds'][0]['validation'],test=old['folds'][0]['test'],clock_sha256=old['folds'][0]['clock_sha256']))
|
||||
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
|
||||
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed()
|
||||
model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes']
|
||||
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=job['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters())))
|
||||
for epoch in range(1,201):
|
||||
stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0.
|
||||
if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch')
|
||||
order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts)
|
||||
for index in order:
|
||||
case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda')
|
||||
result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss']
|
||||
record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip']))
|
||||
del data
|
||||
if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch')
|
||||
metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch))
|
||||
stage='EPOCH_COMPLETE';record(metrics,epoch_file=True)
|
||||
if epoch==1:
|
||||
stage='WAIT_OWN_FIRST_EPOCH_COST_GATE';record(dict(epoch_parts=seen,epoch_nodes=nodes))
|
||||
while not first_epoch_release(root,fold_id):time.sleep(.5)
|
||||
if epoch<200:
|
||||
for group in opt.param_groups:group['lr']=lr_epoch(epoch+1)
|
||||
stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch)
|
||||
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=fold_id,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name)
|
||||
save(out/'result.json',dict(status='COMPLETE',job=fold_id,fold=fold_id,completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks()))
|
||||
except BaseException as exc:
|
||||
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
|
||||
save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
|
||||
if __name__=='__main__':run(Path(sys.argv[1]),int(sys.argv[2]))
|
||||
Reference in new issue
Block a user