Archive both protective stops and publish the reviewed-scope Track A four-split binding
This commit is contained in:
1 parent
18398c2b2f
commit
482c490afb
35 files changed
+229770
No files matched your search
@@ -0,0 +1,26 @@
|
||||
test_all_node_proxy_not_relative_l2 (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_bootstrap_order_invariant_and_deterministic (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_bootstrap_zero_difference_degenerate_interval (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_fixed_last_epoch_identity (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_missing_visit_not_silently_averaged (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_no_target_io_before_actual_review (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_no_target_io_before_predictions_frozen (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_postfit_reserve_and_training_charged_once (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_prediction_manifest_native_id_and_duplicate_rejection (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_same_paired_support_five_summary_means (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_test_loader_rejects_foreign_case_before_io (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_undefined_candidates_and_visit_counts_retained (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_undefined_pairs_excluded_symmetrically (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_unique_part_bootstrap_not_visit_weighted (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_zero_truth_is_explicitly_undefined (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_exact_split_ids_and_visit_counts (test_track_a_postfit_rebind.RebindTests) ... ok
|
||||
test_four_split_summary_uses_original_part_bootstrap (test_track_a_postfit_rebind.RebindTests) ... ok
|
||||
test_incomplete_duplicate_or_extra_visits_rejected (test_track_a_postfit_rebind.RebindTests) ... ok
|
||||
test_mixed_source_checkpoint_identity (test_track_a_postfit_rebind.RebindTests) ... ok
|
||||
test_scoring_requires_new_review_before_target_access (test_track_a_postfit_rebind.RebindTests) ... ok
|
||||
test_scoring_requires_prediction_freeze_before_target_access (test_track_a_postfit_rebind.RebindTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 21 tests in 0.014s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,117 @@
|
||||
{
|
||||
"ruling": "1795022093.308479",
|
||||
"track_a": {
|
||||
"status": "RECOVERY_INCOMPLETE_NO_FURTHER_RECOVERY",
|
||||
"available_folds": [
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"gpu_seconds": 78579.79901044304,
|
||||
"original_files": [
|
||||
{
|
||||
"path": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/complete.json",
|
||||
"sha256": "2e9568c68ba03acfa099039b801c878474a9a5f227da0f2b425c1799b12d673b",
|
||||
"bytes": 1094
|
||||
},
|
||||
{
|
||||
"path": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/terminal-git-freeze.json",
|
||||
"sha256": "47cecc48e3427d40d9a6dcf9ada09e0bdff1a9a95bf1f22b035a27223c96ff76",
|
||||
"bytes": 1387
|
||||
},
|
||||
{
|
||||
"path": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/terminal-ledger.json",
|
||||
"sha256": "285a097f9e26f9cb3412ecfa24a7ca67a35d80b7e9fdceaade1898baf7d050cb",
|
||||
"bytes": 845
|
||||
},
|
||||
{
|
||||
"path": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/available-models-incomplete.json",
|
||||
"sha256": "9acef8095110af43d18d03345b39a1e5029e5a4c721b705cb896e2dabdf3a055",
|
||||
"bytes": 2814
|
||||
},
|
||||
{
|
||||
"path": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/incomplete-models-git-freeze.json",
|
||||
"sha256": "c33dafe09d393445b4735050167e305734ada09152d536b04df43b8a705d8d8a",
|
||||
"bytes": 3688
|
||||
},
|
||||
{
|
||||
"path": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/3/stop.json",
|
||||
"sha256": "e17a02303fc3e1f9ee0a5a2e284ee1c48efd84d2737b42862e2d2182169a72e5",
|
||||
"bytes": 113
|
||||
},
|
||||
{
|
||||
"path": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/3/first-epoch-projection.json",
|
||||
"sha256": "c37524b96d70fd68159df223d93d397acdfa5923c66b284a89adc54320b79161",
|
||||
"bytes": 759
|
||||
},
|
||||
{
|
||||
"path": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/3/complete.json",
|
||||
"sha256": "a6a5fd3d6b457a22179540caf5c25c9b0cb96914b944b81a8522631e8f1af58a",
|
||||
"bytes": 49437
|
||||
},
|
||||
{
|
||||
"path": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/complete.json",
|
||||
"sha256": "6c7fb910b9fc928596a080e5ca254dc9c6af6a06d2eea6f16c89183434df4b4f",
|
||||
"bytes": 8985187
|
||||
}
|
||||
]
|
||||
},
|
||||
"item1b": {
|
||||
"status": "PARKED_NO_FURTHER_COST",
|
||||
"terminal_status": "STOPPED_INCOMPLETE",
|
||||
"completed_checkpoints": [],
|
||||
"attempted": [
|
||||
1,
|
||||
0,
|
||||
2
|
||||
],
|
||||
"never_attempted": [
|
||||
3,
|
||||
4
|
||||
],
|
||||
"gpu_seconds": 19262.057604262838,
|
||||
"original_files": [
|
||||
{
|
||||
"path": "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/complete.json",
|
||||
"sha256": "430247e44f301f984f396c662e6fa5419d09649476f6b2a81076444731f55dde",
|
||||
"bytes": 1118
|
||||
},
|
||||
{
|
||||
"path": "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/training-stop-git-freeze.json",
|
||||
"sha256": "0529c018e67f8abc3a77edc0d37dc287576afe25bff85ad18ada79fcdf461604",
|
||||
"bytes": 739
|
||||
},
|
||||
{
|
||||
"path": "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/global-stop.json",
|
||||
"sha256": "d7e8b6c3d06a826f267bb1b873c0d73ac7af45ea222a8dd7b9414645837a4bfd",
|
||||
"bytes": 93
|
||||
},
|
||||
{
|
||||
"path": "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/latest-cost-projection.json",
|
||||
"sha256": "96792d9c310e1eebc91bce286187df96d8af76692302ff0699476d61e41f7ff3",
|
||||
"bytes": 678
|
||||
},
|
||||
{
|
||||
"path": "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/post-terminal-cost.json",
|
||||
"sha256": "7241fe629e3227b81655886c6c208336deef443d67c8d6efd7530000de8477bf",
|
||||
"bytes": 305
|
||||
},
|
||||
{
|
||||
"path": "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/1/complete.json",
|
||||
"sha256": "5005fd88a828079f2fce35993a59ef973905c2b0491443a06bf4e65b82acb2b8",
|
||||
"bytes": 2966
|
||||
},
|
||||
{
|
||||
"path": "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/complete.json",
|
||||
"sha256": "e699c79fb5101613016323eebcf90489ba5549599dc3aeff168735be51bdbb81",
|
||||
"bytes": 2971
|
||||
},
|
||||
{
|
||||
"path": "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/2/complete.json",
|
||||
"sha256": "4880ac220620b67839f79b042e7f47ca42024d0d896495a2a747ef5f3342687a",
|
||||
"bytes": 2964
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
# Track A postfit binding for four of five splits
|
||||
|
||||
Claude approved this scope in `1795022093.308479`. Use original splits 0, 1 and 2 and recovered split 4. Split 3 stopped at its first-epoch budget gate and has no completed checkpoint. The original and recovery training receipts, the incomplete four-model freeze and the original five-split postfit binding stay available for audit.
|
||||
|
||||
The four original test lists contain 304, 311, 308 and 305 valid visits respectively, for 1,228 paired visits over 571 unique parts. A is the original frozen GBRT prediction on each of those visits. H2 uses each original last-epoch checkpoint and its own learning-only normalization. The checkpoint's training contract must match its original or recovery source. The 14 original preparation failures remain disclosed. The split 3 omission is carried in the contract and in any eventual score report. No split is replaced or relabelled as a pass.
|
||||
|
||||
Keep full-node `RMSE / max(full truth)`, paired A and H2 support, per-split summaries and their mean over the four evaluated splits. The existing descriptive bootstrap still averages visits within each part first, then weights parts equally. It uses 2,000 replicates, seed `2026111408` and the same percentile interval. No significance gate is introduced. The exact new independent pre-score PASS and a complete frozen four-split prediction manifest are required before reading the 571 target arrays. The previous five-split review belongs to the previous binding.
|
||||
|
||||
The new root is `/home/frank14f/MeltMind-runs/track-a-postfit-four-split-2026-11-19`. The original and recovery training charges total 78,579.79901044304 GPU seconds, counted once. GPU2, the idle condition, per-fold guards, 7,200 GPU-second and 3,600 CPU-second postfit caps and 162,000 GPU-second Track A total remain as approved. Preparation performs metadata and opaque hash checks plus CPU synthetic verification. This delivery goes to MeltAssist for independent pre-score review before any launch.
|
||||
|
||||
Item 1b is parked without further cost. Its three attempted folds had no complete 100-epoch checkpoint. The phase 3 ledger retains the 19,262.057604262838 GPU seconds already charged, including the terminal freeze tail. Any new 1b design is a later stage-review decision once seed 1 and c have reported.
|
||||
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"status": "STOPPED_INCOMPLETE",
|
||||
"failure": "RuntimeError: TOTAL_PROJECTION_OR_ACTUAL_CAP",
|
||||
"entries": [
|
||||
{
|
||||
"id": 1,
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"returncode": 1,
|
||||
"gpu_seconds_charged": 3732.392637782963,
|
||||
"guard_sha256": "5005fd88a828079f2fce35993a59ef973905c2b0491443a06bf4e65b82acb2b8"
|
||||
},
|
||||
{
|
||||
"id": 0,
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"returncode": 1,
|
||||
"gpu_seconds_charged": 3732.394393248949,
|
||||
"guard_sha256": "e699c79fb5101613016323eebcf90489ba5549599dc3aeff168735be51bdbb81"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"returncode": 1,
|
||||
"gpu_seconds_charged": 3732.395846397965,
|
||||
"guard_sha256": "4880ac220620b67839f79b042e7f47ca42024d0d896495a2a747ef5f3342687a"
|
||||
}
|
||||
],
|
||||
"never_attempted": [
|
||||
3,
|
||||
4
|
||||
],
|
||||
"never_attempted_wait_charges": {
|
||||
"3": 3732.397045093938,
|
||||
"4": 3732.3970463059377
|
||||
},
|
||||
"cumulative_gpu_seconds": 19261.976968829753,
|
||||
"postfit_gpu_reserve": 7200,
|
||||
"no_retry": true,
|
||||
"test_targets_read": false,
|
||||
"prediction_started": false,
|
||||
"scoring_started": false
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"reason": "RuntimeError: TOTAL_PROJECTION_OR_ACTUAL_CAP",
|
||||
"unix": 1795021405.4632003
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
{
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"reasons": [
|
||||
"GLOBAL_STOP",
|
||||
"SIGNAL_15",
|
||||
"WORKER_NOT_PASS",
|
||||
"NO_COMPLETE_IDENTITY"
|
||||
],
|
||||
"worker": {
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/phase3_item1b_worker.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18",
|
||||
"0",
|
||||
"050e8b6748a3e2a287eb905f75179daa9941bd5690660f2c2716298fbf71bd1d"
|
||||
],
|
||||
"returncode": -9,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 104.23424299999999,
|
||||
"wall_seconds": 108.10220872296486,
|
||||
"peak_tree_rss_bytes": 1431515136,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 8192,
|
||||
"max_poll_gap_seconds": 0.8025802979245782,
|
||||
"poll_target_seconds": 0.2,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": true,
|
||||
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null,
|
||||
"monitor_policy": {
|
||||
"rule": "three_consecutive_failed_samples_per_operation",
|
||||
"failure_limit": 3,
|
||||
"failed": false,
|
||||
"errors": 0,
|
||||
"streaks": {
|
||||
"initial_disk": 0,
|
||||
"tree_usage": 0,
|
||||
"live_disk": 0,
|
||||
"final_disk": 0
|
||||
},
|
||||
"max_streaks": {}
|
||||
}
|
||||
},
|
||||
"worker_result_sha256": null,
|
||||
"cpu_seconds_charged": 120.435661578,
|
||||
"pre_dispatch_wait_seconds": 3558.4185484119225,
|
||||
"guard_wall_seconds": 108.39543320192024,
|
||||
"recorded_memory": {
|
||||
"maxima": {
|
||||
"peak_allocated_bytes": 1334860288,
|
||||
"peak_reserved_bytes": 1388314624
|
||||
},
|
||||
"witnesses": {
|
||||
"peak_allocated_bytes": {
|
||||
"path": "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/worker/progress.json",
|
||||
"sha256": "ebbefc1b4e3a1485a2cd906d04aedf8cfc6e94abd249cc1f6c56b5f2257e6667",
|
||||
"observed_unix": 1795021338.519215,
|
||||
"peaks": {
|
||||
"peak_allocated_bytes": 1334860288,
|
||||
"peak_reserved_bytes": 1388314624
|
||||
}
|
||||
},
|
||||
"peak_reserved_bytes": {
|
||||
"path": "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/worker/progress.json",
|
||||
"sha256": "ebbefc1b4e3a1485a2cd906d04aedf8cfc6e94abd249cc1f6c56b5f2257e6667",
|
||||
"observed_unix": 1795021338.519215,
|
||||
"peaks": {
|
||||
"peak_allocated_bytes": 1334860288,
|
||||
"peak_reserved_bytes": 1388314624
|
||||
}
|
||||
}
|
||||
},
|
||||
"observations_count": 159
|
||||
},
|
||||
"monitor_policy": {
|
||||
"rule": "three_consecutive_failed_samples_per_operation",
|
||||
"failure_limit": 3,
|
||||
"failed": false,
|
||||
"errors": 0,
|
||||
"streaks": {
|
||||
"recorded_peaks": 0,
|
||||
"tree": 0,
|
||||
"disk": 0,
|
||||
"queue_identity": 0,
|
||||
"gpu_processes": 0,
|
||||
"final_peaks": 0
|
||||
},
|
||||
"max_streaks": {}
|
||||
},
|
||||
"monitor_errors": [],
|
||||
"no_retry": true
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
{
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"reasons": [
|
||||
"GLOBAL_STOP",
|
||||
"SIGNAL_15",
|
||||
"WORKER_NOT_PASS",
|
||||
"NO_COMPLETE_IDENTITY"
|
||||
],
|
||||
"worker": {
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/phase3_item1b_worker.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18",
|
||||
"1",
|
||||
"df4ac7547e8e7d6d7b034f8cdb6bbb6b41b3ce840f18110f8718f64aff414332"
|
||||
],
|
||||
"returncode": -9,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 3663.864256,
|
||||
"wall_seconds": 3666.8987807390513,
|
||||
"peak_tree_rss_bytes": 1460621312,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 4096,
|
||||
"max_poll_gap_seconds": 1.098231279058382,
|
||||
"poll_target_seconds": 0.2,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": true,
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null,
|
||||
"monitor_policy": {
|
||||
"rule": "three_consecutive_failed_samples_per_operation",
|
||||
"failure_limit": 3,
|
||||
"failed": false,
|
||||
"errors": 0,
|
||||
"streaks": {
|
||||
"initial_disk": 0,
|
||||
"tree_usage": 0,
|
||||
"live_disk": 0,
|
||||
"final_disk": 0
|
||||
},
|
||||
"max_streaks": {}
|
||||
}
|
||||
},
|
||||
"worker_result_sha256": null,
|
||||
"cpu_seconds_charged": 4204.10390978,
|
||||
"pre_dispatch_wait_seconds": 0.24245897098444402,
|
||||
"guard_wall_seconds": 3666.913833159022,
|
||||
"recorded_memory": {
|
||||
"maxima": {
|
||||
"peak_allocated_bytes": 1334860288,
|
||||
"peak_reserved_bytes": 1390411776
|
||||
},
|
||||
"witnesses": {
|
||||
"peak_allocated_bytes": {
|
||||
"path": "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/1/worker/progress.json",
|
||||
"sha256": "5bd22093443fc327b09b91c043ccd318af7ad1edf62a6e9d8fbaca36d2f91b2e",
|
||||
"observed_unix": 1795017756.1968787,
|
||||
"peaks": {
|
||||
"peak_allocated_bytes": 1334860288,
|
||||
"peak_reserved_bytes": 1390411776
|
||||
}
|
||||
},
|
||||
"peak_reserved_bytes": {
|
||||
"path": "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/1/worker/progress.json",
|
||||
"sha256": "5bd22093443fc327b09b91c043ccd318af7ad1edf62a6e9d8fbaca36d2f91b2e",
|
||||
"observed_unix": 1795017756.1968787,
|
||||
"peaks": {
|
||||
"peak_allocated_bytes": 1334860288,
|
||||
"peak_reserved_bytes": 1390411776
|
||||
}
|
||||
}
|
||||
},
|
||||
"observations_count": 6385
|
||||
},
|
||||
"monitor_policy": {
|
||||
"rule": "three_consecutive_failed_samples_per_operation",
|
||||
"failure_limit": 3,
|
||||
"failed": false,
|
||||
"errors": 0,
|
||||
"streaks": {
|
||||
"recorded_peaks": 0,
|
||||
"tree": 0,
|
||||
"disk": 0,
|
||||
"queue_identity": 0,
|
||||
"gpu_processes": 0,
|
||||
"final_peaks": 0
|
||||
},
|
||||
"max_streaks": {}
|
||||
},
|
||||
"monitor_errors": [],
|
||||
"no_retry": true
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
{
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"reasons": [
|
||||
"GLOBAL_STOP",
|
||||
"SIGNAL_15",
|
||||
"WORKER_NOT_PASS",
|
||||
"NO_COMPLETE_IDENTITY"
|
||||
],
|
||||
"worker": {
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/phase3_item1b_worker.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18",
|
||||
"2",
|
||||
"a2699b9e9174eb7ba8a4b54a4764f58f28fb794be0008083f36765c738035b26"
|
||||
],
|
||||
"returncode": -9,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 105.068737,
|
||||
"wall_seconds": 107.40005897299852,
|
||||
"peak_tree_rss_bytes": 1471344640,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 4096,
|
||||
"max_poll_gap_seconds": 0.9828807739540935,
|
||||
"poll_target_seconds": 0.2,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": true,
|
||||
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null,
|
||||
"monitor_policy": {
|
||||
"rule": "three_consecutive_failed_samples_per_operation",
|
||||
"failure_limit": 3,
|
||||
"failed": false,
|
||||
"errors": 0,
|
||||
"streaks": {
|
||||
"initial_disk": 0,
|
||||
"tree_usage": 0,
|
||||
"live_disk": 0,
|
||||
"final_disk": 0
|
||||
},
|
||||
"max_streaks": {}
|
||||
}
|
||||
},
|
||||
"worker_result_sha256": null,
|
||||
"cpu_seconds_charged": 121.041777704,
|
||||
"pre_dispatch_wait_seconds": 3559.733168193954,
|
||||
"guard_wall_seconds": 107.41873304301407,
|
||||
"recorded_memory": {
|
||||
"maxima": {
|
||||
"peak_allocated_bytes": 1461572608,
|
||||
"peak_reserved_bytes": 1516240896
|
||||
},
|
||||
"witnesses": {
|
||||
"peak_allocated_bytes": {
|
||||
"path": "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/2/worker/progress.json",
|
||||
"sha256": "e9bc1e7e7e61795f5de473ddaff66d9fc5f3a068edbaf20c3da03d75f4fc4926",
|
||||
"observed_unix": 1795021349.2222252,
|
||||
"peaks": {
|
||||
"peak_allocated_bytes": 1461572608,
|
||||
"peak_reserved_bytes": 1516240896
|
||||
}
|
||||
},
|
||||
"peak_reserved_bytes": {
|
||||
"path": "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/2/worker/progress.json",
|
||||
"sha256": "e9bc1e7e7e61795f5de473ddaff66d9fc5f3a068edbaf20c3da03d75f4fc4926",
|
||||
"observed_unix": 1795021349.2222252,
|
||||
"peaks": {
|
||||
"peak_allocated_bytes": 1461572608,
|
||||
"peak_reserved_bytes": 1516240896
|
||||
}
|
||||
}
|
||||
},
|
||||
"observations_count": 157
|
||||
},
|
||||
"monitor_policy": {
|
||||
"rule": "three_consecutive_failed_samples_per_operation",
|
||||
"failure_limit": 3,
|
||||
"failed": false,
|
||||
"errors": 0,
|
||||
"streaks": {
|
||||
"recorded_peaks": 0,
|
||||
"tree": 0,
|
||||
"disk": 0,
|
||||
"queue_identity": 0,
|
||||
"gpu_processes": 0,
|
||||
"final_peaks": 0
|
||||
},
|
||||
"max_streaks": {}
|
||||
},
|
||||
"monitor_errors": [],
|
||||
"no_retry": true
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"status": "STOP_BUDGET",
|
||||
"projected_gpu_seconds": 88798.19167636364,
|
||||
"actual_training_and_preparation_gpu_seconds": 19248.94018448668,
|
||||
"folds": {
|
||||
"0": {
|
||||
"charged": 3729.7880347049795,
|
||||
"projected": 18317.55197484768
|
||||
},
|
||||
"1": {
|
||||
"charged": 3729.788036578917,
|
||||
"projected": 14081.724695697776
|
||||
},
|
||||
"2": {
|
||||
"charged": 3729.7880372299114,
|
||||
"projected": 16199.638335272728
|
||||
},
|
||||
"3": {
|
||||
"charged": 3729.7880377409747,
|
||||
"projected": 16199.638335272728
|
||||
},
|
||||
"4": {
|
||||
"charged": 3729.788038231898,
|
||||
"projected": 16199.638335272728
|
||||
}
|
||||
},
|
||||
"postfit_gpu_reserve": 7200,
|
||||
"safety_factor": 1.5
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"cumulative_gpu_seconds": 19262.057604262838,
|
||||
"within_cap": true,
|
||||
"freeze_tail_charged_once": true,
|
||||
"postfit_gpu_reserve": 7200,
|
||||
"postfit_started": false,
|
||||
"training_complete_sha256": "430247e44f301f984f396c662e6fa5419d09649476f6b2a81076444731f55dde",
|
||||
"required_before_any_postfit": true
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"commit": "2d0a3c6456ef2bcd11fef94b60d23122b9254237",
|
||||
"files": {
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/contract.json": "d33f036adc48cde904fbb6bb0f615a5464f94b039906d0b45a4dbd15f0681bd3",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/complete.json": "430247e44f301f984f396c662e6fa5419d09649476f6b2a81076444731f55dde"
|
||||
},
|
||||
"git_paths": {
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/contract.json": "home-runs/phase3-item-1b-2026-11-18/contract.json",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/complete.json": "home-runs/phase3-item-1b-2026-11-18/complete.json"
|
||||
},
|
||||
"remote_published": false,
|
||||
"local_repository": "/tmp/meltmind-phase2-local-fixed-20261117"
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"reviewed_commit": "28236c5df91329fa52ef98a584688496d2b4e272",
|
||||
"contract_sha256": "cc00b599f584d722d246c94dd5b7db9ecc082adf10581c4b32514e55c72c6e63",
|
||||
"implementation_proof_sha256": "d9ff47cb647cb5e7af6ad7f314a1ad9969b639d1063a5d135b2fd4d73b2a7804",
|
||||
"comparison_contract_sha256": "8f0f3c1088d899b85aa3f578b7552f04cb409fd7a5dcfca3e17eaa3d7e25d0d6",
|
||||
"code_sha256": {
|
||||
"/home/frank14f/optane/MeltMind/tools/boundary_distance.py": "a45c1cbce92521a96eb814913b4f465b926c0b3883932ab0b7c37d11db113ed7",
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_features_v2.py": "7dcd5ac218ffdc1ca149edcd87e8658fb3c59e18c0fb9ce47985c7f1357d2cdf",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_data.py": "de9cbcc147677c9144c706acbe52c0f7d505dad17d741d86dcbd04d052f940e2",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_postfit_rebind.py": "e5fcb0c1ca594aa74e6c3cb70dada60f79baa0d8bcaa0622b19da191ad596536",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_common.py": "27543fff987975ece767167f9771e6621c825035a053805604e4a696794dfcbd",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "437a756ddf10b125dce450f463e76e28df52724f8c5766e0a989dfa536805628",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_postfit_rebind.py": "aefdab5d2bcb0e4cf63b7ebe4e736307726f2bd8f98dc2985cc85b0d814315b3",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_common.py": "8b7d425ab702eb033d4027d24d2812042b66411943b004d6be08a13d4ffcd1c9",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind.py": "54389c854f8ee43bdf66834675c22b5e705446cab363ebf55d340d8e3c79381b",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_common.py": "b0f283577da25142201c6d2c7fbfdc486e0f21a75a554a6dd8c2a1da58d29778",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_guard.py": "0aaacc9518041235eecb7f078cebba6d0bc62dadfbd4951f0066bd8271db14a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_statistics.py": "07b1d26dbdb898d3ffe26bec497204a59226749d299d35ad55b074a4e0ad03e9",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_worker.py": "e34dbcd50bd91b9f84367b61f7568e0cdac471e9e1262a082bb9cfba80b79751",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_statistics.py": "b7d346d1762d7737eb1ebd723989a71f8c65b597d4cdb8c5d512b48a0fd984ce",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py": "a932d069340909465a5a1e66be1504e28c8df159a5ce304b0586295003d2e734"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"contract_sha256": "cc00b599f584d722d246c94dd5b7db9ecc082adf10581c4b32514e55c72c6e63",
|
||||
"proof_sha256": "d9ff47cb647cb5e7af6ad7f314a1ad9969b639d1063a5d135b2fd4d73b2a7804"
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
{
|
||||
"commit": "28236c5df91329fa52ef98a584688496d2b4e272",
|
||||
"files": {
|
||||
"/home/frank14f/MeltMind-runs/track-a-postfit-four-split-2026-11-19/contract.json": "cc00b599f584d722d246c94dd5b7db9ecc082adf10581c4b32514e55c72c6e63",
|
||||
"/home/frank14f/MeltMind-runs/track-a-postfit-four-split-2026-11-19/preparation-receipt.json": "3f7d14c510d538e8d0cc2b44de943ded82479381034187df0e1e4986c43369a3",
|
||||
"/home/frank14f/optane/MeltMind/tools/boundary_distance.py": "a45c1cbce92521a96eb814913b4f465b926c0b3883932ab0b7c37d11db113ed7",
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_features_v2.py": "7dcd5ac218ffdc1ca149edcd87e8658fb3c59e18c0fb9ce47985c7f1357d2cdf",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_data.py": "de9cbcc147677c9144c706acbe52c0f7d505dad17d741d86dcbd04d052f940e2",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_postfit_rebind.py": "e5fcb0c1ca594aa74e6c3cb70dada60f79baa0d8bcaa0622b19da191ad596536",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_common.py": "27543fff987975ece767167f9771e6621c825035a053805604e4a696794dfcbd",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "437a756ddf10b125dce450f463e76e28df52724f8c5766e0a989dfa536805628",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_postfit_rebind.py": "aefdab5d2bcb0e4cf63b7ebe4e736307726f2bd8f98dc2985cc85b0d814315b3",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_common.py": "8b7d425ab702eb033d4027d24d2812042b66411943b004d6be08a13d4ffcd1c9",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind.py": "54389c854f8ee43bdf66834675c22b5e705446cab363ebf55d340d8e3c79381b",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_common.py": "b0f283577da25142201c6d2c7fbfdc486e0f21a75a554a6dd8c2a1da58d29778",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_guard.py": "0aaacc9518041235eecb7f078cebba6d0bc62dadfbd4951f0066bd8271db14a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_statistics.py": "07b1d26dbdb898d3ffe26bec497204a59226749d299d35ad55b074a4e0ad03e9",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_worker.py": "e34dbcd50bd91b9f84367b61f7568e0cdac471e9e1262a082bb9cfba80b79751",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_statistics.py": "b7d346d1762d7737eb1ebd723989a71f8c65b597d4cdb8c5d512b48a0fd984ce",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py": "a932d069340909465a5a1e66be1504e28c8df159a5ce304b0586295003d2e734",
|
||||
"/home/frank14f/MeltMind-runs/track-a-postfit-2026-11-18/contract.json": "19faa4a9e14a19cedfeac3cbc5334462e10892f49269384d655449b525f6068e",
|
||||
"/home/frank14f/MeltMind-runs/track-a-postfit-2026-11-18/expected-review-binding.json": "a1787bfcce696ca46df245694295f3fbc270da355d9670835296ba447614edc8",
|
||||
"/home/frank14f/MeltMind-runs/track-a-postfit-2026-11-18/implementation-git-freeze.json": "83e178efae084bd39994a22b21d2d020a9cb46f144ba11dbc6a4e260f1b46124",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/complete.json": "299a73b8435877cf6f35af7e2340f27a833122588b3e1a7740f2a014606499ad",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/infrastructure-stop-git-freeze.json": "72eec78b4a6f6c803a2f30a7fd9a7e0acbde94473c182ec9eb25ec22e8f2897a",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/infrastructure-stop-authorization.json": "9a53b0f927670d7176b859f1a7b4517995e4a37c13617c4c2bf7e279e67e0f27",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/contract.json": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/complete.json": "2e9568c68ba03acfa099039b801c878474a9a5f227da0f2b425c1799b12d673b",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/terminal-ledger.json": "285a097f9e26f9cb3412ecfa24a7ca67a35d80b7e9fdceaade1898baf7d050cb",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/terminal-git-freeze.json": "47cecc48e3427d40d9a6dcf9ada09e0bdff1a9a95bf1f22b035a27223c96ff76",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/terminal-authorization.json": "6cbfb58fc45aa270e97fb961ec7abfc8b311d378ba365b06e914c12fd27a78b3",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/available-models-incomplete.json": "9acef8095110af43d18d03345b39a1e5029e5a4c721b705cb896e2dabdf3a055",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/incomplete-models-git-freeze.json": "c33dafe09d393445b4735050167e305734ada09152d536b04df43b8a705d8d8a",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/incomplete-models-authorization.json": "9f1b604181c818b0607fd8e0c7b3016663955075ceb3aea7c76a4502ae5d1741",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/3/stop.json": "e17a02303fc3e1f9ee0a5a2e284ee1c48efd84d2737b42862e2d2182169a72e5",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/3/first-epoch-projection.json": "c37524b96d70fd68159df223d93d397acdfa5923c66b284a89adc54320b79161",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/worker/result.json": "9dd0170ca5d6918b4ac30971034de227bba981aff285809f97b17aec8f9879db",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/3/complete.json": "a6a5fd3d6b457a22179540caf5c25c9b0cb96914b944b81a8522631e8f1af58a",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/complete.json": "6c7fb910b9fc928596a080e5ca254dc9c6af6a06d2eea6f16c89183434df4b4f",
|
||||
"/home/frank14f/optane/MeltMind/docs/track-a-four-split-postfit-2026-11-19.md": "44a0792d2b915fe1fdccd5b7dea1cdd35a24bfff68ecab7bc4cbbbd849839c7f",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-postfit-rebind-2026-11-19/acceptance.log": "2a787701645b9d9bd7d29b1862308e51958aeba9ecbe8e4dcd8c443af331c4d6",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-postfit-rebind-2026-11-19/protective-stops.json": "d9511838bc935c068c26d56933c8219ceaef02f18a3bab9905b8b7bb9a5fbf47"
|
||||
},
|
||||
"git_paths": {
|
||||
"/home/frank14f/MeltMind-runs/track-a-postfit-four-split-2026-11-19/contract.json": "home-runs/track-a-postfit-four-split-2026-11-19/contract.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-postfit-four-split-2026-11-19/preparation-receipt.json": "home-runs/track-a-postfit-four-split-2026-11-19/preparation-receipt.json",
|
||||
"/home/frank14f/optane/MeltMind/tools/boundary_distance.py": "tools/boundary_distance.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "tools/freeze_g1_components.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_features_v2.py": "tools/gre24_features_v2.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "tools/gre24_models.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "tools/phase2_gate_outcome_review.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "tools/phase2_h1_worker_v2.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "tools/phase2_h2_data.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "tools/phase2_h2_model.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "tools/phase2_home_freeze_io.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "tools/phase2_local_freeze_io.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "tools/phase2_residual_math.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_data.py": "tools/phase3_item1_postfit_data.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_postfit_rebind.py": "tools/prepare_track_a_postfit_rebind.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_common.py": "tools/r4bc_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "tools/r4bc_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "tools/s2_gpu_idle_gate.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "tools/s2_main_memory_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "tools/s2_pilot_train.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "tools/s2_precision_continuation_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "tools/s2_recovery_gpu_supervisor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "tools/s2_recovery_monitoring.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "tools/s2_recovery_supervisor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_postfit_rebind.py": "tools/test_track_a_postfit_rebind.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "tools/track_a_h2_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "tools/track_a_h2_prepare.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_common.py": "tools/track_a_postfit_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind.py": "tools/track_a_postfit_rebind.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_common.py": "tools/track_a_postfit_rebind_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_guard.py": "tools/track_a_postfit_rebind_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_statistics.py": "tools/track_a_postfit_rebind_statistics.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_worker.py": "tools/track_a_postfit_rebind_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_statistics.py": "tools/track_a_postfit_statistics.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py": "tools/track_a_recovery_common.py",
|
||||
"/home/frank14f/MeltMind-runs/track-a-postfit-2026-11-18/contract.json": "home-runs/track-a-postfit-2026-11-18/contract.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-postfit-2026-11-18/expected-review-binding.json": "home-runs/track-a-postfit-2026-11-18/expected-review-binding.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-postfit-2026-11-18/implementation-git-freeze.json": "home-runs/track-a-postfit-2026-11-18/implementation-git-freeze.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/complete.json": "home-runs/track-a-h2-2026-11-18/complete.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/infrastructure-stop-git-freeze.json": "home-runs/track-a-h2-2026-11-18/infrastructure-stop-git-freeze.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/infrastructure-stop-authorization.json": "home-runs/track-a-h2-2026-11-18/infrastructure-stop-authorization.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/contract.json": "home-runs/track-a-recovery-2026-11-18/contract.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/complete.json": "home-runs/track-a-recovery-2026-11-18/complete.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/terminal-ledger.json": "home-runs/track-a-recovery-2026-11-18/terminal-ledger.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/terminal-git-freeze.json": "home-runs/track-a-recovery-2026-11-18/terminal-git-freeze.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/terminal-authorization.json": "home-runs/track-a-recovery-2026-11-18/terminal-authorization.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/available-models-incomplete.json": "home-runs/track-a-recovery-2026-11-18/available-models-incomplete.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/incomplete-models-git-freeze.json": "home-runs/track-a-recovery-2026-11-18/incomplete-models-git-freeze.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/incomplete-models-authorization.json": "home-runs/track-a-recovery-2026-11-18/incomplete-models-authorization.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/3/stop.json": "home-runs/track-a-recovery-2026-11-18/jobs/3/stop.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/3/first-epoch-projection.json": "home-runs/track-a-recovery-2026-11-18/jobs/3/first-epoch-projection.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/worker/result.json": "home-runs/track-a-recovery-2026-11-18/jobs/4/worker/result.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/3/complete.json": "home-runs/track-a-recovery-2026-11-18/jobs/3/complete.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/complete.json": "home-runs/track-a-recovery-2026-11-18/jobs/4/complete.json",
|
||||
"/home/frank14f/optane/MeltMind/docs/track-a-four-split-postfit-2026-11-19.md": "docs/track-a-four-split-postfit-2026-11-19.md",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-postfit-rebind-2026-11-19/acceptance.log": "artifacts/track-a-postfit-rebind-2026-11-19/acceptance.log",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-postfit-rebind-2026-11-19/protective-stops.json": "artifacts/track-a-postfit-rebind-2026-11-19/protective-stops.json"
|
||||
},
|
||||
"remote_published": false,
|
||||
"local_repository": "/tmp/meltmind-phase2-local-fixed-20261117"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"status": "PREPARED_PENDING_INDEPENDENT_PRE_SCORE_REVIEW",
|
||||
"rebind_ruling": "1795022093.308479",
|
||||
"prior_postfit_implementation": "b164071e391c92ad8ee99dfa96c37f5b17991731",
|
||||
"evaluated_splits": [
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"excluded_split": {
|
||||
"split": 3,
|
||||
"status": "NOT_TRAINED_TO_COMPLETION_BUDGET_STOP",
|
||||
"reason": "OWN_PROJECTED_CAP",
|
||||
"recovery_completed_epochs": 1
|
||||
},
|
||||
"pair_visits": 1228,
|
||||
"unique_parts": 571,
|
||||
"training_gpu_seconds_once": 78579.79901044304,
|
||||
"source_pins": 36,
|
||||
"code_pins": 33,
|
||||
"bootstrap": {
|
||||
"replicates": 2000,
|
||||
"seed": 2026111408,
|
||||
"confidence": 0.95,
|
||||
"unit": "unique_part_mean_of_split_visit_differences",
|
||||
"interval": "percentile_linear"
|
||||
},
|
||||
"opaque_checkpoint_hashes_only": true,
|
||||
"metadata_cpu_seconds": 3.887246857,
|
||||
"review_adopted": false,
|
||||
"launched": false,
|
||||
"predicted": false,
|
||||
"scored": false,
|
||||
"test_targets_read": false
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"models": [
|
||||
{
|
||||
"job": 0,
|
||||
"path": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/last-epoch200.pt",
|
||||
"sha256": "767c186bdd0c99d9bcd195cadd5e7b15374c4c07bcb53aa03c157219c74b326d",
|
||||
"standardization": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/standardization.json",
|
||||
"standardization_sha256": "6fab06dd34d30d10409198bc177627293d4d8da559be3b33585c410964e6a15c",
|
||||
"origin": "ORIGINAL_HASH_REUSE",
|
||||
"training_contract_sha256": "be4ac1f6d5a5d6c0183aa50d2e046b22d7e5fa77c7714fef7eab6d616e68bee7",
|
||||
"guard_complete_sha256": "024bb110576da7c351ad8e98227cace2f40f4247a8f349c01ea4ebfde954891c"
|
||||
},
|
||||
{
|
||||
"job": 1,
|
||||
"path": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/last-epoch200.pt",
|
||||
"sha256": "9b1356403017f5c351729ce3d33494c48033b0addff4b7b9ca6c1aad6bf90b93",
|
||||
"standardization": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/standardization.json",
|
||||
"standardization_sha256": "6734f28dc7ca6ffd3a39a43ad2964ffb747c77e55d24d0705c3b5b41f56f397b",
|
||||
"origin": "ORIGINAL_HASH_REUSE",
|
||||
"training_contract_sha256": "be4ac1f6d5a5d6c0183aa50d2e046b22d7e5fa77c7714fef7eab6d616e68bee7",
|
||||
"guard_complete_sha256": "9259261f44a5813451e9f6f7878fd69d65db6e509889f7c78adcb6f417e00b3d"
|
||||
},
|
||||
{
|
||||
"job": 2,
|
||||
"path": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/last-epoch200.pt",
|
||||
"sha256": "0f71453359e8c6df759434b00759b5721e0cd3131cfb474a17a3b47f0c3c2e18",
|
||||
"standardization": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/standardization.json",
|
||||
"standardization_sha256": "e1f6c14617ed019379998142d49bf9359474db41e1614e2b4e20834ccc37b09d",
|
||||
"origin": "ORIGINAL_HASH_REUSE",
|
||||
"training_contract_sha256": "be4ac1f6d5a5d6c0183aa50d2e046b22d7e5fa77c7714fef7eab6d616e68bee7",
|
||||
"guard_complete_sha256": "b60b2575b9efdbb28eabcc2682dc04ce3519c9e7b2f6954cc4dcea0e29b1ab2f"
|
||||
},
|
||||
{
|
||||
"job": 4,
|
||||
"path": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/worker/last-epoch200.pt",
|
||||
"sha256": "d6aa7dc1b572c827a09cce9b9c23e9c744f1b283ee352686b3c507588677eb10",
|
||||
"standardization": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/worker/standardization.json",
|
||||
"standardization_sha256": "e8a85e95ad3829856c897be1257e0557363a9293be10e0e25edc5a8c6be7bd0e",
|
||||
"origin": "RECOVERY_FRESH_SEED0",
|
||||
"training_contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"guard_complete_sha256": "6c7fb910b9fc928596a080e5ca254dc9c6af6a06d2eea6f16c89183434df4b4f"
|
||||
}
|
||||
],
|
||||
"new_fits": 2,
|
||||
"reused_folds": [
|
||||
0,
|
||||
1,
|
||||
2
|
||||
],
|
||||
"all_five_complete": false,
|
||||
"postfit_started": false
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"status": "RECOVERY_INCOMPLETE_NO_FURTHER_RECOVERY",
|
||||
"entries": [
|
||||
{
|
||||
"id": 4,
|
||||
"status": "PASS",
|
||||
"returncode": 0,
|
||||
"gpu_seconds_charged": 13403.338334045024,
|
||||
"guard_complete_sha256": "6c7fb910b9fc928596a080e5ca254dc9c6af6a06d2eea6f16c89183434df4b4f",
|
||||
"pre_dispatch_wait_seconds": 1.9354809819487855,
|
||||
"receipt_error": null
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"returncode": 1,
|
||||
"gpu_seconds_charged": 17232.51348184503,
|
||||
"guard_complete_sha256": "a6a5fd3d6b457a22179540caf5c25c9b0cb96914b944b81a8522631e8f1af58a",
|
||||
"pre_dispatch_wait_seconds": 17085.811470279004,
|
||||
"receipt_error": null
|
||||
}
|
||||
],
|
||||
"old_gpu_seconds_once": 47943.94719455298,
|
||||
"new_training_gpu_seconds": 30635.851815890055,
|
||||
"cumulative_training_gpu_seconds": 78579.79901044304,
|
||||
"available_folds": [
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
4
|
||||
],
|
||||
"coordinator_cpu_seconds": 581.334575427,
|
||||
"postfit_started": false,
|
||||
"test_targets_read": false,
|
||||
"no_further_recovery": true,
|
||||
"original_postfit_exact_binding_compatible": false
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"proof_sha256": "c33dafe09d393445b4735050167e305734ada09152d536b04df43b8a705d8d8a"
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"commit": "cf27dde1cd3c952534e6999ac34a14427027079c",
|
||||
"files": {
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/contract.json": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/available-models-incomplete.json": "9acef8095110af43d18d03345b39a1e5029e5a4c721b705cb896e2dabdf3a055",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/terminal-ledger.json": "285a097f9e26f9cb3412ecfa24a7ca67a35d80b7e9fdceaade1898baf7d050cb",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/last-epoch200.pt": "767c186bdd0c99d9bcd195cadd5e7b15374c4c07bcb53aa03c157219c74b326d",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/last-epoch200.pt": "9b1356403017f5c351729ce3d33494c48033b0addff4b7b9ca6c1aad6bf90b93",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/last-epoch200.pt": "0f71453359e8c6df759434b00759b5721e0cd3131cfb474a17a3b47f0c3c2e18",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/worker/last-epoch200.pt": "d6aa7dc1b572c827a09cce9b9c23e9c744f1b283ee352686b3c507588677eb10",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/standardization.json": "6fab06dd34d30d10409198bc177627293d4d8da559be3b33585c410964e6a15c",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/standardization.json": "6734f28dc7ca6ffd3a39a43ad2964ffb747c77e55d24d0705c3b5b41f56f397b",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/standardization.json": "e1f6c14617ed019379998142d49bf9359474db41e1614e2b4e20834ccc37b09d",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/worker/standardization.json": "e8a85e95ad3829856c897be1257e0557363a9293be10e0e25edc5a8c6be7bd0e"
|
||||
},
|
||||
"git_paths": {
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/contract.json": "home-runs/track-a-recovery-2026-11-18/contract.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/available-models-incomplete.json": "home-runs/track-a-recovery-2026-11-18/available-models-incomplete.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/terminal-ledger.json": "home-runs/track-a-recovery-2026-11-18/terminal-ledger.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/last-epoch200.pt": "home-runs/track-a-h2-2026-11-18/jobs/0/worker/last-epoch200.pt",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/last-epoch200.pt": "home-runs/track-a-h2-2026-11-18/jobs/1/worker/last-epoch200.pt",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/last-epoch200.pt": "home-runs/track-a-h2-2026-11-18/jobs/2/worker/last-epoch200.pt",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/worker/last-epoch200.pt": "home-runs/track-a-recovery-2026-11-18/jobs/4/worker/last-epoch200.pt",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/standardization.json": "home-runs/track-a-h2-2026-11-18/jobs/0/worker/standardization.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/standardization.json": "home-runs/track-a-h2-2026-11-18/jobs/1/worker/standardization.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/standardization.json": "home-runs/track-a-h2-2026-11-18/jobs/2/worker/standardization.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/worker/standardization.json": "home-runs/track-a-recovery-2026-11-18/jobs/4/worker/standardization.json"
|
||||
},
|
||||
"remote_published": false,
|
||||
"local_repository": "/tmp/meltmind-phase2-local-fixed-20261117"
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"status": "STOP_OWN_COST_PROJECTION",
|
||||
"cell": {
|
||||
"id": 3,
|
||||
"projected_gpu_seconds": 34426.17240025735,
|
||||
"within_cap": false
|
||||
},
|
||||
"total": {
|
||||
"status": "PASS",
|
||||
"rows": [
|
||||
{
|
||||
"id": 3,
|
||||
"projected_gpu_seconds": 30960,
|
||||
"basis": "locally_stopping_own_cap_or_actual"
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"projected_gpu_seconds": 13403.338334045024,
|
||||
"basis": "terminal_actual"
|
||||
}
|
||||
],
|
||||
"new_projected_gpu_seconds": 51563.338334045024,
|
||||
"old_gpu_seconds_once": 47943.94719455298,
|
||||
"cumulative_projected_gpu_seconds": 99507.285528598,
|
||||
"postfit_reserve": 7200
|
||||
},
|
||||
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"unix": 1795021376.0306857
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"id": 3,
|
||||
"reason": "OWN_PROJECTED_CAP",
|
||||
"unix": 1795021376.0280583,
|
||||
"other_fold_must_continue": true
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"status": "COMPLETE",
|
||||
"job": 4,
|
||||
"fold": 4,
|
||||
"completed_epochs": 200,
|
||||
"optimizer_steps": 62000,
|
||||
"learning_parts": 310,
|
||||
"learning_nodes_per_epoch": 8642607,
|
||||
"checkpoint": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/worker/last-epoch200.pt",
|
||||
"checkpoint_sha256": "d6aa7dc1b572c827a09cce9b9c23e9c744f1b283ee352686b3c507588677eb10",
|
||||
"standardization_sha256": "e8a85e95ad3829856c897be1257e0557363a9293be10e0e25edc5a8c6be7bd0e",
|
||||
"test_targets_read": false,
|
||||
"test_prediction_started": false,
|
||||
"validation_used_for_selection": false,
|
||||
"worker_wall_seconds": 13327.702773115016,
|
||||
"worker_cpu_seconds": 13323.528446017,
|
||||
"peak_allocated_bytes": 1461572608,
|
||||
"peak_reserved_bytes": 1516240896
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"proof_sha256": "47cecc48e3427d40d9a6dcf9ada09e0bdff1a9a95bf1f22b035a27223c96ff76"
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"commit": "73fcb4213ec50afe8cb9e0b1558d507725ba8715",
|
||||
"files": {
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/contract.json": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/complete.json": "2e9568c68ba03acfa099039b801c878474a9a5f227da0f2b425c1799b12d673b",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/available-models-incomplete.json": "9acef8095110af43d18d03345b39a1e5029e5a4c721b705cb896e2dabdf3a055",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/terminal-ledger.json": "285a097f9e26f9cb3412ecfa24a7ca67a35d80b7e9fdceaade1898baf7d050cb"
|
||||
},
|
||||
"git_paths": {
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/contract.json": "home-runs/track-a-recovery-2026-11-18/contract.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/complete.json": "home-runs/track-a-recovery-2026-11-18/complete.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/available-models-incomplete.json": "home-runs/track-a-recovery-2026-11-18/available-models-incomplete.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/terminal-ledger.json": "home-runs/track-a-recovery-2026-11-18/terminal-ledger.json"
|
||||
},
|
||||
"remote_published": false,
|
||||
"local_repository": "/tmp/meltmind-phase2-local-fixed-20261117"
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"entries": [
|
||||
{
|
||||
"id": 4,
|
||||
"status": "PASS",
|
||||
"returncode": 0,
|
||||
"gpu_seconds_charged": 13403.338334045024,
|
||||
"guard_complete_sha256": "6c7fb910b9fc928596a080e5ca254dc9c6af6a06d2eea6f16c89183434df4b4f",
|
||||
"pre_dispatch_wait_seconds": 1.9354809819487855,
|
||||
"receipt_error": null
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"returncode": 1,
|
||||
"gpu_seconds_charged": 17232.51348184503,
|
||||
"guard_complete_sha256": "a6a5fd3d6b457a22179540caf5c25c9b0cb96914b944b81a8522631e8f1af58a",
|
||||
"pre_dispatch_wait_seconds": 17085.811470279004,
|
||||
"receipt_error": null
|
||||
}
|
||||
],
|
||||
"old_gpu_seconds_once": 47943.94719455298,
|
||||
"new_settled_gpu_seconds": 30635.851815890055,
|
||||
"new_charged_gpu_seconds": 30635.851815890055,
|
||||
"cumulative_training_gpu_seconds": 78579.79901044304
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Freeze the authorized four-split Track A binding without opening targets or models."""
|
||||
|
||||
import fcntl
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from phase2_home_freeze_io import freeze, verify
|
||||
from track_a_h2_prepare import local_code_closure
|
||||
from track_a_postfit_common import config as original_config
|
||||
from track_a_postfit_rebind_common import (
|
||||
ORIGINAL, ORIGINAL_POSTFIT, RECOVERY, ROOT, RULING, config, read, scope, terminal,
|
||||
)
|
||||
|
||||
BASE = Path(__file__).resolve().parents[1]
|
||||
RECORDS = BASE / 'artifacts/track-a-postfit-rebind-2026-11-19'
|
||||
|
||||
|
||||
def source_files():
|
||||
return [
|
||||
*(ORIGINAL_POSTFIT / name for name in ('contract.json', 'expected-review-binding.json',
|
||||
'implementation-git-freeze.json')),
|
||||
*(ORIGINAL / name for name in ('complete.json', 'infrastructure-stop-git-freeze.json',
|
||||
'infrastructure-stop-authorization.json')),
|
||||
*(RECOVERY / name for name in ('contract.json', 'complete.json', 'terminal-ledger.json',
|
||||
'terminal-git-freeze.json', 'terminal-authorization.json',
|
||||
'available-models-incomplete.json', 'incomplete-models-git-freeze.json',
|
||||
'incomplete-models-authorization.json')),
|
||||
RECOVERY / 'jobs/3/stop.json',
|
||||
RECOVERY / 'jobs/3/first-epoch-projection.json',
|
||||
RECOVERY / 'jobs/4/worker/result.json',
|
||||
*(RECOVERY / f'jobs/{i}/complete.json' for i in (3, 4)),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
start = time.process_time()
|
||||
old, training, _ = original_config(ORIGINAL_POSTFIT)
|
||||
if ROOT.exists():
|
||||
raise FileExistsError('The rebind root already exists')
|
||||
sources = {str(p): sha(p) for p in source_files()}
|
||||
if set(sources).intersection(old['source_sha256']):
|
||||
for p in set(sources).intersection(old['source_sha256']):
|
||||
checked(p, old['source_sha256'][p])
|
||||
models = terminal(training)
|
||||
codes = local_code_closure([Path(__file__), *(BASE / 'tools' / name for name in (
|
||||
'track_a_postfit_rebind.py', 'track_a_postfit_rebind_worker.py',
|
||||
'track_a_postfit_rebind_guard.py', 'test_track_a_postfit_rebind.py',
|
||||
))])
|
||||
contract = dict(old, **scope(old, training), version='track-a-postfit-four-split',
|
||||
rebind_ruling=RULING, training_evidence=models,
|
||||
source_sha256={**old['source_sha256'], **sources}, code_sha256=codes)
|
||||
del contract['all_five_training_terminals_before_prediction']
|
||||
contract['all_selected_training_terminals_before_prediction'] = True
|
||||
if (contract['expected_pair_visits'], contract['expected_unique_target_reads']) != (1228, 571):
|
||||
raise ValueError('Four-split overlap/domain differs from the original metadata')
|
||||
with Path('/tmp/meltmind-single-writer.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
ROOT.mkdir()
|
||||
save(ROOT / 'contract.json', contract)
|
||||
save(ROOT / 'preparation-receipt.json', dict(
|
||||
status='PREPARED_PENDING_INDEPENDENT_PRE_SCORE_REVIEW', rebind_ruling=RULING,
|
||||
prior_postfit_implementation=verify(ORIGINAL_POSTFIT, 'implementation')['commit'],
|
||||
evaluated_splits=[x['id'] for x in contract['jobs']], excluded_split=contract['excluded_split'],
|
||||
pair_visits=contract['expected_pair_visits'], unique_parts=contract['expected_unique_target_reads'],
|
||||
training_gpu_seconds_once=models['training_gpu_seconds_once'],
|
||||
source_pins=len(contract['source_sha256']), code_pins=len(codes),
|
||||
bootstrap=contract['bootstrap'], opaque_checkpoint_hashes_only=True,
|
||||
metadata_cpu_seconds=time.process_time()-start, review_adopted=False,
|
||||
launched=False, predicted=False, scored=False, test_targets_read=False,
|
||||
))
|
||||
freeze(ROOT, 'implementation', [ROOT / 'contract.json', ROOT / 'preparation-receipt.json',
|
||||
*(Path(p) for p in codes), *source_files(),
|
||||
BASE / 'docs/track-a-four-split-postfit-2026-11-19.md',
|
||||
RECORDS / 'acceptance.log', RECORDS / 'protective-stops.json'])
|
||||
proof = verify(ROOT, 'implementation')
|
||||
frozen, _, _ = config(ROOT)
|
||||
save(ROOT / 'expected-review-binding.json', dict(
|
||||
reviewed_commit=proof['commit'], contract_sha256=sha(ROOT / 'contract.json'),
|
||||
implementation_proof_sha256=sha(ROOT / 'implementation-git-freeze.json'),
|
||||
comparison_contract_sha256=frozen['comparison_contract_sha256'], code_sha256=codes))
|
||||
receipt = read(ROOT / 'preparation-receipt.json')
|
||||
print(dict(receipt, implementation=proof['commit'], contract=sha(ROOT / 'contract.json'),
|
||||
binding_sha256=sha(ROOT / 'expected-review-binding.json')))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Synthetic checks of the four-split boundary and unchanged paired estimator."""
|
||||
|
||||
from pathlib import Path
|
||||
from unittest import TestCase, main
|
||||
from unittest.mock import patch
|
||||
|
||||
from track_a_postfit_rebind_common import OMITTED, SPLITS, scope, selected_training
|
||||
from track_a_postfit_rebind_statistics import paired_uncertainty, summarize
|
||||
from track_a_postfit_statistics import paired_uncertainty as original_uncertainty
|
||||
|
||||
|
||||
class RebindTests(TestCase):
|
||||
def setUp(self):
|
||||
self.jobs = [dict(id=i, test=['a', str(i)], missing_test=[]) for i in range(5)]
|
||||
self.rows = [dict(split=i, case=case, A=1., H2=2.+i)
|
||||
for i in SPLITS for case in self.jobs[i]['test']]
|
||||
|
||||
def test_exact_split_ids_and_visit_counts(self):
|
||||
original = dict(A_predictions={str(i): {str(i): {'sha256': str(i)}} for i in range(5)})
|
||||
result = scope(original, {'jobs': self.jobs})
|
||||
self.assertEqual([job['id'] for job in result['jobs']], list(SPLITS))
|
||||
self.assertEqual(set(result['A_predictions']), {'0', '1', '2', '4'})
|
||||
self.assertEqual(result['excluded_split'], OMITTED)
|
||||
self.assertEqual(result['expected_pair_visits'], 8)
|
||||
self.assertEqual(result['expected_unique_target_reads'], 5)
|
||||
|
||||
def test_four_split_summary_uses_original_part_bootstrap(self):
|
||||
outcome = summarize(self.rows, selected_training({'jobs': self.jobs})['jobs'])
|
||||
self.assertIs(paired_uncertainty, original_uncertainty)
|
||||
self.assertEqual(outcome['descriptive_paired_uncertainty'], original_uncertainty(self.rows))
|
||||
self.assertEqual(outcome['mean_of_four_split_summaries']['H2']['median'], 3.75)
|
||||
self.assertEqual(outcome['excluded_split']['split'], 3)
|
||||
self.assertNotIn('mean_of_five_split_summaries', outcome)
|
||||
|
||||
def test_incomplete_duplicate_or_extra_visits_rejected(self):
|
||||
jobs = selected_training({'jobs': self.jobs})['jobs']
|
||||
for rows in (self.rows[:-1], self.rows+[self.rows[0]],
|
||||
self.rows+[dict(split=3, case='a', A=1., H2=4.)]):
|
||||
with self.assertRaises(ValueError):
|
||||
summarize(rows, jobs)
|
||||
|
||||
def test_mixed_source_checkpoint_identity(self):
|
||||
from track_a_postfit_common import checkpoint_identity
|
||||
for i in SPLITS:
|
||||
training_sha = 'recovery' if i == 4 else 'original'
|
||||
blob = dict(seed=0, fold=i, completed_epochs=200, optimizer_steps=62000,
|
||||
contract_sha256=training_sha, role='last_epoch_only')
|
||||
checkpoint_identity(blob, dict(id=i, target_steps=62000), training_sha)
|
||||
with self.assertRaises(ValueError):
|
||||
checkpoint_identity(blob, dict(id=i, target_steps=62000), 'other')
|
||||
|
||||
def test_scoring_requires_new_review_before_target_access(self):
|
||||
from track_a_postfit_rebind_worker import score
|
||||
with patch('track_a_postfit_rebind_worker.config', return_value=({}, {}, {})), \
|
||||
patch('track_a_postfit_rebind_worker.validate_review', side_effect=PermissionError), \
|
||||
patch('phase2_h2_data.cached') as targets:
|
||||
with self.assertRaises(PermissionError):
|
||||
score(Path('/synthetic'))
|
||||
targets.assert_not_called()
|
||||
|
||||
def test_scoring_requires_prediction_freeze_before_target_access(self):
|
||||
from track_a_postfit_rebind_worker import score
|
||||
with patch('track_a_postfit_rebind_worker.config', return_value=({}, {}, {})), \
|
||||
patch('track_a_postfit_rebind_worker.validate_review'), \
|
||||
patch('track_a_postfit_rebind_worker.adopted'), \
|
||||
patch('track_a_postfit_rebind_worker.verify', side_effect=PermissionError), \
|
||||
patch('phase2_h2_data.cached') as targets:
|
||||
with self.assertRaises(PermissionError):
|
||||
score(Path('/synthetic'))
|
||||
targets.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Four-split Track A postfit after exact mixed-source review and training terminal verification."""
|
||||
import fcntl,json,os,resource,sys,time
|
||||
from pathlib import Path
|
||||
import psutil
|
||||
from track_a_postfit_common import prediction_identity,budget
|
||||
from track_a_postfit_rebind_common import config,read,terminal,selected_training,SPLITS,RECOVERY,ENTRYPOINTS
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_home_freeze_io import freeze,verify
|
||||
from phase2_h1_worker_v2 import save,sha,checked
|
||||
from r4bc_guard import supervise
|
||||
|
||||
def run(root):
|
||||
gpu_clock=time.monotonic();start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);c,tc,entry=config(root);validate_review(root);adopted=terminal(tc);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if adopted!=c['training_evidence']:raise ValueError('Training terminal changed since rebind')
|
||||
if (root/'started.json').exists():raise FileExistsError('One Track A postfit attempt, no retry')
|
||||
for p in psutil.process_iter(['pid','cmdline']):
|
||||
if p.pid!=os.getpid() and any(Path(x).name in ENTRYPOINTS for x in p.info['cmdline'] or []):raise RuntimeError('Existing postfit process')
|
||||
gpu_lock=Path('/tmp/meltmind-track-a-postfit-'+c['gpu_uuid']+'.lock').open('a');fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
os.sched_setaffinity(0,c['coordinator_cpus']);os.environ['PYTORCH_CUDA_ALLOC_CONF']='expandable_segments:True';gpu=0.;charged=0.;records=[];prior=adopted['training_gpu_seconds_once'];prediction_start=None
|
||||
def cpu():
|
||||
u=resource.getrusage(resource.RUSAGE_CHILDREN);return 180+time.process_time()-start+max(charged,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)
|
||||
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),training_gpu_seconds_once=prior,postfit_gpu_cap=7200,track_a_gpu_cap=162000,review_sha256=sha(root/'review.json'),cpu_start_counters=dict(self_process_time=start,children_user=u0.ru_utime,children_system=u0.ru_stime),cpu_setup_allowance=180,new_fits=0,new_decodes=0))
|
||||
try:
|
||||
budget(prior,gpu,cpu(),len(SPLITS));save(root/'models-adopted.json',adopted);train=Path(c['training_root']);freeze(root,'models',[root/'contract.json',root/'models-adopted.json',train/'complete.json',RECOVERY/'complete.json',RECOVERY/'available-models-incomplete.json',RECOVERY/'incomplete-models-git-freeze.json',RECOVERY/'terminal-git-freeze.json',*[Path(m['path']) for m in adopted['models']],*[Path(m['standardization']) for m in adopted['models']]])
|
||||
entries=[]
|
||||
from track_a_postfit_rebind_guard import run_prediction
|
||||
for position,index in enumerate(SPLITS):
|
||||
gpu=max(gpu,60+time.monotonic()-gpu_clock);budget(prior,gpu,cpu(),len(SPLITS)-position);(root/'jobs'/str(index)).mkdir(parents=True,exist_ok=False);prediction_start=time.monotonic();g=run_prediction(root,c,tc,index,prior+gpu,300);observed=60+time.monotonic()-prediction_start;prediction_start=None;cost=max(observed,g['gpu_seconds_charged']);gpu+=cost;charged+=g['worker']['cpu_seconds'] if g['worker'] else 0.;records.append(dict(mode='predict',index=index,charged_gpu_seconds=cost,guard=g));save(root/'resource-progress.json',dict(records=records,prediction_gpu_seconds=gpu,postfit_cpu_seconds=cpu(),cumulative_track_a_gpu_seconds=prior+gpu))
|
||||
if g['status']!='PASS' or cost>900 or gpu>7200 or prior+gpu>162000:raise RuntimeError('Prediction/resource guard failure')
|
||||
m=read(root/'jobs'/str(index)/'prediction-records/manifest.json');entries.extend(m['entries'])
|
||||
prediction_identity(root,c,selected_training(tc),entries);save(root/'predictions-manifest.json',dict(entries=entries,A_predictions_reused=c['expected_pair_visits'],evaluated_splits=list(SPLITS),excluded_split=c['excluded_split'],new_fits=0,targets_read=False));freeze(root,'predictions',[root/'contract.json',root/'models-adopted.json',root/'predictions-manifest.json',root/'resource-progress.json',*[Path(p) for p in c['code_sha256']]])
|
||||
gpu=max(gpu,60+time.monotonic()-gpu_clock);budget(prior,gpu,cpu(),0);validate_review(root);verify(root,'models');verify(root,'predictions')
|
||||
g=supervise([sys.executable,'tools/track_a_postfit_rebind_worker.py','score',str(root)],root/'score-guard',dict(cpu_seconds=900,wall_seconds=1200,rss_bytes=16*2**30,disk_bytes=20*2**30));charged+=g['cpu_seconds'];records.append(dict(mode='score',guard=g))
|
||||
if g['status']!='PASS' or cpu()+120>3600:raise RuntimeError('Score/resource failure')
|
||||
save(root/'complete.json',dict(status='COMPLETE_FOUR_OF_FIVE',evaluated_splits=list(SPLITS),excluded_split=c['excluded_split'],records=records,prediction_gpu_seconds=gpu,postfit_cpu_seconds=cpu(),training_gpu_seconds_once=prior,cumulative_track_a_gpu_seconds=prior+gpu,track_a_gpu_cap=162000,new_fits=0,new_decodes=0,future_training_started=False,no_retry=True));freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'score/result.json',root/'review.json'])
|
||||
final=cpu();save(root/'post-terminal-cost.json',dict(postfit_cpu_seconds=final,prediction_gpu_seconds=gpu,cumulative_track_a_gpu_seconds=prior+gpu,within_cap=final<=3600 and gpu<=7200 and prior+gpu<=162000,cpu_start_counters=read(root/'started.json')['cpu_start_counters']))
|
||||
if final>3600:raise RuntimeError('Final CPU cap after freeze')
|
||||
except BaseException as exc:
|
||||
if prediction_start is not None:gpu+=60+time.monotonic()-prediction_start
|
||||
if (root/'complete.json').exists():save(root/'pre-failure-complete.json',read(root/'complete.json'))
|
||||
save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),records=records,postfit_cpu_seconds=cpu(),prediction_gpu_seconds=gpu,training_gpu_seconds_once=prior,cumulative_track_a_gpu_seconds=prior+gpu,new_fits=0,no_retry=True));raise
|
||||
if __name__=='__main__':run(Path(sys.argv[1]))
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Identity and custody for the four completed splits of the Track A rebind."""
|
||||
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from phase2_h1_worker_v2 import checked, sha
|
||||
from phase2_home_freeze_io import verify
|
||||
from track_a_h2_common import read
|
||||
from track_a_postfit_common import config as original_postfit_config
|
||||
from track_a_recovery_common import ORIGINAL, ROOT as RECOVERY, accepted_guard, execution_command, reuse_identity
|
||||
|
||||
|
||||
RULING = '1795022093.308479'
|
||||
ROOT = ORIGINAL.with_name('track-a-postfit-four-split-2026-11-19')
|
||||
ORIGINAL_POSTFIT = ORIGINAL.with_name('track-a-postfit-2026-11-18')
|
||||
SPLITS = (0, 1, 2, 4)
|
||||
OMITTED = dict(split=3, status='NOT_TRAINED_TO_COMPLETION_BUDGET_STOP',
|
||||
reason='OWN_PROJECTED_CAP', recovery_completed_epochs=1)
|
||||
ENTRYPOINTS = {
|
||||
'track_a_postfit.py', 'track_a_postfit_worker.py',
|
||||
'track_a_postfit_rebind.py', 'track_a_postfit_rebind_worker.py',
|
||||
}
|
||||
|
||||
|
||||
def selected_training(training):
|
||||
return dict(training, jobs=[training['jobs'][split] for split in SPLITS])
|
||||
|
||||
|
||||
def scope(original, training):
|
||||
jobs = selected_training(training)['jobs']
|
||||
if tuple(job['id'] for job in jobs) != SPLITS:
|
||||
raise ValueError('Original split identities changed')
|
||||
return dict(
|
||||
jobs=[dict(id=j['id'], expected_parts=len(j['test'])) for j in jobs],
|
||||
A_predictions={str(split): original['A_predictions'][str(split)] for split in SPLITS},
|
||||
expected_pair_visits=sum(len(j['test']) for j in jobs),
|
||||
expected_unique_target_reads=len({part for j in jobs for part in j['test']}),
|
||||
excluded_split=OMITTED,
|
||||
)
|
||||
|
||||
|
||||
def config(root):
|
||||
if root.resolve() != ROOT:
|
||||
raise PermissionError('Only the four-split rebind root is approved')
|
||||
contract = read(root / 'contract.json')
|
||||
verify(root, 'implementation')
|
||||
original, training, entry = original_postfit_config(ORIGINAL_POSTFIT)
|
||||
expected = scope(original, training)
|
||||
for key in ('approval', 'budget_approval', 'estimator_approval', 'training_root',
|
||||
'training_contract_sha256', 'comparison_contract', 'comparison_contract_sha256',
|
||||
'gpu_uuid', 'worker_cpus', 'coordinator_cpus', 'gpu_cap', 'postfit_gpu_cap',
|
||||
'postfit_cpu_cap', 'idle_wait_seconds', 'no_retry', 'bootstrap',
|
||||
'all_predictions_frozen_before_targets', 'A_refit', 'H2_refit', 'new_decodes',
|
||||
'metric', 'original_failed'):
|
||||
expected[key] = original[key]
|
||||
if (contract['rebind_ruling'] != RULING or contract['version'] != 'track-a-postfit-four-split'
|
||||
or not contract['no_auto_launch'] or any(contract[k] != v for k, v in expected.items())
|
||||
or (contract['expected_pair_visits'], contract['expected_unique_target_reads']) != (1228, 571)):
|
||||
raise PermissionError('Frozen four-of-five postfit scope')
|
||||
for p, h in {**contract['source_sha256'], **contract['code_sha256']}.items():
|
||||
checked(p, h)
|
||||
return contract, training, entry
|
||||
|
||||
|
||||
def terminal(training):
|
||||
import psutil
|
||||
|
||||
for process in psutil.process_iter(['pid', 'cmdline']):
|
||||
args = process.info['cmdline'] or []
|
||||
if execution_command(args) or (process.pid != os.getpid()
|
||||
and any(Path(a).name in ENTRYPOINTS for a in args)):
|
||||
raise RuntimeError('Track A training or another postfit process still present')
|
||||
verify(RECOVERY, 'terminal')
|
||||
model_proof = verify(RECOVERY, 'incomplete-models')
|
||||
recovered = read(RECOVERY / 'complete.json')
|
||||
incomplete = read(RECOVERY / 'available-models-incomplete.json')
|
||||
recovery = read(RECOVERY / 'contract.json')
|
||||
originals = reuse_identity(recovery)
|
||||
if (recovered['status'] != 'RECOVERY_INCOMPLETE_NO_FURTHER_RECOVERY'
|
||||
or recovered['available_folds'] != list(SPLITS) or not recovered['no_further_recovery']
|
||||
or recovered['postfit_started'] or recovered['test_targets_read']
|
||||
or incomplete['all_five_complete'] or incomplete['postfit_started']
|
||||
or (RECOVERY / 'global-stop.json').exists() or (ORIGINAL_POSTFIT / 'started.json').exists()):
|
||||
raise ValueError('Track A terminal scope differs')
|
||||
entries = {e['id']: e for e in recovered['entries']}
|
||||
if len(recovered['entries']) != 2 or set(entries) != {3, 4}:
|
||||
raise ValueError('Exact two recovery receipts required')
|
||||
fold3, fold4 = (RECOVERY / 'jobs' / str(i) for i in (3, 4))
|
||||
stopped = read(checked(fold3 / 'complete.json', entries[3]['guard_complete_sha256']))
|
||||
stop = read(fold3 / 'stop.json')
|
||||
projection = read(fold3 / 'first-epoch-projection.json')
|
||||
if (entries[3]['status'] != 'STOPPED_OR_FAILED' or 'OWN_FOLD_STOP' not in stopped['reasons']
|
||||
or stop['id'] != 3 or stop['reason'] != OMITTED['reason']
|
||||
or projection['cell']['id'] != 3 or projection['cell']['within_cap']
|
||||
or projection['cell']['projected_gpu_seconds'] <= recovery['jobs'][3]['gpu_seconds']
|
||||
or (fold3 / 'worker' / 'result.json').exists()):
|
||||
raise ValueError('Split 3 budget stop identity')
|
||||
g = read(checked(fold4 / 'complete.json', entries[4]['guard_complete_sha256']))
|
||||
if (entries[4]['status'] != 'PASS' or not accepted_guard(entries[4]['returncode'], g)
|
||||
or g['worker'] != read(fold4 / 'guards' / 'train.receipt.json')
|
||||
or g['worker']['monitor_errors'] != 0 or g['monitor_errors']):
|
||||
raise ValueError('Split 4 clean recovery PASS required')
|
||||
result = read(checked(fold4 / 'worker' / 'result.json', g['worker_result_sha256']))
|
||||
job = training['jobs'][4]
|
||||
if (result['status'] != 'COMPLETE' or result['fold'] != 4 or result['job'] != 4
|
||||
or result['completed_epochs'] != 200 or result['optimizer_steps'] != job['target_steps']
|
||||
or result['test_targets_read'] or result['test_prediction_started']
|
||||
or result['validation_used_for_selection']):
|
||||
raise ValueError('Split 4 last epoch identity')
|
||||
model = dict(job=4, path=result['checkpoint'], sha256=result['checkpoint_sha256'],
|
||||
standardization=str(fold4 / 'worker' / 'standardization.json'),
|
||||
standardization_sha256=result['standardization_sha256'],
|
||||
origin='RECOVERY_FRESH_SEED0', training_contract_sha256=sha(RECOVERY / 'contract.json'),
|
||||
guard_complete_sha256=entries[4]['guard_complete_sha256'])
|
||||
models = originals + [model]
|
||||
if models != incomplete['models'] or tuple(m['job'] for m in models) != SPLITS:
|
||||
raise ValueError('Mixed-source manifest differs from recovered models')
|
||||
for m in models:
|
||||
for p, h in ((m['path'], m['sha256']), (m['standardization'], m['standardization_sha256'])):
|
||||
checked(p, h)
|
||||
if model_proof['files'].get(str(Path(p).resolve())) != h:
|
||||
raise ValueError('Model or normalization is missing from the recovery freeze')
|
||||
stats = read(m['standardization'])
|
||||
j = training['jobs'][m['job']]
|
||||
if stats['parts'] != j['learning'] or stats['nodes'] != j['learning_nodes']:
|
||||
raise ValueError('Learning-only normalization identity')
|
||||
old = read(ORIGINAL / 'complete.json')['training_gpu_seconds']
|
||||
new = sum(e['gpu_seconds_charged'] for e in entries.values())
|
||||
prior = recovered['cumulative_training_gpu_seconds']
|
||||
if (old != recovered['old_gpu_seconds_once'] or new != recovered['new_training_gpu_seconds']
|
||||
or not math.isclose(prior, old + new, rel_tol=0, abs_tol=1e-8) or not 0 <= prior <= 154800):
|
||||
raise ValueError('Old and recovery training charges must be included exactly once')
|
||||
return dict(models=models, excluded_split=OMITTED, training_gpu_seconds_once=prior,
|
||||
original_gpu_seconds_once=old, recovery_gpu_seconds_once=new,
|
||||
original_terminal_sha256=sha(ORIGINAL / 'complete.json'),
|
||||
recovery_terminal_sha256=sha(RECOVERY / 'complete.json'))
|
||||
|
||||
|
||||
def adopted(root, contract):
|
||||
verify(root, 'models')
|
||||
receipt = read(root / 'models-adopted.json')
|
||||
if receipt != contract['training_evidence']:
|
||||
raise ValueError('Adopted evidence differs from the frozen binding')
|
||||
return receipt
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Track A postfit guard: bounded selected-GPU wait, no retries."""
|
||||
import json,os,signal,sys,threading,time
|
||||
import psutil
|
||||
from phase2_h1_worker_v2 import save
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
from s2_precision_continuation_guard import device_usage
|
||||
from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage
|
||||
|
||||
def final_reasons(reasons,receipt,result,tracker,gpu_seconds,cpu_seconds,cpu_cap,expected_parts):
|
||||
out=list(reasons)
|
||||
def add(s):
|
||||
if s not in out:out.append(s)
|
||||
if not receipt or receipt['status']!='PASS':add('WORKER_NOT_PASS')
|
||||
if receipt and receipt.get('monitor_errors'):add('WORKER_MONITOR_ERRORS')
|
||||
if not result or result.get('status')!='COMPLETE' or result.get('parts')!=expected_parts:add('INCOMPLETE_FOLD')
|
||||
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):add('MISSING_PEAKS')
|
||||
if any(v>15*2**30 for v in tracker.maxima.values()):add('RECORDED_VRAM_LIMIT')
|
||||
if gpu_seconds>900 or cpu_seconds>cpu_cap:add('FINAL_RESOURCE_LIMIT')
|
||||
return out
|
||||
|
||||
def run_prediction(queue,c,pilot,fold,prior_gpu,cpu_cap):
|
||||
root=queue/'jobs'/str(fold);gpu_uuid=c['gpu_uuid'];expected_parts=next(j['expected_parts'] for j in c['jobs'] if j['id']==fold)
|
||||
start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None
|
||||
def elapsed():return 60+time.monotonic()-start
|
||||
def reason(x):
|
||||
if x not in reasons:reasons.append(x)
|
||||
command=[sys.executable,'tools/track_a_postfit_rebind_worker.py','predict',str(queue),str(fold)]
|
||||
def kill():
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:stop_group(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
def scan():
|
||||
if tracker.scan(root/'prediction-records'):reason('RECORDED_VRAM_LIMIT')
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
try:
|
||||
scan();mem,util=device_usage(gpu_uuid);_,rss,_=tree_usage(os.getpid())
|
||||
if mem>15*2**30:reason('DEVICE_VRAM_LIMIT')
|
||||
if rss>16*2**30:reason('TREE_RSS_LIMIT')
|
||||
if disk_usage(queue)>20*2**30:reason('DISK_LIMIT')
|
||||
if elapsed()>=895:reason('PREDICTION_TIME_LIMIT')
|
||||
for x in requested:reason(x)
|
||||
save(root/'prediction-live.json',dict(gpu_seconds_charged=elapsed(),device_bytes=mem,device_utilization=util,rss_bytes=rss,peaks=tracker.maxima,reasons=reasons,unix=time.time()))
|
||||
except Exception as exc:reason('MONITOR_ERROR');errors.append(str(exc))
|
||||
if reasons:kill()
|
||||
done.wait(.5)
|
||||
old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGTERM,signal.SIGINT]};thread=None
|
||||
try:
|
||||
wait_start=time.monotonic();waiting=[]
|
||||
while True:
|
||||
mem,util=device_usage(gpu_uuid);waiting.append(dict(unix=time.time(),bytes=mem,utilization=util))
|
||||
save(root/'recovery-idle-wait.json',dict(samples=waiting,limit_seconds=c['idle_wait_seconds'],worker_started=False))
|
||||
if requested:raise RuntimeError('Signal received during idle wait')
|
||||
duration=time.monotonic()-wait_start
|
||||
if duration>c['idle_wait_seconds']:raise RuntimeError('SELECTED_GPU_BUSY_AFTER_BOUNDED_WAIT_NO_RETRY')
|
||||
if mem<=128*2**20 and util==0:break
|
||||
if duration>=c['idle_wait_seconds']:raise RuntimeError('SELECTED_GPU_BUSY_AFTER_BOUNDED_WAIT_NO_RETRY')
|
||||
time.sleep(min(.5,max(0.,c['idle_wait_seconds']-(time.monotonic()-wait_start))))
|
||||
for i in range(3):
|
||||
mem,util=device_usage(gpu_uuid);samples.append(dict(unix=time.time(),bytes=mem,utilization=util))
|
||||
if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY_NO_RETRY')
|
||||
if i<2:time.sleep(2)
|
||||
save(root/'prediction-idle-polls.json',samples)
|
||||
if requested:raise RuntimeError('Signal received during setup')
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
remaining=900-elapsed()-10
|
||||
if remaining<=0:raise RuntimeError('Setup exhausted prediction budget')
|
||||
receipt=supervise(command,root/'prediction-guard','predict',dict(total_cpu_seconds=cpu_cap-60,job_cpu_seconds=cpu_cap-60,job_wall_seconds=remaining,tree_rss_bytes=16*2**30,disk_bytes=20*2**30,max_processes=1),poll=.2,gpu_uuid=gpu_uuid)
|
||||
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill()
|
||||
finally:
|
||||
done.set()
|
||||
if thread:thread.join(timeout=4)
|
||||
if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT')
|
||||
try:scan()
|
||||
except Exception as exc:reason('FINAL_SCAN_ERROR');errors.append(str(exc))
|
||||
for s,h in old.items():signal.signal(s,h)
|
||||
result=None;p=root/'prediction-records/result.json'
|
||||
try:
|
||||
if p.exists():result=json.loads(p.read_text())
|
||||
except Exception as exc:reason('FINAL_RESULT_READ_ERROR');errors.append(str(exc))
|
||||
totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu
|
||||
duration=elapsed();reasons=final_reasons(reasons,receipt,result,tracker,duration,totalcpu,cpu_cap,expected_parts)
|
||||
out=dict(status='PASS' if not reasons else 'STOPPED',reasons=reasons,monitor_errors=errors,worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=duration,cpu_seconds_charged=totalcpu,cumulative_track_a_gpu_seconds=prior_gpu+duration,no_retry=True)
|
||||
save(root/'prediction-guard.json',out);return out
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Frozen full-domain RMSE/max(truth) summaries on identical paired support."""
|
||||
import numpy as np
|
||||
|
||||
from track_a_postfit_rebind_common import OMITTED, SPLITS
|
||||
from track_a_postfit_statistics import paired_uncertainty
|
||||
|
||||
|
||||
def summarize(rows, jobs):
|
||||
if tuple(j['id'] for j in jobs) != SPLITS or {r['split'] for r in rows} != set(SPLITS):
|
||||
raise ValueError('Exactly the four approved original splits required')
|
||||
per = []
|
||||
for j in jobs:
|
||||
chosen = [r for r in rows if r['split'] == j['id']]
|
||||
if {r['case'] for r in chosen} != set(j['test']) or len(chosen) != len(j['test']):
|
||||
raise ValueError('Complete paired prediction support required')
|
||||
valid = [r for r in chosen if r['A'] is not None and r['H2'] is not None]
|
||||
summaries = {}
|
||||
for method in ['A', 'H2']:
|
||||
values = np.asarray([r[method] for r in valid], np.float64)
|
||||
if not np.isfinite(values).all() or np.any(values < 0):
|
||||
raise ValueError('Invalid paired metric')
|
||||
summaries[method] = {name: float(fn(values)) if len(values) else None
|
||||
for name, fn in [('min',np.min),('median',np.median),('max',np.max)]}
|
||||
delta = np.asarray([r['H2']-r['A'] for r in valid],np.float64)
|
||||
per.append(dict(split=j['id'],requested_test_parts=315,paired_predictions=len(chosen),
|
||||
scored_pairs=len(valid),missing_original=list(j['missing_test']),
|
||||
undefined=[r['case'] for r in chosen if r['A'] is None or r['H2'] is None],
|
||||
summaries=summaries,paired_mean_H2_minus_A=float(delta.mean()) if len(delta) else None,
|
||||
paired_median_H2_minus_A=float(np.median(delta)) if len(delta) else None))
|
||||
means = {method: {name: float(np.mean([r['summaries'][method][name] for r in per]))
|
||||
if all(r['summaries'][method][name] is not None for r in per) else None
|
||||
for name in ['min','median','max']} for method in ['A','H2']}
|
||||
return dict(per_split=per,mean_of_four_split_summaries=means,
|
||||
evaluated_splits=list(SPLITS), excluded_split=OMITTED,
|
||||
direction='Negative paired H2-minus-A favours H2; descriptive only',
|
||||
descriptive_paired_uncertainty=paired_uncertainty(rows),significance_gate=False,metric='RMSE / max(full truth)',
|
||||
full_requested_315_coverage=all(not r['missing_original'] for r in per))
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Four fixed test split predictions then independently gated paired full-node scoring."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
import numpy as np
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_h1_worker_v2 import checked, sha, save
|
||||
from phase2_home_freeze_io import verify
|
||||
from track_a_postfit_common import checkpoint_identity, prediction_identity
|
||||
from track_a_postfit_rebind_common import config, read, adopted, selected_training, SPLITS
|
||||
from track_a_postfit_rebind_statistics import summarize
|
||||
from phase2_residual_math import decode_seconds
|
||||
|
||||
|
||||
def predict(root,index):
|
||||
import torch
|
||||
from phase2_h2_model import H2Geo
|
||||
from phase2_h2_data import to_device,arguments
|
||||
from phase3_item1_postfit_data import load_evaluation_part
|
||||
c,tc,entry=config(root);validate_review(root);a=adopted(root,c)
|
||||
if index not in SPLITS:raise ValueError('Split outside approved four-of-five support')
|
||||
j=tc['jobs'][index];m=next(m for m in a['models'] if m['job']==index)
|
||||
out=root/'jobs'/str(index)/'prediction-records';out.mkdir(exist_ok=False)
|
||||
if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU allocator identity')
|
||||
if torch.__version__!=tc['versions']['torch'] or np.__version__!=tc['versions']['numpy']:raise ValueError('Library identity')
|
||||
os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
|
||||
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False;torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats()
|
||||
def peaks():return dict(peak_allocated_bytes=int(torch.cuda.max_memory_allocated()),peak_reserved_bytes=int(torch.cuda.max_memory_reserved()))
|
||||
parts=0;entries=[];start=time.monotonic();cpu=time.process_time()
|
||||
try:
|
||||
blob=torch.load(checked(m['path'],m['sha256']),map_location='cpu',weights_only=False)
|
||||
checkpoint_identity(blob,j,m['training_contract_sha256']);model=H2Geo().cuda();model.load_state_dict(blob['model'],strict=True);model.eval();del blob
|
||||
stats=read(checked(m['standardization'],m['standardization_sha256']));view=dict(tc,folds=[j])
|
||||
with torch.no_grad():
|
||||
for case in j['test']:
|
||||
data=to_device(load_evaluation_part(view,case,stats,'test'),'cuda')
|
||||
residual=model(*arguments(data),checkpoint_blocks=False)
|
||||
if not bool(torch.isfinite(residual).all()):raise ValueError('Nonfinite test residual')
|
||||
seconds=decode_seconds(residual.cpu().numpy(),data['F'],j['c'],data['t_set']);torch.cuda.synchronize()
|
||||
if seconds.shape!=(tc['parts'][case]['nodes'],) or not np.isfinite(seconds).all():raise ValueError('Full-node prediction invalid')
|
||||
p=root/'predictions'/str(index)/(case+'.npz');p.parent.mkdir(parents=True,exist_ok=True)
|
||||
if p.exists():raise FileExistsError('No prediction overwrite/retry')
|
||||
np.savez_compressed(p,seconds=seconds,raw_ids=data['raw_ids'])
|
||||
entries.append(dict(split=index,case=case,path=str(p),sha256=sha(p),nodes=len(seconds),
|
||||
raw_ids_sha256=tc['parts'][case]['output_sha256']['raw_ids.npy'],negative_predictions=int(np.sum(seconds<0))))
|
||||
parts+=1;save(out/'progress.json',dict(stage='TEST_PREDICTION_NO_TARGETS',parts=parts,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()))
|
||||
del data,residual,seconds
|
||||
save(out/'manifest.json',dict(entries=entries,test_targets_read=False))
|
||||
save(out/'result.json',dict(status='COMPLETE',parts=parts,checkpoint_sha256=m['sha256'],test_targets_read=False,**peaks()))
|
||||
except BaseException as exc:
|
||||
save(out/'failure.json',dict(status='FAILED',parts=parts,reason=type(exc).__name__+': '+str(exc),**peaks()));raise
|
||||
|
||||
|
||||
def score(root):
|
||||
import json
|
||||
from phase2_h2_data import cached
|
||||
from gre24_features_v2 import per_geometry_metrics
|
||||
c,tc,entry=config(root);validate_review(root);adopted(root,c);verify(root,'predictions')
|
||||
selected=selected_training(tc)
|
||||
entries=read(root/'predictions-manifest.json')['entries'];prediction_identity(root,c,selected,entries)
|
||||
out=root/'score';out.mkdir(exist_ok=False);bycase={}
|
||||
save(out/'review-consumed.json',dict(review_sha256=sha(root/'review.json'),prediction_manifest_sha256=sha(root/'predictions-manifest.json'),before_target_values=True))
|
||||
for e in entries:bycase.setdefault(e['case'],[]).append(e)
|
||||
expected=set().union(*(set(j['test']) for j in selected['jobs']))
|
||||
if set(bycase)!=expected or len(expected)!=c['expected_unique_target_reads']:raise ValueError('Frozen unique test domain')
|
||||
rows=[];count=0
|
||||
for case in sorted(bycase):
|
||||
raw=cached(tc,case,'raw_ids')
|
||||
with (out/'target-access.jsonl').open('a') as log:
|
||||
log.write(json.dumps(dict(case=case,role='test',source_sha256=tc['parts'][case]['output_sha256']['seconds.npy'],unix=time.time()))+'\n')
|
||||
truth=cached(tc,case,'seconds');count+=1
|
||||
if truth.shape!=raw.shape:raise ValueError('Target native node domain')
|
||||
for e in bycase[case]:
|
||||
ref=c['A_predictions'][str(e['split'])][case]
|
||||
A=np.load(checked(ref['path'],ref['sha256']),allow_pickle=False)
|
||||
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z:
|
||||
if not np.array_equal(z['raw_ids'],raw):raise ValueError('H2 native raw-ID ordering')
|
||||
H2=z['seconds'].copy()
|
||||
am=per_geometry_metrics(truth,A);hm=per_geometry_metrics(truth,H2)
|
||||
if am['nodes']!=e['nodes'] or hm['nodes']!=e['nodes'] or int(np.sum(H2<0))!=e['negative_predictions']:raise ValueError('Paired full-node coverage')
|
||||
av=am['rrmse_maximum'];hv=hm['rrmse_maximum']
|
||||
rows.append(dict(split=e['split'],case=case,nodes=e['nodes'],A=av,H2=hv,
|
||||
delta_H2_minus_A=hv-av if av is not None and hv is not None else None,
|
||||
truth_maximum=am['denominators']['maximum'],undefined=av is None or hv is None,
|
||||
A_rmse=am['rmse'],H2_rmse=hm['rmse'],A_negative=int(np.sum(A<0)),H2_negative=e['negative_predictions']))
|
||||
del truth
|
||||
save(out/'progress.json',dict(unique_test_target_reads=count,paired_rows=len(rows)))
|
||||
if len(rows)!=c['expected_pair_visits'] or count!=c['expected_unique_target_reads']:raise ValueError('Complete paired scoring required')
|
||||
save(out/'result.json',dict(status='COMPLETE_FOUR_OF_FIVE',rows=rows,summary=summarize(rows,selected['jobs']),
|
||||
evaluated_splits=list(SPLITS),excluded_split=c['excluded_split'],
|
||||
unique_test_target_reads=count,paired_visits=len(rows),original_failed=tc['original_failed'],
|
||||
A_refit=False,H2_refit=False,clipped=False,new_decodes=0,comparison_contract_sha256=c['comparison_contract_sha256'],
|
||||
review_sha256=sha(root/'review.json'),limitations=['Four of five fixed splits; split 3 has no completed training checkpoint after the budget stop',
|
||||
'Protocol-level re-creation; original author split IDs unavailable',
|
||||
'Random-part splits can share geometry families','Original14 preparation failures are retained, not resampled',
|
||||
'RMSE/max(full truth) is the frozen documented proxy; paper numbers are context only']))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
if sys.argv[1]=='predict':predict(Path(sys.argv[2]),int(sys.argv[3]))
|
||||
elif sys.argv[1]=='score' and len(sys.argv)==3:score(Path(sys.argv[2]))
|
||||
else:raise ValueError('predict root index or score root')
|
||||
Reference in new issue
Block a user