Record authorized recovery actual start and first fold cost gate

This commit is contained in:
MeltAstra committed 2026-11-18 20:19:00 +08:00
1 parent bfb4b7c84d
commit ff0033d733
25 files changed
+905

No files matched your search

@@ -0,0 +1,18 @@
{
"unix": 1795004292.8713422,
"status": "SAME_ACTION_REVIEW_ALLOWED_AND_WORKER_OBSERVED",
"previous_block": "approval-block.json",
"authorization_relay": "user-confirmation-relay.json",
"review_provenance": "Relayed human question and assent submitted transparently to the same automatic approval workflow; original user item not independently returned by history tool.",
"service_launch": {
"unit": "meltmind-track-a-recovery-20261118.service",
"returncode": 0,
"stdout": "",
"stderr": "Running as unit: meltmind-track-a-recovery-20261118.service\n",
"unix": 1795004202.9873908,
"submitted_only": true
},
"actual_start_audit": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/actual-start-audit.json",
"original_rejection_preserved": true,
"alternate_execution_route_used": false
}
@@ -0,0 +1,24 @@
{
"received_unix": 1795004170.198358,
"source": "native codex_delegation",
"source_thread_id": "6afb1249-81d4-8191-b190-6fa32c3880de",
"question": "是否允许在 seed 1 继续运行时利用空闲 GPU 一次性恢复 Track A,复用三折、重训中断两折、总预算 162000 GPU 秒含已有成本,不再自动追加恢复。",
"answer": "我同意",
"attribution": "Yanqi, relayed by TRON",
"verification": "Exact question and reply supplied in incoming native relay. read_thread for source returned in-progress turn with no items; direct original user message was not independently retrieved. Submit this provenance to the same automatic approval reviewer; do not bypass any renewed rejection.",
"scope": {
"reused_folds": [
0,
1,
2
],
"fresh_folds": [
3,
4
],
"total_gpu_seconds": 162000,
"old_gpu_seconds_once": 47943.94719455298,
"seed1_unchanged": true,
"no_further_recovery": true
}
}
@@ -0,0 +1,55 @@
{
"unix": 1795004330.5901194,
"status": "FIRST_DISPATCHED_FOLD_COST_PASS",
"fold": 4,
"first_epoch_seconds": 57.02044904301874,
"own_projection": 14437.42601964093,
"cumulative_projection": 100541.37321419391,
"original_old_cost_once": 47943.94719455298,
"postfit_reserve": 7200,
"pending_fold3_unmeasured": true,
"worker_progress": {
"fold": 4,
"stage": "TRAIN",
"epoch": 2,
"optimizer_steps": 600,
"wall_seconds": 112.14072392100934,
"cpu_seconds": 111.596696969,
"case": "00000787_a25fb37392a44447b9684f16_step_012",
"epoch_parts": 290,
"epoch_nodes": 8188943,
"lr": 0.00019999999999999998,
"loss": 0.0017632129602134228,
"gradient_norm_before_clip": 0.06885933130979538,
"peak_allocated_bytes": 1461572608,
"peak_reserved_bytes": 1516240896
},
"files": [
{
"source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/first-epoch-projection.json",
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/first-epoch-projection.json",
"sha256": "e524294579063ab269f9996ee6025dad198372b0eeb3d642f4a838e279e99b56"
},
{
"source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/worker/history/001.json",
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/worker/history/001.json",
"sha256": "9538696224a700851e7868f6d2b36c4512e4b2eedd79725af6b05596c849497e"
},
{
"source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/worker/progress.json",
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/worker/progress.json",
"sha256": "bd39f8aa3916e7d7eea424c35b5581312abd7ac67231ca70325f4002b3092f82"
},
{
"source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/live-progress.json",
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/live-progress.json",
"sha256": "b577571afa3aab4c0b83dc4c79a3ea6ca7be8aa00f49f65f0840b4f89b9d742e"
},
{
"source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/3/dispatch-idle.json",
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/3/dispatch-idle.json",
"sha256": "d0e38b8d59b1dffcc02640e3d506e4ced0238115c2fdc0d42ab820f7599ec116"
}
],
"no_relaunch": true
}
@@ -0,0 +1,18 @@
{
"start_unix": 1795004329.9132068,
"end_unix": 1795004329.9554157,
"valid": true,
"memory_bytes": 1928331264,
"utilization": 20,
"monitor_policy": {
"rule": "three_consecutive_failed_samples_per_operation",
"failure_limit": 3,
"failed": false,
"errors": 0,
"streaks": {
"gpu_usage": 0
},
"max_streaks": {}
},
"charged_gpu_seconds": 184.04915182094555
}
@@ -0,0 +1,29 @@
{
"status": "PASS",
"cell": {
"id": 4,
"projected_gpu_seconds": 14437.42601964093,
"within_cap": true
},
"total": {
"status": "PASS",
"rows": [
{
"id": 3,
"projected_gpu_seconds": 30960,
"basis": "continuing_projection_or_pending_cap"
},
{
"id": 4,
"projected_gpu_seconds": 14437.42601964093,
"basis": "continuing_projection_or_pending_cap"
}
],
"new_projected_gpu_seconds": 52597.42601964093,
"old_gpu_seconds_once": 47943.94719455298,
"cumulative_projected_gpu_seconds": 100541.37321419391,
"postfit_reserve": 7200
},
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
"unix": 1795004279.4981134
}
@@ -0,0 +1,15 @@
{
"fold": 4,
"stage": "EPOCH_COMPLETE",
"epoch": 1,
"optimizer_steps": 310,
"wall_seconds": 60.80659728299361,
"cpu_seconds": 60.766106956,
"train_mean_weighted_loss": 0.005705016106361244,
"train_epoch_seconds": 57.02044904301874,
"epoch_parts": 310,
"epoch_nodes": 8642607,
"lr": 0.0001,
"peak_allocated_bytes": 1461572608,
"peak_reserved_bytes": 1516240896
}
@@ -0,0 +1,16 @@
{
"fold": 4,
"stage": "TRAIN",
"epoch": 2,
"optimizer_steps": 600,
"wall_seconds": 112.14072392100934,
"cpu_seconds": 111.596696969,
"case": "00000787_a25fb37392a44447b9684f16_step_012",
"epoch_parts": 290,
"epoch_nodes": 8188943,
"lr": 0.00019999999999999998,
"loss": 0.0017632129602134228,
"gradient_norm_before_clip": 0.06885933130979538,
"peak_allocated_bytes": 1461572608,
"peak_reserved_bytes": 1516240896
}
@@ -0,0 +1,17 @@
{
"unix": 1795004329.9561625,
"active": [
{
"id": 4,
"guard_pid": 1650109
}
],
"pending": [
3
],
"terminal": [],
"old_gpu_seconds_once": 47943.94719455298,
"new_training_gpu_seconds": 368.09981042100117,
"cumulative_training_gpu_seconds": 48312.04701779806,
"postfit_started": false
}