Record authorized recovery actual start and first fold cost gate
This commit is contained in:
1 parent
bfb4b7c84d
commit
ff0033d733
25 files changed
+905
No files matched your search
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"unix": 1795004292.8713422,
|
||||
"status": "SAME_ACTION_REVIEW_ALLOWED_AND_WORKER_OBSERVED",
|
||||
"previous_block": "approval-block.json",
|
||||
"authorization_relay": "user-confirmation-relay.json",
|
||||
"review_provenance": "Relayed human question and assent submitted transparently to the same automatic approval workflow; original user item not independently returned by history tool.",
|
||||
"service_launch": {
|
||||
"unit": "meltmind-track-a-recovery-20261118.service",
|
||||
"returncode": 0,
|
||||
"stdout": "",
|
||||
"stderr": "Running as unit: meltmind-track-a-recovery-20261118.service\n",
|
||||
"unix": 1795004202.9873908,
|
||||
"submitted_only": true
|
||||
},
|
||||
"actual_start_audit": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/actual-start-audit.json",
|
||||
"original_rejection_preserved": true,
|
||||
"alternate_execution_route_used": false
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"received_unix": 1795004170.198358,
|
||||
"source": "native codex_delegation",
|
||||
"source_thread_id": "6afb1249-81d4-8191-b190-6fa32c3880de",
|
||||
"question": "是否允许在 seed 1 继续运行时利用空闲 GPU 一次性恢复 Track A,复用三折、重训中断两折、总预算 162000 GPU 秒含已有成本,不再自动追加恢复。",
|
||||
"answer": "我同意",
|
||||
"attribution": "Yanqi, relayed by TRON",
|
||||
"verification": "Exact question and reply supplied in incoming native relay. read_thread for source returned in-progress turn with no items; direct original user message was not independently retrieved. Submit this provenance to the same automatic approval reviewer; do not bypass any renewed rejection.",
|
||||
"scope": {
|
||||
"reused_folds": [
|
||||
0,
|
||||
1,
|
||||
2
|
||||
],
|
||||
"fresh_folds": [
|
||||
3,
|
||||
4
|
||||
],
|
||||
"total_gpu_seconds": 162000,
|
||||
"old_gpu_seconds_once": 47943.94719455298,
|
||||
"seed1_unchanged": true,
|
||||
"no_further_recovery": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
{
|
||||
"unix": 1795004330.5901194,
|
||||
"status": "FIRST_DISPATCHED_FOLD_COST_PASS",
|
||||
"fold": 4,
|
||||
"first_epoch_seconds": 57.02044904301874,
|
||||
"own_projection": 14437.42601964093,
|
||||
"cumulative_projection": 100541.37321419391,
|
||||
"original_old_cost_once": 47943.94719455298,
|
||||
"postfit_reserve": 7200,
|
||||
"pending_fold3_unmeasured": true,
|
||||
"worker_progress": {
|
||||
"fold": 4,
|
||||
"stage": "TRAIN",
|
||||
"epoch": 2,
|
||||
"optimizer_steps": 600,
|
||||
"wall_seconds": 112.14072392100934,
|
||||
"cpu_seconds": 111.596696969,
|
||||
"case": "00000787_a25fb37392a44447b9684f16_step_012",
|
||||
"epoch_parts": 290,
|
||||
"epoch_nodes": 8188943,
|
||||
"lr": 0.00019999999999999998,
|
||||
"loss": 0.0017632129602134228,
|
||||
"gradient_norm_before_clip": 0.06885933130979538,
|
||||
"peak_allocated_bytes": 1461572608,
|
||||
"peak_reserved_bytes": 1516240896
|
||||
},
|
||||
"files": [
|
||||
{
|
||||
"source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/first-epoch-projection.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/first-epoch-projection.json",
|
||||
"sha256": "e524294579063ab269f9996ee6025dad198372b0eeb3d642f4a838e279e99b56"
|
||||
},
|
||||
{
|
||||
"source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/worker/history/001.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/worker/history/001.json",
|
||||
"sha256": "9538696224a700851e7868f6d2b36c4512e4b2eedd79725af6b05596c849497e"
|
||||
},
|
||||
{
|
||||
"source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/worker/progress.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/worker/progress.json",
|
||||
"sha256": "bd39f8aa3916e7d7eea424c35b5581312abd7ac67231ca70325f4002b3092f82"
|
||||
},
|
||||
{
|
||||
"source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/live-progress.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/live-progress.json",
|
||||
"sha256": "b577571afa3aab4c0b83dc4c79a3ea6ca7be8aa00f49f65f0840b4f89b9d742e"
|
||||
},
|
||||
{
|
||||
"source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/3/dispatch-idle.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/3/dispatch-idle.json",
|
||||
"sha256": "d0e38b8d59b1dffcc02640e3d506e4ced0238115c2fdc0d42ab820f7599ec116"
|
||||
}
|
||||
],
|
||||
"no_relaunch": true
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"start_unix": 1795004329.9132068,
|
||||
"end_unix": 1795004329.9554157,
|
||||
"valid": true,
|
||||
"memory_bytes": 1928331264,
|
||||
"utilization": 20,
|
||||
"monitor_policy": {
|
||||
"rule": "three_consecutive_failed_samples_per_operation",
|
||||
"failure_limit": 3,
|
||||
"failed": false,
|
||||
"errors": 0,
|
||||
"streaks": {
|
||||
"gpu_usage": 0
|
||||
},
|
||||
"max_streaks": {}
|
||||
},
|
||||
"charged_gpu_seconds": 184.04915182094555
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"cell": {
|
||||
"id": 4,
|
||||
"projected_gpu_seconds": 14437.42601964093,
|
||||
"within_cap": true
|
||||
},
|
||||
"total": {
|
||||
"status": "PASS",
|
||||
"rows": [
|
||||
{
|
||||
"id": 3,
|
||||
"projected_gpu_seconds": 30960,
|
||||
"basis": "continuing_projection_or_pending_cap"
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"projected_gpu_seconds": 14437.42601964093,
|
||||
"basis": "continuing_projection_or_pending_cap"
|
||||
}
|
||||
],
|
||||
"new_projected_gpu_seconds": 52597.42601964093,
|
||||
"old_gpu_seconds_once": 47943.94719455298,
|
||||
"cumulative_projected_gpu_seconds": 100541.37321419391,
|
||||
"postfit_reserve": 7200
|
||||
},
|
||||
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"unix": 1795004279.4981134
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"fold": 4,
|
||||
"stage": "EPOCH_COMPLETE",
|
||||
"epoch": 1,
|
||||
"optimizer_steps": 310,
|
||||
"wall_seconds": 60.80659728299361,
|
||||
"cpu_seconds": 60.766106956,
|
||||
"train_mean_weighted_loss": 0.005705016106361244,
|
||||
"train_epoch_seconds": 57.02044904301874,
|
||||
"epoch_parts": 310,
|
||||
"epoch_nodes": 8642607,
|
||||
"lr": 0.0001,
|
||||
"peak_allocated_bytes": 1461572608,
|
||||
"peak_reserved_bytes": 1516240896
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"fold": 4,
|
||||
"stage": "TRAIN",
|
||||
"epoch": 2,
|
||||
"optimizer_steps": 600,
|
||||
"wall_seconds": 112.14072392100934,
|
||||
"cpu_seconds": 111.596696969,
|
||||
"case": "00000787_a25fb37392a44447b9684f16_step_012",
|
||||
"epoch_parts": 290,
|
||||
"epoch_nodes": 8188943,
|
||||
"lr": 0.00019999999999999998,
|
||||
"loss": 0.0017632129602134228,
|
||||
"gradient_norm_before_clip": 0.06885933130979538,
|
||||
"peak_allocated_bytes": 1461572608,
|
||||
"peak_reserved_bytes": 1516240896
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"unix": 1795004329.9561625,
|
||||
"active": [
|
||||
{
|
||||
"id": 4,
|
||||
"guard_pid": 1650109
|
||||
}
|
||||
],
|
||||
"pending": [
|
||||
3
|
||||
],
|
||||
"terminal": [],
|
||||
"old_gpu_seconds_once": 47943.94719455298,
|
||||
"new_training_gpu_seconds": 368.09981042100117,
|
||||
"cumulative_training_gpu_seconds": 48312.04701779806,
|
||||
"postfit_started": false
|
||||
}
|
||||
Reference in new issue
Block a user