Preserve GINO measurement startup stop and Torch UUID parser evidence
This commit is contained in:
1 parent
bc3f68c667
commit
17a40a6c1b
12 files changed
+228
No files matched your search
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"status": "STOPPED",
|
||||
"trigger": "WORKER_FAILED",
|
||||
"contract_sha256": "e4545bb5e9eea0a5a501cbfd538ee67468224da36ea151951c7affc5bfc4719f",
|
||||
"result_sha256": null,
|
||||
"worker": {
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19"
|
||||
],
|
||||
"returncode": 1,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 3.02495,
|
||||
"wall_seconds": 3.1360727309947833,
|
||||
"peak_tree_rss_bytes": 343724032,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 94208,
|
||||
"max_poll_gap_seconds": 0.2760110989911482,
|
||||
"poll_target_seconds": 0.2,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": true,
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
},
|
||||
"booked_idle_seconds": 20.40600436902605,
|
||||
"cumulative_gpu_wall_seconds": 23.69431395700667,
|
||||
"gpu_wall_cap_seconds": 3600,
|
||||
"recorded_peaks": {
|
||||
"peak_allocated_bytes": 0,
|
||||
"peak_reserved_bytes": 0
|
||||
},
|
||||
"no_retry": true
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"contract_sha256": "e4545bb5e9eea0a5a501cbfd538ee67468224da36ea151951c7affc5bfc4719f",
|
||||
"coordinator": {
|
||||
"pid": 2549759,
|
||||
"created": 1795072040.56
|
||||
},
|
||||
"start_by_monotonic": 686324.938349955
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
Traceback (most recent call last):
|
||||
File "/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2_worker.py", line 21, in <module>
|
||||
FunctionType(work.__code__, namespace, work.__name__)(root, c)
|
||||
File "/home/frank14f/optane/MeltMind/tools/track_b_gino_shapenet_worker.py", line 235, in work
|
||||
if "V100" not in torch.cuda.get_device_name(0):
|
||||
File "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/lib/python3.10/site-packages/torch/cuda/__init__.py", line 341, in get_device_name
|
||||
return get_device_properties(device).name
|
||||
File "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/lib/python3.10/site-packages/torch/cuda/__init__.py", line 374, in get_device_properties
|
||||
raise AssertionError("Invalid device id")
|
||||
AssertionError: Invalid device id
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19"
|
||||
],
|
||||
"returncode": 1,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 3.02495,
|
||||
"wall_seconds": 3.1360727309947833,
|
||||
"peak_tree_rss_bytes": 343724032,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 94208,
|
||||
"max_poll_gap_seconds": 0.2760110989911482,
|
||||
"poll_target_seconds": 0.2,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": true,
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19"
|
||||
],
|
||||
"limits": {
|
||||
"total_cpu_seconds": 3500,
|
||||
"job_cpu_seconds": 3500,
|
||||
"job_wall_seconds": 3519.4860540250083,
|
||||
"tree_rss_bytes": 12884901888,
|
||||
"disk_bytes": 2147483648,
|
||||
"max_processes": 1
|
||||
},
|
||||
"started_unix": 1795072063.4696167
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"status": "IDLE_CONFIRMED",
|
||||
"uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"polls": 3,
|
||||
"consecutive_idle": 3,
|
||||
"wall_seconds": 20.310800467035733,
|
||||
"last": {
|
||||
"unix": 1795072063.3707962,
|
||||
"uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"memory_MiB": 5,
|
||||
"utilization_percent": 0,
|
||||
"occupants": [],
|
||||
"idle": true
|
||||
}
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"checked_unix": 1795072040.0946133,
|
||||
"gpu": {
|
||||
"unix": 1795072040.0942595,
|
||||
"uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"memory_MiB": 5,
|
||||
"utilization_percent": 0,
|
||||
"occupants": [],
|
||||
"idle": true
|
||||
},
|
||||
"exact_review": "PASS",
|
||||
"historical_roots_started": 0
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"cpu_seconds_charged": 3.02495,
|
||||
"attempts": [
|
||||
{
|
||||
"name": "gino",
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"cpu_seconds": 3.02495
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"unit": "meltmind-track-b-gino-gpu2-20261119.service",
|
||||
"contract_sha256": "e4545bb5e9eea0a5a501cbfd538ee67468224da36ea151951c7affc5bfc4719f",
|
||||
"review_sha256": "67b8263bc43d8f176779e6c9f39b320760a2d6c36db475c43b7aef461a905422",
|
||||
"stage_approval_sha256": "80b63ceb8ca3e781d8c25383c3a7be9d3cf83be3d3f9acece172875ae44225d6",
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"unix": 1795072042.8671427,
|
||||
"epoch_measurement_only": true
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"coordinator": {
|
||||
"pid": 2549759,
|
||||
"created": 1795072040.56
|
||||
},
|
||||
"contract_sha256": "e4545bb5e9eea0a5a501cbfd538ee67468224da36ea151951c7affc5bfc4719f",
|
||||
"cpu_acceptance_sha256": "093e646890929af6f520eb5eecf8c751025c0a24b8b5689094b1a0ae4da08dc3",
|
||||
"interpreter_path": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python",
|
||||
"interpreter_sha256": "cd4a09f01b941a4de48669ffe81e055d001c79b44409ddb14781f437b8ec3f45"
|
||||
}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"audited_utc": "2026-11-19T07:10:27.309958+00:00",
|
||||
"status": "STOPPED",
|
||||
"trigger": "WORKER_FAILED",
|
||||
"original_exit_context": "Installed Torch CUDA property wrapper rejected logical device 0 with physical UUID mask",
|
||||
"exact_independent_pass_commit": "bc3f68c667a441ab5c95af5a9c12fe815c99f168",
|
||||
"scope_authority": "h-8d45c127",
|
||||
"launch_count": 1,
|
||||
"coordinator_pid_at_launch": 2549759,
|
||||
"worker_exit": 1,
|
||||
"original_guard_gpu_seconds": 23.69431395700667,
|
||||
"worker_wall_seconds": 3.1360727309947833,
|
||||
"device_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"recorded_peaks": {
|
||||
"peak_allocated_bytes": 0,
|
||||
"peak_reserved_bytes": 0
|
||||
},
|
||||
"heldout_targets_read": 0,
|
||||
"learning_samples_read": 0,
|
||||
"optimizer_steps": 0,
|
||||
"measurement_result": null,
|
||||
"host_terminal_observation": "MainPID=0\nExecMainStatus=1\nActiveState=failed\nSubState=failed\n PID PPID STAT ELAPSED COMMAND\nGPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c, 2512405, 820 MiB\nGPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36, 2517067, 1708 MiB\n",
|
||||
"same_error_reproduced_using_mock_nvml_and_no_cuda": true,
|
||||
"retry_started": false,
|
||||
"team_report_slack": "1795072202.479989",
|
||||
"evidence_sha256": {
|
||||
"artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/service-launch.json": "1cadc651f508d314f2faf8d63bc4c2413f45d2f8943f545f4ca0e9bd1f4f3669",
|
||||
"artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/last-pre-admission.json": "a9d2316b4094837c6ff8aea2642df03150a3790156fdb5b2e6adb938fbd8b868",
|
||||
"artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/started.json": "1967c4e3d7f5423eb966b2a8f565edcd85524dfeb12010068d6f569e42909cf3",
|
||||
"artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/idle-gate.json": "10f1a79315316a3160319226d293d3fedc4e2c5457664342ade8b8d0bd61b40f",
|
||||
"artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/dispatch.json": "93cbd1525a0cae8a06771d750cdd22545bd8d4da586ec80df6e9d10fbb12327b",
|
||||
"artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/gino.started.json": "8fdda2432fb032404af8d5c803a6e73989c7527a52d115c60fdba4878334159a",
|
||||
"artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/gino.receipt.json": "d947dda1ef7ac76d13792bbd6c81a86194ab2eccb65296b2b2fd2647bacc3837",
|
||||
"artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/gino.log": "4c0f619d6bcbb3402dbda364bb8031ef878c7370fb5e9f35401a10df9f31a674",
|
||||
"artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/ledger.json": "8bc9b505e224bfed74d2c5005d713ee8b095c7b7a0293b239b3c84c748d8b141",
|
||||
"artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/complete.json": "c983c05f9531119dad905d71c73399d788837299c44e4736b50a42262b4e9e6e",
|
||||
"artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/uuid-parser-cpu-diagnostic.json": "07cc919a949f6a7995348ea3f33e8a022c98b7d5de372baed2114de4f02ba9d5"
|
||||
}
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"status": "REPRODUCED_WITHOUT_CUDA",
|
||||
"torch_version": "1.13.1+cu116",
|
||||
"cuda_python_source": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/lib/python3.10/site-packages/torch/cuda/__init__.py",
|
||||
"cuda_python_source_sha256": "2f359b138403461d8df4a824d887ad39786ca44757d309ee6946f95380957f18",
|
||||
"results": {
|
||||
"masked": {
|
||||
"mask": "",
|
||||
"parsed": [
|
||||
-1
|
||||
],
|
||||
"nvml_count_with_mock_four_devices": 0,
|
||||
"properties_error": "Invalid device id"
|
||||
},
|
||||
"uuid": {
|
||||
"mask": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"parsed": [
|
||||
-1
|
||||
],
|
||||
"nvml_count_with_mock_four_devices": 0,
|
||||
"properties_error": "Invalid device id"
|
||||
},
|
||||
"ordinal_control": {
|
||||
"mask": "2",
|
||||
"parsed": [
|
||||
2
|
||||
],
|
||||
"nvml_count_with_mock_four_devices": 1,
|
||||
"properties_error": null
|
||||
}
|
||||
},
|
||||
"real_nvml_calls": 0,
|
||||
"real_cuda_calls": 0,
|
||||
"measurement_relaunched": false,
|
||||
"recorded_mask": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76"
|
||||
}
|
||||
Reference in new issue
Block a user