diff --git a/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/complete.json b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/complete.json new file mode 100644 index 000000000..f7a99e29d --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/complete.json @@ -0,0 +1,38 @@ +{ + "status": "STOPPED", + "trigger": "WORKER_FAILED", + "contract_sha256": "e4545bb5e9eea0a5a501cbfd538ee67468224da36ea151951c7affc5bfc4719f", + "result_sha256": null, + "worker": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2_worker.py", + "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19" + ], + "returncode": 1, + "stop_reason": null, + "cpu_seconds": 3.02495, + "wall_seconds": 3.1360727309947833, + "peak_tree_rss_bytes": 343724032, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 94208, + "max_poll_gap_seconds": 0.2760110989911482, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": true, + "gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "booked_idle_seconds": 20.40600436902605, + "cumulative_gpu_wall_seconds": 23.69431395700667, + "gpu_wall_cap_seconds": 3600, + "recorded_peaks": { + "peak_allocated_bytes": 0, + "peak_reserved_bytes": 0 + }, + "no_retry": true +} diff --git a/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/dispatch.json b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/dispatch.json new file mode 100644 index 000000000..de2982321 --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/dispatch.json @@ -0,0 +1,8 @@ +{ + "contract_sha256": "e4545bb5e9eea0a5a501cbfd538ee67468224da36ea151951c7affc5bfc4719f", + "coordinator": { + "pid": 2549759, + "created": 1795072040.56 + }, + "start_by_monotonic": 686324.938349955 +} diff --git a/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/gino.log b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/gino.log new file mode 100644 index 000000000..194f92c6a --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/gino.log @@ -0,0 +1,10 @@ +Traceback (most recent call last): + File "/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2_worker.py", line 21, in + FunctionType(work.__code__, namespace, work.__name__)(root, c) + File "/home/frank14f/optane/MeltMind/tools/track_b_gino_shapenet_worker.py", line 235, in work + if "V100" not in torch.cuda.get_device_name(0): + File "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/lib/python3.10/site-packages/torch/cuda/__init__.py", line 341, in get_device_name + return get_device_properties(device).name + File "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/lib/python3.10/site-packages/torch/cuda/__init__.py", line 374, in get_device_properties + raise AssertionError("Invalid device id") +AssertionError: Invalid device id diff --git a/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/gino.receipt.json b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/gino.receipt.json new file mode 100644 index 000000000..1fed77a92 --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/gino.receipt.json @@ -0,0 +1,24 @@ +{ + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2_worker.py", + "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19" + ], + "returncode": 1, + "stop_reason": null, + "cpu_seconds": 3.02495, + "wall_seconds": 3.1360727309947833, + "peak_tree_rss_bytes": 343724032, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 94208, + "max_poll_gap_seconds": 0.2760110989911482, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": true, + "gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null +} diff --git a/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/gino.started.json b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/gino.started.json new file mode 100644 index 000000000..4921ad76c --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/gino.started.json @@ -0,0 +1,16 @@ +{ + "command": [ + "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2_worker.py", + "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19" + ], + "limits": { + "total_cpu_seconds": 3500, + "job_cpu_seconds": 3500, + "job_wall_seconds": 3519.4860540250083, + "tree_rss_bytes": 12884901888, + "disk_bytes": 2147483648, + "max_processes": 1 + }, + "started_unix": 1795072063.4696167 +} diff --git a/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/idle-gate.json b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/idle-gate.json new file mode 100644 index 000000000..79b73b6ef --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/idle-gate.json @@ -0,0 +1,15 @@ +{ + "status": "IDLE_CONFIRMED", + "uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "polls": 3, + "consecutive_idle": 3, + "wall_seconds": 20.310800467035733, + "last": { + "unix": 1795072063.3707962, + "uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "memory_MiB": 5, + "utilization_percent": 0, + "occupants": [], + "idle": true + } +} diff --git a/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/last-pre-admission.json b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/last-pre-admission.json new file mode 100644 index 000000000..8dce15ab1 --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/last-pre-admission.json @@ -0,0 +1,13 @@ +{ + "checked_unix": 1795072040.0946133, + "gpu": { + "unix": 1795072040.0942595, + "uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "memory_MiB": 5, + "utilization_percent": 0, + "occupants": [], + "idle": true + }, + "exact_review": "PASS", + "historical_roots_started": 0 +} diff --git a/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/ledger.json b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/ledger.json new file mode 100644 index 000000000..d60fad231 --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/ledger.json @@ -0,0 +1,10 @@ +{ + "cpu_seconds_charged": 3.02495, + "attempts": [ + { + "name": "gino", + "status": "STOPPED_OR_FAILED", + "cpu_seconds": 3.02495 + } + ] +} diff --git a/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/service-launch.json b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/service-launch.json new file mode 100644 index 000000000..c55c533b9 --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/service-launch.json @@ -0,0 +1,9 @@ +{ + "unit": "meltmind-track-b-gino-gpu2-20261119.service", + "contract_sha256": "e4545bb5e9eea0a5a501cbfd538ee67468224da36ea151951c7affc5bfc4719f", + "review_sha256": "67b8263bc43d8f176779e6c9f39b320760a2d6c36db475c43b7aef461a905422", + "stage_approval_sha256": "80b63ceb8ca3e781d8c25383c3a7be9d3cf83be3d3f9acece172875ae44225d6", + "gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "unix": 1795072042.8671427, + "epoch_measurement_only": true +} diff --git a/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/started.json b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/started.json new file mode 100644 index 000000000..c97705a5e --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/started.json @@ -0,0 +1,10 @@ +{ + "coordinator": { + "pid": 2549759, + "created": 1795072040.56 + }, + "contract_sha256": "e4545bb5e9eea0a5a501cbfd538ee67468224da36ea151951c7affc5bfc4719f", + "cpu_acceptance_sha256": "093e646890929af6f520eb5eecf8c751025c0a24b8b5689094b1a0ae4da08dc3", + "interpreter_path": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python", + "interpreter_sha256": "cd4a09f01b941a4de48669ffe81e055d001c79b44409ddb14781f437b8ec3f45" +} diff --git a/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/terminal-uuid-enumeration-audit.json b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/terminal-uuid-enumeration-audit.json new file mode 100644 index 000000000..0c1af14aa --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/terminal-uuid-enumeration-audit.json @@ -0,0 +1,39 @@ +{ + "audited_utc": "2026-11-19T07:10:27.309958+00:00", + "status": "STOPPED", + "trigger": "WORKER_FAILED", + "original_exit_context": "Installed Torch CUDA property wrapper rejected logical device 0 with physical UUID mask", + "exact_independent_pass_commit": "bc3f68c667a441ab5c95af5a9c12fe815c99f168", + "scope_authority": "h-8d45c127", + "launch_count": 1, + "coordinator_pid_at_launch": 2549759, + "worker_exit": 1, + "original_guard_gpu_seconds": 23.69431395700667, + "worker_wall_seconds": 3.1360727309947833, + "device_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "recorded_peaks": { + "peak_allocated_bytes": 0, + "peak_reserved_bytes": 0 + }, + "heldout_targets_read": 0, + "learning_samples_read": 0, + "optimizer_steps": 0, + "measurement_result": null, + "host_terminal_observation": "MainPID=0\nExecMainStatus=1\nActiveState=failed\nSubState=failed\n PID PPID STAT ELAPSED COMMAND\nGPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c, 2512405, 820 MiB\nGPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36, 2517067, 1708 MiB\n", + "same_error_reproduced_using_mock_nvml_and_no_cuda": true, + "retry_started": false, + "team_report_slack": "1795072202.479989", + "evidence_sha256": { + "artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/service-launch.json": "1cadc651f508d314f2faf8d63bc4c2413f45d2f8943f545f4ca0e9bd1f4f3669", + "artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/last-pre-admission.json": "a9d2316b4094837c6ff8aea2642df03150a3790156fdb5b2e6adb938fbd8b868", + "artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/started.json": "1967c4e3d7f5423eb966b2a8f565edcd85524dfeb12010068d6f569e42909cf3", + "artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/idle-gate.json": "10f1a79315316a3160319226d293d3fedc4e2c5457664342ade8b8d0bd61b40f", + "artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/dispatch.json": "93cbd1525a0cae8a06771d750cdd22545bd8d4da586ec80df6e9d10fbb12327b", + "artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/gino.started.json": "8fdda2432fb032404af8d5c803a6e73989c7527a52d115c60fdba4878334159a", + "artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/gino.receipt.json": "d947dda1ef7ac76d13792bbd6c81a86194ab2eccb65296b2b2fd2647bacc3837", + "artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/gino.log": "4c0f619d6bcbb3402dbda364bb8031ef878c7370fb5e9f35401a10df9f31a674", + "artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/ledger.json": "8bc9b505e224bfed74d2c5005d713ee8b095c7b7a0293b239b3c84c748d8b141", + "artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/complete.json": "c983c05f9531119dad905d71c73399d788837299c44e4736b50a42262b4e9e6e", + "artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/uuid-parser-cpu-diagnostic.json": "07cc919a949f6a7995348ea3f33e8a022c98b7d5de372baed2114de4f02ba9d5" + } +} diff --git a/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/uuid-parser-cpu-diagnostic.json b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/uuid-parser-cpu-diagnostic.json new file mode 100644 index 000000000..d9b4975e3 --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/uuid-parser-cpu-diagnostic.json @@ -0,0 +1,36 @@ +{ + "status": "REPRODUCED_WITHOUT_CUDA", + "torch_version": "1.13.1+cu116", + "cuda_python_source": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/lib/python3.10/site-packages/torch/cuda/__init__.py", + "cuda_python_source_sha256": "2f359b138403461d8df4a824d887ad39786ca44757d309ee6946f95380957f18", + "results": { + "masked": { + "mask": "", + "parsed": [ + -1 + ], + "nvml_count_with_mock_four_devices": 0, + "properties_error": "Invalid device id" + }, + "uuid": { + "mask": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "parsed": [ + -1 + ], + "nvml_count_with_mock_four_devices": 0, + "properties_error": "Invalid device id" + }, + "ordinal_control": { + "mask": "2", + "parsed": [ + 2 + ], + "nvml_count_with_mock_four_devices": 1, + "properties_error": null + } + }, + "real_nvml_calls": 0, + "real_cuda_calls": 0, + "measurement_relaunched": false, + "recorded_mask": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76" +}