test_exact_device_memory_units (__main__.GPUUsageTests) ... ok
test_explicit_visibility_reaches_worker_and_receipt_without_gpu_use (__main__.GPUUsageTests) ... ok
test_missing_or_ambiguous_counters_fail_closed (__main__.GPUUsageTests) ... ok
test_vram_monitor_error_and_whole_host_rss_stop_without_second_run (__main__.GPUUsageTests) ... {"status": "PILOT_INCOMPLETE", "reason": "PILOT_RUN_FAILED_NO_RETRY", "runs": [{"run": {"name": "N128", "arm": "N", "width": 128, "seed": 0}, "status": "STOPPED_OR_FAILED", "stop_reason": "DEVICE_VRAM_LIMIT", "gpu_visible": true, "gpu_uuid": "synthetic-device", "device_monitor": {"peak_device_memory_bytes": 17179869184, "peak_whole_run_tree_rss_bytes": 0, "max_poll_gap_seconds": 2.6650377549231052e-06, "reason": "DEVICE_VRAM_LIMIT", "samples": 1}, "total_run_wall_seconds": 0.00118093698984012, "result_sha256": null}], "selected_global_width": null, "selection_rule": "mean of N/E validation error at each width, ties smaller; all four runs required", "pilot_wall_seconds": 0.0018433370278216898, "test_labels_read": false, "test_predictions": false, "retries": 0}
/home/frank14f/optane/MeltMind/tools/test_s2_pilot_gpu_guard.py:42: ResourceWarning: unclosed file <_io.TextIOWrapper name='/tmp/tmp8t5zwxfd/pilot.lock' mode='a' encoding='UTF-8'>
  guard.run(root)
ResourceWarning: Enable tracemalloc to get the object allocation traceback
{"status": "PILOT_INCOMPLETE", "reason": "PILOT_RUN_FAILED_NO_RETRY", "runs": [{"run": {"name": "N128", "arm": "N", "width": 128, "seed": 0}, "status": "STOPPED_OR_FAILED", "stop_reason": "GPU_MONITOR_FAILED", "gpu_visible": true, "gpu_uuid": "synthetic-device", "device_monitor": {"peak_device_memory_bytes": 5242880, "peak_whole_run_tree_rss_bytes": 0, "max_poll_gap_seconds": 1.6629928722977638e-06, "reason": "GPU_MONITOR_FAILED", "samples": 0, "error": "synthetic monitor failure"}, "total_run_wall_seconds": 0.0004254329833202064, "result_sha256": null}], "selected_global_width": null, "selection_rule": "mean of N/E validation error at each width, ties smaller; all four runs required", "pilot_wall_seconds": 0.0009375379886478186, "test_labels_read": false, "test_predictions": false, "retries": 0}
/home/frank14f/optane/MeltMind/tools/test_s2_pilot_gpu_guard.py:42: ResourceWarning: unclosed file <_io.TextIOWrapper name='/tmp/tmpackzyjel/pilot.lock' mode='a' encoding='UTF-8'>
  guard.run(root)
ResourceWarning: Enable tracemalloc to get the object allocation traceback
{"status": "PILOT_INCOMPLETE", "reason": "PILOT_RUN_FAILED_NO_RETRY", "runs": [{"run": {"name": "N128", "arm": "N", "width": 128, "seed": 0}, "status": "STOPPED_OR_FAILED", "stop_reason": "WHOLE_RUN_HOST_RSS_LIMIT", "gpu_visible": true, "gpu_uuid": "synthetic-device", "device_monitor": {"peak_device_memory_bytes": 5242880, "peak_whole_run_tree_rss_bytes": 18253611008, "max_poll_gap_seconds": 1.4830147847533226e-06, "reason": "WHOLE_RUN_HOST_RSS_LIMIT", "samples": 1}, "total_run_wall_seconds": 0.0007409480167552829, "result_sha256": null}], "selected_global_width": null, "selection_rule": "mean of N/E validation error at each width, ties smaller; all four runs required", "pilot_wall_seconds": 0.0012677110498771071, "test_labels_read": false, "test_predictions": false, "retries": 0}
/home/frank14f/optane/MeltMind/tools/test_s2_pilot_gpu_guard.py:42: ResourceWarning: unclosed file <_io.TextIOWrapper name='/tmp/tmpeb4lk0qx/pilot.lock' mode='a' encoding='UTF-8'>
  guard.run(root)
ResourceWarning: Enable tracemalloc to get the object allocation traceback
ok

----------------------------------------------------------------------
Ran 4 tests in 0.039s

OK
