test_exact_device_memory_units (__main__.GPUUsageTests) ... ok
test_explicit_visibility_reaches_worker_and_receipt_without_gpu_use (__main__.GPUUsageTests) ... ok
test_missing_or_ambiguous_counters_fail_closed (__main__.GPUUsageTests) ... ok
test_vram_monitor_error_and_whole_host_rss_stop_without_second_run (__main__.GPUUsageTests) ... {"status": "PILOT_INCOMPLETE", "reason": "PILOT_RUN_FAILED_NO_RETRY", "runs": [{"run": {"name": "N128", "arm": "N", "width": 128, "seed": 0}, "status": "STOPPED_OR_FAILED", "stop_reason": "DEVICE_VRAM_LIMIT", "gpu_visible": true, "gpu_uuid": "synthetic-device", "device_monitor": {"peak_device_memory_bytes": 17179869184, "peak_whole_run_tree_rss_bytes": 0, "max_poll_gap_seconds": 2.435001078993082e-06, "reason": "DEVICE_VRAM_LIMIT", "samples": 1}, "total_run_wall_seconds": 0.0010300629655830562, "result_sha256": null}], "selected_global_width": null, "selection_rule": "mean of N/E validation error at each width, ties smaller; all four runs required", "pilot_wall_seconds": 0.0018631540006026626, "test_labels_read": false, "test_predictions": false, "retries": 0}
{"status": "PILOT_INCOMPLETE", "reason": "PILOT_RUN_FAILED_NO_RETRY", "runs": [{"run": {"name": "N128", "arm": "N", "width": 128, "seed": 0}, "status": "STOPPED_OR_FAILED", "stop_reason": "GPU_MONITOR_FAILED", "gpu_visible": true, "gpu_uuid": "synthetic-device", "device_monitor": {"peak_device_memory_bytes": 5242880, "peak_whole_run_tree_rss_bytes": 0, "max_poll_gap_seconds": 1.4629913493990898e-06, "reason": "GPU_MONITOR_FAILED", "samples": 0, "error": "synthetic monitor failure"}, "total_run_wall_seconds": 0.00040111603448167443, "result_sha256": null}], "selected_global_width": null, "selection_rule": "mean of N/E validation error at each width, ties smaller; all four runs required", "pilot_wall_seconds": 0.000856155005749315, "test_labels_read": false, "test_predictions": false, "retries": 0}
{"status": "PILOT_INCOMPLETE", "reason": "PILOT_RUN_FAILED_NO_RETRY", "runs": [{"run": {"name": "N128", "arm": "N", "width": 128, "seed": 0}, "status": "STOPPED_OR_FAILED", "stop_reason": "WHOLE_RUN_HOST_RSS_LIMIT", "gpu_visible": true, "gpu_uuid": "synthetic-device", "device_monitor": {"peak_device_memory_bytes": 5242880, "peak_whole_run_tree_rss_bytes": 18253611008, "max_poll_gap_seconds": 1.4929682947695255e-06, "reason": "WHOLE_RUN_HOST_RSS_LIMIT", "samples": 1}, "total_run_wall_seconds": 0.0006760350079275668, "result_sha256": null}], "selected_global_width": null, "selection_rule": "mean of N/E validation error at each width, ties smaller; all four runs required", "pilot_wall_seconds": 0.0011414529872126877, "test_labels_read": false, "test_predictions": false, "retries": 0}
ok

----------------------------------------------------------------------
Ran 4 tests in 0.036s

OK
