Files
MeltMind/artifacts/s2-pilot-v2-continuation-2026-11-15/N256-seed0/failure.json
T

16 lines
841 B
JSON

{
"status": "FAILED_NO_RETRY",
"error_type": "FloatingPointError",
"message": "AMP skipped steps exceed 1 percent after epoch 0",
"traceback": "Traceback (most recent call last):\n File \"/home/frank14f/optane/MeltMind/tools/run_s2_pilot_training_v2.py\", line 113, in run\n skip_policy.finish_epoch(epoch)\n File \"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py\", line 52, in finish_epoch\n if epoch>0 and self.skipped>self.attempted*.01:raise FloatingPointError('AMP skipped steps exceed 1 percent after epoch 0')\nFloatingPointError: AMP skipped steps exceed 1 percent after epoch 0\n",
"wall_seconds": 93.35228163201828,
"epoch": 2,
"amp": {
"attempted_steps": 382,
"skipped_steps": 4,
"successful_updates": 378,
"skipped_fraction": 0.010471204188481676,
"consecutive_skipped": 0
}
}