16 lines
841 B
JSON
16 lines
841 B
JSON
{
|
|
"status": "FAILED_NO_RETRY",
|
|
"error_type": "FloatingPointError",
|
|
"message": "AMP skipped steps exceed 1 percent after epoch 0",
|
|
"traceback": "Traceback (most recent call last):\n File \"/home/frank14f/optane/MeltMind/tools/run_s2_pilot_training_v2.py\", line 113, in run\n skip_policy.finish_epoch(epoch)\n File \"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py\", line 52, in finish_epoch\n if epoch>0 and self.skipped>self.attempted*.01:raise FloatingPointError('AMP skipped steps exceed 1 percent after epoch 0')\nFloatingPointError: AMP skipped steps exceed 1 percent after epoch 0\n",
|
|
"wall_seconds": 93.35228163201828,
|
|
"epoch": 2,
|
|
"amp": {
|
|
"attempted_steps": 382,
|
|
"skipped_steps": 4,
|
|
"successful_updates": 378,
|
|
"skipped_fraction": 0.010471204188481676,
|
|
"consecutive_skipped": 0
|
|
}
|
|
}
|