Ignore Runtime error

opendatahub-io · Dec 6, 2024 · 9a422b5 · 9a422b5
1 parent fcc8fc7
commit 9a422b5
Showing 1 changed file with 7 additions and 1 deletion.
diff --git a/tests/odh/resources/mnist_ray_mini.ipynb b/tests/odh/resources/mnist_ray_mini.ipynb
@@ -189,7 +189,13 @@
     "finished = False\n",
     "while not finished:\n",
     "    sleep(1)\n",
-    "    status = client.get_job_status(submission_id)\n",
+    "    try:\n",
+    "        status = client.get_job_status(submission_id)\n",
+    "    except RuntimeError:\n",
+    "    # At times, the ray dashboard displays a \"RuntimeError: Request failed with status code 504: <html><body><h1>504 Gateway Time-out</h1>\" \n",
+    "    # message, leading to a crashloopback error in the notebook pod. However, the ray job continues running and disregards the error. \n",
+    "    # Consider eliminating the try-except block when using the updated version of Ray 2.38.\n",
+    "        pass\n",
     "    finished = (status == \"SUCCEEDED\")\n",
     "if finished:\n",
     "    print(\"Job completed Successfully !\")\n",