Change conda env validity checking to activate and run Python

python/ray/_private/runtime_env/conda.py

@@ -18,8 +18,7 @@
     create_conda_env_if_needed,
     delete_conda_env,
     get_conda_activate_commands,
-    get_conda_info_json,
-    get_conda_envs,
+    check_if_conda_environment_valid,
 )
 from ray._private.runtime_env.context import RuntimeEnvContext
 from ray._private.runtime_env.packaging import Protocol, parse_uri
@@ -343,15 +342,13 @@ def _create():
                 if result in self._validated_named_conda_env:
                     return 0
 
-                conda_info = get_conda_info_json()
-                envs = get_conda_envs(conda_info)
-
                 # We accept `result` as a conda name or full path.
-                if not any(result == env[0] or result == env[1] for env in envs):
+                if not check_if_conda_environment_valid(result):
                     raise ValueError(
                         f"The given conda environment '{result}' "
-                        f"from the runtime env {runtime_env} doesn't "
-                        "exist from the output of `conda info --json`. "
+                        f"from the runtime env {runtime_env} can't "
+                        f"be activated with conda activate {result} 1>&2 "
+                        "&& python --version"
                         "You can only specify an env that already exists. "
                         f"Please make sure to create an env {result} "
                     )

python/ray/_private/runtime_env/conda_utils.py

@@ -4,7 +4,7 @@
 import subprocess
 import hashlib
 import json
-from typing import Optional, List, Union, Tuple
+from typing import Optional, List, Tuple
 
 """Utilities for conda.  Adapted from https://github.com/mlflow/mlflow."""
 
@@ -213,8 +213,11 @@ class ShellCommandException(Exception):
 
 
 def exec_cmd(
-    cmd: List[str], throw_on_error: bool = True, logger: Optional[logging.Logger] = None
-) -> Union[int, Tuple[int, str, str]]:
+    cmd: List[str],
+    throw_on_error: bool = True,
+    shell: bool = False,
+    logger: Optional[logging.Logger] = None,
+) -> Tuple[int, str, str]:
     """
     Runs a command as a child process.
 
@@ -234,6 +237,7 @@ def exec_cmd(
         stdin=subprocess.PIPE,
         stderr=subprocess.PIPE,
         universal_newlines=True,
+        shell=shell,
     )
     (stdout, stderr) = child.communicate()
     exit_code = child.wait()
@@ -276,3 +280,15 @@ def exec_cmd_stream_to_logger(
 
     exit_code = child.wait()
     return exit_code, "\n".join(last_n_lines)
+
+
+def check_if_conda_environment_valid(conda_env_name: str) -> bool:
+    """
+    Check if the given conda environment exists and is valid.
+    We try to `conda activate` it and check that python is runnable
+    """
+    activate_cmd = get_conda_activate_commands(conda_env_name) + ["python", "--version"]
+    exit_code, _, _ = exec_cmd(
+        [" ".join(activate_cmd)], throw_on_error=False, shell=True
+    )
+    return exit_code == 0

python/ray/tests/test_runtime_env_conda_and_pip.py

@@ -225,9 +225,7 @@ def f():
     for ref in refs:
         with pytest.raises(ray.exceptions.RuntimeEnvSetupError) as exc_info:
             ray.get(ref)
-        assert "doesn't exist from the output of `conda info --json`" in str(
-            exc_info.value
-        )  # noqa
+        assert "can't be activated with" in str(exc_info.value)  # noqa
 
 
 def test_get_requirements_file():