Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

FEAT-#7459: Add methods to get and set backend. #7460

Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Respond to comments
Signed-off-by: sfc-gh-mvashishtha <mahesh.vashishtha@snowflake.com>
  • Loading branch information
sfc-gh-mvashishtha committed Mar 11, 2025
commit c605ffdfe136e1df6d5bd7976afc6a448107665e
38 changes: 27 additions & 11 deletions modin/config/envvars.py
Original file line number Diff line number Diff line change
@@ -35,6 +35,22 @@
)


class DisallowExecutionAndBackendInEnvironmentMixin:
"""Mixin to disallow setting both execution and backend in environment."""

@classmethod
@doc(Parameter._get_value_from_config.__doc__)
def _get_value_from_config(cls) -> str:
if Backend.varname in os.environ and (
Engine.varname in os.environ or StorageFormat.varname in os.environ
):
# Handling this case is tricky, in part because the combination of
# Backend and Engine/StorageFormat may be invalid. For now just
# disallow it.
raise Exception("Can't specify both execution and backend in environment")
return Parameter._get_value_from_config()


class EnvironmentVariable(Parameter, type=str, abstract=True):
"""Base class for environment variables-based configuration."""

@@ -56,6 +72,8 @@ def _get_value_from_config(cls) -> Any:
return _UNSET
raw = os.environ[cls.varname]
if not _TYPE_PARAMS[cls.type].verify(raw):
# TODO: use and test a better error message, like "Invalid value
# for {cls.varname}: {raw}"
raise ValueError(f"Unsupported raw value: {raw}")
return _TYPE_PARAMS[cls.type].decode(raw)

@@ -170,7 +188,9 @@ class IsDebug(EnvironmentVariable, type=bool):
varname = "MODIN_DEBUG"


class Engine(EnvironmentVariable, type=str):
class Engine(
DisallowExecutionAndBackendInEnvironmentMixin, EnvironmentVariable, type=str
):
"""Distribution engine to run queries by."""

varname = "MODIN_ENGINE"
@@ -307,7 +327,9 @@ def get(cls) -> str:
return cls._value


class StorageFormat(EnvironmentVariable, type=str):
class StorageFormat(
DisallowExecutionAndBackendInEnvironmentMixin, EnvironmentVariable, type=str
):
"""Engine to run on a single node of distribution."""

@classmethod
@@ -373,7 +395,9 @@ def get(cls) -> str:
Execution = namedtuple("Execution", ["storage_format", "engine"])


class Backend(EnvironmentVariable, type=str):
class Backend(
DisallowExecutionAndBackendInEnvironmentMixin, EnvironmentVariable, type=str
):
"""
An alias for execution, i.e. the combination of StorageFormat and Engine.

@@ -1264,13 +1288,5 @@ def _check_vars() -> None:
FutureWarning,
)

if Backend.varname in os.environ and (
Engine.varname in os.environ or StorageFormat.varname in os.environ
):
# Handling this case is tricky, in part because the combination of
# Backend and Engine/StorageFormat may be invalid. For now just
# disallow it.
raise Exception("Can't specify both execution and backend in environment")


_check_vars()
7 changes: 6 additions & 1 deletion modin/tests/config/test_envvars.py
Original file line number Diff line number Diff line change
@@ -750,20 +750,25 @@ def test_environment_not_set_and_pick_up_default_engine(
"execution_variable, value",
[(cfg.Engine, "Python"), (cfg.StorageFormat, "Pandas")],
)
@pytest.mark.parametrize(
"variable_to_get",
[cfg.Backend, cfg.Engine, cfg.StorageFormat],
)
def test_conflicting_execution_and_backend_in_environment(
self,
monkeypatch,
clear_backend_execution_and_storage_format,
execution_variable,
value,
variable_to_get,
):
monkeypatch.setitem(os.environ, cfg.Backend.varname, "Ray")
monkeypatch.setitem(os.environ, execution_variable.varname, value)
with pytest.raises(
Exception,
match=re.escape("Can't specify both execution and backend in environment"),
):
_check_vars()
variable_to_get.get()

def test_get_execution_for_unknown_backend(self):
with pytest.raises(
Loading