diff --git a/dvc/repo/experiments/queue/base.py b/dvc/repo/experiments/queue/base.py index b7999f4f7..23d5ec45d 100644 --- a/dvc/repo/experiments/queue/base.py +++ b/dvc/repo/experiments/queue/base.py @@ -1,6 +1,5 @@ import logging import os -import sys from abc import ABC, abstractmethod from dataclasses import asdict, dataclass from typing import ( @@ -526,16 +525,9 @@ class BaseStashQueue(ABC): .. _Hydra Override: https://hydra.cc/docs/advanced/override_grammar/basic/ """ - logger.debug("Using experiment params '%s'", params) + from dvc.utils.hydra import apply_overrides, compose_and_dump - try: - from dvc.utils.hydra import apply_overrides, compose_and_dump - except ValueError: - if sys.version_info >= (3, 11): - raise DvcException( - "--set-param is not supported in Python >= 3.11" - ) - raise + logger.debug("Using experiment params '%s'", params) hydra_config = self.repo.config.get("hydra", {}) hydra_enabled = hydra_config.get("enabled", False) diff --git a/dvc/repo/experiments/run.py b/dvc/repo/experiments/run.py index 043f6cc51..af6a24c6c 100644 --- a/dvc/repo/experiments/run.py +++ b/dvc/repo/experiments/run.py @@ -1,9 +1,8 @@ import logging -import sys from typing import Dict, Iterable, Optional from dvc.dependency.param import ParamsDependency -from dvc.exceptions import DvcException, InvalidArgumentError +from dvc.exceptions import InvalidArgumentError from dvc.repo import locked from dvc.ui import ui from dvc.utils.cli_parse import to_path_overrides @@ -29,19 +28,24 @@ def run( Returns a dict mapping new experiment SHAs to the results of `repro` for that experiment. """ - try: - from dvc.utils.hydra import to_hydra_overrides - except ValueError: - if sys.version_info >= (3, 11): - raise DvcException("exp run is not supported in Python >= 3.11") - raise - if run_all: entries = list(repo.experiments.celery_queue.iter_queued()) return repo.experiments.reproduce_celery(entries, jobs=jobs) if params: + from dvc.utils.hydra import to_hydra_overrides + path_overrides = to_path_overrides(params) + hydra_sweep = any( + x.is_sweep_override() + for param_file in path_overrides + for x in to_hydra_overrides(path_overrides[param_file]) + ) + + if hydra_sweep and not queue: + raise InvalidArgumentError( + "Sweep overrides can't be used without `--queue`" + ) else: path_overrides = {} @@ -51,17 +55,6 @@ def run( # Force `_update_params` even if `--set-param` was not used path_overrides[hydra_output_file] = [] - hydra_sweep = any( - x.is_sweep_override() - for param_file in path_overrides - for x in to_hydra_overrides(path_overrides[param_file]) - ) - - if hydra_sweep and not queue: - raise InvalidArgumentError( - "Sweep overrides can't be used without `--queue`" - ) - if not queue: return repo.experiments.reproduce_one( targets=targets, params=path_overrides, tmp_dir=tmp_dir, **kwargs diff --git a/dvc/utils/hydra.py b/dvc/utils/hydra.py index 5ee3180d4..793d9664f 100644 --- a/dvc/utils/hydra.py +++ b/dvc/utils/hydra.py @@ -1,15 +1,24 @@ +import sys from pathlib import Path from typing import TYPE_CHECKING, List -from hydra import compose, initialize_config_dir -from hydra._internal.config_loader_impl import ConfigLoaderImpl -from hydra._internal.core_plugins.basic_sweeper import BasicSweeper -from hydra.core.override_parser.overrides_parser import OverridesParser -from hydra.core.override_parser.types import ValueType -from hydra.errors import ConfigCompositionException, OverrideParseException -from omegaconf import OmegaConf +try: + from hydra import compose, initialize_config_dir + from hydra._internal.config_loader_impl import ConfigLoaderImpl + from hydra._internal.core_plugins.basic_sweeper import BasicSweeper + from hydra.core.override_parser.overrides_parser import OverridesParser + from hydra.core.override_parser.types import ValueType + from hydra.errors import ConfigCompositionException, OverrideParseException + from omegaconf import OmegaConf -from dvc.exceptions import InvalidArgumentError + hydra_compatible = True +except ValueError: + if sys.version_info >= (3, 11): + hydra_compatible = False + else: + raise + +from dvc.exceptions import DvcException, InvalidArgumentError from .collections import merge_dicts, remove_missing_keys, to_omegaconf from .serialize import DUMPERS, MODIFIERS @@ -37,6 +46,11 @@ def compose_and_dump( .. _Hydra Override: https://hydra.cc/docs/advanced/override_grammar/basic/ """ + if not hydra_compatible: + raise DvcException( + "hydra functionality is not supported in Python >= 3.11" + ) + with initialize_config_dir(config_dir, version_base=None): cfg = compose(config_name=config_name, overrides=overrides) @@ -53,6 +67,11 @@ def apply_overrides(path: "StrPath", overrides: List[str]) -> None: .. _Hydra Override: https://hydra.cc/docs/next/advanced/override_grammar/basic/ """ + if not hydra_compatible: + raise DvcException( + "hydra functionality is not supported in Python >= 3.11" + ) + suffix = Path(path).suffix.lower() hydra_errors = (ConfigCompositionException, OverrideParseException) @@ -78,6 +97,11 @@ def apply_overrides(path: "StrPath", overrides: List[str]) -> None: def to_hydra_overrides(path_overrides): + if not hydra_compatible: + raise DvcException( + "hydra functionality is not supported in Python >= 3.11" + ) + parser = OverridesParser.create() return parser.parse_overrides(overrides=path_overrides) @@ -89,6 +113,11 @@ def dict_product(dicts): def get_hydra_sweeps(path_overrides): + if not hydra_compatible: + raise DvcException( + "hydra functionality is not supported in Python >= 3.11" + ) + path_sweeps = {} for path, overrides in path_overrides.items(): overrides = to_hydra_overrides(overrides)