From 3b532b7d59a272f143b4db5b9aef842bc6bf1642 Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Tue, 13 Sep 2022 12:32:51 +0200 Subject: [PATCH] exp run: Support hydra basic sweeper. Requires `--queue` to be used alongside `--set-param`. Doesn't strictly require `hydra.config.enabled`. Add `get_hydra_sweeps` to `dvc/utils/hydra.py`. Uses hydra `BasicSweeper` to generate the different sweeps. `glob` syntax for sweeps is not supported in this P.R. Closes #8083 --- Using setup from https://hydra.cc/docs/tutorials/basic/your_first_app/composition/ : ```console $ dvc exp run -S "db=mysql,postgresql" -S "schema=warehouse,school" --queue Queueing with '{'params.yaml': ['db=mysql', 'schema=warehouse']}'. Queued experiment '5ab98b8' for future execution. Queueing with '{'params.yaml': ['db=mysql', 'schema=school']}'. Queued experiment '57c2fb6' for future execution. Queueing with '{'params.yaml': ['db=postgresql', 'schema=warehouse']}'. Queued experiment 'b9d6391' for future execution. Queueing with '{'params.yaml': ['db=postgresql', 'schema=school']}'. Queued experiment '145cd55' for future execution. ``` --- dvc/repo/experiments/run.py | 42 +++++++++++++---- dvc/utils/hydra.py | 41 ++++++++++++++--- tests/func/experiments/test_set_params.py | 56 +++++++++++++++++++++++ tests/func/utils/test_hydra.py | 51 +++++++++++++++++++++ 4 files changed, 176 insertions(+), 14 deletions(-) diff --git a/dvc/repo/experiments/run.py b/dvc/repo/experiments/run.py index 5090ff4d9..668d0582f 100644 --- a/dvc/repo/experiments/run.py +++ b/dvc/repo/experiments/run.py @@ -2,6 +2,7 @@ import logging from typing import Dict, Iterable, Optional from dvc.dependency.param import ParamsDependency +from dvc.exceptions import InvalidArgumentError from dvc.repo import locked from dvc.ui import ui from dvc.utils.cli_parse import to_path_overrides @@ -27,6 +28,8 @@ def run( Returns a dict mapping new experiment SHAs to the results of `repro` for that experiment. """ + from dvc.utils.hydra import to_hydra_overrides + if run_all: entries = list(repo.experiments.celery_queue.iter_queued()) return repo.experiments.reproduce_celery(entries, jobs=jobs) @@ -42,19 +45,42 @@ def run( # Force `_update_params` even if `--set-param` was not used path_overrides[hydra_output_file] = [] - if queue: - if not kwargs.get("checkpoint_resume", None): - kwargs["reset"] = True + hydra_sweep = any( + x.is_sweep_override() + for param_file in path_overrides + for x in to_hydra_overrides(path_overrides[param_file]) + ) + + if hydra_sweep and not queue: + raise InvalidArgumentError( + "Sweep overrides can't be used without `--queue`" + ) + + if not queue: + return repo.experiments.reproduce_one( + targets=targets, params=path_overrides, tmp_dir=tmp_dir, **kwargs + ) + + if hydra_sweep: + from dvc.utils.hydra import get_hydra_sweeps + + sweeps = get_hydra_sweeps(path_overrides) + else: + sweeps = [path_overrides] + + if not kwargs.get("checkpoint_resume", None): + kwargs["reset"] = True + + for sweep_overrides in sweeps: queue_entry = repo.experiments.queue_one( repo.experiments.celery_queue, targets=targets, - params=path_overrides, + params=sweep_overrides, **kwargs, ) + if sweep_overrides: + ui.write(f"Queueing with overrides '{sweep_overrides}'.") name = queue_entry.name or queue_entry.stash_rev[:7] ui.write(f"Queued experiment '{name}' for future execution.") - return {} - return repo.experiments.reproduce_one( - targets=targets, params=path_overrides, tmp_dir=tmp_dir, **kwargs - ) + return {} diff --git a/dvc/utils/hydra.py b/dvc/utils/hydra.py index f06fb7c8f..281ef0c95 100644 --- a/dvc/utils/hydra.py +++ b/dvc/utils/hydra.py @@ -1,11 +1,13 @@ +from collections import defaultdict from pathlib import Path from typing import TYPE_CHECKING, List from hydra import compose, initialize_config_dir from hydra._internal.config_loader_impl import ConfigLoaderImpl +from hydra._internal.core_plugins.basic_sweeper import BasicSweeper from hydra.core.override_parser.overrides_parser import OverridesParser +from hydra.core.override_parser.types import ValueType from hydra.errors import ConfigCompositionException, OverrideParseException -from hydra.types import RunMode from omegaconf import OmegaConf from dvc.exceptions import InvalidArgumentError @@ -59,11 +61,7 @@ def apply_overrides(path: "StrPath", overrides: List[str]) -> None: modify_data = MODIFIERS[suffix] with modify_data(path) as original_data: try: - parser = OverridesParser.create() - parsed = parser.parse_overrides(overrides=overrides) - ConfigLoaderImpl.validate_sweep_overrides_legal( - parsed, run_mode=RunMode.RUN, from_shell=True - ) + parsed = to_hydra_overrides(overrides) new_data = OmegaConf.create( to_omegaconf(original_data), @@ -78,3 +76,34 @@ def apply_overrides(path: "StrPath", overrides: List[str]) -> None: merge_dicts(original_data, new_data) remove_missing_keys(original_data, new_data) + + +def to_hydra_overrides(path_overrides): + parser = OverridesParser.create() + return parser.parse_overrides(overrides=path_overrides) + + +def get_hydra_sweeps(path_overrides): + merged_overrides = [] + for path, overrides in path_overrides.items(): + merged_overrides.extend( + [f"{path}___{override}" for override in overrides] + ) + + hydra_overrides = to_hydra_overrides(merged_overrides) + for hydra_override in hydra_overrides: + if hydra_override.value_type == ValueType.GLOB_CHOICE_SWEEP: + raise InvalidArgumentError( + f"Glob override '{hydra_override.input_line}' " + "is not supported." + ) + + splits = BasicSweeper.split_arguments(hydra_overrides, None)[0] + sweeps = [] + for split in splits: + sweep_overrides = defaultdict(list) + for merged_override in split: + path, override = merged_override.split("___") + sweep_overrides[path].append(override) + sweeps.append(dict(sweep_overrides)) + return sweeps diff --git a/tests/func/experiments/test_set_params.py b/tests/func/experiments/test_set_params.py index 77e3197dc..5be97527d 100644 --- a/tests/func/experiments/test_set_params.py +++ b/tests/func/experiments/test_set_params.py @@ -2,6 +2,8 @@ import sys import pytest +from dvc.exceptions import InvalidArgumentError + from ..utils.test_hydra import hydra_setup @@ -85,3 +87,57 @@ def test_hydra_compose_and_dump( "goo": {"bag": 3.0}, "lorem": False, } + + +@pytest.mark.parametrize( + "hydra_enabled,overrides,expected", + [ + ( + pytest.param( + True, + marks=pytest.mark.xfail( + sys.version_info >= (3, 11), reason="unsupported on 3.11" + ), + ), + ["db=mysql,postgresql"], + [ + {"params.yaml": ["db=mysql"]}, + {"params.yaml": ["db=postgresql"]}, + ], + ), + ( + False, + ["foo=bar,baz"], + [{"params.yaml": ["foo=bar"]}, {"params.yaml": ["foo=baz"]}], + ), + ], +) +def test_hydra_sweep( + tmp_dir, params_repo, dvc, mocker, hydra_enabled, overrides, expected +): + patched = mocker.patch.object(dvc.experiments, "queue_one") + + if hydra_enabled: + hydra_setup( + tmp_dir, + config_dir="conf", + config_name="config", + ) + with dvc.config.edit() as conf: + conf["hydra"]["enabled"] = True + + dvc.experiments.run(params=overrides, queue=True) + + assert patched.call_count == len(expected) + for e in expected: + patched.assert_any_call( + mocker.ANY, + params=e, + reset=True, + targets=None, + ) + + +def test_hydra_sweep_requires_queue(params_repo, dvc): + with pytest.raises(InvalidArgumentError): + dvc.experiments.run(params=["db=mysql,postgresql"]) diff --git a/tests/func/utils/test_hydra.py b/tests/func/utils/test_hydra.py index 2afb45cae..f7942fa84 100644 --- a/tests/func/utils/test_hydra.py +++ b/tests/func/utils/test_hydra.py @@ -178,3 +178,54 @@ def test_compose_and_dump(tmp_dir, suffix, overrides, expected): output_file = tmp_dir / f"params.{suffix}" compose_and_dump(output_file, config_dir, config_name, overrides) assert output_file.parse() == expected + + +@pytest.mark.parametrize( + "overrides, expected", + [ + ( + {"params.yaml": ["foo=1,2", "bar=3,4"]}, + [ + {"params.yaml": ["foo=1", "bar=3"]}, + {"params.yaml": ["foo=1", "bar=4"]}, + {"params.yaml": ["foo=2", "bar=3"]}, + {"params.yaml": ["foo=2", "bar=4"]}, + ], + ), + ( + {"params.yaml": ["foo=choice(1,2)"]}, + [{"params.yaml": ["foo=1"]}, {"params.yaml": ["foo=2"]}], + ), + ( + {"params.yaml": ["foo=range(1, 3)"]}, + [{"params.yaml": ["foo=1"]}, {"params.yaml": ["foo=2"]}], + ), + ( + {"params.yaml": ["foo=1,2"], "others.yaml": ["bar=3"]}, + [ + {"params.yaml": ["foo=1"], "others.yaml": ["bar=3"]}, + {"params.yaml": ["foo=2"], "others.yaml": ["bar=3"]}, + ], + ), + ( + {"params.yaml": ["foo=1,2"], "others.yaml": ["bar=3,4"]}, + [ + {"params.yaml": ["foo=1"], "others.yaml": ["bar=3"]}, + {"params.yaml": ["foo=1"], "others.yaml": ["bar=4"]}, + {"params.yaml": ["foo=2"], "others.yaml": ["bar=3"]}, + {"params.yaml": ["foo=2"], "others.yaml": ["bar=4"]}, + ], + ), + ], +) +def test_hydra_sweeps(overrides, expected): + from dvc.utils.hydra import get_hydra_sweeps + + assert get_hydra_sweeps(overrides) == expected + + +def test_invalid_sweep(): + from dvc.utils.hydra import get_hydra_sweeps + + with pytest.raises(InvalidArgumentError): + get_hydra_sweeps({"params.yaml": ["foo=glob(*)"]})