exp run: Support hydra basic sweeper.

Requires `--queue` to be used alongside `--set-param`.

Doesn't strictly require `hydra.config.enabled`.

Add `get_hydra_sweeps` to `dvc/utils/hydra.py`.  Uses hydra `BasicSweeper` to generate the different sweeps.

`glob` syntax for sweeps is not supported in this P.R.

Closes #8083

---

Using setup from https://hydra.cc/docs/tutorials/basic/your_first_app/composition/ :

```console
$ dvc exp run -S "db=mysql,postgresql" -S "schema=warehouse,school" --queue
Queueing with '{'params.yaml': ['db=mysql', 'schema=warehouse']}'.
Queued experiment '5ab98b8' for future execution.
Queueing with '{'params.yaml': ['db=mysql', 'schema=school']}'.
Queued experiment '57c2fb6' for future execution.
Queueing with '{'params.yaml': ['db=postgresql', 'schema=warehouse']}'.
Queued experiment 'b9d6391' for future execution.
Queueing with '{'params.yaml': ['db=postgresql', 'schema=school']}'.
Queued experiment '145cd55' for future execution.
```
This commit is contained in:
David de la Iglesia Castro
2022-09-13 13:43:30 +02:00
parent e78c4455de
commit 3b532b7d59
4 changed files with 176 additions and 14 deletions
+34 -8
View File
@@ -2,6 +2,7 @@ import logging
from typing import Dict, Iterable, Optional
from dvc.dependency.param import ParamsDependency
from dvc.exceptions import InvalidArgumentError
from dvc.repo import locked
from dvc.ui import ui
from dvc.utils.cli_parse import to_path_overrides
@@ -27,6 +28,8 @@ def run(
Returns a dict mapping new experiment SHAs to the results
of `repro` for that experiment.
"""
from dvc.utils.hydra import to_hydra_overrides
if run_all:
entries = list(repo.experiments.celery_queue.iter_queued())
return repo.experiments.reproduce_celery(entries, jobs=jobs)
@@ -42,19 +45,42 @@ def run(
# Force `_update_params` even if `--set-param` was not used
path_overrides[hydra_output_file] = []
if queue:
if not kwargs.get("checkpoint_resume", None):
kwargs["reset"] = True
hydra_sweep = any(
x.is_sweep_override()
for param_file in path_overrides
for x in to_hydra_overrides(path_overrides[param_file])
)
if hydra_sweep and not queue:
raise InvalidArgumentError(
"Sweep overrides can't be used without `--queue`"
)
if not queue:
return repo.experiments.reproduce_one(
targets=targets, params=path_overrides, tmp_dir=tmp_dir, **kwargs
)
if hydra_sweep:
from dvc.utils.hydra import get_hydra_sweeps
sweeps = get_hydra_sweeps(path_overrides)
else:
sweeps = [path_overrides]
if not kwargs.get("checkpoint_resume", None):
kwargs["reset"] = True
for sweep_overrides in sweeps:
queue_entry = repo.experiments.queue_one(
repo.experiments.celery_queue,
targets=targets,
params=path_overrides,
params=sweep_overrides,
**kwargs,
)
if sweep_overrides:
ui.write(f"Queueing with overrides '{sweep_overrides}'.")
name = queue_entry.name or queue_entry.stash_rev[:7]
ui.write(f"Queued experiment '{name}' for future execution.")
return {}
return repo.experiments.reproduce_one(
targets=targets, params=path_overrides, tmp_dir=tmp_dir, **kwargs
)
return {}
+35 -6
View File
@@ -1,11 +1,13 @@
from collections import defaultdict
from pathlib import Path
from typing import TYPE_CHECKING, List
from hydra import compose, initialize_config_dir
from hydra._internal.config_loader_impl import ConfigLoaderImpl
from hydra._internal.core_plugins.basic_sweeper import BasicSweeper
from hydra.core.override_parser.overrides_parser import OverridesParser
from hydra.core.override_parser.types import ValueType
from hydra.errors import ConfigCompositionException, OverrideParseException
from hydra.types import RunMode
from omegaconf import OmegaConf
from dvc.exceptions import InvalidArgumentError
@@ -59,11 +61,7 @@ def apply_overrides(path: "StrPath", overrides: List[str]) -> None:
modify_data = MODIFIERS[suffix]
with modify_data(path) as original_data:
try:
parser = OverridesParser.create()
parsed = parser.parse_overrides(overrides=overrides)
ConfigLoaderImpl.validate_sweep_overrides_legal(
parsed, run_mode=RunMode.RUN, from_shell=True
)
parsed = to_hydra_overrides(overrides)
new_data = OmegaConf.create(
to_omegaconf(original_data),
@@ -78,3 +76,34 @@ def apply_overrides(path: "StrPath", overrides: List[str]) -> None:
merge_dicts(original_data, new_data)
remove_missing_keys(original_data, new_data)
def to_hydra_overrides(path_overrides):
parser = OverridesParser.create()
return parser.parse_overrides(overrides=path_overrides)
def get_hydra_sweeps(path_overrides):
merged_overrides = []
for path, overrides in path_overrides.items():
merged_overrides.extend(
[f"{path}___{override}" for override in overrides]
)
hydra_overrides = to_hydra_overrides(merged_overrides)
for hydra_override in hydra_overrides:
if hydra_override.value_type == ValueType.GLOB_CHOICE_SWEEP:
raise InvalidArgumentError(
f"Glob override '{hydra_override.input_line}' "
"is not supported."
)
splits = BasicSweeper.split_arguments(hydra_overrides, None)[0]
sweeps = []
for split in splits:
sweep_overrides = defaultdict(list)
for merged_override in split:
path, override = merged_override.split("___")
sweep_overrides[path].append(override)
sweeps.append(dict(sweep_overrides))
return sweeps
+56
View File
@@ -2,6 +2,8 @@ import sys
import pytest
from dvc.exceptions import InvalidArgumentError
from ..utils.test_hydra import hydra_setup
@@ -85,3 +87,57 @@ def test_hydra_compose_and_dump(
"goo": {"bag": 3.0},
"lorem": False,
}
@pytest.mark.parametrize(
"hydra_enabled,overrides,expected",
[
(
pytest.param(
True,
marks=pytest.mark.xfail(
sys.version_info >= (3, 11), reason="unsupported on 3.11"
),
),
["db=mysql,postgresql"],
[
{"params.yaml": ["db=mysql"]},
{"params.yaml": ["db=postgresql"]},
],
),
(
False,
["foo=bar,baz"],
[{"params.yaml": ["foo=bar"]}, {"params.yaml": ["foo=baz"]}],
),
],
)
def test_hydra_sweep(
tmp_dir, params_repo, dvc, mocker, hydra_enabled, overrides, expected
):
patched = mocker.patch.object(dvc.experiments, "queue_one")
if hydra_enabled:
hydra_setup(
tmp_dir,
config_dir="conf",
config_name="config",
)
with dvc.config.edit() as conf:
conf["hydra"]["enabled"] = True
dvc.experiments.run(params=overrides, queue=True)
assert patched.call_count == len(expected)
for e in expected:
patched.assert_any_call(
mocker.ANY,
params=e,
reset=True,
targets=None,
)
def test_hydra_sweep_requires_queue(params_repo, dvc):
with pytest.raises(InvalidArgumentError):
dvc.experiments.run(params=["db=mysql,postgresql"])
+51
View File
@@ -178,3 +178,54 @@ def test_compose_and_dump(tmp_dir, suffix, overrides, expected):
output_file = tmp_dir / f"params.{suffix}"
compose_and_dump(output_file, config_dir, config_name, overrides)
assert output_file.parse() == expected
@pytest.mark.parametrize(
"overrides, expected",
[
(
{"params.yaml": ["foo=1,2", "bar=3,4"]},
[
{"params.yaml": ["foo=1", "bar=3"]},
{"params.yaml": ["foo=1", "bar=4"]},
{"params.yaml": ["foo=2", "bar=3"]},
{"params.yaml": ["foo=2", "bar=4"]},
],
),
(
{"params.yaml": ["foo=choice(1,2)"]},
[{"params.yaml": ["foo=1"]}, {"params.yaml": ["foo=2"]}],
),
(
{"params.yaml": ["foo=range(1, 3)"]},
[{"params.yaml": ["foo=1"]}, {"params.yaml": ["foo=2"]}],
),
(
{"params.yaml": ["foo=1,2"], "others.yaml": ["bar=3"]},
[
{"params.yaml": ["foo=1"], "others.yaml": ["bar=3"]},
{"params.yaml": ["foo=2"], "others.yaml": ["bar=3"]},
],
),
(
{"params.yaml": ["foo=1,2"], "others.yaml": ["bar=3,4"]},
[
{"params.yaml": ["foo=1"], "others.yaml": ["bar=3"]},
{"params.yaml": ["foo=1"], "others.yaml": ["bar=4"]},
{"params.yaml": ["foo=2"], "others.yaml": ["bar=3"]},
{"params.yaml": ["foo=2"], "others.yaml": ["bar=4"]},
],
),
],
)
def test_hydra_sweeps(overrides, expected):
from dvc.utils.hydra import get_hydra_sweeps
assert get_hydra_sweeps(overrides) == expected
def test_invalid_sweep():
from dvc.utils.hydra import get_hydra_sweeps
with pytest.raises(InvalidArgumentError):
get_hydra_sweeps({"params.yaml": ["foo=glob(*)"]})