mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
exp run: Support hydra basic sweeper.
Requires `--queue` to be used alongside `--set-param`. Doesn't strictly require `hydra.config.enabled`. Add `get_hydra_sweeps` to `dvc/utils/hydra.py`. Uses hydra `BasicSweeper` to generate the different sweeps. `glob` syntax for sweeps is not supported in this P.R. Closes #8083 --- Using setup from https://hydra.cc/docs/tutorials/basic/your_first_app/composition/ : ```console $ dvc exp run -S "db=mysql,postgresql" -S "schema=warehouse,school" --queue Queueing with '{'params.yaml': ['db=mysql', 'schema=warehouse']}'. Queued experiment '5ab98b8' for future execution. Queueing with '{'params.yaml': ['db=mysql', 'schema=school']}'. Queued experiment '57c2fb6' for future execution. Queueing with '{'params.yaml': ['db=postgresql', 'schema=warehouse']}'. Queued experiment 'b9d6391' for future execution. Queueing with '{'params.yaml': ['db=postgresql', 'schema=school']}'. Queued experiment '145cd55' for future execution. ```
This commit is contained in:
@@ -2,6 +2,7 @@ import logging
|
||||
from typing import Dict, Iterable, Optional
|
||||
|
||||
from dvc.dependency.param import ParamsDependency
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
from dvc.repo import locked
|
||||
from dvc.ui import ui
|
||||
from dvc.utils.cli_parse import to_path_overrides
|
||||
@@ -27,6 +28,8 @@ def run(
|
||||
Returns a dict mapping new experiment SHAs to the results
|
||||
of `repro` for that experiment.
|
||||
"""
|
||||
from dvc.utils.hydra import to_hydra_overrides
|
||||
|
||||
if run_all:
|
||||
entries = list(repo.experiments.celery_queue.iter_queued())
|
||||
return repo.experiments.reproduce_celery(entries, jobs=jobs)
|
||||
@@ -42,19 +45,42 @@ def run(
|
||||
# Force `_update_params` even if `--set-param` was not used
|
||||
path_overrides[hydra_output_file] = []
|
||||
|
||||
if queue:
|
||||
if not kwargs.get("checkpoint_resume", None):
|
||||
kwargs["reset"] = True
|
||||
hydra_sweep = any(
|
||||
x.is_sweep_override()
|
||||
for param_file in path_overrides
|
||||
for x in to_hydra_overrides(path_overrides[param_file])
|
||||
)
|
||||
|
||||
if hydra_sweep and not queue:
|
||||
raise InvalidArgumentError(
|
||||
"Sweep overrides can't be used without `--queue`"
|
||||
)
|
||||
|
||||
if not queue:
|
||||
return repo.experiments.reproduce_one(
|
||||
targets=targets, params=path_overrides, tmp_dir=tmp_dir, **kwargs
|
||||
)
|
||||
|
||||
if hydra_sweep:
|
||||
from dvc.utils.hydra import get_hydra_sweeps
|
||||
|
||||
sweeps = get_hydra_sweeps(path_overrides)
|
||||
else:
|
||||
sweeps = [path_overrides]
|
||||
|
||||
if not kwargs.get("checkpoint_resume", None):
|
||||
kwargs["reset"] = True
|
||||
|
||||
for sweep_overrides in sweeps:
|
||||
queue_entry = repo.experiments.queue_one(
|
||||
repo.experiments.celery_queue,
|
||||
targets=targets,
|
||||
params=path_overrides,
|
||||
params=sweep_overrides,
|
||||
**kwargs,
|
||||
)
|
||||
if sweep_overrides:
|
||||
ui.write(f"Queueing with overrides '{sweep_overrides}'.")
|
||||
name = queue_entry.name or queue_entry.stash_rev[:7]
|
||||
ui.write(f"Queued experiment '{name}' for future execution.")
|
||||
return {}
|
||||
|
||||
return repo.experiments.reproduce_one(
|
||||
targets=targets, params=path_overrides, tmp_dir=tmp_dir, **kwargs
|
||||
)
|
||||
return {}
|
||||
|
||||
+35
-6
@@ -1,11 +1,13 @@
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, List
|
||||
|
||||
from hydra import compose, initialize_config_dir
|
||||
from hydra._internal.config_loader_impl import ConfigLoaderImpl
|
||||
from hydra._internal.core_plugins.basic_sweeper import BasicSweeper
|
||||
from hydra.core.override_parser.overrides_parser import OverridesParser
|
||||
from hydra.core.override_parser.types import ValueType
|
||||
from hydra.errors import ConfigCompositionException, OverrideParseException
|
||||
from hydra.types import RunMode
|
||||
from omegaconf import OmegaConf
|
||||
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
@@ -59,11 +61,7 @@ def apply_overrides(path: "StrPath", overrides: List[str]) -> None:
|
||||
modify_data = MODIFIERS[suffix]
|
||||
with modify_data(path) as original_data:
|
||||
try:
|
||||
parser = OverridesParser.create()
|
||||
parsed = parser.parse_overrides(overrides=overrides)
|
||||
ConfigLoaderImpl.validate_sweep_overrides_legal(
|
||||
parsed, run_mode=RunMode.RUN, from_shell=True
|
||||
)
|
||||
parsed = to_hydra_overrides(overrides)
|
||||
|
||||
new_data = OmegaConf.create(
|
||||
to_omegaconf(original_data),
|
||||
@@ -78,3 +76,34 @@ def apply_overrides(path: "StrPath", overrides: List[str]) -> None:
|
||||
|
||||
merge_dicts(original_data, new_data)
|
||||
remove_missing_keys(original_data, new_data)
|
||||
|
||||
|
||||
def to_hydra_overrides(path_overrides):
|
||||
parser = OverridesParser.create()
|
||||
return parser.parse_overrides(overrides=path_overrides)
|
||||
|
||||
|
||||
def get_hydra_sweeps(path_overrides):
|
||||
merged_overrides = []
|
||||
for path, overrides in path_overrides.items():
|
||||
merged_overrides.extend(
|
||||
[f"{path}___{override}" for override in overrides]
|
||||
)
|
||||
|
||||
hydra_overrides = to_hydra_overrides(merged_overrides)
|
||||
for hydra_override in hydra_overrides:
|
||||
if hydra_override.value_type == ValueType.GLOB_CHOICE_SWEEP:
|
||||
raise InvalidArgumentError(
|
||||
f"Glob override '{hydra_override.input_line}' "
|
||||
"is not supported."
|
||||
)
|
||||
|
||||
splits = BasicSweeper.split_arguments(hydra_overrides, None)[0]
|
||||
sweeps = []
|
||||
for split in splits:
|
||||
sweep_overrides = defaultdict(list)
|
||||
for merged_override in split:
|
||||
path, override = merged_override.split("___")
|
||||
sweep_overrides[path].append(override)
|
||||
sweeps.append(dict(sweep_overrides))
|
||||
return sweeps
|
||||
|
||||
@@ -2,6 +2,8 @@ import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
|
||||
from ..utils.test_hydra import hydra_setup
|
||||
|
||||
|
||||
@@ -85,3 +87,57 @@ def test_hydra_compose_and_dump(
|
||||
"goo": {"bag": 3.0},
|
||||
"lorem": False,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"hydra_enabled,overrides,expected",
|
||||
[
|
||||
(
|
||||
pytest.param(
|
||||
True,
|
||||
marks=pytest.mark.xfail(
|
||||
sys.version_info >= (3, 11), reason="unsupported on 3.11"
|
||||
),
|
||||
),
|
||||
["db=mysql,postgresql"],
|
||||
[
|
||||
{"params.yaml": ["db=mysql"]},
|
||||
{"params.yaml": ["db=postgresql"]},
|
||||
],
|
||||
),
|
||||
(
|
||||
False,
|
||||
["foo=bar,baz"],
|
||||
[{"params.yaml": ["foo=bar"]}, {"params.yaml": ["foo=baz"]}],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_hydra_sweep(
|
||||
tmp_dir, params_repo, dvc, mocker, hydra_enabled, overrides, expected
|
||||
):
|
||||
patched = mocker.patch.object(dvc.experiments, "queue_one")
|
||||
|
||||
if hydra_enabled:
|
||||
hydra_setup(
|
||||
tmp_dir,
|
||||
config_dir="conf",
|
||||
config_name="config",
|
||||
)
|
||||
with dvc.config.edit() as conf:
|
||||
conf["hydra"]["enabled"] = True
|
||||
|
||||
dvc.experiments.run(params=overrides, queue=True)
|
||||
|
||||
assert patched.call_count == len(expected)
|
||||
for e in expected:
|
||||
patched.assert_any_call(
|
||||
mocker.ANY,
|
||||
params=e,
|
||||
reset=True,
|
||||
targets=None,
|
||||
)
|
||||
|
||||
|
||||
def test_hydra_sweep_requires_queue(params_repo, dvc):
|
||||
with pytest.raises(InvalidArgumentError):
|
||||
dvc.experiments.run(params=["db=mysql,postgresql"])
|
||||
|
||||
@@ -178,3 +178,54 @@ def test_compose_and_dump(tmp_dir, suffix, overrides, expected):
|
||||
output_file = tmp_dir / f"params.{suffix}"
|
||||
compose_and_dump(output_file, config_dir, config_name, overrides)
|
||||
assert output_file.parse() == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides, expected",
|
||||
[
|
||||
(
|
||||
{"params.yaml": ["foo=1,2", "bar=3,4"]},
|
||||
[
|
||||
{"params.yaml": ["foo=1", "bar=3"]},
|
||||
{"params.yaml": ["foo=1", "bar=4"]},
|
||||
{"params.yaml": ["foo=2", "bar=3"]},
|
||||
{"params.yaml": ["foo=2", "bar=4"]},
|
||||
],
|
||||
),
|
||||
(
|
||||
{"params.yaml": ["foo=choice(1,2)"]},
|
||||
[{"params.yaml": ["foo=1"]}, {"params.yaml": ["foo=2"]}],
|
||||
),
|
||||
(
|
||||
{"params.yaml": ["foo=range(1, 3)"]},
|
||||
[{"params.yaml": ["foo=1"]}, {"params.yaml": ["foo=2"]}],
|
||||
),
|
||||
(
|
||||
{"params.yaml": ["foo=1,2"], "others.yaml": ["bar=3"]},
|
||||
[
|
||||
{"params.yaml": ["foo=1"], "others.yaml": ["bar=3"]},
|
||||
{"params.yaml": ["foo=2"], "others.yaml": ["bar=3"]},
|
||||
],
|
||||
),
|
||||
(
|
||||
{"params.yaml": ["foo=1,2"], "others.yaml": ["bar=3,4"]},
|
||||
[
|
||||
{"params.yaml": ["foo=1"], "others.yaml": ["bar=3"]},
|
||||
{"params.yaml": ["foo=1"], "others.yaml": ["bar=4"]},
|
||||
{"params.yaml": ["foo=2"], "others.yaml": ["bar=3"]},
|
||||
{"params.yaml": ["foo=2"], "others.yaml": ["bar=4"]},
|
||||
],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_hydra_sweeps(overrides, expected):
|
||||
from dvc.utils.hydra import get_hydra_sweeps
|
||||
|
||||
assert get_hydra_sweeps(overrides) == expected
|
||||
|
||||
|
||||
def test_invalid_sweep():
|
||||
from dvc.utils.hydra import get_hydra_sweeps
|
||||
|
||||
with pytest.raises(InvalidArgumentError):
|
||||
get_hydra_sweeps({"params.yaml": ["foo=glob(*)"]})
|
||||
|
||||
Reference in New Issue
Block a user