Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-09 23:10:24 +01:00
Code Issues Packages Projects Releases Wiki Activity

tests: cover exp init

Browse Source
This commit is contained in:
Saugat Pachhai (सौगात)
2021-11-19 18:58:23 +05:45
committed by Saugat Pachhai
parent a70b3087b4
commit cb941a836d
5 changed files with 624 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/repo/experiments/init.py
+35 -22
View File
@@ -9,6 +9,8 @@ from typing import (
Dict,
Iterable,
List,
Optional,
TextIO,
Tuple,
Union,
cast,
@@ -49,6 +51,7 @@ def _prompts(
defaults: Dict[str, str] = None,
validator: Callable[[str, str], Union[str, Tuple[str, str]]] = None,
allow_omission: bool = True,
stream: Optional[TextIO] = None,
) -> Dict[str, OptStr]:
from dvc.ui.prompt import Prompt
@@ -60,6 +63,7 @@ def _prompts(
default=defaults.get(key),
validator=partial(validator, key) if validator else None,
allow_omission=allow_omission,
stream=stream,
)
for key in keys
}
@@ -99,8 +103,8 @@ def init_interactive(
defaults: Dict[str, str],
provided: Dict[str, str],
validator: Callable[[str, str], Union[str, Tuple[str, str]]] = None,
show_tree: bool = False,
live: bool = False,
stream: Optional[TextIO] = None,
) -> Dict[str, str]:
command = provided.pop("cmd", None)
primary = lremove(provided.keys(), ["code", "data", "models", "params"])
@@ -131,18 +135,27 @@ def init_interactive(
)
if not command:
ret.update(compact(_prompts(["cmd"], allow_omission=False)))
ret.update(
compact(_prompts(["cmd"], allow_omission=False, stream=stream))
)
if prompts:
ui.error_write()
ui.error_write(styled=True)
if not prompts:
return ret
ui.write("Enter the paths for dependencies and outputs of the command.")
if show_tree and workspace:
ui.error_write(
"Enter the paths for dependencies and outputs of the command.",
styled=True,
)
if workspace:
ui.error_write(build_workspace_tree(workspace), styled=True)
ui.error_write()
ret.update(compact(_prompts(prompts, defaults, validator=validator)))
ui.error_write(styled=True)
ret.update(
compact(
_prompts(prompts, defaults, validator=validator, stream=stream)
)
)
return ret
@@ -170,16 +183,14 @@ def validate_prompts(key: str, value: str) -> Union[Any, Tuple[Any, str]]:
if key == "params":
assert isinstance(value, str)
try:
loadd_params(value)
except (FileNotFoundError, IsADirectoryError) as exc:
reason = "does not exist"
if isinstance(exc, IsADirectoryError):
reason = "is a directory"
raise InvalidResponse(
f"[prompt.invalid]'{value}' {reason}. "
"Please retry with an existing parameters file."
)
msg_format = (
"[prompt.invalid]'{0}' {1}. "
"Please retry with an existing parameters file."
)
if not os.path.exists(value):
raise InvalidResponse(msg_format.format(value, "does not exist"))
if os.path.isdir(value):
raise InvalidResponse(msg_format.format(value, "is a directory"))
elif key in ("code", "data"):
if not os.path.exists(value):
return value, (
@@ -197,6 +208,7 @@ def init(
overrides: Dict[str, str] = None,
interactive: bool = False,
force: bool = False,
stream: Optional[TextIO] = None,
) -> "Stage":
from dvc.dvcfile import make_dvcfile
@@ -205,8 +217,8 @@ def init(
_check_stage_exists(dvcfile, name, force=force)
defaults = defaults or {}
overrides = overrides or {}
defaults = defaults.copy() if defaults else {}
overrides = overrides.copy() if overrides else {}
with_live = type == "live"
@@ -217,7 +229,7 @@ def init(
defaults=defaults,
live=with_live,
provided=overrides,
show_tree=True,
stream=stream,
)
else:
if with_live:
@@ -252,7 +264,7 @@ def init(
)
if interactive:
ui.write(Rule(style="green"), styled=True)
ui.error_write(Rule(style="green"), styled=True)
_yaml = dumps_yaml(to_pipeline_file(cast(PipelineStage, stage)))
syn = Syntax(_yaml, "yaml", theme="ansi_dark")
ui.error_write(syn, styled=True)
@@ -263,8 +275,9 @@ def init(
"Do you want to add the above contents to dvc.yaml?",
console=ui.error_console,
default=True,
stream=stream,
):
with repo.scm_context(autostage=True, quiet=True), _disable_logging():
with _disable_logging(), repo.scm_context(autostage=True, quiet=True):
stage.dump(update_lock=False)
stage.ignore_outs()
if params:
tests/func/experiments/test_init.py
+416 -3
View File
@@ -1,11 +1,16 @@
import io
import os
import pytest
from dvc.command.experiments import CmdExperimentsInit
from dvc.exceptions import DvcException
from dvc.main import main
from dvc.utils.serialize import load_yaml
from dvc.repo.experiments.init import init
from dvc.stage.exceptions import DuplicateStageName
def test_init(tmp_dir, dvc):
def test_init_simple(tmp_dir, scm, dvc, capsys):
tmp_dir.gen(
{
CmdExperimentsInit.CODE: {"copy.py": ""},
@@ -18,8 +23,12 @@ def test_init(tmp_dir, dvc):
code_path = os.path.join(CmdExperimentsInit.CODE, "copy.py")
script = f"python {code_path}"
capsys.readouterr()
assert main(["exp", "init", script]) == 0
assert load_yaml(tmp_dir / "dvc.yaml") == {
out, err = capsys.readouterr()
assert not err
assert "Created default stage in dvc.yaml" in out
assert (tmp_dir / "dvc.yaml").parse() == {
"stages": {
"default": {
"cmd": script,
@@ -31,3 +40,407 @@ def test_init(tmp_dir, dvc):
}
}
}
@pytest.mark.parametrize("interactive", [True, False])
@pytest.mark.parametrize("typ", ["default", "live"])
def test_when_stage_already_exists_with_same_name(
tmp_dir, dvc, interactive, typ
):
(tmp_dir / "dvc.yaml").dump({"stages": {typ: {"cmd": "test"}}})
with pytest.raises(DuplicateStageName) as exc:
init(
dvc,
interactive=interactive,
type=typ,
overrides={"cmd": "true"},
defaults=CmdExperimentsInit.DEFAULTS,
)
assert (
str(exc.value) == f"Stage '{typ}' already exists in 'dvc.yaml'. "
"Use '--force' to overwrite."
)
@pytest.mark.parametrize("typ", ["default", "live"])
def test_when_stage_force_if_already_exists(tmp_dir, dvc, typ):
(tmp_dir / "params.yaml").dump({"foo": 1})
(tmp_dir / "dvc.yaml").dump({"stages": {typ: {"cmd": "test"}}})
init(
dvc,
type=typ,
force=True,
overrides={"cmd": "true"},
defaults=CmdExperimentsInit.DEFAULTS,
)
d = (tmp_dir / "dvc.yaml").parse()
assert d["stages"][typ]["cmd"] == "true"
def test_with_a_custom_name(tmp_dir, dvc):
init(dvc, name="custom", overrides={"cmd": "cmd"})
assert (tmp_dir / "dvc.yaml").parse() == {
"stages": {"custom": {"cmd": "cmd"}}
}
def test_init_with_no_defaults_non_interactive(tmp_dir, scm, dvc):
init(dvc, defaults={}, overrides={"cmd": "python script.py"})
assert (tmp_dir / "dvc.yaml").parse() == {
"stages": {"default": {"cmd": "python script.py"}}
}
scm._reset()
assert not (tmp_dir / "dvc.lock").exists()
assert scm.is_tracked("dvc.yaml")
def test_abort_confirmation(tmp_dir, dvc):
(tmp_dir / "param").dump({"foo": 1})
inp = io.StringIO("./script\nscript\ndata\nmodel\nparam\nmetric\nplt\nn")
with pytest.raises(DvcException) as exc:
init(
dvc,
interactive=True,
defaults=CmdExperimentsInit.DEFAULTS,
stream=inp,
)
assert str(exc.value) == "Aborting ..."
assert not (tmp_dir / "dvc.yaml").exists()
assert not (tmp_dir / "dvc.lock").exists()
@pytest.mark.parametrize(
"extra_overrides, inp",
[
({"cmd": "cmd"}, io.StringIO()),
({}, io.StringIO("cmd")),
],
)
def test_init_interactive_when_no_path_prompts_need_to_be_asked(
tmp_dir, dvc, extra_overrides, inp
):
"""When we pass everything that's required of, it should not prompt us."""
(tmp_dir / "params.yaml").dump({"foo": 1})
init(
dvc,
interactive=True,
defaults=CmdExperimentsInit.DEFAULTS,
overrides={**CmdExperimentsInit.DEFAULTS, **extra_overrides},
stream=inp, # we still need to confirm
)
assert (tmp_dir / "dvc.yaml").parse() == {
"stages": {
"default": {
"cmd": "cmd",
"deps": ["data", "src"],
"live": {"dvclive": {"html": True, "summary": True}},
"metrics": [{"metrics.json": {"cache": False}}],
# we specify `live` through `overrides`,
# so it creates checkpoint-based output.
"outs": [{"models": {"checkpoint": True}}],
"params": ["foo"],
"plots": [{"plots": {"cache": False}}],
}
}
}
def test_when_params_is_omitted_in_interactive_mode(tmp_dir, scm, dvc):
(tmp_dir / "params.yaml").dump({"foo": 1})
inp = io.StringIO("python script.py\nscript.py\ndata\nmodels\nn")
init(
dvc, interactive=True, stream=inp, defaults=CmdExperimentsInit.DEFAULTS
)
assert (tmp_dir / "dvc.yaml").parse() == {
"stages": {
"default": {
"cmd": "python script.py",
"deps": ["data", "script.py"],
"metrics": [{"metrics.json": {"cache": False}}],
"outs": ["models"],
"plots": [{"plots": {"cache": False}}],
}
}
}
assert not (tmp_dir / "dvc.lock").exists()
scm._reset()
assert scm.is_tracked("dvc.yaml")
assert not scm.is_tracked("params.yaml")
assert scm.is_tracked(".gitignore")
assert scm.is_ignored("models")
def test_init_interactive_params_validation(tmp_dir, dvc, capsys):
tmp_dir.gen({"data": {"foo": "foo"}})
(tmp_dir / "params.yaml").dump({"foo": 1})
inp = io.StringIO(
"python script.py\nscript.py\ndata\nmodels\nparams.json\ndata\n"
)
init(
dvc, stream=inp, interactive=True, defaults=CmdExperimentsInit.DEFAULTS
)
assert (tmp_dir / "dvc.yaml").parse() == {
"stages": {
"default": {
"cmd": "python script.py",
"deps": ["data", "script.py"],
"metrics": [{"metrics.json": {"cache": False}}],
"outs": ["models"],
"params": ["foo"],
"plots": [{"plots": {"cache": False}}],
}
}
}
out, err = capsys.readouterr()
assert (
"Path to a parameters file [params.yaml, n to omit]: "
"'params.json' does not exist. "
"Please retry with an existing parameters file.\n"
"Path to a parameters file [params.yaml, n to omit]: "
"'data' is a directory. "
"Please retry with an existing parameters file.\n"
"Path to a parameters file [params.yaml, n to omit]:"
) in err
assert not out
def test_init_with_no_defaults_interactive(tmp_dir, dvc):
inp = io.StringIO(
"python script.py\n"
"script.py\n"
"data\n"
"model\n"
"n\n"
"metric\n"
"n\n"
)
init(
dvc,
defaults={},
overrides={"cmd": "python script.py"},
interactive=True,
stream=inp,
)
assert (tmp_dir / "dvc.yaml").parse() == {
"stages": {
"default": {
"cmd": "python script.py",
"deps": ["python script.py", "script.py"],
"metrics": [{"metric": {"cache": False}}],
"outs": ["data"],
}
}
}
@pytest.mark.parametrize(
"interactive, overrides, inp",
[
(False, {"cmd": "python script.py", "code": "script.py"}, None),
(
True,
{},
io.StringIO(
"python script.py\n"
"script.py\n"
"data\n"
"models\n"
"params.yaml\n"
"metrics.json\n"
"plots\n"
"y"
),
),
],
ids=["non-interactive", "interactive"],
)
def test_init_interactive_default(
tmp_dir, scm, dvc, interactive, overrides, inp, capsys
):
(tmp_dir / "params.yaml").dump({"foo": {"bar": 1}})
init(
dvc,
interactive=interactive,
defaults=CmdExperimentsInit.DEFAULTS,
overrides=overrides,
stream=inp,
)
assert (tmp_dir / "dvc.yaml").parse() == {
"stages": {
"default": {
"cmd": "python script.py",
"deps": ["data", "script.py"],
"metrics": [{"metrics.json": {"cache": False}}],
"outs": ["models"],
"params": ["foo"],
"plots": [{"plots": {"cache": False}}],
}
}
}
assert not (tmp_dir / "dvc.lock").exists()
scm._reset()
assert scm.is_tracked("dvc.yaml")
assert scm.is_tracked("params.yaml")
assert scm.is_tracked(".gitignore")
assert scm.is_ignored("models")
out, err = capsys.readouterr()
if interactive:
assert "'script.py' does not exist in the workspace." in err
assert "'data' does not exist in the workspace." in err
assert not out
@pytest.mark.parametrize(
"interactive, overrides, inp",
[
(False, {"cmd": "python script.py", "code": "script.py"}, None),
(
True,
{},
io.StringIO(
"python script.py\n"
"script.py\n"
"data\n"
"models\n"
"params.yaml\n"
"dvclive\n"
"y"
),
),
(
True,
{"cmd": "python script.py"},
io.StringIO(
"script.py\n"
"data\n"
"models\n"
"params.yaml\n"
"dvclive\n"
"y"
),
),
(
True,
{"cmd": "python script.py", "models": "models"},
io.StringIO("script.py\ndata\nparams.yaml\ndvclive\ny"),
),
],
ids=[
"non-interactive",
"interactive",
"interactive-cmd-provided",
"interactive-cmd-models-provided",
],
)
def test_init_interactive_live(
tmp_dir, scm, dvc, interactive, overrides, inp, capsys
):
(tmp_dir / "params.yaml").dump({"foo": {"bar": 1}})
init(
dvc,
type="live",
interactive=interactive,
defaults=CmdExperimentsInit.DEFAULTS,
overrides=overrides,
stream=inp,
)
assert (tmp_dir / "dvc.yaml").parse() == {
"stages": {
"live": {
"cmd": "python script.py",
"deps": ["data", "script.py"],
"live": {"dvclive": {"html": True, "summary": True}},
"outs": [{"models": {"checkpoint": True}}],
"params": ["foo"],
}
}
}
assert not (tmp_dir / "dvc.lock").exists()
scm._reset()
assert scm.is_tracked("dvc.yaml")
assert scm.is_tracked("params.yaml")
assert scm.is_tracked(".gitignore")
assert scm.is_ignored("models")
out, err = capsys.readouterr()
if interactive:
assert "'script.py' does not exist in the workspace." in err
assert "'data' does not exist in the workspace." in err
assert not out
@pytest.mark.parametrize(
"interactive, inp",
[
(False, None),
(True, io.StringIO()),
],
)
def test_init_with_type_live_and_models_plots_provided(
tmp_dir, dvc, interactive, inp
):
(tmp_dir / "params.yaml").dump({"foo": 1})
init(
dvc,
type="live",
interactive=interactive,
stream=inp,
defaults=CmdExperimentsInit.DEFAULTS,
overrides={"cmd": "cmd", "metrics": "m", "plots": "p"},
)
assert (tmp_dir / "dvc.yaml").parse() == {
"stages": {
"live": {
"cmd": "cmd",
"deps": ["data", "src"],
"live": {"dvclive": {"html": True, "summary": True}},
"metrics": [{"m": {"cache": False}}],
"outs": [{"models": {"checkpoint": True}}],
"params": ["foo"],
"plots": [{"p": {"cache": False}}],
}
}
}
@pytest.mark.parametrize(
"interactive, inp",
[
(False, None),
(True, io.StringIO()),
],
)
def test_init_with_type_default_and_live_provided(
tmp_dir, dvc, interactive, inp
):
(tmp_dir / "params.yaml").dump({"foo": 1})
init(
dvc,
interactive=interactive,
stream=inp,
defaults=CmdExperimentsInit.DEFAULTS,
overrides={"cmd": "cmd", "live": "live"},
)
assert (tmp_dir / "dvc.yaml").parse() == {
"stages": {
"default": {
"cmd": "cmd",
"deps": ["data", "src"],
"live": {"live": {"html": True, "summary": True}},
"metrics": [{"metrics.json": {"cache": False}}],
"outs": [{"models": {"checkpoint": True}}],
"params": ["foo"],
"plots": [{"plots": {"cache": False}}],
}
}
}
tests/func/test_analytics.py
+1 -11
View File
@@ -6,6 +6,7 @@ import pytest
from dvc.analytics import _scm_in_use, collect_and_send_report
from dvc.main import main
from dvc.repo import Repo
from tests.utils import ANY
def test_daemon_analytics(mocker, tmp_path):
@@ -34,17 +35,6 @@ def mock_daemon(mocker):
yield m
class ANY:
def __init__(self, expected_type):
self.expected_type = expected_type
def __repr__(self):
return "Any" + self.expected_type.__name__.capitalize()
def __eq__(self, other):
return isinstance(other, self.expected_type)
def test_collect_and_send_report(mocker, dvc, mock_daemon):
mock_post = mocker.patch("requests.post")
collect_and_send_report()
tests/unit/command/test_experiments.py
+161 -12
View File
@@ -10,6 +10,7 @@ from dvc.command.experiments import (
CmdExperimentsBranch,
CmdExperimentsDiff,
CmdExperimentsGC,
CmdExperimentsInit,
CmdExperimentsList,
CmdExperimentsPull,
CmdExperimentsPush,
@@ -18,7 +19,11 @@ from dvc.command.experiments import (
CmdExperimentsShow,
show_experiments,
)
from dvc.exceptions import InvalidArgumentError
from dvc.exceptions import DvcParserError, InvalidArgumentError
from dvc.repo import Repo
from dvc.stage import PipelineStage
from tests.utils import ANY
from tests.utils.asserts import called_once_with_subset
from .test_repro import default_arguments as repro_arguments
@@ -594,7 +599,49 @@ def test_show_experiments_sort_by(capsys, sort_order):
assert params == (2, 1, 0)
def test_experiments_init_config(dvc, mocker):
@pytest.mark.parametrize("extra_args", [(), ("--run",)])
def test_experiments_init(dvc, scm, mocker, capsys, extra_args):
m = mocker.patch(
"dvc.repo.experiments.init.init",
return_value=PipelineStage(dvc, path="dvc.yaml", name="stage1"),
)
runner = mocker.patch("dvc.repo.experiments.run.run", return_value=0)
cli_args = parse_args(["exp", "init", *extra_args, "cmd"])
cmd = cli_args.func(cli_args)
assert isinstance(cmd, CmdExperimentsInit)
assert cmd.run() == 0
m.assert_called_once_with(
ANY(Repo),
name=None,
type="default",
defaults={
"code": "src",
"models": "models",
"data": "data",
"metrics": "metrics.json",
"params": "params.yaml",
"plots": "plots",
"live": "dvclive",
},
overrides={"cmd": "cmd"},
interactive=False,
force=False,
)
expected = "Created default stage in dvc.yaml."
if not extra_args:
expected += (
' To run, use "dvc exp run".\n'
"See https://dvc.org/doc/user-guide/experiment-management"
"/running-experiments."
)
assert capsys.readouterr() == (expected + "\n", "")
if extra_args:
# `parse_args` creates a new `Repo` object
runner.assert_called_once_with(ANY(Repo), targets=["stage1"])
def test_experiments_init_config(dvc, scm, mocker):
with dvc.config.edit() as conf:
conf["exp"] = {"code": "new_src", "models": "new_models"}
@@ -602,14 +649,116 @@ def test_experiments_init_config(dvc, mocker):
cli_args = parse_args(["exp", "init", "cmd"])
cmd = cli_args.func(cli_args)
assert isinstance(cmd, CmdExperimentsInit)
assert cmd.run() == 0
assert m.call_args[1]["defaults"] == {
"code": "new_src",
"models": "new_models",
"data": "data",
"metrics": "metrics.json",
"params": "params.yaml",
"plots": "plots",
"live": "dvclive",
}
assert m.call_args[1]["overrides"] == {"cmd": "cmd"}
m.assert_called_once_with(
ANY(Repo),
name=None,
type="default",
defaults={
"code": "new_src",
"models": "new_models",
"data": "data",
"metrics": "metrics.json",
"params": "params.yaml",
"plots": "plots",
"live": "dvclive",
},
overrides={"cmd": "cmd"},
interactive=False,
force=False,
)
def test_experiments_init_explicit(dvc, mocker):
m = mocker.patch("dvc.repo.experiments.init.init")
cli_args = parse_args(["exp", "init", "--explicit", "cmd"])
cmd = cli_args.func(cli_args)
assert cmd.run() == 0
m.assert_called_once_with(
ANY(Repo),
name=None,
type="default",
defaults={},
overrides={"cmd": "cmd"},
interactive=False,
force=False,
)
def test_experiments_init_cmd_required_for_non_interactive_mode(dvc):
cli_args = parse_args(["exp", "init"])
cmd = cli_args.func(cli_args)
assert isinstance(cmd, CmdExperimentsInit)
with pytest.raises(InvalidArgumentError) as exc:
cmd.run()
assert str(exc.value) == "command is not specified"
def test_experiments_init_cmd_not_required_for_interactive_mode(dvc, mocker):
cli_args = parse_args(["exp", "init", "--interactive"])
cmd = cli_args.func(cli_args)
assert isinstance(cmd, CmdExperimentsInit)
m = mocker.patch("dvc.repo.experiments.init.init")
assert cmd.run() == 0
assert called_once_with_subset(m, ANY(Repo), interactive=True)
@pytest.mark.parametrize(
"extra_args, expected_kw",
[
(["--type", "default"], {"type": "default"}),
(["--type", "live"], {"type": "live"}),
(["--force"], {"force": True}),
(
["--name", "name", "--type", "live"],
{"name": "name", "type": "live"},
),
(
[
"--plots",
"p",
"--models",
"m",
"--code",
"c",
"--metrics",
"m.json",
"--params",
"p.yaml",
"--data",
"d",
"--live",
"live",
],
{
"overrides": {
"plots": "p",
"models": "m",
"code": "c",
"metrics": "m.json",
"params": "p.yaml",
"data": "d",
"live": "live",
"cmd": "cmd",
}
},
),
],
)
def test_experiments_init_extra_args(extra_args, expected_kw, mocker):
cli_args = parse_args(["exp", "init", *extra_args, "cmd"])
cmd = cli_args.func(cli_args)
assert isinstance(cmd, CmdExperimentsInit)
m = mocker.patch("dvc.repo.experiments.init.init")
assert cmd.run() == 0
assert called_once_with_subset(m, ANY(Repo), **expected_kw)
def test_experiments_init_type_invalid_choice():
with pytest.raises(DvcParserError):
parse_args(["exp", "init", "--type=invalid", "cmd"])
tests/utils/__init__.py
+11
View File
@@ -69,3 +69,14 @@ def console_width(console, width):
finally:
console_options.max_width = original
console._width = con_width
class ANY:
def __init__(self, expected_type):
self.expected_type = expected_type
def __repr__(self):
return "Any" + self.expected_type.__name__.capitalize()
def __eq__(self, other):
return isinstance(other, self.expected_type)