From c65fdcabae9632edb4e8741570631732204b10f8 Mon Sep 17 00:00:00 2001 From: Saugat Pachhai Date: Fri, 20 Nov 2020 19:42:59 +0545 Subject: [PATCH] tests: add error message tests for resolving stages (#4927) --- dvc/parsing/context.py | 2 +- tests/func/test_stage_resolver_error_msg.py | 339 ++++++++++++++++++++ tests/unit/test_context.py | 11 +- 3 files changed, 346 insertions(+), 6 deletions(-) create mode 100644 tests/func/test_stage_resolver_error_msg.py diff --git a/dvc/parsing/context.py b/dvc/parsing/context.py index 75260e678..3f219e261 100644 --- a/dvc/parsing/context.py +++ b/dvc/parsing/context.py @@ -319,7 +319,7 @@ class Context(CtxDict): _, ext = os.path.splitext(file) loader = LOADERS[ext] - meta = Meta(source=str(path), local=False) + meta = Meta(source=file, local=False) return cls(loader(path, tree=tree), meta=meta) def merge_from(self, tree, path: PathInfo, overwrite=False): diff --git a/tests/func/test_stage_resolver_error_msg.py b/tests/func/test_stage_resolver_error_msg.py new file mode 100644 index 000000000..4b7e5af3e --- /dev/null +++ b/tests/func/test_stage_resolver_error_msg.py @@ -0,0 +1,339 @@ +import logging + +import pytest + +from dvc.parsing import DEFAULT_PARAMS_FILE, DataResolver, ResolveError +from dvc.utils.fs import remove +from dvc.utils.serialize import dump_json, dump_yaml + +DATA = {"models": {"bar": "bar", "foo": "foo"}} + + +@pytest.fixture +def repo(tmp_dir, dvc): + dump_yaml(tmp_dir / DEFAULT_PARAMS_FILE, DATA) + dump_json(tmp_dir / "params.json", DATA) + yield dvc + + +def test_global_overwrite_error_on_imports(tmp_dir, repo): + d = {"vars": [DEFAULT_PARAMS_FILE, "params.json"]} + with pytest.raises(ResolveError) as exc_info: + DataResolver(repo, tmp_dir, d) + assert str(exc_info.value) == ( + "failed to parse 'vars' in 'dvc.yaml':\n" + "cannot redefine 'models.bar' from 'params.json' " + "as it already exists in 'params.yaml'" + ) + + +def test_global_overwrite_vars(tmp_dir, repo): + d = {"vars": [DATA]} + with pytest.raises(ResolveError) as exc_info: + DataResolver(repo, tmp_dir, d) + assert str(exc_info.value) == ( + "failed to parse 'vars' in 'dvc.yaml':\n" + "cannot redefine 'models.bar' from 'vars[0]' " + "as it already exists in 'params.yaml'" + ) + + +def test_local_declared_vars_overwrite(tmp_dir, repo): + d = {"vars": [DATA["models"], DATA["models"]]} + with pytest.raises(ResolveError) as exc_info: + DataResolver(repo, tmp_dir, d) + assert str(exc_info.value) == ( + "failed to parse 'vars' in 'dvc.yaml':\n" + "cannot redefine 'bar' from 'vars[1]' " + "as it already exists in 'vars[0]'" + ) + + +def test_default_params_file_not_exist(tmp_dir, repo): + remove(tmp_dir / DEFAULT_PARAMS_FILE) + d = {"vars": [DATA["models"]]} + DataResolver(repo, tmp_dir, d) + + +def test_specified_params_file_not_exist(tmp_dir, repo): + d = {"vars": ["not_existing_params.yaml"]} + with pytest.raises(ResolveError) as exc_info: + DataResolver(repo, tmp_dir, d) + assert str(exc_info.value) == ( + "failed to parse 'vars' in 'dvc.yaml': " + "'not_existing_params.yaml' does not exist" + ) + + +def test_failed_to_interpolate(tmp_dir, repo): + d = { + "vars": [DEFAULT_PARAMS_FILE], + "stages": {"build": {"cmd": "echo ${models.foo.}"}}, + } + with pytest.raises(ResolveError) as exc_info: + resolver = DataResolver(repo, tmp_dir, d) + resolver.resolve() + assert str(exc_info.value) == ( + "failed to parse 'stages.build.cmd' in 'dvc.yaml':\n" + "${models.foo.}\n" + " ^\n" + "ParseException: Expected end of text, found '.'" + " (at char 12), (line:1, col:13)" + ) + + +@pytest.mark.parametrize( + "vars_, loc", [(DATA, "build.vars[0]"), ("params.json", "params.json")], +) +def test_local_overwrite_error(tmp_dir, repo, vars_, loc): + d = {"stages": {"build": {"cmd": "echo ${models.foo}", "vars": [vars_]}}} + resolver = DataResolver(repo, tmp_dir, d) + with pytest.raises(ResolveError) as exc_info: + resolver.resolve() + assert str(exc_info.value) == ( + "failed to parse stage 'build' in 'dvc.yaml':\n" + f"cannot redefine 'models.bar' from '{loc}' " + "as it already exists in 'params.yaml'" + ) + + +def test_local_vars_params_file_not_exist(tmp_dir, repo): + d = { + "stages": { + "build": { + "cmd": "echo ${models.foo}", + "vars": ["not_existing_params.yaml"], + } + } + } + resolver = DataResolver(repo, tmp_dir, d) + with pytest.raises(ResolveError) as exc_info: + resolver.resolve() + assert str(exc_info.value) == ( + "failed to parse stage 'build' in 'dvc.yaml': " + "'not_existing_params.yaml' does not exist" + ) + + +def test_specified_key_does_not_exist(tmp_dir, repo): + d = {"stages": {"build": {"cmd": "echo ${models.foobar}"}}} + resolver = DataResolver(repo, tmp_dir, d) + with pytest.raises(ResolveError) as exc_info: + resolver.resolve() + assert str(exc_info.value) == ( + "failed to parse 'stages.build.cmd' in 'dvc.yaml': " + "Could not find 'models.foobar'" + ) + + +@pytest.mark.parametrize( + "local_vars, expected_text", + [ + ({"item": "item"}, "item is"), + ({"key": "key"}, "key is"), + ({"item": "item", "key": "key"}, "item and key are"), + ], +) +def test_foreach_item_key_already_exists( + tmp_dir, repo, caplog, local_vars, expected_text +): + d = { + "vars": [local_vars], + "stages": {"build": {"foreach": "${models}", "in": {}}}, + } + resolver = DataResolver(repo, tmp_dir, d) + + caplog.clear() + with caplog.at_level(logging.WARNING, logger="dvc.parsing"): + resolver.resolve() + assert expected_text in caplog.text + assert "generated from 'build'" in caplog.text + + +def test_foreach_interpolation_key_does_not_exist(tmp_dir, repo): + d = { + "stages": {"build": {"foreach": "${modelss}", "in": {}}}, + } + resolver = DataResolver(repo, tmp_dir, d) + with pytest.raises(ResolveError) as exc_info: + resolver.resolve() + assert str(exc_info.value) == ( + "failed to parse 'stages.build.foreach' in 'dvc.yaml': " + "Could not find 'modelss'" + ) + + +def test_foreach_interpolation_key_error(tmp_dir, repo): + d = { + "stages": {"build": {"foreach": "${models[123}", "in": {}}}, + } + resolver = DataResolver(repo, tmp_dir, d) + with pytest.raises(ResolveError) as exc_info: + resolver.resolve() + assert str(exc_info.value) == ( + "failed to parse 'stages.build.foreach' in 'dvc.yaml':\n" + "${models[123}\n" + " ^\n" + "ParseException: Expected end of text, found '['" + " (at char 8), (line:1, col:9)" + ) + + +@pytest.mark.parametrize( + "syn, msg", + [ + ("${ite}", " Could not find 'ite'"), + ( + "${ite.}", + "\n${ite.}\n" + " ^\n" + "ParseException: Expected end of text, found '.'" + " (at char 5), (line:1, col:6)", + ), + ], +) +def test_foreach_generated_errors(tmp_dir, repo, syn, msg): + d = { + "stages": { + "build": {"foreach": "${models}", "in": {"cmd": f"echo {syn}"}} + }, + } + resolver = DataResolver(repo, tmp_dir, d) + with pytest.raises(ResolveError) as exc_info: + resolver.resolve() + assert str(exc_info.value) == ( + "failed to parse 'stages.build-bar.cmd' in 'dvc.yaml':" + msg + ) + + +@pytest.mark.parametrize( + "vars_, msg", + [ + (["params-does-not-exist"], " 'params-does-not-exist' does not exist"), + ( + [DATA], + "\ncannot redefine 'models.bar' from 'build-bar.vars[0]' " + f"as it already exists in '{DEFAULT_PARAMS_FILE}'", + ), + ], +) +def test_foreach_generated_local_vars_error(tmp_dir, repo, vars_, msg): + d = { + "stages": { + "build": { + "foreach": "${models}", + "in": {"vars": vars_, "cmd": "echo ${item}"}, + } + }, + } + resolver = DataResolver(repo, tmp_dir, d) + with pytest.raises(ResolveError) as exc_info: + resolver.resolve() + assert str(exc_info.value) == ( + "failed to parse stage 'build-bar' (gen. from 'build') in 'dvc.yaml':" + + msg + ) + + +@pytest.mark.parametrize( + "wdir, msg", + [ + ("${ite}", " Could not find 'ite'"), + ( + "${item.}", + "\n${item.}\n" + " ^\n" + "ParseException: Expected end of text, found '.'" + " (at char 6), (line:1, col:7)", + ), + ], +) +def test_foreach_wdir_interpolation_issues(tmp_dir, repo, wdir, msg): + d = { + "stages": { + "build": { + "foreach": "${models}", + "in": {"wdir": wdir, "cmd": "echo ${item}"}, + } + }, + } + resolver = DataResolver(repo, tmp_dir, d) + with pytest.raises(ResolveError) as exc_info: + resolver.resolve() + assert str(exc_info.value) == ( + "failed to parse 'stages.build-bar.wdir' in 'dvc.yaml':" + msg + ) + + +@pytest.mark.parametrize("foreach", ["${models.bar}", "foo"]) +def test_foreach_expects_dict_or_list(tmp_dir, repo, foreach): + d = { + "stages": {"build": {"foreach": foreach, "in": {}}}, + } + resolver = DataResolver(repo, tmp_dir, d) + with pytest.raises(ResolveError) as exc_info: + resolver.resolve() + assert str(exc_info.value) == ( + "failed to resolve 'stages.build.foreach' in 'dvc.yaml': " + "expected list/dictionary, got str" + ) + + +@pytest.mark.parametrize( + "wdir, expected_msg", + [ + ("${models[foobar]}", " Could not find 'models.foobar'"), + ( + "${models.foo]}", + "\n${models.foo]}\n" + " ^\n" + "ParseException: Expected end of text, found ']'" + " (at char 12), (line:1, col:13)", + ), + ], +) +def test_wdir_failed_to_interpolate(tmp_dir, repo, wdir, expected_msg): + d = {"stages": {"build": {"wdir": wdir, "cmd": "echo ${models.bar}"}}} + resolver = DataResolver(repo, tmp_dir, d) + with pytest.raises(ResolveError) as exc_info: + resolver.resolve() + assert str(exc_info.value) == ( + "failed to parse 'stages.build.wdir' in 'dvc.yaml':" + expected_msg + ) + + +@pytest.mark.parametrize( + "foreach, redefine", + [ + ("${models}", "item"), + ("${models}", "key"), + (list(DATA["models"].keys()), "item"), + ], +) +def test_item_key_in_generated_stage_vars(tmp_dir, repo, foreach, redefine): + d = { + "stages": { + "build": { + "foreach": foreach, + "in": {"cmd": "echo ${item}", "vars": [{redefine: "value"}]}, + } + }, + } + resolver = DataResolver(repo, tmp_dir, d) + with pytest.raises(ResolveError) as exc_info: + resolver.resolve() + assert str(exc_info.value) == ( + f"attempted to redefine '{redefine}' in stage 'build-bar'" + " generated through 'foreach'" + ) + + +def test_interpolate_non_string(tmp_dir, repo): + d = {"stages": {"build": {"cmd": "echo ${models}"}}} + resolver = DataResolver(repo, tmp_dir, d) + with pytest.raises(ResolveError) as exc_info: + resolver.resolve() + assert str(exc_info.value) == ( + "failed to parse 'stages.build.cmd' in 'dvc.yaml':\n" + "Cannot interpolate data of type 'dict'" + ) diff --git a/tests/unit/test_context.py b/tests/unit/test_context.py index 4828664b2..924b8f986 100644 --- a/tests/unit/test_context.py +++ b/tests/unit/test_context.py @@ -14,6 +14,7 @@ from dvc.parsing.context import ( Value, ) from dvc.tree.local import LocalTree +from dvc.utils import relpath from dvc.utils.serialize import dump_yaml from tests.func.test_stage_resolver import recurse_not_a_node @@ -291,11 +292,11 @@ def test_track(tmp_dir): path = tmp_dir / "params.yaml" dump_yaml(path, d, tree) - context = Context.load_from(tree, str(path)) + context = Context.load_from(tree, path) def key_tracked(key): assert len(context.tracked) == 1 - return key in context.tracked[str(path)] + return key in context.tracked[relpath(path)] with context.track(): context.select("lst") @@ -328,12 +329,12 @@ def test_track_from_multiple_files(tmp_dir): dump_yaml(path1, d1, tree) dump_yaml(path2, d2, tree) - context = Context.load_from(tree, str(path1)) - c = Context.load_from(tree, str(path2)) + context = Context.load_from(tree, path1) + c = Context.load_from(tree, path2) context.merge_update(c) def key_tracked(path, key): - return key in context.tracked[str(path)] + return key in context.tracked[relpath(path)] with context.track(): context.select("Train")