From dca48fe8d7f86e49dad856bad1e970b837912793 Mon Sep 17 00:00:00 2001 From: Saugat Pachhai Date: Sat, 7 Nov 2020 09:30:12 +0545 Subject: [PATCH] Skip loading globally used params section again locally for stages (#4859) We were still trying to load params file used in `use` for local stages which was throwing overwrite error. Thanks to @dmpetrov for pointing out. --- dvc/parsing/__init__.py | 19 +++++++---- tests/func/test_stage_resolver.py | 53 +++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/dvc/parsing/__init__.py b/dvc/parsing/__init__.py index fecc209b1..92e96d151 100644 --- a/dvc/parsing/__init__.py +++ b/dvc/parsing/__init__.py @@ -89,12 +89,19 @@ class DataResolver: "%s does not exist for stage %s", params_yaml_file, name ) - params_file = definition.get(PARAMS_KWD, []) - for item in params_file: - if item and isinstance(item, dict): - contexts.append( - Context.load_from(self.repo.tree, str(wdir / first(item))) - ) + params_deps = definition.get(PARAMS_KWD, []) + params_files = { + wdir / first(item) + for item in params_deps + if item and isinstance(item, dict) + } + for params_file in params_files - { + self.global_ctx_source, + params_yaml_file, + }: + contexts.append( + Context.load_from(self.repo.tree, str(params_file)) + ) context.merge_update(*contexts) diff --git a/tests/func/test_stage_resolver.py b/tests/func/test_stage_resolver.py index 28e4c6e12..0a000e863 100644 --- a/tests/func/test_stage_resolver.py +++ b/tests/func/test_stage_resolver.py @@ -429,3 +429,56 @@ def test_set_with_foreach_and_on_stage_definition(tmp_dir, dvc): } }, ) + + +def test_resolve_local_tries_to_load_globally_used_files(tmp_dir, dvc): + iterable = {"bar": "bar", "foo": "foo"} + dump_json(tmp_dir / "params.json", iterable) + + d = { + "use": "params.json", + "stages": { + "build": { + "cmd": "command --value ${bar}", + "params": [{"params.json": ["foo"]}], + }, + }, + } + resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) + assert_stage_equal( + resolver.resolve(), + { + "stages": { + "build": { + "cmd": "command --value bar", + "params": [{"params.json": ["bar", "foo"]}], + }, + } + }, + ) + + +def test_resolve_local_tries_to_load_globally_used_params_yaml(tmp_dir, dvc): + iterable = {"bar": "bar", "foo": "foo"} + dump_yaml(tmp_dir / "params.yaml", iterable) + + d = { + "stages": { + "build": { + "cmd": "command --value ${bar}", + "params": [{"params.yaml": ["foo"]}], + }, + }, + } + resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) + assert_stage_equal( + resolver.resolve(), + { + "stages": { + "build": { + "cmd": "command --value bar", + "params": [{"params.yaml": ["bar", "foo"]}], + }, + } + }, + )