params diff: skip params dependency for vars (bp #5123) (#5127)

* Skip paramdependency for vars

Previously, `configs` which was a list of PathInfo was being used
to check if the `vars` should be skipped or not.

But, the file is stored as a string that made that check redundant
which might have caused #5117 as they were being stored twice,
once as a expanded paramdependency and next as a flattened dict
from vars, which later during `flatten`, it might have panicked
from that duplication.

* Use defaultdict

* change add_vars to collect_vars

(cherry picked from commit dcce3a31a9)

Co-authored-by: Saugat Pachhai <[email protected]>
This commit is contained in:
mergify[bot]
2020-12-18 12:09:22 +05:45
committed by GitHub
co-authored by Saugat Pachhai
parent a3891995e6
commit fe0d6ec3e0
+11 -5
View File
@@ -1,4 +1,5 @@
import logging
from collections import defaultdict
from typing import TYPE_CHECKING, List
from dvc.dependency.param import ParamsDependency
@@ -50,17 +51,18 @@ def _read_params(repo, configs, rev):
return res
def _add_vars(repo, configs, params):
def _collect_vars(repo, params):
vars_params = defaultdict(dict)
for stage in repo.stages:
if isinstance(stage, PipelineStage) and stage.tracked_vars:
for file, vars_ in stage.tracked_vars.items():
# `params` file are shown regardless of `tracked` or not
# to reduce noise and duplication, they are skipped
if file in configs:
if file in params:
continue
params[file] = params.get(file, {})
params[file].update(vars_)
vars_params[file].update(vars_)
return vars_params
@locked
@@ -70,7 +72,11 @@ def show(repo, revs=None):
for branch in repo.brancher(revs=revs):
configs = _collect_configs(repo, branch)
params = _read_params(repo, configs, branch)
_add_vars(repo, configs, params)
vars_params = _collect_vars(repo, params)
# NOTE: only those that are not added as a ParamDependency are included
# so we don't need to recursively merge them yet.
params.update(vars_params)
if params:
res[branch] = params