dvc: normalize targets before entering brancher

Fixes #7888
This commit is contained in:
Ruslan Kuprieiev
2022-07-04 16:34:49 +03:00
parent 683d22c42b
commit 0d71194741
8 changed files with 61 additions and 2 deletions
+5 -2
View File
@@ -5,6 +5,7 @@ from typing import Dict, List
from dvc.exceptions import PathMissingError
from dvc.repo import locked
from dvc.utils.collections import ensure_list
logger = logging.getLogger(__name__)
@@ -25,6 +26,8 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
from dvc.fs.dvc import DvcFileSystem
dvcfs = DvcFileSystem(repo=self)
targets = ensure_list(targets)
targets = [dvcfs.from_os_path(target) for target in targets]
b_rev = b_rev if b_rev else "workspace"
results = {}
@@ -36,7 +39,7 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
continue
targets_paths = None
if targets is not None:
if targets:
# convert targets to paths, and capture any missing targets
targets_paths, missing_targets[rev] = _targets_to_paths(
dvcfs, targets
@@ -44,7 +47,7 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
results[rev] = _paths_checksums(self, targets_paths)
if targets is not None:
if targets:
# check for overlapping missing targets between a_rev and b_rev
for target in set(missing_targets[a_rev]) & set(
missing_targets[b_rev]
+4
View File
@@ -11,6 +11,7 @@ from dvc.repo.collect import StrPaths, collect
from dvc.repo.live import summary_fs_path
from dvc.scm import NoSCMError
from dvc.utils import error_handler, errored_revisions, onerror_collect
from dvc.utils.collections import ensure_list
from dvc.utils.serialize import load_yaml
logger = logging.getLogger(__name__)
@@ -113,6 +114,9 @@ def show(
if onerror is None:
onerror = onerror_collect
targets = ensure_list(targets)
targets = [repo.dvcfs.from_os_path(target) for target in targets]
res = {}
for rev in repo.brancher(
revs=revs,
+4
View File
@@ -21,6 +21,7 @@ from dvc.scm import NoSCMError
from dvc.stage import PipelineStage
from dvc.ui import ui
from dvc.utils import error_handler, errored_revisions, onerror_collect
from dvc.utils.collections import ensure_list
from dvc.utils.serialize import LOADERS
if TYPE_CHECKING:
@@ -131,6 +132,9 @@ def show(
onerror = onerror_collect
res = {}
targets = ensure_list(targets)
targets = [repo.dvcfs.from_os_path(target) for target in targets]
for branch in repo.brancher(revs=revs):
params = error_handler(_gather_params)(
repo=repo,
+2
View File
@@ -112,6 +112,8 @@ class Plots:
from dvc.utils.collections import ensure_list
targets = ensure_list(targets)
targets = [self.repo.dvcfs.from_os_path(target) for target in targets]
for rev in self.repo.brancher(revs=revs):
# .brancher() adds unwanted workspace
if revs is not None and rev not in revs:
+13
View File
@@ -31,6 +31,19 @@ def test_show(tmp_dir, dvc, run_copy_metrics):
}
def test_show_targets(tmp_dir, dvc, run_copy_metrics):
tmp_dir.gen("metrics_t.yaml", "foo: 1.1")
run_copy_metrics(
"metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"]
)
expected = {"": {"data": {"metrics.yaml": {"data": {"foo": 1.1}}}}}
assert dvc.metrics.show(targets=["metrics.yaml"]) == expected
assert (
dvc.metrics.show(targets=(tmp_dir / "metrics.yaml").fs_path)
== expected
)
def test_show_multiple(tmp_dir, dvc, run_copy_metrics):
tmp_dir.gen("foo_temp", "foo: 1\n")
tmp_dir.gen("baz_temp", "baz: 2\n")
+10
View File
@@ -20,6 +20,16 @@ def test_show(tmp_dir, dvc):
}
def test_show_targets(tmp_dir, dvc):
tmp_dir.gen("params.yaml", "foo: bar")
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
expected = {"": {"data": {"params.yaml": {"data": {"foo": "bar"}}}}}
assert dvc.params.show(targets=["params.yaml"]) == expected
assert (
dvc.params.show(targets=(tmp_dir / "params.yaml").fs_path) == expected
)
def test_show_toml(tmp_dir, dvc):
tmp_dir.gen("params.toml", "[foo]\nbar = 42\nbaz = [1, 2]\n")
dvc.run(
+11
View File
@@ -13,6 +13,17 @@ from dvc.utils.serialize import EncodingError, YAMLFileCorruptedError
from tests.utils.plots import get_plot
def test_show_targets(tmp_dir, dvc):
metric = [{"first_val": 100, "val": 2}, {"first_val": 200, "val": 3}]
(tmp_dir / "metric.json").dump_json(metric, sort_keys=True)
plots = dvc.plots.show(targets=["metric.json"])
assert get_plot(plots, "workspace", file="metric.json") == metric
plots = dvc.plots.show(targets=(tmp_dir / "metric.json").fs_path)
assert get_plot(plots, "workspace", file="metric.json") == metric
def test_plot_cache_missing(tmp_dir, scm, dvc, caplog, run_copy_metrics):
metric1 = [{"y": 2}, {"y": 3}]
(tmp_dir / "metric_t.json").dump_json(metric1, sort_keys=True)
+12
View File
@@ -331,6 +331,18 @@ def test_no_commits(tmp_dir):
assert Repo.init().diff() == {}
def test_abs_target(tmp_dir, scm, dvc):
tmp_dir.dvc_gen("file", "text")
assert dvc.diff(targets=(tmp_dir / "file").fs_path) == {
"added": [{"path": "file", "hash": digest("text")}],
"deleted": [],
"modified": [],
"not in cache": [],
"renamed": [],
}
def setup_targets_test(tmp_dir):
tmp_dir.dvc_gen("file", "first", commit="add a file")