Run graph checks on collect/find_outs_by_path (#5035)

We try to optimize `tree.exists` calls and probably few others
in that they either look directly into the workspace or,
to the cache without running graph checks. It does not seem
to be possible just to run graph checks on `find_outs_by_path`
due to those optimizations.

So, that's why, the `collect` also does a graph check for this
reason.

Fixes #5027
Fixes #4010
This commit is contained in:
Saugat Pachhai
2020-12-10 12:20:45 +02:00
committed by GitHub
parent 9967d52885
commit 189e88f1db
4 changed files with 60 additions and 4 deletions
+2 -2
View File
@@ -380,8 +380,8 @@ class Repo:
return self.stage.collect_repo(onerror=error_handler)
def find_outs_by_path(self, path, outs=None, recursive=False, strict=True):
if not outs:
outs = [out for stage in self.stages for out in stage.outs]
# using `outs_graph` to ensure graph checks are run
outs = outs or self.outs_graph
abs_path = os.path.abspath(path)
path_info = PathInfo(abs_path)
+1 -1
View File
@@ -21,7 +21,7 @@ def _collect_outs(
) -> Outputs:
outs = {
out
for stage in repo.stages
for stage in repo.graph # using `graph` to ensure graph checks run
for out in (stage.deps if deps else stage.outs)
}
return set(filter(output_filter, outs)) if output_filter else outs
+46 -1
View File
@@ -2,7 +2,14 @@ import logging
import pytest
from dvc.exceptions import NoMetricsFoundError, NoMetricsParsedError
from dvc.exceptions import (
NoMetricsFoundError,
NoMetricsParsedError,
OverlappingOutputPathsError,
)
from dvc.path_info import PathInfo
from dvc.utils.fs import remove
from dvc.utils.serialize import dump_yaml, modify_yaml
from tests.func.metrics.utils import _write_json
@@ -61,3 +68,41 @@ def test_show_malformed_metric(
def test_show_no_metrics_files(tmp_dir, dvc, caplog, show):
with pytest.raises(NoMetricsFoundError):
show(dvc)
@pytest.mark.parametrize("clear_before_run", [True, False])
@pytest.mark.parametrize("typ", ["metrics", "plots"])
def test_metrics_show_overlap(
tmp_dir, dvc, run_copy_metrics, clear_before_run, typ
):
data_dir = PathInfo("data")
(tmp_dir / data_dir).mkdir()
outs = {typ: [str(data_dir / "m1.yaml")]}
dump_yaml(data_dir / "m1_temp.yaml", {"a": {"b": {"c": 2, "d": 1}}})
run_copy_metrics(
str(data_dir / "m1_temp.yaml"),
str(data_dir / "m1.yaml"),
single_stage=False,
commit=f"add m1 {typ}",
name="cp-m1",
**outs,
)
with modify_yaml("dvc.yaml") as d:
# trying to make an output overlaps error
d["stages"]["corrupted-stage"] = {
"cmd": "mkdir data",
"outs": ["data"],
}
# running by clearing and not clearing stuffs
# so as it works even for optimized cases
if clear_before_run:
remove(data_dir)
remove(dvc.cache.local.cache_dir)
dvc._reset()
show = dvc.metrics.show if typ == "metrics" else dvc.plots.show
with pytest.raises(OverlappingOutputPathsError):
show()
+11
View File
@@ -1,7 +1,9 @@
import os
import shutil
import pytest
from dvc.exceptions import OutputDuplicationError
from dvc.repo import NotDvcRepoError, Repo, locked
from dvc.utils.fs import remove
@@ -29,6 +31,15 @@ def test_find_outs_by_path(tmp_dir, dvc, path):
assert outs[0].path_info == stage.outs[0].path_info
def test_find_outs_by_path_does_graph_checks(tmp_dir, dvc):
tmp_dir.dvc_gen("foo", "foo")
shutil.copyfile("foo.dvc", "foo-2.dvc")
dvc._reset()
with pytest.raises(OutputDuplicationError):
dvc.find_outs_by_path("foo")
@pytest.mark.parametrize(
"path",
[os.path.join("dir", "subdir", "file"), os.path.join("dir", "subdir")],