diff --git a/dvc/repo/collect.py b/dvc/repo/collect.py new file mode 100644 index 000000000..e0abdec09 --- /dev/null +++ b/dvc/repo/collect.py @@ -0,0 +1,74 @@ +import logging +import os +from typing import Iterable + +from dvc.path_info import PathInfo +from dvc.repo import Repo +from dvc.tree.repo import RepoTree + +logger = logging.getLogger(__name__) + + +def _collect_outs( + repo: Repo, output_filter: callable = None, deps: bool = False +): + outs = { + out + for stage in repo.stages + for out in (stage.deps if deps else stage.outs) + } + return set(filter(output_filter, outs)) if output_filter else outs + + +def _collect_paths( + repo: Repo, targets: Iterable, recursive: bool = False, rev: str = None +): + path_infos = {PathInfo(os.path.abspath(target)) for target in targets} + tree = RepoTree(repo) + + target_infos = set() + for path_info in path_infos: + + if recursive and tree.isdir(path_info): + target_infos.update(set(tree.walk_files(path_info))) + + if not tree.isfile(path_info): + if not recursive: + logger.warning( + "'%s' was not found at: '%s'.", path_info, rev, + ) + continue + target_infos.add(path_info) + return target_infos + + +def _filter_duplicates(outs: Iterable, path_infos: Iterable): + res_outs = set() + res_infos = set(path_infos) + + for out in outs: + if out.path_info in path_infos: + res_outs.add(out) + res_infos.remove(out.path_info) + + return res_outs, res_infos + + +def collect( + repo: Repo, + deps: bool = False, + targets: Iterable = None, + output_filter: callable = None, + rev: str = None, + recursive: bool = False, +): + assert targets or output_filter + + outs = _collect_outs(repo, output_filter=output_filter, deps=deps) + + if not targets: + return outs, set() + + target_infos = _collect_paths(repo, targets, recursive=recursive, rev=rev) + + return _filter_duplicates(outs, target_infos) diff --git a/dvc/repo/experiments/show.py b/dvc/repo/experiments/show.py index af63d35d0..7d66b2aeb 100644 --- a/dvc/repo/experiments/show.py +++ b/dvc/repo/experiments/show.py @@ -23,7 +23,7 @@ def _collect_experiment(repo, rev, stash=False, sha_only=True): commit = _resolve_commit(repo, rev) res["timestamp"] = datetime.fromtimestamp(commit.committed_date) - configs = _collect_configs(repo) + configs = _collect_configs(repo, rev=rev) params = _read_params(repo, configs, rev) if params: res["params"] = params diff --git a/dvc/repo/metrics/show.py b/dvc/repo/metrics/show.py index d189bd659..81a34151e 100644 --- a/dvc/repo/metrics/show.py +++ b/dvc/repo/metrics/show.py @@ -1,41 +1,23 @@ import logging -import os from dvc.exceptions import NoMetricsError -from dvc.path_info import PathInfo from dvc.repo import locked +from dvc.repo.collect import collect from dvc.tree.repo import RepoTree from dvc.utils.serialize import YAMLFileCorruptedError, load_yaml logger = logging.getLogger(__name__) +def _is_metric(out): + return bool(out.metric) + + def _collect_metrics(repo, targets, recursive): - - if targets: - target_infos = [ - PathInfo(os.path.abspath(target)) for target in targets - ] - tree = RepoTree(repo) - - rec_files = [] - if recursive: - for target_info in target_infos: - if tree.isdir(target_info): - rec_files.extend(list(tree.walk_files(target_info))) - - result = [t for t in target_infos if tree.isfile(t)] - result.extend(rec_files) - - return result - - metrics = set() - for stage in repo.stages: - for out in stage.outs: - if not out.metric: - continue - metrics.add(out.path_info) - return list(metrics) + metrics, path_infos = collect( + repo, targets=targets, output_filter=_is_metric, recursive=recursive + ) + return [m.path_info for m in metrics] + list(path_infos) def _extract_metrics(metrics, path, rev): diff --git a/dvc/repo/params/show.py b/dvc/repo/params/show.py index 7b2f59457..ac9459d51 100644 --- a/dvc/repo/params/show.py +++ b/dvc/repo/params/show.py @@ -4,6 +4,7 @@ from dvc.dependency.param import ParamsDependency from dvc.exceptions import DvcException from dvc.path_info import PathInfo from dvc.repo import locked +from dvc.repo.collect import collect from dvc.utils.serialize import LOADERS, ParseError logger = logging.getLogger(__name__) @@ -13,15 +14,14 @@ class NoParamsError(DvcException): pass -def _collect_configs(repo): - configs = set() - configs.add(PathInfo(repo.root_dir) / ParamsDependency.DEFAULT_PARAMS_FILE) - for stage in repo.stages: - for dep in stage.deps: - if not isinstance(dep, ParamsDependency): - continue +def _is_params(dep): + return isinstance(dep, ParamsDependency) - configs.add(dep.path_info) + +def _collect_configs(repo, rev): + params, _ = collect(repo, deps=True, output_filter=_is_params, rev=rev) + configs = {p.path_info for p in params} + configs.add(PathInfo(repo.root_dir) / ParamsDependency.DEFAULT_PARAMS_FILE) return list(configs) @@ -49,7 +49,7 @@ def show(repo, revs=None): res = {} for branch in repo.brancher(revs=revs): - configs = _collect_configs(repo) + configs = _collect_configs(repo, branch) params = _read_params(repo, configs, branch) if params: diff --git a/dvc/repo/plots/__init__.py b/dvc/repo/plots/__init__.py index c5bbe6fd5..2748baa5b 100644 --- a/dvc/repo/plots/__init__.py +++ b/dvc/repo/plots/__init__.py @@ -1,10 +1,9 @@ import logging -import os from funcy import cached_property, first, project from dvc.exceptions import DvcException, NoPlotsError -from dvc.path_info import PathInfo +from dvc.repo.collect import collect from dvc.schema import PLOT_PROPS from dvc.tree.repo import RepoTree from dvc.utils import relpath @@ -149,34 +148,16 @@ class Plots: return PlotTemplates(self.repo.dvc_dir) +def _is_plot(out): + return bool(out.plot) + + def _collect_plots(repo, targets=None, rev=None): - plots = {out for stage in repo.stages for out in stage.outs if out.plot} - - def to_result(plots): - return {plot.path_info: _plot_props(plot) for plot in plots} - - if not targets: - return to_result(plots) - - target_infos = {PathInfo(os.path.abspath(target)) for target in targets} - - target_plots = set() - for p in plots: - if p.path_info in target_infos: - target_plots.add(p) - target_infos.remove(p.path_info) - - tree = RepoTree(repo) - result = to_result(target_plots) - - for t in target_infos: - if tree.isfile(t): - result[t] = {} - else: - logger.warning( - "'%s' was not found at: '%s'. It will not be plotted.", t, rev, - ) - + plots, path_infos = collect( + repo, output_filter=_is_plot, targets=targets, rev=rev + ) + result = {plot.path_info: _plot_props(plot) for plot in plots} + result.update({path_info: {} for path_info in path_infos}) return result diff --git a/tests/func/plots/test_plots.py b/tests/func/plots/test_plots.py index cdb478629..0d5f22f56 100644 --- a/tests/func/plots/test_plots.py +++ b/tests/func/plots/test_plots.py @@ -330,10 +330,7 @@ def test_plot_even_if_metric_missing( caplog.clear() with caplog.at_level(logging.WARNING, "dvc"): plots = dvc.plots.show(revs=["v1", "v2"], targets=["metric.json"]) - assert ( - "'metric.json' was not found at: 'v1'. " - "It will not be plotted." in caplog.text - ) + assert "'metric.json' was not found at: 'v1'." in caplog.text plot_content = json.loads(plots["metric.json"]) assert plot_content["data"]["values"] == [