mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 15:28:28 +01:00
params, plots, metrics: unify collection (#4603)
* params, plots, metrics: unify collection * collect: split into smaller parts
This commit is contained in:
@@ -0,0 +1,74 @@
|
||||
import logging
|
||||
import os
|
||||
from typing import Iterable
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import Repo
|
||||
from dvc.tree.repo import RepoTree
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _collect_outs(
|
||||
repo: Repo, output_filter: callable = None, deps: bool = False
|
||||
):
|
||||
outs = {
|
||||
out
|
||||
for stage in repo.stages
|
||||
for out in (stage.deps if deps else stage.outs)
|
||||
}
|
||||
return set(filter(output_filter, outs)) if output_filter else outs
|
||||
|
||||
|
||||
def _collect_paths(
|
||||
repo: Repo, targets: Iterable, recursive: bool = False, rev: str = None
|
||||
):
|
||||
path_infos = {PathInfo(os.path.abspath(target)) for target in targets}
|
||||
tree = RepoTree(repo)
|
||||
|
||||
target_infos = set()
|
||||
for path_info in path_infos:
|
||||
|
||||
if recursive and tree.isdir(path_info):
|
||||
target_infos.update(set(tree.walk_files(path_info)))
|
||||
|
||||
if not tree.isfile(path_info):
|
||||
if not recursive:
|
||||
logger.warning(
|
||||
"'%s' was not found at: '%s'.", path_info, rev,
|
||||
)
|
||||
continue
|
||||
target_infos.add(path_info)
|
||||
return target_infos
|
||||
|
||||
|
||||
def _filter_duplicates(outs: Iterable, path_infos: Iterable):
|
||||
res_outs = set()
|
||||
res_infos = set(path_infos)
|
||||
|
||||
for out in outs:
|
||||
if out.path_info in path_infos:
|
||||
res_outs.add(out)
|
||||
res_infos.remove(out.path_info)
|
||||
|
||||
return res_outs, res_infos
|
||||
|
||||
|
||||
def collect(
|
||||
repo: Repo,
|
||||
deps: bool = False,
|
||||
targets: Iterable = None,
|
||||
output_filter: callable = None,
|
||||
rev: str = None,
|
||||
recursive: bool = False,
|
||||
):
|
||||
assert targets or output_filter
|
||||
|
||||
outs = _collect_outs(repo, output_filter=output_filter, deps=deps)
|
||||
|
||||
if not targets:
|
||||
return outs, set()
|
||||
|
||||
target_infos = _collect_paths(repo, targets, recursive=recursive, rev=rev)
|
||||
|
||||
return _filter_duplicates(outs, target_infos)
|
||||
@@ -23,7 +23,7 @@ def _collect_experiment(repo, rev, stash=False, sha_only=True):
|
||||
commit = _resolve_commit(repo, rev)
|
||||
res["timestamp"] = datetime.fromtimestamp(commit.committed_date)
|
||||
|
||||
configs = _collect_configs(repo)
|
||||
configs = _collect_configs(repo, rev=rev)
|
||||
params = _read_params(repo, configs, rev)
|
||||
if params:
|
||||
res["params"] = params
|
||||
|
||||
@@ -1,41 +1,23 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
from dvc.exceptions import NoMetricsError
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import locked
|
||||
from dvc.repo.collect import collect
|
||||
from dvc.tree.repo import RepoTree
|
||||
from dvc.utils.serialize import YAMLFileCorruptedError, load_yaml
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _is_metric(out):
|
||||
return bool(out.metric)
|
||||
|
||||
|
||||
def _collect_metrics(repo, targets, recursive):
|
||||
|
||||
if targets:
|
||||
target_infos = [
|
||||
PathInfo(os.path.abspath(target)) for target in targets
|
||||
]
|
||||
tree = RepoTree(repo)
|
||||
|
||||
rec_files = []
|
||||
if recursive:
|
||||
for target_info in target_infos:
|
||||
if tree.isdir(target_info):
|
||||
rec_files.extend(list(tree.walk_files(target_info)))
|
||||
|
||||
result = [t for t in target_infos if tree.isfile(t)]
|
||||
result.extend(rec_files)
|
||||
|
||||
return result
|
||||
|
||||
metrics = set()
|
||||
for stage in repo.stages:
|
||||
for out in stage.outs:
|
||||
if not out.metric:
|
||||
continue
|
||||
metrics.add(out.path_info)
|
||||
return list(metrics)
|
||||
metrics, path_infos = collect(
|
||||
repo, targets=targets, output_filter=_is_metric, recursive=recursive
|
||||
)
|
||||
return [m.path_info for m in metrics] + list(path_infos)
|
||||
|
||||
|
||||
def _extract_metrics(metrics, path, rev):
|
||||
|
||||
@@ -4,6 +4,7 @@ from dvc.dependency.param import ParamsDependency
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import locked
|
||||
from dvc.repo.collect import collect
|
||||
from dvc.utils.serialize import LOADERS, ParseError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -13,15 +14,14 @@ class NoParamsError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
def _collect_configs(repo):
|
||||
configs = set()
|
||||
configs.add(PathInfo(repo.root_dir) / ParamsDependency.DEFAULT_PARAMS_FILE)
|
||||
for stage in repo.stages:
|
||||
for dep in stage.deps:
|
||||
if not isinstance(dep, ParamsDependency):
|
||||
continue
|
||||
def _is_params(dep):
|
||||
return isinstance(dep, ParamsDependency)
|
||||
|
||||
configs.add(dep.path_info)
|
||||
|
||||
def _collect_configs(repo, rev):
|
||||
params, _ = collect(repo, deps=True, output_filter=_is_params, rev=rev)
|
||||
configs = {p.path_info for p in params}
|
||||
configs.add(PathInfo(repo.root_dir) / ParamsDependency.DEFAULT_PARAMS_FILE)
|
||||
return list(configs)
|
||||
|
||||
|
||||
@@ -49,7 +49,7 @@ def show(repo, revs=None):
|
||||
res = {}
|
||||
|
||||
for branch in repo.brancher(revs=revs):
|
||||
configs = _collect_configs(repo)
|
||||
configs = _collect_configs(repo, branch)
|
||||
params = _read_params(repo, configs, branch)
|
||||
|
||||
if params:
|
||||
|
||||
+10
-29
@@ -1,10 +1,9 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
from funcy import cached_property, first, project
|
||||
|
||||
from dvc.exceptions import DvcException, NoPlotsError
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo.collect import collect
|
||||
from dvc.schema import PLOT_PROPS
|
||||
from dvc.tree.repo import RepoTree
|
||||
from dvc.utils import relpath
|
||||
@@ -149,34 +148,16 @@ class Plots:
|
||||
return PlotTemplates(self.repo.dvc_dir)
|
||||
|
||||
|
||||
def _is_plot(out):
|
||||
return bool(out.plot)
|
||||
|
||||
|
||||
def _collect_plots(repo, targets=None, rev=None):
|
||||
plots = {out for stage in repo.stages for out in stage.outs if out.plot}
|
||||
|
||||
def to_result(plots):
|
||||
return {plot.path_info: _plot_props(plot) for plot in plots}
|
||||
|
||||
if not targets:
|
||||
return to_result(plots)
|
||||
|
||||
target_infos = {PathInfo(os.path.abspath(target)) for target in targets}
|
||||
|
||||
target_plots = set()
|
||||
for p in plots:
|
||||
if p.path_info in target_infos:
|
||||
target_plots.add(p)
|
||||
target_infos.remove(p.path_info)
|
||||
|
||||
tree = RepoTree(repo)
|
||||
result = to_result(target_plots)
|
||||
|
||||
for t in target_infos:
|
||||
if tree.isfile(t):
|
||||
result[t] = {}
|
||||
else:
|
||||
logger.warning(
|
||||
"'%s' was not found at: '%s'. It will not be plotted.", t, rev,
|
||||
)
|
||||
|
||||
plots, path_infos = collect(
|
||||
repo, output_filter=_is_plot, targets=targets, rev=rev
|
||||
)
|
||||
result = {plot.path_info: _plot_props(plot) for plot in plots}
|
||||
result.update({path_info: {} for path_info in path_infos})
|
||||
return result
|
||||
|
||||
|
||||
|
||||
@@ -330,10 +330,7 @@ def test_plot_even_if_metric_missing(
|
||||
caplog.clear()
|
||||
with caplog.at_level(logging.WARNING, "dvc"):
|
||||
plots = dvc.plots.show(revs=["v1", "v2"], targets=["metric.json"])
|
||||
assert (
|
||||
"'metric.json' was not found at: 'v1'. "
|
||||
"It will not be plotted." in caplog.text
|
||||
)
|
||||
assert "'metric.json' was not found at: 'v1'." in caplog.text
|
||||
|
||||
plot_content = json.loads(plots["metric.json"])
|
||||
assert plot_content["data"]["values"] == [
|
||||
|
||||
Reference in New Issue
Block a user