mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
Refactor plots (#3994)
- separate data collection and rendering - provide a way to use custom template loader - drop hard-coding datafiles in template feature
This commit is contained in:
+182
-3
@@ -1,11 +1,80 @@
|
||||
import logging
|
||||
|
||||
from funcy import first, project
|
||||
|
||||
from dvc.exceptions import DvcException, NoPlotsError, OutputNotFoundError
|
||||
from dvc.repo.tree import RepoTree
|
||||
from dvc.schema import PLOT_PROPS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Plots:
|
||||
def __init__(self, repo):
|
||||
self.repo = repo
|
||||
|
||||
def show(self, *args, **kwargs):
|
||||
from .show import show
|
||||
def collect(self, targets=None, revs=None):
|
||||
"""Collects all props and data for plots.
|
||||
|
||||
return show(self.repo, *args, **kwargs)
|
||||
Returns a structure like:
|
||||
{rev: {plots.csv: {
|
||||
props: {x: ..., "csv_header": ..., ...},
|
||||
data: "...data as a string...",
|
||||
}}}
|
||||
Data parsing is postponed, since it's affected by props.
|
||||
"""
|
||||
targets = [targets] if isinstance(targets, str) else targets or []
|
||||
data = {}
|
||||
for rev in self.repo.brancher(revs=revs):
|
||||
# .brancher() adds unwanted workspace
|
||||
if revs is not None and rev not in revs:
|
||||
continue
|
||||
rev = rev or "workspace"
|
||||
|
||||
tree = RepoTree(self.repo)
|
||||
plots = _collect_plots(self.repo, targets, rev)
|
||||
for datafile, props in plots.items():
|
||||
data[rev] = {datafile: {"props": props}}
|
||||
|
||||
# Load data from git or dvc cache
|
||||
try:
|
||||
with tree.open(datafile) as fd:
|
||||
data[rev][datafile]["data"] = fd.read()
|
||||
except FileNotFoundError:
|
||||
# This might happen simply because cache is absent
|
||||
pass
|
||||
|
||||
return data
|
||||
|
||||
def render(self, data, revs=None, props=None, templates=None):
|
||||
"""Renders plots"""
|
||||
props = props or {}
|
||||
templates = templates or self.repo.plot_templates
|
||||
|
||||
# Merge data by plot file and apply overriding props
|
||||
plots = _prepare_plots(data, revs, props)
|
||||
|
||||
return {
|
||||
datafile: _render(datafile, desc["data"], desc["props"], templates)
|
||||
for datafile, desc in plots.items()
|
||||
}
|
||||
|
||||
def show(self, targets=None, revs=None, props=None):
|
||||
from .data import NoMetricInHistoryError
|
||||
|
||||
data = self.collect(targets, revs)
|
||||
|
||||
# If any mentioned plot doesn't have any data then that's an error
|
||||
targets = [targets] if isinstance(targets, str) else targets or []
|
||||
for target in targets:
|
||||
if not any("data" in d[target] for d in data.values()):
|
||||
raise NoMetricInHistoryError(target)
|
||||
|
||||
# No data at all is a special error with a special message
|
||||
if not data:
|
||||
raise NoPlotsError()
|
||||
|
||||
return self.render(data, revs, props)
|
||||
|
||||
def diff(self, *args, **kwargs):
|
||||
from .diff import diff
|
||||
@@ -33,3 +102,113 @@ class Plots:
|
||||
|
||||
dvcfile = Dvcfile(self.repo, out.stage.path)
|
||||
dvcfile.dump(out.stage, update_pipeline=True)
|
||||
|
||||
|
||||
def _collect_plots(repo, targets=None, rev=None):
|
||||
def _targets_to_outs(targets):
|
||||
for t in targets:
|
||||
try:
|
||||
(out,) = repo.find_outs_by_path(t)
|
||||
yield out
|
||||
except OutputNotFoundError:
|
||||
logger.warning(
|
||||
"File '{}' was not found at: '{}'. It will not be "
|
||||
"plotted.".format(t, rev)
|
||||
)
|
||||
|
||||
if targets:
|
||||
outs = _targets_to_outs(targets)
|
||||
else:
|
||||
outs = (out for stage in repo.stages for out in stage.outs if out.plot)
|
||||
|
||||
return {str(out): _plot_props(out) for out in outs}
|
||||
|
||||
|
||||
def _plot_props(out):
|
||||
if not out.plot:
|
||||
raise DvcException(
|
||||
f"'{out}' is not a plot. Use `dvc plots modify` to change that."
|
||||
)
|
||||
if isinstance(out.plot, list):
|
||||
raise DvcException("Multiple plots per data file not supported.")
|
||||
if isinstance(out.plot, bool):
|
||||
return {}
|
||||
|
||||
return project(out.plot, PLOT_PROPS)
|
||||
|
||||
|
||||
def _prepare_plots(data, revs, props):
|
||||
"""Groups data by plot file.
|
||||
|
||||
Also resolves props conflicts between revs and applies global props.
|
||||
"""
|
||||
# we go in order revs are supplied on props conflict first ones win.
|
||||
revs = iter(data) if revs is None else revs
|
||||
|
||||
plots, props_revs = {}, {}
|
||||
for rev in revs:
|
||||
# Asked for revision without data
|
||||
if rev not in data:
|
||||
continue
|
||||
|
||||
for datafile, desc in data[rev].items():
|
||||
# props from command line overwrite plot props from out definition
|
||||
full_props = {**desc["props"], **props}
|
||||
|
||||
if datafile in plots:
|
||||
saved = plots[datafile]
|
||||
if saved["props"] != full_props:
|
||||
logger.warning(
|
||||
f"Inconsistent plot props for '{datafile}' in "
|
||||
f"'{props_revs[datafile]}' and '{rev}'. "
|
||||
f"Going to use ones from '{props_revs[datafile]}'"
|
||||
)
|
||||
|
||||
saved["data"][rev] = desc["data"]
|
||||
else:
|
||||
plots[datafile] = {
|
||||
"props": full_props,
|
||||
"data": {rev: desc["data"]},
|
||||
}
|
||||
# Save rev we got props from
|
||||
props_revs[datafile] = rev
|
||||
|
||||
return plots
|
||||
|
||||
|
||||
def _render(datafile, datas, props, templates):
|
||||
from .data import plot_data, PlotData
|
||||
|
||||
# Copy it to not modify a passed value
|
||||
props = props.copy()
|
||||
|
||||
# Add x and y to fields if set
|
||||
fields = props.get("fields")
|
||||
if fields is not None:
|
||||
fields = {*fields, props.get("x"), props.get("y")} - {None}
|
||||
|
||||
template = templates.load(props.get("template") or "default")
|
||||
|
||||
# If x is not set add index field
|
||||
if not props.get("x") and template.has_anchor("x"):
|
||||
props["append_index"] = True
|
||||
props["x"] = PlotData.INDEX_FIELD
|
||||
|
||||
# Parse all data, preprocess it and collect as a list of dicts
|
||||
data = []
|
||||
for rev, datablob in datas.items():
|
||||
rev_data = plot_data(datafile, rev, datablob).to_datapoints(
|
||||
fields=fields,
|
||||
path=props.get("path"),
|
||||
csv_header=props.get("csv_header", True),
|
||||
append_index=props.get("append_index", False),
|
||||
)
|
||||
data.extend(rev_data)
|
||||
|
||||
# If y is not set then use last field not used yet
|
||||
if not props.get("y") and template.has_anchor("y"):
|
||||
fields = list(first(data))
|
||||
skip = (PlotData.REVISION_FIELD, props.get("x"))
|
||||
props["y"] = first(f for f in reversed(fields) if f not in skip)
|
||||
|
||||
return template.render(data, props=props)
|
||||
|
||||
+2
-64
@@ -1,7 +1,6 @@
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from collections import OrderedDict
|
||||
from copy import copy
|
||||
@@ -10,9 +9,7 @@ import yaml
|
||||
from funcy import first
|
||||
from yaml import SafeLoader
|
||||
|
||||
from dvc.exceptions import DvcException, PathMissingError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
class PlotMetricTypeError(DvcException):
|
||||
@@ -31,21 +28,6 @@ class PlotDataStructureError(DvcException):
|
||||
)
|
||||
|
||||
|
||||
class JsonParsingError(DvcException):
|
||||
def __init__(self, file):
|
||||
super().__init__(
|
||||
"Failed to infer data structure from '{}'. Did you forget "
|
||||
"to specify JSONpath?".format(file)
|
||||
)
|
||||
|
||||
|
||||
class NoMetricOnRevisionError(DvcException):
|
||||
def __init__(self, path, revision):
|
||||
self.path = path
|
||||
self.revision = revision
|
||||
super().__init__(f"Could not find '{path}' on revision '{revision}'")
|
||||
|
||||
|
||||
class NoMetricInHistoryError(DvcException):
|
||||
def __init__(self, path):
|
||||
super().__init__(f"Could not find '{path}'.")
|
||||
@@ -77,7 +59,7 @@ def _filter_fields(data_points, filename, revision, fields=None, **kwargs):
|
||||
if keys & fields != fields:
|
||||
raise DvcException(
|
||||
"Could not find fields: '{}' for '{}' at '{}'.".format(
|
||||
", " "".join(fields), filename, revision
|
||||
", ".join(fields), filename, revision
|
||||
)
|
||||
)
|
||||
|
||||
@@ -240,47 +222,3 @@ class YAMLPlotData(PlotData):
|
||||
def _processors(self):
|
||||
parent_processors = super()._processors()
|
||||
return [_find_data] + parent_processors
|
||||
|
||||
|
||||
def _load_from_revision(repo, datafile, revision):
|
||||
from dvc.repo.tree import RepoTree
|
||||
|
||||
tree = RepoTree(repo)
|
||||
|
||||
try:
|
||||
with tree.open(datafile) as fobj:
|
||||
datafile_content = fobj.read()
|
||||
|
||||
except (FileNotFoundError, PathMissingError):
|
||||
raise NoMetricOnRevisionError(datafile, revision)
|
||||
|
||||
return plot_data(datafile, revision, datafile_content)
|
||||
|
||||
|
||||
def _load_from_revisions(repo, datafile, revisions):
|
||||
data = []
|
||||
exceptions = []
|
||||
|
||||
for rev in repo.brancher(revs=revisions):
|
||||
if rev == "workspace" and rev not in revisions:
|
||||
continue
|
||||
|
||||
try:
|
||||
data.append(_load_from_revision(repo, datafile, rev))
|
||||
except NoMetricOnRevisionError as e:
|
||||
exceptions.append(e)
|
||||
except PlotMetricTypeError:
|
||||
raise
|
||||
except (yaml.error.YAMLError, json.decoder.JSONDecodeError, csv.Error):
|
||||
logger.error(f"Failed to parse '{datafile}' at '{rev}'.")
|
||||
raise
|
||||
|
||||
if not data and exceptions:
|
||||
raise NoMetricInHistoryError(datafile)
|
||||
else:
|
||||
for e in exceptions:
|
||||
logger.warning(
|
||||
"File '{}' was not found at: '{}'. It will not be "
|
||||
"plotted.".format(e.path, e.revision)
|
||||
)
|
||||
return data
|
||||
|
||||
@@ -1,217 +0,0 @@
|
||||
import copy
|
||||
import logging
|
||||
import os
|
||||
|
||||
from funcy import first, last, project
|
||||
|
||||
from dvc.exceptions import DvcException, NoPlotsError
|
||||
from dvc.repo import locked
|
||||
from dvc.schema import PLOT_PROPS
|
||||
|
||||
from .data import NoMetricInHistoryError, PlotData
|
||||
from .template import NoDataForTemplateError, Template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TooManyDataSourcesError(DvcException):
|
||||
def __init__(self, datafile, template_datafiles):
|
||||
super().__init__(
|
||||
"Unable to infer which of possible data sources: '{}' "
|
||||
"should be replaced with '{}'.".format(
|
||||
", ".join(template_datafiles), datafile
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class NoDataOrTemplateProvided(DvcException):
|
||||
def __init__(self):
|
||||
super().__init__("Datafile or template is not specified.")
|
||||
|
||||
|
||||
class TooManyTemplatesError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
def _evaluate_templatepath(repo, template=None):
|
||||
if not template:
|
||||
return repo.plot_templates.default_template
|
||||
|
||||
if os.path.exists(template):
|
||||
return template
|
||||
return repo.plot_templates.get_template(template)
|
||||
|
||||
|
||||
@locked
|
||||
def fill_template(
|
||||
repo, datafile, template_path, revisions, props,
|
||||
):
|
||||
# Copy things to not modify passed values
|
||||
props = props.copy()
|
||||
fields = copy.copy(props.get("fields"))
|
||||
|
||||
if props.get("x") and fields:
|
||||
fields.add(props.get("x"))
|
||||
|
||||
if props.get("y") and fields:
|
||||
fields.add(props.get("y"))
|
||||
|
||||
template_datafiles, x_anchor, y_anchor = _parse_template(
|
||||
template_path, datafile
|
||||
)
|
||||
append_index = x_anchor and not props.get("x")
|
||||
if append_index:
|
||||
props["x"] = PlotData.INDEX_FIELD
|
||||
|
||||
template_data = {}
|
||||
for template_datafile in template_datafiles:
|
||||
from .data import _load_from_revisions
|
||||
|
||||
plot_datas = _load_from_revisions(repo, template_datafile, revisions)
|
||||
tmp_data = []
|
||||
for pd in plot_datas:
|
||||
rev_data_points = pd.to_datapoints(
|
||||
fields=fields,
|
||||
path=props.get("path"),
|
||||
csv_header=props.get("csv_header", True),
|
||||
append_index=append_index,
|
||||
)
|
||||
|
||||
if y_anchor and not props.get("y"):
|
||||
props["y"] = _infer_y_field(rev_data_points, props.get("x"))
|
||||
tmp_data.extend(rev_data_points)
|
||||
|
||||
template_data[template_datafile] = tmp_data
|
||||
|
||||
if len(template_data) == 0:
|
||||
raise NoDataForTemplateError(template_path)
|
||||
|
||||
content = Template.fill(
|
||||
template_path, template_data, priority_datafile=datafile, props=props,
|
||||
)
|
||||
|
||||
path = datafile or ",".join(template_datafiles)
|
||||
|
||||
return path, content
|
||||
|
||||
|
||||
def _infer_y_field(rev_data_points, x_field):
|
||||
all_fields = list(first(rev_data_points).keys())
|
||||
all_fields.remove(PlotData.REVISION_FIELD)
|
||||
if x_field and x_field in all_fields:
|
||||
all_fields.remove(x_field)
|
||||
y_field = last(all_fields)
|
||||
return y_field
|
||||
|
||||
|
||||
def _show(repo, datafile=None, revs=None, props=None):
|
||||
if revs is None:
|
||||
revs = ["workspace"]
|
||||
|
||||
if props is None:
|
||||
props = {}
|
||||
|
||||
if not datafile and not props.get("template"):
|
||||
raise NoDataOrTemplateProvided()
|
||||
|
||||
template_path = _evaluate_templatepath(repo, props.get("template"))
|
||||
|
||||
plot_datafile, plot_content = fill_template(
|
||||
repo, datafile, template_path, revs, props
|
||||
)
|
||||
|
||||
return plot_datafile, plot_content
|
||||
|
||||
|
||||
def _parse_template(template_path, priority_datafile):
|
||||
with open(template_path) as fobj:
|
||||
tempalte_content = fobj.read()
|
||||
|
||||
template_datafiles = Template.parse_data_anchors(tempalte_content)
|
||||
if priority_datafile:
|
||||
if len(template_datafiles) > 1:
|
||||
raise TooManyDataSourcesError(
|
||||
priority_datafile, template_datafiles
|
||||
)
|
||||
template_datafiles = {priority_datafile}
|
||||
|
||||
return (
|
||||
template_datafiles,
|
||||
Template.X_ANCHOR in tempalte_content,
|
||||
Template.Y_ANCHOR in tempalte_content,
|
||||
)
|
||||
|
||||
|
||||
def _collect_plots(repo, targets=None):
|
||||
from dvc.exceptions import OutputNotFoundError
|
||||
from contextlib import suppress
|
||||
|
||||
def _targets_to_outs(targets):
|
||||
for t in targets:
|
||||
with suppress(OutputNotFoundError):
|
||||
(out,) = repo.find_outs_by_path(t)
|
||||
yield out
|
||||
|
||||
if targets:
|
||||
outs = _targets_to_outs(targets)
|
||||
else:
|
||||
outs = (out for stage in repo.stages for out in stage.outs if out.plot)
|
||||
|
||||
return {str(out): _plot_props(out) for out in outs}
|
||||
|
||||
|
||||
def _plot_props(out):
|
||||
if not out.plot:
|
||||
raise DvcException(
|
||||
f"'{out}' is not a plot. Use `dvc plots modify` to change that."
|
||||
)
|
||||
if isinstance(out.plot, list):
|
||||
raise DvcException("Multiple plots per data file not supported yet.")
|
||||
if isinstance(out.plot, bool):
|
||||
return {}
|
||||
|
||||
return project(out.plot, PLOT_PROPS)
|
||||
|
||||
|
||||
def show(repo, targets=None, revs=None, props=None) -> dict:
|
||||
if isinstance(targets, str):
|
||||
targets = [targets]
|
||||
if props is None:
|
||||
props = {}
|
||||
|
||||
# Collect plot data files with associated props
|
||||
plots = {}
|
||||
for rev in repo.brancher(revs=revs):
|
||||
if revs is not None and rev not in revs:
|
||||
continue
|
||||
|
||||
for datafile, file_props in _collect_plots(repo, targets).items():
|
||||
# props from command line overwrite plot props from out definition
|
||||
full_props = {**file_props, **props}
|
||||
|
||||
if datafile in plots:
|
||||
saved_rev, saved_props = plots[datafile]
|
||||
if saved_props != props:
|
||||
logger.warning(
|
||||
f"Inconsistent plot props for '{datafile}' in "
|
||||
f"'{saved_rev}' and '{rev}'. "
|
||||
f"Going to use ones from '{saved_rev}'"
|
||||
)
|
||||
else:
|
||||
plots[datafile] = rev, full_props
|
||||
|
||||
if not plots:
|
||||
if targets:
|
||||
raise NoMetricInHistoryError(", ".join(targets))
|
||||
|
||||
try:
|
||||
datafile, plot = _show(repo, datafile=None, revs=revs, props=props)
|
||||
except NoDataOrTemplateProvided:
|
||||
raise NoPlotsError()
|
||||
|
||||
return {datafile: plot}
|
||||
|
||||
return {
|
||||
datafile: _show(repo, datafile=datafile, revs=revs, props=props)[1]
|
||||
for datafile, (_, props) in plots.items()
|
||||
}
|
||||
+59
-112
@@ -1,28 +1,17 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
|
||||
from funcy import cached_property
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.utils.fs import makedirs
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TemplateNotFoundError(DvcException):
|
||||
def __init__(self, path):
|
||||
super().__init__(f"Template '{path}' not found.")
|
||||
|
||||
|
||||
class NoDataForTemplateError(DvcException):
|
||||
def __init__(self, template_path):
|
||||
super().__init__(
|
||||
"No data provided for '{}'.".format(os.path.relpath(template_path))
|
||||
)
|
||||
|
||||
|
||||
class NoFieldInDataError(DvcException):
|
||||
def __init__(self, field_name):
|
||||
super().__init__(
|
||||
@@ -34,6 +23,7 @@ class Template:
|
||||
INDENT = 4
|
||||
SEPARATORS = (",", ": ")
|
||||
EXTENSION = ".json"
|
||||
ANCHOR = '"<DVC_METRIC_{}>"'
|
||||
METRIC_DATA_ANCHOR = "<DVC_METRIC_DATA>"
|
||||
X_ANCHOR = "<DVC_METRIC_X>"
|
||||
Y_ANCHOR = "<DVC_METRIC_Y>"
|
||||
@@ -41,125 +31,60 @@ class Template:
|
||||
X_LABEL_ANCHOR = "<DVC_METRIC_X_LABEL>"
|
||||
Y_LABEL_ANCHOR = "<DVC_METRIC_Y_LABEL>"
|
||||
|
||||
def __init__(self, templates_dir):
|
||||
self.plot_templates_dir = templates_dir
|
||||
def __init__(self, content=None, name=None):
|
||||
self.content = self.DEFAULT_CONTENT if content is None else content
|
||||
self.name = name or self.DEFAULT_NAME
|
||||
self.filename = self.name + self.EXTENSION
|
||||
|
||||
def dump(self):
|
||||
makedirs(self.plot_templates_dir, exist_ok=True)
|
||||
|
||||
with open(
|
||||
os.path.join(
|
||||
self.plot_templates_dir, self.TEMPLATE_NAME + self.EXTENSION
|
||||
),
|
||||
"w",
|
||||
) as fobj:
|
||||
json.dump(
|
||||
self.DEFAULT_CONTENT,
|
||||
fobj,
|
||||
indent=self.INDENT,
|
||||
separators=self.SEPARATORS,
|
||||
)
|
||||
fobj.write("\n")
|
||||
|
||||
@staticmethod
|
||||
def get_data_anchor(template_content):
|
||||
regex = re.compile('"<DVC_METRIC_DATA[^>"]*>"')
|
||||
return regex.findall(template_content)
|
||||
|
||||
@staticmethod
|
||||
def parse_data_anchors(template_content):
|
||||
data_files = {
|
||||
Template.get_datafile(m)
|
||||
for m in Template.get_data_anchor(template_content)
|
||||
}
|
||||
return {df for df in data_files if df}
|
||||
|
||||
@staticmethod
|
||||
def get_datafile(anchor_string):
|
||||
return (
|
||||
anchor_string.replace("<", "")
|
||||
.replace(">", "")
|
||||
.replace('"', "")
|
||||
.replace("DVC_METRIC_DATA", "")
|
||||
.replace(",", "")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def fill(
|
||||
template_path, data, priority_datafile=None, props=None,
|
||||
):
|
||||
def render(self, data, props=None):
|
||||
props = props or {}
|
||||
|
||||
with open(template_path) as fobj:
|
||||
result_content = fobj.read()
|
||||
|
||||
if props.get("x"):
|
||||
Template._check_field_exists(data, props.get("x"))
|
||||
if props.get("y"):
|
||||
Template._check_field_exists(data, props.get("y"))
|
||||
|
||||
result_content = Template._replace_data_anchors(
|
||||
result_content, data, priority_datafile
|
||||
content = self._fill_anchor(self.content, "data", data)
|
||||
content = self._fill_metadata(content, props)
|
||||
|
||||
return content
|
||||
|
||||
def has_anchor(self, name):
|
||||
return self._anchor(name) in self.content
|
||||
|
||||
@classmethod
|
||||
def _fill_anchor(cls, content, name, value):
|
||||
value_str = json.dumps(
|
||||
value, indent=cls.INDENT, separators=cls.SEPARATORS, sort_keys=True
|
||||
)
|
||||
return content.replace(cls._anchor(name), value_str)
|
||||
|
||||
result_content = Template._replace_metadata_anchors(
|
||||
result_content, props
|
||||
)
|
||||
@classmethod
|
||||
def _anchor(cls, name):
|
||||
return cls.ANCHOR.format(name.upper())
|
||||
|
||||
return result_content
|
||||
|
||||
@staticmethod
|
||||
def _check_field_exists(data, field):
|
||||
for file, data_points in data.items():
|
||||
if not any(
|
||||
field in data_point.keys() for data_point in data_points
|
||||
):
|
||||
raise NoFieldInDataError(field)
|
||||
|
||||
@staticmethod
|
||||
def _replace_metadata_anchors(result_content, props):
|
||||
@classmethod
|
||||
def _fill_metadata(cls, content, props):
|
||||
props.setdefault("title", "")
|
||||
props.setdefault("x_label", props.get("x"))
|
||||
props.setdefault("y_label", props.get("y"))
|
||||
|
||||
replace_pairs = [
|
||||
(Template.TITLE_ANCHOR, "title"),
|
||||
(Template.X_ANCHOR, "x"),
|
||||
(Template.Y_ANCHOR, "y"),
|
||||
(Template.X_LABEL_ANCHOR, "x_label"),
|
||||
(Template.Y_LABEL_ANCHOR, "y_label"),
|
||||
]
|
||||
for anchor, key in replace_pairs:
|
||||
value = props.get(key)
|
||||
if anchor in result_content and value is not None:
|
||||
result_content = result_content.replace(anchor, value)
|
||||
names = ["title", "x", "y", "x_label", "y_label"]
|
||||
for name in names:
|
||||
value = props.get(name)
|
||||
if value is not None:
|
||||
content = cls._fill_anchor(content, name, value)
|
||||
|
||||
return result_content
|
||||
return content
|
||||
|
||||
@staticmethod
|
||||
def _replace_data_anchors(result_content, data, priority_datafile):
|
||||
for anchor in Template.get_data_anchor(result_content):
|
||||
file = Template.get_datafile(anchor)
|
||||
|
||||
if not file or priority_datafile:
|
||||
key = priority_datafile
|
||||
else:
|
||||
key = file
|
||||
|
||||
result_content = result_content.replace(
|
||||
anchor,
|
||||
json.dumps(
|
||||
data[key],
|
||||
indent=Template.INDENT,
|
||||
separators=Template.SEPARATORS,
|
||||
sort_keys=True,
|
||||
),
|
||||
)
|
||||
return result_content
|
||||
def _check_field_exists(data, field):
|
||||
if not any(field in row for row in data):
|
||||
raise NoFieldInDataError(field)
|
||||
|
||||
|
||||
class DefaultLinearTemplate(Template):
|
||||
TEMPLATE_NAME = "default"
|
||||
DEFAULT_NAME = "default"
|
||||
|
||||
DEFAULT_CONTENT = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
@@ -184,7 +109,7 @@ class DefaultLinearTemplate(Template):
|
||||
|
||||
|
||||
class DefaultConfusionTemplate(Template):
|
||||
TEMPLATE_NAME = "confusion"
|
||||
DEFAULT_NAME = "confusion"
|
||||
DEFAULT_CONTENT = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
"data": {"values": Template.METRIC_DATA_ANCHOR},
|
||||
@@ -210,7 +135,7 @@ class DefaultConfusionTemplate(Template):
|
||||
|
||||
|
||||
class DefaultScatterTemplate(Template):
|
||||
TEMPLATE_NAME = "scatter"
|
||||
DEFAULT_NAME = "scatter"
|
||||
DEFAULT_CONTENT = {
|
||||
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
|
||||
"data": {"values": Template.METRIC_DATA_ANCHOR},
|
||||
@@ -253,6 +178,9 @@ class PlotTemplates:
|
||||
return default_plot_path
|
||||
|
||||
def get_template(self, path):
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
|
||||
t_path = os.path.join(self.templates_dir, path)
|
||||
if os.path.exists(t_path):
|
||||
return t_path
|
||||
@@ -279,4 +207,23 @@ class PlotTemplates:
|
||||
if not os.path.exists(self.templates_dir):
|
||||
makedirs(self.templates_dir, exist_ok=True)
|
||||
for t in self.TEMPLATES:
|
||||
t(self.templates_dir).dump()
|
||||
self.dump(t())
|
||||
|
||||
def dump(self, template):
|
||||
path = os.path.join(self.templates_dir, template.filename)
|
||||
with open(path, "w") as fd:
|
||||
json.dump(
|
||||
template.content,
|
||||
fd,
|
||||
indent=template.INDENT,
|
||||
separators=template.SEPARATORS,
|
||||
)
|
||||
fd.write("\n")
|
||||
|
||||
def load(self, name):
|
||||
path = self.get_template(name)
|
||||
|
||||
with open(path) as fd:
|
||||
content = fd.read()
|
||||
|
||||
return Template(content, name=name)
|
||||
|
||||
@@ -32,10 +32,10 @@ def test_diff_dirty(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
|
||||
plot_content = json.loads(plot_string)
|
||||
assert plot_content["data"]["values"] == [
|
||||
{"y": 5, PlotData.INDEX_FIELD: 0, "rev": "workspace"},
|
||||
{"y": 6, PlotData.INDEX_FIELD: 1, "rev": "workspace"},
|
||||
{"y": 3, PlotData.INDEX_FIELD: 0, "rev": "HEAD"},
|
||||
{"y": 5, PlotData.INDEX_FIELD: 1, "rev": "HEAD"},
|
||||
{"y": 5, PlotData.INDEX_FIELD: 0, "rev": "workspace"},
|
||||
{"y": 6, PlotData.INDEX_FIELD: 1, "rev": "workspace"},
|
||||
]
|
||||
assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
|
||||
assert plot_content["encoding"]["y"]["field"] == "y"
|
||||
|
||||
@@ -16,11 +16,7 @@ from dvc.repo.plots.data import (
|
||||
PlotMetricTypeError,
|
||||
YAMLPlotData,
|
||||
)
|
||||
from dvc.repo.plots.template import (
|
||||
NoDataForTemplateError,
|
||||
NoFieldInDataError,
|
||||
TemplateNotFoundError,
|
||||
)
|
||||
from dvc.repo.plots.template import NoFieldInDataError, TemplateNotFoundError
|
||||
|
||||
|
||||
def _write_csv(metric, filename, header=True):
|
||||
@@ -262,7 +258,11 @@ def test_plot_multiple_revs_default(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
|
||||
|
||||
def test_plot_multiple_revs(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
shutil.copy(tmp_dir / ".dvc" / "plots" / "default.json", "template.json")
|
||||
templates_dir = dvc.plot_templates.templates_dir
|
||||
shutil.copy(
|
||||
os.path.join(templates_dir, "default.json"),
|
||||
os.path.join(templates_dir, "template.json"),
|
||||
)
|
||||
|
||||
metric_1 = [{"y": 2}, {"y": 3}]
|
||||
_write_json(tmp_dir, metric_1, "metric_t.json")
|
||||
@@ -323,13 +323,13 @@ def test_plot_even_if_metric_missing(
|
||||
|
||||
caplog.clear()
|
||||
with caplog.at_level(logging.WARNING, "dvc"):
|
||||
plot_string = dvc.plots.show(revs=["v1", "v2"])["metric.json"]
|
||||
plots = dvc.plots.show(revs=["v1", "v2"], targets=["metric.json"])
|
||||
assert (
|
||||
"File 'metric.json' was not found at: 'v1'. "
|
||||
"It will not be plotted." in caplog.text
|
||||
)
|
||||
|
||||
plot_content = json.loads(plot_string)
|
||||
plot_content = json.loads(plots["metric.json"])
|
||||
assert plot_content["data"]["values"] == [
|
||||
{"y": 2, PlotData.INDEX_FIELD: 0, "rev": "v2"},
|
||||
{"y": 3, PlotData.INDEX_FIELD: 1, "rev": "v2"},
|
||||
@@ -392,72 +392,6 @@ def _replace(path, src, dst):
|
||||
path.write_text(path.read_text().replace(src, dst))
|
||||
|
||||
|
||||
def test_custom_template_with_specified_data(
|
||||
tmp_dir, scm, dvc, custom_template, run_copy_metrics
|
||||
):
|
||||
_replace(
|
||||
custom_template, "DVC_METRIC_DATA", "DVC_METRIC_DATA,metric.json",
|
||||
)
|
||||
|
||||
metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
outs_no_cache=["metric.json"],
|
||||
commit="init",
|
||||
tag="v1",
|
||||
)
|
||||
|
||||
props = {"template": os.fspath(custom_template), "x": "a", "y": "b"}
|
||||
plot_string = dvc.plots.show(props=props)["metric.json"]
|
||||
|
||||
plot_content = json.loads(plot_string)
|
||||
assert plot_content["data"]["values"] == [
|
||||
{"a": 1, "b": 2, "rev": "workspace"},
|
||||
{"a": 2, "b": 3, "rev": "workspace"},
|
||||
]
|
||||
assert plot_content["encoding"]["x"]["field"] == "a"
|
||||
assert plot_content["encoding"]["y"]["field"] == "b"
|
||||
|
||||
|
||||
def test_plot_override_specified_data_source(
|
||||
tmp_dir, scm, dvc, run_copy_metrics
|
||||
):
|
||||
shutil.copy(
|
||||
tmp_dir / ".dvc" / "plots" / "default.json",
|
||||
tmp_dir / "newtemplate.json",
|
||||
)
|
||||
_replace(
|
||||
tmp_dir / "newtemplate.json",
|
||||
"DVC_METRIC_DATA",
|
||||
"DVC_METRIC_DATA,metric.json",
|
||||
)
|
||||
|
||||
metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
_write_json(tmp_dir, metric, "metric1.json")
|
||||
run_copy_metrics(
|
||||
"metric1.json",
|
||||
"metric2.json",
|
||||
plots_no_cache=["metric2.json"],
|
||||
commit="init",
|
||||
tag="v1",
|
||||
)
|
||||
|
||||
props = {"template": "newtemplate.json", "x": "a"}
|
||||
plot_string = dvc.plots.show(targets=["metric2.json"], props=props)[
|
||||
"metric2.json"
|
||||
]
|
||||
|
||||
plot_content = json.loads(plot_string)
|
||||
assert plot_content["data"]["values"] == [
|
||||
{"a": 1, "b": 2, "rev": "workspace"},
|
||||
{"a": 2, "b": 3, "rev": "workspace"},
|
||||
]
|
||||
assert plot_content["encoding"]["x"]["field"] == "a"
|
||||
assert plot_content["encoding"]["y"]["field"] == "b"
|
||||
|
||||
|
||||
def test_no_plots(tmp_dir, dvc):
|
||||
from dvc.exceptions import NoPlotsError
|
||||
|
||||
@@ -480,11 +414,6 @@ def test_should_raise_on_no_template(tmp_dir, dvc, run_copy_metrics):
|
||||
dvc.plots.show("metric.json", props=props)
|
||||
|
||||
|
||||
def test_plot_no_data(tmp_dir, dvc):
|
||||
with pytest.raises(NoDataForTemplateError):
|
||||
dvc.plots.show(props={"template": "default"})
|
||||
|
||||
|
||||
def test_plot_wrong_metric_type(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
tmp_dir.gen("metric_t.txt", "some text")
|
||||
run_copy_metrics(
|
||||
|
||||
@@ -72,13 +72,13 @@ def test_metrics_show(dvc, mocker):
|
||||
cmd = cli_args.func(cli_args)
|
||||
|
||||
m = mocker.patch(
|
||||
"dvc.repo.plots.show.show", return_value={"datafile": "filledtemplate"}
|
||||
"dvc.repo.plots.Plots.show",
|
||||
return_value={"datafile": "filledtemplate"},
|
||||
)
|
||||
|
||||
assert cmd.run() == 0
|
||||
|
||||
m.assert_called_once_with(
|
||||
cmd.repo,
|
||||
targets=["datafile"],
|
||||
props={"template": "template", "csv_header": False},
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user