render: Extract to dvc_render

Create and use separate package for rendering logic.
Closes #6944

dvc/commands/experiments/show.py

@@ -432,14 +432,15 @@ def show_experiments(
             for x in td.column("Experiment")
         ]
         out = kwargs.get("out") or "dvc_plots"
+        output_file = os.path.join(out, "index.html")
         ui.write(
             td.to_parallel_coordinates(
-                output_path=os.path.abspath(out),
+                output_path=os.path.abspath(output_file),
                 color_by=kwargs.get("sort_by") or "Experiment",
             )
         )
         if kwargs.get("open"):
-            return ui.open_browser(os.path.join(out, "index.html"))
+            return ui.open_browser(output_file)
 
     else:
         td.render(

dvc/commands/live.py

@@ -10,17 +10,21 @@ class CmdLive(CmdBase):
     UNINITIALIZED = True
 
     def _run(self, target, revs=None):
-        from dvc.render.utils import match_renderers, render
+        from dvc_render import render_html
+
+        from dvc.render.match import match_renderers
 
         metrics, plots = self.repo.live.show(target=target, revs=revs)
 
         if plots:
             from pathlib import Path
 
-            html_path = Path.cwd() / (self.args.target + "_html")
+            output = Path.cwd() / (self.args.target + "_html") / "index.html"
 
-            renderers = match_renderers(plots, self.repo.plots.templates)
-            index_path = render(self.repo, renderers, metrics, html_path)
+            renderers = match_renderers(
+                plots, templates_dir=self.repo.plots.templates_dir
+            )
+            index_path = render_html(renderers, output, metrics)
             ui.write(index_path.as_uri())
             return 0
         return 1

dvc/commands/plots.py

@@ -1,6 +1,7 @@
 import argparse
 import json
 import logging
+import os
 
 from funcy import first
 
@@ -14,15 +15,11 @@ from dvc.utils import format_link
 logger = logging.getLogger(__name__)
 
 
-def _show_json(renderers, path: None, split=False):
-    if any(r.needs_output_path for r in renderers) and not path:
-        raise DvcException("Output path ('-o') is required!")
+def _show_json(renderers, split=False):
+    from dvc.render.convert import to_json
 
     result = {
-        renderer.filename: json.loads(
-            renderer.as_json(path=path, fill_data=not split)
-        )
-        for renderer in renderers
+        renderer.name: to_json(renderer, split) for renderer in renderers
     }
     if result:
         ui.write_json(result)
@@ -42,8 +39,9 @@ class CmdPlots(CmdBase):
     def run(self):
         from pathlib import Path
 
-        from dvc.render.utils import match_renderers, render
-        from dvc.render.vega import VegaRenderer
+        from dvc_render import render_html
+
+        from dvc.render.match import match_renderers
 
         if self.args.show_vega:
             if not self.args.targets:
@@ -74,31 +72,40 @@ class CmdPlots(CmdBase):
                 )
 
             renderers = match_renderers(
-                plots_data=plots_data, templates=self.repo.plots.templates
+                plots_data=plots_data, out=self.args.out
             )
 
             if self.args.show_vega:
-                renderer = first(
-                    filter(lambda r: isinstance(r, VegaRenderer), renderers)
-                )
+                renderer = first(filter(lambda r: r.TYPE == "vega", renderers))
                 if renderer:
-                    content = renderer.asdict()
-                    ui.write_json(content)
+                    ui.write_json(json.loads(renderer.partial_html()))
                 return 0
             if self.args.json:
-                _show_json(renderers, self.args.out, self.args.split)
+                _show_json(renderers, self.args.split)
                 return 0
 
+            html_template_path = self.args.html_template
+            if not html_template_path:
+                html_template_path = self.repo.config.get("plots", {}).get(
+                    "html_template", None
+                )
+                if html_template_path and not os.path.isabs(
+                    html_template_path
+                ):
+                    html_template_path = os.path.join(
+                        self.repo.dvc_dir, html_template_path
+                    )
+
             rel: str = self.args.out or "dvc_plots"
-            path: Path = (Path.cwd() / rel).resolve()
-            index_path = render(
-                self.repo,
-                renderers,
-                path=path,
-                html_template_path=self.args.html_template,
+            output_file: Path = (Path.cwd() / rel).resolve() / "index.html"
+
+            render_html(
+                renderers=renderers,
+                output_file=output_file,
+                template_path=html_template_path,
             )
 
-            ui.write(index_path.as_uri())
+            ui.write(output_file.as_uri())
             auto_open = self.repo.config["plots"].get("auto_open", False)
             if self.args.open or auto_open:
                 if not auto_open:
@@ -107,7 +114,7 @@ class CmdPlots(CmdBase):
                         "\n"
                         "\tdvc config plots.auto_open true"
                     )
-                return ui.open_browser(index_path)
+                return ui.open_browser(output_file)
 
             return 0
 
@@ -154,17 +161,17 @@ class CmdPlotsTemplates(CmdBase):
     ]
 
     def run(self):
-        import os
+        from dvc_render.vega_templates import dump_templates
 
         try:
             out = (
                 os.path.join(os.getcwd(), self.args.out)
                 if self.args.out
-                else self.repo.plots.templates.templates_dir
+                else self.repo.plots.templates_dir
             )
 
             targets = [self.args.target] if self.args.target else None
-            self.repo.plots.templates.init(output=out, targets=targets)
+            dump_templates(output=out, targets=targets)
             templates_path = os.path.relpath(out, os.getcwd())
             ui.write(f"Templates have been written into '{templates_path}'.")
 

dvc/compare.py

@@ -168,17 +168,21 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
 
     def to_parallel_coordinates(
         self, output_path: "StrPath", color_by: str = None
-    ) -> str:
-        from dvc.render.html import write
-        from dvc.render.plotly import ParallelCoordinatesRenderer
+    ) -> "StrPath":
+        from dvc_render.html import render_html
+        from dvc_render.plotly import ParallelCoordinatesRenderer
 
-        index_path = write(
-            output_path,
+        render_html(
             renderers=[
-                ParallelCoordinatesRenderer(self, color_by, self._fill_value)
+                ParallelCoordinatesRenderer(
+                    self.as_dict(),
+                    color_by=color_by,
+                    fill_value=self._fill_value,
+                )
             ],
+            output_file=output_path,
         )
-        return index_path.as_uri()
+        return output_path
 
     def add_column(self, name: str) -> None:
         self._columns[name] = Column([self._fill_value] * len(self))

dvc/render/__init__.py

@@ -1,4 +1,5 @@
-from .image import ImageRenderer
-from .vega import VegaRenderer
-
-RENDERERS = [ImageRenderer, VegaRenderer]
+INDEX_FIELD = "step"
+REVISION_FIELD = "rev"
+REVISIONS_KEY = "revisions"
+TYPE_KEY = "type"
+SRC_FIELD = "src"

dvc/render/base.py

@@ -1,77 +0,0 @@
-import abc
-from typing import TYPE_CHECKING, Dict
-
-from dvc.exceptions import DvcException
-
-if TYPE_CHECKING:
-    from dvc.types import StrPath
-
-
-REVISION_FIELD = "rev"
-INDEX_FIELD = "step"
-
-
-class BadTemplateError(DvcException):
-    pass
-
-
-class Renderer(abc.ABC):
-    REVISIONS_KEY = "revisions"
-    TYPE_KEY = "type"
-
-    def __init__(self, data: Dict, **kwargs):
-        self.data = data
-
-        from dvc.render.utils import get_files
-
-        files = get_files(self.data)
-
-        # we assume comparison of same file between revisions for now
-        assert len(files) == 1
-        self.filename = files.pop()
-
-    def partial_html(self, **kwargs):
-        """
-        Us this method to generate partial HTML content,
-        that will fill self.DIV
-        """
-
-        raise NotImplementedError
-
-    @property
-    @abc.abstractmethod
-    def TYPE(self):
-        raise NotImplementedError
-
-    @property
-    @abc.abstractmethod
-    def DIV(self):
-        raise NotImplementedError
-
-    @property
-    @abc.abstractmethod
-    def SCRIPTS(self):
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def as_json(self, **kwargs):
-        raise NotImplementedError
-
-    @staticmethod
-    def _remove_special_chars(string: str):
-        return string.translate(
-            {ord(c): "_" for c in r"!@#$%^&*()[]{};,<>?\/:.|`~=_+"}
-        )
-
-    @property
-    def needs_output_path(self):
-        return False
-
-    def generate_html(self, path: "StrPath"):
-        """this method might edit content of path"""
-        partial = self.partial_html(path=path)
-
-        div_id = self._remove_special_chars(self.filename)
-        div_id = f"plot_{div_id}"
-
-        return self.DIV.format(id=div_id, partial=partial)

dvc/render/convert.py

@@ -0,0 +1,68 @@
+import json
+from collections import defaultdict
+from typing import Dict, List, Union
+
+from dvc.render import REVISION_FIELD, REVISIONS_KEY, SRC_FIELD, TYPE_KEY
+from dvc.render.image_converter import ImageConverter
+from dvc.render.vega_converter import VegaConverter
+
+
+def get_converter(
+    renderer_class, props
+) -> Union[VegaConverter, ImageConverter]:
+    from dvc_render import ImageRenderer, VegaRenderer
+
+    if renderer_class.TYPE == VegaRenderer.TYPE:
+        return VegaConverter(props)
+    if renderer_class.TYPE == ImageRenderer.TYPE:
+        return ImageConverter(props)
+
+    raise ValueError(f"Invalid renderer class {renderer_class}")
+
+
+def to_datapoints(renderer_class, data: Dict, props: Dict):
+    converter = get_converter(renderer_class, props)
+    datapoints: List[Dict] = []
+    for revision, rev_data in data.items():
+        for filename, file_data in rev_data.get("data", {}).items():
+            if "data" in file_data:
+                processed, final_props = converter.convert(
+                    revision, filename, file_data.get("data")
+                )
+                datapoints.extend(processed)
+    return datapoints, final_props
+
+
+def _group_by_rev(datapoints):
+    grouped = defaultdict(list)
+    for datapoint in datapoints:
+        rev = datapoint.pop(REVISION_FIELD)
+        grouped[rev].append(datapoint)
+    return dict(grouped)
+
+
+def to_json(renderer, split: bool = False) -> List[Dict]:
+    if renderer.TYPE == "vega":
+        grouped = _group_by_rev(renderer.datapoints)
+        if split:
+            content = renderer.get_filled_template(skip_anchors=["data"])
+        else:
+            content = renderer.get_filled_template()
+        return [
+            {
+                TYPE_KEY: renderer.TYPE,
+                REVISIONS_KEY: sorted(grouped.keys()),
+                "content": json.loads(content),
+                "datapoints": grouped,
+            }
+        ]
+    if renderer.TYPE == "image":
+        return [
+            {
+                TYPE_KEY: renderer.TYPE,
+                REVISIONS_KEY: datapoint.get(REVISION_FIELD),
+                "url": datapoint.get(SRC_FIELD),
+            }
+            for datapoint in renderer.datapoints
+        ]
+    raise ValueError(f"Invalid renderer: {renderer.TYPE}")

dvc/render/html.py

@@ -1,114 +0,0 @@
-import os
-from pathlib import Path
-from typing import TYPE_CHECKING, Dict, List, Optional
-
-from dvc.exceptions import DvcException
-
-if TYPE_CHECKING:
-    from dvc.render.base import Renderer
-    from dvc.types import StrPath
-
-PAGE_HTML = """<!DOCTYPE html>
-<html>
-<head>
-    {refresh_tag}
-    <title>DVC Plot</title>
-    {scripts}
-</head>
-<body>
-    {plot_divs}
-</body>
-</html>"""
-
-
-class MissingPlaceholderError(DvcException):
-    def __init__(self, placeholder):
-        super().__init__(f"HTML template has to contain '{placeholder}'.")
-
-
-class HTML:
-    SCRIPTS_PLACEHOLDER = "scripts"
-    PLOTS_PLACEHOLDER = "plot_divs"
-    PLOTS_PLACEHOLDER_FORMAT_STR = f"{{{PLOTS_PLACEHOLDER}}}"
-    REFRESH_PLACEHOLDER = "refresh_tag"
-    REFRESH_TAG = '<meta http-equiv="refresh" content="{}">'
-
-    def __init__(
-        self,
-        template: Optional[str] = None,
-        refresh_seconds: Optional[int] = None,
-    ):
-        template = template or PAGE_HTML
-        if self.PLOTS_PLACEHOLDER_FORMAT_STR not in template:
-            raise MissingPlaceholderError(self.PLOTS_PLACEHOLDER_FORMAT_STR)
-
-        self.template = template
-        self.elements: List[str] = []
-        self.scripts: str = ""
-        self.refresh_tag = ""
-        if refresh_seconds is not None:
-            self.refresh_tag = self.REFRESH_TAG.format(refresh_seconds)
-
-    def with_metrics(self, metrics: Dict[str, Dict]) -> "HTML":
-        import tabulate
-
-        header: List[str] = []
-        rows: List[List[str]] = []
-
-        for _, rev_data in metrics.items():
-            for _, data in rev_data.items():
-                if not header:
-                    header.extend(sorted(data.keys()))
-
-                rows.append([data[key] for key in header])
-
-        self.elements.append(tabulate.tabulate(rows, header, tablefmt="html"))
-        return self
-
-    def with_scripts(self, scripts: str) -> "HTML":
-        if scripts not in self.scripts:
-            self.scripts += f"\n{scripts}"
-        return self
-
-    def with_element(self, html: str) -> "HTML":
-        self.elements.append(html)
-        return self
-
-    def embed(self) -> str:
-        kwargs = {
-            self.SCRIPTS_PLACEHOLDER: self.scripts,
-            self.PLOTS_PLACEHOLDER: "\n".join(self.elements),
-            self.REFRESH_PLACEHOLDER: self.refresh_tag,
-        }
-        return self.template.format(**kwargs)
-
-
-def write(
-    path: "StrPath",
-    renderers: List["Renderer"],
-    metrics: Optional[Dict[str, Dict]] = None,
-    template_path: Optional["StrPath"] = None,
-    refresh_seconds: Optional[int] = None,
-):
-
-    os.makedirs(path, exist_ok=True)
-
-    page_html = None
-    if template_path:
-        with open(template_path, encoding="utf-8") as fobj:
-            page_html = fobj.read()
-
-    document = HTML(page_html, refresh_seconds=refresh_seconds)
-    if metrics:
-        document.with_metrics(metrics)
-        document.with_element("<br>")
-
-    for renderer in renderers:
-        document.with_scripts(renderer.SCRIPTS)
-        document.with_element(renderer.generate_html(path))
-
-    index = Path(os.path.join(path, "index.html"))
-
-    with open(index, "w", encoding="utf-8") as fd:
-        fd.write(document.embed())
-    return index

dvc/render/image.py

@@ -1,110 +0,0 @@
-import json
-import os
-from typing import TYPE_CHECKING
-
-from funcy import reraise
-
-from dvc.exceptions import DvcException
-from dvc.render.base import Renderer
-from dvc.render.utils import get_files
-from dvc.utils import relpath
-
-if TYPE_CHECKING:
-    from dvc.types import StrPath
-
-
-class ImageRenderer(Renderer):
-    TYPE = "image"
-    DIV = """
-        <div
-            id="{id}"
-            style="border:1px solid black;text-align:center;
-            white-space: nowrap;overflow-y:hidden;">
-            {partial}
-        </div>"""
-
-    SCRIPTS = ""
-
-    @property
-    def needs_output_path(self):
-        return True
-
-    def _write_image(
-        self,
-        path: "StrPath",
-        revision: str,
-        filename: str,
-        image_data: bytes,
-    ):
-        img_path = os.path.join(
-            path, f"{revision}_{filename.replace(os.sep, '_')}"
-        )
-        with open(img_path, "wb") as fd:
-            fd.write(image_data)
-
-        return img_path
-
-    def _save_images(self, path: "StrPath"):
-
-        for rev, rev_data in self.data.items():
-            if "data" in rev_data:
-                for file, file_data in rev_data.get("data", {}).items():
-                    if "data" in file_data:
-                        if not os.path.isdir(path):
-                            os.makedirs(path, exist_ok=True)
-                        yield rev, file, self._write_image(
-                            os.path.abspath(path), rev, file, file_data["data"]
-                        )
-
-    def partial_html(self, **kwargs):
-        path = kwargs.get("path", None)
-        if not path:
-            raise DvcException("Can't save here")
-        static = os.path.join(path, "static")
-
-        div_content = []
-        for rev, _, img_path in self._save_images(static):
-            div_content.append(
-                """
-        <div
-            style="border:1px dotted black;margin:2px;display: inline-block;
-            overflow:hidden;margin-left:8px;">
-            <p>{title}</p>
-            <img src="{src}">
-        </div>""".format(
-                    title=rev, src=(relpath(img_path, path))
-                )
-            )
-        if div_content:
-            div_content.insert(0, f"<p>{self.filename}</p>")
-            return "\n".join(div_content)
-        return ""
-
-    def as_json(self, **kwargs):
-
-        with reraise(
-            KeyError,
-            DvcException(
-                f"{type(self).__name__} needs 'path' to store images."
-            ),
-        ):
-            path = kwargs["path"]
-
-        results = []
-
-        for revision, _, img_path in self._save_images(path):
-            results.append(
-                {
-                    self.TYPE_KEY: self.TYPE,
-                    self.REVISIONS_KEY: [revision],
-                    "url": img_path,
-                }
-            )
-
-        return json.dumps(results)
-
-    @staticmethod
-    def matches(data):
-        files = get_files(data)
-        extensions = set(map(lambda f: os.path.splitext(f)[1], files))
-        return extensions.issubset({".jpg", ".jpeg", ".gif", ".png"})

dvc/render/image_converter.py

@@ -0,0 +1,58 @@
+import base64
+import os
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
+
+from dvc.render import REVISION_FIELD, SRC_FIELD
+
+if TYPE_CHECKING:
+    from dvc.types import StrPath
+
+
+class ImageConverter:
+    def __init__(self, plot_properties: Optional[Dict] = None):
+        self.plot_properties = plot_properties or {}
+
+    @staticmethod
+    def _write_image(
+        path: "StrPath",
+        revision: str,
+        filename: str,
+        image_data: bytes,
+    ) -> "StrPath":
+        img_path = os.path.join(
+            path, f"{revision}_{filename.replace(os.sep, '_')}"
+        )
+        with open(img_path, "wb") as fd:
+            fd.write(image_data)
+
+        return img_path
+
+    @staticmethod
+    def _encode_image(
+        image_data: bytes,
+    ) -> str:
+        base64_str = base64.b64encode(image_data).decode()
+        return f"data:image;base64,{base64_str}"
+
+    def convert(
+        self, revision, filename, data: bytes
+    ) -> Tuple[List[Dict], Dict]:
+        """
+        Convert the DVC Plots content to DVC Render datapoints.
+        Return both generated datapoints and updated properties.
+        """
+        path = self.plot_properties.get("out")
+        if path:
+            if not os.path.isdir(path):
+                os.makedirs(path, exist_ok=True)
+            src = self._write_image(
+                os.path.abspath(path), revision, filename, data
+            )
+        else:
+            src = self._encode_image(data)
+        datapoint = {
+            REVISION_FIELD: revision,
+            "filename": filename,
+            SRC_FIELD: src,
+        }
+        return [datapoint], self.plot_properties

dvc/render/match.py

@@ -0,0 +1,55 @@
+from typing import TYPE_CHECKING, Dict, List, Optional
+
+import dpath.util
+from dvc_render import RENDERERS
+
+from .convert import to_datapoints
+
+if TYPE_CHECKING:
+    from dvc.types import StrPath
+
+
+def get_files(plots_data: Dict) -> List:
+    values = dpath.util.values(plots_data, ["*", "*"])
+    return sorted({key for submap in values for key in submap.keys()})
+
+
+def group_by_filename(plots_data: Dict) -> Dict:
+    files = get_files(plots_data)
+    grouped = {
+        file: dpath.util.search(plots_data, ["*", "*", file]) for file in files
+    }
+    return grouped
+
+
+def squash_plots_properties(data: Dict) -> Dict:
+    resolved: Dict[str, str] = {}
+    for rev_data in data.values():
+        for file_data in rev_data.get("data", {}).values():
+            props = file_data.get("props", {})
+            resolved = {**resolved, **props}
+    return resolved
+
+
+def match_renderers(
+    plots_data,
+    out: Optional["StrPath"] = None,
+    templates_dir: Optional["StrPath"] = None,
+):
+    renderers = []
+    for filename, group in group_by_filename(plots_data).items():
+        plot_properties = squash_plots_properties(group)
+        for renderer_class in RENDERERS:
+            if renderer_class.matches(filename, plot_properties):
+                if out is not None:
+                    plot_properties["out"] = out
+                if templates_dir is not None:
+                    plot_properties["templates_dir"] = templates_dir
+                datapoints, plot_properties = to_datapoints(
+                    renderer_class, group, plot_properties
+                )
+                renderers.append(
+                    renderer_class(datapoints, filename, **plot_properties)
+                )
+                break
+    return renderers

dvc/render/plotly.py

@@ -1,92 +0,0 @@
-import json
-from collections import defaultdict
-from typing import TYPE_CHECKING, Any, Dict, Optional
-
-from dvc.render.base import Renderer
-
-if TYPE_CHECKING:
-    from dvc.compare import TabularData
-
-
-class ParallelCoordinatesRenderer(Renderer):
-    TYPE = "plotly"
-
-    DIV = """
-    <div id = "{id}">
-        <script type = "text/javascript">
-            var plotly_data = {partial};
-            Plotly.newPlot("{id}", plotly_data.data, plotly_data.layout);
-        </script>
-    </div>
-    """
-
-    SCRIPTS = """
-    <script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
-    """
-
-    # pylint: disable=W0231
-    def __init__(
-        self,
-        tabular_data: "TabularData",
-        color_by: Optional[str] = None,
-        fill_value: str = "",
-    ):
-        self.tabular_data = tabular_data
-        self.color_by = color_by
-        self.filename = "experiments"
-        self.fill_value = fill_value
-
-    def partial_html(self, **kwargs):
-        return self.as_json()
-
-    def as_json(self, **kwargs) -> str:
-        tabular_dict = defaultdict(list)
-        for row in self.tabular_data.as_dict():
-            for col_name, value in row.items():
-                tabular_dict[col_name].append(str(value))
-
-        trace: Dict[str, Any] = {"type": "parcoords", "dimensions": []}
-        for label, values in tabular_dict.items():
-            is_categorical = False
-            try:
-                float_values = [
-                    float(x) if x != self.fill_value else None for x in values
-                ]
-            except ValueError:
-                is_categorical = True
-
-            if is_categorical:
-                non_missing = [x for x in values if x != self.fill_value]
-                unique_values = sorted(set(non_missing))
-                unique_values.append(self.fill_value)
-
-                dummy_values = [unique_values.index(x) for x in values]
-
-                values = [
-                    x if x != self.fill_value else "Missing" for x in values
-                ]
-                trace["dimensions"].append(
-                    {
-                        "label": label,
-                        "values": dummy_values,
-                        "tickvals": dummy_values,
-                        "ticktext": values,
-                    }
-                )
-            else:
-                trace["dimensions"].append(
-                    {"label": label, "values": float_values}
-                )
-
-            if label == self.color_by:
-                trace["line"] = {
-                    "color": dummy_values if is_categorical else float_values,
-                    "showscale": True,
-                    "colorbar": {"title": self.color_by},
-                }
-                if is_categorical:
-                    trace["line"]["colorbar"]["tickmode"] = "array"
-                    trace["line"]["colorbar"]["tickvals"] = dummy_values
-                    trace["line"]["colorbar"]["ticktext"] = values
-
-        return json.dumps({"data": [trace], "layout": {}})

dvc/render/utils.py

@@ -1,70 +0,0 @@
-import os.path
-from typing import Dict, List
-
-import dpath.util
-
-
-def get_files(plots_data: Dict) -> List:
-    values = dpath.util.values(plots_data, ["*", "*"])
-    return sorted({key for submap in values for key in submap.keys()})
-
-
-def group_by_filename(plots_data: Dict) -> List[Dict]:
-    files = get_files(plots_data)
-    grouped = []
-    for file in files:
-        grouped.append(dpath.util.search(plots_data, ["*", "*", file]))
-    return grouped
-
-
-def squash_plots_properties(data: Dict) -> Dict:
-    resolved: Dict[str, str] = {}
-    for rev_data in data.values():
-        for file_data in rev_data.get("data", {}).values():
-            props = file_data.get("props", {})
-            resolved = {**resolved, **props}
-    return resolved
-
-
-def match_renderers(plots_data, templates):
-    from dvc.render import RENDERERS
-
-    renderers = []
-    for group in group_by_filename(plots_data):
-        plot_properties = squash_plots_properties(group)
-        template = templates.load(plot_properties.get("template", None))
-
-        for renderer_class in RENDERERS:
-            if renderer_class.matches(group):
-                renderers.append(
-                    renderer_class(
-                        group, template=template, properties=plot_properties
-                    )
-                )
-    return renderers
-
-
-def render(
-    repo,
-    renderers,
-    metrics=None,
-    path=None,
-    html_template_path=None,
-    refresh_seconds=None,
-):
-    if not html_template_path:
-        html_template_path = repo.config.get("plots", {}).get(
-            "html_template", None
-        )
-        if html_template_path and not os.path.isabs(html_template_path):
-            html_template_path = os.path.join(repo.dvc_dir, html_template_path)
-
-    from dvc.render.html import write
-
-    return write(
-        path,
-        renderers,
-        metrics=metrics,
-        template_path=html_template_path,
-        refresh_seconds=refresh_seconds,
-    )

dvc/render/vega.py

@@ -1,134 +0,0 @@
-import json
-import os
-from copy import deepcopy
-from typing import Dict, List, Optional
-
-from funcy import cached_property
-
-from dvc.render.base import REVISION_FIELD, BadTemplateError, Renderer
-from dvc.render.data import Converter, to_datapoints
-from dvc.render.utils import get_files
-from dvc.repo.plots.template import Template
-
-
-def _flatten_datapoints(rev_datapoints: Dict[str, List[Dict]]) -> List[Dict]:
-    flat = []
-    for revision, datapoints in rev_datapoints.items():
-        datapoints_cp = deepcopy(datapoints)
-        Converter.update(datapoints_cp, {REVISION_FIELD: revision})
-        flat.extend(datapoints_cp)
-    return flat
-
-
-class VegaRenderer(Renderer):
-    TYPE = "vega"
-
-    DIV = """
-    <div id = "{id}">
-        <script type = "text/javascript">
-            var spec = {partial};
-            vegaEmbed('#{id}', spec);
-        </script>
-    </div>
-    """
-
-    SCRIPTS = """
-    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
-    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
-    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
-    """
-
-    def __init__(
-        self, data: Dict, template: Template, properties: Dict = None
-    ):
-        super().__init__(data)
-        self.properties = properties or {}
-        self.template = template
-
-    def _revisions(self):
-        return list(self.data.keys())
-
-    # TODO Shouldn't it be a part of plots?
-    def _fill_template(self, template, datapoints, props=None, fill_data=True):
-        props = props or {}
-
-        content = deepcopy(template.content)
-        if template.anchor_str("data") not in template.content:
-            anchor = template.anchor("data")
-            raise BadTemplateError(
-                f"Template '{template.name}' is not using '{anchor}' anchor"
-            )
-
-        if props.get("x"):
-            template.check_field_exists(datapoints, props.get("x"))
-        if props.get("y"):
-            template.check_field_exists(datapoints, props.get("y"))
-
-        if fill_data:
-            content = template.fill_anchor(content, "data", datapoints)
-
-        props.setdefault("title", "")
-        props.setdefault("x_label", props.get("x"))
-        props.setdefault("y_label", props.get("y"))
-
-        names = ["title", "x", "y", "x_label", "y_label"]
-        for name in names:
-            value = props.get(name)
-            if value is not None:
-                content = template.fill_anchor(content, name, value)
-
-        return content
-
-    @cached_property
-    def _converted_data(self):
-        return to_datapoints(self.data, self.properties)
-
-    def _get_filled_template(self, fill_data=True):
-        datapoints, final_props = self._converted_data
-        flat_datapoints = _flatten_datapoints(datapoints)
-
-        if datapoints:
-            filled_template = self._fill_template(
-                self.template,
-                flat_datapoints,
-                final_props,
-                fill_data=fill_data,
-            )
-
-            return filled_template
-        return ""
-
-    # TODO rename
-    def asdict(self, fill_data=True):
-        filled_template = self._get_filled_template(fill_data)
-        if filled_template:
-            return json.loads(filled_template)
-        return {}
-
-    def as_json(self, **kwargs) -> Optional[str]:
-
-        fill_data = kwargs.get("fill_data", True)
-
-        content = self.asdict(fill_data)
-        datapoints, _ = self._converted_data
-
-        return json.dumps(
-            [
-                {
-                    self.TYPE_KEY: self.TYPE,
-                    self.REVISIONS_KEY: self._revisions(),
-                    "content": content,
-                    "datapoints": datapoints,
-                }
-            ],
-        )
-
-    def partial_html(self, **kwargs):
-        filled_template = self._get_filled_template()
-        return filled_template
-
-    @staticmethod
-    def matches(data):
-        files = get_files(data)
-        extensions = set(map(lambda f: os.path.splitext(f)[1], files))
-        return extensions.issubset({".yml", ".yaml", ".json", ".csv", ".tsv"})

dvc/render/vega_converter.py

@@ -5,7 +5,7 @@ from typing import Dict, List, Optional, Set, Union
 from funcy import first, project
 
 from dvc.exceptions import DvcException
-from dvc.render.base import INDEX_FIELD, REVISION_FIELD
+from dvc.render import INDEX_FIELD, REVISION_FIELD
 
 
 class FieldsNotFoundError(DvcException):
@@ -76,7 +76,7 @@ def _append_index(datapoints: List[Dict]) -> List[Dict]:
     return datapoints
 
 
-class Converter:
+class VegaConverter:
     """
     Class that takes care of converting unspecified data blob
     (Dict or List[Dict]) into datapoints (List[Dict]).
@@ -84,12 +84,6 @@ class Converter:
     ('x', 'y') it will attempt to fill in the blanks.
     """
 
-    @staticmethod
-    def update(datapoints: List[Dict], update_dict: Dict):
-        for data_point in datapoints:
-            data_point.update(update_dict)
-        return datapoints
-
     def __init__(self, plot_properties: Optional[Dict] = None):
         plot_properties = plot_properties or {}
         self.props = deepcopy(plot_properties)
@@ -162,7 +156,7 @@ class Converter:
             else:
                 self.inferred_props["y"] = inferred_y
 
-    def convert(self, data):
+    def convert(self, revision: str, filename: str, data: Dict):
         """
         Convert the data. Fill necessary fields ('x', 'y') and return both
         generated datapoints and updated properties.
@@ -174,20 +168,8 @@ class Converter:
 
         self._infer_y(processed)
 
+        for datapoint in processed:
+            datapoint[REVISION_FIELD] = revision
+            datapoint["filename"] = filename
+
         return processed, {**self.props, **self.inferred_props}
-
-
-def to_datapoints(data: Dict, props: Dict):
-    converter = Converter(props)
-
-    datapoints: Dict[str, List[Dict]] = {}
-    for revision, rev_data in data.items():
-        datapoints[revision] = []
-        for _, file_data in rev_data.get("data", {}).items():
-            if "data" in file_data:
-                processed, final_props = converter.convert(
-                    file_data.get("data")
-                )
-
-                datapoints[revision].extend(processed)
-    return datapoints, final_props

dvc/repo/live.py

@@ -1,9 +1,11 @@
 import logging
+import os
 from typing import TYPE_CHECKING, List, Optional
 
+from dvc_render import render_html
 from funcy import once_per_args
 
-from dvc.render.utils import match_renderers, render
+from dvc.render.match import match_renderers
 
 logger = logging.getLogger(__name__)
 
@@ -24,11 +26,13 @@ def create_summary(out):
 
     metrics, plots = out.repo.live.show(out.fs_path)
 
-    html_path = out.fs_path + "_dvc_plots"
+    html_path = os.path.join(out.fs_path + "_dvc_plots", "index.html")
 
-    renderers = match_renderers(plots, out.repo.plots.templates)
-    index_path = render(
-        out.repo, renderers, metrics=metrics, path=html_path, refresh_seconds=5
+    renderers = match_renderers(
+        plots, templates_dir=out.repo.plots.templates_dir
+    )
+    index_path = render_html(
+        renderers, output_file=html_path, metrics=metrics, refresh_seconds=5
     )
     if out.repo.config["plots"].get("auto_open", False):
         webbrowser_open(index_path)

dvc/repo/plots/__init__.py

@@ -195,12 +195,14 @@ class Plots:
             out.plot.pop(prop)
 
     def modify(self, path, props=None, unset=None):
+        from dvc_render.vega_templates import get_template
+
         from dvc.dvcfile import Dvcfile
 
         props = props or {}
         template = props.get("template")
         if template:
-            self.templates.get_template(template)
+            get_template(template, self.templates_dir)
 
         (out,) = self.repo.find_outs_by_path(path)
         if not out.plot and unset is not None:
@@ -225,10 +227,8 @@ class Plots:
         dvcfile.dump(out.stage, update_lock=False)
 
     @cached_property
-    def templates(self):
-        from .template import PlotTemplates
-
-        return PlotTemplates(self.repo.dvc_dir)
+    def templates_dir(self):
+        return os.path.join(self.repo.dvc_dir, "plots")
 
 
 def _is_plot(out: "Output") -> bool:

dvc/repo/plots/template.py

@@ -1,595 +0,0 @@
-import json
-import os
-from typing import Any, Dict, List, Optional
-
-from funcy import cached_property
-
-from dvc.exceptions import DvcException
-
-
-class TemplateNotFoundError(DvcException):
-    def __init__(self, path):
-        super().__init__(f"Template '{path}' not found.")
-
-
-class NoFieldInDataError(DvcException):
-    def __init__(self, field_name):
-        super().__init__(
-            f"Field '{field_name}' does not exist in provided data."
-        )
-
-
-class TemplateContentDoesNotMatch(DvcException):
-    def __init__(self, template_name: str, path: str):
-        super().__init__(
-            f"Template '{path}' already exists "
-            f"and its content is different than '{template_name}' content. "
-            "Remove it manually if you want to recreate it."
-        )
-
-
-class Template:
-    INDENT = 4
-    SEPARATORS = (",", ": ")
-    EXTENSION = ".json"
-    ANCHOR = "<DVC_METRIC_{}>"
-
-    DEFAULT_CONTENT: Optional[Dict[str, Any]] = None
-    DEFAULT_NAME: Optional[str] = None
-
-    def __init__(self, content=None, name=None):
-        if content:
-            self.content = content
-        else:
-            self.content = (
-                json.dumps(
-                    self.DEFAULT_CONTENT,
-                    indent=self.INDENT,
-                    separators=self.SEPARATORS,
-                )
-                + "\n"
-            )
-
-        self.name = name or self.DEFAULT_NAME
-        assert self.content and self.name
-        self.filename = self.name + self.EXTENSION
-
-    @classmethod
-    def anchor(cls, name):
-        return cls.ANCHOR.format(name.upper())
-
-    def has_anchor(self, name):
-        return self.anchor_str(name) in self.content
-
-    @classmethod
-    def fill_anchor(cls, content, name, value):
-        value_str = json.dumps(
-            value, indent=cls.INDENT, separators=cls.SEPARATORS, sort_keys=True
-        )
-        return content.replace(cls.anchor_str(name), value_str)
-
-    @classmethod
-    def anchor_str(cls, name):
-        return f'"{cls.anchor(name)}"'
-
-    @staticmethod
-    def check_field_exists(data, field):
-        if not any(field in row for row in data):
-            raise NoFieldInDataError(field)
-
-
-class SimpleLinearTemplate(Template):
-    DEFAULT_NAME = "simple"
-
-    DEFAULT_CONTENT = {
-        "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
-        "data": {"values": Template.anchor("data")},
-        "title": Template.anchor("title"),
-        "width": 300,
-        "height": 300,
-        "mark": {"type": "line"},
-        "encoding": {
-            "x": {
-                "field": Template.anchor("x"),
-                "type": "quantitative",
-                "title": Template.anchor("x_label"),
-            },
-            "y": {
-                "field": Template.anchor("y"),
-                "type": "quantitative",
-                "title": Template.anchor("y_label"),
-                "scale": {"zero": False},
-            },
-            "color": {"field": "rev", "type": "nominal"},
-        },
-    }
-
-
-class ConfusionTemplate(Template):
-    DEFAULT_NAME = "confusion"
-    DEFAULT_CONTENT = {
-        "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
-        "data": {"values": Template.anchor("data")},
-        "title": Template.anchor("title"),
-        "facet": {"field": "rev", "type": "nominal"},
-        "spec": {
-            "transform": [
-                {
-                    "aggregate": [{"op": "count", "as": "xy_count"}],
-                    "groupby": [Template.anchor("y"), Template.anchor("x")],
-                },
-                {
-                    "impute": "xy_count",
-                    "groupby": ["rev", Template.anchor("y")],
-                    "key": Template.anchor("x"),
-                    "value": 0,
-                },
-                {
-                    "impute": "xy_count",
-                    "groupby": ["rev", Template.anchor("x")],
-                    "key": Template.anchor("y"),
-                    "value": 0,
-                },
-                {
-                    "joinaggregate": [
-                        {"op": "max", "field": "xy_count", "as": "max_count"}
-                    ],
-                    "groupby": [],
-                },
-                {
-                    "calculate": "datum.xy_count / datum.max_count",
-                    "as": "percent_of_max",
-                },
-            ],
-            "encoding": {
-                "x": {
-                    "field": Template.anchor("x"),
-                    "type": "nominal",
-                    "sort": "ascending",
-                    "title": Template.anchor("x_label"),
-                },
-                "y": {
-                    "field": Template.anchor("y"),
-                    "type": "nominal",
-                    "sort": "ascending",
-                    "title": Template.anchor("y_label"),
-                },
-            },
-            "layer": [
-                {
-                    "mark": "rect",
-                    "width": 300,
-                    "height": 300,
-                    "encoding": {
-                        "color": {
-                            "field": "xy_count",
-                            "type": "quantitative",
-                            "title": "",
-                            "scale": {"domainMin": 0, "nice": True},
-                        }
-                    },
-                },
-                {
-                    "mark": "text",
-                    "encoding": {
-                        "text": {"field": "xy_count", "type": "quantitative"},
-                        "color": {
-                            "condition": {
-                                "test": "datum.percent_of_max > 0.5",
-                                "value": "white",
-                            },
-                            "value": "black",
-                        },
-                    },
-                },
-            ],
-        },
-    }
-
-
-class NormalizedConfusionTemplate(Template):
-    DEFAULT_NAME = "confusion_normalized"
-    DEFAULT_CONTENT = {
-        "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
-        "data": {"values": Template.anchor("data")},
-        "title": Template.anchor("title"),
-        "facet": {"field": "rev", "type": "nominal"},
-        "spec": {
-            "transform": [
-                {
-                    "aggregate": [{"op": "count", "as": "xy_count"}],
-                    "groupby": [Template.anchor("y"), Template.anchor("x")],
-                },
-                {
-                    "impute": "xy_count",
-                    "groupby": ["rev", Template.anchor("y")],
-                    "key": Template.anchor("x"),
-                    "value": 0,
-                },
-                {
-                    "impute": "xy_count",
-                    "groupby": ["rev", Template.anchor("x")],
-                    "key": Template.anchor("y"),
-                    "value": 0,
-                },
-                {
-                    "joinaggregate": [
-                        {"op": "sum", "field": "xy_count", "as": "sum_y"}
-                    ],
-                    "groupby": [Template.anchor("y")],
-                },
-                {
-                    "calculate": "datum.xy_count / datum.sum_y",
-                    "as": "percent_of_y",
-                },
-            ],
-            "encoding": {
-                "x": {
-                    "field": Template.anchor("x"),
-                    "type": "nominal",
-                    "sort": "ascending",
-                    "title": Template.anchor("x_label"),
-                },
-                "y": {
-                    "field": Template.anchor("y"),
-                    "type": "nominal",
-                    "sort": "ascending",
-                    "title": Template.anchor("y_label"),
-                },
-            },
-            "layer": [
-                {
-                    "mark": "rect",
-                    "width": 300,
-                    "height": 300,
-                    "encoding": {
-                        "color": {
-                            "field": "percent_of_y",
-                            "type": "quantitative",
-                            "title": "",
-                            "scale": {"domain": [0, 1]},
-                        }
-                    },
-                },
-                {
-                    "mark": "text",
-                    "encoding": {
-                        "text": {
-                            "field": "percent_of_y",
-                            "type": "quantitative",
-                            "format": ".2f",
-                        },
-                        "color": {
-                            "condition": {
-                                "test": "datum.percent_of_y > 0.5",
-                                "value": "white",
-                            },
-                            "value": "black",
-                        },
-                    },
-                },
-            ],
-        },
-    }
-
-
-class ScatterTemplate(Template):
-    DEFAULT_NAME = "scatter"
-
-    DEFAULT_CONTENT = {
-        "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
-        "data": {"values": Template.anchor("data")},
-        "title": Template.anchor("title"),
-        "width": 300,
-        "height": 300,
-        "layer": [
-            {
-                "encoding": {
-                    "x": {
-                        "field": Template.anchor("x"),
-                        "type": "quantitative",
-                        "title": Template.anchor("x_label"),
-                    },
-                    "y": {
-                        "field": Template.anchor("y"),
-                        "type": "quantitative",
-                        "title": Template.anchor("y_label"),
-                        "scale": {"zero": False},
-                    },
-                    "color": {"field": "rev", "type": "nominal"},
-                },
-                "layer": [
-                    {"mark": "point"},
-                    {
-                        "selection": {
-                            "label": {
-                                "type": "single",
-                                "nearest": True,
-                                "on": "mouseover",
-                                "encodings": ["x"],
-                                "empty": "none",
-                                "clear": "mouseout",
-                            }
-                        },
-                        "mark": "point",
-                        "encoding": {
-                            "opacity": {
-                                "condition": {
-                                    "selection": "label",
-                                    "value": 1,
-                                },
-                                "value": 0,
-                            }
-                        },
-                    },
-                ],
-            },
-            {
-                "transform": [{"filter": {"selection": "label"}}],
-                "layer": [
-                    {
-                        "encoding": {
-                            "text": {
-                                "type": "quantitative",
-                                "field": Template.anchor("y"),
-                            },
-                            "x": {
-                                "field": Template.anchor("x"),
-                                "type": "quantitative",
-                            },
-                            "y": {
-                                "field": Template.anchor("y"),
-                                "type": "quantitative",
-                            },
-                        },
-                        "layer": [
-                            {
-                                "mark": {
-                                    "type": "text",
-                                    "align": "left",
-                                    "dx": 5,
-                                    "dy": -5,
-                                },
-                                "encoding": {
-                                    "color": {
-                                        "type": "nominal",
-                                        "field": "rev",
-                                    }
-                                },
-                            }
-                        ],
-                    }
-                ],
-            },
-        ],
-    }
-
-
-class SmoothLinearTemplate(Template):
-    DEFAULT_NAME = "smooth"
-
-    DEFAULT_CONTENT = {
-        "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
-        "data": {"values": Template.anchor("data")},
-        "title": Template.anchor("title"),
-        "mark": {"type": "line"},
-        "encoding": {
-            "x": {
-                "field": Template.anchor("x"),
-                "type": "quantitative",
-                "title": Template.anchor("x_label"),
-            },
-            "y": {
-                "field": Template.anchor("y"),
-                "type": "quantitative",
-                "title": Template.anchor("y_label"),
-                "scale": {"zero": False},
-            },
-            "color": {"field": "rev", "type": "nominal"},
-        },
-        "transform": [
-            {
-                "loess": Template.anchor("y"),
-                "on": Template.anchor("x"),
-                "groupby": ["rev"],
-                "bandwidth": 0.3,
-            }
-        ],
-    }
-
-
-class LinearTemplate(Template):
-    DEFAULT_NAME = "linear"
-
-    DEFAULT_CONTENT = {
-        "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
-        "data": {"values": Template.anchor("data")},
-        "title": Template.anchor("title"),
-        "width": 300,
-        "height": 300,
-        "layer": [
-            {
-                "encoding": {
-                    "x": {
-                        "field": Template.anchor("x"),
-                        "type": "quantitative",
-                        "title": Template.anchor("x_label"),
-                    },
-                    "y": {
-                        "field": Template.anchor("y"),
-                        "type": "quantitative",
-                        "title": Template.anchor("y_label"),
-                        "scale": {"zero": False},
-                    },
-                    "color": {"field": "rev", "type": "nominal"},
-                },
-                "layer": [
-                    {"mark": "line"},
-                    {
-                        "selection": {
-                            "label": {
-                                "type": "single",
-                                "nearest": True,
-                                "on": "mouseover",
-                                "encodings": ["x"],
-                                "empty": "none",
-                                "clear": "mouseout",
-                            }
-                        },
-                        "mark": "point",
-                        "encoding": {
-                            "opacity": {
-                                "condition": {
-                                    "selection": "label",
-                                    "value": 1,
-                                },
-                                "value": 0,
-                            }
-                        },
-                    },
-                ],
-            },
-            {
-                "transform": [{"filter": {"selection": "label"}}],
-                "layer": [
-                    {
-                        "mark": {"type": "rule", "color": "gray"},
-                        "encoding": {
-                            "x": {
-                                "field": Template.anchor("x"),
-                                "type": "quantitative",
-                            }
-                        },
-                    },
-                    {
-                        "encoding": {
-                            "text": {
-                                "type": "quantitative",
-                                "field": Template.anchor("y"),
-                            },
-                            "x": {
-                                "field": Template.anchor("x"),
-                                "type": "quantitative",
-                            },
-                            "y": {
-                                "field": Template.anchor("y"),
-                                "type": "quantitative",
-                            },
-                        },
-                        "layer": [
-                            {
-                                "mark": {
-                                    "type": "text",
-                                    "align": "left",
-                                    "dx": 5,
-                                    "dy": -5,
-                                },
-                                "encoding": {
-                                    "color": {
-                                        "type": "nominal",
-                                        "field": "rev",
-                                    }
-                                },
-                            }
-                        ],
-                    },
-                ],
-            },
-        ],
-    }
-
-
-class PlotTemplates:
-    TEMPLATES_DIR = "plots"
-    TEMPLATES = [
-        SimpleLinearTemplate,
-        LinearTemplate,
-        ConfusionTemplate,
-        NormalizedConfusionTemplate,
-        ScatterTemplate,
-        SmoothLinearTemplate,
-    ]
-
-    @cached_property
-    def templates_dir(self):
-        return os.path.join(self.dvc_dir, self.TEMPLATES_DIR)
-
-    def get_template(self, template_name: str):
-        template_path = os.path.abspath(template_name)
-        if os.path.exists(template_path):
-            return os.path.abspath(template_path)
-
-        if self.dvc_dir and os.path.exists(self.dvc_dir):
-            template_path = os.path.join(self.templates_dir, template_name)
-            if os.path.exists(template_path):
-                return template_path
-
-            all_templates = [
-                os.path.join(root, file)
-                for root, _, files in os.walk(self.templates_dir)
-                for file in files
-            ]
-            matches = [
-                template
-                for template in all_templates
-                if os.path.splitext(template)[0] == template_path
-            ]
-            if matches:
-                assert len(matches) == 1
-                return matches[0]
-        raise TemplateNotFoundError(template_name)
-
-    def __init__(self, dvc_dir):
-        self.dvc_dir = dvc_dir
-
-    def init(
-        self, output: Optional[str] = None, targets: Optional[List] = None
-    ):
-        from dvc.utils.fs import makedirs
-
-        output = output or self.templates_dir
-
-        makedirs(output, exist_ok=True)
-
-        if targets:
-            templates = [
-                template
-                for template in self.TEMPLATES
-                if template.DEFAULT_NAME in targets
-            ]
-        else:
-            templates = self.TEMPLATES
-
-        for template in templates:
-            self._dump(template(), output)
-
-    def _dump(self, template: Template, output: str):
-        path = os.path.join(output, template.filename)
-
-        if os.path.exists(path):
-            with open(path, "r", encoding="utf-8") as fd:
-                content = fd.read()
-            if content != template.content:
-                raise TemplateContentDoesNotMatch(
-                    template.DEFAULT_NAME or "", path
-                )
-        else:
-            with open(path, "w", encoding="utf-8") as fd:
-                fd.write(template.content)
-
-    def load(self, template_name=None):
-        if not template_name:
-            template_name = "linear"
-
-        try:
-            path = self.get_template(template_name)
-
-            with open(path, "r", encoding="utf-8") as fd:
-                content = fd.read()
-
-            return Template(content, name=template_name)
-        except TemplateNotFoundError:
-            for template in self.TEMPLATES:
-                if template.DEFAULT_NAME == template_name:
-                    return template()
-            raise

dvc/repo/plots/templates/confusion.json

@@ -1,107 +0,0 @@
-{
-    "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
-    "data": {
-        "values": "<DVC_METRIC_DATA>"
-    },
-    "title": "<DVC_METRIC_TITLE>",
-    "facet": {
-        "field": "rev",
-        "type": "nominal"
-    },
-    "spec": {
-        "transform": [
-            {
-                "aggregate": [
-                    {
-                        "op": "count",
-                        "as": "xy_count"
-                    }
-                ],
-                "groupby": [
-                    "<DVC_METRIC_Y>",
-                    "<DVC_METRIC_X>"
-                ]
-            },
-            {
-                "impute": "xy_count",
-                "groupby": [
-                    "rev",
-                    "<DVC_METRIC_Y>"
-                ],
-                "key": "<DVC_METRIC_X>",
-                "value": 0
-            },
-            {
-                "impute": "xy_count",
-                "groupby": [
-                    "rev",
-                    "<DVC_METRIC_X>"
-                ],
-                "key": "<DVC_METRIC_Y>",
-                "value": 0
-            },
-            {
-                "joinaggregate": [
-                    {
-                        "op": "max",
-                        "field": "xy_count",
-                        "as": "max_count"
-                    }
-                ],
-                "groupby": []
-            },
-            {
-                "calculate": "datum.xy_count / datum.max_count",
-                "as": "percent_of_max"
-            }
-        ],
-        "encoding": {
-            "x": {
-                "field": "<DVC_METRIC_X>",
-                "type": "nominal",
-                "sort": "ascending",
-                "title": "<DVC_METRIC_X_LABEL>"
-            },
-            "y": {
-                "field": "<DVC_METRIC_Y>",
-                "type": "nominal",
-                "sort": "ascending",
-                "title": "<DVC_METRIC_Y_LABEL>"
-            }
-        },
-        "layer": [
-            {
-                "mark": "rect",
-                "width": 300,
-                "height": 300,
-                "encoding": {
-                    "color": {
-                        "field": "xy_count",
-                        "type": "quantitative",
-                        "title": "",
-                        "scale": {
-                            "domainMin": 0,
-                            "nice": true
-                        }
-                    }
-                }
-            },
-            {
-                "mark": "text",
-                "encoding": {
-                    "text": {
-                        "field": "xy_count",
-                        "type": "quantitative"
-                    },
-                    "color": {
-                        "condition": {
-                            "test": "datum.percent_of_max > 0.5",
-                            "value": "white"
-                        },
-                        "value": "black"
-                    }
-                }
-            }
-        ]
-    }
-}

dvc/repo/plots/templates/confusion_normalized.json

@@ -1,112 +0,0 @@
-{
-    "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
-    "data": {
-        "values": "<DVC_METRIC_DATA>"
-    },
-    "title": "<DVC_METRIC_TITLE>",
-    "facet": {
-        "field": "rev",
-        "type": "nominal"
-    },
-    "spec": {
-        "transform": [
-            {
-                "aggregate": [
-                    {
-                        "op": "count",
-                        "as": "xy_count"
-                    }
-                ],
-                "groupby": [
-                    "<DVC_METRIC_Y>",
-                    "<DVC_METRIC_X>"
-                ]
-            },
-            {
-                "impute": "xy_count",
-                "groupby": [
-                    "rev",
-                    "<DVC_METRIC_Y>"
-                ],
-                "key": "<DVC_METRIC_X>",
-                "value": 0
-            },
-            {
-                "impute": "xy_count",
-                "groupby": [
-                    "rev",
-                    "<DVC_METRIC_X>"
-                ],
-                "key": "<DVC_METRIC_Y>",
-                "value": 0
-            },
-            {
-                "joinaggregate": [
-                    {
-                        "op": "sum",
-                        "field": "xy_count",
-                        "as": "sum_y"
-                    }
-                ],
-                "groupby": [
-                    "<DVC_METRIC_Y>"
-                ]
-            },
-            {
-                "calculate": "datum.xy_count / datum.sum_y",
-                "as": "percent_of_y"
-            }
-        ],
-        "encoding": {
-            "x": {
-                "field": "<DVC_METRIC_X>",
-                "type": "nominal",
-                "sort": "ascending",
-                "title": "<DVC_METRIC_X_LABEL>"
-            },
-            "y": {
-                "field": "<DVC_METRIC_Y>",
-                "type": "nominal",
-                "sort": "ascending",
-                "title": "<DVC_METRIC_Y_LABEL>"
-            }
-        },
-        "layer": [
-            {
-                "mark": "rect",
-                "width": 300,
-                "height": 300,
-                "encoding": {
-                    "color": {
-                        "field": "percent_of_y",
-                        "type": "quantitative",
-                        "title": "",
-                        "scale": {
-                            "domain": [
-                                0,
-                                1
-                            ]
-                        }
-                    }
-                }
-            },
-            {
-                "mark": "text",
-                "encoding": {
-                    "text": {
-                        "field": "percent_of_y",
-                        "type": "quantitative",
-                        "format": ".2f"
-                    },
-                    "color": {
-                        "condition": {
-                            "test": "datum.percent_of_y > 0.5",
-                            "value": "white"
-                        },
-                        "value": "black"
-                    }
-                }
-            }
-        ]
-    }
-}

dvc/repo/plots/templates/linear.json

@@ -1,116 +0,0 @@
-{
-    "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
-    "data": {
-        "values": "<DVC_METRIC_DATA>"
-    },
-    "title": "<DVC_METRIC_TITLE>",
-    "width": 300,
-    "height": 300,
-    "layer": [
-        {
-            "encoding": {
-                "x": {
-                    "field": "<DVC_METRIC_X>",
-                    "type": "quantitative",
-                    "title": "<DVC_METRIC_X_LABEL>"
-                },
-                "y": {
-                    "field": "<DVC_METRIC_Y>",
-                    "type": "quantitative",
-                    "title": "<DVC_METRIC_Y_LABEL>",
-                    "scale": {
-                        "zero": false
-                    }
-                },
-                "color": {
-                    "field": "rev",
-                    "type": "nominal"
-                }
-            },
-            "layer": [
-                {
-                    "mark": "line"
-                },
-                {
-                    "selection": {
-                        "label": {
-                            "type": "single",
-                            "nearest": true,
-                            "on": "mouseover",
-                            "encodings": [
-                                "x"
-                            ],
-                            "empty": "none",
-                            "clear": "mouseout"
-                        }
-                    },
-                    "mark": "point",
-                    "encoding": {
-                        "opacity": {
-                            "condition": {
-                                "selection": "label",
-                                "value": 1
-                            },
-                            "value": 0
-                        }
-                    }
-                }
-            ]
-        },
-        {
-            "transform": [
-                {
-                    "filter": {
-                        "selection": "label"
-                    }
-                }
-            ],
-            "layer": [
-                {
-                    "mark": {
-                        "type": "rule",
-                        "stroke": "gray"
-                    },
-                    "encoding": {
-                        "x": {
-                            "field": "<DVC_METRIC_X>",
-                            "type": "quantitative"
-                        }
-                    }
-                },
-                {
-                    "encoding": {
-                        "text": {
-                            "type": "quantitative",
-                            "field": "<DVC_METRIC_Y>"
-                        },
-                        "x": {
-                            "field": "<DVC_METRIC_X>",
-                            "type": "quantitative"
-                        },
-                        "y": {
-                            "field": "<DVC_METRIC_Y>",
-                            "type": "quantitative"
-                        }
-                    },
-                    "layer": [
-                        {
-                            "mark": {
-                                "type": "text",
-                                "align": "left",
-                                "dx": 5,
-                                "dy": -5
-                            },
-                            "encoding": {
-                                "color": {
-                                    "type": "nominal",
-                                    "field": "rev"
-                                }
-                            }
-                        }
-                    ]
-                }
-            ]
-        }
-    ]
-}

dvc/repo/plots/templates/scatter.json

@@ -1,104 +0,0 @@
-{
-    "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
-    "data": {
-        "values": "<DVC_METRIC_DATA>"
-    },
-    "title": "<DVC_METRIC_TITLE>",
-    "width": 300,
-    "height": 300,
-    "layer": [
-        {
-            "encoding": {
-                "x": {
-                    "field": "<DVC_METRIC_X>",
-                    "type": "quantitative",
-                    "title": "<DVC_METRIC_X_LABEL>"
-                },
-                "y": {
-                    "field": "<DVC_METRIC_Y>",
-                    "type": "quantitative",
-                    "title": "<DVC_METRIC_Y_LABEL>",
-                    "scale": {
-                        "zero": false
-                    }
-                },
-                "color": {
-                    "field": "rev",
-                    "type": "nominal"
-                }
-            },
-            "layer": [
-                {
-                    "mark": "point"
-                },
-                {
-                    "selection": {
-                        "label": {
-                            "type": "single",
-                            "nearest": true,
-                            "on": "mouseover",
-                            "encodings": [
-                                "x"
-                            ],
-                            "empty": "none",
-                            "clear": "mouseout"
-                        }
-                    },
-                    "mark": "point",
-                    "encoding": {
-                        "opacity": {
-                            "condition": {
-                                "selection": "label",
-                                "value": 1
-                            },
-                            "value": 0
-                        }
-                    }
-                }
-            ]
-        },
-        {
-            "transform": [
-                {
-                    "filter": {
-                        "selection": "label"
-                    }
-                }
-            ],
-            "layer": [
-                {
-                    "encoding": {
-                        "text": {
-                            "type": "quantitative",
-                            "field": "<DVC_METRIC_Y>"
-                        },
-                        "x": {
-                            "field": "<DVC_METRIC_X>",
-                            "type": "quantitative"
-                        },
-                        "y": {
-                            "field": "<DVC_METRIC_Y>",
-                            "type": "quantitative"
-                        }
-                    },
-                    "layer": [
-                        {
-                            "mark": {
-                                "type": "text",
-                                "align": "left",
-                                "dx": 5,
-                                "dy": -5
-                            },
-                            "encoding": {
-                                "color": {
-                                    "type": "nominal",
-                                    "field": "rev"
-                                }
-                            }
-                        }
-                    ]
-                }
-            ]
-        }
-    ]
-}

dvc/repo/plots/templates/simple.json

@@ -1,31 +0,0 @@
-{
-    "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
-    "data": {
-        "values": "<DVC_METRIC_DATA>"
-    },
-    "title": "<DVC_METRIC_TITLE>",
-    "width": 300,
-    "height": 300,
-    "mark": {
-        "type": "line"
-    },
-    "encoding": {
-        "x": {
-            "field": "<DVC_METRIC_X>",
-            "type": "quantitative",
-            "title": "<DVC_METRIC_X_LABEL>"
-        },
-        "y": {
-            "field": "<DVC_METRIC_Y>",
-            "type": "quantitative",
-            "title": "<DVC_METRIC_Y_LABEL>",
-            "scale": {
-                "zero": false
-            }
-        },
-        "color": {
-            "field": "rev",
-            "type": "nominal"
-        }
-    }
-}

dvc/repo/plots/templates/smooth.json

@@ -1,39 +0,0 @@
-{
-    "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
-    "data": {
-        "values": "<DVC_METRIC_DATA>"
-    },
-    "title": "<DVC_METRIC_TITLE>",
-    "mark": {
-        "type": "line"
-    },
-    "encoding": {
-        "x": {
-            "field": "<DVC_METRIC_X>",
-            "type": "quantitative",
-            "title": "<DVC_METRIC_X_LABEL>"
-        },
-        "y": {
-            "field": "<DVC_METRIC_Y>",
-            "type": "quantitative",
-            "title": "<DVC_METRIC_Y_LABEL>",
-            "scale": {
-                "zero": false
-            }
-        },
-        "color": {
-            "field": "rev",
-            "type": "nominal"
-        }
-    },
-    "transform": [
-        {
-            "loess": "<DVC_METRIC_Y>",
-            "on": "<DVC_METRIC_X>",
-            "groupby": [
-                "rev"
-            ],
-            "bandwidth": 0.3
-        }
-    ]
-}

setup.cfg

@@ -76,6 +76,7 @@ install_requires =
     diskcache>=5.2.1
     jaraco.windows>=5.7.0; python_version < '3.8' and sys_platform == 'win32'
     scmrepo==0.0.13
+    dvc_render==0.0.2
 
 [options.extras_require]
 all =

tests/conftest.py

@@ -1,3 +1,4 @@
+import json
 import os
 import sys
 from contextlib import suppress
@@ -164,19 +165,10 @@ def pytest_configure(config):
 
 @pytest.fixture
 def custom_template(tmp_dir, dvc):
-    try:
-        import importlib_resources
-    except ImportError:
-        import importlib.resources as importlib_resources
-
-    content = (
-        importlib_resources.files("dvc.repo.plots")
-        / "templates"
-        / "simple.json"
-    ).read_text()
+    from dvc_render.vega_templates import SimpleLinearTemplate
 
     template = tmp_dir / "custom_template.json"
-    template.write_text(content)
+    template.write_text(json.dumps(SimpleLinearTemplate.DEFAULT_CONTENT))
     return template
 
 

tests/func/plots/test_modify.py

@@ -4,7 +4,6 @@ import pytest
 
 from dvc.dvcfile import PIPELINE_LOCK
 from dvc.repo.plots import PropsNotFoundError
-from dvc.repo.plots.template import TemplateNotFoundError
 from dvc.utils import relpath
 
 
@@ -47,6 +46,8 @@ def test_plots_modify_should_not_change_lockfile(
 
 
 def test_plots_modify_not_existing_template(dvc):
+    from dvc_render.vega_templates import TemplateNotFoundError
+
     with pytest.raises(TemplateNotFoundError):
         dvc.plots.modify(
             "metric.json", props={"template": "not-existing-template.json"}

tests/func/test_live.py

@@ -6,7 +6,7 @@ import pytest
 from funcy import first
 
 from dvc import stage as stage_module
-from dvc.render.utils import get_files
+from dvc.render.match import get_files
 
 LIVE_SCRIPT = dedent(
     """

tests/unit/command/test_experiments.py

@@ -862,5 +862,5 @@ def test_show_experiments_pcp(tmp_dir, mocker):
 
     kwargs = td.to_parallel_coordinates.call_args[1]
 
-    assert kwargs["output_path"] == str(tmp_dir / "dvc_plots")
+    assert kwargs["output_path"] == str(tmp_dir / "dvc_plots" / "index.html")
     assert kwargs["color_by"] == "Experiment"

tests/unit/command/test_plots.py

@@ -55,7 +55,7 @@ def test_plots_diff(dvc, mocker, plots_data):
     cmd = cli_args.func(cli_args)
     m = mocker.patch("dvc.repo.plots.diff.diff", return_value=plots_data)
     render_mock = mocker.patch(
-        "dvc.render.utils.render", return_value="html_path"
+        "dvc_render.render_html", return_value="html_path"
     )
 
     assert cmd.run() == 0
@@ -100,7 +100,7 @@ def test_plots_show_vega(dvc, mocker, plots_data):
         return_value=plots_data,
     )
     render_mock = mocker.patch(
-        "dvc.render.utils.render", return_value="html_path"
+        "dvc_render.render_html", return_value="html_path"
     )
 
     assert cmd.run() == 0
@@ -127,10 +127,10 @@ def test_plots_diff_vega(dvc, mocker, capsys, plots_data):
     cmd = cli_args.func(cli_args)
     mocker.patch("dvc.repo.plots.diff.diff", return_value=plots_data)
     mocker.patch(
-        "dvc.render.VegaRenderer.asdict",
-        return_value={"this": "is vega json"},
+        "dvc_render.VegaRenderer.partial_html",
+        return_value=json.dumps({"this": "is vega json"}),
     )
-    render_mock = mocker.patch("dvc.render.utils.render")
+    render_mock = mocker.patch("dvc_render.render_html")
     assert cmd.run() == 0
 
     out, _ = capsys.readouterr()
@@ -156,7 +156,7 @@ def test_plots_diff_open(tmp_dir, dvc, mocker, capsys, plots_data, auto_open):
     mocker.patch("dvc.repo.plots.diff.diff", return_value=plots_data)
 
     index_path = tmp_dir / "dvc_plots" / "index.html"
-    mocker.patch("dvc.render.utils.render", return_value=index_path)
+    mocker.patch("dvc_render.render_html", return_value=index_path)
 
     assert cmd.run() == 0
     mocked_open.assert_called_once_with(index_path.as_uri())
@@ -178,7 +178,7 @@ def test_plots_diff_open_WSL(tmp_dir, dvc, mocker, plots_data):
     mocker.patch("dvc.repo.plots.diff.diff", return_value=plots_data)
 
     index_path = tmp_dir / "dvc_plots" / "index.html"
-    mocker.patch("dvc.render.utils.render", return_value=index_path)
+    mocker.patch("dvc_render.render_html", return_value=index_path)
 
     assert cmd.run() == 0
     mocked_open.assert_called_once_with(str(Path("dvc_plots") / "index.html"))
@@ -253,9 +253,9 @@ def test_should_call_render(tmp_dir, mocker, capsys, plots_data, output):
     output = output or "dvc_plots"
     index_path = tmp_dir / output / "index.html"
     renderers = mocker.MagicMock()
-    mocker.patch("dvc.render.utils.match_renderers", return_value=renderers)
+    mocker.patch("dvc.render.match.match_renderers", return_value=renderers)
     render_mock = mocker.patch(
-        "dvc.render.utils.render", return_value=index_path
+        "dvc_render.render_html", return_value=index_path
     )
 
     assert cmd.run() == 0
@@ -264,7 +264,9 @@ def test_should_call_render(tmp_dir, mocker, capsys, plots_data, output):
     assert index_path.as_uri() in out
 
     render_mock.assert_called_once_with(
-        cmd.repo, renderers, path=tmp_dir / output, html_template_path=None
+        renderers=renderers,
+        output_file=Path(tmp_dir / output / "index.html"),
+        template_path=None,
     )
 
 
@@ -289,66 +291,21 @@ def test_plots_diff_json(dvc, mocker, capsys):
     mocker.patch("dvc.repo.plots.diff.diff", return_value=data)
 
     renderers = mocker.MagicMock()
-    mocker.patch("dvc.render.utils.match_renderers", return_value=renderers)
-    render_mock = mocker.patch("dvc.render.utils.render")
+    mocker.patch("dvc.render.match.match_renderers", return_value=renderers)
+    render_mock = mocker.patch("dvc_render.render_html")
 
     show_json_mock = mocker.patch("dvc.commands.plots._show_json")
 
     assert cmd.run() == 0
 
-    show_json_mock.assert_called_once_with(renderers, "out", True)
+    show_json_mock.assert_called_once_with(renderers, True)
 
     render_mock.assert_not_called()
 
 
-def test_show_json_requires_out(dvc, mocker, capsys):
-    cli_args = parse_args(
-        [
-            "plots",
-            "diff",
-            "HEAD~10",
-            "HEAD~1",
-            "--json",
-            "--targets",
-            "plot.csv",
-        ]
-    )
-    cmd = cli_args.func(cli_args)
-
-    mocker.patch("dvc.repo.plots.diff.diff")
-
-    renderer = mocker.MagicMock()
-    renderer.needs_output_path = False
-    renderer.filename = "foo"
-    renderer.as_json.return_value = "{}"
-    mocker.patch("dvc.render.utils.match_renderers", return_value=[renderer])
-
-    assert cmd.run() == 0
-
-    renderer.needs_output_path = True
-
-    assert cmd.run() == 1
-
-    cli_args = parse_args(
-        [
-            "plots",
-            "diff",
-            "HEAD~10",
-            "HEAD~1",
-            "--json",
-            "--targets",
-            "plot.csv",
-            "-o",
-            "out",
-        ]
-    )
-    cmd = cli_args.func(cli_args)
-    assert cmd.run() == 0
-
-
 @pytest.mark.parametrize("target", (("t1"), (None)))
 def test_plots_templates(tmp_dir, dvc, mocker, capsys, target):
-    assert not os.path.exists(dvc.plots.templates.templates_dir)
+    assert not os.path.exists(dvc.plots.templates_dir)
     mocker.patch(
         "dvc.commands.plots.CmdPlotsTemplates.TEMPLATES_CHOICES",
         ["t1", "t2"],
@@ -361,21 +318,21 @@ def test_plots_templates(tmp_dir, dvc, mocker, capsys, target):
     cli_args = parse_args(arguments)
     assert cli_args.func == CmdPlotsTemplates
 
-    init_mock = mocker.patch("dvc.repo.plots.template.PlotTemplates.init")
+    dump_mock = mocker.patch("dvc_render.vega_templates.dump_templates")
     cmd = cli_args.func(cli_args)
 
     assert cmd.run() == 0
     out, _ = capsys.readouterr()
 
-    init_mock.assert_called_once_with(
+    dump_mock.assert_called_once_with(
         output=os.path.abspath("output"), targets=[target] if target else None
     )
     assert "Templates have been written into 'output'." in out
 
 
 def test_plots_templates_choices(tmp_dir, dvc):
-    from dvc.repo.plots.template import PlotTemplates
+    from dvc_render import TEMPLATES
 
     assert CmdPlotsTemplates.TEMPLATES_CHOICES == list(
-        pluck_attr("DEFAULT_NAME", PlotTemplates.TEMPLATES)
+        pluck_attr("DEFAULT_NAME", TEMPLATES)
     )

tests/unit/render/test_convert.py

@@ -0,0 +1,219 @@
+from dvc.render import (
+    INDEX_FIELD,
+    REVISION_FIELD,
+    REVISIONS_KEY,
+    SRC_FIELD,
+    TYPE_KEY,
+)
+from dvc.render.convert import to_datapoints, to_json
+
+
+def test_to_datapoints_single_revision(mocker):
+    renderer = mocker.MagicMock()
+    renderer.TYPE = "vega"
+
+    input_data = {
+        "revision": {
+            "data": {
+                "filename": {
+                    "data": {
+                        "metric": [
+                            {"v": 1, "v2": 0.1, "v3": 0.01, "v4": 0.001},
+                            {"v": 2, "v2": 0.2, "v3": 0.02, "v4": 0.002},
+                        ]
+                    }
+                }
+            }
+        }
+    }
+    props = {"fields": {"v"}, "x": "v2", "y": "v3"}
+
+    datapoints, resolved_properties = to_datapoints(
+        renderer, input_data, props
+    )
+
+    assert datapoints == [
+        {
+            "v": 1,
+            "v2": 0.1,
+            "v3": 0.01,
+            "rev": "revision",
+            "filename": "filename",
+        },
+        {
+            "v": 2,
+            "v2": 0.2,
+            "v3": 0.02,
+            "rev": "revision",
+            "filename": "filename",
+        },
+    ]
+    assert resolved_properties == {
+        "fields": {"v", "v2", "v3"},
+        "x": "v2",
+        "y": "v3",
+    }
+
+
+def test_to_datapoints_revision_with_error(mocker):
+    renderer = mocker.MagicMock()
+    renderer.TYPE = "vega"
+
+    data = {
+        "v2": {
+            "data": {"file.json": {"data": [{"y": 2}, {"y": 3}], "props": {}}}
+        },
+        "workspace": {
+            "data": {"file.json": {"error": FileNotFoundError(), "props": {}}}
+        },
+    }
+    datapoints, final_props = to_datapoints(renderer, data, {})
+
+    assert datapoints == [
+        {
+            "y": 2,
+            INDEX_FIELD: 0,
+            REVISION_FIELD: "v2",
+            "filename": "file.json",
+        },
+        {
+            "y": 3,
+            INDEX_FIELD: 1,
+            REVISION_FIELD: "v2",
+            "filename": "file.json",
+        },
+    ]
+    assert final_props == {"x": INDEX_FIELD, "y": "y"}
+
+
+def test_to_datapoints_multiple_revisions(mocker):
+    renderer = mocker.MagicMock()
+    renderer.TYPE = "vega"
+
+    metric_1 = [{"y": 2}, {"y": 3}]
+    metric_2 = [{"y": 3}, {"y": 5}]
+    metric_3 = [{"y": 5}, {"y": 6}]
+
+    data = {
+        "HEAD": {
+            "data": {
+                "file.json": {"data": metric_3, "props": {"fields": {"y"}}}
+            }
+        },
+        "v2": {
+            "data": {
+                "file.json": {"data": metric_2, "props": {"fields": {"y"}}}
+            }
+        },
+        "v1": {
+            "data": {
+                "file.json": {"data": metric_1, "props": {"fields": {"y"}}}
+            }
+        },
+    }
+    props = {"fields": {"y"}}
+
+    datapoints, final_props = to_datapoints(renderer, data, props)
+
+    assert datapoints == [
+        {
+            "y": 5,
+            INDEX_FIELD: 0,
+            REVISION_FIELD: "HEAD",
+            "filename": "file.json",
+        },
+        {
+            "y": 6,
+            INDEX_FIELD: 1,
+            REVISION_FIELD: "HEAD",
+            "filename": "file.json",
+        },
+        {
+            "y": 3,
+            INDEX_FIELD: 0,
+            REVISION_FIELD: "v2",
+            "filename": "file.json",
+        },
+        {
+            "y": 5,
+            INDEX_FIELD: 1,
+            REVISION_FIELD: "v2",
+            "filename": "file.json",
+        },
+        {
+            "y": 2,
+            INDEX_FIELD: 0,
+            REVISION_FIELD: "v1",
+            "filename": "file.json",
+        },
+        {
+            "y": 3,
+            INDEX_FIELD: 1,
+            REVISION_FIELD: "v1",
+            "filename": "file.json",
+        },
+    ]
+    assert final_props == {"x": INDEX_FIELD, "y": "y", "fields": {"y", "step"}}
+
+
+def test_to_json_vega(mocker):
+    vega_renderer = mocker.MagicMock()
+    vega_renderer.TYPE = "vega"
+    vega_renderer.get_filled_template.return_value = '{"this": "is vega"}'
+    vega_renderer.datapoints = [
+        {"x": 1, "y": 2, REVISION_FIELD: "foo", "filename": "foo.json"},
+        {"x": 2, "y": 1, REVISION_FIELD: "bar", "filename": "foo.json"},
+    ]
+    result = to_json(vega_renderer)
+    assert result[0] == {
+        TYPE_KEY: vega_renderer.TYPE,
+        REVISIONS_KEY: ["bar", "foo"],
+        "content": {"this": "is vega"},
+        "datapoints": {
+            "foo": [
+                {"x": 1, "y": 2, "filename": "foo.json"},
+            ],
+            "bar": [
+                {"x": 2, "y": 1, "filename": "foo.json"},
+            ],
+        },
+    }
+    vega_renderer.get_filled_template.assert_called()
+
+
+def test_to_json_vega_split(mocker):
+    vega_renderer = mocker.MagicMock()
+    vega_renderer.TYPE = "vega"
+    vega_renderer.get_filled_template.return_value = (
+        '{"this": "is split vega"}'
+    )
+    vega_renderer.datapoints = [
+        {"x": 1, "y": 2, REVISION_FIELD: "foo", "filename": "foo.json"},
+        {"x": 2, "y": 1, REVISION_FIELD: "bar", "filename": "foo.json"},
+    ]
+    result = to_json(vega_renderer, split=True)
+    assert result[0] == {
+        TYPE_KEY: vega_renderer.TYPE,
+        REVISIONS_KEY: ["bar", "foo"],
+        "content": {"this": "is split vega"},
+        "datapoints": {
+            "foo": [{"x": 1, "y": 2, "filename": "foo.json"}],
+            "bar": [{"x": 2, "y": 1, "filename": "foo.json"}],
+        },
+    }
+    vega_renderer.get_filled_template.assert_called_with(skip_anchors=["data"])
+
+
+def test_to_json_image(mocker):
+    image_renderer = mocker.MagicMock()
+    image_renderer.TYPE = "image"
+    image_renderer.datapoints = [
+        {SRC_FIELD: "contentfoo", REVISION_FIELD: "foo"},
+        {SRC_FIELD: "contentbar", REVISION_FIELD: "bar"},
+    ]
+    result = to_json(image_renderer)
+    assert result[0] == {
+        "url": image_renderer.datapoints[0].get(SRC_FIELD),
+        REVISIONS_KEY: image_renderer.datapoints[0].get(REVISION_FIELD),
+        TYPE_KEY: image_renderer.TYPE,
+    }

tests/unit/render/test_html.py

@@ -1,47 +0,0 @@
-import pytest
-
-from dvc.render.html import HTML, PAGE_HTML, MissingPlaceholderError
-
-CUSTOM_PAGE_HTML = """<!DOCTYPE html>
-<html>
-<head>
-    <title>TITLE</title>
-    <script type="text/javascript" src="vega"></script>
-    <script type="text/javascript" src="vega-lite"></script>
-    <script type="text/javascript" src="vega-embed"></script>
-</head>
-<body>
-    {plot_divs}
-</body>
-</html>"""
-
-
-@pytest.mark.parametrize(
-    "template,page_elements,expected_page",
-    [
-        (
-            None,
-            ["content"],
-            PAGE_HTML.format(plot_divs="content", refresh_tag="", scripts=""),
-        ),
-        (
-            CUSTOM_PAGE_HTML,
-            ["content"],
-            CUSTOM_PAGE_HTML.format(plot_divs="content"),
-        ),
-    ],
-)
-def test_html(tmp_dir, template, page_elements, expected_page):
-    page = HTML(template)
-    page.elements = page_elements
-
-    result = page.embed()
-
-    assert result == expected_page
-
-
-def test_no_placeholder(tmp_dir):
-    template = "<head></head><body></body>"
-
-    with pytest.raises(MissingPlaceholderError):
-        HTML(template)

tests/unit/render/test_image.py

@@ -1,79 +0,0 @@
-import json
-import os
-
-import pytest
-
-from dvc.render.image import ImageRenderer
-
-
-@pytest.mark.parametrize(
-    "extension, matches",
-    (
-        (".csv", False),
-        (".json", False),
-        (".tsv", False),
-        (".yaml", False),
-        (".jpg", True),
-        (".gif", True),
-        (".jpeg", True),
-        (".png", True),
-    ),
-)
-def test_matches(extension, matches):
-    filename = "file" + extension
-    data = {
-        "HEAD": {"data": {filename: {}}},
-        "v1": {"data": {filename: {}}},
-    }
-    assert ImageRenderer.matches(data) == matches
-
-
-def test_render(tmp_dir):
-    data = {"workspace": {"data": {"file.jpg": {"data": b"content"}}}}
-
-    page_dir = os.path.join("some", "path")
-    html = ImageRenderer(data).generate_html(page_dir)
-
-    assert (tmp_dir / page_dir).is_dir()
-    image_file = tmp_dir / page_dir / "static" / "workspace_file.jpg"
-    assert image_file.is_file()
-
-    with open(image_file, "rb") as fobj:
-        assert fobj.read() == b"content"
-
-    assert "<p>file.jpg</p>" in html
-    assert (
-        f'<img src="{os.path.join("static", "workspace_file.jpg")}">' in html
-    )
-
-
-def test_as_json(tmp_dir):
-    data = {
-        "workspace": {"data": {"file.jpg": {"data": b"content"}}},
-        "HEAD": {"data": {"file.jpg": {"data": b"content2"}}},
-    }
-
-    page_dir = os.path.join("some", "path")
-    json_content = json.loads(ImageRenderer(data).as_json(path=page_dir))
-
-    img1 = os.path.join(page_dir, "workspace_file.jpg")
-    img2 = os.path.join(page_dir, "HEAD_file.jpg")
-
-    assert {
-        "type": "image",
-        "revisions": ["workspace"],
-        "url": os.path.abspath(img1),
-    } in json_content
-    assert {
-        "type": "image",
-        "revisions": ["HEAD"],
-        "url": os.path.abspath(img2),
-    } in json_content
-
-    def assert_file_with_content(path, content):
-        assert os.path.isfile(path)
-        with open(path, "rb") as fd:
-            assert fd.read() == content
-
-    assert_file_with_content(img1, b"content")
-    assert_file_with_content(img2, b"content2")

tests/unit/render/test_image_converter.py

@@ -0,0 +1,30 @@
+from dvc.render import REVISION_FIELD, SRC_FIELD
+from dvc.render.image_converter import ImageConverter
+
+
+def test_image_converter_no_out():
+    data = b"content"
+    converter = ImageConverter()
+
+    datapoints, _ = converter.convert("r", "image.png", data)
+
+    assert datapoints[0] == {
+        REVISION_FIELD: "r",
+        "filename": "image.png",
+        SRC_FIELD: converter._encode_image(b"content"),
+    }
+
+
+def test_image_converter_with_out(tmp_dir):
+    data = b"content"
+    converter = ImageConverter({"out": tmp_dir / "foo"})
+
+    datapoints, _ = converter.convert("r", "image.png", data)
+
+    assert datapoints[0] == {
+        REVISION_FIELD: "r",
+        "filename": "image.png",
+        SRC_FIELD: str(tmp_dir / "foo" / "r_image.png"),
+    }
+
+    assert (tmp_dir / "foo" / "r_image.png").read_bytes() == b"content"

tests/unit/render/test_match.py

@@ -0,0 +1,168 @@
+from dvc.render.match import (
+    group_by_filename,
+    match_renderers,
+    squash_plots_properties,
+)
+
+
+def test_group_by_filename():
+    error = FileNotFoundError()
+    data = {
+        "v2": {
+            "data": {
+                "file.json": {"data": [{"y": 2}, {"y": 3}], "props": {}},
+                "other_file.jpg": {"data": "content"},
+            }
+        },
+        "v1": {
+            "data": {"file.json": {"data": [{"y": 4}, {"y": 5}], "props": {}}}
+        },
+        "workspace": {
+            "data": {
+                "file.json": {"error": error, "props": {}},
+                "other_file.jpg": {"data": "content2"},
+            }
+        },
+    }
+
+    results = group_by_filename(data)
+    assert results["file.json"] == {
+        "v2": {
+            "data": {
+                "file.json": {"data": [{"y": 2}, {"y": 3}], "props": {}},
+            }
+        },
+        "v1": {
+            "data": {"file.json": {"data": [{"y": 4}, {"y": 5}], "props": {}}}
+        },
+        "workspace": {
+            "data": {
+                "file.json": {"error": error, "props": {}},
+            }
+        },
+    }
+    assert results["other_file.jpg"] == {
+        "v2": {
+            "data": {
+                "other_file.jpg": {"data": "content"},
+            }
+        },
+        "workspace": {
+            "data": {
+                "other_file.jpg": {"data": "content2"},
+            }
+        },
+    }
+
+
+def test_squash_plots_properties():
+    error = FileNotFoundError()
+    group = {
+        "v2": {
+            "data": {
+                "file.json": {
+                    "data": [{"y": 2}, {"y": 3}],
+                    "props": {"foo": 1},
+                },
+            }
+        },
+        "v1": {
+            "data": {
+                "file.json": {
+                    "data": [{"y": 4}, {"y": 5}],
+                    "props": {"bar": 1},
+                }
+            }
+        },
+        "workspace": {
+            "data": {
+                "file.json": {"error": error, "props": {}},
+            }
+        },
+    }
+
+    plot_properties = squash_plots_properties(group)
+
+    assert plot_properties == {"foo": 1, "bar": 1}
+
+
+def test_match_renderers_no_out(mocker):
+    from dvc import render
+
+    vega_convert = mocker.spy(render.vega_converter.VegaConverter, "convert")
+    image_convert = mocker.spy(
+        render.image_converter.ImageConverter, "convert"
+    )
+    image_encode = mocker.spy(
+        render.image_converter.ImageConverter, "_encode_image"
+    )
+    image_write = mocker.spy(
+        render.image_converter.ImageConverter, "_write_image"
+    )
+
+    error = FileNotFoundError()
+    data = {
+        "v2": {
+            "data": {
+                "file.json": {"data": [{"y": 2}, {"y": 3}], "props": {}},
+                "other_file.jpg": {"data": b"content"},
+            }
+        },
+        "v1": {
+            "data": {"file.json": {"data": [{"y": 4}, {"y": 5}], "props": {}}}
+        },
+        "workspace": {
+            "data": {
+                "file.json": {"error": error, "props": {}},
+                "other_file.jpg": {"data": b"content2"},
+            }
+        },
+    }
+
+    renderers = match_renderers(data)
+
+    assert {r.TYPE for r in renderers} == {"vega", "image"}
+    vega_convert.assert_called()
+    image_convert.assert_called()
+    image_encode.assert_called()
+    image_write.assert_not_called()
+
+
+def test_match_renderers_with_out(tmp_dir, mocker):
+    from dvc import render
+
+    image_encode = mocker.spy(
+        render.image_converter.ImageConverter, "_encode_image"
+    )
+    image_write = mocker.spy(
+        render.image_converter.ImageConverter, "_write_image"
+    )
+
+    error = FileNotFoundError()
+    data = {
+        "v2": {
+            "data": {
+                "file.json": {"data": [{"y": 2}, {"y": 3}], "props": {}},
+                "other_file.jpg": {"data": b"content"},
+            }
+        },
+        "v1": {
+            "data": {"file.json": {"data": [{"y": 4}, {"y": 5}], "props": {}}}
+        },
+        "workspace": {
+            "data": {
+                "file.json": {"error": error, "props": {}},
+                "other_file.jpg": {"data": b"content2"},
+            }
+        },
+    }
+
+    match_renderers(data, out=tmp_dir / "foo")
+
+    image_encode.assert_not_called()
+    image_write.assert_called()
+
+    assert (tmp_dir / "foo" / "v2_other_file.jpg").read_bytes() == b"content"
+    assert (
+        tmp_dir / "foo" / "workspace_other_file.jpg"
+    ).read_bytes() == b"content2"

tests/unit/render/test_parallel_coordinates.py

@@ -1,156 +0,0 @@
-import json
-
-from dvc.compare import TabularData
-from dvc.render.html import write
-from dvc.render.plotly import ParallelCoordinatesRenderer
-
-# pylint: disable=W1514
-
-
-def expected_format(result):
-    assert "data" in result
-    assert "layout" in result
-    assert isinstance(result["data"], list)
-    assert result["data"][0]["type"] == "parcoords"
-    assert isinstance(result["data"][0]["dimensions"], list)
-    return True
-
-
-def test_scalar_columns():
-    td = TabularData(["col-1", "col-2", "col-3"])
-    td.extend([["0.1", "1", ""], ["2", "0.2", "0"]])
-    renderer = ParallelCoordinatesRenderer(td)
-
-    result = json.loads(renderer.as_json())
-
-    assert expected_format(result)
-
-    assert result["data"][0]["dimensions"][0] == {
-        "label": "col-1",
-        "values": [0.1, 2.0],
-    }
-    assert result["data"][0]["dimensions"][1] == {
-        "label": "col-2",
-        "values": [1.0, 0.2],
-    }
-    assert result["data"][0]["dimensions"][2] == {
-        "label": "col-3",
-        "values": [None, 0],
-    }
-
-
-def test_categorical_columns():
-    td = TabularData(["col-1", "col-2"])
-    td.extend([["foo", ""], ["bar", "foobar"], ["foo", ""]])
-    renderer = ParallelCoordinatesRenderer(td)
-
-    result = json.loads(renderer.as_json())
-
-    assert expected_format(result)
-
-    assert result["data"][0]["dimensions"][0] == {
-        "label": "col-1",
-        "values": [1, 0, 1],
-        "tickvals": [1, 0, 1],
-        "ticktext": ["foo", "bar", "foo"],
-    }
-    assert result["data"][0]["dimensions"][1] == {
-        "label": "col-2",
-        "values": [1, 0, 1],
-        "tickvals": [1, 0, 1],
-        "ticktext": ["Missing", "foobar", "Missing"],
-    }
-
-
-def test_mixed_columns():
-    td = TabularData(["categorical", "scalar"])
-    td.extend([["foo", "0.1"], ["bar", "2"]])
-    renderer = ParallelCoordinatesRenderer(td)
-
-    result = json.loads(renderer.as_json())
-
-    assert expected_format(result)
-
-    assert result["data"][0]["dimensions"][0] == {
-        "label": "categorical",
-        "values": [1, 0],
-        "tickvals": [1, 0],
-        "ticktext": ["foo", "bar"],
-    }
-    assert result["data"][0]["dimensions"][1] == {
-        "label": "scalar",
-        "values": [0.1, 2.0],
-    }
-
-
-def test_color_by_scalar():
-    td = TabularData(["categorical", "scalar"])
-    td.extend([["foo", "0.1"], ["bar", "2"]])
-    renderer = ParallelCoordinatesRenderer(td, color_by="scalar")
-
-    result = json.loads(renderer.as_json())
-
-    assert expected_format(result)
-    assert result["data"][0]["line"] == {
-        "color": [0.1, 2.0],
-        "showscale": True,
-        "colorbar": {"title": "scalar"},
-    }
-
-
-def test_color_by_categorical():
-    td = TabularData(["categorical", "scalar"])
-    td.extend([["foo", "0.1"], ["bar", "2"]])
-    renderer = ParallelCoordinatesRenderer(td, color_by="categorical")
-
-    result = json.loads(renderer.as_json())
-
-    assert expected_format(result)
-    assert result["data"][0]["line"] == {
-        "color": [1, 0],
-        "showscale": True,
-        "colorbar": {
-            "title": "categorical",
-            "tickmode": "array",
-            "tickvals": [1, 0],
-            "ticktext": ["foo", "bar"],
-        },
-    }
-
-
-def test_write_parallel_coordinates(tmp_dir):
-    td = TabularData(["categorical", "scalar"])
-    td.extend([["foo", "0.1"], ["bar", "2"]])
-
-    renderer = ParallelCoordinatesRenderer(td)
-    html_path = write(tmp_dir, renderers=[renderer])
-
-    html_text = html_path.read_text()
-
-    assert ParallelCoordinatesRenderer.SCRIPTS in html_text
-
-    div = ParallelCoordinatesRenderer.DIV.format(
-        id="plot_experiments", partial=renderer.as_json()
-    )
-    assert div in html_text
-
-
-def test_fill_value():
-    td = TabularData(["categorical", "scalar"])
-    td.extend([["foo", "-"], ["-", "2"]])
-    renderer = ParallelCoordinatesRenderer(td, fill_value="-")
-
-    result = json.loads(renderer.as_json())
-
-    assert expected_format(result)
-
-    assert result["data"][0]["dimensions"][0] == {
-        "label": "categorical",
-        "values": [0, 1],
-        "tickvals": [0, 1],
-        "ticktext": ["foo", "Missing"],
-    }
-    assert result["data"][0]["dimensions"][1] == {
-        "label": "scalar",
-        "values": [None, 2.0],
-    }

tests/unit/render/test_render.py

@@ -1,78 +0,0 @@
-import os
-
-import dpath
-
-from dvc.render.utils import match_renderers, render
-from dvc.render.vega import VegaRenderer
-
-
-def assert_website_has_image(page_path, revision, filename, image_content):
-    index_path = page_path / "index.html"
-    assert index_path.is_file()
-    index_content = index_path.read_text()
-
-    resources_filename = f"{revision}_{filename.replace(os.sep, '_')}"
-    image_path = page_path / "static" / resources_filename
-    assert image_path.is_file()
-
-    img_html = f'<img src="{os.path.join("static", resources_filename)}">'
-    assert img_html in index_content
-    with open(image_path, "rb") as fobj:
-        assert fobj.read() == image_content
-
-
-def test_render(tmp_dir, dvc):
-    data = {
-        "HEAD": {
-            "data": {
-                "file.json": {
-                    "data": [{"y": 5}, {"y": 6}],
-                    "props": {"fields": {"y"}},
-                },
-                os.path.join("sub", "other_file.jpg"): {"data": b"content"},
-            }
-        },
-        "v2": {
-            "data": {
-                "file.json": {
-                    "data": [{"y": 3}, {"y": 5}],
-                    "props": {"fields": {"y"}},
-                },
-                "other_file.jpg": {"data": b"content2"},
-            }
-        },
-        "v1": {
-            "data": {
-                "some.csv": {
-                    "data": [{"y": 2}, {"y": 3}],
-                    "props": {"fields": {"y"}},
-                },
-                "another.gif": {"data": b"content3"},
-            }
-        },
-    }
-    renderers = match_renderers(data, dvc.plots.templates)
-
-    render(dvc, renderers, path=os.path.join("results", "dir"))
-    page_path = tmp_dir / "results" / "dir"
-    index_path = page_path / "index.html"
-
-    assert index_path.is_file()
-    assert_website_has_image(
-        page_path, "HEAD", os.path.join("sub", "other_file.jpg"), b"content"
-    )
-    assert_website_has_image(page_path, "v2", "other_file.jpg", b"content2")
-    assert_website_has_image(page_path, "v1", "another.gif", b"content3")
-
-    def clean(txt: str) -> str:
-        return txt.replace("\n", "").replace("\r", "").replace(" ", "")
-
-    def get_vega_string(data, filename):
-        file_data = dpath.util.search(data, ["*", "*", filename])
-        return VegaRenderer(
-            file_data, dvc.plots.templates.load()
-        ).partial_html()
-
-    index_content = index_path.read_text()
-    assert clean(get_vega_string(data, "file.json")) in clean(index_content)
-    assert clean(get_vega_string(data, "some.csv")) in clean(index_content)

tests/unit/render/test_renderer.py

@@ -1,9 +0,0 @@
-from dvc.render.base import Renderer
-
-
-def test_remove_special_characters():
-    special_chars = r"!@#$%^&*()[]{};,<>?\/:.|`~=_+"
-    dirty = f"plot_name{special_chars}"
-    assert Renderer._remove_special_chars(dirty) == "plot_name" + "_" * len(
-        special_chars
-    )

tests/unit/render/test_vega.py

@@ -1,375 +0,0 @@
-import json
-import os
-from collections import OrderedDict
-
-import pytest
-from funcy import first
-
-from dvc.render.base import BadTemplateError
-from dvc.render.data import INDEX_FIELD, REVISION_FIELD
-from dvc.render.utils import group_by_filename
-from dvc.render.vega import VegaRenderer
-from dvc.repo.plots.template import NoFieldInDataError
-
-
-def test_group_plots_data():
-    error = FileNotFoundError()
-    data = {
-        "v2": {
-            "data": {
-                "file.json": {"data": [{"y": 2}, {"y": 3}], "props": {}},
-                "other_file.jpg": {"data": "content"},
-            }
-        },
-        "v1": {
-            "data": {"file.json": {"data": [{"y": 4}, {"y": 5}], "props": {}}}
-        },
-        "workspace": {
-            "data": {
-                "file.json": {"error": error, "props": {}},
-                "other_file.jpg": {"data": "content2"},
-            }
-        },
-    }
-
-    results = group_by_filename(data)
-    assert {
-        "v2": {
-            "data": {
-                "file.json": {"data": [{"y": 2}, {"y": 3}], "props": {}},
-            }
-        },
-        "v1": {
-            "data": {"file.json": {"data": [{"y": 4}, {"y": 5}], "props": {}}}
-        },
-        "workspace": {
-            "data": {
-                "file.json": {"error": error, "props": {}},
-            }
-        },
-    } in results
-    assert {
-        "v2": {
-            "data": {
-                "other_file.jpg": {"data": "content"},
-            }
-        },
-        "workspace": {
-            "data": {
-                "other_file.jpg": {"data": "content2"},
-            }
-        },
-    } in results
-
-
-def test_one_column(tmp_dir, dvc):
-    props = {
-        "x_label": "x_title",
-        "y_label": "y_title",
-        "title": "mytitle",
-    }
-    data = {
-        "workspace": {
-            "data": {"file.json": {"data": [{"val": 2}, {"val": 3}]}}
-        }
-    }
-
-    plot_content = VegaRenderer(
-        data, template=dvc.plots.templates.load(), properties=props
-    ).asdict()
-
-    assert plot_content["title"] == "mytitle"
-    assert plot_content["data"]["values"] == [
-        {"val": 2, INDEX_FIELD: 0, REVISION_FIELD: "workspace"},
-        {"val": 3, INDEX_FIELD: 1, REVISION_FIELD: "workspace"},
-    ]
-    assert (
-        first(plot_content["layer"])["encoding"]["x"]["field"] == INDEX_FIELD
-    )
-    assert first(plot_content["layer"])["encoding"]["y"]["field"] == "val"
-    assert first(plot_content["layer"])["encoding"]["x"]["title"] == "x_title"
-    assert first(plot_content["layer"])["encoding"]["y"]["title"] == "y_title"
-
-
-def test_multiple_columns(tmp_dir, scm, dvc):
-    metric = [
-        OrderedDict([("first_val", 100), ("second_val", 100), ("val", 2)]),
-        OrderedDict([("first_val", 200), ("second_val", 300), ("val", 3)]),
-    ]
-
-    data = {
-        "workspace": {"data": {"file.json": {"data": metric, "props": {}}}}
-    }
-
-    plot_content = VegaRenderer(data, dvc.plots.templates.load()).asdict()
-
-    assert plot_content["data"]["values"] == [
-        {
-            "val": 2,
-            INDEX_FIELD: 0,
-            REVISION_FIELD: "workspace",
-            "first_val": 100,
-            "second_val": 100,
-        },
-        {
-            "val": 3,
-            INDEX_FIELD: 1,
-            REVISION_FIELD: "workspace",
-            "first_val": 200,
-            "second_val": 300,
-        },
-    ]
-    assert (
-        first(plot_content["layer"])["encoding"]["x"]["field"] == INDEX_FIELD
-    )
-    assert first(plot_content["layer"])["encoding"]["y"]["field"] == "val"
-
-
-def test_choose_axes(tmp_dir, scm, dvc):
-    metric = [
-        OrderedDict([("first_val", 100), ("second_val", 100), ("val", 2)]),
-        OrderedDict([("first_val", 200), ("second_val", 300), ("val", 3)]),
-    ]
-
-    props = {"x": "first_val", "y": "second_val"}
-
-    data = {
-        "workspace": {"data": {"file.json": {"data": metric, "props": props}}}
-    }
-    plot_content = VegaRenderer(
-        data, template=dvc.plots.templates.load(), properties=props
-    ).asdict()
-
-    assert plot_content["data"]["values"] == [
-        {
-            "val": 2,
-            REVISION_FIELD: "workspace",
-            "first_val": 100,
-            "second_val": 100,
-        },
-        {
-            "val": 3,
-            REVISION_FIELD: "workspace",
-            "first_val": 200,
-            "second_val": 300,
-        },
-    ]
-    assert (
-        first(plot_content["layer"])["encoding"]["x"]["field"] == "first_val"
-    )
-    assert (
-        first(plot_content["layer"])["encoding"]["y"]["field"] == "second_val"
-    )
-
-
-def test_confusion(tmp_dir, dvc):
-    confusion_matrix = [
-        {"predicted": "B", "actual": "A"},
-        {"predicted": "A", "actual": "A"},
-    ]
-    props = {"template": "confusion", "x": "predicted", "y": "actual"}
-
-    data = {"workspace": {"data": {"file.json": {"data": confusion_matrix}}}}
-
-    plot_content = VegaRenderer(
-        data, template=dvc.plots.templates.load("confusion"), properties=props
-    ).asdict()
-
-    assert plot_content["data"]["values"] == [
-        {"predicted": "B", "actual": "A", REVISION_FIELD: "workspace"},
-        {"predicted": "A", "actual": "A", REVISION_FIELD: "workspace"},
-    ]
-    assert plot_content["spec"]["transform"][0]["groupby"] == [
-        "actual",
-        "predicted",
-    ]
-    assert plot_content["spec"]["encoding"]["x"]["field"] == "predicted"
-    assert plot_content["spec"]["encoding"]["y"]["field"] == "actual"
-
-
-def test_multiple_revs_default(tmp_dir, scm, dvc):
-    metric_1 = [{"y": 2}, {"y": 3}]
-    metric_2 = [{"y": 3}, {"y": 5}]
-    metric_3 = [{"y": 5}, {"y": 6}]
-
-    data = {
-        "HEAD": {
-            "data": {
-                "file.json": {"data": metric_3, "props": {"fields": {"y"}}}
-            }
-        },
-        "v2": {
-            "data": {
-                "file.json": {"data": metric_2, "props": {"fields": {"y"}}}
-            }
-        },
-        "v1": {
-            "data": {
-                "file.json": {"data": metric_1, "props": {"fields": {"y"}}}
-            }
-        },
-    }
-
-    plot_content = VegaRenderer(data, dvc.plots.templates.load()).asdict()
-
-    assert plot_content["data"]["values"] == [
-        {"y": 5, INDEX_FIELD: 0, REVISION_FIELD: "HEAD"},
-        {"y": 6, INDEX_FIELD: 1, REVISION_FIELD: "HEAD"},
-        {"y": 3, INDEX_FIELD: 0, REVISION_FIELD: "v2"},
-        {"y": 5, INDEX_FIELD: 1, REVISION_FIELD: "v2"},
-        {"y": 2, INDEX_FIELD: 0, REVISION_FIELD: "v1"},
-        {"y": 3, INDEX_FIELD: 1, REVISION_FIELD: "v1"},
-    ]
-    assert (
-        first(plot_content["layer"])["encoding"]["x"]["field"] == INDEX_FIELD
-    )
-    assert first(plot_content["layer"])["encoding"]["y"]["field"] == "y"
-
-
-def test_metric_missing(tmp_dir, scm, dvc, caplog):
-
-    metric = [{"y": 2}, {"y": 3}]
-    data = {
-        "v2": {"data": {"file.json": {"data": metric, "props": {}}}},
-        "workspace": {
-            "data": {"file.json": {"error": FileNotFoundError(), "props": {}}}
-        },
-    }
-    plot_content = VegaRenderer(data, dvc.plots.templates.load()).asdict()
-
-    assert plot_content["data"]["values"] == [
-        {"y": 2, INDEX_FIELD: 0, REVISION_FIELD: "v2"},
-        {"y": 3, INDEX_FIELD: 1, REVISION_FIELD: "v2"},
-    ]
-    assert (
-        first(plot_content["layer"])["encoding"]["x"]["field"] == INDEX_FIELD
-    )
-    assert first(plot_content["layer"])["encoding"]["y"]["field"] == "y"
-
-
-def test_bad_template(tmp_dir, dvc):
-    metric = [{"val": 2}, {"val": 3}]
-    data = {"workspace": {"data": {"file.json": {"data": metric}}}}
-
-    from dvc.repo.plots.template import Template
-
-    with pytest.raises(BadTemplateError):
-        VegaRenderer(data, Template("name", "content")).asdict()
-
-
-def test_plot_choose_columns(tmp_dir, scm, dvc, custom_template):
-    metric = [{"a": 1, "b": 2, "c": 3}, {"a": 2, "b": 3, "c": 4}]
-    props = {
-        "fields": {"b", "c"},
-        "x": "b",
-        "y": "c",
-    }
-    data = {
-        "workspace": {"data": {"file.json": {"data": metric, "props": props}}}
-    }
-
-    plot_content = VegaRenderer(
-        data,
-        template=dvc.plots.templates.load(os.fspath(custom_template)),
-        properties=props,
-    ).asdict()
-
-    assert plot_content["data"]["values"] == [
-        {"b": 2, "c": 3, REVISION_FIELD: "workspace"},
-        {"b": 3, "c": 4, REVISION_FIELD: "workspace"},
-    ]
-    assert plot_content["encoding"]["x"]["field"] == "b"
-    assert plot_content["encoding"]["y"]["field"] == "c"
-
-
-def test_raise_on_wrong_field(tmp_dir, scm, dvc):
-    metric = [{"val": 2}, {"val": 3}]
-    props = {"x": "no_val"}
-    data = {"workspace": {"data": {"file.json": {"data": metric}}}}
-
-    with pytest.raises(NoFieldInDataError):
-        VegaRenderer(
-            data, template=dvc.plots.templates.load(), properties=props
-        ).asdict()
-
-
-@pytest.mark.parametrize(
-    "extension, matches",
-    (
-        (".csv", True),
-        (".json", True),
-        (".tsv", True),
-        (".yaml", True),
-        (".jpg", False),
-        (".gif", False),
-        (".jpeg", False),
-        (".png", False),
-    ),
-)
-def test_matches(extension, matches):
-    filename = "file" + extension
-    data = {
-        "HEAD": {"data": {filename: {}}},
-        "v1": {"data": {filename: {}}},
-    }
-    assert VegaRenderer.matches(data) == matches
-
-
-@pytest.mark.parametrize(
-    "template_path, target_name",
-    [
-        (os.path.join(".dvc", "plots", "template.json"), "template"),
-        (os.path.join(".dvc", "plots", "template.json"), "template.json"),
-        (
-            os.path.join(".dvc", "plots", "subdir", "template.json"),
-            os.path.join("subdir", "template.json"),
-        ),
-        (
-            os.path.join(".dvc", "plots", "subdir", "template.json"),
-            os.path.join("subdir", "template"),
-        ),
-        ("template.json", "template.json"),
-    ],
-)
-def test_should_resolve_template(tmp_dir, dvc, template_path, target_name):
-    os.makedirs(os.path.abspath(os.path.dirname(template_path)), exist_ok=True)
-    with open(template_path, "w", encoding="utf-8") as fd:
-        fd.write("template_content")
-
-    assert dvc.plots.templates.get_template(target_name) == os.path.abspath(
-        template_path
-    )
-
-
-# TODO remove content checks
-def test_as_json(tmp_dir, scm, dvc):
-    metric = [{"a": 1, "b": 2, "c": 3}, {"a": 2, "b": 3, "c": 4}]
-    data = {"workspace": {"data": {"file.json": {"data": metric}}}}
-    props = {"fields": {"b"}}
-
-    renderer = VegaRenderer(
-        data, template=dvc.plots.templates.load(), properties=props
-    )
-    plot_content = renderer.asdict()
-    plot_as_json = first(json.loads(renderer.as_json()))
-
-    assert plot_as_json["type"] == "vega"
-    assert plot_as_json["revisions"] == ["workspace"]
-    assert plot_as_json["content"] == plot_content
-
-
-def test_as_json_split(tmp_dir, scm, dvc):
-    metric = [{"a": 1, "b": 2, "c": 3}, {"a": 2, "b": 3, "c": 4}]
-    data = {"workspace": {"data": {"file.json": {"data": metric}}}}
-    props = {"fields": {"b"}}
-
-    renderer = VegaRenderer(
-        data, template=dvc.plots.templates.load(), properties=props
-    )
-    plot_content = renderer.asdict(fill_data=False)
-    plot_as_json = first(json.loads(renderer.as_json(fill_data=False)))
-
-    assert plot_as_json["type"] == "vega"
-    assert plot_as_json["revisions"] == ["workspace"]
-    assert plot_as_json["content"] == plot_content
-    assert plot_as_json["datapoints"] == renderer._converted_data[0]

tests/unit/render/test_vega_converter.py

@@ -2,13 +2,13 @@ from collections import OrderedDict
 
 import pytest
 
-from dvc.render.data import (
-    Converter,
+from dvc.render.vega_converter import (
     FieldsNotFoundError,
+    PlotDataStructureError,
+    VegaConverter,
     _filter_fields,
     _find_first_list,
     _lists,
-    to_datapoints,
 )
 
 
@@ -20,6 +20,9 @@ def test_find_first_list_in_dict():
     assert _find_first_list(dmetric, fields=set()) == m1
     assert _find_first_list(dmetric, fields={"x"}) == m2
 
+    with pytest.raises(PlotDataStructureError):
+        _find_first_list(dmetric, fields={"foo"})
+
 
 def test_filter_fields():
     m = [{"accuracy": 1, "loss": 2}, {"accuracy": 3, "loss": 4}]
@@ -57,14 +60,20 @@ def test_finding_lists(dictionary, expected_result):
             # default x and y
             {"metric": [{"v": 1}, {"v": 2}]},
             {},
-            [{"v": 1, "step": 0}, {"v": 2, "step": 1}],
+            [
+                {"v": 1, "step": 0, "filename": "f", "rev": "r"},
+                {"v": 2, "step": 1, "filename": "f", "rev": "r"},
+            ],
             {"x": "step", "y": "v"},
         ),
         (
             # filter fields
             {"metric": [{"v": 1, "v2": 0.1}, {"v": 2, "v2": 0.2}]},
             {"fields": {"v"}},
-            [{"v": 1, "step": 0}, {"v": 2, "step": 1}],
+            [
+                {"v": 1, "step": 0, "filename": "f", "rev": "r"},
+                {"v": 2, "step": 1, "filename": "f", "rev": "r"},
+            ],
             {
                 "x": "step",
                 "y": "v",
@@ -75,7 +84,10 @@ def test_finding_lists(dictionary, expected_result):
             # choose x and y
             {"metric": [{"v": 1, "v2": 0.1}, {"v": 2, "v2": 0.2}]},
             {"x": "v", "y": "v2"},
-            [{"v": 1, "v2": 0.1}, {"v": 2, "v2": 0.2}],
+            [
+                {"v": 1, "v2": 0.1, "filename": "f", "rev": "r"},
+                {"v": 2, "v2": 0.2, "filename": "f", "rev": "r"},
+            ],
             {"x": "v", "y": "v2"},
         ),
         (
@@ -88,8 +100,8 @@ def test_finding_lists(dictionary, expected_result):
             },
             {"x": "v3", "y": "v4", "fields": {"v"}},
             [
-                {"v": 1, "v3": 0.01, "v4": 0.001},
-                {"v": 2, "v3": 0.02, "v4": 0.002},
+                {"v": 1, "v3": 0.01, "v4": 0.001, "filename": "f", "rev": "r"},
+                {"v": 2, "v3": 0.02, "v4": 0.002, "filename": "f", "rev": "r"},
             ],
             {"x": "v3", "y": "v4", "fields": {"v", "v3", "v4"}},
         ),
@@ -104,60 +116,41 @@ def test_finding_lists(dictionary, expected_result):
                 },
             },
             {"x": "v", "y": "v2"},
-            [{"v": 1, "v2": 0.1}, {"v": 2, "v2": 0.2}],
+            [
+                {"v": 1, "v2": 0.1, "filename": "f", "rev": "r"},
+                {"v": 2, "v2": 0.2, "filename": "f", "rev": "r"},
+            ],
             {"x": "v", "y": "v2"},
         ),
     ],
 )
 def test_convert(
-    input_data, properties, expected_datapoints, expected_properties
+    input_data,
+    properties,
+    expected_datapoints,
+    expected_properties,
 ):
-    converter = Converter(properties)
-    datapoints, resolved_properties = converter.convert(input_data)
+    converter = VegaConverter(properties)
+    datapoints, resolved_properties = converter.convert(
+        revision="r", filename="f", data=input_data
+    )
 
     assert datapoints == expected_datapoints
     assert resolved_properties == expected_properties
 
 
 def test_convert_skip_step():
-    converter = Converter()
+    converter = VegaConverter()
     converter.skip_step("append_index")
 
     datapoints, resolved_properties = converter.convert(
-        {"a": "b", "metric": [{"v": 1}, {"v": 2}]}
+        revision="r",
+        filename="f",
+        data={"a": "b", "metric": [{"v": 1}, {"v": 2}]},
     )
 
-    assert datapoints == [{"v": 1}, {"v": 2}]
+    assert datapoints == [
+        {"v": 1, "filename": "f", "rev": "r"},
+        {"v": 2, "filename": "f", "rev": "r"},
+    ]
     assert resolved_properties == {"x": "step", "y": "v"}
-
-
-def test_to_datapoints():
-    input_data = {
-        "revision": {
-            "data": {
-                "filename": {
-                    "data": {
-                        "metric": [
-                            {"v": 1, "v2": 0.1, "v3": 0.01, "v4": 0.001},
-                            {"v": 2, "v2": 0.2, "v3": 0.02, "v4": 0.002},
-                        ]
-                    }
-                }
-            }
-        }
-    }
-    props = {"fields": {"v"}, "x": "v2", "y": "v3"}
-
-    datapoints, resolved_properties = to_datapoints(input_data, props)
-
-    assert datapoints == {
-        "revision": [
-            {"v": 1, "v2": 0.1, "v3": 0.01},
-            {"v": 2, "v2": 0.2, "v3": 0.02},
-        ]
-    }
-    assert resolved_properties == {
-        "fields": {"v", "v2", "v3"},
-        "x": "v2",
-        "y": "v3",
-    }

tests/unit/repo/plots/test_templates.py

@@ -1,73 +0,0 @@
-import os
-
-import pytest
-
-from dvc.repo.plots.template import (
-    LinearTemplate,
-    PlotTemplates,
-    ScatterTemplate,
-    TemplateContentDoesNotMatch,
-    TemplateNotFoundError,
-)
-
-
-def test_raise_on_no_template(tmp_dir, dvc):
-    with pytest.raises(TemplateNotFoundError):
-        dvc.plots.templates.load("non_existing_template.json")
-
-
-@pytest.mark.parametrize(
-    "template_path, target_name",
-    [
-        (os.path.join(".dvc", "plots", "template.json"), "template"),
-        (os.path.join(".dvc", "plots", "template.json"), "template.json"),
-        (
-            os.path.join(".dvc", "plots", "subdir", "template.json"),
-            os.path.join("subdir", "template.json"),
-        ),
-        (
-            os.path.join(".dvc", "plots", "subdir", "template.json"),
-            os.path.join("subdir", "template"),
-        ),
-        ("template.json", "template.json"),
-    ],
-)
-def test_load_template(tmp_dir, dvc, template_path, target_name):
-    os.makedirs(os.path.abspath(os.path.dirname(template_path)), exist_ok=True)
-    with open(template_path, "w", encoding="utf-8") as fd:
-        fd.write("template_content")
-
-    assert dvc.plots.templates.load(target_name).content == "template_content"
-
-
-def test_load_default_template(tmp_dir, dvc):
-    assert dvc.plots.templates.load(None).content == LinearTemplate().content
-
-
-@pytest.mark.parametrize("output", ("output", None))
-@pytest.mark.parametrize(
-    "targets,expected_templates",
-    (
-        ([None, PlotTemplates.TEMPLATES]),
-        (["linear", "scatter"], [ScatterTemplate, LinearTemplate]),
-    ),
-)
-def test_init(tmp_dir, dvc, output, targets, expected_templates):
-    output = output or dvc.plots.templates.templates_dir
-    dvc.plots.templates.init(output, targets)
-
-    assert set(os.listdir(output)) == {
-        cls.DEFAULT_NAME + ".json" for cls in expected_templates
-    }
-
-
-def test_raise_on_init_modified(tmp_dir, dvc):
-    dvc.plots.templates.init(output=None, targets=["linear"])
-
-    with open(
-        tmp_dir / ".dvc" / "plots" / "linear.json", "a", encoding="utf-8"
-    ) as fd:
-        fd.write("modification")
-
-    with pytest.raises(TemplateContentDoesNotMatch):
-        dvc.plots.templates.init(output=None, targets=["linear"])

tests/unit/test_plots.py

@@ -1,7 +1,7 @@
 import json
 import os
 
-from dvc.render.utils import get_files
+from dvc.render.match import get_files
 
 
 def test_plots_order(tmp_dir, dvc):

tests/unit/test_tabular_data.py

@@ -383,14 +383,16 @@ def test_to_parallel_coordinates(tmp_dir, mocker):
     td = TabularData(["categorical", "scalar"])
     td.extend([["foo", "0.1"], ["bar", "2"]])
 
-    write = mocker.patch("dvc.render.html.write")
+    write = mocker.patch("dvc_render.html.render_html")
     renderer_class = mocker.patch(
-        "dvc.render.plotly.ParallelCoordinatesRenderer"
+        "dvc_render.plotly.ParallelCoordinatesRenderer"
     )
     renderer = renderer_class.return_value
 
     td.to_parallel_coordinates(output_path="foo")
 
-    renderer_class.assert_called_with(td, None, td._fill_value)
+    renderer_class.assert_called_with(
+        td.as_dict(), color_by=None, fill_value=td._fill_value
+    )
 
-    write.assert_called_with("foo", renderers=[renderer])
+    write.assert_called_with(renderers=[renderer], output_file="foo")