diff --git a/dvc/commands/experiments/show.py b/dvc/commands/experiments/show.py index 1f0e06283..c00df5a4f 100644 --- a/dvc/commands/experiments/show.py +++ b/dvc/commands/experiments/show.py @@ -432,14 +432,15 @@ def show_experiments( for x in td.column("Experiment") ] out = kwargs.get("out") or "dvc_plots" + output_file = os.path.join(out, "index.html") ui.write( td.to_parallel_coordinates( - output_path=os.path.abspath(out), + output_path=os.path.abspath(output_file), color_by=kwargs.get("sort_by") or "Experiment", ) ) if kwargs.get("open"): - return ui.open_browser(os.path.join(out, "index.html")) + return ui.open_browser(output_file) else: td.render( diff --git a/dvc/commands/live.py b/dvc/commands/live.py index d947ee897..e08e6d21e 100644 --- a/dvc/commands/live.py +++ b/dvc/commands/live.py @@ -10,17 +10,21 @@ class CmdLive(CmdBase): UNINITIALIZED = True def _run(self, target, revs=None): - from dvc.render.utils import match_renderers, render + from dvc_render import render_html + + from dvc.render.match import match_renderers metrics, plots = self.repo.live.show(target=target, revs=revs) if plots: from pathlib import Path - html_path = Path.cwd() / (self.args.target + "_html") + output = Path.cwd() / (self.args.target + "_html") / "index.html" - renderers = match_renderers(plots, self.repo.plots.templates) - index_path = render(self.repo, renderers, metrics, html_path) + renderers = match_renderers( + plots, templates_dir=self.repo.plots.templates_dir + ) + index_path = render_html(renderers, output, metrics) ui.write(index_path.as_uri()) return 0 return 1 diff --git a/dvc/commands/plots.py b/dvc/commands/plots.py index 62ee6ebc3..06c7f617c 100644 --- a/dvc/commands/plots.py +++ b/dvc/commands/plots.py @@ -1,6 +1,7 @@ import argparse import json import logging +import os from funcy import first @@ -14,15 +15,11 @@ from dvc.utils import format_link logger = logging.getLogger(__name__) -def _show_json(renderers, path: None, split=False): - if any(r.needs_output_path for r in renderers) and not path: - raise DvcException("Output path ('-o') is required!") +def _show_json(renderers, split=False): + from dvc.render.convert import to_json result = { - renderer.filename: json.loads( - renderer.as_json(path=path, fill_data=not split) - ) - for renderer in renderers + renderer.name: to_json(renderer, split) for renderer in renderers } if result: ui.write_json(result) @@ -42,8 +39,9 @@ class CmdPlots(CmdBase): def run(self): from pathlib import Path - from dvc.render.utils import match_renderers, render - from dvc.render.vega import VegaRenderer + from dvc_render import render_html + + from dvc.render.match import match_renderers if self.args.show_vega: if not self.args.targets: @@ -74,31 +72,40 @@ class CmdPlots(CmdBase): ) renderers = match_renderers( - plots_data=plots_data, templates=self.repo.plots.templates + plots_data=plots_data, out=self.args.out ) if self.args.show_vega: - renderer = first( - filter(lambda r: isinstance(r, VegaRenderer), renderers) - ) + renderer = first(filter(lambda r: r.TYPE == "vega", renderers)) if renderer: - content = renderer.asdict() - ui.write_json(content) + ui.write_json(json.loads(renderer.partial_html())) return 0 if self.args.json: - _show_json(renderers, self.args.out, self.args.split) + _show_json(renderers, self.args.split) return 0 + html_template_path = self.args.html_template + if not html_template_path: + html_template_path = self.repo.config.get("plots", {}).get( + "html_template", None + ) + if html_template_path and not os.path.isabs( + html_template_path + ): + html_template_path = os.path.join( + self.repo.dvc_dir, html_template_path + ) + rel: str = self.args.out or "dvc_plots" - path: Path = (Path.cwd() / rel).resolve() - index_path = render( - self.repo, - renderers, - path=path, - html_template_path=self.args.html_template, + output_file: Path = (Path.cwd() / rel).resolve() / "index.html" + + render_html( + renderers=renderers, + output_file=output_file, + template_path=html_template_path, ) - ui.write(index_path.as_uri()) + ui.write(output_file.as_uri()) auto_open = self.repo.config["plots"].get("auto_open", False) if self.args.open or auto_open: if not auto_open: @@ -107,7 +114,7 @@ class CmdPlots(CmdBase): "\n" "\tdvc config plots.auto_open true" ) - return ui.open_browser(index_path) + return ui.open_browser(output_file) return 0 @@ -154,17 +161,17 @@ class CmdPlotsTemplates(CmdBase): ] def run(self): - import os + from dvc_render.vega_templates import dump_templates try: out = ( os.path.join(os.getcwd(), self.args.out) if self.args.out - else self.repo.plots.templates.templates_dir + else self.repo.plots.templates_dir ) targets = [self.args.target] if self.args.target else None - self.repo.plots.templates.init(output=out, targets=targets) + dump_templates(output=out, targets=targets) templates_path = os.path.relpath(out, os.getcwd()) ui.write(f"Templates have been written into '{templates_path}'.") diff --git a/dvc/compare.py b/dvc/compare.py index aa3a2cd45..c90f99799 100644 --- a/dvc/compare.py +++ b/dvc/compare.py @@ -168,17 +168,21 @@ class TabularData(MutableSequence[Sequence["CellT"]]): def to_parallel_coordinates( self, output_path: "StrPath", color_by: str = None - ) -> str: - from dvc.render.html import write - from dvc.render.plotly import ParallelCoordinatesRenderer + ) -> "StrPath": + from dvc_render.html import render_html + from dvc_render.plotly import ParallelCoordinatesRenderer - index_path = write( - output_path, + render_html( renderers=[ - ParallelCoordinatesRenderer(self, color_by, self._fill_value) + ParallelCoordinatesRenderer( + self.as_dict(), + color_by=color_by, + fill_value=self._fill_value, + ) ], + output_file=output_path, ) - return index_path.as_uri() + return output_path def add_column(self, name: str) -> None: self._columns[name] = Column([self._fill_value] * len(self)) diff --git a/dvc/render/__init__.py b/dvc/render/__init__.py index 0db485254..9cf67b884 100644 --- a/dvc/render/__init__.py +++ b/dvc/render/__init__.py @@ -1,4 +1,5 @@ -from .image import ImageRenderer -from .vega import VegaRenderer - -RENDERERS = [ImageRenderer, VegaRenderer] +INDEX_FIELD = "step" +REVISION_FIELD = "rev" +REVISIONS_KEY = "revisions" +TYPE_KEY = "type" +SRC_FIELD = "src" diff --git a/dvc/render/base.py b/dvc/render/base.py deleted file mode 100644 index d321384a2..000000000 --- a/dvc/render/base.py +++ /dev/null @@ -1,77 +0,0 @@ -import abc -from typing import TYPE_CHECKING, Dict - -from dvc.exceptions import DvcException - -if TYPE_CHECKING: - from dvc.types import StrPath - - -REVISION_FIELD = "rev" -INDEX_FIELD = "step" - - -class BadTemplateError(DvcException): - pass - - -class Renderer(abc.ABC): - REVISIONS_KEY = "revisions" - TYPE_KEY = "type" - - def __init__(self, data: Dict, **kwargs): - self.data = data - - from dvc.render.utils import get_files - - files = get_files(self.data) - - # we assume comparison of same file between revisions for now - assert len(files) == 1 - self.filename = files.pop() - - def partial_html(self, **kwargs): - """ - Us this method to generate partial HTML content, - that will fill self.DIV - """ - - raise NotImplementedError - - @property - @abc.abstractmethod - def TYPE(self): - raise NotImplementedError - - @property - @abc.abstractmethod - def DIV(self): - raise NotImplementedError - - @property - @abc.abstractmethod - def SCRIPTS(self): - raise NotImplementedError - - @abc.abstractmethod - def as_json(self, **kwargs): - raise NotImplementedError - - @staticmethod - def _remove_special_chars(string: str): - return string.translate( - {ord(c): "_" for c in r"!@#$%^&*()[]{};,<>?\/:.|`~=_+"} - ) - - @property - def needs_output_path(self): - return False - - def generate_html(self, path: "StrPath"): - """this method might edit content of path""" - partial = self.partial_html(path=path) - - div_id = self._remove_special_chars(self.filename) - div_id = f"plot_{div_id}" - - return self.DIV.format(id=div_id, partial=partial) diff --git a/dvc/render/convert.py b/dvc/render/convert.py new file mode 100644 index 000000000..bfb93644a --- /dev/null +++ b/dvc/render/convert.py @@ -0,0 +1,68 @@ +import json +from collections import defaultdict +from typing import Dict, List, Union + +from dvc.render import REVISION_FIELD, REVISIONS_KEY, SRC_FIELD, TYPE_KEY +from dvc.render.image_converter import ImageConverter +from dvc.render.vega_converter import VegaConverter + + +def get_converter( + renderer_class, props +) -> Union[VegaConverter, ImageConverter]: + from dvc_render import ImageRenderer, VegaRenderer + + if renderer_class.TYPE == VegaRenderer.TYPE: + return VegaConverter(props) + if renderer_class.TYPE == ImageRenderer.TYPE: + return ImageConverter(props) + + raise ValueError(f"Invalid renderer class {renderer_class}") + + +def to_datapoints(renderer_class, data: Dict, props: Dict): + converter = get_converter(renderer_class, props) + datapoints: List[Dict] = [] + for revision, rev_data in data.items(): + for filename, file_data in rev_data.get("data", {}).items(): + if "data" in file_data: + processed, final_props = converter.convert( + revision, filename, file_data.get("data") + ) + datapoints.extend(processed) + return datapoints, final_props + + +def _group_by_rev(datapoints): + grouped = defaultdict(list) + for datapoint in datapoints: + rev = datapoint.pop(REVISION_FIELD) + grouped[rev].append(datapoint) + return dict(grouped) + + +def to_json(renderer, split: bool = False) -> List[Dict]: + if renderer.TYPE == "vega": + grouped = _group_by_rev(renderer.datapoints) + if split: + content = renderer.get_filled_template(skip_anchors=["data"]) + else: + content = renderer.get_filled_template() + return [ + { + TYPE_KEY: renderer.TYPE, + REVISIONS_KEY: sorted(grouped.keys()), + "content": json.loads(content), + "datapoints": grouped, + } + ] + if renderer.TYPE == "image": + return [ + { + TYPE_KEY: renderer.TYPE, + REVISIONS_KEY: datapoint.get(REVISION_FIELD), + "url": datapoint.get(SRC_FIELD), + } + for datapoint in renderer.datapoints + ] + raise ValueError(f"Invalid renderer: {renderer.TYPE}") diff --git a/dvc/render/html.py b/dvc/render/html.py deleted file mode 100644 index 89da9677c..000000000 --- a/dvc/render/html.py +++ /dev/null @@ -1,114 +0,0 @@ -import os -from pathlib import Path -from typing import TYPE_CHECKING, Dict, List, Optional - -from dvc.exceptions import DvcException - -if TYPE_CHECKING: - from dvc.render.base import Renderer - from dvc.types import StrPath - -PAGE_HTML = """ - - - {refresh_tag} - DVC Plot - {scripts} - - - {plot_divs} - -""" - - -class MissingPlaceholderError(DvcException): - def __init__(self, placeholder): - super().__init__(f"HTML template has to contain '{placeholder}'.") - - -class HTML: - SCRIPTS_PLACEHOLDER = "scripts" - PLOTS_PLACEHOLDER = "plot_divs" - PLOTS_PLACEHOLDER_FORMAT_STR = f"{{{PLOTS_PLACEHOLDER}}}" - REFRESH_PLACEHOLDER = "refresh_tag" - REFRESH_TAG = '' - - def __init__( - self, - template: Optional[str] = None, - refresh_seconds: Optional[int] = None, - ): - template = template or PAGE_HTML - if self.PLOTS_PLACEHOLDER_FORMAT_STR not in template: - raise MissingPlaceholderError(self.PLOTS_PLACEHOLDER_FORMAT_STR) - - self.template = template - self.elements: List[str] = [] - self.scripts: str = "" - self.refresh_tag = "" - if refresh_seconds is not None: - self.refresh_tag = self.REFRESH_TAG.format(refresh_seconds) - - def with_metrics(self, metrics: Dict[str, Dict]) -> "HTML": - import tabulate - - header: List[str] = [] - rows: List[List[str]] = [] - - for _, rev_data in metrics.items(): - for _, data in rev_data.items(): - if not header: - header.extend(sorted(data.keys())) - - rows.append([data[key] for key in header]) - - self.elements.append(tabulate.tabulate(rows, header, tablefmt="html")) - return self - - def with_scripts(self, scripts: str) -> "HTML": - if scripts not in self.scripts: - self.scripts += f"\n{scripts}" - return self - - def with_element(self, html: str) -> "HTML": - self.elements.append(html) - return self - - def embed(self) -> str: - kwargs = { - self.SCRIPTS_PLACEHOLDER: self.scripts, - self.PLOTS_PLACEHOLDER: "\n".join(self.elements), - self.REFRESH_PLACEHOLDER: self.refresh_tag, - } - return self.template.format(**kwargs) - - -def write( - path: "StrPath", - renderers: List["Renderer"], - metrics: Optional[Dict[str, Dict]] = None, - template_path: Optional["StrPath"] = None, - refresh_seconds: Optional[int] = None, -): - - os.makedirs(path, exist_ok=True) - - page_html = None - if template_path: - with open(template_path, encoding="utf-8") as fobj: - page_html = fobj.read() - - document = HTML(page_html, refresh_seconds=refresh_seconds) - if metrics: - document.with_metrics(metrics) - document.with_element("
") - - for renderer in renderers: - document.with_scripts(renderer.SCRIPTS) - document.with_element(renderer.generate_html(path)) - - index = Path(os.path.join(path, "index.html")) - - with open(index, "w", encoding="utf-8") as fd: - fd.write(document.embed()) - return index diff --git a/dvc/render/image.py b/dvc/render/image.py deleted file mode 100644 index f7db58949..000000000 --- a/dvc/render/image.py +++ /dev/null @@ -1,110 +0,0 @@ -import json -import os -from typing import TYPE_CHECKING - -from funcy import reraise - -from dvc.exceptions import DvcException -from dvc.render.base import Renderer -from dvc.render.utils import get_files -from dvc.utils import relpath - -if TYPE_CHECKING: - from dvc.types import StrPath - - -class ImageRenderer(Renderer): - TYPE = "image" - DIV = """ -
- {partial} -
""" - - SCRIPTS = "" - - @property - def needs_output_path(self): - return True - - def _write_image( - self, - path: "StrPath", - revision: str, - filename: str, - image_data: bytes, - ): - img_path = os.path.join( - path, f"{revision}_{filename.replace(os.sep, '_')}" - ) - with open(img_path, "wb") as fd: - fd.write(image_data) - - return img_path - - def _save_images(self, path: "StrPath"): - - for rev, rev_data in self.data.items(): - if "data" in rev_data: - for file, file_data in rev_data.get("data", {}).items(): - if "data" in file_data: - if not os.path.isdir(path): - os.makedirs(path, exist_ok=True) - yield rev, file, self._write_image( - os.path.abspath(path), rev, file, file_data["data"] - ) - - def partial_html(self, **kwargs): - path = kwargs.get("path", None) - if not path: - raise DvcException("Can't save here") - static = os.path.join(path, "static") - - div_content = [] - for rev, _, img_path in self._save_images(static): - div_content.append( - """ -
-

{title}

- -
""".format( - title=rev, src=(relpath(img_path, path)) - ) - ) - if div_content: - div_content.insert(0, f"

{self.filename}

") - return "\n".join(div_content) - return "" - - def as_json(self, **kwargs): - - with reraise( - KeyError, - DvcException( - f"{type(self).__name__} needs 'path' to store images." - ), - ): - path = kwargs["path"] - - results = [] - - for revision, _, img_path in self._save_images(path): - results.append( - { - self.TYPE_KEY: self.TYPE, - self.REVISIONS_KEY: [revision], - "url": img_path, - } - ) - - return json.dumps(results) - - @staticmethod - def matches(data): - files = get_files(data) - extensions = set(map(lambda f: os.path.splitext(f)[1], files)) - return extensions.issubset({".jpg", ".jpeg", ".gif", ".png"}) diff --git a/dvc/render/image_converter.py b/dvc/render/image_converter.py new file mode 100644 index 000000000..39c8980ac --- /dev/null +++ b/dvc/render/image_converter.py @@ -0,0 +1,58 @@ +import base64 +import os +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple + +from dvc.render import REVISION_FIELD, SRC_FIELD + +if TYPE_CHECKING: + from dvc.types import StrPath + + +class ImageConverter: + def __init__(self, plot_properties: Optional[Dict] = None): + self.plot_properties = plot_properties or {} + + @staticmethod + def _write_image( + path: "StrPath", + revision: str, + filename: str, + image_data: bytes, + ) -> "StrPath": + img_path = os.path.join( + path, f"{revision}_{filename.replace(os.sep, '_')}" + ) + with open(img_path, "wb") as fd: + fd.write(image_data) + + return img_path + + @staticmethod + def _encode_image( + image_data: bytes, + ) -> str: + base64_str = base64.b64encode(image_data).decode() + return f"data:image;base64,{base64_str}" + + def convert( + self, revision, filename, data: bytes + ) -> Tuple[List[Dict], Dict]: + """ + Convert the DVC Plots content to DVC Render datapoints. + Return both generated datapoints and updated properties. + """ + path = self.plot_properties.get("out") + if path: + if not os.path.isdir(path): + os.makedirs(path, exist_ok=True) + src = self._write_image( + os.path.abspath(path), revision, filename, data + ) + else: + src = self._encode_image(data) + datapoint = { + REVISION_FIELD: revision, + "filename": filename, + SRC_FIELD: src, + } + return [datapoint], self.plot_properties diff --git a/dvc/render/match.py b/dvc/render/match.py new file mode 100644 index 000000000..052c65222 --- /dev/null +++ b/dvc/render/match.py @@ -0,0 +1,55 @@ +from typing import TYPE_CHECKING, Dict, List, Optional + +import dpath.util +from dvc_render import RENDERERS + +from .convert import to_datapoints + +if TYPE_CHECKING: + from dvc.types import StrPath + + +def get_files(plots_data: Dict) -> List: + values = dpath.util.values(plots_data, ["*", "*"]) + return sorted({key for submap in values for key in submap.keys()}) + + +def group_by_filename(plots_data: Dict) -> Dict: + files = get_files(plots_data) + grouped = { + file: dpath.util.search(plots_data, ["*", "*", file]) for file in files + } + return grouped + + +def squash_plots_properties(data: Dict) -> Dict: + resolved: Dict[str, str] = {} + for rev_data in data.values(): + for file_data in rev_data.get("data", {}).values(): + props = file_data.get("props", {}) + resolved = {**resolved, **props} + return resolved + + +def match_renderers( + plots_data, + out: Optional["StrPath"] = None, + templates_dir: Optional["StrPath"] = None, +): + renderers = [] + for filename, group in group_by_filename(plots_data).items(): + plot_properties = squash_plots_properties(group) + for renderer_class in RENDERERS: + if renderer_class.matches(filename, plot_properties): + if out is not None: + plot_properties["out"] = out + if templates_dir is not None: + plot_properties["templates_dir"] = templates_dir + datapoints, plot_properties = to_datapoints( + renderer_class, group, plot_properties + ) + renderers.append( + renderer_class(datapoints, filename, **plot_properties) + ) + break + return renderers diff --git a/dvc/render/plotly.py b/dvc/render/plotly.py deleted file mode 100644 index 8092e18fa..000000000 --- a/dvc/render/plotly.py +++ /dev/null @@ -1,92 +0,0 @@ -import json -from collections import defaultdict -from typing import TYPE_CHECKING, Any, Dict, Optional - -from dvc.render.base import Renderer - -if TYPE_CHECKING: - from dvc.compare import TabularData - - -class ParallelCoordinatesRenderer(Renderer): - TYPE = "plotly" - - DIV = """ -
- -
- """ - - SCRIPTS = """ - - """ - - # pylint: disable=W0231 - def __init__( - self, - tabular_data: "TabularData", - color_by: Optional[str] = None, - fill_value: str = "", - ): - self.tabular_data = tabular_data - self.color_by = color_by - self.filename = "experiments" - self.fill_value = fill_value - - def partial_html(self, **kwargs): - return self.as_json() - - def as_json(self, **kwargs) -> str: - tabular_dict = defaultdict(list) - for row in self.tabular_data.as_dict(): - for col_name, value in row.items(): - tabular_dict[col_name].append(str(value)) - - trace: Dict[str, Any] = {"type": "parcoords", "dimensions": []} - for label, values in tabular_dict.items(): - is_categorical = False - try: - float_values = [ - float(x) if x != self.fill_value else None for x in values - ] - except ValueError: - is_categorical = True - - if is_categorical: - non_missing = [x for x in values if x != self.fill_value] - unique_values = sorted(set(non_missing)) - unique_values.append(self.fill_value) - - dummy_values = [unique_values.index(x) for x in values] - - values = [ - x if x != self.fill_value else "Missing" for x in values - ] - trace["dimensions"].append( - { - "label": label, - "values": dummy_values, - "tickvals": dummy_values, - "ticktext": values, - } - ) - else: - trace["dimensions"].append( - {"label": label, "values": float_values} - ) - - if label == self.color_by: - trace["line"] = { - "color": dummy_values if is_categorical else float_values, - "showscale": True, - "colorbar": {"title": self.color_by}, - } - if is_categorical: - trace["line"]["colorbar"]["tickmode"] = "array" - trace["line"]["colorbar"]["tickvals"] = dummy_values - trace["line"]["colorbar"]["ticktext"] = values - - return json.dumps({"data": [trace], "layout": {}}) diff --git a/dvc/render/utils.py b/dvc/render/utils.py deleted file mode 100644 index e27b20c1c..000000000 --- a/dvc/render/utils.py +++ /dev/null @@ -1,70 +0,0 @@ -import os.path -from typing import Dict, List - -import dpath.util - - -def get_files(plots_data: Dict) -> List: - values = dpath.util.values(plots_data, ["*", "*"]) - return sorted({key for submap in values for key in submap.keys()}) - - -def group_by_filename(plots_data: Dict) -> List[Dict]: - files = get_files(plots_data) - grouped = [] - for file in files: - grouped.append(dpath.util.search(plots_data, ["*", "*", file])) - return grouped - - -def squash_plots_properties(data: Dict) -> Dict: - resolved: Dict[str, str] = {} - for rev_data in data.values(): - for file_data in rev_data.get("data", {}).values(): - props = file_data.get("props", {}) - resolved = {**resolved, **props} - return resolved - - -def match_renderers(plots_data, templates): - from dvc.render import RENDERERS - - renderers = [] - for group in group_by_filename(plots_data): - plot_properties = squash_plots_properties(group) - template = templates.load(plot_properties.get("template", None)) - - for renderer_class in RENDERERS: - if renderer_class.matches(group): - renderers.append( - renderer_class( - group, template=template, properties=plot_properties - ) - ) - return renderers - - -def render( - repo, - renderers, - metrics=None, - path=None, - html_template_path=None, - refresh_seconds=None, -): - if not html_template_path: - html_template_path = repo.config.get("plots", {}).get( - "html_template", None - ) - if html_template_path and not os.path.isabs(html_template_path): - html_template_path = os.path.join(repo.dvc_dir, html_template_path) - - from dvc.render.html import write - - return write( - path, - renderers, - metrics=metrics, - template_path=html_template_path, - refresh_seconds=refresh_seconds, - ) diff --git a/dvc/render/vega.py b/dvc/render/vega.py deleted file mode 100644 index f44484481..000000000 --- a/dvc/render/vega.py +++ /dev/null @@ -1,134 +0,0 @@ -import json -import os -from copy import deepcopy -from typing import Dict, List, Optional - -from funcy import cached_property - -from dvc.render.base import REVISION_FIELD, BadTemplateError, Renderer -from dvc.render.data import Converter, to_datapoints -from dvc.render.utils import get_files -from dvc.repo.plots.template import Template - - -def _flatten_datapoints(rev_datapoints: Dict[str, List[Dict]]) -> List[Dict]: - flat = [] - for revision, datapoints in rev_datapoints.items(): - datapoints_cp = deepcopy(datapoints) - Converter.update(datapoints_cp, {REVISION_FIELD: revision}) - flat.extend(datapoints_cp) - return flat - - -class VegaRenderer(Renderer): - TYPE = "vega" - - DIV = """ -
- -
- """ - - SCRIPTS = """ - - - - """ - - def __init__( - self, data: Dict, template: Template, properties: Dict = None - ): - super().__init__(data) - self.properties = properties or {} - self.template = template - - def _revisions(self): - return list(self.data.keys()) - - # TODO Shouldn't it be a part of plots? - def _fill_template(self, template, datapoints, props=None, fill_data=True): - props = props or {} - - content = deepcopy(template.content) - if template.anchor_str("data") not in template.content: - anchor = template.anchor("data") - raise BadTemplateError( - f"Template '{template.name}' is not using '{anchor}' anchor" - ) - - if props.get("x"): - template.check_field_exists(datapoints, props.get("x")) - if props.get("y"): - template.check_field_exists(datapoints, props.get("y")) - - if fill_data: - content = template.fill_anchor(content, "data", datapoints) - - props.setdefault("title", "") - props.setdefault("x_label", props.get("x")) - props.setdefault("y_label", props.get("y")) - - names = ["title", "x", "y", "x_label", "y_label"] - for name in names: - value = props.get(name) - if value is not None: - content = template.fill_anchor(content, name, value) - - return content - - @cached_property - def _converted_data(self): - return to_datapoints(self.data, self.properties) - - def _get_filled_template(self, fill_data=True): - datapoints, final_props = self._converted_data - flat_datapoints = _flatten_datapoints(datapoints) - - if datapoints: - filled_template = self._fill_template( - self.template, - flat_datapoints, - final_props, - fill_data=fill_data, - ) - - return filled_template - return "" - - # TODO rename - def asdict(self, fill_data=True): - filled_template = self._get_filled_template(fill_data) - if filled_template: - return json.loads(filled_template) - return {} - - def as_json(self, **kwargs) -> Optional[str]: - - fill_data = kwargs.get("fill_data", True) - - content = self.asdict(fill_data) - datapoints, _ = self._converted_data - - return json.dumps( - [ - { - self.TYPE_KEY: self.TYPE, - self.REVISIONS_KEY: self._revisions(), - "content": content, - "datapoints": datapoints, - } - ], - ) - - def partial_html(self, **kwargs): - filled_template = self._get_filled_template() - return filled_template - - @staticmethod - def matches(data): - files = get_files(data) - extensions = set(map(lambda f: os.path.splitext(f)[1], files)) - return extensions.issubset({".yml", ".yaml", ".json", ".csv", ".tsv"}) diff --git a/dvc/render/data.py b/dvc/render/vega_converter.py similarity index 86% rename from dvc/render/data.py rename to dvc/render/vega_converter.py index 9ff3d09f6..643d7cfb0 100644 --- a/dvc/render/data.py +++ b/dvc/render/vega_converter.py @@ -5,7 +5,7 @@ from typing import Dict, List, Optional, Set, Union from funcy import first, project from dvc.exceptions import DvcException -from dvc.render.base import INDEX_FIELD, REVISION_FIELD +from dvc.render import INDEX_FIELD, REVISION_FIELD class FieldsNotFoundError(DvcException): @@ -76,7 +76,7 @@ def _append_index(datapoints: List[Dict]) -> List[Dict]: return datapoints -class Converter: +class VegaConverter: """ Class that takes care of converting unspecified data blob (Dict or List[Dict]) into datapoints (List[Dict]). @@ -84,12 +84,6 @@ class Converter: ('x', 'y') it will attempt to fill in the blanks. """ - @staticmethod - def update(datapoints: List[Dict], update_dict: Dict): - for data_point in datapoints: - data_point.update(update_dict) - return datapoints - def __init__(self, plot_properties: Optional[Dict] = None): plot_properties = plot_properties or {} self.props = deepcopy(plot_properties) @@ -162,7 +156,7 @@ class Converter: else: self.inferred_props["y"] = inferred_y - def convert(self, data): + def convert(self, revision: str, filename: str, data: Dict): """ Convert the data. Fill necessary fields ('x', 'y') and return both generated datapoints and updated properties. @@ -174,20 +168,8 @@ class Converter: self._infer_y(processed) + for datapoint in processed: + datapoint[REVISION_FIELD] = revision + datapoint["filename"] = filename + return processed, {**self.props, **self.inferred_props} - - -def to_datapoints(data: Dict, props: Dict): - converter = Converter(props) - - datapoints: Dict[str, List[Dict]] = {} - for revision, rev_data in data.items(): - datapoints[revision] = [] - for _, file_data in rev_data.get("data", {}).items(): - if "data" in file_data: - processed, final_props = converter.convert( - file_data.get("data") - ) - - datapoints[revision].extend(processed) - return datapoints, final_props diff --git a/dvc/repo/live.py b/dvc/repo/live.py index fdeb3f6ba..40a349de0 100644 --- a/dvc/repo/live.py +++ b/dvc/repo/live.py @@ -1,9 +1,11 @@ import logging +import os from typing import TYPE_CHECKING, List, Optional +from dvc_render import render_html from funcy import once_per_args -from dvc.render.utils import match_renderers, render +from dvc.render.match import match_renderers logger = logging.getLogger(__name__) @@ -24,11 +26,13 @@ def create_summary(out): metrics, plots = out.repo.live.show(out.fs_path) - html_path = out.fs_path + "_dvc_plots" + html_path = os.path.join(out.fs_path + "_dvc_plots", "index.html") - renderers = match_renderers(plots, out.repo.plots.templates) - index_path = render( - out.repo, renderers, metrics=metrics, path=html_path, refresh_seconds=5 + renderers = match_renderers( + plots, templates_dir=out.repo.plots.templates_dir + ) + index_path = render_html( + renderers, output_file=html_path, metrics=metrics, refresh_seconds=5 ) if out.repo.config["plots"].get("auto_open", False): webbrowser_open(index_path) diff --git a/dvc/repo/plots/__init__.py b/dvc/repo/plots/__init__.py index da58c83ca..303308d63 100644 --- a/dvc/repo/plots/__init__.py +++ b/dvc/repo/plots/__init__.py @@ -195,12 +195,14 @@ class Plots: out.plot.pop(prop) def modify(self, path, props=None, unset=None): + from dvc_render.vega_templates import get_template + from dvc.dvcfile import Dvcfile props = props or {} template = props.get("template") if template: - self.templates.get_template(template) + get_template(template, self.templates_dir) (out,) = self.repo.find_outs_by_path(path) if not out.plot and unset is not None: @@ -225,10 +227,8 @@ class Plots: dvcfile.dump(out.stage, update_lock=False) @cached_property - def templates(self): - from .template import PlotTemplates - - return PlotTemplates(self.repo.dvc_dir) + def templates_dir(self): + return os.path.join(self.repo.dvc_dir, "plots") def _is_plot(out: "Output") -> bool: diff --git a/dvc/repo/plots/template.py b/dvc/repo/plots/template.py deleted file mode 100644 index e8b850968..000000000 --- a/dvc/repo/plots/template.py +++ /dev/null @@ -1,595 +0,0 @@ -import json -import os -from typing import Any, Dict, List, Optional - -from funcy import cached_property - -from dvc.exceptions import DvcException - - -class TemplateNotFoundError(DvcException): - def __init__(self, path): - super().__init__(f"Template '{path}' not found.") - - -class NoFieldInDataError(DvcException): - def __init__(self, field_name): - super().__init__( - f"Field '{field_name}' does not exist in provided data." - ) - - -class TemplateContentDoesNotMatch(DvcException): - def __init__(self, template_name: str, path: str): - super().__init__( - f"Template '{path}' already exists " - f"and its content is different than '{template_name}' content. " - "Remove it manually if you want to recreate it." - ) - - -class Template: - INDENT = 4 - SEPARATORS = (",", ": ") - EXTENSION = ".json" - ANCHOR = "" - - DEFAULT_CONTENT: Optional[Dict[str, Any]] = None - DEFAULT_NAME: Optional[str] = None - - def __init__(self, content=None, name=None): - if content: - self.content = content - else: - self.content = ( - json.dumps( - self.DEFAULT_CONTENT, - indent=self.INDENT, - separators=self.SEPARATORS, - ) - + "\n" - ) - - self.name = name or self.DEFAULT_NAME - assert self.content and self.name - self.filename = self.name + self.EXTENSION - - @classmethod - def anchor(cls, name): - return cls.ANCHOR.format(name.upper()) - - def has_anchor(self, name): - return self.anchor_str(name) in self.content - - @classmethod - def fill_anchor(cls, content, name, value): - value_str = json.dumps( - value, indent=cls.INDENT, separators=cls.SEPARATORS, sort_keys=True - ) - return content.replace(cls.anchor_str(name), value_str) - - @classmethod - def anchor_str(cls, name): - return f'"{cls.anchor(name)}"' - - @staticmethod - def check_field_exists(data, field): - if not any(field in row for row in data): - raise NoFieldInDataError(field) - - -class SimpleLinearTemplate(Template): - DEFAULT_NAME = "simple" - - DEFAULT_CONTENT = { - "$schema": "https://vega.github.io/schema/vega-lite/v4.json", - "data": {"values": Template.anchor("data")}, - "title": Template.anchor("title"), - "width": 300, - "height": 300, - "mark": {"type": "line"}, - "encoding": { - "x": { - "field": Template.anchor("x"), - "type": "quantitative", - "title": Template.anchor("x_label"), - }, - "y": { - "field": Template.anchor("y"), - "type": "quantitative", - "title": Template.anchor("y_label"), - "scale": {"zero": False}, - }, - "color": {"field": "rev", "type": "nominal"}, - }, - } - - -class ConfusionTemplate(Template): - DEFAULT_NAME = "confusion" - DEFAULT_CONTENT = { - "$schema": "https://vega.github.io/schema/vega-lite/v4.json", - "data": {"values": Template.anchor("data")}, - "title": Template.anchor("title"), - "facet": {"field": "rev", "type": "nominal"}, - "spec": { - "transform": [ - { - "aggregate": [{"op": "count", "as": "xy_count"}], - "groupby": [Template.anchor("y"), Template.anchor("x")], - }, - { - "impute": "xy_count", - "groupby": ["rev", Template.anchor("y")], - "key": Template.anchor("x"), - "value": 0, - }, - { - "impute": "xy_count", - "groupby": ["rev", Template.anchor("x")], - "key": Template.anchor("y"), - "value": 0, - }, - { - "joinaggregate": [ - {"op": "max", "field": "xy_count", "as": "max_count"} - ], - "groupby": [], - }, - { - "calculate": "datum.xy_count / datum.max_count", - "as": "percent_of_max", - }, - ], - "encoding": { - "x": { - "field": Template.anchor("x"), - "type": "nominal", - "sort": "ascending", - "title": Template.anchor("x_label"), - }, - "y": { - "field": Template.anchor("y"), - "type": "nominal", - "sort": "ascending", - "title": Template.anchor("y_label"), - }, - }, - "layer": [ - { - "mark": "rect", - "width": 300, - "height": 300, - "encoding": { - "color": { - "field": "xy_count", - "type": "quantitative", - "title": "", - "scale": {"domainMin": 0, "nice": True}, - } - }, - }, - { - "mark": "text", - "encoding": { - "text": {"field": "xy_count", "type": "quantitative"}, - "color": { - "condition": { - "test": "datum.percent_of_max > 0.5", - "value": "white", - }, - "value": "black", - }, - }, - }, - ], - }, - } - - -class NormalizedConfusionTemplate(Template): - DEFAULT_NAME = "confusion_normalized" - DEFAULT_CONTENT = { - "$schema": "https://vega.github.io/schema/vega-lite/v4.json", - "data": {"values": Template.anchor("data")}, - "title": Template.anchor("title"), - "facet": {"field": "rev", "type": "nominal"}, - "spec": { - "transform": [ - { - "aggregate": [{"op": "count", "as": "xy_count"}], - "groupby": [Template.anchor("y"), Template.anchor("x")], - }, - { - "impute": "xy_count", - "groupby": ["rev", Template.anchor("y")], - "key": Template.anchor("x"), - "value": 0, - }, - { - "impute": "xy_count", - "groupby": ["rev", Template.anchor("x")], - "key": Template.anchor("y"), - "value": 0, - }, - { - "joinaggregate": [ - {"op": "sum", "field": "xy_count", "as": "sum_y"} - ], - "groupby": [Template.anchor("y")], - }, - { - "calculate": "datum.xy_count / datum.sum_y", - "as": "percent_of_y", - }, - ], - "encoding": { - "x": { - "field": Template.anchor("x"), - "type": "nominal", - "sort": "ascending", - "title": Template.anchor("x_label"), - }, - "y": { - "field": Template.anchor("y"), - "type": "nominal", - "sort": "ascending", - "title": Template.anchor("y_label"), - }, - }, - "layer": [ - { - "mark": "rect", - "width": 300, - "height": 300, - "encoding": { - "color": { - "field": "percent_of_y", - "type": "quantitative", - "title": "", - "scale": {"domain": [0, 1]}, - } - }, - }, - { - "mark": "text", - "encoding": { - "text": { - "field": "percent_of_y", - "type": "quantitative", - "format": ".2f", - }, - "color": { - "condition": { - "test": "datum.percent_of_y > 0.5", - "value": "white", - }, - "value": "black", - }, - }, - }, - ], - }, - } - - -class ScatterTemplate(Template): - DEFAULT_NAME = "scatter" - - DEFAULT_CONTENT = { - "$schema": "https://vega.github.io/schema/vega-lite/v4.json", - "data": {"values": Template.anchor("data")}, - "title": Template.anchor("title"), - "width": 300, - "height": 300, - "layer": [ - { - "encoding": { - "x": { - "field": Template.anchor("x"), - "type": "quantitative", - "title": Template.anchor("x_label"), - }, - "y": { - "field": Template.anchor("y"), - "type": "quantitative", - "title": Template.anchor("y_label"), - "scale": {"zero": False}, - }, - "color": {"field": "rev", "type": "nominal"}, - }, - "layer": [ - {"mark": "point"}, - { - "selection": { - "label": { - "type": "single", - "nearest": True, - "on": "mouseover", - "encodings": ["x"], - "empty": "none", - "clear": "mouseout", - } - }, - "mark": "point", - "encoding": { - "opacity": { - "condition": { - "selection": "label", - "value": 1, - }, - "value": 0, - } - }, - }, - ], - }, - { - "transform": [{"filter": {"selection": "label"}}], - "layer": [ - { - "encoding": { - "text": { - "type": "quantitative", - "field": Template.anchor("y"), - }, - "x": { - "field": Template.anchor("x"), - "type": "quantitative", - }, - "y": { - "field": Template.anchor("y"), - "type": "quantitative", - }, - }, - "layer": [ - { - "mark": { - "type": "text", - "align": "left", - "dx": 5, - "dy": -5, - }, - "encoding": { - "color": { - "type": "nominal", - "field": "rev", - } - }, - } - ], - } - ], - }, - ], - } - - -class SmoothLinearTemplate(Template): - DEFAULT_NAME = "smooth" - - DEFAULT_CONTENT = { - "$schema": "https://vega.github.io/schema/vega-lite/v4.json", - "data": {"values": Template.anchor("data")}, - "title": Template.anchor("title"), - "mark": {"type": "line"}, - "encoding": { - "x": { - "field": Template.anchor("x"), - "type": "quantitative", - "title": Template.anchor("x_label"), - }, - "y": { - "field": Template.anchor("y"), - "type": "quantitative", - "title": Template.anchor("y_label"), - "scale": {"zero": False}, - }, - "color": {"field": "rev", "type": "nominal"}, - }, - "transform": [ - { - "loess": Template.anchor("y"), - "on": Template.anchor("x"), - "groupby": ["rev"], - "bandwidth": 0.3, - } - ], - } - - -class LinearTemplate(Template): - DEFAULT_NAME = "linear" - - DEFAULT_CONTENT = { - "$schema": "https://vega.github.io/schema/vega-lite/v4.json", - "data": {"values": Template.anchor("data")}, - "title": Template.anchor("title"), - "width": 300, - "height": 300, - "layer": [ - { - "encoding": { - "x": { - "field": Template.anchor("x"), - "type": "quantitative", - "title": Template.anchor("x_label"), - }, - "y": { - "field": Template.anchor("y"), - "type": "quantitative", - "title": Template.anchor("y_label"), - "scale": {"zero": False}, - }, - "color": {"field": "rev", "type": "nominal"}, - }, - "layer": [ - {"mark": "line"}, - { - "selection": { - "label": { - "type": "single", - "nearest": True, - "on": "mouseover", - "encodings": ["x"], - "empty": "none", - "clear": "mouseout", - } - }, - "mark": "point", - "encoding": { - "opacity": { - "condition": { - "selection": "label", - "value": 1, - }, - "value": 0, - } - }, - }, - ], - }, - { - "transform": [{"filter": {"selection": "label"}}], - "layer": [ - { - "mark": {"type": "rule", "color": "gray"}, - "encoding": { - "x": { - "field": Template.anchor("x"), - "type": "quantitative", - } - }, - }, - { - "encoding": { - "text": { - "type": "quantitative", - "field": Template.anchor("y"), - }, - "x": { - "field": Template.anchor("x"), - "type": "quantitative", - }, - "y": { - "field": Template.anchor("y"), - "type": "quantitative", - }, - }, - "layer": [ - { - "mark": { - "type": "text", - "align": "left", - "dx": 5, - "dy": -5, - }, - "encoding": { - "color": { - "type": "nominal", - "field": "rev", - } - }, - } - ], - }, - ], - }, - ], - } - - -class PlotTemplates: - TEMPLATES_DIR = "plots" - TEMPLATES = [ - SimpleLinearTemplate, - LinearTemplate, - ConfusionTemplate, - NormalizedConfusionTemplate, - ScatterTemplate, - SmoothLinearTemplate, - ] - - @cached_property - def templates_dir(self): - return os.path.join(self.dvc_dir, self.TEMPLATES_DIR) - - def get_template(self, template_name: str): - template_path = os.path.abspath(template_name) - if os.path.exists(template_path): - return os.path.abspath(template_path) - - if self.dvc_dir and os.path.exists(self.dvc_dir): - template_path = os.path.join(self.templates_dir, template_name) - if os.path.exists(template_path): - return template_path - - all_templates = [ - os.path.join(root, file) - for root, _, files in os.walk(self.templates_dir) - for file in files - ] - matches = [ - template - for template in all_templates - if os.path.splitext(template)[0] == template_path - ] - if matches: - assert len(matches) == 1 - return matches[0] - raise TemplateNotFoundError(template_name) - - def __init__(self, dvc_dir): - self.dvc_dir = dvc_dir - - def init( - self, output: Optional[str] = None, targets: Optional[List] = None - ): - from dvc.utils.fs import makedirs - - output = output or self.templates_dir - - makedirs(output, exist_ok=True) - - if targets: - templates = [ - template - for template in self.TEMPLATES - if template.DEFAULT_NAME in targets - ] - else: - templates = self.TEMPLATES - - for template in templates: - self._dump(template(), output) - - def _dump(self, template: Template, output: str): - path = os.path.join(output, template.filename) - - if os.path.exists(path): - with open(path, "r", encoding="utf-8") as fd: - content = fd.read() - if content != template.content: - raise TemplateContentDoesNotMatch( - template.DEFAULT_NAME or "", path - ) - else: - with open(path, "w", encoding="utf-8") as fd: - fd.write(template.content) - - def load(self, template_name=None): - if not template_name: - template_name = "linear" - - try: - path = self.get_template(template_name) - - with open(path, "r", encoding="utf-8") as fd: - content = fd.read() - - return Template(content, name=template_name) - except TemplateNotFoundError: - for template in self.TEMPLATES: - if template.DEFAULT_NAME == template_name: - return template() - raise diff --git a/dvc/repo/plots/templates/confusion.json b/dvc/repo/plots/templates/confusion.json deleted file mode 100644 index 84ec022f8..000000000 --- a/dvc/repo/plots/templates/confusion.json +++ /dev/null @@ -1,107 +0,0 @@ -{ - "$schema": "https://vega.github.io/schema/vega-lite/v5.json", - "data": { - "values": "" - }, - "title": "", - "facet": { - "field": "rev", - "type": "nominal" - }, - "spec": { - "transform": [ - { - "aggregate": [ - { - "op": "count", - "as": "xy_count" - } - ], - "groupby": [ - "", - "" - ] - }, - { - "impute": "xy_count", - "groupby": [ - "rev", - "" - ], - "key": "", - "value": 0 - }, - { - "impute": "xy_count", - "groupby": [ - "rev", - "" - ], - "key": "", - "value": 0 - }, - { - "joinaggregate": [ - { - "op": "max", - "field": "xy_count", - "as": "max_count" - } - ], - "groupby": [] - }, - { - "calculate": "datum.xy_count / datum.max_count", - "as": "percent_of_max" - } - ], - "encoding": { - "x": { - "field": "", - "type": "nominal", - "sort": "ascending", - "title": "" - }, - "y": { - "field": "", - "type": "nominal", - "sort": "ascending", - "title": "" - } - }, - "layer": [ - { - "mark": "rect", - "width": 300, - "height": 300, - "encoding": { - "color": { - "field": "xy_count", - "type": "quantitative", - "title": "", - "scale": { - "domainMin": 0, - "nice": true - } - } - } - }, - { - "mark": "text", - "encoding": { - "text": { - "field": "xy_count", - "type": "quantitative" - }, - "color": { - "condition": { - "test": "datum.percent_of_max > 0.5", - "value": "white" - }, - "value": "black" - } - } - } - ] - } -} diff --git a/dvc/repo/plots/templates/confusion_normalized.json b/dvc/repo/plots/templates/confusion_normalized.json deleted file mode 100644 index 92c77739c..000000000 --- a/dvc/repo/plots/templates/confusion_normalized.json +++ /dev/null @@ -1,112 +0,0 @@ -{ - "$schema": "https://vega.github.io/schema/vega-lite/v5.json", - "data": { - "values": "" - }, - "title": "", - "facet": { - "field": "rev", - "type": "nominal" - }, - "spec": { - "transform": [ - { - "aggregate": [ - { - "op": "count", - "as": "xy_count" - } - ], - "groupby": [ - "", - "" - ] - }, - { - "impute": "xy_count", - "groupby": [ - "rev", - "" - ], - "key": "", - "value": 0 - }, - { - "impute": "xy_count", - "groupby": [ - "rev", - "" - ], - "key": "", - "value": 0 - }, - { - "joinaggregate": [ - { - "op": "sum", - "field": "xy_count", - "as": "sum_y" - } - ], - "groupby": [ - "" - ] - }, - { - "calculate": "datum.xy_count / datum.sum_y", - "as": "percent_of_y" - } - ], - "encoding": { - "x": { - "field": "", - "type": "nominal", - "sort": "ascending", - "title": "" - }, - "y": { - "field": "", - "type": "nominal", - "sort": "ascending", - "title": "" - } - }, - "layer": [ - { - "mark": "rect", - "width": 300, - "height": 300, - "encoding": { - "color": { - "field": "percent_of_y", - "type": "quantitative", - "title": "", - "scale": { - "domain": [ - 0, - 1 - ] - } - } - } - }, - { - "mark": "text", - "encoding": { - "text": { - "field": "percent_of_y", - "type": "quantitative", - "format": ".2f" - }, - "color": { - "condition": { - "test": "datum.percent_of_y > 0.5", - "value": "white" - }, - "value": "black" - } - } - } - ] - } -} diff --git a/dvc/repo/plots/templates/linear.json b/dvc/repo/plots/templates/linear.json deleted file mode 100644 index 2e94f67c3..000000000 --- a/dvc/repo/plots/templates/linear.json +++ /dev/null @@ -1,116 +0,0 @@ -{ - "$schema": "https://vega.github.io/schema/vega-lite/v5.json", - "data": { - "values": "" - }, - "title": "", - "width": 300, - "height": 300, - "layer": [ - { - "encoding": { - "x": { - "field": "", - "type": "quantitative", - "title": "" - }, - "y": { - "field": "", - "type": "quantitative", - "title": "", - "scale": { - "zero": false - } - }, - "color": { - "field": "rev", - "type": "nominal" - } - }, - "layer": [ - { - "mark": "line" - }, - { - "selection": { - "label": { - "type": "single", - "nearest": true, - "on": "mouseover", - "encodings": [ - "x" - ], - "empty": "none", - "clear": "mouseout" - } - }, - "mark": "point", - "encoding": { - "opacity": { - "condition": { - "selection": "label", - "value": 1 - }, - "value": 0 - } - } - } - ] - }, - { - "transform": [ - { - "filter": { - "selection": "label" - } - } - ], - "layer": [ - { - "mark": { - "type": "rule", - "stroke": "gray" - }, - "encoding": { - "x": { - "field": "", - "type": "quantitative" - } - } - }, - { - "encoding": { - "text": { - "type": "quantitative", - "field": "" - }, - "x": { - "field": "", - "type": "quantitative" - }, - "y": { - "field": "", - "type": "quantitative" - } - }, - "layer": [ - { - "mark": { - "type": "text", - "align": "left", - "dx": 5, - "dy": -5 - }, - "encoding": { - "color": { - "type": "nominal", - "field": "rev" - } - } - } - ] - } - ] - } - ] -} diff --git a/dvc/repo/plots/templates/scatter.json b/dvc/repo/plots/templates/scatter.json deleted file mode 100644 index 6e8cf5b48..000000000 --- a/dvc/repo/plots/templates/scatter.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "$schema": "https://vega.github.io/schema/vega-lite/v5.json", - "data": { - "values": "" - }, - "title": "", - "width": 300, - "height": 300, - "layer": [ - { - "encoding": { - "x": { - "field": "", - "type": "quantitative", - "title": "" - }, - "y": { - "field": "", - "type": "quantitative", - "title": "", - "scale": { - "zero": false - } - }, - "color": { - "field": "rev", - "type": "nominal" - } - }, - "layer": [ - { - "mark": "point" - }, - { - "selection": { - "label": { - "type": "single", - "nearest": true, - "on": "mouseover", - "encodings": [ - "x" - ], - "empty": "none", - "clear": "mouseout" - } - }, - "mark": "point", - "encoding": { - "opacity": { - "condition": { - "selection": "label", - "value": 1 - }, - "value": 0 - } - } - } - ] - }, - { - "transform": [ - { - "filter": { - "selection": "label" - } - } - ], - "layer": [ - { - "encoding": { - "text": { - "type": "quantitative", - "field": "" - }, - "x": { - "field": "", - "type": "quantitative" - }, - "y": { - "field": "", - "type": "quantitative" - } - }, - "layer": [ - { - "mark": { - "type": "text", - "align": "left", - "dx": 5, - "dy": -5 - }, - "encoding": { - "color": { - "type": "nominal", - "field": "rev" - } - } - } - ] - } - ] - } - ] -} diff --git a/dvc/repo/plots/templates/simple.json b/dvc/repo/plots/templates/simple.json deleted file mode 100644 index 1cebce9ba..000000000 --- a/dvc/repo/plots/templates/simple.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "$schema": "https://vega.github.io/schema/vega-lite/v5.json", - "data": { - "values": "" - }, - "title": "", - "width": 300, - "height": 300, - "mark": { - "type": "line" - }, - "encoding": { - "x": { - "field": "", - "type": "quantitative", - "title": "" - }, - "y": { - "field": "", - "type": "quantitative", - "title": "", - "scale": { - "zero": false - } - }, - "color": { - "field": "rev", - "type": "nominal" - } - } -} diff --git a/dvc/repo/plots/templates/smooth.json b/dvc/repo/plots/templates/smooth.json deleted file mode 100644 index 42b1ecfff..000000000 --- a/dvc/repo/plots/templates/smooth.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "$schema": "https://vega.github.io/schema/vega-lite/v5.json", - "data": { - "values": "" - }, - "title": "", - "mark": { - "type": "line" - }, - "encoding": { - "x": { - "field": "", - "type": "quantitative", - "title": "" - }, - "y": { - "field": "", - "type": "quantitative", - "title": "", - "scale": { - "zero": false - } - }, - "color": { - "field": "rev", - "type": "nominal" - } - }, - "transform": [ - { - "loess": "", - "on": "", - "groupby": [ - "rev" - ], - "bandwidth": 0.3 - } - ] -} diff --git a/setup.cfg b/setup.cfg index f8254aa19..634793132 100644 --- a/setup.cfg +++ b/setup.cfg @@ -76,6 +76,7 @@ install_requires = diskcache>=5.2.1 jaraco.windows>=5.7.0; python_version < '3.8' and sys_platform == 'win32' scmrepo==0.0.13 + dvc_render==0.0.2 [options.extras_require] all = diff --git a/tests/conftest.py b/tests/conftest.py index 8bb652e04..492924190 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +import json import os import sys from contextlib import suppress @@ -164,19 +165,10 @@ def pytest_configure(config): @pytest.fixture def custom_template(tmp_dir, dvc): - try: - import importlib_resources - except ImportError: - import importlib.resources as importlib_resources - - content = ( - importlib_resources.files("dvc.repo.plots") - / "templates" - / "simple.json" - ).read_text() + from dvc_render.vega_templates import SimpleLinearTemplate template = tmp_dir / "custom_template.json" - template.write_text(content) + template.write_text(json.dumps(SimpleLinearTemplate.DEFAULT_CONTENT)) return template diff --git a/tests/func/plots/test_modify.py b/tests/func/plots/test_modify.py index ef4e2918e..2cea446d5 100644 --- a/tests/func/plots/test_modify.py +++ b/tests/func/plots/test_modify.py @@ -4,7 +4,6 @@ import pytest from dvc.dvcfile import PIPELINE_LOCK from dvc.repo.plots import PropsNotFoundError -from dvc.repo.plots.template import TemplateNotFoundError from dvc.utils import relpath @@ -47,6 +46,8 @@ def test_plots_modify_should_not_change_lockfile( def test_plots_modify_not_existing_template(dvc): + from dvc_render.vega_templates import TemplateNotFoundError + with pytest.raises(TemplateNotFoundError): dvc.plots.modify( "metric.json", props={"template": "not-existing-template.json"} diff --git a/tests/func/test_live.py b/tests/func/test_live.py index 83166450c..e5aca8859 100644 --- a/tests/func/test_live.py +++ b/tests/func/test_live.py @@ -6,7 +6,7 @@ import pytest from funcy import first from dvc import stage as stage_module -from dvc.render.utils import get_files +from dvc.render.match import get_files LIVE_SCRIPT = dedent( """ diff --git a/tests/unit/command/test_experiments.py b/tests/unit/command/test_experiments.py index 7ff828936..80f38f383 100644 --- a/tests/unit/command/test_experiments.py +++ b/tests/unit/command/test_experiments.py @@ -862,5 +862,5 @@ def test_show_experiments_pcp(tmp_dir, mocker): kwargs = td.to_parallel_coordinates.call_args[1] - assert kwargs["output_path"] == str(tmp_dir / "dvc_plots") + assert kwargs["output_path"] == str(tmp_dir / "dvc_plots" / "index.html") assert kwargs["color_by"] == "Experiment" diff --git a/tests/unit/command/test_plots.py b/tests/unit/command/test_plots.py index 124d3bb84..d2b625d97 100644 --- a/tests/unit/command/test_plots.py +++ b/tests/unit/command/test_plots.py @@ -55,7 +55,7 @@ def test_plots_diff(dvc, mocker, plots_data): cmd = cli_args.func(cli_args) m = mocker.patch("dvc.repo.plots.diff.diff", return_value=plots_data) render_mock = mocker.patch( - "dvc.render.utils.render", return_value="html_path" + "dvc_render.render_html", return_value="html_path" ) assert cmd.run() == 0 @@ -100,7 +100,7 @@ def test_plots_show_vega(dvc, mocker, plots_data): return_value=plots_data, ) render_mock = mocker.patch( - "dvc.render.utils.render", return_value="html_path" + "dvc_render.render_html", return_value="html_path" ) assert cmd.run() == 0 @@ -127,10 +127,10 @@ def test_plots_diff_vega(dvc, mocker, capsys, plots_data): cmd = cli_args.func(cli_args) mocker.patch("dvc.repo.plots.diff.diff", return_value=plots_data) mocker.patch( - "dvc.render.VegaRenderer.asdict", - return_value={"this": "is vega json"}, + "dvc_render.VegaRenderer.partial_html", + return_value=json.dumps({"this": "is vega json"}), ) - render_mock = mocker.patch("dvc.render.utils.render") + render_mock = mocker.patch("dvc_render.render_html") assert cmd.run() == 0 out, _ = capsys.readouterr() @@ -156,7 +156,7 @@ def test_plots_diff_open(tmp_dir, dvc, mocker, capsys, plots_data, auto_open): mocker.patch("dvc.repo.plots.diff.diff", return_value=plots_data) index_path = tmp_dir / "dvc_plots" / "index.html" - mocker.patch("dvc.render.utils.render", return_value=index_path) + mocker.patch("dvc_render.render_html", return_value=index_path) assert cmd.run() == 0 mocked_open.assert_called_once_with(index_path.as_uri()) @@ -178,7 +178,7 @@ def test_plots_diff_open_WSL(tmp_dir, dvc, mocker, plots_data): mocker.patch("dvc.repo.plots.diff.diff", return_value=plots_data) index_path = tmp_dir / "dvc_plots" / "index.html" - mocker.patch("dvc.render.utils.render", return_value=index_path) + mocker.patch("dvc_render.render_html", return_value=index_path) assert cmd.run() == 0 mocked_open.assert_called_once_with(str(Path("dvc_plots") / "index.html")) @@ -253,9 +253,9 @@ def test_should_call_render(tmp_dir, mocker, capsys, plots_data, output): output = output or "dvc_plots" index_path = tmp_dir / output / "index.html" renderers = mocker.MagicMock() - mocker.patch("dvc.render.utils.match_renderers", return_value=renderers) + mocker.patch("dvc.render.match.match_renderers", return_value=renderers) render_mock = mocker.patch( - "dvc.render.utils.render", return_value=index_path + "dvc_render.render_html", return_value=index_path ) assert cmd.run() == 0 @@ -264,7 +264,9 @@ def test_should_call_render(tmp_dir, mocker, capsys, plots_data, output): assert index_path.as_uri() in out render_mock.assert_called_once_with( - cmd.repo, renderers, path=tmp_dir / output, html_template_path=None + renderers=renderers, + output_file=Path(tmp_dir / output / "index.html"), + template_path=None, ) @@ -289,66 +291,21 @@ def test_plots_diff_json(dvc, mocker, capsys): mocker.patch("dvc.repo.plots.diff.diff", return_value=data) renderers = mocker.MagicMock() - mocker.patch("dvc.render.utils.match_renderers", return_value=renderers) - render_mock = mocker.patch("dvc.render.utils.render") + mocker.patch("dvc.render.match.match_renderers", return_value=renderers) + render_mock = mocker.patch("dvc_render.render_html") show_json_mock = mocker.patch("dvc.commands.plots._show_json") assert cmd.run() == 0 - show_json_mock.assert_called_once_with(renderers, "out", True) + show_json_mock.assert_called_once_with(renderers, True) render_mock.assert_not_called() -def test_show_json_requires_out(dvc, mocker, capsys): - cli_args = parse_args( - [ - "plots", - "diff", - "HEAD~10", - "HEAD~1", - "--json", - "--targets", - "plot.csv", - ] - ) - cmd = cli_args.func(cli_args) - - mocker.patch("dvc.repo.plots.diff.diff") - - renderer = mocker.MagicMock() - renderer.needs_output_path = False - renderer.filename = "foo" - renderer.as_json.return_value = "{}" - mocker.patch("dvc.render.utils.match_renderers", return_value=[renderer]) - - assert cmd.run() == 0 - - renderer.needs_output_path = True - - assert cmd.run() == 1 - - cli_args = parse_args( - [ - "plots", - "diff", - "HEAD~10", - "HEAD~1", - "--json", - "--targets", - "plot.csv", - "-o", - "out", - ] - ) - cmd = cli_args.func(cli_args) - assert cmd.run() == 0 - - @pytest.mark.parametrize("target", (("t1"), (None))) def test_plots_templates(tmp_dir, dvc, mocker, capsys, target): - assert not os.path.exists(dvc.plots.templates.templates_dir) + assert not os.path.exists(dvc.plots.templates_dir) mocker.patch( "dvc.commands.plots.CmdPlotsTemplates.TEMPLATES_CHOICES", ["t1", "t2"], @@ -361,21 +318,21 @@ def test_plots_templates(tmp_dir, dvc, mocker, capsys, target): cli_args = parse_args(arguments) assert cli_args.func == CmdPlotsTemplates - init_mock = mocker.patch("dvc.repo.plots.template.PlotTemplates.init") + dump_mock = mocker.patch("dvc_render.vega_templates.dump_templates") cmd = cli_args.func(cli_args) assert cmd.run() == 0 out, _ = capsys.readouterr() - init_mock.assert_called_once_with( + dump_mock.assert_called_once_with( output=os.path.abspath("output"), targets=[target] if target else None ) assert "Templates have been written into 'output'." in out def test_plots_templates_choices(tmp_dir, dvc): - from dvc.repo.plots.template import PlotTemplates + from dvc_render import TEMPLATES assert CmdPlotsTemplates.TEMPLATES_CHOICES == list( - pluck_attr("DEFAULT_NAME", PlotTemplates.TEMPLATES) + pluck_attr("DEFAULT_NAME", TEMPLATES) ) diff --git a/tests/unit/render/test_convert.py b/tests/unit/render/test_convert.py new file mode 100644 index 000000000..077a86c7c --- /dev/null +++ b/tests/unit/render/test_convert.py @@ -0,0 +1,219 @@ +from dvc.render import ( + INDEX_FIELD, + REVISION_FIELD, + REVISIONS_KEY, + SRC_FIELD, + TYPE_KEY, +) +from dvc.render.convert import to_datapoints, to_json + + +def test_to_datapoints_single_revision(mocker): + renderer = mocker.MagicMock() + renderer.TYPE = "vega" + + input_data = { + "revision": { + "data": { + "filename": { + "data": { + "metric": [ + {"v": 1, "v2": 0.1, "v3": 0.01, "v4": 0.001}, + {"v": 2, "v2": 0.2, "v3": 0.02, "v4": 0.002}, + ] + } + } + } + } + } + props = {"fields": {"v"}, "x": "v2", "y": "v3"} + + datapoints, resolved_properties = to_datapoints( + renderer, input_data, props + ) + + assert datapoints == [ + { + "v": 1, + "v2": 0.1, + "v3": 0.01, + "rev": "revision", + "filename": "filename", + }, + { + "v": 2, + "v2": 0.2, + "v3": 0.02, + "rev": "revision", + "filename": "filename", + }, + ] + assert resolved_properties == { + "fields": {"v", "v2", "v3"}, + "x": "v2", + "y": "v3", + } + + +def test_to_datapoints_revision_with_error(mocker): + renderer = mocker.MagicMock() + renderer.TYPE = "vega" + + data = { + "v2": { + "data": {"file.json": {"data": [{"y": 2}, {"y": 3}], "props": {}}} + }, + "workspace": { + "data": {"file.json": {"error": FileNotFoundError(), "props": {}}} + }, + } + datapoints, final_props = to_datapoints(renderer, data, {}) + + assert datapoints == [ + { + "y": 2, + INDEX_FIELD: 0, + REVISION_FIELD: "v2", + "filename": "file.json", + }, + { + "y": 3, + INDEX_FIELD: 1, + REVISION_FIELD: "v2", + "filename": "file.json", + }, + ] + assert final_props == {"x": INDEX_FIELD, "y": "y"} + + +def test_to_datapoints_multiple_revisions(mocker): + renderer = mocker.MagicMock() + renderer.TYPE = "vega" + + metric_1 = [{"y": 2}, {"y": 3}] + metric_2 = [{"y": 3}, {"y": 5}] + metric_3 = [{"y": 5}, {"y": 6}] + + data = { + "HEAD": { + "data": { + "file.json": {"data": metric_3, "props": {"fields": {"y"}}} + } + }, + "v2": { + "data": { + "file.json": {"data": metric_2, "props": {"fields": {"y"}}} + } + }, + "v1": { + "data": { + "file.json": {"data": metric_1, "props": {"fields": {"y"}}} + } + }, + } + props = {"fields": {"y"}} + + datapoints, final_props = to_datapoints(renderer, data, props) + + assert datapoints == [ + { + "y": 5, + INDEX_FIELD: 0, + REVISION_FIELD: "HEAD", + "filename": "file.json", + }, + { + "y": 6, + INDEX_FIELD: 1, + REVISION_FIELD: "HEAD", + "filename": "file.json", + }, + { + "y": 3, + INDEX_FIELD: 0, + REVISION_FIELD: "v2", + "filename": "file.json", + }, + { + "y": 5, + INDEX_FIELD: 1, + REVISION_FIELD: "v2", + "filename": "file.json", + }, + { + "y": 2, + INDEX_FIELD: 0, + REVISION_FIELD: "v1", + "filename": "file.json", + }, + { + "y": 3, + INDEX_FIELD: 1, + REVISION_FIELD: "v1", + "filename": "file.json", + }, + ] + assert final_props == {"x": INDEX_FIELD, "y": "y", "fields": {"y", "step"}} + + +def test_to_json_vega(mocker): + vega_renderer = mocker.MagicMock() + vega_renderer.TYPE = "vega" + vega_renderer.get_filled_template.return_value = '{"this": "is vega"}' + vega_renderer.datapoints = [ + {"x": 1, "y": 2, REVISION_FIELD: "foo", "filename": "foo.json"}, + {"x": 2, "y": 1, REVISION_FIELD: "bar", "filename": "foo.json"}, + ] + result = to_json(vega_renderer) + assert result[0] == { + TYPE_KEY: vega_renderer.TYPE, + REVISIONS_KEY: ["bar", "foo"], + "content": {"this": "is vega"}, + "datapoints": { + "foo": [ + {"x": 1, "y": 2, "filename": "foo.json"}, + ], + "bar": [ + {"x": 2, "y": 1, "filename": "foo.json"}, + ], + }, + } + vega_renderer.get_filled_template.assert_called() + + +def test_to_json_vega_split(mocker): + vega_renderer = mocker.MagicMock() + vega_renderer.TYPE = "vega" + vega_renderer.get_filled_template.return_value = ( + '{"this": "is split vega"}' + ) + vega_renderer.datapoints = [ + {"x": 1, "y": 2, REVISION_FIELD: "foo", "filename": "foo.json"}, + {"x": 2, "y": 1, REVISION_FIELD: "bar", "filename": "foo.json"}, + ] + result = to_json(vega_renderer, split=True) + assert result[0] == { + TYPE_KEY: vega_renderer.TYPE, + REVISIONS_KEY: ["bar", "foo"], + "content": {"this": "is split vega"}, + "datapoints": { + "foo": [{"x": 1, "y": 2, "filename": "foo.json"}], + "bar": [{"x": 2, "y": 1, "filename": "foo.json"}], + }, + } + vega_renderer.get_filled_template.assert_called_with(skip_anchors=["data"]) + + +def test_to_json_image(mocker): + image_renderer = mocker.MagicMock() + image_renderer.TYPE = "image" + image_renderer.datapoints = [ + {SRC_FIELD: "contentfoo", REVISION_FIELD: "foo"}, + {SRC_FIELD: "contentbar", REVISION_FIELD: "bar"}, + ] + result = to_json(image_renderer) + assert result[0] == { + "url": image_renderer.datapoints[0].get(SRC_FIELD), + REVISIONS_KEY: image_renderer.datapoints[0].get(REVISION_FIELD), + TYPE_KEY: image_renderer.TYPE, + } diff --git a/tests/unit/render/test_html.py b/tests/unit/render/test_html.py deleted file mode 100644 index fe3878b07..000000000 --- a/tests/unit/render/test_html.py +++ /dev/null @@ -1,47 +0,0 @@ -import pytest - -from dvc.render.html import HTML, PAGE_HTML, MissingPlaceholderError - -CUSTOM_PAGE_HTML = """ - - - TITLE - - - - - - {plot_divs} - -""" - - -@pytest.mark.parametrize( - "template,page_elements,expected_page", - [ - ( - None, - ["content"], - PAGE_HTML.format(plot_divs="content", refresh_tag="", scripts=""), - ), - ( - CUSTOM_PAGE_HTML, - ["content"], - CUSTOM_PAGE_HTML.format(plot_divs="content"), - ), - ], -) -def test_html(tmp_dir, template, page_elements, expected_page): - page = HTML(template) - page.elements = page_elements - - result = page.embed() - - assert result == expected_page - - -def test_no_placeholder(tmp_dir): - template = "" - - with pytest.raises(MissingPlaceholderError): - HTML(template) diff --git a/tests/unit/render/test_image.py b/tests/unit/render/test_image.py deleted file mode 100644 index 737eff189..000000000 --- a/tests/unit/render/test_image.py +++ /dev/null @@ -1,79 +0,0 @@ -import json -import os - -import pytest - -from dvc.render.image import ImageRenderer - - -@pytest.mark.parametrize( - "extension, matches", - ( - (".csv", False), - (".json", False), - (".tsv", False), - (".yaml", False), - (".jpg", True), - (".gif", True), - (".jpeg", True), - (".png", True), - ), -) -def test_matches(extension, matches): - filename = "file" + extension - data = { - "HEAD": {"data": {filename: {}}}, - "v1": {"data": {filename: {}}}, - } - assert ImageRenderer.matches(data) == matches - - -def test_render(tmp_dir): - data = {"workspace": {"data": {"file.jpg": {"data": b"content"}}}} - - page_dir = os.path.join("some", "path") - html = ImageRenderer(data).generate_html(page_dir) - - assert (tmp_dir / page_dir).is_dir() - image_file = tmp_dir / page_dir / "static" / "workspace_file.jpg" - assert image_file.is_file() - - with open(image_file, "rb") as fobj: - assert fobj.read() == b"content" - - assert "

file.jpg

" in html - assert ( - f'' in html - ) - - -def test_as_json(tmp_dir): - data = { - "workspace": {"data": {"file.jpg": {"data": b"content"}}}, - "HEAD": {"data": {"file.jpg": {"data": b"content2"}}}, - } - - page_dir = os.path.join("some", "path") - json_content = json.loads(ImageRenderer(data).as_json(path=page_dir)) - - img1 = os.path.join(page_dir, "workspace_file.jpg") - img2 = os.path.join(page_dir, "HEAD_file.jpg") - - assert { - "type": "image", - "revisions": ["workspace"], - "url": os.path.abspath(img1), - } in json_content - assert { - "type": "image", - "revisions": ["HEAD"], - "url": os.path.abspath(img2), - } in json_content - - def assert_file_with_content(path, content): - assert os.path.isfile(path) - with open(path, "rb") as fd: - assert fd.read() == content - - assert_file_with_content(img1, b"content") - assert_file_with_content(img2, b"content2") diff --git a/tests/unit/render/test_image_converter.py b/tests/unit/render/test_image_converter.py new file mode 100644 index 000000000..08cc16e14 --- /dev/null +++ b/tests/unit/render/test_image_converter.py @@ -0,0 +1,30 @@ +from dvc.render import REVISION_FIELD, SRC_FIELD +from dvc.render.image_converter import ImageConverter + + +def test_image_converter_no_out(): + data = b"content" + converter = ImageConverter() + + datapoints, _ = converter.convert("r", "image.png", data) + + assert datapoints[0] == { + REVISION_FIELD: "r", + "filename": "image.png", + SRC_FIELD: converter._encode_image(b"content"), + } + + +def test_image_converter_with_out(tmp_dir): + data = b"content" + converter = ImageConverter({"out": tmp_dir / "foo"}) + + datapoints, _ = converter.convert("r", "image.png", data) + + assert datapoints[0] == { + REVISION_FIELD: "r", + "filename": "image.png", + SRC_FIELD: str(tmp_dir / "foo" / "r_image.png"), + } + + assert (tmp_dir / "foo" / "r_image.png").read_bytes() == b"content" diff --git a/tests/unit/render/test_match.py b/tests/unit/render/test_match.py new file mode 100644 index 000000000..39be95d58 --- /dev/null +++ b/tests/unit/render/test_match.py @@ -0,0 +1,168 @@ +from dvc.render.match import ( + group_by_filename, + match_renderers, + squash_plots_properties, +) + + +def test_group_by_filename(): + error = FileNotFoundError() + data = { + "v2": { + "data": { + "file.json": {"data": [{"y": 2}, {"y": 3}], "props": {}}, + "other_file.jpg": {"data": "content"}, + } + }, + "v1": { + "data": {"file.json": {"data": [{"y": 4}, {"y": 5}], "props": {}}} + }, + "workspace": { + "data": { + "file.json": {"error": error, "props": {}}, + "other_file.jpg": {"data": "content2"}, + } + }, + } + + results = group_by_filename(data) + assert results["file.json"] == { + "v2": { + "data": { + "file.json": {"data": [{"y": 2}, {"y": 3}], "props": {}}, + } + }, + "v1": { + "data": {"file.json": {"data": [{"y": 4}, {"y": 5}], "props": {}}} + }, + "workspace": { + "data": { + "file.json": {"error": error, "props": {}}, + } + }, + } + assert results["other_file.jpg"] == { + "v2": { + "data": { + "other_file.jpg": {"data": "content"}, + } + }, + "workspace": { + "data": { + "other_file.jpg": {"data": "content2"}, + } + }, + } + + +def test_squash_plots_properties(): + error = FileNotFoundError() + group = { + "v2": { + "data": { + "file.json": { + "data": [{"y": 2}, {"y": 3}], + "props": {"foo": 1}, + }, + } + }, + "v1": { + "data": { + "file.json": { + "data": [{"y": 4}, {"y": 5}], + "props": {"bar": 1}, + } + } + }, + "workspace": { + "data": { + "file.json": {"error": error, "props": {}}, + } + }, + } + + plot_properties = squash_plots_properties(group) + + assert plot_properties == {"foo": 1, "bar": 1} + + +def test_match_renderers_no_out(mocker): + from dvc import render + + vega_convert = mocker.spy(render.vega_converter.VegaConverter, "convert") + image_convert = mocker.spy( + render.image_converter.ImageConverter, "convert" + ) + image_encode = mocker.spy( + render.image_converter.ImageConverter, "_encode_image" + ) + image_write = mocker.spy( + render.image_converter.ImageConverter, "_write_image" + ) + + error = FileNotFoundError() + data = { + "v2": { + "data": { + "file.json": {"data": [{"y": 2}, {"y": 3}], "props": {}}, + "other_file.jpg": {"data": b"content"}, + } + }, + "v1": { + "data": {"file.json": {"data": [{"y": 4}, {"y": 5}], "props": {}}} + }, + "workspace": { + "data": { + "file.json": {"error": error, "props": {}}, + "other_file.jpg": {"data": b"content2"}, + } + }, + } + + renderers = match_renderers(data) + + assert {r.TYPE for r in renderers} == {"vega", "image"} + vega_convert.assert_called() + image_convert.assert_called() + image_encode.assert_called() + image_write.assert_not_called() + + +def test_match_renderers_with_out(tmp_dir, mocker): + from dvc import render + + image_encode = mocker.spy( + render.image_converter.ImageConverter, "_encode_image" + ) + image_write = mocker.spy( + render.image_converter.ImageConverter, "_write_image" + ) + + error = FileNotFoundError() + data = { + "v2": { + "data": { + "file.json": {"data": [{"y": 2}, {"y": 3}], "props": {}}, + "other_file.jpg": {"data": b"content"}, + } + }, + "v1": { + "data": {"file.json": {"data": [{"y": 4}, {"y": 5}], "props": {}}} + }, + "workspace": { + "data": { + "file.json": {"error": error, "props": {}}, + "other_file.jpg": {"data": b"content2"}, + } + }, + } + + match_renderers(data, out=tmp_dir / "foo") + + image_encode.assert_not_called() + image_write.assert_called() + + assert (tmp_dir / "foo" / "v2_other_file.jpg").read_bytes() == b"content" + assert ( + tmp_dir / "foo" / "workspace_other_file.jpg" + ).read_bytes() == b"content2" diff --git a/tests/unit/render/test_parallel_coordinates.py b/tests/unit/render/test_parallel_coordinates.py deleted file mode 100644 index 8eb6a06cb..000000000 --- a/tests/unit/render/test_parallel_coordinates.py +++ /dev/null @@ -1,156 +0,0 @@ -import json - -from dvc.compare import TabularData -from dvc.render.html import write -from dvc.render.plotly import ParallelCoordinatesRenderer - -# pylint: disable=W1514 - - -def expected_format(result): - assert "data" in result - assert "layout" in result - assert isinstance(result["data"], list) - assert result["data"][0]["type"] == "parcoords" - assert isinstance(result["data"][0]["dimensions"], list) - return True - - -def test_scalar_columns(): - td = TabularData(["col-1", "col-2", "col-3"]) - td.extend([["0.1", "1", ""], ["2", "0.2", "0"]]) - renderer = ParallelCoordinatesRenderer(td) - - result = json.loads(renderer.as_json()) - - assert expected_format(result) - - assert result["data"][0]["dimensions"][0] == { - "label": "col-1", - "values": [0.1, 2.0], - } - assert result["data"][0]["dimensions"][1] == { - "label": "col-2", - "values": [1.0, 0.2], - } - assert result["data"][0]["dimensions"][2] == { - "label": "col-3", - "values": [None, 0], - } - - -def test_categorical_columns(): - td = TabularData(["col-1", "col-2"]) - td.extend([["foo", ""], ["bar", "foobar"], ["foo", ""]]) - renderer = ParallelCoordinatesRenderer(td) - - result = json.loads(renderer.as_json()) - - assert expected_format(result) - - assert result["data"][0]["dimensions"][0] == { - "label": "col-1", - "values": [1, 0, 1], - "tickvals": [1, 0, 1], - "ticktext": ["foo", "bar", "foo"], - } - assert result["data"][0]["dimensions"][1] == { - "label": "col-2", - "values": [1, 0, 1], - "tickvals": [1, 0, 1], - "ticktext": ["Missing", "foobar", "Missing"], - } - - -def test_mixed_columns(): - td = TabularData(["categorical", "scalar"]) - td.extend([["foo", "0.1"], ["bar", "2"]]) - renderer = ParallelCoordinatesRenderer(td) - - result = json.loads(renderer.as_json()) - - assert expected_format(result) - - assert result["data"][0]["dimensions"][0] == { - "label": "categorical", - "values": [1, 0], - "tickvals": [1, 0], - "ticktext": ["foo", "bar"], - } - assert result["data"][0]["dimensions"][1] == { - "label": "scalar", - "values": [0.1, 2.0], - } - - -def test_color_by_scalar(): - td = TabularData(["categorical", "scalar"]) - td.extend([["foo", "0.1"], ["bar", "2"]]) - renderer = ParallelCoordinatesRenderer(td, color_by="scalar") - - result = json.loads(renderer.as_json()) - - assert expected_format(result) - assert result["data"][0]["line"] == { - "color": [0.1, 2.0], - "showscale": True, - "colorbar": {"title": "scalar"}, - } - - -def test_color_by_categorical(): - td = TabularData(["categorical", "scalar"]) - td.extend([["foo", "0.1"], ["bar", "2"]]) - renderer = ParallelCoordinatesRenderer(td, color_by="categorical") - - result = json.loads(renderer.as_json()) - - assert expected_format(result) - assert result["data"][0]["line"] == { - "color": [1, 0], - "showscale": True, - "colorbar": { - "title": "categorical", - "tickmode": "array", - "tickvals": [1, 0], - "ticktext": ["foo", "bar"], - }, - } - - -def test_write_parallel_coordinates(tmp_dir): - td = TabularData(["categorical", "scalar"]) - td.extend([["foo", "0.1"], ["bar", "2"]]) - - renderer = ParallelCoordinatesRenderer(td) - html_path = write(tmp_dir, renderers=[renderer]) - - html_text = html_path.read_text() - - assert ParallelCoordinatesRenderer.SCRIPTS in html_text - - div = ParallelCoordinatesRenderer.DIV.format( - id="plot_experiments", partial=renderer.as_json() - ) - assert div in html_text - - -def test_fill_value(): - td = TabularData(["categorical", "scalar"]) - td.extend([["foo", "-"], ["-", "2"]]) - renderer = ParallelCoordinatesRenderer(td, fill_value="-") - - result = json.loads(renderer.as_json()) - - assert expected_format(result) - - assert result["data"][0]["dimensions"][0] == { - "label": "categorical", - "values": [0, 1], - "tickvals": [0, 1], - "ticktext": ["foo", "Missing"], - } - assert result["data"][0]["dimensions"][1] == { - "label": "scalar", - "values": [None, 2.0], - } diff --git a/tests/unit/render/test_render.py b/tests/unit/render/test_render.py deleted file mode 100644 index cb3a45b42..000000000 --- a/tests/unit/render/test_render.py +++ /dev/null @@ -1,78 +0,0 @@ -import os - -import dpath - -from dvc.render.utils import match_renderers, render -from dvc.render.vega import VegaRenderer - - -def assert_website_has_image(page_path, revision, filename, image_content): - index_path = page_path / "index.html" - assert index_path.is_file() - index_content = index_path.read_text() - - resources_filename = f"{revision}_{filename.replace(os.sep, '_')}" - image_path = page_path / "static" / resources_filename - assert image_path.is_file() - - img_html = f'' - assert img_html in index_content - with open(image_path, "rb") as fobj: - assert fobj.read() == image_content - - -def test_render(tmp_dir, dvc): - data = { - "HEAD": { - "data": { - "file.json": { - "data": [{"y": 5}, {"y": 6}], - "props": {"fields": {"y"}}, - }, - os.path.join("sub", "other_file.jpg"): {"data": b"content"}, - } - }, - "v2": { - "data": { - "file.json": { - "data": [{"y": 3}, {"y": 5}], - "props": {"fields": {"y"}}, - }, - "other_file.jpg": {"data": b"content2"}, - } - }, - "v1": { - "data": { - "some.csv": { - "data": [{"y": 2}, {"y": 3}], - "props": {"fields": {"y"}}, - }, - "another.gif": {"data": b"content3"}, - } - }, - } - renderers = match_renderers(data, dvc.plots.templates) - - render(dvc, renderers, path=os.path.join("results", "dir")) - page_path = tmp_dir / "results" / "dir" - index_path = page_path / "index.html" - - assert index_path.is_file() - assert_website_has_image( - page_path, "HEAD", os.path.join("sub", "other_file.jpg"), b"content" - ) - assert_website_has_image(page_path, "v2", "other_file.jpg", b"content2") - assert_website_has_image(page_path, "v1", "another.gif", b"content3") - - def clean(txt: str) -> str: - return txt.replace("\n", "").replace("\r", "").replace(" ", "") - - def get_vega_string(data, filename): - file_data = dpath.util.search(data, ["*", "*", filename]) - return VegaRenderer( - file_data, dvc.plots.templates.load() - ).partial_html() - - index_content = index_path.read_text() - assert clean(get_vega_string(data, "file.json")) in clean(index_content) - assert clean(get_vega_string(data, "some.csv")) in clean(index_content) diff --git a/tests/unit/render/test_renderer.py b/tests/unit/render/test_renderer.py deleted file mode 100644 index 86e3f15f9..000000000 --- a/tests/unit/render/test_renderer.py +++ /dev/null @@ -1,9 +0,0 @@ -from dvc.render.base import Renderer - - -def test_remove_special_characters(): - special_chars = r"!@#$%^&*()[]{};,<>?\/:.|`~=_+" - dirty = f"plot_name{special_chars}" - assert Renderer._remove_special_chars(dirty) == "plot_name" + "_" * len( - special_chars - ) diff --git a/tests/unit/render/test_vega.py b/tests/unit/render/test_vega.py deleted file mode 100644 index f9d3e996b..000000000 --- a/tests/unit/render/test_vega.py +++ /dev/null @@ -1,375 +0,0 @@ -import json -import os -from collections import OrderedDict - -import pytest -from funcy import first - -from dvc.render.base import BadTemplateError -from dvc.render.data import INDEX_FIELD, REVISION_FIELD -from dvc.render.utils import group_by_filename -from dvc.render.vega import VegaRenderer -from dvc.repo.plots.template import NoFieldInDataError - - -def test_group_plots_data(): - error = FileNotFoundError() - data = { - "v2": { - "data": { - "file.json": {"data": [{"y": 2}, {"y": 3}], "props": {}}, - "other_file.jpg": {"data": "content"}, - } - }, - "v1": { - "data": {"file.json": {"data": [{"y": 4}, {"y": 5}], "props": {}}} - }, - "workspace": { - "data": { - "file.json": {"error": error, "props": {}}, - "other_file.jpg": {"data": "content2"}, - } - }, - } - - results = group_by_filename(data) - assert { - "v2": { - "data": { - "file.json": {"data": [{"y": 2}, {"y": 3}], "props": {}}, - } - }, - "v1": { - "data": {"file.json": {"data": [{"y": 4}, {"y": 5}], "props": {}}} - }, - "workspace": { - "data": { - "file.json": {"error": error, "props": {}}, - } - }, - } in results - assert { - "v2": { - "data": { - "other_file.jpg": {"data": "content"}, - } - }, - "workspace": { - "data": { - "other_file.jpg": {"data": "content2"}, - } - }, - } in results - - -def test_one_column(tmp_dir, dvc): - props = { - "x_label": "x_title", - "y_label": "y_title", - "title": "mytitle", - } - data = { - "workspace": { - "data": {"file.json": {"data": [{"val": 2}, {"val": 3}]}} - } - } - - plot_content = VegaRenderer( - data, template=dvc.plots.templates.load(), properties=props - ).asdict() - - assert plot_content["title"] == "mytitle" - assert plot_content["data"]["values"] == [ - {"val": 2, INDEX_FIELD: 0, REVISION_FIELD: "workspace"}, - {"val": 3, INDEX_FIELD: 1, REVISION_FIELD: "workspace"}, - ] - assert ( - first(plot_content["layer"])["encoding"]["x"]["field"] == INDEX_FIELD - ) - assert first(plot_content["layer"])["encoding"]["y"]["field"] == "val" - assert first(plot_content["layer"])["encoding"]["x"]["title"] == "x_title" - assert first(plot_content["layer"])["encoding"]["y"]["title"] == "y_title" - - -def test_multiple_columns(tmp_dir, scm, dvc): - metric = [ - OrderedDict([("first_val", 100), ("second_val", 100), ("val", 2)]), - OrderedDict([("first_val", 200), ("second_val", 300), ("val", 3)]), - ] - - data = { - "workspace": {"data": {"file.json": {"data": metric, "props": {}}}} - } - - plot_content = VegaRenderer(data, dvc.plots.templates.load()).asdict() - - assert plot_content["data"]["values"] == [ - { - "val": 2, - INDEX_FIELD: 0, - REVISION_FIELD: "workspace", - "first_val": 100, - "second_val": 100, - }, - { - "val": 3, - INDEX_FIELD: 1, - REVISION_FIELD: "workspace", - "first_val": 200, - "second_val": 300, - }, - ] - assert ( - first(plot_content["layer"])["encoding"]["x"]["field"] == INDEX_FIELD - ) - assert first(plot_content["layer"])["encoding"]["y"]["field"] == "val" - - -def test_choose_axes(tmp_dir, scm, dvc): - metric = [ - OrderedDict([("first_val", 100), ("second_val", 100), ("val", 2)]), - OrderedDict([("first_val", 200), ("second_val", 300), ("val", 3)]), - ] - - props = {"x": "first_val", "y": "second_val"} - - data = { - "workspace": {"data": {"file.json": {"data": metric, "props": props}}} - } - plot_content = VegaRenderer( - data, template=dvc.plots.templates.load(), properties=props - ).asdict() - - assert plot_content["data"]["values"] == [ - { - "val": 2, - REVISION_FIELD: "workspace", - "first_val": 100, - "second_val": 100, - }, - { - "val": 3, - REVISION_FIELD: "workspace", - "first_val": 200, - "second_val": 300, - }, - ] - assert ( - first(plot_content["layer"])["encoding"]["x"]["field"] == "first_val" - ) - assert ( - first(plot_content["layer"])["encoding"]["y"]["field"] == "second_val" - ) - - -def test_confusion(tmp_dir, dvc): - confusion_matrix = [ - {"predicted": "B", "actual": "A"}, - {"predicted": "A", "actual": "A"}, - ] - props = {"template": "confusion", "x": "predicted", "y": "actual"} - - data = {"workspace": {"data": {"file.json": {"data": confusion_matrix}}}} - - plot_content = VegaRenderer( - data, template=dvc.plots.templates.load("confusion"), properties=props - ).asdict() - - assert plot_content["data"]["values"] == [ - {"predicted": "B", "actual": "A", REVISION_FIELD: "workspace"}, - {"predicted": "A", "actual": "A", REVISION_FIELD: "workspace"}, - ] - assert plot_content["spec"]["transform"][0]["groupby"] == [ - "actual", - "predicted", - ] - assert plot_content["spec"]["encoding"]["x"]["field"] == "predicted" - assert plot_content["spec"]["encoding"]["y"]["field"] == "actual" - - -def test_multiple_revs_default(tmp_dir, scm, dvc): - metric_1 = [{"y": 2}, {"y": 3}] - metric_2 = [{"y": 3}, {"y": 5}] - metric_3 = [{"y": 5}, {"y": 6}] - - data = { - "HEAD": { - "data": { - "file.json": {"data": metric_3, "props": {"fields": {"y"}}} - } - }, - "v2": { - "data": { - "file.json": {"data": metric_2, "props": {"fields": {"y"}}} - } - }, - "v1": { - "data": { - "file.json": {"data": metric_1, "props": {"fields": {"y"}}} - } - }, - } - - plot_content = VegaRenderer(data, dvc.plots.templates.load()).asdict() - - assert plot_content["data"]["values"] == [ - {"y": 5, INDEX_FIELD: 0, REVISION_FIELD: "HEAD"}, - {"y": 6, INDEX_FIELD: 1, REVISION_FIELD: "HEAD"}, - {"y": 3, INDEX_FIELD: 0, REVISION_FIELD: "v2"}, - {"y": 5, INDEX_FIELD: 1, REVISION_FIELD: "v2"}, - {"y": 2, INDEX_FIELD: 0, REVISION_FIELD: "v1"}, - {"y": 3, INDEX_FIELD: 1, REVISION_FIELD: "v1"}, - ] - assert ( - first(plot_content["layer"])["encoding"]["x"]["field"] == INDEX_FIELD - ) - assert first(plot_content["layer"])["encoding"]["y"]["field"] == "y" - - -def test_metric_missing(tmp_dir, scm, dvc, caplog): - - metric = [{"y": 2}, {"y": 3}] - data = { - "v2": {"data": {"file.json": {"data": metric, "props": {}}}}, - "workspace": { - "data": {"file.json": {"error": FileNotFoundError(), "props": {}}} - }, - } - plot_content = VegaRenderer(data, dvc.plots.templates.load()).asdict() - - assert plot_content["data"]["values"] == [ - {"y": 2, INDEX_FIELD: 0, REVISION_FIELD: "v2"}, - {"y": 3, INDEX_FIELD: 1, REVISION_FIELD: "v2"}, - ] - assert ( - first(plot_content["layer"])["encoding"]["x"]["field"] == INDEX_FIELD - ) - assert first(plot_content["layer"])["encoding"]["y"]["field"] == "y" - - -def test_bad_template(tmp_dir, dvc): - metric = [{"val": 2}, {"val": 3}] - data = {"workspace": {"data": {"file.json": {"data": metric}}}} - - from dvc.repo.plots.template import Template - - with pytest.raises(BadTemplateError): - VegaRenderer(data, Template("name", "content")).asdict() - - -def test_plot_choose_columns(tmp_dir, scm, dvc, custom_template): - metric = [{"a": 1, "b": 2, "c": 3}, {"a": 2, "b": 3, "c": 4}] - props = { - "fields": {"b", "c"}, - "x": "b", - "y": "c", - } - data = { - "workspace": {"data": {"file.json": {"data": metric, "props": props}}} - } - - plot_content = VegaRenderer( - data, - template=dvc.plots.templates.load(os.fspath(custom_template)), - properties=props, - ).asdict() - - assert plot_content["data"]["values"] == [ - {"b": 2, "c": 3, REVISION_FIELD: "workspace"}, - {"b": 3, "c": 4, REVISION_FIELD: "workspace"}, - ] - assert plot_content["encoding"]["x"]["field"] == "b" - assert plot_content["encoding"]["y"]["field"] == "c" - - -def test_raise_on_wrong_field(tmp_dir, scm, dvc): - metric = [{"val": 2}, {"val": 3}] - props = {"x": "no_val"} - data = {"workspace": {"data": {"file.json": {"data": metric}}}} - - with pytest.raises(NoFieldInDataError): - VegaRenderer( - data, template=dvc.plots.templates.load(), properties=props - ).asdict() - - -@pytest.mark.parametrize( - "extension, matches", - ( - (".csv", True), - (".json", True), - (".tsv", True), - (".yaml", True), - (".jpg", False), - (".gif", False), - (".jpeg", False), - (".png", False), - ), -) -def test_matches(extension, matches): - filename = "file" + extension - data = { - "HEAD": {"data": {filename: {}}}, - "v1": {"data": {filename: {}}}, - } - assert VegaRenderer.matches(data) == matches - - -@pytest.mark.parametrize( - "template_path, target_name", - [ - (os.path.join(".dvc", "plots", "template.json"), "template"), - (os.path.join(".dvc", "plots", "template.json"), "template.json"), - ( - os.path.join(".dvc", "plots", "subdir", "template.json"), - os.path.join("subdir", "template.json"), - ), - ( - os.path.join(".dvc", "plots", "subdir", "template.json"), - os.path.join("subdir", "template"), - ), - ("template.json", "template.json"), - ], -) -def test_should_resolve_template(tmp_dir, dvc, template_path, target_name): - os.makedirs(os.path.abspath(os.path.dirname(template_path)), exist_ok=True) - with open(template_path, "w", encoding="utf-8") as fd: - fd.write("template_content") - - assert dvc.plots.templates.get_template(target_name) == os.path.abspath( - template_path - ) - - -# TODO remove content checks -def test_as_json(tmp_dir, scm, dvc): - metric = [{"a": 1, "b": 2, "c": 3}, {"a": 2, "b": 3, "c": 4}] - data = {"workspace": {"data": {"file.json": {"data": metric}}}} - props = {"fields": {"b"}} - - renderer = VegaRenderer( - data, template=dvc.plots.templates.load(), properties=props - ) - plot_content = renderer.asdict() - plot_as_json = first(json.loads(renderer.as_json())) - - assert plot_as_json["type"] == "vega" - assert plot_as_json["revisions"] == ["workspace"] - assert plot_as_json["content"] == plot_content - - -def test_as_json_split(tmp_dir, scm, dvc): - metric = [{"a": 1, "b": 2, "c": 3}, {"a": 2, "b": 3, "c": 4}] - data = {"workspace": {"data": {"file.json": {"data": metric}}}} - props = {"fields": {"b"}} - - renderer = VegaRenderer( - data, template=dvc.plots.templates.load(), properties=props - ) - plot_content = renderer.asdict(fill_data=False) - plot_as_json = first(json.loads(renderer.as_json(fill_data=False))) - - assert plot_as_json["type"] == "vega" - assert plot_as_json["revisions"] == ["workspace"] - assert plot_as_json["content"] == plot_content - assert plot_as_json["datapoints"] == renderer._converted_data[0] diff --git a/tests/unit/render/test_data.py b/tests/unit/render/test_vega_converter.py similarity index 66% rename from tests/unit/render/test_data.py rename to tests/unit/render/test_vega_converter.py index 3559b2962..2a6984c2e 100644 --- a/tests/unit/render/test_data.py +++ b/tests/unit/render/test_vega_converter.py @@ -2,13 +2,13 @@ from collections import OrderedDict import pytest -from dvc.render.data import ( - Converter, +from dvc.render.vega_converter import ( FieldsNotFoundError, + PlotDataStructureError, + VegaConverter, _filter_fields, _find_first_list, _lists, - to_datapoints, ) @@ -20,6 +20,9 @@ def test_find_first_list_in_dict(): assert _find_first_list(dmetric, fields=set()) == m1 assert _find_first_list(dmetric, fields={"x"}) == m2 + with pytest.raises(PlotDataStructureError): + _find_first_list(dmetric, fields={"foo"}) + def test_filter_fields(): m = [{"accuracy": 1, "loss": 2}, {"accuracy": 3, "loss": 4}] @@ -57,14 +60,20 @@ def test_finding_lists(dictionary, expected_result): # default x and y {"metric": [{"v": 1}, {"v": 2}]}, {}, - [{"v": 1, "step": 0}, {"v": 2, "step": 1}], + [ + {"v": 1, "step": 0, "filename": "f", "rev": "r"}, + {"v": 2, "step": 1, "filename": "f", "rev": "r"}, + ], {"x": "step", "y": "v"}, ), ( # filter fields {"metric": [{"v": 1, "v2": 0.1}, {"v": 2, "v2": 0.2}]}, {"fields": {"v"}}, - [{"v": 1, "step": 0}, {"v": 2, "step": 1}], + [ + {"v": 1, "step": 0, "filename": "f", "rev": "r"}, + {"v": 2, "step": 1, "filename": "f", "rev": "r"}, + ], { "x": "step", "y": "v", @@ -75,7 +84,10 @@ def test_finding_lists(dictionary, expected_result): # choose x and y {"metric": [{"v": 1, "v2": 0.1}, {"v": 2, "v2": 0.2}]}, {"x": "v", "y": "v2"}, - [{"v": 1, "v2": 0.1}, {"v": 2, "v2": 0.2}], + [ + {"v": 1, "v2": 0.1, "filename": "f", "rev": "r"}, + {"v": 2, "v2": 0.2, "filename": "f", "rev": "r"}, + ], {"x": "v", "y": "v2"}, ), ( @@ -88,8 +100,8 @@ def test_finding_lists(dictionary, expected_result): }, {"x": "v3", "y": "v4", "fields": {"v"}}, [ - {"v": 1, "v3": 0.01, "v4": 0.001}, - {"v": 2, "v3": 0.02, "v4": 0.002}, + {"v": 1, "v3": 0.01, "v4": 0.001, "filename": "f", "rev": "r"}, + {"v": 2, "v3": 0.02, "v4": 0.002, "filename": "f", "rev": "r"}, ], {"x": "v3", "y": "v4", "fields": {"v", "v3", "v4"}}, ), @@ -104,60 +116,41 @@ def test_finding_lists(dictionary, expected_result): }, }, {"x": "v", "y": "v2"}, - [{"v": 1, "v2": 0.1}, {"v": 2, "v2": 0.2}], + [ + {"v": 1, "v2": 0.1, "filename": "f", "rev": "r"}, + {"v": 2, "v2": 0.2, "filename": "f", "rev": "r"}, + ], {"x": "v", "y": "v2"}, ), ], ) def test_convert( - input_data, properties, expected_datapoints, expected_properties + input_data, + properties, + expected_datapoints, + expected_properties, ): - converter = Converter(properties) - datapoints, resolved_properties = converter.convert(input_data) + converter = VegaConverter(properties) + datapoints, resolved_properties = converter.convert( + revision="r", filename="f", data=input_data + ) assert datapoints == expected_datapoints assert resolved_properties == expected_properties def test_convert_skip_step(): - converter = Converter() + converter = VegaConverter() converter.skip_step("append_index") datapoints, resolved_properties = converter.convert( - {"a": "b", "metric": [{"v": 1}, {"v": 2}]} + revision="r", + filename="f", + data={"a": "b", "metric": [{"v": 1}, {"v": 2}]}, ) - assert datapoints == [{"v": 1}, {"v": 2}] + assert datapoints == [ + {"v": 1, "filename": "f", "rev": "r"}, + {"v": 2, "filename": "f", "rev": "r"}, + ] assert resolved_properties == {"x": "step", "y": "v"} - - -def test_to_datapoints(): - input_data = { - "revision": { - "data": { - "filename": { - "data": { - "metric": [ - {"v": 1, "v2": 0.1, "v3": 0.01, "v4": 0.001}, - {"v": 2, "v2": 0.2, "v3": 0.02, "v4": 0.002}, - ] - } - } - } - } - } - props = {"fields": {"v"}, "x": "v2", "y": "v3"} - - datapoints, resolved_properties = to_datapoints(input_data, props) - - assert datapoints == { - "revision": [ - {"v": 1, "v2": 0.1, "v3": 0.01}, - {"v": 2, "v2": 0.2, "v3": 0.02}, - ] - } - assert resolved_properties == { - "fields": {"v", "v2", "v3"}, - "x": "v2", - "y": "v3", - } diff --git a/tests/unit/repo/plots/test_templates.py b/tests/unit/repo/plots/test_templates.py deleted file mode 100644 index 7bc9ffba4..000000000 --- a/tests/unit/repo/plots/test_templates.py +++ /dev/null @@ -1,73 +0,0 @@ -import os - -import pytest - -from dvc.repo.plots.template import ( - LinearTemplate, - PlotTemplates, - ScatterTemplate, - TemplateContentDoesNotMatch, - TemplateNotFoundError, -) - - -def test_raise_on_no_template(tmp_dir, dvc): - with pytest.raises(TemplateNotFoundError): - dvc.plots.templates.load("non_existing_template.json") - - -@pytest.mark.parametrize( - "template_path, target_name", - [ - (os.path.join(".dvc", "plots", "template.json"), "template"), - (os.path.join(".dvc", "plots", "template.json"), "template.json"), - ( - os.path.join(".dvc", "plots", "subdir", "template.json"), - os.path.join("subdir", "template.json"), - ), - ( - os.path.join(".dvc", "plots", "subdir", "template.json"), - os.path.join("subdir", "template"), - ), - ("template.json", "template.json"), - ], -) -def test_load_template(tmp_dir, dvc, template_path, target_name): - os.makedirs(os.path.abspath(os.path.dirname(template_path)), exist_ok=True) - with open(template_path, "w", encoding="utf-8") as fd: - fd.write("template_content") - - assert dvc.plots.templates.load(target_name).content == "template_content" - - -def test_load_default_template(tmp_dir, dvc): - assert dvc.plots.templates.load(None).content == LinearTemplate().content - - -@pytest.mark.parametrize("output", ("output", None)) -@pytest.mark.parametrize( - "targets,expected_templates", - ( - ([None, PlotTemplates.TEMPLATES]), - (["linear", "scatter"], [ScatterTemplate, LinearTemplate]), - ), -) -def test_init(tmp_dir, dvc, output, targets, expected_templates): - output = output or dvc.plots.templates.templates_dir - dvc.plots.templates.init(output, targets) - - assert set(os.listdir(output)) == { - cls.DEFAULT_NAME + ".json" for cls in expected_templates - } - - -def test_raise_on_init_modified(tmp_dir, dvc): - dvc.plots.templates.init(output=None, targets=["linear"]) - - with open( - tmp_dir / ".dvc" / "plots" / "linear.json", "a", encoding="utf-8" - ) as fd: - fd.write("modification") - - with pytest.raises(TemplateContentDoesNotMatch): - dvc.plots.templates.init(output=None, targets=["linear"]) diff --git a/tests/unit/test_plots.py b/tests/unit/test_plots.py index 306cb822c..7bbc4a77f 100644 --- a/tests/unit/test_plots.py +++ b/tests/unit/test_plots.py @@ -1,7 +1,7 @@ import json import os -from dvc.render.utils import get_files +from dvc.render.match import get_files def test_plots_order(tmp_dir, dvc): diff --git a/tests/unit/test_tabular_data.py b/tests/unit/test_tabular_data.py index 7b6b0f409..697b6b41e 100644 --- a/tests/unit/test_tabular_data.py +++ b/tests/unit/test_tabular_data.py @@ -383,14 +383,16 @@ def test_to_parallel_coordinates(tmp_dir, mocker): td = TabularData(["categorical", "scalar"]) td.extend([["foo", "0.1"], ["bar", "2"]]) - write = mocker.patch("dvc.render.html.write") + write = mocker.patch("dvc_render.html.render_html") renderer_class = mocker.patch( - "dvc.render.plotly.ParallelCoordinatesRenderer" + "dvc_render.plotly.ParallelCoordinatesRenderer" ) renderer = renderer_class.return_value td.to_parallel_coordinates(output_path="foo") - renderer_class.assert_called_with(td, None, td._fill_value) + renderer_class.assert_called_with( + td.as_dict(), color_by=None, fill_value=td._fill_value + ) - write.assert_called_with("foo", renderers=[renderer]) + write.assert_called_with(renderers=[renderer], output_file="foo")