From 4a1985d7cdcaf5b079745b63ffe76a340b0b9e9f Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Tue, 2 Nov 2021 20:32:42 +0100 Subject: [PATCH] dvc render: Add RENDERERS registry. Related: #6840, #4455, #6943 --- dvc/render/__init__.py | 35 ++---------------- dvc/render/base.py | 52 +++++++++++++++++++++++++++ dvc/render/html.py | 25 ++++++++----- dvc/render/image.py | 7 +++- dvc/render/utils.py | 12 +++---- dvc/render/vega.py | 65 ++++++++++++++++++++++++---------- dvc/repo/plots/__init__.py | 9 ++++- dvc/repo/plots/template.py | 47 +++--------------------- tests/func/plots/test_show.py | 2 +- tests/unit/render/test_html.py | 2 +- tests/unit/render/test_vega.py | 31 ++++++++-------- 11 files changed, 157 insertions(+), 130 deletions(-) create mode 100644 dvc/render/base.py diff --git a/dvc/render/__init__.py b/dvc/render/__init__.py index 30d26f3a7..0db485254 100644 --- a/dvc/render/__init__.py +++ b/dvc/render/__init__.py @@ -1,33 +1,4 @@ -import logging -from typing import TYPE_CHECKING, Dict +from .image import ImageRenderer +from .vega import VegaRenderer -if TYPE_CHECKING: - from dvc.types import StrPath - -logger = logging.getLogger(__name__) - - -class Renderer: - def __init__(self, data: Dict): - self.data = data - - from dvc.render.utils import get_files - - files = get_files(self.data) - - # we assume comparison of same file between revisions for now - assert len(files) == 1 - self.filename = files.pop() - - def _convert(self, path: "StrPath"): - raise NotImplementedError - - @property - def DIV(self): - raise NotImplementedError - - def generate_html(self, path: "StrPath"): - """this method might edit content of path""" - partial = self._convert(path) - div_id = f"plot_{self.filename.replace('.', '_').replace('/', '_')}" - return self.DIV.format(id=div_id, partial=partial) +RENDERERS = [ImageRenderer, VegaRenderer] diff --git a/dvc/render/base.py b/dvc/render/base.py new file mode 100644 index 000000000..b92976b83 --- /dev/null +++ b/dvc/render/base.py @@ -0,0 +1,52 @@ +import abc +from typing import TYPE_CHECKING, Dict + +from dvc.exceptions import DvcException + +if TYPE_CHECKING: + from dvc.types import StrPath + + +REVISION_FIELD = "rev" +INDEX_FIELD = "step" + + +class BadTemplateError(DvcException): + pass + + +class Renderer(abc.ABC): + def __init__(self, data: Dict, templates=None): + self.data = data + self.templates = templates + + from dvc.render.utils import get_files + + files = get_files(self.data) + + # we assume comparison of same file between revisions for now + assert len(files) == 1 + self.filename = files.pop() + + def _convert(self, path: "StrPath"): + raise NotImplementedError + + @property + @abc.abstractmethod + def DIV(self): + raise NotImplementedError + + @property + @abc.abstractmethod + def SCRIPTS(self): + raise NotImplementedError + + @abc.abstractmethod + def as_json(self): + raise NotImplementedError + + def generate_html(self, path: "StrPath"): + """this method might edit content of path""" + partial = self._convert(path) + div_id = f"plot_{self.filename.replace('.', '_').replace('/', '_')}" + return self.DIV.format(id=div_id, partial=partial) diff --git a/dvc/render/html.py b/dvc/render/html.py index d66b2cd1d..89da9677c 100644 --- a/dvc/render/html.py +++ b/dvc/render/html.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Dict, List, Optional from dvc.exceptions import DvcException if TYPE_CHECKING: - from dvc.render import Renderer + from dvc.render.base import Renderer from dvc.types import StrPath PAGE_HTML = """ @@ -13,9 +13,7 @@ PAGE_HTML = """ {refresh_tag} DVC Plot - - - + {scripts} {plot_divs} @@ -29,8 +27,9 @@ class MissingPlaceholderError(DvcException): class HTML: - PLACEHOLDER = "plot_divs" - PLACEHOLDER_FORMAT_STR = f"{{{PLACEHOLDER}}}" + SCRIPTS_PLACEHOLDER = "scripts" + PLOTS_PLACEHOLDER = "plot_divs" + PLOTS_PLACEHOLDER_FORMAT_STR = f"{{{PLOTS_PLACEHOLDER}}}" REFRESH_PLACEHOLDER = "refresh_tag" REFRESH_TAG = '' @@ -40,11 +39,12 @@ class HTML: refresh_seconds: Optional[int] = None, ): template = template or PAGE_HTML - if self.PLACEHOLDER_FORMAT_STR not in template: - raise MissingPlaceholderError(self.PLACEHOLDER_FORMAT_STR) + if self.PLOTS_PLACEHOLDER_FORMAT_STR not in template: + raise MissingPlaceholderError(self.PLOTS_PLACEHOLDER_FORMAT_STR) self.template = template self.elements: List[str] = [] + self.scripts: str = "" self.refresh_tag = "" if refresh_seconds is not None: self.refresh_tag = self.REFRESH_TAG.format(refresh_seconds) @@ -65,13 +65,19 @@ class HTML: self.elements.append(tabulate.tabulate(rows, header, tablefmt="html")) return self + def with_scripts(self, scripts: str) -> "HTML": + if scripts not in self.scripts: + self.scripts += f"\n{scripts}" + return self + def with_element(self, html: str) -> "HTML": self.elements.append(html) return self def embed(self) -> str: kwargs = { - self.PLACEHOLDER: "\n".join(self.elements), + self.SCRIPTS_PLACEHOLDER: self.scripts, + self.PLOTS_PLACEHOLDER: "\n".join(self.elements), self.REFRESH_PLACEHOLDER: self.refresh_tag, } return self.template.format(**kwargs) @@ -98,6 +104,7 @@ def write( document.with_element("
") for renderer in renderers: + document.with_scripts(renderer.SCRIPTS) document.with_element(renderer.generate_html(path)) index = Path(os.path.join(path, "index.html")) diff --git a/dvc/render/image.py b/dvc/render/image.py index 60ba45a39..0715f211e 100644 --- a/dvc/render/image.py +++ b/dvc/render/image.py @@ -1,7 +1,7 @@ import os from typing import TYPE_CHECKING -from dvc.render import Renderer +from dvc.render.base import Renderer from dvc.render.utils import get_files from dvc.utils import relpath @@ -17,6 +17,8 @@ class ImageRenderer(Renderer): {partial} """ + SCRIPTS = "" + def _write_image( self, page_dir_path: "StrPath", @@ -56,6 +58,9 @@ class ImageRenderer(Renderer): return "\n".join(div_content) return "" + def as_json(self): + raise NotImplementedError + @staticmethod def matches(data): files = get_files(data) diff --git a/dvc/render/utils.py b/dvc/render/utils.py index ff74d210f..cc6d4e1bb 100644 --- a/dvc/render/utils.py +++ b/dvc/render/utils.py @@ -26,20 +26,18 @@ def find_vega(repo, plots_data, target): from dvc.render.vega import VegaRenderer if found and VegaRenderer.matches(found): - return VegaRenderer(found, repo.plots.templates).get_vega() + return VegaRenderer(found, repo.plots.templates).as_json() return "" def match_renderers(plots_data, templates): - from dvc.render.image import ImageRenderer - from dvc.render.vega import VegaRenderer + from dvc.render import RENDERERS renderers = [] for g in group_by_filename(plots_data): - if VegaRenderer.matches(g): - renderers.append(VegaRenderer(g, templates)) - if ImageRenderer.matches(g): - renderers.append(ImageRenderer(g)) + for renderer_class in RENDERERS: + if renderer_class.matches(g): + renderers.append(renderer_class(g, templates)) return renderers diff --git a/dvc/render/vega.py b/dvc/render/vega.py index 281398826..fb8930cee 100644 --- a/dvc/render/vega.py +++ b/dvc/render/vega.py @@ -5,24 +5,17 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Union from funcy import first from dvc.exceptions import DvcException -from dvc.render import Renderer +from dvc.render.base import ( + INDEX_FIELD, + REVISION_FIELD, + BadTemplateError, + Renderer, +) from dvc.render.utils import get_files if TYPE_CHECKING: - from dvc.repo.plots.template import PlotTemplates from dvc.types import StrPath -REVISION_FIELD = "rev" -INDEX_FIELD = "step" - - -class PlotMetricTypeError(DvcException): - def __init__(self, file): - super().__init__( - "'{}' - file type error\n" - "Only JSON, YAML, CSV and TSV formats are supported.".format(file) - ) - class PlotDataStructureError(DvcException): def __init__(self): @@ -108,9 +101,11 @@ class VegaRenderer(Renderer): """ - def __init__(self, data: Dict, templates: "PlotTemplates"): - super().__init__(data) - self.templates = templates + SCRIPTS = """ + + + + """ def _squash_props(self) -> Dict: resolved: Dict[str, str] = {} @@ -144,7 +139,36 @@ class VegaRenderer(Renderer): datapoints.extend(tmp) return datapoints - def get_vega(self) -> Optional[str]: + def fill_template(self, template, datapoints, props=None): + props = props or {} + + content = deepcopy(template.content) + if template.anchor_str("data") not in template.content: + anchor = template.anchor("data") + raise BadTemplateError( + f"Template '{template.name}' is not using '{anchor}' anchor" + ) + + if props.get("x"): + template.check_field_exists(datapoints, props.get("x")) + if props.get("y"): + template.check_field_exists(datapoints, props.get("y")) + + content = template.fill_anchor(content, "data", datapoints) + + props.setdefault("title", "") + props.setdefault("x_label", props.get("x")) + props.setdefault("y_label", props.get("y")) + + names = ["title", "x", "y", "x_label", "y_label"] + for name in names: + value = props.get(name) + if value is not None: + content = template.fill_anchor(content, name, value) + + return content + + def as_json(self) -> Optional[str]: props = self._squash_props() template = self.templates.load(props.get("template", None)) @@ -162,11 +186,14 @@ class VegaRenderer(Renderer): props["y"] = first( f for f in reversed(fields) if f not in skip ) - return template.render(datapoints, props=props) + filled_template = self.fill_template(template, datapoints, props) + + return filled_template + return None def _convert(self, path: "StrPath"): - return self.get_vega() + return self.as_json() @staticmethod def matches(data): diff --git a/dvc/repo/plots/__init__.py b/dvc/repo/plots/__init__.py index 6a98b50b5..59e02c883 100644 --- a/dvc/repo/plots/__init__.py +++ b/dvc/repo/plots/__init__.py @@ -8,7 +8,6 @@ from typing import TYPE_CHECKING, Callable, Dict, Generator, List, Optional from funcy import cached_property, first, project from dvc.exceptions import DvcException -from dvc.render.vega import PlotMetricTypeError from dvc.utils import ( error_handler, errored_revisions, @@ -24,6 +23,14 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +class PlotMetricTypeError(DvcException): + def __init__(self, file): + super().__init__( + "'{}' - file type error\n" + "Only JSON, YAML, CSV and TSV formats are supported.".format(file) + ) + + class NotAPlotError(DvcException): def __init__(self, out): super().__init__( diff --git a/dvc/repo/plots/template.py b/dvc/repo/plots/template.py index 4e8cab56f..9c8dffde2 100644 --- a/dvc/repo/plots/template.py +++ b/dvc/repo/plots/template.py @@ -20,10 +20,6 @@ class TemplateNotFoundError(DvcException): super().__init__(f"Template '{name}' not found.") -class BadTemplateError(DvcException): - pass - - class NoFieldInDataError(DvcException): def __init__(self, field_name): super().__init__( @@ -41,59 +37,26 @@ class Template: self.content = content self.name = name - def render(self, data, props=None): - props = props or {} - - if self._anchor_str("data") not in self.content: - anchor = self.anchor("data") - raise BadTemplateError( - f"Template '{self.name}' is not using '{anchor}' anchor" - ) - - if props.get("x"): - Template._check_field_exists(data, props.get("x")) - if props.get("y"): - Template._check_field_exists(data, props.get("y")) - - content = self._fill_anchor(self.content, "data", data) - content = self._fill_metadata(content, props) - - return content - @classmethod def anchor(cls, name): return cls.ANCHOR.format(name.upper()) def has_anchor(self, name): - return self._anchor_str(name) in self.content + return self.anchor_str(name) in self.content @classmethod - def _fill_anchor(cls, content, name, value): + def fill_anchor(cls, content, name, value): value_str = json.dumps( value, indent=cls.INDENT, separators=cls.SEPARATORS, sort_keys=True ) - return content.replace(cls._anchor_str(name), value_str) + return content.replace(cls.anchor_str(name), value_str) @classmethod - def _anchor_str(cls, name): + def anchor_str(cls, name): return f'"{cls.anchor(name)}"' - @classmethod - def _fill_metadata(cls, content, props): - props.setdefault("title", "") - props.setdefault("x_label", props.get("x")) - props.setdefault("y_label", props.get("y")) - - names = ["title", "x", "y", "x_label", "y_label"] - for name in names: - value = props.get(name) - if value is not None: - content = cls._fill_anchor(content, name, value) - - return content - @staticmethod - def _check_field_exists(data, field): + def check_field_exists(data, field): if not any(field in row for row in data): raise NoFieldInDataError(field) diff --git a/tests/func/plots/test_show.py b/tests/func/plots/test_show.py index b5141ae51..d3662084e 100644 --- a/tests/func/plots/test_show.py +++ b/tests/func/plots/test_show.py @@ -6,8 +6,8 @@ from funcy import get_in from dvc.dvcfile import PIPELINE_FILE from dvc.exceptions import OverlappingOutputPathsError from dvc.main import main -from dvc.render.vega import PlotMetricTypeError from dvc.repo import Repo +from dvc.repo.plots import PlotMetricTypeError from dvc.utils import onerror_collect from dvc.utils.fs import remove from dvc.utils.serialize import EncodingError, YAMLFileCorruptedError diff --git a/tests/unit/render/test_html.py b/tests/unit/render/test_html.py index 5b4648a1a..fe3878b07 100644 --- a/tests/unit/render/test_html.py +++ b/tests/unit/render/test_html.py @@ -22,7 +22,7 @@ CUSTOM_PAGE_HTML = """ ( None, ["content"], - PAGE_HTML.format(plot_divs="content", refresh_tag=""), + PAGE_HTML.format(plot_divs="content", refresh_tag="", scripts=""), ), ( CUSTOM_PAGE_HTML, diff --git a/tests/unit/render/test_vega.py b/tests/unit/render/test_vega.py index 5ff80b55c..ca9dad68b 100644 --- a/tests/unit/render/test_vega.py +++ b/tests/unit/render/test_vega.py @@ -9,15 +9,12 @@ from dvc.render.utils import find_vega, group_by_filename from dvc.render.vega import ( INDEX_FIELD, REVISION_FIELD, + BadTemplateError, VegaRenderer, _find_data, _lists, ) -from dvc.repo.plots.template import ( - BadTemplateError, - NoFieldInDataError, - TemplateNotFoundError, -) +from dvc.repo.plots.template import NoFieldInDataError, TemplateNotFoundError @pytest.mark.parametrize( @@ -110,7 +107,7 @@ def test_one_column(tmp_dir, scm, dvc): } } - plot_string = VegaRenderer(data, dvc.plots.templates).get_vega() + plot_string = VegaRenderer(data, dvc.plots.templates).as_json() plot_content = json.loads(plot_string) assert plot_content["title"] == "mytitle" @@ -136,7 +133,7 @@ def test_multiple_columns(tmp_dir, scm, dvc): "workspace": {"data": {"file.json": {"data": metric, "props": {}}}} } - plot_string = VegaRenderer(data, dvc.plots.templates).get_vega() + plot_string = VegaRenderer(data, dvc.plots.templates).as_json() plot_content = json.loads(plot_string) assert plot_content["data"]["values"] == [ @@ -172,7 +169,7 @@ def test_choose_axes(tmp_dir, scm, dvc): data = { "workspace": {"data": {"file.json": {"data": metric, "props": props}}} } - plot_string = VegaRenderer(data, dvc.plots.templates).get_vega() + plot_string = VegaRenderer(data, dvc.plots.templates).as_json() plot_content = json.loads(plot_string) assert plot_content["data"]["values"] == [ @@ -210,7 +207,7 @@ def test_confusion(tmp_dir, dvc): } } - plot_string = VegaRenderer(data, dvc.plots.templates).get_vega() + plot_string = VegaRenderer(data, dvc.plots.templates).as_json() plot_content = json.loads(plot_string) assert plot_content["data"]["values"] == [ @@ -248,7 +245,7 @@ def test_multiple_revs_default(tmp_dir, scm, dvc): }, } - plot_string = VegaRenderer(data, dvc.plots.templates).get_vega() + plot_string = VegaRenderer(data, dvc.plots.templates).as_json() plot_content = json.loads(plot_string) assert plot_content["data"]["values"] == [ @@ -274,7 +271,7 @@ def test_metric_missing(tmp_dir, scm, dvc, caplog): "data": {"file.json": {"error": FileNotFoundError(), "props": {}}} }, } - plot_string = VegaRenderer(data, dvc.plots.templates).get_vega() + plot_string = VegaRenderer(data, dvc.plots.templates).as_json() plot_content = json.loads(plot_string) assert plot_content["data"]["values"] == [ @@ -294,7 +291,7 @@ def test_custom_template(tmp_dir, scm, dvc, custom_template): "workspace": {"data": {"file.json": {"data": metric, "props": props}}} } - plot_string = VegaRenderer(data, dvc.plots.templates).get_vega() + plot_string = VegaRenderer(data, dvc.plots.templates).as_json() plot_content = json.loads(plot_string) assert plot_content["data"]["values"] == [ @@ -313,7 +310,7 @@ def test_raise_on_no_template(tmp_dir, dvc): } with pytest.raises(TemplateNotFoundError): - VegaRenderer(data, dvc.plots.templates).get_vega() + VegaRenderer(data, dvc.plots.templates).as_json() def test_bad_template(tmp_dir, dvc): @@ -325,7 +322,7 @@ def test_bad_template(tmp_dir, dvc): } with pytest.raises(BadTemplateError): - VegaRenderer(data, dvc.plots.templates).get_vega() + VegaRenderer(data, dvc.plots.templates).as_json() def test_plot_choose_columns(tmp_dir, scm, dvc, custom_template): @@ -340,7 +337,7 @@ def test_plot_choose_columns(tmp_dir, scm, dvc, custom_template): "workspace": {"data": {"file.json": {"data": metric, "props": props}}} } - plot_string = VegaRenderer(data, dvc.plots.templates).get_vega() + plot_string = VegaRenderer(data, dvc.plots.templates).as_json() plot_content = json.loads(plot_string) assert plot_content["data"]["values"] == [ @@ -359,7 +356,7 @@ def test_plot_default_choose_column(tmp_dir, scm, dvc): } } - plot_string = VegaRenderer(data, dvc.plots.templates).get_vega() + plot_string = VegaRenderer(data, dvc.plots.templates).as_json() plot_content = json.loads(plot_string) assert plot_content["data"]["values"] == [ @@ -381,7 +378,7 @@ def test_raise_on_wrong_field(tmp_dir, scm, dvc): } with pytest.raises(NoFieldInDataError): - VegaRenderer(data, dvc.plots.templates).get_vega() + VegaRenderer(data, dvc.plots.templates).as_json() @pytest.mark.parametrize(