From 10bcc9971d06796e9f991e1672ca3eb0121b432f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Redzy=C5=84ski?= Date: Thu, 21 Apr 2022 17:27:45 +0200 Subject: [PATCH] plots: tests: integration tests for studio Related: #7549 --- dvc/render/convert.py | 1 + pyproject.toml | 2 + tests/integration/__init__.py | 0 tests/integration/plots/__init__.py | 0 tests/integration/plots/conftest.py | 81 ++++++++++++ tests/integration/plots/test_json.py | 105 ++++------------ .../integration/plots/test_repo_plots_api.py | 118 ++++++++++++++++++ 7 files changed, 228 insertions(+), 79 deletions(-) create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/plots/__init__.py create mode 100644 tests/integration/plots/conftest.py create mode 100644 tests/integration/plots/test_repo_plots_api.py diff --git a/dvc/render/convert.py b/dvc/render/convert.py index 36967eff5..e98c5a68f 100644 --- a/dvc/render/convert.py +++ b/dvc/render/convert.py @@ -24,6 +24,7 @@ def _get_converter( def to_datapoints(renderer_class, data: Dict, props: Dict): converter = _get_converter(renderer_class, props) datapoints: List[Dict] = [] + final_props: Dict = {} for revision, rev_data in data.items(): for filename, file_data in rev_data.get("data", {}).items(): if "data" in file_data: diff --git a/pyproject.toml b/pyproject.toml index 34c272779..b3196edb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,8 @@ log_level = "debug" addopts = "-ra" markers = [ "needs_internet: Might need network access for the tests", +"vscode: Tests verifying contract between DVC and VSCode plugin", +"studio: Tests verifying contract between DVC and Studio" ] xfail_strict = true diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/plots/__init__.py b/tests/integration/plots/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/plots/conftest.py b/tests/integration/plots/conftest.py new file mode 100644 index 000000000..8b607d566 --- /dev/null +++ b/tests/integration/plots/conftest.py @@ -0,0 +1,81 @@ +import pytest + + +@pytest.fixture +def repo_with_plots(tmp_dir, scm, dvc, run_copy_metrics): + def make(): + linear_v1 = [ + {"x": 1, "y": 0.1}, + {"x": 2, "y": 0.2}, + {"x": 3, "y": 0.3}, + ] + + confusion_v1 = [ + {"actual": 0, "predicted": 1}, + {"actual": 0, "predicted": 1}, + {"actual": 1, "predicted": 0}, + {"actual": 1, "predicted": 0}, + ] + + image_v1 = b"content" + + (tmp_dir / "linear_src.json").dump_json(linear_v1) + (tmp_dir / "confusion_src.json").dump_json(confusion_v1) + (tmp_dir / "image_src.png").write_bytes(image_v1) + + scm.add(["linear_src.json", "confusion_src.json", "image_src.png"]) + scm.commit("add data sources") + + run_copy_metrics( + "linear_src.json", + "linear.json", + name="linear", + plots=["linear.json"], + commit="linear", + ) + run_copy_metrics( + "confusion_src.json", + "confusion.json", + name="confusion", + plots=["confusion.json"], + commit="confusion", + ) + confusion_props = { + "title": "confusion matrix", + "x": "predicted", + "y": "actual", + "template": "confusion", + } + dvc.plots.modify("confusion.json", confusion_props) + run_copy_metrics( + "image_src.png", + "image.png", + name="image", + plots=["image.png"], + commit="image", + ) + + scm.add(["dvc.yaml", "dvc.lock"]) + scm.commit("commit dvc files") + yield image_v1, linear_v1, confusion_v1, confusion_props + linear_v2 = [ + {"x": 1, "y": 0.2}, + {"x": 2, "y": 0.3}, + {"x": 3, "y": 0.4}, + ] + confusion_v2 = [ + {"actual": 0, "predicted": 0}, + {"actual": 0, "predicted": 0}, + {"actual": 1, "predicted": 1}, + {"actual": 1, "predicted": 1}, + ] + image_v2 = b"content2" + + (tmp_dir / "linear_src.json").dump_json(linear_v2) + (tmp_dir / "confusion_src.json").dump_json(confusion_v2) + (tmp_dir / "image_src.png").write_bytes(image_v2) + + dvc.reproduce() + yield image_v2, linear_v2, confusion_v2, confusion_props + + return make diff --git a/tests/integration/plots/test_json.py b/tests/integration/plots/test_json.py index 85bfe7bed..f3bbb8cd4 100644 --- a/tests/integration/plots/test_json.py +++ b/tests/integration/plots/test_json.py @@ -6,6 +6,7 @@ from urllib.parse import urlparse from urllib.request import url2pathname import dpath.util +import pytest from funcy import first from dvc.cli import main @@ -36,13 +37,6 @@ def call(capsys, subcommand="show"): return abspath, json_result, split_json_result -def test_no_plots(tmp_dir, scm, dvc, capsys): - html_path, json_result, split_json_result = call(capsys) - assert os.path.exists(html_path) - assert json_result == {} - assert split_json_result == {} - - def filter_fields(datapoints: List[Dict], fields: List[str]): tmp = deepcopy(datapoints) @@ -77,6 +71,9 @@ def verify_image(tmp_dir, version, filename, content, html_path, json_result): os.path.join("dvc_plots", "static", f"{version}_{filename}") in html_content ) + + # there should be no absolute paths in produced HTML + assert str(tmp_dir) not in html_content assert ( tmp_dir / "dvc_plots" / "static" / f"{version}_{filename}" ).read_bytes() == content @@ -147,7 +144,7 @@ def verify_vega( assert json_content == split_json_content -def verify_vega_props(filename, json_result, title, x, y): +def verify_vega_props(filename, json_result, title, x, y, **kwargs): data = json_result[filename] assert len(data) == 1 data = first(data) @@ -164,85 +161,26 @@ def verify_vega_props(filename, json_result, title, x, y): ) -def repo_with_plots(tmp_dir, scm, dvc, run_copy_metrics): - linear_v1 = [{"x": 1, "y": 0.1}, {"x": 2, "y": 0.2}, {"x": 3, "y": 0.3}] - - confusion_v1 = [ - {"actual": 0, "predicted": 1}, - {"actual": 0, "predicted": 1}, - {"actual": 1, "predicted": 0}, - {"actual": 1, "predicted": 0}, - ] - - image_v1 = b"content" - - (tmp_dir / "linear_src.json").dump_json(linear_v1) - (tmp_dir / "confusion_src.json").dump_json(confusion_v1) - (tmp_dir / "image_src.png").write_bytes(image_v1) - - scm.add(["linear_src.json", "confusion_src.json", "image_src.png"]) - scm.commit("add data sources") - - run_copy_metrics( - "linear_src.json", - "linear.json", - name="linear", - plots=["linear.json"], - commit="linear", - ) - run_copy_metrics( - "confusion_src.json", - "confusion.json", - name="confusion", - plots=["confusion.json"], - commit="confusion", - ) - confusion_props = { - "title": "confusion matrix", - "x": "predicted", - "y": "actual", - } - dvc.plots.modify( - "confusion.json", - {**confusion_props, **{"template": "confusion"}}, - ) - run_copy_metrics( - "image_src.png", - "image.png", - name="image", - plots=["image.png"], - commit="image", - ) - - scm.add(["dvc.yaml", "dvc.lock"]) - scm.commit("commit dvc files") - yield image_v1, linear_v1, confusion_v1, confusion_props - linear_v2 = [{"x": 1, "y": 0.2}, {"x": 2, "y": 0.3}, {"x": 3, "y": 0.4}] - confusion_v2 = [ - {"actual": 0, "predicted": 0}, - {"actual": 0, "predicted": 0}, - {"actual": 1, "predicted": 1}, - {"actual": 1, "predicted": 1}, - ] - image_v2 = b"content2" - - (tmp_dir / "linear_src.json").dump_json(linear_v2) - (tmp_dir / "confusion_src.json").dump_json(confusion_v2) - (tmp_dir / "image_src.png").write_bytes(image_v2) - - dvc.reproduce() - yield image_v2, linear_v2, confusion_v2, confusion_props +@pytest.mark.vscode +def test_no_plots(tmp_dir, scm, dvc, capsys): + html_path, json_result, split_json_result = call(capsys) + assert os.path.exists(html_path) + assert json_result == {} + assert split_json_result == {} -def test_repo_with_plots(tmp_dir, scm, dvc, capsys, run_copy_metrics): - repo_state = repo_with_plots(tmp_dir, scm, dvc, run_copy_metrics) +@pytest.mark.vscode +def test_repo_with_plots( + tmp_dir, scm, dvc, capsys, run_copy_metrics, repo_with_plots +): + repo_state = repo_with_plots() image_v1, linear_v1, confusion_v1, confusion_props = next(repo_state) html_path, json_result, split_json_result = call(capsys) verify_image( - tmp_dir, "workspace", "image.png", b"content", html_path, json_result + tmp_dir, "workspace", "image.png", image_v1, html_path, json_result ) verify_vega( @@ -297,3 +235,12 @@ def test_repo_with_plots(tmp_dir, scm, dvc, capsys, run_copy_metrics): split_json_result, ) verify_vega_props("confusion.json", json_result, **confusion_props) + from dvc.utils.fs import remove + + # even if there is no data, call should be successful + remove(tmp_dir / ".dvc" / "cache") + remove("linear.json") + remove("confusion.json") + remove("image.png") + call(capsys) + call(capsys, subcommand="diff") diff --git a/tests/integration/plots/test_repo_plots_api.py b/tests/integration/plots/test_repo_plots_api.py new file mode 100644 index 000000000..24fcdd9fc --- /dev/null +++ b/tests/integration/plots/test_repo_plots_api.py @@ -0,0 +1,118 @@ +import dpath.util +import pytest + + +@pytest.mark.studio +def test_api(tmp_dir, dvc, repo_with_plots): + repo_state = repo_with_plots() + image_v1, linear_v1, confusion_v1, confusion_params = next(repo_state) + + workspace_data = next(dvc.plots.collect()) + + assert ( + dpath.util.get( + workspace_data, ["workspace", "data", "image.png", "props"] + ) + == {} + ) + image_source = dpath.util.get( + workspace_data, ["workspace", "data", "image.png", "data_source"] + ) + assert callable(image_source) + assert image_source() == {"data": image_v1} + + assert ( + dpath.util.get( + workspace_data, ["workspace", "data", "linear.json", "props"] + ) + == {} + ) + linear_source = dpath.util.get( + workspace_data, ["workspace", "data", "linear.json", "data_source"] + ) + assert callable(linear_source) + assert linear_source() == {"data": linear_v1} + + assert ( + dpath.util.get( + workspace_data, ["workspace", "data", "confusion.json", "props"] + ) + == confusion_params + ) + confusion_source = dpath.util.get( + workspace_data, ["workspace", "data", "confusion.json", "data_source"] + ) + assert callable(confusion_source) + assert confusion_source() == {"data": confusion_v1} + + image_v2, linear_v2, confusion_v2, _ = next(repo_state) + data_generator = dvc.plots.collect(revs=["workspace", "HEAD"]) + + workspace_data = next(data_generator) + + assert ( + dpath.util.get( + workspace_data, ["workspace", "data", "image.png", "props"] + ) + == {} + ) + image_source = dpath.util.get( + workspace_data, ["workspace", "data", "image.png", "data_source"] + ) + assert callable(image_source) + assert image_source() == {"data": image_v2} + + assert ( + dpath.util.get( + workspace_data, ["workspace", "data", "linear.json", "props"] + ) + == {} + ) + linear_source = dpath.util.get( + workspace_data, ["workspace", "data", "linear.json", "data_source"] + ) + assert callable(linear_source) + assert linear_source() == {"data": linear_v2} + + assert ( + dpath.util.get( + workspace_data, ["workspace", "data", "confusion.json", "props"] + ) + == confusion_params + ) + confusion_source = dpath.util.get( + workspace_data, ["workspace", "data", "confusion.json", "data_source"] + ) + assert callable(confusion_source) + assert confusion_source() == {"data": confusion_v2} + + head_data = next(data_generator) + + assert ( + dpath.util.get(head_data, ["HEAD", "data", "image.png", "props"]) == {} + ) + image_source = dpath.util.get( + head_data, ["HEAD", "data", "image.png", "data_source"] + ) + assert callable(image_source) + assert image_source() == {"data": image_v1} + + assert ( + dpath.util.get(head_data, ["HEAD", "data", "linear.json", "props"]) + == {} + ) + linear_source = dpath.util.get( + head_data, ["HEAD", "data", "linear.json", "data_source"] + ) + assert callable(linear_source) + assert linear_source() == {"data": linear_v1} + + assert ( + dpath.util.get(head_data, ["HEAD", "data", "confusion.json", "props"]) + == confusion_params + ) + confusion_source = dpath.util.get( + head_data, ["HEAD", "data", "confusion.json", "data_source"] + ) + assert callable(confusion_source) + assert confusion_source() == {"data": confusion_v1}