From 6c79f6bbfc89f5f4b366d8a2a05883a0ed63383e Mon Sep 17 00:00:00 2001 From: Peter Rowlands Date: Mon, 1 May 2023 16:28:47 +0900 Subject: [PATCH] testing: register pytest plugin --- dvc/testing/README.rst | 19 +++++ .../benchmarks/cli/commands/test_plots.py | 6 +- dvc/testing/benchmarks/conftest.py | 84 ------------------- dvc/testing/benchmarks/fixtures.py | 8 +- dvc/testing/benchmarks/plugin.py | 83 ++++++++++++++++++ dvc/testing/conftest.py | 6 -- dvc/testing/plugin.py | 20 +++++ pyproject.toml | 3 +- 8 files changed, 132 insertions(+), 97 deletions(-) create mode 100644 dvc/testing/README.rst create mode 100644 dvc/testing/benchmarks/plugin.py create mode 100644 dvc/testing/plugin.py diff --git a/dvc/testing/README.rst b/dvc/testing/README.rst new file mode 100644 index 000000000..0ec501b79 --- /dev/null +++ b/dvc/testing/README.rst @@ -0,0 +1,19 @@ +DVC pytest plugin + +dvc.testing.benchmarks +====================== + +Benchmark test definitions are now part of ``dvc.testing``. +For CLI usage and `bench.dvc.org `_ details see `dvc-bench `_. + +``dvc.testing.benchmarks`` structure: + +* cli: should be able to run these with any dvc (rpm, deb, pypi, snap, etc) (could be used in dvc-test repo too) + + * commands: granular tests for individual commands. These should have a cached setup, so that we could use them during rapid development instead of our hand-written scripts. Every test could be run in a separate machine. + * stories: multistage start-to-end benchmarks, useful for testing workflows (e.g. in documentation, see test_sharing inspired by `Storing and sharing `_. Every full story could be run in a separate machine. + +* api: for python api only. + + * methods: granular tests for individual methods (e.g. ``api.open/read``). Same reasoning as in ``cli.commands`` + * stories: same as ``cli.stories`` but for our api. E.g. imagine using our api with pandas or smth like that. diff --git a/dvc/testing/benchmarks/cli/commands/test_plots.py b/dvc/testing/benchmarks/cli/commands/test_plots.py index 80d07b9ab..29fecb520 100644 --- a/dvc/testing/benchmarks/cli/commands/test_plots.py +++ b/dvc/testing/benchmarks/cli/commands/test_plots.py @@ -1,7 +1,9 @@ import pytest from dvc.repo import Repo -from tests.benchmarks.conftest import pull +from dvc.testing.benchmarks.fixtures import _pull + +# pylint: disable=redefined-outer-name,unused-argument # pylint: disable=unused-argument @@ -9,7 +11,7 @@ from tests.benchmarks.conftest import pull @pytest.mark.requires(minversion=(2, 34, 0), reason="top-level plots not supported") def test_plots(project, bench_dvc): with Repo() as dvc: - pull(dvc) + _pull(dvc) kwargs = {"rounds": 5, "iterations": 3, "warmup_rounds": 2} bench_dvc("plots", "show", name="show", **kwargs) diff --git a/dvc/testing/benchmarks/conftest.py b/dvc/testing/benchmarks/conftest.py index dd5ed276f..12c22cdf0 100644 --- a/dvc/testing/benchmarks/conftest.py +++ b/dvc/testing/benchmarks/conftest.py @@ -1,85 +1 @@ from .fixtures import * # noqa, pylint: disable=wildcard-import - -DEFAULT_DVC_BIN = "dvc" -DEFAULT_DVC_GIT_REPO = "https://github.com/iterative/dvc" -DEFAULT_DVC_BENCH_GIT_REPO = "https://github.com/iterative/dvc-bench" -DEFAULT_PROJECT_GIT_REPO = "https://github.com/iterative/example-get-started" - - -def pytest_generate_tests(metafunc): - str_revs = metafunc.config.getoption("--dvc-revs") - revs = str_revs.split(",") if str_revs else [None] - if "dvc_rev" in metafunc.fixturenames: - metafunc.parametrize("dvc_rev", revs, scope="session") - - -class DVCBenchConfig: - def __init__(self): - self.size = "small" - self.dvc_bin = DEFAULT_DVC_BIN - self.dvc_revs = None - self.dvc_git_repo = DEFAULT_DVC_GIT_REPO - self.dvc_bench_git_repo = DEFAULT_DVC_BENCH_GIT_REPO - self.project_rev = None - self.project_git_repo = DEFAULT_PROJECT_GIT_REPO - - -def pytest_configure(config): - config.bench_config = DVCBenchConfig() - - config.bench_config.size = config.getoption("--size") - config.bench_config.dvc_bin = config.getoption("--dvc-bin") - config.bench_config.dvc_revs = config.getoption("--dvc-revs") - config.bench_config.dvc_git_repo = config.getoption("--dvc-git-repo") - config.bench_config.dvc_bench_git_repo = config.getoption("--dvc-bench-git-repo") - config.bench_config.project_rev = config.getoption("--project-rev") - config.bench_config.project_git_repo = config.getoption("--project-git-repo") - - -def pytest_addoption(parser): - parser.addoption( - "--size", - choices=["tiny", "small", "large", "mnist"], - default="small", - help="Size of the dataset/datafile to use in tests", - ) - - parser.addoption( - "--dvc-bin", - type=str, - default=DEFAULT_DVC_BIN, - help="Path to dvc binary", - ) - - parser.addoption( - "--dvc-revs", - type=str, - help=("Comma-separated list of DVC revisions to test (overrides `--dvc-bin`)"), - ) - - parser.addoption( - "--dvc-git-repo", - type=str, - default=DEFAULT_DVC_GIT_REPO, - help="Path or url to dvc git repo", - ) - - parser.addoption( - "--dvc-bench-git-repo", - type=str, - default=DEFAULT_DVC_BENCH_GIT_REPO, - help="Path or url to dvc-bench git repo", - ) - - parser.addoption( - "--project-rev", - type=str, - help="Project revision to test", - ) - - parser.addoption( - "--project-git-repo", - type=str, - default=DEFAULT_PROJECT_GIT_REPO, - help="Path or url to dvc project", - ) diff --git a/dvc/testing/benchmarks/fixtures.py b/dvc/testing/benchmarks/fixtures.py index 1767021c7..eb8c40acf 100644 --- a/dvc/testing/benchmarks/fixtures.py +++ b/dvc/testing/benchmarks/fixtures.py @@ -63,7 +63,7 @@ def dvc_bin( make_dvc_venv, dvc_git_repo, bench_config, - request, # noqa: ARG001 + request, ): if dvc_rev: venv = dvc_venvs.get(dvc_rev) @@ -82,6 +82,7 @@ def dvc_bin( return check_output([dvc_bin, *args], text=True) # nosec B603 _dvc_bin.version = parse_tuple(_dvc_bin("--version")) # type: ignore[attr-defined] + _skip_if_dvc_version_lt_required(request, _dvc_bin) return _dvc_bin @@ -93,8 +94,7 @@ def parse_tuple(version_string): return (parsed.major, parsed.minor, parsed.micro) -@pytest.fixture(autouse=True) -def skip_if_dvc_version_lt_required(request, dvc_bin): +def _skip_if_dvc_version_lt_required(request, dvc_bin): if marker := request.node.get_closest_marker("requires"): minversion = marker.kwargs.get("minversion") or first(marker.args) assert minversion, ( @@ -153,7 +153,7 @@ def _pull(repo, *args): @pytest.fixture -def make_dataset(request, bench_config, tmp_dir, dvc_bench_git_repo): # noqa: ARG001 +def make_dataset(request, bench_config, tmp_dir, dvc_bench_git_repo): def _make_dataset( dvcfile=False, files=True, cache=False, commit=False, remote=False ): diff --git a/dvc/testing/benchmarks/plugin.py b/dvc/testing/benchmarks/plugin.py new file mode 100644 index 000000000..633456d57 --- /dev/null +++ b/dvc/testing/benchmarks/plugin.py @@ -0,0 +1,83 @@ +DEFAULT_DVC_BIN = "dvc" +DEFAULT_DVC_GIT_REPO = "https://github.com/iterative/dvc" +DEFAULT_DVC_BENCH_GIT_REPO = "https://github.com/iterative/dvc-bench" +DEFAULT_PROJECT_GIT_REPO = "https://github.com/iterative/example-get-started" + + +def pytest_generate_tests(metafunc): + str_revs = metafunc.config.getoption("--dvc-revs") + revs = str_revs.split(",") if str_revs else [None] + if "dvc_rev" in metafunc.fixturenames: + metafunc.parametrize("dvc_rev", revs, scope="session") + + +class DVCBenchConfig: + def __init__(self): + self.size = "small" + self.dvc_bin = DEFAULT_DVC_BIN + self.dvc_revs = None + self.dvc_git_repo = DEFAULT_DVC_GIT_REPO + self.dvc_bench_git_repo = DEFAULT_DVC_BENCH_GIT_REPO + self.project_rev = None + self.project_git_repo = DEFAULT_PROJECT_GIT_REPO + + +def pytest_configure(config): + config.bench_config = DVCBenchConfig() + + config.bench_config.size = config.getoption("--size") + config.bench_config.dvc_bin = config.getoption("--dvc-bin") + config.bench_config.dvc_revs = config.getoption("--dvc-revs") + config.bench_config.dvc_git_repo = config.getoption("--dvc-git-repo") + config.bench_config.dvc_bench_git_repo = config.getoption("--dvc-bench-git-repo") + config.bench_config.project_rev = config.getoption("--project-rev") + config.bench_config.project_git_repo = config.getoption("--project-git-repo") + + +def pytest_addoption(parser): + parser.addoption( + "--size", + choices=["tiny", "small", "large", "mnist"], + default="small", + help="Size of the dataset/datafile to use in tests", + ) + + parser.addoption( + "--dvc-bin", + type=str, + default=DEFAULT_DVC_BIN, + help="Path to dvc binary", + ) + + parser.addoption( + "--dvc-revs", + type=str, + help=("Comma-separated list of DVC revisions to test (overrides `--dvc-bin`)"), + ) + + parser.addoption( + "--dvc-git-repo", + type=str, + default=DEFAULT_DVC_GIT_REPO, + help="Path or url to dvc git repo", + ) + + parser.addoption( + "--dvc-bench-git-repo", + type=str, + default=DEFAULT_DVC_BENCH_GIT_REPO, + help="Path or url to dvc-bench git repo (for loading benchmark datasets)", + ) + + parser.addoption( + "--project-rev", + type=str, + help="Project revision to test", + ) + + parser.addoption( + "--project-git-repo", + type=str, + default=DEFAULT_PROJECT_GIT_REPO, + help="Path or url to dvc project", + ) diff --git a/dvc/testing/conftest.py b/dvc/testing/conftest.py index 9f2e9712d..12c22cdf0 100644 --- a/dvc/testing/conftest.py +++ b/dvc/testing/conftest.py @@ -1,7 +1 @@ from .fixtures import * # noqa, pylint: disable=wildcard-import - - -def pytest_addoption(parser): - from .benchmarks.conftest import pytest_addoption as bench_addoption - - bench_addoption(parser) diff --git a/dvc/testing/plugin.py b/dvc/testing/plugin.py new file mode 100644 index 000000000..435f9140c --- /dev/null +++ b/dvc/testing/plugin.py @@ -0,0 +1,20 @@ +from .benchmarks.fixtures import * # noqa, pylint: disable=wildcard-import +from .fixtures import * # noqa, pylint: disable=wildcard-import + + +def pytest_generate_tests(metafunc): + from .benchmarks.plugin import pytest_generate_tests as bench_generate_tests + + bench_generate_tests(metafunc) + + +def pytest_addoption(parser): + from .benchmarks.plugin import pytest_addoption as bench_addoption + + bench_addoption(parser) + + +def pytest_configure(config): + from .benchmarks.plugin import pytest_configure as bench_configure + + bench_configure(config) diff --git a/pyproject.toml b/pyproject.toml index f6306551b..ec6c479df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,6 +93,7 @@ testing = [ "pytest-virtualenv", ] tests = [ + "dvc[testing]", "beautifulsoup4>=4.4", "dvc-ssh", "filelock", @@ -119,7 +120,7 @@ dvc = "dvc.cli:main" [project.entry-points."fsspec.specs"] dvc = "dvc.api:DVCFileSystem" [project.entry-points."pytest11"] -dvc-testing = "dvc.testing" +dvc-testing = "dvc.testing.plugin" [tool.setuptools] license-files = ["LICENSE"]