testing: register pytest plugin

dvc/testing/README.rst

@@ -0,0 +1,19 @@
+DVC pytest plugin
+
+dvc.testing.benchmarks
+======================
+
+Benchmark test definitions are now part of ``dvc.testing``.
+For CLI usage and `bench.dvc.org <https://bench.dvc.org>`_ details see `dvc-bench <https://github.com/iterative/dvc-bench>`_.
+
+``dvc.testing.benchmarks`` structure:
+
+* cli: should be able to run these with any dvc (rpm, deb, pypi, snap, etc) (could be used in dvc-test repo too)
+
+  * commands: granular tests for individual commands. These should have a cached setup, so that we could use them during rapid development instead of our hand-written scripts. Every test could be run in a separate machine.
+  * stories: multistage start-to-end benchmarks, useful for testing workflows (e.g. in documentation, see test_sharing inspired by `Storing and sharing <https://dvc.org/doc/start/data-management/data-versioning#storing-and-sharing>`_. Every full story could be run in a separate machine.
+
+* api: for python api only.
+
+  * methods: granular tests for individual methods (e.g. ``api.open/read``). Same reasoning as in ``cli.commands``
+  * stories: same as ``cli.stories`` but for our api. E.g. imagine using our api with pandas or smth like that.

dvc/testing/benchmarks/cli/commands/test_plots.py

@@ -1,7 +1,9 @@
 import pytest
 
 from dvc.repo import Repo
-from tests.benchmarks.conftest import pull
+from dvc.testing.benchmarks.fixtures import _pull
+
+# pylint: disable=redefined-outer-name,unused-argument
 
 # pylint: disable=unused-argument
 
@@ -9,7 +11,7 @@ from tests.benchmarks.conftest import pull
 @pytest.mark.requires(minversion=(2, 34, 0), reason="top-level plots not supported")
 def test_plots(project, bench_dvc):
     with Repo() as dvc:
-        pull(dvc)
+        _pull(dvc)
 
     kwargs = {"rounds": 5, "iterations": 3, "warmup_rounds": 2}
     bench_dvc("plots", "show", name="show", **kwargs)

dvc/testing/benchmarks/conftest.py

@@ -1,85 +1 @@
 from .fixtures import *  # noqa, pylint: disable=wildcard-import
-
-DEFAULT_DVC_BIN = "dvc"
-DEFAULT_DVC_GIT_REPO = "https://github.com/iterative/dvc"
-DEFAULT_DVC_BENCH_GIT_REPO = "https://github.com/iterative/dvc-bench"
-DEFAULT_PROJECT_GIT_REPO = "https://github.com/iterative/example-get-started"
-
-
-def pytest_generate_tests(metafunc):
-    str_revs = metafunc.config.getoption("--dvc-revs")
-    revs = str_revs.split(",") if str_revs else [None]
-    if "dvc_rev" in metafunc.fixturenames:
-        metafunc.parametrize("dvc_rev", revs, scope="session")
-
-
-class DVCBenchConfig:
-    def __init__(self):
-        self.size = "small"
-        self.dvc_bin = DEFAULT_DVC_BIN
-        self.dvc_revs = None
-        self.dvc_git_repo = DEFAULT_DVC_GIT_REPO
-        self.dvc_bench_git_repo = DEFAULT_DVC_BENCH_GIT_REPO
-        self.project_rev = None
-        self.project_git_repo = DEFAULT_PROJECT_GIT_REPO
-
-
-def pytest_configure(config):
-    config.bench_config = DVCBenchConfig()
-
-    config.bench_config.size = config.getoption("--size")
-    config.bench_config.dvc_bin = config.getoption("--dvc-bin")
-    config.bench_config.dvc_revs = config.getoption("--dvc-revs")
-    config.bench_config.dvc_git_repo = config.getoption("--dvc-git-repo")
-    config.bench_config.dvc_bench_git_repo = config.getoption("--dvc-bench-git-repo")
-    config.bench_config.project_rev = config.getoption("--project-rev")
-    config.bench_config.project_git_repo = config.getoption("--project-git-repo")
-
-
-def pytest_addoption(parser):
-    parser.addoption(
-        "--size",
-        choices=["tiny", "small", "large", "mnist"],
-        default="small",
-        help="Size of the dataset/datafile to use in tests",
-    )
-
-    parser.addoption(
-        "--dvc-bin",
-        type=str,
-        default=DEFAULT_DVC_BIN,
-        help="Path to dvc binary",
-    )
-
-    parser.addoption(
-        "--dvc-revs",
-        type=str,
-        help=("Comma-separated list of DVC revisions to test (overrides `--dvc-bin`)"),
-    )
-
-    parser.addoption(
-        "--dvc-git-repo",
-        type=str,
-        default=DEFAULT_DVC_GIT_REPO,
-        help="Path or url to dvc git repo",
-    )
-
-    parser.addoption(
-        "--dvc-bench-git-repo",
-        type=str,
-        default=DEFAULT_DVC_BENCH_GIT_REPO,
-        help="Path or url to dvc-bench git repo",
-    )
-
-    parser.addoption(
-        "--project-rev",
-        type=str,
-        help="Project revision to test",
-    )
-
-    parser.addoption(
-        "--project-git-repo",
-        type=str,
-        default=DEFAULT_PROJECT_GIT_REPO,
-        help="Path or url to dvc project",
-    )

dvc/testing/benchmarks/fixtures.py

@@ -63,7 +63,7 @@ def dvc_bin(
     make_dvc_venv,
     dvc_git_repo,
     bench_config,
-    request,  # noqa: ARG001
+    request,
 ):
     if dvc_rev:
         venv = dvc_venvs.get(dvc_rev)
@@ -82,6 +82,7 @@ def dvc_bin(
         return check_output([dvc_bin, *args], text=True)  # nosec B603
 
     _dvc_bin.version = parse_tuple(_dvc_bin("--version"))  # type: ignore[attr-defined]
+    _skip_if_dvc_version_lt_required(request, _dvc_bin)
     return _dvc_bin
 
 
@@ -93,8 +94,7 @@ def parse_tuple(version_string):
     return (parsed.major, parsed.minor, parsed.micro)
 
 
-@pytest.fixture(autouse=True)
+def _skip_if_dvc_version_lt_required(request, dvc_bin):
-def skip_if_dvc_version_lt_required(request, dvc_bin):
     if marker := request.node.get_closest_marker("requires"):
         minversion = marker.kwargs.get("minversion") or first(marker.args)
         assert minversion, (
@@ -153,7 +153,7 @@ def _pull(repo, *args):
 
 
 @pytest.fixture
-def make_dataset(request, bench_config, tmp_dir, dvc_bench_git_repo):  # noqa: ARG001
+def make_dataset(request, bench_config, tmp_dir, dvc_bench_git_repo):
     def _make_dataset(
         dvcfile=False, files=True, cache=False, commit=False, remote=False
     ):

dvc/testing/benchmarks/plugin.py

@@ -0,0 +1,83 @@
+DEFAULT_DVC_BIN = "dvc"
+DEFAULT_DVC_GIT_REPO = "https://github.com/iterative/dvc"
+DEFAULT_DVC_BENCH_GIT_REPO = "https://github.com/iterative/dvc-bench"
+DEFAULT_PROJECT_GIT_REPO = "https://github.com/iterative/example-get-started"
+
+
+def pytest_generate_tests(metafunc):
+    str_revs = metafunc.config.getoption("--dvc-revs")
+    revs = str_revs.split(",") if str_revs else [None]
+    if "dvc_rev" in metafunc.fixturenames:
+        metafunc.parametrize("dvc_rev", revs, scope="session")
+
+
+class DVCBenchConfig:
+    def __init__(self):
+        self.size = "small"
+        self.dvc_bin = DEFAULT_DVC_BIN
+        self.dvc_revs = None
+        self.dvc_git_repo = DEFAULT_DVC_GIT_REPO
+        self.dvc_bench_git_repo = DEFAULT_DVC_BENCH_GIT_REPO
+        self.project_rev = None
+        self.project_git_repo = DEFAULT_PROJECT_GIT_REPO
+
+
+def pytest_configure(config):
+    config.bench_config = DVCBenchConfig()
+
+    config.bench_config.size = config.getoption("--size")
+    config.bench_config.dvc_bin = config.getoption("--dvc-bin")
+    config.bench_config.dvc_revs = config.getoption("--dvc-revs")
+    config.bench_config.dvc_git_repo = config.getoption("--dvc-git-repo")
+    config.bench_config.dvc_bench_git_repo = config.getoption("--dvc-bench-git-repo")
+    config.bench_config.project_rev = config.getoption("--project-rev")
+    config.bench_config.project_git_repo = config.getoption("--project-git-repo")
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--size",
+        choices=["tiny", "small", "large", "mnist"],
+        default="small",
+        help="Size of the dataset/datafile to use in tests",
+    )
+
+    parser.addoption(
+        "--dvc-bin",
+        type=str,
+        default=DEFAULT_DVC_BIN,
+        help="Path to dvc binary",
+    )
+
+    parser.addoption(
+        "--dvc-revs",
+        type=str,
+        help=("Comma-separated list of DVC revisions to test (overrides `--dvc-bin`)"),
+    )
+
+    parser.addoption(
+        "--dvc-git-repo",
+        type=str,
+        default=DEFAULT_DVC_GIT_REPO,
+        help="Path or url to dvc git repo",
+    )
+
+    parser.addoption(
+        "--dvc-bench-git-repo",
+        type=str,
+        default=DEFAULT_DVC_BENCH_GIT_REPO,
+        help="Path or url to dvc-bench git repo (for loading benchmark datasets)",
+    )
+
+    parser.addoption(
+        "--project-rev",
+        type=str,
+        help="Project revision to test",
+    )
+
+    parser.addoption(
+        "--project-git-repo",
+        type=str,
+        default=DEFAULT_PROJECT_GIT_REPO,
+        help="Path or url to dvc project",
+    )

dvc/testing/conftest.py

@@ -1,7 +1 @@
 from .fixtures import *  # noqa, pylint: disable=wildcard-import
-
-
-def pytest_addoption(parser):
-    from .benchmarks.conftest import pytest_addoption as bench_addoption
-
-    bench_addoption(parser)

dvc/testing/plugin.py

@@ -0,0 +1,20 @@
+from .benchmarks.fixtures import *  # noqa, pylint: disable=wildcard-import
+from .fixtures import *  # noqa, pylint: disable=wildcard-import
+
+
+def pytest_generate_tests(metafunc):
+    from .benchmarks.plugin import pytest_generate_tests as bench_generate_tests
+
+    bench_generate_tests(metafunc)
+
+
+def pytest_addoption(parser):
+    from .benchmarks.plugin import pytest_addoption as bench_addoption
+
+    bench_addoption(parser)
+
+
+def pytest_configure(config):
+    from .benchmarks.plugin import pytest_configure as bench_configure
+
+    bench_configure(config)

pyproject.toml

@@ -93,6 +93,7 @@ testing = [
   "pytest-virtualenv",
 ]
 tests = [
+    "dvc[testing]",
     "beautifulsoup4>=4.4",
     "dvc-ssh",
     "filelock",
@@ -119,7 +120,7 @@ dvc = "dvc.cli:main"
 [project.entry-points."fsspec.specs"]
 dvc = "dvc.api:DVCFileSystem"
 [project.entry-points."pytest11"]
-dvc-testing = "dvc.testing"
+dvc-testing = "dvc.testing.plugin"
 
 [tool.setuptools]
 license-files = ["LICENSE"]