mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-01 23:08:43 +01:00
testing: register pytest plugin
This commit is contained in:
Peter Rowlands
committed by
Peter Rowlands (변기호)
Peter Rowlands (변기호)
parent
58cd1cbc09
commit
6c79f6bbfc
@@ -0,0 +1,19 @@
|
||||
DVC pytest plugin
|
||||
|
||||
dvc.testing.benchmarks
|
||||
======================
|
||||
|
||||
Benchmark test definitions are now part of ``dvc.testing``.
|
||||
For CLI usage and `bench.dvc.org <https://bench.dvc.org>`_ details see `dvc-bench <https://github.com/iterative/dvc-bench>`_.
|
||||
|
||||
``dvc.testing.benchmarks`` structure:
|
||||
|
||||
* cli: should be able to run these with any dvc (rpm, deb, pypi, snap, etc) (could be used in dvc-test repo too)
|
||||
|
||||
* commands: granular tests for individual commands. These should have a cached setup, so that we could use them during rapid development instead of our hand-written scripts. Every test could be run in a separate machine.
|
||||
* stories: multistage start-to-end benchmarks, useful for testing workflows (e.g. in documentation, see test_sharing inspired by `Storing and sharing <https://dvc.org/doc/start/data-management/data-versioning#storing-and-sharing>`_. Every full story could be run in a separate machine.
|
||||
|
||||
* api: for python api only.
|
||||
|
||||
* methods: granular tests for individual methods (e.g. ``api.open/read``). Same reasoning as in ``cli.commands``
|
||||
* stories: same as ``cli.stories`` but for our api. E.g. imagine using our api with pandas or smth like that.
|
||||
@@ -1,7 +1,9 @@
|
||||
import pytest
|
||||
|
||||
from dvc.repo import Repo
|
||||
from tests.benchmarks.conftest import pull
|
||||
from dvc.testing.benchmarks.fixtures import _pull
|
||||
|
||||
# pylint: disable=redefined-outer-name,unused-argument
|
||||
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
@@ -9,7 +11,7 @@ from tests.benchmarks.conftest import pull
|
||||
@pytest.mark.requires(minversion=(2, 34, 0), reason="top-level plots not supported")
|
||||
def test_plots(project, bench_dvc):
|
||||
with Repo() as dvc:
|
||||
pull(dvc)
|
||||
_pull(dvc)
|
||||
|
||||
kwargs = {"rounds": 5, "iterations": 3, "warmup_rounds": 2}
|
||||
bench_dvc("plots", "show", name="show", **kwargs)
|
||||
|
||||
@@ -1,85 +1 @@
|
||||
from .fixtures import * # noqa, pylint: disable=wildcard-import
|
||||
|
||||
DEFAULT_DVC_BIN = "dvc"
|
||||
DEFAULT_DVC_GIT_REPO = "https://github.com/iterative/dvc"
|
||||
DEFAULT_DVC_BENCH_GIT_REPO = "https://github.com/iterative/dvc-bench"
|
||||
DEFAULT_PROJECT_GIT_REPO = "https://github.com/iterative/example-get-started"
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
str_revs = metafunc.config.getoption("--dvc-revs")
|
||||
revs = str_revs.split(",") if str_revs else [None]
|
||||
if "dvc_rev" in metafunc.fixturenames:
|
||||
metafunc.parametrize("dvc_rev", revs, scope="session")
|
||||
|
||||
|
||||
class DVCBenchConfig:
|
||||
def __init__(self):
|
||||
self.size = "small"
|
||||
self.dvc_bin = DEFAULT_DVC_BIN
|
||||
self.dvc_revs = None
|
||||
self.dvc_git_repo = DEFAULT_DVC_GIT_REPO
|
||||
self.dvc_bench_git_repo = DEFAULT_DVC_BENCH_GIT_REPO
|
||||
self.project_rev = None
|
||||
self.project_git_repo = DEFAULT_PROJECT_GIT_REPO
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
config.bench_config = DVCBenchConfig()
|
||||
|
||||
config.bench_config.size = config.getoption("--size")
|
||||
config.bench_config.dvc_bin = config.getoption("--dvc-bin")
|
||||
config.bench_config.dvc_revs = config.getoption("--dvc-revs")
|
||||
config.bench_config.dvc_git_repo = config.getoption("--dvc-git-repo")
|
||||
config.bench_config.dvc_bench_git_repo = config.getoption("--dvc-bench-git-repo")
|
||||
config.bench_config.project_rev = config.getoption("--project-rev")
|
||||
config.bench_config.project_git_repo = config.getoption("--project-git-repo")
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
parser.addoption(
|
||||
"--size",
|
||||
choices=["tiny", "small", "large", "mnist"],
|
||||
default="small",
|
||||
help="Size of the dataset/datafile to use in tests",
|
||||
)
|
||||
|
||||
parser.addoption(
|
||||
"--dvc-bin",
|
||||
type=str,
|
||||
default=DEFAULT_DVC_BIN,
|
||||
help="Path to dvc binary",
|
||||
)
|
||||
|
||||
parser.addoption(
|
||||
"--dvc-revs",
|
||||
type=str,
|
||||
help=("Comma-separated list of DVC revisions to test (overrides `--dvc-bin`)"),
|
||||
)
|
||||
|
||||
parser.addoption(
|
||||
"--dvc-git-repo",
|
||||
type=str,
|
||||
default=DEFAULT_DVC_GIT_REPO,
|
||||
help="Path or url to dvc git repo",
|
||||
)
|
||||
|
||||
parser.addoption(
|
||||
"--dvc-bench-git-repo",
|
||||
type=str,
|
||||
default=DEFAULT_DVC_BENCH_GIT_REPO,
|
||||
help="Path or url to dvc-bench git repo",
|
||||
)
|
||||
|
||||
parser.addoption(
|
||||
"--project-rev",
|
||||
type=str,
|
||||
help="Project revision to test",
|
||||
)
|
||||
|
||||
parser.addoption(
|
||||
"--project-git-repo",
|
||||
type=str,
|
||||
default=DEFAULT_PROJECT_GIT_REPO,
|
||||
help="Path or url to dvc project",
|
||||
)
|
||||
|
||||
@@ -63,7 +63,7 @@ def dvc_bin(
|
||||
make_dvc_venv,
|
||||
dvc_git_repo,
|
||||
bench_config,
|
||||
request, # noqa: ARG001
|
||||
request,
|
||||
):
|
||||
if dvc_rev:
|
||||
venv = dvc_venvs.get(dvc_rev)
|
||||
@@ -82,6 +82,7 @@ def dvc_bin(
|
||||
return check_output([dvc_bin, *args], text=True) # nosec B603
|
||||
|
||||
_dvc_bin.version = parse_tuple(_dvc_bin("--version")) # type: ignore[attr-defined]
|
||||
_skip_if_dvc_version_lt_required(request, _dvc_bin)
|
||||
return _dvc_bin
|
||||
|
||||
|
||||
@@ -93,8 +94,7 @@ def parse_tuple(version_string):
|
||||
return (parsed.major, parsed.minor, parsed.micro)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def skip_if_dvc_version_lt_required(request, dvc_bin):
|
||||
def _skip_if_dvc_version_lt_required(request, dvc_bin):
|
||||
if marker := request.node.get_closest_marker("requires"):
|
||||
minversion = marker.kwargs.get("minversion") or first(marker.args)
|
||||
assert minversion, (
|
||||
@@ -153,7 +153,7 @@ def _pull(repo, *args):
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def make_dataset(request, bench_config, tmp_dir, dvc_bench_git_repo): # noqa: ARG001
|
||||
def make_dataset(request, bench_config, tmp_dir, dvc_bench_git_repo):
|
||||
def _make_dataset(
|
||||
dvcfile=False, files=True, cache=False, commit=False, remote=False
|
||||
):
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
DEFAULT_DVC_BIN = "dvc"
|
||||
DEFAULT_DVC_GIT_REPO = "https://github.com/iterative/dvc"
|
||||
DEFAULT_DVC_BENCH_GIT_REPO = "https://github.com/iterative/dvc-bench"
|
||||
DEFAULT_PROJECT_GIT_REPO = "https://github.com/iterative/example-get-started"
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
str_revs = metafunc.config.getoption("--dvc-revs")
|
||||
revs = str_revs.split(",") if str_revs else [None]
|
||||
if "dvc_rev" in metafunc.fixturenames:
|
||||
metafunc.parametrize("dvc_rev", revs, scope="session")
|
||||
|
||||
|
||||
class DVCBenchConfig:
|
||||
def __init__(self):
|
||||
self.size = "small"
|
||||
self.dvc_bin = DEFAULT_DVC_BIN
|
||||
self.dvc_revs = None
|
||||
self.dvc_git_repo = DEFAULT_DVC_GIT_REPO
|
||||
self.dvc_bench_git_repo = DEFAULT_DVC_BENCH_GIT_REPO
|
||||
self.project_rev = None
|
||||
self.project_git_repo = DEFAULT_PROJECT_GIT_REPO
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
config.bench_config = DVCBenchConfig()
|
||||
|
||||
config.bench_config.size = config.getoption("--size")
|
||||
config.bench_config.dvc_bin = config.getoption("--dvc-bin")
|
||||
config.bench_config.dvc_revs = config.getoption("--dvc-revs")
|
||||
config.bench_config.dvc_git_repo = config.getoption("--dvc-git-repo")
|
||||
config.bench_config.dvc_bench_git_repo = config.getoption("--dvc-bench-git-repo")
|
||||
config.bench_config.project_rev = config.getoption("--project-rev")
|
||||
config.bench_config.project_git_repo = config.getoption("--project-git-repo")
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
parser.addoption(
|
||||
"--size",
|
||||
choices=["tiny", "small", "large", "mnist"],
|
||||
default="small",
|
||||
help="Size of the dataset/datafile to use in tests",
|
||||
)
|
||||
|
||||
parser.addoption(
|
||||
"--dvc-bin",
|
||||
type=str,
|
||||
default=DEFAULT_DVC_BIN,
|
||||
help="Path to dvc binary",
|
||||
)
|
||||
|
||||
parser.addoption(
|
||||
"--dvc-revs",
|
||||
type=str,
|
||||
help=("Comma-separated list of DVC revisions to test (overrides `--dvc-bin`)"),
|
||||
)
|
||||
|
||||
parser.addoption(
|
||||
"--dvc-git-repo",
|
||||
type=str,
|
||||
default=DEFAULT_DVC_GIT_REPO,
|
||||
help="Path or url to dvc git repo",
|
||||
)
|
||||
|
||||
parser.addoption(
|
||||
"--dvc-bench-git-repo",
|
||||
type=str,
|
||||
default=DEFAULT_DVC_BENCH_GIT_REPO,
|
||||
help="Path or url to dvc-bench git repo (for loading benchmark datasets)",
|
||||
)
|
||||
|
||||
parser.addoption(
|
||||
"--project-rev",
|
||||
type=str,
|
||||
help="Project revision to test",
|
||||
)
|
||||
|
||||
parser.addoption(
|
||||
"--project-git-repo",
|
||||
type=str,
|
||||
default=DEFAULT_PROJECT_GIT_REPO,
|
||||
help="Path or url to dvc project",
|
||||
)
|
||||
@@ -1,7 +1 @@
|
||||
from .fixtures import * # noqa, pylint: disable=wildcard-import
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
from .benchmarks.conftest import pytest_addoption as bench_addoption
|
||||
|
||||
bench_addoption(parser)
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
from .benchmarks.fixtures import * # noqa, pylint: disable=wildcard-import
|
||||
from .fixtures import * # noqa, pylint: disable=wildcard-import
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
from .benchmarks.plugin import pytest_generate_tests as bench_generate_tests
|
||||
|
||||
bench_generate_tests(metafunc)
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
from .benchmarks.plugin import pytest_addoption as bench_addoption
|
||||
|
||||
bench_addoption(parser)
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
from .benchmarks.plugin import pytest_configure as bench_configure
|
||||
|
||||
bench_configure(config)
|
||||
+2
-1
@@ -93,6 +93,7 @@ testing = [
|
||||
"pytest-virtualenv",
|
||||
]
|
||||
tests = [
|
||||
"dvc[testing]",
|
||||
"beautifulsoup4>=4.4",
|
||||
"dvc-ssh",
|
||||
"filelock",
|
||||
@@ -119,7 +120,7 @@ dvc = "dvc.cli:main"
|
||||
[project.entry-points."fsspec.specs"]
|
||||
dvc = "dvc.api:DVCFileSystem"
|
||||
[project.entry-points."pytest11"]
|
||||
dvc-testing = "dvc.testing"
|
||||
dvc-testing = "dvc.testing.plugin"
|
||||
|
||||
[tool.setuptools]
|
||||
license-files = ["LICENSE"]
|
||||
|
||||
Reference in New Issue
Block a user