diff --git a/dvc/testing/benchmarks/conftest.py b/dvc/testing/benchmarks/conftest.py index c4eba4c7b..dd5ed276f 100644 --- a/dvc/testing/benchmarks/conftest.py +++ b/dvc/testing/benchmarks/conftest.py @@ -1,12 +1,9 @@ -import os -import shutil -from subprocess import check_output +from .fixtures import * # noqa, pylint: disable=wildcard-import -import pytest -from dulwich.porcelain import clone -from funcy import first -from packaging import version -from pytest_virtualenv import VirtualEnv +DEFAULT_DVC_BIN = "dvc" +DEFAULT_DVC_GIT_REPO = "https://github.com/iterative/dvc" +DEFAULT_DVC_BENCH_GIT_REPO = "https://github.com/iterative/dvc-bench" +DEFAULT_PROJECT_GIT_REPO = "https://github.com/iterative/example-get-started" def pytest_generate_tests(metafunc): @@ -16,192 +13,73 @@ def pytest_generate_tests(metafunc): metafunc.parametrize("dvc_rev", revs, scope="session") -@pytest.fixture(scope="session") -def make_dvc_venv(tmp_path_factory): - def _make_dvc_venv(name): - venv_dir = tmp_path_factory.mktemp(f"dvc-venv-{name}") - return VirtualEnv(workspace=venv_dir) - - return _make_dvc_venv +class DVCBenchConfig: + def __init__(self): + self.size = "small" + self.dvc_bin = DEFAULT_DVC_BIN + self.dvc_revs = None + self.dvc_git_repo = DEFAULT_DVC_GIT_REPO + self.dvc_bench_git_repo = DEFAULT_DVC_BENCH_GIT_REPO + self.project_rev = None + self.project_git_repo = DEFAULT_PROJECT_GIT_REPO -@pytest.fixture(scope="session") -def dvc_venvs(): - return {} +def pytest_configure(config): + config.bench_config = DVCBenchConfig() + + config.bench_config.size = config.getoption("--size") + config.bench_config.dvc_bin = config.getoption("--dvc-bin") + config.bench_config.dvc_revs = config.getoption("--dvc-revs") + config.bench_config.dvc_git_repo = config.getoption("--dvc-git-repo") + config.bench_config.dvc_bench_git_repo = config.getoption("--dvc-bench-git-repo") + config.bench_config.project_rev = config.getoption("--project-rev") + config.bench_config.project_git_repo = config.getoption("--project-git-repo") -@pytest.fixture(scope="session") -def dvc_git_repo(tmp_path_factory, test_config): - url = test_config.dvc_git_repo +def pytest_addoption(parser): + parser.addoption( + "--size", + choices=["tiny", "small", "large", "mnist"], + default="small", + help="Size of the dataset/datafile to use in tests", + ) - if os.path.isdir(url): - return url + parser.addoption( + "--dvc-bin", + type=str, + default=DEFAULT_DVC_BIN, + help="Path to dvc binary", + ) - tmp_path = os.fspath(tmp_path_factory.mktemp("dvc-git-repo")) - clone(url, tmp_path) + parser.addoption( + "--dvc-revs", + type=str, + help=("Comma-separated list of DVC revisions to test (overrides `--dvc-bin`)"), + ) - return tmp_path + parser.addoption( + "--dvc-git-repo", + type=str, + default=DEFAULT_DVC_GIT_REPO, + help="Path or url to dvc git repo", + ) + parser.addoption( + "--dvc-bench-git-repo", + type=str, + default=DEFAULT_DVC_BENCH_GIT_REPO, + help="Path or url to dvc-bench git repo", + ) -@pytest.fixture(scope="session") -def dvc_bin(dvc_rev, dvc_venvs, make_dvc_venv, dvc_git_repo, test_config, request): - if dvc_rev: - venv = dvc_venvs.get(dvc_rev) - if not venv: - venv = make_dvc_venv(dvc_rev) - venv.run("pip install -U pip") - if dvc_rev in ["2.18.1", "2.11.0", "2.6.3"]: - venv.run("pip install fsspec==2022.11.0") - venv.run(f"pip install git+file://{dvc_git_repo}@{dvc_rev}") - dvc_venvs[dvc_rev] = venv - dvc_bin = venv.virtualenv / "bin" / "dvc" - else: - dvc_bin = test_config.dvc_bin + parser.addoption( + "--project-rev", + type=str, + help="Project revision to test", + ) - def _dvc_bin(*args): - return check_output([dvc_bin, *args], text=True) - - _dvc_bin.version = parse_tuple(_dvc_bin("--version")) - return _dvc_bin - - -def parse_tuple(version_string): - parsed = version.parse(version_string) - return (parsed.major, parsed.minor, parsed.micro) - - -@pytest.fixture(autouse=True) -def skip_if_dvc_version_lt_required(request, dvc_bin): - if marker := request.node.get_closest_marker("requires"): - minversion = marker.kwargs.get("minversion") or first(marker.args) - assert minversion, ( - "'minversion' needs to be specified as" - " a positional or a keyword argument" - ) - reason = marker.kwargs.get("reason", "") - if isinstance(minversion, str): - minversion = parse_tuple(minversion) - if dvc_bin.version < minversion: - version_repr = ".".join(map(str, minversion)) - pytest.skip(f"requires dvc>={version_repr}: {reason}") - - -@pytest.fixture(scope="function") -def make_bench(request): - def _make_bench(name): - import pytest_benchmark.plugin - - # hack from https://github.com/ionelmc/pytest-benchmark/issues/166 - bench = pytest_benchmark.plugin.benchmark.__pytest_wrapped__.obj(request) - - suffix = f"-{name}" - - def add_suffix(_name): - start, sep, end = _name.partition("[") - return start + suffix + sep + end - - bench.name = add_suffix(bench.name) - bench.fullname = add_suffix(bench.fullname) - - return bench - - return _make_bench - - -@pytest.fixture(scope="function") -def bench_dvc(dvc_bin, make_bench): - def _bench_dvc(*args, **kwargs): - name = kwargs.pop("name", None) - name = f"-{name}" if name else "" - bench = make_bench(args[0] + name) - return bench.pedantic(dvc_bin, args=args, **kwargs) - - return _bench_dvc - - -def pull(repo, *args): - from dvc.exceptions import CheckoutError, DownloadError - - while True: - try: - return repo.pull(*args) - except (CheckoutError, DownloadError): - pass - - -@pytest.fixture -def make_dataset(request, test_config, tmp_dir, pytestconfig): - def _make_dataset( - dvcfile=False, files=True, cache=False, commit=False, remote=False - ): - from dvc.repo import Repo - - path = tmp_dir / "dataset" - root = pytestconfig.rootpath - src = root / "data" / test_config.size / "dataset" - src_dvc = src.with_suffix(".dvc") - - dvc = Repo(root) - - pull(dvc, [str(src_dvc)]) - if files: - shutil.copytree(src, path) - if dvcfile: - shutil.copy(src.with_suffix(".dvc"), path.with_suffix(".dvc")) - if cache: - shutil.copytree(root / ".dvc" / "cache", tmp_dir / ".dvc" / "cache") - if remote: - assert dvcfile - assert not cache - assert tmp_dir.dvc - # FIXME temporary hack, we should try to push from home repo - # directly to this remote instead - shutil.copytree(root / ".dvc" / "cache", tmp_dir / ".dvc" / "cache") - tmp_dir.dvc.push([str(path.with_suffix(".dvc").relative_to(tmp_dir))]) - shutil.rmtree(tmp_dir / ".dvc" / "cache") - if commit: - assert dvcfile - assert tmp_dir.scm - tmp_dir.scm.add([str(path.with_suffix(".dvc").relative_to(tmp_dir))]) - tmp_dir.scm.commit("add dataset") - return path - - return _make_dataset - - -@pytest.fixture -def dataset(make_dataset): - return make_dataset(dvcfile=False, files=True, cache=False) - - -@pytest.fixture -def remote_dataset(test_config): - pytest.skip("fixme") - - -@pytest.fixture -def make_project(tmp_path_factory): - def _make_project(url, rev=None): - path = os.fspath(tmp_path_factory.mktemp("dvc-project")) - - if rev: - rev = rev.encode("ascii") - - clone(url, path, branch=rev) - return path - - return _make_project - - -@pytest.fixture -def project(test_config, monkeypatch, make_project): - rev = test_config.project_rev - url = test_config.project_git_repo - - if os.path.isdir(url): - path = url - assert not rev - else: - path = make_project(url, rev=rev) - - monkeypatch.chdir(path) + parser.addoption( + "--project-git-repo", + type=str, + default=DEFAULT_PROJECT_GIT_REPO, + help="Path or url to dvc project", + ) diff --git a/dvc/testing/benchmarks/fixtures.py b/dvc/testing/benchmarks/fixtures.py new file mode 100644 index 000000000..1767021c7 --- /dev/null +++ b/dvc/testing/benchmarks/fixtures.py @@ -0,0 +1,230 @@ +import os +import shutil +from subprocess import check_output # nosec B404 + +import pytest +from dulwich.porcelain import clone +from funcy import first +from packaging import version +from pytest_virtualenv import VirtualEnv + +# pylint: disable=redefined-outer-name,unused-argument + + +@pytest.fixture(scope="session") +def bench_config(request): + return request.config.bench_config + + +@pytest.fixture(scope="session") +def make_dvc_venv(tmp_path_factory): + def _make_dvc_venv(name): + venv_dir = tmp_path_factory.mktemp(f"dvc-venv-{name}") + return VirtualEnv(workspace=venv_dir) + + return _make_dvc_venv + + +@pytest.fixture(scope="session") +def dvc_venvs(): + return {} + + +@pytest.fixture(scope="session") +def dvc_git_repo(tmp_path_factory, bench_config): + url = bench_config.dvc_git_repo + + if os.path.isdir(url): + return url + + tmp_path = tmp_path_factory.mktemp("dvc-git-repo") + clone(url, os.fspath(tmp_path)) + + return tmp_path + + +@pytest.fixture(scope="session") +def dvc_bench_git_repo(tmp_path_factory, bench_config): + url = bench_config.dvc_bench_git_repo + + if os.path.isdir(url): + return url + + tmp_path = tmp_path_factory.mktemp("dvc-bench-git-repo") + clone(url, os.fspath(tmp_path)) + + return tmp_path + + +@pytest.fixture(scope="session") +def dvc_bin( + dvc_rev, + dvc_venvs, + make_dvc_venv, + dvc_git_repo, + bench_config, + request, # noqa: ARG001 +): + if dvc_rev: + venv = dvc_venvs.get(dvc_rev) + if not venv: + venv = make_dvc_venv(dvc_rev) + venv.run("pip install -U pip") + if dvc_rev in ["2.18.1", "2.11.0", "2.6.3"]: + venv.run("pip install fsspec==2022.11.0") + venv.run(f"pip install git+file://{dvc_git_repo}@{dvc_rev}") + dvc_venvs[dvc_rev] = venv + dvc_bin = venv.virtualenv / "bin" / "dvc" + else: + dvc_bin = bench_config.dvc_bin + + def _dvc_bin(*args): + return check_output([dvc_bin, *args], text=True) # nosec B603 + + _dvc_bin.version = parse_tuple(_dvc_bin("--version")) # type: ignore[attr-defined] + return _dvc_bin + + +def parse_tuple(version_string): + from packaging.version import Version + + parsed = version.parse(version_string) + assert isinstance(parsed, Version) + return (parsed.major, parsed.minor, parsed.micro) + + +@pytest.fixture(autouse=True) +def skip_if_dvc_version_lt_required(request, dvc_bin): + if marker := request.node.get_closest_marker("requires"): + minversion = marker.kwargs.get("minversion") or first(marker.args) + assert minversion, ( + "'minversion' needs to be specified as" + " a positional or a keyword argument" + ) + reason = marker.kwargs.get("reason", "") + if isinstance(minversion, str): + minversion = parse_tuple(minversion) + if dvc_bin.version < minversion: + version_repr = ".".join(map(str, minversion)) + pytest.skip(f"requires dvc>={version_repr}: {reason}") + + +@pytest.fixture +def make_bench(request): + def _make_bench(name): + import pytest_benchmark.plugin + + # hack from https://github.com/ionelmc/pytest-benchmark/issues/166 + bench = pytest_benchmark.plugin.benchmark.__pytest_wrapped__.obj(request) + + suffix = f"-{name}" + + def add_suffix(_name): + start, sep, end = _name.partition("[") + return start + suffix + sep + end + + bench.name = add_suffix(bench.name) + bench.fullname = add_suffix(bench.fullname) + + return bench + + return _make_bench + + +@pytest.fixture +def bench_dvc(dvc_bin, make_bench): + def _bench_dvc(*args, **kwargs): + name = kwargs.pop("name", None) + name = f"-{name}" if name else "" + bench = make_bench(args[0] + name) + return bench.pedantic(dvc_bin, args=args, **kwargs) + + return _bench_dvc + + +def _pull(repo, *args): + from dvc.exceptions import CheckoutError, DownloadError + + while True: + try: + return repo.pull(*args) + except (CheckoutError, DownloadError): + pass + + +@pytest.fixture +def make_dataset(request, bench_config, tmp_dir, dvc_bench_git_repo): # noqa: ARG001 + def _make_dataset( + dvcfile=False, files=True, cache=False, commit=False, remote=False + ): + from dvc.repo import Repo + + path = tmp_dir / "dataset" + root = dvc_bench_git_repo + src = root / "data" / bench_config.size / "dataset" + src_dvc = src.with_suffix(".dvc") + + dvc = Repo(root) + + _pull(dvc, [str(src_dvc)]) + if files: + shutil.copytree(src, path) + if dvcfile: + shutil.copy(src.with_suffix(".dvc"), path.with_suffix(".dvc")) + if cache: + shutil.copytree(root / ".dvc" / "cache", tmp_dir / ".dvc" / "cache") + if remote: + assert dvcfile + assert not cache + assert tmp_dir.dvc + # FIXME temporary hack, we should try to push from home repo + # directly to this remote instead + shutil.copytree(root / ".dvc" / "cache", tmp_dir / ".dvc" / "cache") + tmp_dir.dvc.push([str(path.with_suffix(".dvc").relative_to(tmp_dir))]) + shutil.rmtree(tmp_dir / ".dvc" / "cache") + if commit: + assert dvcfile + assert tmp_dir.scm + tmp_dir.scm.add([str(path.with_suffix(".dvc").relative_to(tmp_dir))]) + tmp_dir.scm.commit("add dataset") + return path + + return _make_dataset + + +@pytest.fixture +def dataset(make_dataset): + return make_dataset(dvcfile=False, files=True, cache=False) + + +@pytest.fixture +def remote_dataset(): + pytest.skip("fixme") + + +@pytest.fixture +def make_project(tmp_path_factory): + def _make_project(url, rev=None): + path = os.fspath(tmp_path_factory.mktemp("dvc-project")) + + if rev: + rev = rev.encode("ascii") + + clone(url, path, branch=rev) + return path + + return _make_project + + +@pytest.fixture +def project(bench_config, monkeypatch, make_project): + rev = bench_config.project_rev + url = bench_config.project_git_repo + + if os.path.isdir(url): + path = url + assert not rev + else: + path = make_project(url, rev=rev) + + monkeypatch.chdir(path) diff --git a/dvc/testing/conftest.py b/dvc/testing/conftest.py index 12c22cdf0..9f2e9712d 100644 --- a/dvc/testing/conftest.py +++ b/dvc/testing/conftest.py @@ -1 +1,7 @@ from .fixtures import * # noqa, pylint: disable=wildcard-import + + +def pytest_addoption(parser): + from .benchmarks.conftest import pytest_addoption as bench_addoption + + bench_addoption(parser) diff --git a/pyproject.toml b/pyproject.toml index a6601c8a5..f6306551b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,7 +87,11 @@ s3 = ["dvc-s3==2.22.0"] ssh = ["dvc-ssh>=2.22.1,<3"] ssh_gssapi = ["dvc-ssh[gssapi]>=2.22.1,<3"] terraform = ["tpi[ssh]>=2.1"] -testing = ["pytest-test-utils"] +testing = [ + "pytest-test-utils", + "pytest-benchmark[histogram]", + "pytest-virtualenv", +] tests = [ "beautifulsoup4>=4.4", "dvc-ssh", @@ -114,6 +118,8 @@ Source = "https://github.com/iterative/dvc" dvc = "dvc.cli:main" [project.entry-points."fsspec.specs"] dvc = "dvc.api:DVCFileSystem" +[project.entry-points."pytest11"] +dvc-testing = "dvc.testing" [tool.setuptools] license-files = ["LICENSE"] @@ -223,7 +229,9 @@ module = [ "pygtrie.*", "pyinstrument", "pyparsing", + "pytest_benchmark.*", "pytest_docker.plugin", + "pytest_virtualenv.*", "ruamel.*", "ruamel.yaml.*", "shortuuid",