testing: import dvc-bench benchmarks

dvc/testing/benchmarks/cli/commands/test_add.py

@@ -0,0 +1,3 @@
+# pylint: disable=unused-argument
+def test_add(bench_dvc, tmp_dir, dvc, dataset):
+    bench_dvc("add", dataset)

dvc/testing/benchmarks/cli/commands/test_checkout.py

@@ -0,0 +1,39 @@
+# pylint: disable=unused-argument
+import os
+
+import pytest
+
+from dvc.fs import localfs
+from dvc_objects.fs import generic
+from dvc_objects.fs.utils import tmp_fname
+
+
+def _skip_unsupported_link(src, dest, link_type):
+    src_test_file = os.path.join(src, tmp_fname())
+    dest_test_file = os.path.join(dest, tmp_fname())
+    if not generic.test_links(
+        [link_type], localfs, src_test_file, localfs, dest_test_file
+    ):
+        pytest.skip(f"{link_type} not supported")
+
+
+def generate_test(*, link_type="copy"):
+    def _test_checkout_func(bench_dvc, tmp_dir, dvc, make_dataset):
+        dataset = make_dataset(dvcfile=True, cache=True, files=False)
+
+        _skip_unsupported_link((tmp_dir / ".dvc" / "cache"), tmp_dir, link_type)
+
+        with dvc.config.edit() as conf:
+            conf["cache"]["type"] = link_type
+
+        bench_dvc("checkout", dataset)
+        bench_dvc("checkout", name="noop")
+        (dataset / "new").write_text("new")
+        bench_dvc("checkout", "--force", name="update")
+
+    return _test_checkout_func
+
+
+test_checkout_copy = generate_test(link_type="copy")
+test_checkout_symlink = generate_test(link_type="symlink")
+test_checkout_hardlink = generate_test(link_type="hardlink")

dvc/testing/benchmarks/cli/commands/test_data_status.py

@@ -0,0 +1,49 @@
+# pylint: disable=unused-argument
+from shutil import rmtree
+
+import pytest
+
+pytestmark = pytest.mark.requires(minversion=(2, 15, 0), reason="new command")
+
+
+def test_data_status(
+    bench_dvc,
+    tmp_dir,
+    scm,
+    dvc,
+    make_dataset,
+):
+    args = ("data", "status")
+    dataset = make_dataset(cache=True, files=True, dvcfile=True, commit=False)
+    rmtree(dvc.tmp_dir)
+
+    bench_dvc(*args, name="new")
+    bench_dvc(*args, name="noop")
+
+    tmp_dir.scm_add(dataset.with_suffix(".dvc").name, commit="add dataset")
+
+    (dataset / "new").write_text("new")
+    bench_dvc(*args, name="changed")
+    bench_dvc(*args, name="changed-noop")
+
+
+def test_data_status_all_flags(bench_dvc, tmp_dir, scm, dvc, make_dataset):
+    args = (
+        "data",
+        "status",
+        "--granular",
+        "--unchanged",
+        "--untracked-files",
+        "--json",
+    )
+    dataset = make_dataset(cache=True, files=True, dvcfile=True, commit=False)
+    rmtree(dvc.tmp_dir)
+
+    bench_dvc(*args, name="new")
+    bench_dvc(*args, name="noop")
+
+    tmp_dir.scm_add(dataset.with_suffix(".dvc").name, commit="add dataset")
+
+    (dataset / "new").write_text("new")
+    bench_dvc(*args, name="changed")
+    bench_dvc(*args, name="changed-noop")

dvc/testing/benchmarks/cli/commands/test_diff.py

@@ -0,0 +1,9 @@
+# pylint: disable=unused-argument
+def test_diff(bench_dvc, tmp_dir, scm, dvc, make_dataset):
+    dataset = make_dataset(cache=True, files=True, dvcfile=True, commit=True)
+    bench_dvc("diff")
+    bench_dvc("diff", name="noop")
+
+    (dataset / "new").write_text("new")
+    bench_dvc("diff", name="changed")
+    bench_dvc("diff", name="changed-noop")

dvc/testing/benchmarks/cli/commands/test_exp_show.py

@@ -0,0 +1,14 @@
+# pylint: disable=unused-argument
+import pytest
+
+
+@pytest.mark.requires(minversion=(2, 10, 0))
+def test_exp_show(make_project, monkeypatch, bench_dvc, dvc_bin):
+    url = "https://github.com/efiop/lstm_seq2seq"
+    rev = "dvc-bench"
+    path = make_project(url, rev=rev)
+    monkeypatch.chdir(path)
+
+    dvc_bin("exp", "pull", "-A", "--no-cache", "origin")
+
+    bench_dvc("exp", "show", "-A", "--no-pager")

dvc/testing/benchmarks/cli/commands/test_gc.py

@@ -0,0 +1,4 @@
+# pylint: disable=unused-argument
+def test_gc(bench_dvc, tmp_dir, dvc, make_dataset):
+    make_dataset(files=False, cache=True, dvcfile=False)
+    bench_dvc("gc", "-f", "-w")

dvc/testing/benchmarks/cli/commands/test_get.py

@@ -0,0 +1,6 @@
+# pylint: disable=unused-argument
+def test_get(bench_dvc, tmp_dir, scm, dvc, make_dataset, remote):
+    dataset = make_dataset(
+        cache=False, files=False, dvcfile=True, commit=True, remote=True
+    )
+    bench_dvc("get", tmp_dir, dataset.name, "-o", "new")

dvc/testing/benchmarks/cli/commands/test_get_url.py

@@ -0,0 +1,6 @@
+# pylint: disable=unused-argument
+def test_get_url(bench_dvc, tmp_dir, scm, dvc, make_dataset):
+    dataset = make_dataset(
+        cache=False, files=True, dvcfile=False, commit=False, remote=False
+    )
+    bench_dvc("get-url", str(dataset), "new")

dvc/testing/benchmarks/cli/commands/test_help.py

@@ -0,0 +1,2 @@
+def test_help(bench_dvc):
+    bench_dvc("--help", rounds=100)

dvc/testing/benchmarks/cli/commands/test_import.py

@@ -0,0 +1,6 @@
+# pylint: disable=unused-argument
+def test_import(bench_dvc, tmp_dir, scm, dvc, make_dataset, remote):
+    dataset = make_dataset(
+        cache=False, files=False, dvcfile=True, commit=True, remote=True
+    )
+    bench_dvc("import", tmp_dir, dataset.name, "-o", "new")

dvc/testing/benchmarks/cli/commands/test_import_url.py

@@ -0,0 +1,6 @@
+# pylint: disable=unused-argument
+def test_import_url(bench_dvc, tmp_dir, scm, dvc, make_dataset):
+    dataset = make_dataset(
+        cache=False, files=True, dvcfile=False, commit=False, remote=False
+    )
+    bench_dvc("import-url", str(dataset), "new")

dvc/testing/benchmarks/cli/commands/test_init.py

@@ -0,0 +1,14 @@
+# pylint: disable=unused-argument
+import shutil
+
+
+def test_init(bench_dvc, tmp_dir, scm):
+    def _cleanup_dir():
+        for item in tmp_dir.iterdir():
+            if item.is_dir():
+                if item.name != ".git":
+                    shutil.rmtree(item)
+            else:
+                item.unlink()
+
+    bench_dvc("init", setup=_cleanup_dir, rounds=100, warmup_rounds=1)

dvc/testing/benchmarks/cli/commands/test_ls.py

@@ -0,0 +1,7 @@
+# pylint: disable=unused-argument
+def test_list(bench_dvc, tmp_dir, scm, dvc, make_dataset, remote):
+    make_dataset(cache=False, files=False, dvcfile=True, commit=True, remote=True)
+    bench_dvc("list", tmp_dir)
+    bench_dvc("list", tmp_dir, "--dvc-only", name="dvc-only")
+    bench_dvc("list", tmp_dir, "--recursive", name="recursive")
+    bench_dvc("list", tmp_dir, "dataset", name="shallow")

dvc/testing/benchmarks/cli/commands/test_plots.py

@@ -0,0 +1,18 @@
+import pytest
+
+from dvc.repo import Repo
+from tests.benchmarks.conftest import pull
+
+# pylint: disable=unused-argument
+
+
+@pytest.mark.requires(minversion=(2, 34, 0), reason="top-level plots not supported")
+def test_plots(project, bench_dvc):
+    with Repo() as dvc:
+        pull(dvc)
+
+    kwargs = {"rounds": 5, "iterations": 3, "warmup_rounds": 2}
+    bench_dvc("plots", "show", name="show", **kwargs)
+    bench_dvc("plots", "show", "--json", name="show-json", **kwargs)
+    bench_dvc("plots", "diff", "HEAD", name="diff", **kwargs)
+    bench_dvc("plots", "diff", "HEAD", "--json", name="diff-json", **kwargs)

dvc/testing/benchmarks/cli/commands/test_pull.py

@@ -0,0 +1,4 @@
+# pylint: disable=unused-argument
+def test_pull(bench_dvc, tmp_dir, dvc, make_dataset, remote):
+    make_dataset(cache=False, dvcfile=True, files=False, remote=True)
+    bench_dvc("pull")

dvc/testing/benchmarks/cli/commands/test_push.py

@@ -0,0 +1,4 @@
+# pylint: disable=unused-argument
+def test_push(bench_dvc, tmp_dir, dvc, make_dataset, remote):
+    make_dataset(cache=True, dvcfile=True, files=False)
+    bench_dvc("push")

dvc/testing/benchmarks/cli/commands/test_status.py

@@ -0,0 +1,9 @@
+# pylint: disable=unused-argument
+def test_status(bench_dvc, tmp_dir, dvc, make_dataset):
+    dataset = make_dataset(files=True, dvcfile=True, cache=True)
+    bench_dvc("status")
+    bench_dvc("status", name="noop")
+
+    (dataset / "new").write_text("new")
+    bench_dvc("status", name="changed")
+    bench_dvc("status", name="changed-noop")

dvc/testing/benchmarks/cli/commands/test_update.py

@@ -0,0 +1,9 @@
+# pylint: disable=unused-argument
+def test_update(bench_dvc, tmp_dir, dvc, make_dataset):
+    dataset = make_dataset(
+        cache=False, files=True, dvcfile=False, commit=False, remote=False
+    )
+    dvc.imp_url(str(dataset), "new")
+    (dataset / "new").write_text("new")
+    bench_dvc("update", "new.dvc")
+    bench_dvc("update", "new.dvc", name="noop")

dvc/testing/benchmarks/cli/stories/test_modify_data.py

@@ -0,0 +1,55 @@
+"""
+Tests for modifications to an existing dataset.
+"""
+import glob
+import os
+import random
+import shutil
+import sys
+
+import pytest
+
+# pylint: disable=unused-argument,unexpected-keyword-arg
+
+
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="requires 3.10 glob.glob")
+def test_partial_add(bench_dvc, tmp_dir, dvc, dataset, remote):
+    # Move some files to create a partial dataset
+    os.makedirs("partial-copy")
+    for f in glob.glob("*", root_dir=dataset, recursive=True):  # type: ignore[call-arg]
+        if random.random() > 0.5:  # noqa: S311 # nosec
+            shutil.move(dataset / f, tmp_dir / "partial-copy" / f)
+
+    # Add/push partial dataset
+    dvc.add(str(dataset))
+    dvc.push()
+
+    # Add more files to the dataset
+    shutil.copytree("partial-copy", dataset, dirs_exist_ok=True)
+
+    # Benchmark operations for adding files to a dataset
+    bench_dvc("add", dataset, name="partial-add")
+    bench_dvc("push", name="partial-add")
+    bench_dvc("gc", "-f", "-w", name="noop")
+    bench_dvc("gc", "-f", "-w", "-c", name="cloud-noop")
+
+
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="requires 3.10 glob.glob")
+def test_partial_remove(bench_dvc, tmp_dir, dvc, dataset, remote):
+    # Add/push full dataset
+    dvc.add(str(dataset))
+    dvc.push()
+
+    # Remove some files
+    for f in glob.glob("*", root_dir=dataset, recursive=True):  # type: ignore[call-arg]
+        if random.random() > 0.5:  # noqa: S311 # nosec
+            if os.path.isfile(dataset / f):
+                os.remove(dataset / f)
+            elif os.path.isdir(dataset / f):
+                shutil.rmtree(dataset / f)
+
+    # Benchmark operations for removing files from dataset
+    bench_dvc("add", dataset)
+    bench_dvc("push")
+    bench_dvc("gc", "-f", "-w")
+    bench_dvc("gc", "-f", "-w", "-c", name="cloud")

dvc/testing/benchmarks/cli/stories/use_cases/test_sharing.py

@@ -0,0 +1,19 @@
+import shutil
+
+# pylint: disable=unused-argument
+
+
+def test_sharing(bench_dvc, tmp_dir, dvc, dataset, remote):
+    bench_dvc("add", dataset)
+    bench_dvc("add", dataset, name="noop")
+
+    bench_dvc("push")
+    bench_dvc("push", name="noop")
+
+    shutil.rmtree(dataset)
+    shutil.rmtree(tmp_dir / ".dvc" / "cache")
+
+    bench_dvc("pull")
+    bench_dvc("pull", name="noop")
+
+    bench_dvc("checkout", name="noop")

dvc/testing/benchmarks/conftest.py

@@ -0,0 +1,207 @@
+import os
+import shutil
+from subprocess import check_output
+
+import pytest
+from dulwich.porcelain import clone
+from funcy import first
+from packaging import version
+from pytest_virtualenv import VirtualEnv
+
+
+def pytest_generate_tests(metafunc):
+    str_revs = metafunc.config.getoption("--dvc-revs")
+    revs = str_revs.split(",") if str_revs else [None]
+    if "dvc_rev" in metafunc.fixturenames:
+        metafunc.parametrize("dvc_rev", revs, scope="session")
+
+
+@pytest.fixture(scope="session")
+def make_dvc_venv(tmp_path_factory):
+    def _make_dvc_venv(name):
+        venv_dir = tmp_path_factory.mktemp(f"dvc-venv-{name}")
+        return VirtualEnv(workspace=venv_dir)
+
+    return _make_dvc_venv
+
+
+@pytest.fixture(scope="session")
+def dvc_venvs():
+    return {}
+
+
+@pytest.fixture(scope="session")
+def dvc_git_repo(tmp_path_factory, test_config):
+    url = test_config.dvc_git_repo
+
+    if os.path.isdir(url):
+        return url
+
+    tmp_path = os.fspath(tmp_path_factory.mktemp("dvc-git-repo"))
+    clone(url, tmp_path)
+
+    return tmp_path
+
+
+@pytest.fixture(scope="session")
+def dvc_bin(dvc_rev, dvc_venvs, make_dvc_venv, dvc_git_repo, test_config, request):
+    if dvc_rev:
+        venv = dvc_venvs.get(dvc_rev)
+        if not venv:
+            venv = make_dvc_venv(dvc_rev)
+            venv.run("pip install -U pip")
+            if dvc_rev in ["2.18.1", "2.11.0", "2.6.3"]:
+                venv.run("pip install fsspec==2022.11.0")
+            venv.run(f"pip install git+file://{dvc_git_repo}@{dvc_rev}")
+            dvc_venvs[dvc_rev] = venv
+        dvc_bin = venv.virtualenv / "bin" / "dvc"
+    else:
+        dvc_bin = test_config.dvc_bin
+
+    def _dvc_bin(*args):
+        return check_output([dvc_bin, *args], text=True)
+
+    _dvc_bin.version = parse_tuple(_dvc_bin("--version"))
+    return _dvc_bin
+
+
+def parse_tuple(version_string):
+    parsed = version.parse(version_string)
+    return (parsed.major, parsed.minor, parsed.micro)
+
+
+@pytest.fixture(autouse=True)
+def skip_if_dvc_version_lt_required(request, dvc_bin):
+    if marker := request.node.get_closest_marker("requires"):
+        minversion = marker.kwargs.get("minversion") or first(marker.args)
+        assert minversion, (
+            "'minversion' needs to be specified as"
+            " a positional or a keyword argument"
+        )
+        reason = marker.kwargs.get("reason", "")
+        if isinstance(minversion, str):
+            minversion = parse_tuple(minversion)
+        if dvc_bin.version < minversion:
+            version_repr = ".".join(map(str, minversion))
+            pytest.skip(f"requires dvc>={version_repr}: {reason}")
+
+
+@pytest.fixture(scope="function")
+def make_bench(request):
+    def _make_bench(name):
+        import pytest_benchmark.plugin
+
+        # hack from https://github.com/ionelmc/pytest-benchmark/issues/166
+        bench = pytest_benchmark.plugin.benchmark.__pytest_wrapped__.obj(request)
+
+        suffix = f"-{name}"
+
+        def add_suffix(_name):
+            start, sep, end = _name.partition("[")
+            return start + suffix + sep + end
+
+        bench.name = add_suffix(bench.name)
+        bench.fullname = add_suffix(bench.fullname)
+
+        return bench
+
+    return _make_bench
+
+
+@pytest.fixture(scope="function")
+def bench_dvc(dvc_bin, make_bench):
+    def _bench_dvc(*args, **kwargs):
+        name = kwargs.pop("name", None)
+        name = f"-{name}" if name else ""
+        bench = make_bench(args[0] + name)
+        return bench.pedantic(dvc_bin, args=args, **kwargs)
+
+    return _bench_dvc
+
+
+def pull(repo, *args):
+    from dvc.exceptions import CheckoutError, DownloadError
+
+    while True:
+        try:
+            return repo.pull(*args)
+        except (CheckoutError, DownloadError):
+            pass
+
+
+@pytest.fixture
+def make_dataset(request, test_config, tmp_dir, pytestconfig):
+    def _make_dataset(
+        dvcfile=False, files=True, cache=False, commit=False, remote=False
+    ):
+        from dvc.repo import Repo
+
+        path = tmp_dir / "dataset"
+        root = pytestconfig.rootpath
+        src = root / "data" / test_config.size / "dataset"
+        src_dvc = src.with_suffix(".dvc")
+
+        dvc = Repo(root)
+
+        pull(dvc, [str(src_dvc)])
+        if files:
+            shutil.copytree(src, path)
+        if dvcfile:
+            shutil.copy(src.with_suffix(".dvc"), path.with_suffix(".dvc"))
+        if cache:
+            shutil.copytree(root / ".dvc" / "cache", tmp_dir / ".dvc" / "cache")
+        if remote:
+            assert dvcfile
+            assert not cache
+            assert tmp_dir.dvc
+            # FIXME temporary hack, we should try to push from home repo
+            # directly to this remote instead
+            shutil.copytree(root / ".dvc" / "cache", tmp_dir / ".dvc" / "cache")
+            tmp_dir.dvc.push([str(path.with_suffix(".dvc").relative_to(tmp_dir))])
+            shutil.rmtree(tmp_dir / ".dvc" / "cache")
+        if commit:
+            assert dvcfile
+            assert tmp_dir.scm
+            tmp_dir.scm.add([str(path.with_suffix(".dvc").relative_to(tmp_dir))])
+            tmp_dir.scm.commit("add dataset")
+        return path
+
+    return _make_dataset
+
+
+@pytest.fixture
+def dataset(make_dataset):
+    return make_dataset(dvcfile=False, files=True, cache=False)
+
+
+@pytest.fixture
+def remote_dataset(test_config):
+    pytest.skip("fixme")
+
+
+@pytest.fixture
+def make_project(tmp_path_factory):
+    def _make_project(url, rev=None):
+        path = os.fspath(tmp_path_factory.mktemp("dvc-project"))
+
+        if rev:
+            rev = rev.encode("ascii")
+
+        clone(url, path, branch=rev)
+        return path
+
+    return _make_project
+
+
+@pytest.fixture
+def project(test_config, monkeypatch, make_project):
+    rev = test_config.project_rev
+    url = test_config.project_git_repo
+
+    if os.path.isdir(url):
+        path = url
+        assert not rev
+    else:
+        path = make_project(url, rev=rev)
+
+    monkeypatch.chdir(path)

pyproject.toml

@@ -287,6 +287,7 @@ ignore-names = ["M", "SCM"]
 [tool.ruff.per-file-ignores]
 "dvc/commands/**" = ["N806"]
 "dvc/testing/**" = ["ARG002"]
+"dvc/testing/benchmarks/**" = ["ARG001"]
 "scripts/**" = ["T201", "INP001", "TRY002"]
 
 [tool.ruff.pylint]