diff --git a/dvc/testing/benchmarks/__init__.py b/dvc/testing/benchmarks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dvc/testing/benchmarks/api/__init__.py b/dvc/testing/benchmarks/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dvc/testing/benchmarks/cli/__init__.py b/dvc/testing/benchmarks/cli/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dvc/testing/benchmarks/cli/commands/__init__.py b/dvc/testing/benchmarks/cli/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dvc/testing/benchmarks/cli/commands/test_add.py b/dvc/testing/benchmarks/cli/commands/test_add.py new file mode 100644 index 000000000..dc46c6b19 --- /dev/null +++ b/dvc/testing/benchmarks/cli/commands/test_add.py @@ -0,0 +1,3 @@ +# pylint: disable=unused-argument +def test_add(bench_dvc, tmp_dir, dvc, dataset): + bench_dvc("add", dataset) diff --git a/dvc/testing/benchmarks/cli/commands/test_checkout.py b/dvc/testing/benchmarks/cli/commands/test_checkout.py new file mode 100644 index 000000000..9c3af95a6 --- /dev/null +++ b/dvc/testing/benchmarks/cli/commands/test_checkout.py @@ -0,0 +1,39 @@ +# pylint: disable=unused-argument +import os + +import pytest + +from dvc.fs import localfs +from dvc_objects.fs import generic +from dvc_objects.fs.utils import tmp_fname + + +def _skip_unsupported_link(src, dest, link_type): + src_test_file = os.path.join(src, tmp_fname()) + dest_test_file = os.path.join(dest, tmp_fname()) + if not generic.test_links( + [link_type], localfs, src_test_file, localfs, dest_test_file + ): + pytest.skip(f"{link_type} not supported") + + +def generate_test(*, link_type="copy"): + def _test_checkout_func(bench_dvc, tmp_dir, dvc, make_dataset): + dataset = make_dataset(dvcfile=True, cache=True, files=False) + + _skip_unsupported_link((tmp_dir / ".dvc" / "cache"), tmp_dir, link_type) + + with dvc.config.edit() as conf: + conf["cache"]["type"] = link_type + + bench_dvc("checkout", dataset) + bench_dvc("checkout", name="noop") + (dataset / "new").write_text("new") + bench_dvc("checkout", "--force", name="update") + + return _test_checkout_func + + +test_checkout_copy = generate_test(link_type="copy") +test_checkout_symlink = generate_test(link_type="symlink") +test_checkout_hardlink = generate_test(link_type="hardlink") diff --git a/dvc/testing/benchmarks/cli/commands/test_data_status.py b/dvc/testing/benchmarks/cli/commands/test_data_status.py new file mode 100644 index 000000000..465584d4c --- /dev/null +++ b/dvc/testing/benchmarks/cli/commands/test_data_status.py @@ -0,0 +1,49 @@ +# pylint: disable=unused-argument +from shutil import rmtree + +import pytest + +pytestmark = pytest.mark.requires(minversion=(2, 15, 0), reason="new command") + + +def test_data_status( + bench_dvc, + tmp_dir, + scm, + dvc, + make_dataset, +): + args = ("data", "status") + dataset = make_dataset(cache=True, files=True, dvcfile=True, commit=False) + rmtree(dvc.tmp_dir) + + bench_dvc(*args, name="new") + bench_dvc(*args, name="noop") + + tmp_dir.scm_add(dataset.with_suffix(".dvc").name, commit="add dataset") + + (dataset / "new").write_text("new") + bench_dvc(*args, name="changed") + bench_dvc(*args, name="changed-noop") + + +def test_data_status_all_flags(bench_dvc, tmp_dir, scm, dvc, make_dataset): + args = ( + "data", + "status", + "--granular", + "--unchanged", + "--untracked-files", + "--json", + ) + dataset = make_dataset(cache=True, files=True, dvcfile=True, commit=False) + rmtree(dvc.tmp_dir) + + bench_dvc(*args, name="new") + bench_dvc(*args, name="noop") + + tmp_dir.scm_add(dataset.with_suffix(".dvc").name, commit="add dataset") + + (dataset / "new").write_text("new") + bench_dvc(*args, name="changed") + bench_dvc(*args, name="changed-noop") diff --git a/dvc/testing/benchmarks/cli/commands/test_diff.py b/dvc/testing/benchmarks/cli/commands/test_diff.py new file mode 100644 index 000000000..f3218ee52 --- /dev/null +++ b/dvc/testing/benchmarks/cli/commands/test_diff.py @@ -0,0 +1,9 @@ +# pylint: disable=unused-argument +def test_diff(bench_dvc, tmp_dir, scm, dvc, make_dataset): + dataset = make_dataset(cache=True, files=True, dvcfile=True, commit=True) + bench_dvc("diff") + bench_dvc("diff", name="noop") + + (dataset / "new").write_text("new") + bench_dvc("diff", name="changed") + bench_dvc("diff", name="changed-noop") diff --git a/dvc/testing/benchmarks/cli/commands/test_exp_show.py b/dvc/testing/benchmarks/cli/commands/test_exp_show.py new file mode 100644 index 000000000..bad5e8d2e --- /dev/null +++ b/dvc/testing/benchmarks/cli/commands/test_exp_show.py @@ -0,0 +1,14 @@ +# pylint: disable=unused-argument +import pytest + + +@pytest.mark.requires(minversion=(2, 10, 0)) +def test_exp_show(make_project, monkeypatch, bench_dvc, dvc_bin): + url = "https://github.com/efiop/lstm_seq2seq" + rev = "dvc-bench" + path = make_project(url, rev=rev) + monkeypatch.chdir(path) + + dvc_bin("exp", "pull", "-A", "--no-cache", "origin") + + bench_dvc("exp", "show", "-A", "--no-pager") diff --git a/dvc/testing/benchmarks/cli/commands/test_gc.py b/dvc/testing/benchmarks/cli/commands/test_gc.py new file mode 100644 index 000000000..47599d0a0 --- /dev/null +++ b/dvc/testing/benchmarks/cli/commands/test_gc.py @@ -0,0 +1,4 @@ +# pylint: disable=unused-argument +def test_gc(bench_dvc, tmp_dir, dvc, make_dataset): + make_dataset(files=False, cache=True, dvcfile=False) + bench_dvc("gc", "-f", "-w") diff --git a/dvc/testing/benchmarks/cli/commands/test_get.py b/dvc/testing/benchmarks/cli/commands/test_get.py new file mode 100644 index 000000000..3ecd9cfaf --- /dev/null +++ b/dvc/testing/benchmarks/cli/commands/test_get.py @@ -0,0 +1,6 @@ +# pylint: disable=unused-argument +def test_get(bench_dvc, tmp_dir, scm, dvc, make_dataset, remote): + dataset = make_dataset( + cache=False, files=False, dvcfile=True, commit=True, remote=True + ) + bench_dvc("get", tmp_dir, dataset.name, "-o", "new") diff --git a/dvc/testing/benchmarks/cli/commands/test_get_url.py b/dvc/testing/benchmarks/cli/commands/test_get_url.py new file mode 100644 index 000000000..91cc4e6ea --- /dev/null +++ b/dvc/testing/benchmarks/cli/commands/test_get_url.py @@ -0,0 +1,6 @@ +# pylint: disable=unused-argument +def test_get_url(bench_dvc, tmp_dir, scm, dvc, make_dataset): + dataset = make_dataset( + cache=False, files=True, dvcfile=False, commit=False, remote=False + ) + bench_dvc("get-url", str(dataset), "new") diff --git a/dvc/testing/benchmarks/cli/commands/test_help.py b/dvc/testing/benchmarks/cli/commands/test_help.py new file mode 100644 index 000000000..b8955c63b --- /dev/null +++ b/dvc/testing/benchmarks/cli/commands/test_help.py @@ -0,0 +1,2 @@ +def test_help(bench_dvc): + bench_dvc("--help", rounds=100) diff --git a/dvc/testing/benchmarks/cli/commands/test_import.py b/dvc/testing/benchmarks/cli/commands/test_import.py new file mode 100644 index 000000000..66d3a43f1 --- /dev/null +++ b/dvc/testing/benchmarks/cli/commands/test_import.py @@ -0,0 +1,6 @@ +# pylint: disable=unused-argument +def test_import(bench_dvc, tmp_dir, scm, dvc, make_dataset, remote): + dataset = make_dataset( + cache=False, files=False, dvcfile=True, commit=True, remote=True + ) + bench_dvc("import", tmp_dir, dataset.name, "-o", "new") diff --git a/dvc/testing/benchmarks/cli/commands/test_import_url.py b/dvc/testing/benchmarks/cli/commands/test_import_url.py new file mode 100644 index 000000000..f3e29bb10 --- /dev/null +++ b/dvc/testing/benchmarks/cli/commands/test_import_url.py @@ -0,0 +1,6 @@ +# pylint: disable=unused-argument +def test_import_url(bench_dvc, tmp_dir, scm, dvc, make_dataset): + dataset = make_dataset( + cache=False, files=True, dvcfile=False, commit=False, remote=False + ) + bench_dvc("import-url", str(dataset), "new") diff --git a/dvc/testing/benchmarks/cli/commands/test_init.py b/dvc/testing/benchmarks/cli/commands/test_init.py new file mode 100644 index 000000000..cfd49e1fe --- /dev/null +++ b/dvc/testing/benchmarks/cli/commands/test_init.py @@ -0,0 +1,14 @@ +# pylint: disable=unused-argument +import shutil + + +def test_init(bench_dvc, tmp_dir, scm): + def _cleanup_dir(): + for item in tmp_dir.iterdir(): + if item.is_dir(): + if item.name != ".git": + shutil.rmtree(item) + else: + item.unlink() + + bench_dvc("init", setup=_cleanup_dir, rounds=100, warmup_rounds=1) diff --git a/dvc/testing/benchmarks/cli/commands/test_ls.py b/dvc/testing/benchmarks/cli/commands/test_ls.py new file mode 100644 index 000000000..2843a91f9 --- /dev/null +++ b/dvc/testing/benchmarks/cli/commands/test_ls.py @@ -0,0 +1,7 @@ +# pylint: disable=unused-argument +def test_list(bench_dvc, tmp_dir, scm, dvc, make_dataset, remote): + make_dataset(cache=False, files=False, dvcfile=True, commit=True, remote=True) + bench_dvc("list", tmp_dir) + bench_dvc("list", tmp_dir, "--dvc-only", name="dvc-only") + bench_dvc("list", tmp_dir, "--recursive", name="recursive") + bench_dvc("list", tmp_dir, "dataset", name="shallow") diff --git a/dvc/testing/benchmarks/cli/commands/test_plots.py b/dvc/testing/benchmarks/cli/commands/test_plots.py new file mode 100644 index 000000000..80d07b9ab --- /dev/null +++ b/dvc/testing/benchmarks/cli/commands/test_plots.py @@ -0,0 +1,18 @@ +import pytest + +from dvc.repo import Repo +from tests.benchmarks.conftest import pull + +# pylint: disable=unused-argument + + +@pytest.mark.requires(minversion=(2, 34, 0), reason="top-level plots not supported") +def test_plots(project, bench_dvc): + with Repo() as dvc: + pull(dvc) + + kwargs = {"rounds": 5, "iterations": 3, "warmup_rounds": 2} + bench_dvc("plots", "show", name="show", **kwargs) + bench_dvc("plots", "show", "--json", name="show-json", **kwargs) + bench_dvc("plots", "diff", "HEAD", name="diff", **kwargs) + bench_dvc("plots", "diff", "HEAD", "--json", name="diff-json", **kwargs) diff --git a/dvc/testing/benchmarks/cli/commands/test_pull.py b/dvc/testing/benchmarks/cli/commands/test_pull.py new file mode 100644 index 000000000..286dd96f7 --- /dev/null +++ b/dvc/testing/benchmarks/cli/commands/test_pull.py @@ -0,0 +1,4 @@ +# pylint: disable=unused-argument +def test_pull(bench_dvc, tmp_dir, dvc, make_dataset, remote): + make_dataset(cache=False, dvcfile=True, files=False, remote=True) + bench_dvc("pull") diff --git a/dvc/testing/benchmarks/cli/commands/test_push.py b/dvc/testing/benchmarks/cli/commands/test_push.py new file mode 100644 index 000000000..2b979a616 --- /dev/null +++ b/dvc/testing/benchmarks/cli/commands/test_push.py @@ -0,0 +1,4 @@ +# pylint: disable=unused-argument +def test_push(bench_dvc, tmp_dir, dvc, make_dataset, remote): + make_dataset(cache=True, dvcfile=True, files=False) + bench_dvc("push") diff --git a/dvc/testing/benchmarks/cli/commands/test_status.py b/dvc/testing/benchmarks/cli/commands/test_status.py new file mode 100644 index 000000000..ce5717187 --- /dev/null +++ b/dvc/testing/benchmarks/cli/commands/test_status.py @@ -0,0 +1,9 @@ +# pylint: disable=unused-argument +def test_status(bench_dvc, tmp_dir, dvc, make_dataset): + dataset = make_dataset(files=True, dvcfile=True, cache=True) + bench_dvc("status") + bench_dvc("status", name="noop") + + (dataset / "new").write_text("new") + bench_dvc("status", name="changed") + bench_dvc("status", name="changed-noop") diff --git a/dvc/testing/benchmarks/cli/commands/test_update.py b/dvc/testing/benchmarks/cli/commands/test_update.py new file mode 100644 index 000000000..86c96679a --- /dev/null +++ b/dvc/testing/benchmarks/cli/commands/test_update.py @@ -0,0 +1,9 @@ +# pylint: disable=unused-argument +def test_update(bench_dvc, tmp_dir, dvc, make_dataset): + dataset = make_dataset( + cache=False, files=True, dvcfile=False, commit=False, remote=False + ) + dvc.imp_url(str(dataset), "new") + (dataset / "new").write_text("new") + bench_dvc("update", "new.dvc") + bench_dvc("update", "new.dvc", name="noop") diff --git a/dvc/testing/benchmarks/cli/stories/__init__.py b/dvc/testing/benchmarks/cli/stories/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dvc/testing/benchmarks/cli/stories/test_modify_data.py b/dvc/testing/benchmarks/cli/stories/test_modify_data.py new file mode 100644 index 000000000..1ccff734e --- /dev/null +++ b/dvc/testing/benchmarks/cli/stories/test_modify_data.py @@ -0,0 +1,55 @@ +""" +Tests for modifications to an existing dataset. +""" +import glob +import os +import random +import shutil +import sys + +import pytest + +# pylint: disable=unused-argument,unexpected-keyword-arg + + +@pytest.mark.skipif(sys.version_info < (3, 10), reason="requires 3.10 glob.glob") +def test_partial_add(bench_dvc, tmp_dir, dvc, dataset, remote): + # Move some files to create a partial dataset + os.makedirs("partial-copy") + for f in glob.glob("*", root_dir=dataset, recursive=True): # type: ignore[call-arg] + if random.random() > 0.5: # noqa: S311 # nosec + shutil.move(dataset / f, tmp_dir / "partial-copy" / f) + + # Add/push partial dataset + dvc.add(str(dataset)) + dvc.push() + + # Add more files to the dataset + shutil.copytree("partial-copy", dataset, dirs_exist_ok=True) + + # Benchmark operations for adding files to a dataset + bench_dvc("add", dataset, name="partial-add") + bench_dvc("push", name="partial-add") + bench_dvc("gc", "-f", "-w", name="noop") + bench_dvc("gc", "-f", "-w", "-c", name="cloud-noop") + + +@pytest.mark.skipif(sys.version_info < (3, 10), reason="requires 3.10 glob.glob") +def test_partial_remove(bench_dvc, tmp_dir, dvc, dataset, remote): + # Add/push full dataset + dvc.add(str(dataset)) + dvc.push() + + # Remove some files + for f in glob.glob("*", root_dir=dataset, recursive=True): # type: ignore[call-arg] + if random.random() > 0.5: # noqa: S311 # nosec + if os.path.isfile(dataset / f): + os.remove(dataset / f) + elif os.path.isdir(dataset / f): + shutil.rmtree(dataset / f) + + # Benchmark operations for removing files from dataset + bench_dvc("add", dataset) + bench_dvc("push") + bench_dvc("gc", "-f", "-w") + bench_dvc("gc", "-f", "-w", "-c", name="cloud") diff --git a/dvc/testing/benchmarks/cli/stories/use_cases/__init__.py b/dvc/testing/benchmarks/cli/stories/use_cases/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dvc/testing/benchmarks/cli/stories/use_cases/test_sharing.py b/dvc/testing/benchmarks/cli/stories/use_cases/test_sharing.py new file mode 100644 index 000000000..268412dfe --- /dev/null +++ b/dvc/testing/benchmarks/cli/stories/use_cases/test_sharing.py @@ -0,0 +1,19 @@ +import shutil + +# pylint: disable=unused-argument + + +def test_sharing(bench_dvc, tmp_dir, dvc, dataset, remote): + bench_dvc("add", dataset) + bench_dvc("add", dataset, name="noop") + + bench_dvc("push") + bench_dvc("push", name="noop") + + shutil.rmtree(dataset) + shutil.rmtree(tmp_dir / ".dvc" / "cache") + + bench_dvc("pull") + bench_dvc("pull", name="noop") + + bench_dvc("checkout", name="noop") diff --git a/dvc/testing/benchmarks/conftest.py b/dvc/testing/benchmarks/conftest.py new file mode 100644 index 000000000..c4eba4c7b --- /dev/null +++ b/dvc/testing/benchmarks/conftest.py @@ -0,0 +1,207 @@ +import os +import shutil +from subprocess import check_output + +import pytest +from dulwich.porcelain import clone +from funcy import first +from packaging import version +from pytest_virtualenv import VirtualEnv + + +def pytest_generate_tests(metafunc): + str_revs = metafunc.config.getoption("--dvc-revs") + revs = str_revs.split(",") if str_revs else [None] + if "dvc_rev" in metafunc.fixturenames: + metafunc.parametrize("dvc_rev", revs, scope="session") + + +@pytest.fixture(scope="session") +def make_dvc_venv(tmp_path_factory): + def _make_dvc_venv(name): + venv_dir = tmp_path_factory.mktemp(f"dvc-venv-{name}") + return VirtualEnv(workspace=venv_dir) + + return _make_dvc_venv + + +@pytest.fixture(scope="session") +def dvc_venvs(): + return {} + + +@pytest.fixture(scope="session") +def dvc_git_repo(tmp_path_factory, test_config): + url = test_config.dvc_git_repo + + if os.path.isdir(url): + return url + + tmp_path = os.fspath(tmp_path_factory.mktemp("dvc-git-repo")) + clone(url, tmp_path) + + return tmp_path + + +@pytest.fixture(scope="session") +def dvc_bin(dvc_rev, dvc_venvs, make_dvc_venv, dvc_git_repo, test_config, request): + if dvc_rev: + venv = dvc_venvs.get(dvc_rev) + if not venv: + venv = make_dvc_venv(dvc_rev) + venv.run("pip install -U pip") + if dvc_rev in ["2.18.1", "2.11.0", "2.6.3"]: + venv.run("pip install fsspec==2022.11.0") + venv.run(f"pip install git+file://{dvc_git_repo}@{dvc_rev}") + dvc_venvs[dvc_rev] = venv + dvc_bin = venv.virtualenv / "bin" / "dvc" + else: + dvc_bin = test_config.dvc_bin + + def _dvc_bin(*args): + return check_output([dvc_bin, *args], text=True) + + _dvc_bin.version = parse_tuple(_dvc_bin("--version")) + return _dvc_bin + + +def parse_tuple(version_string): + parsed = version.parse(version_string) + return (parsed.major, parsed.minor, parsed.micro) + + +@pytest.fixture(autouse=True) +def skip_if_dvc_version_lt_required(request, dvc_bin): + if marker := request.node.get_closest_marker("requires"): + minversion = marker.kwargs.get("minversion") or first(marker.args) + assert minversion, ( + "'minversion' needs to be specified as" + " a positional or a keyword argument" + ) + reason = marker.kwargs.get("reason", "") + if isinstance(minversion, str): + minversion = parse_tuple(minversion) + if dvc_bin.version < minversion: + version_repr = ".".join(map(str, minversion)) + pytest.skip(f"requires dvc>={version_repr}: {reason}") + + +@pytest.fixture(scope="function") +def make_bench(request): + def _make_bench(name): + import pytest_benchmark.plugin + + # hack from https://github.com/ionelmc/pytest-benchmark/issues/166 + bench = pytest_benchmark.plugin.benchmark.__pytest_wrapped__.obj(request) + + suffix = f"-{name}" + + def add_suffix(_name): + start, sep, end = _name.partition("[") + return start + suffix + sep + end + + bench.name = add_suffix(bench.name) + bench.fullname = add_suffix(bench.fullname) + + return bench + + return _make_bench + + +@pytest.fixture(scope="function") +def bench_dvc(dvc_bin, make_bench): + def _bench_dvc(*args, **kwargs): + name = kwargs.pop("name", None) + name = f"-{name}" if name else "" + bench = make_bench(args[0] + name) + return bench.pedantic(dvc_bin, args=args, **kwargs) + + return _bench_dvc + + +def pull(repo, *args): + from dvc.exceptions import CheckoutError, DownloadError + + while True: + try: + return repo.pull(*args) + except (CheckoutError, DownloadError): + pass + + +@pytest.fixture +def make_dataset(request, test_config, tmp_dir, pytestconfig): + def _make_dataset( + dvcfile=False, files=True, cache=False, commit=False, remote=False + ): + from dvc.repo import Repo + + path = tmp_dir / "dataset" + root = pytestconfig.rootpath + src = root / "data" / test_config.size / "dataset" + src_dvc = src.with_suffix(".dvc") + + dvc = Repo(root) + + pull(dvc, [str(src_dvc)]) + if files: + shutil.copytree(src, path) + if dvcfile: + shutil.copy(src.with_suffix(".dvc"), path.with_suffix(".dvc")) + if cache: + shutil.copytree(root / ".dvc" / "cache", tmp_dir / ".dvc" / "cache") + if remote: + assert dvcfile + assert not cache + assert tmp_dir.dvc + # FIXME temporary hack, we should try to push from home repo + # directly to this remote instead + shutil.copytree(root / ".dvc" / "cache", tmp_dir / ".dvc" / "cache") + tmp_dir.dvc.push([str(path.with_suffix(".dvc").relative_to(tmp_dir))]) + shutil.rmtree(tmp_dir / ".dvc" / "cache") + if commit: + assert dvcfile + assert tmp_dir.scm + tmp_dir.scm.add([str(path.with_suffix(".dvc").relative_to(tmp_dir))]) + tmp_dir.scm.commit("add dataset") + return path + + return _make_dataset + + +@pytest.fixture +def dataset(make_dataset): + return make_dataset(dvcfile=False, files=True, cache=False) + + +@pytest.fixture +def remote_dataset(test_config): + pytest.skip("fixme") + + +@pytest.fixture +def make_project(tmp_path_factory): + def _make_project(url, rev=None): + path = os.fspath(tmp_path_factory.mktemp("dvc-project")) + + if rev: + rev = rev.encode("ascii") + + clone(url, path, branch=rev) + return path + + return _make_project + + +@pytest.fixture +def project(test_config, monkeypatch, make_project): + rev = test_config.project_rev + url = test_config.project_git_repo + + if os.path.isdir(url): + path = url + assert not rev + else: + path = make_project(url, rev=rev) + + monkeypatch.chdir(path) diff --git a/pyproject.toml b/pyproject.toml index 70bf3b430..a6601c8a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -287,6 +287,7 @@ ignore-names = ["M", "SCM"] [tool.ruff.per-file-ignores] "dvc/commands/**" = ["N806"] "dvc/testing/**" = ["ARG002"] +"dvc/testing/benchmarks/**" = ["ARG001"] "scripts/**" = ["T201", "INP001", "TRY002"] [tool.ruff.pylint]