diff --git a/dvc/commands/checkout.py b/dvc/commands/checkout.py index 201a5b53c..50f25a0be 100644 --- a/dvc/commands/checkout.py +++ b/dvc/commands/checkout.py @@ -1,5 +1,3 @@ -import operator - from dvc.cli import completion, formatter from dvc.cli.command import CmdBase from dvc.cli.utils import append_doc_link @@ -30,7 +28,7 @@ class CmdCheckout(CmdBase): stats, exc = None, None try: - stats = self.repo.checkout( + result = self.repo.checkout( targets=self.args.targets, with_deps=self.args.with_deps, force=self.args.force, @@ -40,14 +38,17 @@ class CmdCheckout(CmdBase): ) except CheckoutError as _exc: exc = _exc - stats = exc.stats + result = exc.result if self.args.summary: default_message = "No changes." - msg = get_summary(sorted(stats.items(), key=operator.itemgetter(0))) + stats = result["stats"] + assert isinstance(stats, dict) + msg = get_summary(stats.items()) ui.write(msg or default_message) else: - log_changes(stats) + result.pop("stats", {}) + log_changes(result) if exc: raise exc diff --git a/dvc/commands/data_sync.py b/dvc/commands/data_sync.py index 81f573e6b..22c09bed1 100644 --- a/dvc/commands/data_sync.py +++ b/dvc/commands/data_sync.py @@ -22,17 +22,18 @@ class CmdDataBase(CmdBase): class CmdDataPull(CmdDataBase): - def log_summary(self, stats): + def log_summary(self, result): from dvc.commands.checkout import log_changes - log_changes(stats) + stats = result.pop("stats", {}) + log_changes(result) super().log_summary(stats) def run(self): from dvc.exceptions import CheckoutError, DvcException try: - stats = self.repo.pull( + result = self.repo.pull( targets=self.args.targets, jobs=self.args.jobs, remote=self.args.remote, @@ -46,10 +47,10 @@ class CmdDataPull(CmdDataBase): glob=self.args.glob, allow_missing=self.args.allow_missing, ) - self.log_summary(stats) + self.log_summary(result) except (CheckoutError, DvcException) as exc: - if stats := getattr(exc, "stats", {}): - self.log_summary(stats) + if result := getattr(exc, "result", {}): + self.log_summary(result) logger.exception("failed to pull data from the cloud") return 1 diff --git a/dvc/exceptions.py b/dvc/exceptions.py index f72b0dd62..ef72049d7 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -242,11 +242,11 @@ class UploadError(FileTransferError): class CheckoutError(DvcException): - def __init__(self, target_infos: list[str], stats: dict[str, list[str]]): + def __init__(self, target_infos: list[str], result: dict): from dvc.utils import error_link self.target_infos = target_infos - self.stats = stats + self.result = result targets = [str(t) for t in target_infos] m = ( "Checkout failed for following targets:\n{}\nIs your " diff --git a/dvc/repo/checkout.py b/dvc/repo/checkout.py index 5e146eb57..919fb0a45 100644 --- a/dvc/repo/checkout.py +++ b/dvc/repo/checkout.py @@ -1,4 +1,5 @@ import os +from collections import defaultdict from typing import TYPE_CHECKING from dvc.exceptions import ( @@ -14,7 +15,9 @@ from dvc.utils import relpath from . import locked if TYPE_CHECKING: - from dvc_data.index import BaseDataIndex, DataIndexEntry + from dvc.repo.index import IndexView + from dvc_data.index import BaseDataIndex, DataIndexEntry, DataIndexKey + from dvc_data.index.diff import Change from dvc_objects.fs.base import FileSystem logger = logger.getChild(__name__) @@ -36,10 +39,12 @@ def _remove_unused_links(repo): return ret -def _build_out_changes(index, changes): +def _build_out_changes( + index: "IndexView", changes: dict["DataIndexKey", "Change"] +) -> dict["DataIndexKey", tuple[str, dict[str, int]]]: from dvc_data.index.checkout import MODIFY - out_keys = [] + out_keys: list[DataIndexKey] = [] for out in index.outs: if not out.use_cache: continue @@ -47,19 +52,28 @@ def _build_out_changes(index, changes): ws, key = out.index_key if ws != "repo": continue - out_keys.append(key) - out_changes = {} + out_stats: dict[DataIndexKey, dict[str, int]] + out_stats = defaultdict(lambda: defaultdict(int)) + + out_changes: dict[DataIndexKey, tuple[str, dict[str, int]]] = {} for key, change in changes.items(): + typ = change.typ + isdir = change.new and change.new.isdir for out_key in out_keys: if len(out_key) > len(key) or key[: len(out_key)] != out_key: continue + stats = out_stats[out_key] + if not isdir: + stats[typ] += 1 + if key == out_key: - out_changes[out_key] = change.typ - elif not out_changes.get(out_key): - out_changes[out_key] = MODIFY + out_changes[out_key] = typ, stats + elif out_key not in out_changes: + typ = MODIFY + out_changes[out_key] = typ, stats break return out_changes @@ -107,14 +121,13 @@ def checkout( # noqa: C901 from dvc.stage.exceptions import StageFileBadNameError, StageFileDoesNotExistError from dvc_data.index.checkout import ADD, DELETE, MODIFY, apply, compare - stats: dict[str, list[str]] = { - "added": [], - "deleted": [], - "modified": [], - } + stats = {"modified": 0, "added": 0, "deleted": 0} + changes: dict[str, list[str]] = {"modified": [], "added": [], "deleted": []} + if not targets: targets = [None] - stats["deleted"] = _remove_unused_links(self) + changes["deleted"] = _remove_unused_links(self) + stats["deleted"] = len(changes["deleted"]) if isinstance(targets, str): targets = [targets] @@ -176,16 +189,26 @@ def checkout( # noqa: C901 out_changes = _build_out_changes(view, diff.changes) typ_map = {ADD: "added", DELETE: "deleted", MODIFY: "modified"} - for key, typ in out_changes.items(): + for key, (typ, _stats) in out_changes.items(): out_path = self.fs.join(self.root_dir, *key) if out_path in failed: self.fs.remove(out_path, recursive=True) - else: - self.state.save_link(out_path, self.fs) - stats[typ_map[typ]].append(_fspath_dir(out_path)) + continue + self.state.save_link(out_path, self.fs) + for t, count in _stats.items(): + stats_typ = typ_map[t] + stats[stats_typ] += count + + changes[typ_map[typ]].append(_fspath_dir(out_path)) + + for changelist in changes.values(): + # group directories first, then files. But keep them alphabetically sorted + changelist.sort(key=lambda p: (not p.endswith(os.sep), p)) + + result = changes | {"stats": stats} if failed and not allow_missing: - raise CheckoutError([relpath(out_path) for out_path in failed], stats) - - return stats + result["failed"] = [relpath(out_path) for out_path in failed] + raise CheckoutError([relpath(out_path) for out_path in failed], result) + return result diff --git a/dvc/repo/pull.py b/dvc/repo/pull.py index d997c99a7..ef1443977 100644 --- a/dvc/repo/pull.py +++ b/dvc/repo/pull.py @@ -39,7 +39,7 @@ def pull( # noqa: PLR0913 run_cache=run_cache, ) try: - stats = self.checkout( + result = self.checkout( targets=expanded_targets, with_deps=with_deps, force=force, @@ -47,8 +47,10 @@ def pull( # noqa: PLR0913 allow_missing=allow_missing, ) except CheckoutError as exc: - exc.stats["fetched"] = processed_files_count + # put fetched counts first + exc.result["stats"] = {"fetched": processed_files_count} | exc.result["stats"] raise - - stats["fetched"] = processed_files_count - return stats + else: + # put fetched counts first + result["stats"] = {"fetched": processed_files_count} | result["stats"] + return result diff --git a/dvc/testing/remote_tests.py b/dvc/testing/remote_tests.py index 48e8c052b..7a6aa5743 100644 --- a/dvc/testing/remote_tests.py +++ b/dvc/testing/remote_tests.py @@ -137,8 +137,12 @@ class TestRemote: remove(tmp_dir / "bar") stats = dvc.pull() - assert stats["fetched"] == 2 - assert set(stats["added"]) == {"foo", "bar"} + assert stats == { + "added": ["bar", "foo"], + "deleted": [], + "modified": [], + "stats": {"fetched": 2, "added": 2, "deleted": 0, "modified": 0}, + } @pytest.mark.xfail(raises=NotImplementedError, strict=False) def test_pull_no_00_prefix(self, tmp_dir, dvc, remote, monkeypatch): @@ -163,8 +167,12 @@ class TestRemote: remove(tmp_dir / "bar") stats = dvc.pull() - assert stats["fetched"] == 2 - assert set(stats["added"]) == {"foo", "bar"} + assert stats == { + "added": ["bar", "foo"], + "deleted": [], + "modified": [], + "stats": {"fetched": 2, "added": 2, "deleted": 0, "modified": 0}, + } class TestRemoteVersionAware: diff --git a/dvc/testing/workspace_tests.py b/dvc/testing/workspace_tests.py index ba72f7cef..9bec71c53 100644 --- a/dvc/testing/workspace_tests.py +++ b/dvc/testing/workspace_tests.py @@ -163,7 +163,12 @@ class TestImportURLVersionAware: (remote_version_aware / "data_dir" / "foo").write_text("foo") dvc.update(no_download=True) - assert dvc.pull()["fetched"] == 2 + assert dvc.pull() == { + "modified": [], + "added": ["data_dir" + os.sep], + "deleted": [], + "stats": {"fetched": 2, "modified": 0, "added": 2, "deleted": 0}, + } assert (tmp_dir / "data_dir").read_text() == { "foo": "foo", "subdir": {"file": "file"}, @@ -174,7 +179,12 @@ class TestImportURLVersionAware: scm.checkout("v1") dvc.cache.local.clear() remove(tmp_dir / "data_dir") - assert dvc.pull()["fetched"] == 1 + assert dvc.pull() == { + "modified": [], + "added": ["data_dir" + os.sep], + "deleted": [], + "stats": {"fetched": 1, "modified": 0, "added": 1, "deleted": 0}, + } assert (tmp_dir / "data_dir").read_text() == {"subdir": {"file": "file"}} dvc.commit(force=True) diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index 4763fac4a..4759b0bc2 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -36,7 +36,10 @@ def test_checkout(tmp_dir, dvc, copy_script): remove(tmp_dir / "foo") remove("data") - dvc.checkout(force=True) + assert dvc.checkout(force=True) == empty_checkout | { + "added": ["data" + os.sep, "foo"], + "stats": empty_stats | {"added": 2}, + } assert (tmp_dir / "foo").read_text() == "foo" assert (tmp_dir / "data").read_text() == {"file": "file"} @@ -65,6 +68,53 @@ def test_checkout_cli(tmp_dir, dvc, copy_script): assert (tmp_dir / "data").read_text() == {"file": "file"} +@pytest.mark.parametrize( + "summary,expected_lines", + [ + ( + True, + [ + "1 file modified, 3 files added and 1 file deleted", + ], + ), + ( + False, + [ + "M\tbar".expandtabs(), + "A\tdir".expandtabs() + os.sep, + "A\tfoo".expandtabs(), + "A\tlorem".expandtabs(), + "D\tipsum".expandtabs(), + ], + ), + ], +) +def test_checkout_stats(tmp_dir, dvc, capsys, summary, expected_lines): + tmp_dir.dvc_gen( + { + "foo": "foo", + "bar": "bar", + "lorem": "lorem", + "dir": {"file": "file"}, + "ipsum": "ipsum", + "dolor": "dolor", + } + ) + for out in ["foo", "bar", "lorem", "dir"]: + remove(tmp_dir / out) + (tmp_dir / "ipsum.dvc").unlink() + (tmp_dir / "bar").write_text("foobar") + + opts = ["--summary"] if summary else [] + assert main(["checkout", "--force", *opts]) == 0 + out, _ = capsys.readouterr() + assert out.splitlines() == expected_lines + + main(["checkout"]) + out, _ = capsys.readouterr() + assert not out + + def test_remove_files_when_checkout(tmp_dir, dvc, scm): # add the file into a separate branch scm.checkout("branch", True) @@ -167,8 +217,9 @@ def test_checkout_missing_md5_in_lock_file_for_outs_deps(tmp_dir, dvc, copy_scri name="copy-file", ) - with pytest.raises(CheckoutError): + with pytest.raises(CheckoutError) as exc: dvc.checkout(force=True) + assert exc.value.result == empty_checkout | {"failed": ["file1"]} def test_checkout_empty_dir(tmp_dir, dvc): @@ -179,8 +230,10 @@ def test_checkout_empty_dir(tmp_dir, dvc): stage.outs[0].remove() assert not empty_dir.exists() - stats = dvc.checkout(force=True) - assert stats["added"] == [os.path.join("empty_dir", "")] + assert dvc.checkout(force=True) == empty_checkout | { + "added": [os.path.join("empty_dir", "")], + "stats": empty_stats | {"added": 0}, + } assert empty_dir.is_dir() assert not list(empty_dir.iterdir()) @@ -188,8 +241,7 @@ def test_checkout_empty_dir(tmp_dir, dvc): def test_checkout_not_cached_file(tmp_dir, dvc): tmp_dir.dvc_gen("foo", "foo") dvc.run(cmd="cp foo bar", deps=["foo"], outs_no_cache=["bar"], name="copy-file") - stats = dvc.checkout(force=True) - assert not any(stats.values()) + assert dvc.checkout(force=True) == empty_checkout def test_checkout_with_deps_cli(tmp_dir, dvc, copy_script): @@ -254,8 +306,7 @@ def test_checkout_recursive_not_directory(tmp_dir, dvc): ret = main(["add", "foo"]) assert ret == 0 - stats = dvc.checkout(targets=["foo.dvc"], recursive=True) - assert stats == {"added": [], "modified": [], "deleted": []} + assert dvc.checkout(targets=["foo.dvc"], recursive=True) == empty_checkout def test_checkout_moved_cache_dir_with_symlinks(tmp_dir, dvc): @@ -327,8 +378,7 @@ def test_checkout_relink(tmp_dir, dvc, link, link_test_func): assert os.access(data_file, os.W_OK) assert not link_test_func(data_file) - stats = dvc.checkout(["dir.dvc"], relink=True) - assert stats == empty_checkout + assert dvc.checkout(["dir.dvc"], relink=True) == empty_checkout assert link_test_func(data_file) # NOTE: Windows symlink perms don't propagate to the target and @@ -344,12 +394,15 @@ def test_checkout_relink(tmp_dir, dvc, link, link_test_func): def test_partial_checkout(tmp_dir, dvc, target): tmp_dir.dvc_gen({"dir": {"subdir": {"file": "file"}, "other": "other"}}) shutil.rmtree("dir") - stats = dvc.checkout([target]) - assert stats["added"] == ["dir" + os.sep] + assert dvc.checkout([target]) == empty_checkout | { + "added": ["dir" + os.sep], + "stats": empty_stats | {"added": 1}, + } assert list(walk_files("dir")) == [os.path.join("dir", "subdir", "file")] -empty_checkout = {"added": [], "deleted": [], "modified": []} +empty_stats = {"added": 0, "deleted": 0, "modified": 0} +empty_checkout = {"added": [], "deleted": [], "modified": [], "stats": empty_stats} def test_stats_on_empty_checkout(tmp_dir, dvc, scm): @@ -372,11 +425,17 @@ def test_stats_on_checkout(tmp_dir, dvc, scm): ) scm.checkout("HEAD~") stats = dvc.checkout() - assert set(stats["deleted"]) == {"dir" + os.sep, "foo", "bar"} + assert stats == empty_checkout | { + "deleted": ["dir" + os.sep, "bar", "foo"], + "stats": empty_stats | {"deleted": 3}, + } scm.checkout("-") stats = dvc.checkout() - assert set(stats["added"]) == {"bar", "dir" + os.sep, "foo"} + assert stats == empty_checkout | { + "added": ["dir" + os.sep, "bar", "foo"], + "stats": empty_stats | {"added": 4}, + } tmp_dir.gen({"lorem": "lorem", "bar": "new bar", "dir2": {"file": "file"}}) (tmp_dir / "foo").unlink() @@ -385,18 +444,23 @@ def test_stats_on_checkout(tmp_dir, dvc, scm): scm.checkout("HEAD~") stats = dvc.checkout() - assert set(stats["modified"]) == {"bar"} - assert set(stats["added"]) == {"foo"} - assert set(stats["deleted"]) == {"lorem", "dir2" + os.sep} + assert stats == empty_checkout | { + "modified": ["bar"], + "added": ["foo"], + "deleted": ["dir2" + os.sep, "lorem"], + "stats": {"modified": 1, "added": 1, "deleted": 2}, + } scm.checkout("-") stats = dvc.checkout() - assert set(stats["modified"]) == {"bar"} - assert set(stats["added"]) == {"dir2" + os.sep, "lorem"} - assert set(stats["deleted"]) == {"foo"} + assert stats == empty_checkout | { + "modified": ["bar"], + "added": ["dir2" + os.sep, "lorem"], + "deleted": ["foo"], + "stats": {"modified": 1, "added": 2, "deleted": 1}, + } -@pytest.mark.xfail(reason="values relpath") def test_checkout_stats_on_failure(tmp_dir, dvc, scm): tmp_dir.dvc_gen( {"foo": "foo", "dir": {"subdir": {"file": "file"}}, "other": "other"}, @@ -405,20 +469,20 @@ def test_checkout_stats_on_failure(tmp_dir, dvc, scm): stage = load_file(dvc, "foo.dvc").stage tmp_dir.dvc_gen({"foo": "foobar", "other": "other other"}, commit="second") - # corrupt cache + # remove object from cache cache = stage.outs[0].cache_path - os.chmod(cache, 0o644) - with open(cache, "a", encoding="utf-8") as fd: - fd.write("destroy cache") + remove(cache) + + (tmp_dir / "foo").unlink() scm.checkout("HEAD~") with pytest.raises(CheckoutError) as exc: dvc.checkout(force=True) - assert exc.value.stats == { - **empty_checkout, + assert exc.value.result == empty_checkout | { "failed": ["foo"], "modified": ["other"], + "stats": empty_stats | {"modified": 1}, } @@ -431,11 +495,19 @@ def test_stats_on_added_file_from_tracked_dir(tmp_dir, dvc, scm): tmp_dir.gen("dir/subdir/newfile", "newfile") tmp_dir.dvc_add("dir", commit="add newfile") scm.checkout("HEAD~") - assert dvc.checkout() == {**empty_checkout, "modified": ["dir" + os.sep]} + assert dvc.checkout() == { + **empty_checkout, + "modified": ["dir" + os.sep], + "stats": empty_stats | {"deleted": 1}, + } assert dvc.checkout() == empty_checkout scm.checkout("-") - assert dvc.checkout() == {**empty_checkout, "modified": ["dir" + os.sep]} + assert dvc.checkout() == { + **empty_checkout, + "modified": ["dir" + os.sep], + "stats": empty_stats | {"added": 1}, + } assert dvc.checkout() == empty_checkout @@ -448,11 +520,19 @@ def test_stats_on_updated_file_from_tracked_dir(tmp_dir, dvc, scm): tmp_dir.gen("dir/subdir/file", "what file?") tmp_dir.dvc_add("dir", commit="update file") scm.checkout("HEAD~") - assert dvc.checkout() == {**empty_checkout, "modified": ["dir" + os.sep]} + assert dvc.checkout() == { + **empty_checkout, + "modified": ["dir" + os.sep], + "stats": empty_stats | {"modified": 1}, + } assert dvc.checkout() == empty_checkout scm.checkout("-") - assert dvc.checkout() == {**empty_checkout, "modified": ["dir" + os.sep]} + assert dvc.checkout() == { + **empty_checkout, + "modified": ["dir" + os.sep], + "stats": empty_stats | {"modified": 1}, + } assert dvc.checkout() == empty_checkout @@ -465,11 +545,19 @@ def test_stats_on_removed_file_from_tracked_dir(tmp_dir, dvc, scm): (tmp_dir / "dir" / "subdir" / "file").unlink() tmp_dir.dvc_add("dir", commit="removed file from subdir") scm.checkout("HEAD~") - assert dvc.checkout() == {**empty_checkout, "modified": ["dir" + os.sep]} + assert dvc.checkout() == { + **empty_checkout, + "modified": ["dir" + os.sep], + "stats": empty_stats | {"added": 1}, + } assert dvc.checkout() == empty_checkout scm.checkout("-") - assert dvc.checkout() == {**empty_checkout, "modified": ["dir" + os.sep]} + assert dvc.checkout() == { + **empty_checkout, + "modified": ["dir" + os.sep], + "stats": empty_stats | {"deleted": 2}, + } assert dvc.checkout() == empty_checkout @@ -511,7 +599,11 @@ def test_checkout_with_relink_existing(tmp_dir, dvc, link): dvc.cache.local.cache_types = [link] stats = dvc.checkout(relink=True) - assert stats == {**empty_checkout, "added": ["foo"]} + assert stats == { + **empty_checkout, + "added": ["foo"], + "stats": empty_stats | {"added": 1}, + } def test_checkout_with_deps(tmp_dir, dvc): @@ -522,11 +614,19 @@ def test_checkout_with_deps(tmp_dir, dvc): (tmp_dir / "foo").unlink() stats = dvc.checkout(["copy-file"], with_deps=False) - assert stats == {**empty_checkout, "added": ["bar"]} + assert stats == { + **empty_checkout, + "added": ["bar"], + "stats": empty_stats | {"added": 1}, + } (tmp_dir / "bar").unlink() stats = dvc.checkout(["copy-file"], with_deps=True) - assert set(stats["added"]) == {"foo", "bar"} + assert stats == { + **empty_checkout, + "added": ["bar", "foo"], + "stats": empty_stats | {"added": 2}, + } def test_checkout_recursive(tmp_dir, dvc): @@ -537,9 +637,9 @@ def test_checkout_recursive(tmp_dir, dvc): (tmp_dir / "dir" / "bar").unlink() stats = dvc.checkout(["dir"], recursive=True) - assert set(stats["added"]) == { - os.path.join("dir", "foo"), - os.path.join("dir", "bar"), + assert stats == empty_checkout | { + "added": [os.path.join("dir", "bar"), os.path.join("dir", "foo")], + "stats": empty_stats | {"added": 2}, } @@ -550,30 +650,51 @@ def test_checkouts_with_different_addressing(tmp_dir, dvc, run_copy): (tmp_dir / "bar").unlink() (tmp_dir / "ipsum").unlink() - assert set(dvc.checkout(PROJECT_FILE)["added"]) == {"bar", "ipsum"} + assert dvc.checkout(PROJECT_FILE) == empty_checkout | { + "added": ["bar", "ipsum"], + "stats": empty_stats | {"added": 2}, + } (tmp_dir / "bar").unlink() (tmp_dir / "ipsum").unlink() - assert set(dvc.checkout(":")["added"]) == {"bar", "ipsum"} + assert dvc.checkout(":") == empty_checkout | { + "added": ["bar", "ipsum"], + "stats": empty_stats | {"added": 2}, + } (tmp_dir / "bar").unlink() - assert dvc.checkout("copy-foo-bar")["added"] == ["bar"] + assert dvc.checkout("copy-foo-bar") == empty_checkout | { + "added": ["bar"], + "stats": empty_stats | {"added": 1}, + } (tmp_dir / "bar").unlink() - assert dvc.checkout("dvc.yaml:copy-foo-bar")["added"] == ["bar"] + assert dvc.checkout("dvc.yaml:copy-foo-bar") == empty_checkout | { + "added": ["bar"], + "stats": empty_stats | {"added": 1}, + } (tmp_dir / "bar").unlink() - assert dvc.checkout(":copy-foo-bar")["added"] == ["bar"] + assert dvc.checkout(":copy-foo-bar") == empty_checkout | { + "added": ["bar"], + "stats": empty_stats | {"added": 1}, + } (tmp_dir / "bar").unlink() (tmp_dir / "data").mkdir() with (tmp_dir / "data").chdir(): - assert dvc.checkout(relpath(tmp_dir / "dvc.yaml") + ":copy-foo-bar")[ - "added" - ] == [relpath(tmp_dir / "bar")] + assert dvc.checkout( + relpath(tmp_dir / "dvc.yaml") + ":copy-foo-bar" + ) == empty_checkout | { + "added": [relpath(tmp_dir / "bar")], + "stats": empty_stats | {"added": 1}, + } (tmp_dir / "bar").unlink() - assert dvc.checkout("bar")["added"] == ["bar"] + assert dvc.checkout("bar") == empty_checkout | { + "added": ["bar"], + "stats": empty_stats | {"added": 1}, + } def test_checkouts_on_same_stage_name_and_output_name(tmp_dir, dvc, run_copy): @@ -583,8 +704,14 @@ def test_checkouts_on_same_stage_name_and_output_name(tmp_dir, dvc, run_copy): (tmp_dir / "bar").unlink() (tmp_dir / "copy-foo-bar").unlink() - assert dvc.checkout("copy-foo-bar")["added"] == ["bar"] - assert dvc.checkout("./copy-foo-bar")["added"] == ["copy-foo-bar"] + assert dvc.checkout("copy-foo-bar") == empty_checkout | { + "added": ["bar"], + "stats": empty_stats | {"added": 1}, + } + assert dvc.checkout("./copy-foo-bar") == empty_checkout | { + "added": ["copy-foo-bar"], + "stats": empty_stats | {"added": 1}, + } def test_checkouts_for_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy): @@ -595,20 +722,35 @@ def test_checkouts_for_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy): for out in ["bar", "ipsum"]: (tmp_dir / out).unlink() - assert dvc.checkout(["bar"])["added"] == ["bar"] + assert dvc.checkout(["bar"]) == empty_checkout | { + "added": ["bar"], + "stats": empty_stats | {"added": 1}, + } (tmp_dir / "bar").unlink() - assert set(dvc.checkout([PROJECT_FILE])["added"]) == {"bar", "ipsum"} + assert dvc.checkout([PROJECT_FILE]) == empty_checkout | { + "added": ["bar", "ipsum"], + "stats": empty_stats | {"added": 2}, + } for out in ["bar", "ipsum"]: (tmp_dir / out).unlink() - assert set(dvc.checkout([stage1.addressing])["added"]) == {"bar"} + assert dvc.checkout([stage1.addressing]) == empty_checkout | { + "added": ["bar"], + "stats": empty_stats | {"added": 1}, + } (tmp_dir / "bar").unlink() - assert set(dvc.checkout([stage2.addressing])["added"]) == {"ipsum"} + assert dvc.checkout([stage2.addressing]) == empty_checkout | { + "added": ["ipsum"], + "stats": empty_stats | {"added": 1}, + } (tmp_dir / "ipsum").unlink() - assert set(dvc.checkout()["added"]) == {"bar", "ipsum"} + assert dvc.checkout() == empty_checkout | { + "added": ["bar", "ipsum"], + "stats": empty_stats | {"added": 2}, + } def test_checkout_executable(tmp_dir, dvc): @@ -618,7 +760,10 @@ def test_checkout_executable(tmp_dir, dvc): contents["outs"][0]["isexec"] = True (tmp_dir / "foo.dvc").dump(contents) - dvc.checkout("foo") + assert dvc.checkout("foo") == empty_checkout | { + "modified": ["foo"], + "stats": empty_stats | {"modified": 1}, + } isexec = os.stat("foo").st_mode & stat.S_IEXEC if os.name == "nt": @@ -634,13 +779,22 @@ def test_checkout_partial(tmp_dir, dvc): data_dir = tmp_dir / "data" shutil.rmtree(data_dir) - dvc.checkout(str(data_dir / "foo")) + assert dvc.checkout(str(data_dir / "foo")) == empty_checkout | { + "added": ["data" + os.sep], + "stats": empty_stats | {"added": 1}, + } assert data_dir.read_text() == {"foo": "foo"} - dvc.checkout(str(data_dir / "sub_dir" / "baz")) + assert dvc.checkout(str(data_dir / "sub_dir" / "baz")) == empty_checkout | { + "modified": ["data" + os.sep], + "stats": empty_stats | {"added": 1}, + } assert data_dir.read_text() == {"foo": "foo", "sub_dir": {"baz": "baz"}} - dvc.checkout(str(data_dir / "bar")) + assert dvc.checkout(str(data_dir / "bar")) == empty_checkout | { + "modified": ["data" + os.sep], + "stats": empty_stats | {"added": 1}, + } assert data_dir.read_text() == { "foo": "foo", "bar": "bar", @@ -664,36 +818,45 @@ def test_checkout_partial_unchanged(tmp_dir, dvc): sub_dir_file = sub_dir / "baz" # Nothing changed, nothing added/deleted/modified - stats = dvc.checkout(str(bar)) - assert not any(stats.values()) + assert dvc.checkout(str(bar)) == empty_checkout # Irrelevant file changed, still nothing added/deleted/modified foo.unlink() - stats = dvc.checkout(str(bar)) - assert not any(stats.values()) + assert dvc.checkout(str(bar)) == empty_checkout # Relevant change, one modified bar.unlink() stats = dvc.checkout(str(bar)) - assert len(stats["modified"]) == 1 + assert stats == empty_checkout | { + "modified": ["data" + os.sep], + "stats": empty_stats | {"added": 1}, + } # No changes inside data/sub - stats = dvc.checkout(str(sub_dir)) - assert not any(stats.values()) + assert dvc.checkout(str(sub_dir)) == empty_checkout # Relevant change, one modified sub_dir_file.unlink() stats = dvc.checkout(str(sub_dir)) + assert stats == empty_checkout | { + "modified": ["data" + os.sep], + "stats": empty_stats | {"added": 1}, + } assert len(stats["modified"]) == 1 stats = dvc.checkout(str(data_dir / "empty_sub_dir")) - assert stats == {**empty_checkout, "modified": ["data" + os.sep]} + assert stats == empty_checkout | { + "modified": ["data" + os.sep], + "stats": empty_stats | {"deleted": 1}, + } - dvc.checkout(str(data_dir)) + assert dvc.checkout(str(data_dir)) == empty_checkout | { + "modified": ["data" + os.sep], + "stats": empty_stats | {"added": 1}, + } # Everything is in place, no action taken - stats = dvc.checkout(str(data_dir)) - assert not any(stats.values()) + assert dvc.checkout(str(data_dir)) == empty_checkout def test_checkout_partial_subdir(tmp_dir, dvc): @@ -704,14 +867,20 @@ def test_checkout_partial_subdir(tmp_dir, dvc): sub_dir_bar = sub_dir / "baz" shutil.rmtree(sub_dir) - dvc.checkout(str(sub_dir)) + assert dvc.checkout(str(sub_dir)) == empty_checkout | { + "modified": ["data" + os.sep], + "stats": empty_stats | {"added": 2}, + } assert data_dir.read_text() == { "foo": "foo", "sub_dir": {"bar": "bar", "baz": "baz"}, } sub_dir_bar.unlink() - dvc.checkout(str(sub_dir_bar)) + assert dvc.checkout(str(sub_dir_bar)) == empty_checkout | { + "modified": ["data" + os.sep], + "stats": empty_stats | {"added": 1}, + } assert data_dir.read_text() == { "foo": "foo", "sub_dir": {"bar": "bar", "baz": "baz"}, @@ -721,12 +890,14 @@ def test_checkout_partial_subdir(tmp_dir, dvc): def test_checkout_file(tmp_dir, dvc): tmp_dir.dvc_gen("foo", "foo") - stats = dvc.checkout("foo") - assert not any(stats.values()) + assert dvc.checkout("foo") == empty_checkout os.unlink("foo") stats = dvc.checkout("foo") - assert stats["added"] == ["foo"] + assert stats == empty_checkout | { + "added": ["foo"], + "stats": empty_stats | {"added": 1}, + } def test_checkout_dir_compat(tmp_dir, dvc): @@ -743,7 +914,10 @@ def test_checkout_dir_compat(tmp_dir, dvc): ), ) remove("data") - dvc.checkout() + assert dvc.checkout() == empty_checkout | { + "added": ["data" + os.sep], + "stats": empty_stats | {"added": 1}, + } assert (tmp_dir / "data").read_text() == {"foo": "foo"} @@ -751,7 +925,10 @@ def test_checkout_cleanup_properly_on_untracked_nested_directories(tmp_dir, scm, tmp_dir.dvc_gen({"datasets": {"dir1": {"file1": "file1"}}}) tmp_dir.gen({"datasets": {"dir2": {"dir3": {"file2": "file2"}}}}) - dvc.checkout(force=True) + assert dvc.checkout(force=True) == empty_checkout | { + "modified": ["datasets" + os.sep], + "stats": empty_stats | {"deleted": 3}, + } assert (tmp_dir / "datasets").read_text() == {"dir1": {"file1": "file1"}} @@ -766,7 +943,10 @@ def test_checkout_loads_specific_file(tmp_dir, dvc, mocker): f = SingleStageFile(dvc, "foo.dvc") spy = mocker.spy(FileMixin, "_load") - assert dvc.checkout("foo.dvc") == {"added": ["foo"], "deleted": [], "modified": []} + assert dvc.checkout("foo.dvc") == empty_checkout | { + "added": ["foo"], + "stats": empty_stats | {"added": 1}, + } spy.assert_called_with(f) assert (tmp_dir / "foo").exists() diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index ff7081ac7..74f58cb72 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -17,9 +17,13 @@ from dvc.utils.fs import remove from dvc_data.hashfile.db import HashFileDB from dvc_data.hashfile.db.local import LocalHashFileDB from dvc_data.hashfile.hash_info import HashInfo +from tests.func.test_checkout import empty_checkout, empty_stats + +empty_stats = empty_stats | {"fetched": 0} +empty_pull = empty_checkout | {"stats": empty_stats} -def test_cloud_cli(tmp_dir, dvc, remote, mocker): # noqa: PLR0915 +def test_cloud_cli(tmp_dir, dvc, capsys, remote, mocker): # noqa: PLR0915 jobs = 2 args = ["-v", "-j", str(jobs)] @@ -42,6 +46,7 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker): # noqa: PLR0915 oids_exist = mocker.spy(LocalHashFileDB, "oids_exist") assert main(["push", *args]) == 0 + assert capsys.readouterr().out == "5 files pushed\n" assert os.path.exists(cache) assert os.path.isfile(cache) assert os.path.isfile(cache_dir) @@ -54,6 +59,7 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker): # noqa: PLR0915 oids_exist.reset_mock() assert main(["fetch", *args]) == 0 + assert capsys.readouterr().out == "5 files fetched\n" assert os.path.exists(cache) assert os.path.isfile(cache) assert os.path.isfile(cache_dir) @@ -65,6 +71,7 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker): # noqa: PLR0915 oids_exist.reset_mock() assert main(["pull", *args]) == 0 + assert capsys.readouterr().out == "Everything is up to date.\n" assert os.path.exists(cache) assert os.path.isfile(cache) assert os.path.isfile(cache_dir) @@ -93,6 +100,7 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker): # noqa: PLR0915 shutil.move(dvc.cache.local.path, dvc.cache.local.path + ".back") assert main(["fetch", *args]) == 0 + assert capsys.readouterr().out == "5 files fetched\n" assert oids_exist.called assert all( @@ -101,6 +109,7 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker): # noqa: PLR0915 oids_exist.reset_mock() assert main(["pull", "-f", *args]) == 0 + assert capsys.readouterr().out == "Everything is up to date.\n" assert os.path.exists(cache) assert os.path.isfile(cache) assert os.path.isfile(cache_dir) @@ -195,7 +204,11 @@ def test_verify_hashes(tmp_dir, scm, dvc, mocker, tmp_path_factory, local_remote hash_spy = mocker.spy(dvc_data.hashfile.hash, "file_md5") - dvc.pull() + assert dvc.pull() == empty_pull | { + "added": ["dir" + os.sep, "file"], + "stats": empty_stats | {"fetched": 3, "added": 2}, + } + # NOTE: 2 are for index.data_tree building assert hash_spy.call_count == 3 @@ -205,11 +218,11 @@ def test_verify_hashes(tmp_dir, scm, dvc, mocker, tmp_path_factory, local_remote with dvc.config.edit() as conf: conf["remote"]["upstream"]["verify"] = True - dvc.pull() + assert dvc.pull() == empty_pull | {"stats": empty_stats | {"fetched": 3}} assert hash_spy.call_count == 10 -@pytest.mark.flaky(reruns=3) +# @pytest.mark.flaky(reruns=3) @pytest.mark.parametrize("erepo_type", ["git_dir", "erepo_dir"]) def test_pull_git_imports(request, tmp_dir, dvc, scm, erepo_type): erepo = request.getfixturevalue(erepo_type) @@ -220,7 +233,7 @@ def test_pull_git_imports(request, tmp_dir, dvc, scm, erepo_type): dvc.imp(os.fspath(erepo), "foo") dvc.imp(os.fspath(erepo), "dir", out="new_dir", rev="HEAD~") - assert dvc.pull()["fetched"] == 0 + assert dvc.pull() == empty_pull for item in ["foo", "new_dir"]: remove(item) @@ -228,7 +241,10 @@ def test_pull_git_imports(request, tmp_dir, dvc, scm, erepo_type): os.makedirs(dvc.cache.local.path, exist_ok=True) clean_repos() - assert dvc.pull(force=True)["fetched"] == 2 + assert dvc.pull(force=True) == empty_pull | { + "added": ["new_dir" + os.sep, "foo"], + "stats": empty_stats | {"fetched": 2, "added": 2}, + } assert (tmp_dir / "foo").exists() assert (tmp_dir / "foo").read_text() == "foo" @@ -248,11 +264,14 @@ def test_pull_external_dvc_imports(tmp_dir, dvc, scm, erepo_dir): dvc.imp(os.fspath(erepo_dir), "foo") dvc.imp(os.fspath(erepo_dir), "dir", out="new_dir", rev="HEAD~") - assert dvc.pull()["fetched"] == 0 + assert dvc.pull() == empty_pull clean(["foo", "new_dir"], dvc) - assert dvc.pull(force=True)["fetched"] == 2 + assert dvc.pull(force=True) == empty_pull | { + "added": ["new_dir" + os.sep, "foo"], + "stats": empty_stats | {"fetched": 2, "added": 2}, + } assert (tmp_dir / "foo").exists() assert (tmp_dir / "foo").read_text() == "foo" @@ -267,7 +286,10 @@ def test_pull_partial_import(tmp_dir, dvc, local_workspace): stage = dvc.imp_url("remote://workspace/file", os.fspath(dst), no_download=True) result = dvc.pull("file") - assert result["fetched"] == 1 + assert result == empty_checkout | { + "added": ["file"], + "stats": empty_stats | {"fetched": 1, "added": 1}, + } assert dst.exists() assert stage.outs[0].get_hash().value == "d10b4c3ff123b26dc068d43a8bef2d23" @@ -279,8 +301,9 @@ def test_pull_partial_import_missing(tmp_dir, dvc, local_workspace): dvc.imp_url("remote://workspace/file", os.fspath(dst), no_download=True) (local_workspace / "file").unlink() - with pytest.raises(CheckoutError): + with pytest.raises(CheckoutError) as exc: dvc.pull("file") + assert exc.value.result == empty_pull | {"failed": ["file"]} assert not dst.exists() @@ -290,8 +313,9 @@ def test_pull_partial_import_modified(tmp_dir, dvc, local_workspace): dvc.imp_url("remote://workspace/file", os.fspath(dst), no_download=True) local_workspace.gen("file", "updated file content") - with pytest.raises(CheckoutError): + with pytest.raises(CheckoutError) as exc: dvc.pull("file") + assert exc.value.result == empty_pull | {"failed": ["file"]} assert not dst.exists() @@ -309,7 +333,10 @@ def test_pull_external_dvc_imports_mixed(tmp_dir, dvc, scm, erepo_dir, local_rem clean(["foo", "bar"], dvc) - assert dvc.pull()["fetched"] == 2 + assert dvc.pull() == empty_pull | { + "added": ["bar", "foo"], + "stats": empty_stats | {"fetched": 2, "added": 2}, + } assert (tmp_dir / "foo").read_text() == "foo" assert (tmp_dir / "bar").read_text() == "bar" @@ -330,34 +357,44 @@ def recurse_list_dir(d): def test_dvc_pull_pipeline_stages(tmp_dir, dvc, run_copy, local_remote): - (stage0,) = tmp_dir.dvc_gen("foo", "foo") - stage1 = run_copy("foo", "bar", name="copy-foo-bar") - stage2 = run_copy("bar", "foobar", name="copy-bar-foobar") + (stage0,) = tmp_dir.dvc_gen("bar", "bar") + stage1 = run_copy("bar", "foo", name="copy-bar-foo") + stage2 = run_copy("foo", "foobar", name="copy-foo-foobar") dvc.push() - outs = ["foo", "bar", "foobar"] + outs = ["bar", "foo", "foobar"] clean(outs, dvc) - dvc.pull() + assert dvc.pull() == empty_pull | { + "added": outs, + "stats": empty_stats | {"fetched": 1, "added": 3}, + } assert all((tmp_dir / file).exists() for file in outs) for out, stage in zip(outs, [stage0, stage1, stage2]): for target in [stage.addressing, out]: clean(outs, dvc) stats = dvc.pull([target]) - assert stats["fetched"] == 1 - assert stats["added"] == [out] + assert stats == empty_pull | { + "added": [out], + "stats": empty_stats | {"fetched": 1, "added": 1}, + } assert os.path.exists(out) assert not any(os.path.exists(out) for out in set(outs) - {out}) clean(outs, dvc) stats = dvc.pull([stage2.addressing], with_deps=True) - assert len(stats["added"]) == 3 - assert set(stats["added"]) == set(outs) + assert stats == empty_pull | { + "added": outs, + "stats": empty_stats | {"fetched": 1, "added": 3}, + } clean(outs, dvc) stats = dvc.pull([os.curdir], recursive=True) - assert set(stats["added"]) == set(outs) + assert stats == empty_pull | { + "added": outs, + "stats": empty_stats | {"fetched": 1, "added": 3}, + } def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, local_remote): @@ -374,7 +411,7 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, local_remote): remove(dvc.stage_cache.cache_dir) - dvc.push() + assert dvc.push() == 3 outs = ["foo", "lorem", "ipsum", "baz", "lorem2"] @@ -382,20 +419,26 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, local_remote): assert len(recurse_list_dir(path)) == 3 clean(outs, dvc) - assert set(dvc.pull(["dvc.yaml"])["added"]) == {"lorem2", "baz"} + assert dvc.pull(["dvc.yaml"]) == empty_pull | { + "added": ["baz", "lorem2"], + "stats": empty_stats | {"fetched": 2, "added": 2}, + } clean(outs, dvc) - assert set(dvc.pull()["added"]) == set(outs) + assert dvc.pull() == empty_pull | { + "added": ["baz", "foo", "ipsum", "lorem", "lorem2"], + "stats": empty_stats | {"fetched": 3, "added": 5}, + } # clean everything in remote and push from dvc.testing.tmp_dir import TmpDir clean(TmpDir(path).iterdir()) - dvc.push(["dvc.yaml:copy-ipsum-baz"]) + assert dvc.push(["dvc.yaml:copy-ipsum-baz"]) == 1 assert len(recurse_list_dir(path)) == 1 clean(TmpDir(path).iterdir()) - dvc.push(["dvc.yaml"]) + assert dvc.push(["dvc.yaml"]) == 2 assert len(recurse_list_dir(path)) == 2 with pytest.raises(StageNotFound): @@ -440,23 +483,36 @@ def test_fetch_stats(tmp_dir, dvc, fs, msg, capsys, local_remote): def test_pull_stats(tmp_dir, dvc, capsys, local_remote): - tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"}) + tmp_dir.dvc_gen( + { + "foo": "foo", + "bar": "bar", + "lorem": "lorem", + "dir": {"file": "file"}, + "ipsum": "ipsum", + "dolor": "dolor", + } + ) dvc.push() - clean(["foo", "bar"], dvc) + clean(["foo", "bar", "dir", "lorem"], dvc) + + (tmp_dir / "ipsum.dvc").unlink() (tmp_dir / "bar").write_text("foobar") assert main(["pull", "--force"]) == 0 - out, _ = capsys.readouterr() - assert "M\tbar".expandtabs() in out - assert "A\tfoo".expandtabs() in out - assert "2 files fetched" in out - assert "1 file added" in out - assert "1 file modified" in out + assert out.splitlines() == [ + "M\tbar".expandtabs(), + "A\tdir".expandtabs() + os.sep, + "A\tfoo".expandtabs(), + "A\tlorem".expandtabs(), + "D\tipsum".expandtabs(), + "6 files fetched, 1 file modified, 3 files added and 1 file deleted", + ] main(["pull"]) out, _ = capsys.readouterr() - assert "Everything is up to date." in out + assert out == "Everything is up to date.\n" @pytest.mark.parametrize( @@ -475,26 +531,32 @@ def test_push_pull_all(tmp_dir, scm, dvc, local_remote, key, expected): assert dvc.push(**{key: True}) == expected clean(["foo", "bar", "baz"], dvc) - assert dvc.pull(**{key: True})["fetched"] == expected + assert dvc.pull(**{key: True}) == empty_pull | { + "added": ["bar", "foo"], + "stats": empty_stats | {"fetched": expected, "added": 2}, + } def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, local_remote): tmp_dir.dvc_gen("foo", "foo") run_copy("foo", "bar", name="copy-foo-bar") - dvc.push("copy-foo-bar") + assert dvc.push("copy-foo-bar") == 1 assert len(recurse_list_dir(local_remote.url)) == 1 # pushing everything so as we can check pull/fetch only downloads # from specified targets - dvc.push() + assert dvc.push() == 0 clean(["foo", "bar"], dvc) - dvc.pull("copy-foo-bar") + assert dvc.pull("copy-foo-bar") == empty_pull | { + "added": ["bar"], + "stats": empty_stats | {"fetched": 1, "added": 1}, + } assert (tmp_dir / "bar").exists() assert len(recurse_list_dir(dvc.cache.local.path)) == 1 clean(["bar"], dvc) - dvc.fetch("copy-foo-bar") + assert dvc.fetch("copy-foo-bar") == 1 assert len(recurse_list_dir(dvc.cache.local.path)) == 1 @@ -505,7 +567,10 @@ def test_pull_partial(tmp_dir, dvc, local_remote): clean(["foo"], dvc) stats = dvc.pull(os.path.join("foo", "bar")) - assert stats["fetched"] == 2 + assert stats == empty_pull | { + "added": [os.path.join("foo", "")], + "stats": empty_stats | {"fetched": 2, "added": 1}, + } assert (tmp_dir / "foo").read_text() == {"bar": {"baz": "baz"}} @@ -540,7 +605,10 @@ def test_output_remote(tmp_dir, dvc, make_remote): clean(["foo", "bar", "data"], dvc) - dvc.pull() + assert dvc.pull() == empty_pull | { + "added": ["data" + os.sep, "bar", "foo"], + "stats": empty_stats | {"fetched": 5, "added": 4}, + } assert set(dvc.cache.local.all()) == { "37b51d194a7513e45b56f6524f2d51f2", @@ -573,7 +641,10 @@ def test_target_remote(tmp_dir, dvc, make_remote): clean(["foo", "data"], dvc) - dvc.pull(remote="myremote") + assert dvc.pull(remote="myremote") == empty_pull | { + "added": ["data" + os.sep, "foo"], + "stats": empty_stats | {"fetched": 4, "added": 3}, + } assert set(dvc.cache.local.all()) == { "acbd18db4cc2f85cedef654fccc4a4d8", @@ -630,7 +701,10 @@ def test_output_target_remote(tmp_dir, dvc, make_remote): clean(["foo", "bar", "data"], dvc) # pull foo and data from for_foo remote - dvc.pull(remote="for_foo", allow_missing=True) + assert dvc.pull(remote="for_foo", allow_missing=True) == empty_pull | { + "added": ["data" + os.sep, "foo"], + "stats": empty_stats | {"fetched": 4, "added": 3}, + } assert set(dvc.cache.local.all()) == expected @@ -638,15 +712,19 @@ def test_output_target_remote(tmp_dir, dvc, make_remote): def test_pull_allow_missing(tmp_dir, dvc, local_remote): dvc.stage.add(name="bar", outs=["bar"], cmd="echo bar > bar") - with pytest.raises(CheckoutError): + with pytest.raises(CheckoutError) as exc: dvc.pull() + assert exc.value.result == empty_pull | {"failed": ["bar"]} tmp_dir.dvc_gen("foo", "foo") dvc.push() clean(["foo"], dvc) stats = dvc.pull(allow_missing=True) - assert stats["fetched"] == 1 + assert stats == empty_pull | { + "added": ["foo"], + "stats": empty_stats | {"fetched": 1, "added": 1}, + } def test_pull_granular_excluding_import_that_cannot_be_pulled( @@ -668,11 +746,9 @@ def test_pull_granular_excluding_import_that_cannot_be_pulled( shutil.rmtree("dir") dvc.cache.local.clear() - assert dvc.pull(stage.addressing) == { + assert dvc.pull(stage.addressing) == empty_pull | { "added": [join("dir", "")], - "deleted": [], - "modified": [], - "fetched": 3, + "stats": empty_stats | {"added": 2, "fetched": 3}, } with pytest.raises(CloneError, match="SCM error"): @@ -704,8 +780,8 @@ def test_loads_single_file(tmp_dir, dvc, local_remote, mocker): assert dvc.pull("foo.dvc") == { "added": ["foo"], "deleted": [], - "fetched": 1, "modified": [], + "stats": {"added": 1, "deleted": 0, "modified": 0, "fetched": 1}, } spy.assert_called_with(foo_dvcfile) assert (tmp_dir / "foo").exists() diff --git a/tests/func/test_run_cache.py b/tests/func/test_run_cache.py index b25ee537e..f7dc6a983 100644 --- a/tests/func/test_run_cache.py +++ b/tests/func/test_run_cache.py @@ -20,7 +20,12 @@ def test_push_pull(tmp_dir, dvc, erepo_dir, run_copy, local_remote): erepo_dir.add_remote(config=local_remote.config) with erepo_dir.chdir(): assert not os.path.exists(erepo_dir.dvc.stage_cache.cache_dir) - assert erepo_dir.dvc.pull(run_cache=True)["fetched"] == 0 + assert erepo_dir.dvc.pull(run_cache=True) == { + "added": [], + "deleted": [], + "modified": [], + "stats": {"fetched": 0, "added": 0, "deleted": 0, "modified": 0}, + } assert os.listdir(erepo_dir.dvc.stage_cache.cache_dir) diff --git a/tests/func/test_virtual_directory.py b/tests/func/test_virtual_directory.py index 16947749a..3c67db96b 100644 --- a/tests/func/test_virtual_directory.py +++ b/tests/func/test_virtual_directory.py @@ -1,3 +1,4 @@ +import os import shutil from os.path import join @@ -180,7 +181,12 @@ def test_partial_checkout_and_update(M, tmp_dir, dvc, remote): dvc.cache.local.clear() shutil.rmtree("dir") - assert dvc.pull("dir/subdir") == M.dict(added=[join("dir", "")], fetched=11) + assert dvc.pull("dir/subdir") == { + "added": ["dir" + os.sep], + "deleted": [], + "modified": [], + "stats": {"added": 10, "deleted": 0, "fetched": 11, "modified": 0}, + } assert (tmp_dir / "dir").read_text() == {"subdir": dir1} tmp_dir.gen({"dir": {"subdir": {"file": "file"}}})