mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-01 23:08:43 +01:00
checkout/pull: report granular summary of changes (#10842)
This commit is contained in:
skshetry
@@ -1,5 +1,3 @@
|
||||
import operator
|
||||
|
||||
from dvc.cli import completion, formatter
|
||||
from dvc.cli.command import CmdBase
|
||||
from dvc.cli.utils import append_doc_link
|
||||
@@ -30,7 +28,7 @@ class CmdCheckout(CmdBase):
|
||||
|
||||
stats, exc = None, None
|
||||
try:
|
||||
stats = self.repo.checkout(
|
||||
result = self.repo.checkout(
|
||||
targets=self.args.targets,
|
||||
with_deps=self.args.with_deps,
|
||||
force=self.args.force,
|
||||
@@ -40,14 +38,17 @@ class CmdCheckout(CmdBase):
|
||||
)
|
||||
except CheckoutError as _exc:
|
||||
exc = _exc
|
||||
stats = exc.stats
|
||||
result = exc.result
|
||||
|
||||
if self.args.summary:
|
||||
default_message = "No changes."
|
||||
msg = get_summary(sorted(stats.items(), key=operator.itemgetter(0)))
|
||||
stats = result["stats"]
|
||||
assert isinstance(stats, dict)
|
||||
msg = get_summary(stats.items())
|
||||
ui.write(msg or default_message)
|
||||
else:
|
||||
log_changes(stats)
|
||||
result.pop("stats", {})
|
||||
log_changes(result)
|
||||
|
||||
if exc:
|
||||
raise exc
|
||||
|
||||
@@ -22,17 +22,18 @@ class CmdDataBase(CmdBase):
|
||||
|
||||
|
||||
class CmdDataPull(CmdDataBase):
|
||||
def log_summary(self, stats):
|
||||
def log_summary(self, result):
|
||||
from dvc.commands.checkout import log_changes
|
||||
|
||||
log_changes(stats)
|
||||
stats = result.pop("stats", {})
|
||||
log_changes(result)
|
||||
super().log_summary(stats)
|
||||
|
||||
def run(self):
|
||||
from dvc.exceptions import CheckoutError, DvcException
|
||||
|
||||
try:
|
||||
stats = self.repo.pull(
|
||||
result = self.repo.pull(
|
||||
targets=self.args.targets,
|
||||
jobs=self.args.jobs,
|
||||
remote=self.args.remote,
|
||||
@@ -46,10 +47,10 @@ class CmdDataPull(CmdDataBase):
|
||||
glob=self.args.glob,
|
||||
allow_missing=self.args.allow_missing,
|
||||
)
|
||||
self.log_summary(stats)
|
||||
self.log_summary(result)
|
||||
except (CheckoutError, DvcException) as exc:
|
||||
if stats := getattr(exc, "stats", {}):
|
||||
self.log_summary(stats)
|
||||
if result := getattr(exc, "result", {}):
|
||||
self.log_summary(result)
|
||||
logger.exception("failed to pull data from the cloud")
|
||||
return 1
|
||||
|
||||
|
||||
+2
-2
@@ -242,11 +242,11 @@ class UploadError(FileTransferError):
|
||||
|
||||
|
||||
class CheckoutError(DvcException):
|
||||
def __init__(self, target_infos: list[str], stats: dict[str, list[str]]):
|
||||
def __init__(self, target_infos: list[str], result: dict):
|
||||
from dvc.utils import error_link
|
||||
|
||||
self.target_infos = target_infos
|
||||
self.stats = stats
|
||||
self.result = result
|
||||
targets = [str(t) for t in target_infos]
|
||||
m = (
|
||||
"Checkout failed for following targets:\n{}\nIs your "
|
||||
|
||||
+44
-21
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from dvc.exceptions import (
|
||||
@@ -14,7 +15,9 @@ from dvc.utils import relpath
|
||||
from . import locked
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc_data.index import BaseDataIndex, DataIndexEntry
|
||||
from dvc.repo.index import IndexView
|
||||
from dvc_data.index import BaseDataIndex, DataIndexEntry, DataIndexKey
|
||||
from dvc_data.index.diff import Change
|
||||
from dvc_objects.fs.base import FileSystem
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
@@ -36,10 +39,12 @@ def _remove_unused_links(repo):
|
||||
return ret
|
||||
|
||||
|
||||
def _build_out_changes(index, changes):
|
||||
def _build_out_changes(
|
||||
index: "IndexView", changes: dict["DataIndexKey", "Change"]
|
||||
) -> dict["DataIndexKey", tuple[str, dict[str, int]]]:
|
||||
from dvc_data.index.checkout import MODIFY
|
||||
|
||||
out_keys = []
|
||||
out_keys: list[DataIndexKey] = []
|
||||
for out in index.outs:
|
||||
if not out.use_cache:
|
||||
continue
|
||||
@@ -47,19 +52,28 @@ def _build_out_changes(index, changes):
|
||||
ws, key = out.index_key
|
||||
if ws != "repo":
|
||||
continue
|
||||
|
||||
out_keys.append(key)
|
||||
|
||||
out_changes = {}
|
||||
out_stats: dict[DataIndexKey, dict[str, int]]
|
||||
out_stats = defaultdict(lambda: defaultdict(int))
|
||||
|
||||
out_changes: dict[DataIndexKey, tuple[str, dict[str, int]]] = {}
|
||||
for key, change in changes.items():
|
||||
typ = change.typ
|
||||
isdir = change.new and change.new.isdir
|
||||
for out_key in out_keys:
|
||||
if len(out_key) > len(key) or key[: len(out_key)] != out_key:
|
||||
continue
|
||||
|
||||
stats = out_stats[out_key]
|
||||
if not isdir:
|
||||
stats[typ] += 1
|
||||
|
||||
if key == out_key:
|
||||
out_changes[out_key] = change.typ
|
||||
elif not out_changes.get(out_key):
|
||||
out_changes[out_key] = MODIFY
|
||||
out_changes[out_key] = typ, stats
|
||||
elif out_key not in out_changes:
|
||||
typ = MODIFY
|
||||
out_changes[out_key] = typ, stats
|
||||
break
|
||||
|
||||
return out_changes
|
||||
@@ -107,14 +121,13 @@ def checkout( # noqa: C901
|
||||
from dvc.stage.exceptions import StageFileBadNameError, StageFileDoesNotExistError
|
||||
from dvc_data.index.checkout import ADD, DELETE, MODIFY, apply, compare
|
||||
|
||||
stats: dict[str, list[str]] = {
|
||||
"added": [],
|
||||
"deleted": [],
|
||||
"modified": [],
|
||||
}
|
||||
stats = {"modified": 0, "added": 0, "deleted": 0}
|
||||
changes: dict[str, list[str]] = {"modified": [], "added": [], "deleted": []}
|
||||
|
||||
if not targets:
|
||||
targets = [None]
|
||||
stats["deleted"] = _remove_unused_links(self)
|
||||
changes["deleted"] = _remove_unused_links(self)
|
||||
stats["deleted"] = len(changes["deleted"])
|
||||
|
||||
if isinstance(targets, str):
|
||||
targets = [targets]
|
||||
@@ -176,16 +189,26 @@ def checkout( # noqa: C901
|
||||
out_changes = _build_out_changes(view, diff.changes)
|
||||
|
||||
typ_map = {ADD: "added", DELETE: "deleted", MODIFY: "modified"}
|
||||
for key, typ in out_changes.items():
|
||||
for key, (typ, _stats) in out_changes.items():
|
||||
out_path = self.fs.join(self.root_dir, *key)
|
||||
|
||||
if out_path in failed:
|
||||
self.fs.remove(out_path, recursive=True)
|
||||
else:
|
||||
self.state.save_link(out_path, self.fs)
|
||||
stats[typ_map[typ]].append(_fspath_dir(out_path))
|
||||
continue
|
||||
|
||||
self.state.save_link(out_path, self.fs)
|
||||
for t, count in _stats.items():
|
||||
stats_typ = typ_map[t]
|
||||
stats[stats_typ] += count
|
||||
|
||||
changes[typ_map[typ]].append(_fspath_dir(out_path))
|
||||
|
||||
for changelist in changes.values():
|
||||
# group directories first, then files. But keep them alphabetically sorted
|
||||
changelist.sort(key=lambda p: (not p.endswith(os.sep), p))
|
||||
|
||||
result = changes | {"stats": stats}
|
||||
if failed and not allow_missing:
|
||||
raise CheckoutError([relpath(out_path) for out_path in failed], stats)
|
||||
|
||||
return stats
|
||||
result["failed"] = [relpath(out_path) for out_path in failed]
|
||||
raise CheckoutError([relpath(out_path) for out_path in failed], result)
|
||||
return result
|
||||
|
||||
+7
-5
@@ -39,7 +39,7 @@ def pull( # noqa: PLR0913
|
||||
run_cache=run_cache,
|
||||
)
|
||||
try:
|
||||
stats = self.checkout(
|
||||
result = self.checkout(
|
||||
targets=expanded_targets,
|
||||
with_deps=with_deps,
|
||||
force=force,
|
||||
@@ -47,8 +47,10 @@ def pull( # noqa: PLR0913
|
||||
allow_missing=allow_missing,
|
||||
)
|
||||
except CheckoutError as exc:
|
||||
exc.stats["fetched"] = processed_files_count
|
||||
# put fetched counts first
|
||||
exc.result["stats"] = {"fetched": processed_files_count} | exc.result["stats"]
|
||||
raise
|
||||
|
||||
stats["fetched"] = processed_files_count
|
||||
return stats
|
||||
else:
|
||||
# put fetched counts first
|
||||
result["stats"] = {"fetched": processed_files_count} | result["stats"]
|
||||
return result
|
||||
|
||||
@@ -137,8 +137,12 @@ class TestRemote:
|
||||
remove(tmp_dir / "bar")
|
||||
|
||||
stats = dvc.pull()
|
||||
assert stats["fetched"] == 2
|
||||
assert set(stats["added"]) == {"foo", "bar"}
|
||||
assert stats == {
|
||||
"added": ["bar", "foo"],
|
||||
"deleted": [],
|
||||
"modified": [],
|
||||
"stats": {"fetched": 2, "added": 2, "deleted": 0, "modified": 0},
|
||||
}
|
||||
|
||||
@pytest.mark.xfail(raises=NotImplementedError, strict=False)
|
||||
def test_pull_no_00_prefix(self, tmp_dir, dvc, remote, monkeypatch):
|
||||
@@ -163,8 +167,12 @@ class TestRemote:
|
||||
remove(tmp_dir / "bar")
|
||||
|
||||
stats = dvc.pull()
|
||||
assert stats["fetched"] == 2
|
||||
assert set(stats["added"]) == {"foo", "bar"}
|
||||
assert stats == {
|
||||
"added": ["bar", "foo"],
|
||||
"deleted": [],
|
||||
"modified": [],
|
||||
"stats": {"fetched": 2, "added": 2, "deleted": 0, "modified": 0},
|
||||
}
|
||||
|
||||
|
||||
class TestRemoteVersionAware:
|
||||
|
||||
@@ -163,7 +163,12 @@ class TestImportURLVersionAware:
|
||||
|
||||
(remote_version_aware / "data_dir" / "foo").write_text("foo")
|
||||
dvc.update(no_download=True)
|
||||
assert dvc.pull()["fetched"] == 2
|
||||
assert dvc.pull() == {
|
||||
"modified": [],
|
||||
"added": ["data_dir" + os.sep],
|
||||
"deleted": [],
|
||||
"stats": {"fetched": 2, "modified": 0, "added": 2, "deleted": 0},
|
||||
}
|
||||
assert (tmp_dir / "data_dir").read_text() == {
|
||||
"foo": "foo",
|
||||
"subdir": {"file": "file"},
|
||||
@@ -174,7 +179,12 @@ class TestImportURLVersionAware:
|
||||
scm.checkout("v1")
|
||||
dvc.cache.local.clear()
|
||||
remove(tmp_dir / "data_dir")
|
||||
assert dvc.pull()["fetched"] == 1
|
||||
assert dvc.pull() == {
|
||||
"modified": [],
|
||||
"added": ["data_dir" + os.sep],
|
||||
"deleted": [],
|
||||
"stats": {"fetched": 1, "modified": 0, "added": 1, "deleted": 0},
|
||||
}
|
||||
assert (tmp_dir / "data_dir").read_text() == {"subdir": {"file": "file"}}
|
||||
|
||||
dvc.commit(force=True)
|
||||
|
||||
+259
-79
@@ -36,7 +36,10 @@ def test_checkout(tmp_dir, dvc, copy_script):
|
||||
remove(tmp_dir / "foo")
|
||||
remove("data")
|
||||
|
||||
dvc.checkout(force=True)
|
||||
assert dvc.checkout(force=True) == empty_checkout | {
|
||||
"added": ["data" + os.sep, "foo"],
|
||||
"stats": empty_stats | {"added": 2},
|
||||
}
|
||||
assert (tmp_dir / "foo").read_text() == "foo"
|
||||
assert (tmp_dir / "data").read_text() == {"file": "file"}
|
||||
|
||||
@@ -65,6 +68,53 @@ def test_checkout_cli(tmp_dir, dvc, copy_script):
|
||||
assert (tmp_dir / "data").read_text() == {"file": "file"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"summary,expected_lines",
|
||||
[
|
||||
(
|
||||
True,
|
||||
[
|
||||
"1 file modified, 3 files added and 1 file deleted",
|
||||
],
|
||||
),
|
||||
(
|
||||
False,
|
||||
[
|
||||
"M\tbar".expandtabs(),
|
||||
"A\tdir".expandtabs() + os.sep,
|
||||
"A\tfoo".expandtabs(),
|
||||
"A\tlorem".expandtabs(),
|
||||
"D\tipsum".expandtabs(),
|
||||
],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_checkout_stats(tmp_dir, dvc, capsys, summary, expected_lines):
|
||||
tmp_dir.dvc_gen(
|
||||
{
|
||||
"foo": "foo",
|
||||
"bar": "bar",
|
||||
"lorem": "lorem",
|
||||
"dir": {"file": "file"},
|
||||
"ipsum": "ipsum",
|
||||
"dolor": "dolor",
|
||||
}
|
||||
)
|
||||
for out in ["foo", "bar", "lorem", "dir"]:
|
||||
remove(tmp_dir / out)
|
||||
(tmp_dir / "ipsum.dvc").unlink()
|
||||
(tmp_dir / "bar").write_text("foobar")
|
||||
|
||||
opts = ["--summary"] if summary else []
|
||||
assert main(["checkout", "--force", *opts]) == 0
|
||||
out, _ = capsys.readouterr()
|
||||
assert out.splitlines() == expected_lines
|
||||
|
||||
main(["checkout"])
|
||||
out, _ = capsys.readouterr()
|
||||
assert not out
|
||||
|
||||
|
||||
def test_remove_files_when_checkout(tmp_dir, dvc, scm):
|
||||
# add the file into a separate branch
|
||||
scm.checkout("branch", True)
|
||||
@@ -167,8 +217,9 @@ def test_checkout_missing_md5_in_lock_file_for_outs_deps(tmp_dir, dvc, copy_scri
|
||||
name="copy-file",
|
||||
)
|
||||
|
||||
with pytest.raises(CheckoutError):
|
||||
with pytest.raises(CheckoutError) as exc:
|
||||
dvc.checkout(force=True)
|
||||
assert exc.value.result == empty_checkout | {"failed": ["file1"]}
|
||||
|
||||
|
||||
def test_checkout_empty_dir(tmp_dir, dvc):
|
||||
@@ -179,8 +230,10 @@ def test_checkout_empty_dir(tmp_dir, dvc):
|
||||
stage.outs[0].remove()
|
||||
assert not empty_dir.exists()
|
||||
|
||||
stats = dvc.checkout(force=True)
|
||||
assert stats["added"] == [os.path.join("empty_dir", "")]
|
||||
assert dvc.checkout(force=True) == empty_checkout | {
|
||||
"added": [os.path.join("empty_dir", "")],
|
||||
"stats": empty_stats | {"added": 0},
|
||||
}
|
||||
assert empty_dir.is_dir()
|
||||
assert not list(empty_dir.iterdir())
|
||||
|
||||
@@ -188,8 +241,7 @@ def test_checkout_empty_dir(tmp_dir, dvc):
|
||||
def test_checkout_not_cached_file(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
dvc.run(cmd="cp foo bar", deps=["foo"], outs_no_cache=["bar"], name="copy-file")
|
||||
stats = dvc.checkout(force=True)
|
||||
assert not any(stats.values())
|
||||
assert dvc.checkout(force=True) == empty_checkout
|
||||
|
||||
|
||||
def test_checkout_with_deps_cli(tmp_dir, dvc, copy_script):
|
||||
@@ -254,8 +306,7 @@ def test_checkout_recursive_not_directory(tmp_dir, dvc):
|
||||
ret = main(["add", "foo"])
|
||||
assert ret == 0
|
||||
|
||||
stats = dvc.checkout(targets=["foo.dvc"], recursive=True)
|
||||
assert stats == {"added": [], "modified": [], "deleted": []}
|
||||
assert dvc.checkout(targets=["foo.dvc"], recursive=True) == empty_checkout
|
||||
|
||||
|
||||
def test_checkout_moved_cache_dir_with_symlinks(tmp_dir, dvc):
|
||||
@@ -327,8 +378,7 @@ def test_checkout_relink(tmp_dir, dvc, link, link_test_func):
|
||||
assert os.access(data_file, os.W_OK)
|
||||
assert not link_test_func(data_file)
|
||||
|
||||
stats = dvc.checkout(["dir.dvc"], relink=True)
|
||||
assert stats == empty_checkout
|
||||
assert dvc.checkout(["dir.dvc"], relink=True) == empty_checkout
|
||||
assert link_test_func(data_file)
|
||||
|
||||
# NOTE: Windows symlink perms don't propagate to the target and
|
||||
@@ -344,12 +394,15 @@ def test_checkout_relink(tmp_dir, dvc, link, link_test_func):
|
||||
def test_partial_checkout(tmp_dir, dvc, target):
|
||||
tmp_dir.dvc_gen({"dir": {"subdir": {"file": "file"}, "other": "other"}})
|
||||
shutil.rmtree("dir")
|
||||
stats = dvc.checkout([target])
|
||||
assert stats["added"] == ["dir" + os.sep]
|
||||
assert dvc.checkout([target]) == empty_checkout | {
|
||||
"added": ["dir" + os.sep],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
assert list(walk_files("dir")) == [os.path.join("dir", "subdir", "file")]
|
||||
|
||||
|
||||
empty_checkout = {"added": [], "deleted": [], "modified": []}
|
||||
empty_stats = {"added": 0, "deleted": 0, "modified": 0}
|
||||
empty_checkout = {"added": [], "deleted": [], "modified": [], "stats": empty_stats}
|
||||
|
||||
|
||||
def test_stats_on_empty_checkout(tmp_dir, dvc, scm):
|
||||
@@ -372,11 +425,17 @@ def test_stats_on_checkout(tmp_dir, dvc, scm):
|
||||
)
|
||||
scm.checkout("HEAD~")
|
||||
stats = dvc.checkout()
|
||||
assert set(stats["deleted"]) == {"dir" + os.sep, "foo", "bar"}
|
||||
assert stats == empty_checkout | {
|
||||
"deleted": ["dir" + os.sep, "bar", "foo"],
|
||||
"stats": empty_stats | {"deleted": 3},
|
||||
}
|
||||
|
||||
scm.checkout("-")
|
||||
stats = dvc.checkout()
|
||||
assert set(stats["added"]) == {"bar", "dir" + os.sep, "foo"}
|
||||
assert stats == empty_checkout | {
|
||||
"added": ["dir" + os.sep, "bar", "foo"],
|
||||
"stats": empty_stats | {"added": 4},
|
||||
}
|
||||
|
||||
tmp_dir.gen({"lorem": "lorem", "bar": "new bar", "dir2": {"file": "file"}})
|
||||
(tmp_dir / "foo").unlink()
|
||||
@@ -385,18 +444,23 @@ def test_stats_on_checkout(tmp_dir, dvc, scm):
|
||||
|
||||
scm.checkout("HEAD~")
|
||||
stats = dvc.checkout()
|
||||
assert set(stats["modified"]) == {"bar"}
|
||||
assert set(stats["added"]) == {"foo"}
|
||||
assert set(stats["deleted"]) == {"lorem", "dir2" + os.sep}
|
||||
assert stats == empty_checkout | {
|
||||
"modified": ["bar"],
|
||||
"added": ["foo"],
|
||||
"deleted": ["dir2" + os.sep, "lorem"],
|
||||
"stats": {"modified": 1, "added": 1, "deleted": 2},
|
||||
}
|
||||
|
||||
scm.checkout("-")
|
||||
stats = dvc.checkout()
|
||||
assert set(stats["modified"]) == {"bar"}
|
||||
assert set(stats["added"]) == {"dir2" + os.sep, "lorem"}
|
||||
assert set(stats["deleted"]) == {"foo"}
|
||||
assert stats == empty_checkout | {
|
||||
"modified": ["bar"],
|
||||
"added": ["dir2" + os.sep, "lorem"],
|
||||
"deleted": ["foo"],
|
||||
"stats": {"modified": 1, "added": 2, "deleted": 1},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.xfail(reason="values relpath")
|
||||
def test_checkout_stats_on_failure(tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen(
|
||||
{"foo": "foo", "dir": {"subdir": {"file": "file"}}, "other": "other"},
|
||||
@@ -405,20 +469,20 @@ def test_checkout_stats_on_failure(tmp_dir, dvc, scm):
|
||||
stage = load_file(dvc, "foo.dvc").stage
|
||||
tmp_dir.dvc_gen({"foo": "foobar", "other": "other other"}, commit="second")
|
||||
|
||||
# corrupt cache
|
||||
# remove object from cache
|
||||
cache = stage.outs[0].cache_path
|
||||
os.chmod(cache, 0o644)
|
||||
with open(cache, "a", encoding="utf-8") as fd:
|
||||
fd.write("destroy cache")
|
||||
remove(cache)
|
||||
|
||||
(tmp_dir / "foo").unlink()
|
||||
|
||||
scm.checkout("HEAD~")
|
||||
with pytest.raises(CheckoutError) as exc:
|
||||
dvc.checkout(force=True)
|
||||
|
||||
assert exc.value.stats == {
|
||||
**empty_checkout,
|
||||
assert exc.value.result == empty_checkout | {
|
||||
"failed": ["foo"],
|
||||
"modified": ["other"],
|
||||
"stats": empty_stats | {"modified": 1},
|
||||
}
|
||||
|
||||
|
||||
@@ -431,11 +495,19 @@ def test_stats_on_added_file_from_tracked_dir(tmp_dir, dvc, scm):
|
||||
tmp_dir.gen("dir/subdir/newfile", "newfile")
|
||||
tmp_dir.dvc_add("dir", commit="add newfile")
|
||||
scm.checkout("HEAD~")
|
||||
assert dvc.checkout() == {**empty_checkout, "modified": ["dir" + os.sep]}
|
||||
assert dvc.checkout() == {
|
||||
**empty_checkout,
|
||||
"modified": ["dir" + os.sep],
|
||||
"stats": empty_stats | {"deleted": 1},
|
||||
}
|
||||
assert dvc.checkout() == empty_checkout
|
||||
|
||||
scm.checkout("-")
|
||||
assert dvc.checkout() == {**empty_checkout, "modified": ["dir" + os.sep]}
|
||||
assert dvc.checkout() == {
|
||||
**empty_checkout,
|
||||
"modified": ["dir" + os.sep],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
assert dvc.checkout() == empty_checkout
|
||||
|
||||
|
||||
@@ -448,11 +520,19 @@ def test_stats_on_updated_file_from_tracked_dir(tmp_dir, dvc, scm):
|
||||
tmp_dir.gen("dir/subdir/file", "what file?")
|
||||
tmp_dir.dvc_add("dir", commit="update file")
|
||||
scm.checkout("HEAD~")
|
||||
assert dvc.checkout() == {**empty_checkout, "modified": ["dir" + os.sep]}
|
||||
assert dvc.checkout() == {
|
||||
**empty_checkout,
|
||||
"modified": ["dir" + os.sep],
|
||||
"stats": empty_stats | {"modified": 1},
|
||||
}
|
||||
assert dvc.checkout() == empty_checkout
|
||||
|
||||
scm.checkout("-")
|
||||
assert dvc.checkout() == {**empty_checkout, "modified": ["dir" + os.sep]}
|
||||
assert dvc.checkout() == {
|
||||
**empty_checkout,
|
||||
"modified": ["dir" + os.sep],
|
||||
"stats": empty_stats | {"modified": 1},
|
||||
}
|
||||
assert dvc.checkout() == empty_checkout
|
||||
|
||||
|
||||
@@ -465,11 +545,19 @@ def test_stats_on_removed_file_from_tracked_dir(tmp_dir, dvc, scm):
|
||||
(tmp_dir / "dir" / "subdir" / "file").unlink()
|
||||
tmp_dir.dvc_add("dir", commit="removed file from subdir")
|
||||
scm.checkout("HEAD~")
|
||||
assert dvc.checkout() == {**empty_checkout, "modified": ["dir" + os.sep]}
|
||||
assert dvc.checkout() == {
|
||||
**empty_checkout,
|
||||
"modified": ["dir" + os.sep],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
assert dvc.checkout() == empty_checkout
|
||||
|
||||
scm.checkout("-")
|
||||
assert dvc.checkout() == {**empty_checkout, "modified": ["dir" + os.sep]}
|
||||
assert dvc.checkout() == {
|
||||
**empty_checkout,
|
||||
"modified": ["dir" + os.sep],
|
||||
"stats": empty_stats | {"deleted": 2},
|
||||
}
|
||||
assert dvc.checkout() == empty_checkout
|
||||
|
||||
|
||||
@@ -511,7 +599,11 @@ def test_checkout_with_relink_existing(tmp_dir, dvc, link):
|
||||
dvc.cache.local.cache_types = [link]
|
||||
|
||||
stats = dvc.checkout(relink=True)
|
||||
assert stats == {**empty_checkout, "added": ["foo"]}
|
||||
assert stats == {
|
||||
**empty_checkout,
|
||||
"added": ["foo"],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
|
||||
|
||||
def test_checkout_with_deps(tmp_dir, dvc):
|
||||
@@ -522,11 +614,19 @@ def test_checkout_with_deps(tmp_dir, dvc):
|
||||
(tmp_dir / "foo").unlink()
|
||||
|
||||
stats = dvc.checkout(["copy-file"], with_deps=False)
|
||||
assert stats == {**empty_checkout, "added": ["bar"]}
|
||||
assert stats == {
|
||||
**empty_checkout,
|
||||
"added": ["bar"],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
stats = dvc.checkout(["copy-file"], with_deps=True)
|
||||
assert set(stats["added"]) == {"foo", "bar"}
|
||||
assert stats == {
|
||||
**empty_checkout,
|
||||
"added": ["bar", "foo"],
|
||||
"stats": empty_stats | {"added": 2},
|
||||
}
|
||||
|
||||
|
||||
def test_checkout_recursive(tmp_dir, dvc):
|
||||
@@ -537,9 +637,9 @@ def test_checkout_recursive(tmp_dir, dvc):
|
||||
(tmp_dir / "dir" / "bar").unlink()
|
||||
|
||||
stats = dvc.checkout(["dir"], recursive=True)
|
||||
assert set(stats["added"]) == {
|
||||
os.path.join("dir", "foo"),
|
||||
os.path.join("dir", "bar"),
|
||||
assert stats == empty_checkout | {
|
||||
"added": [os.path.join("dir", "bar"), os.path.join("dir", "foo")],
|
||||
"stats": empty_stats | {"added": 2},
|
||||
}
|
||||
|
||||
|
||||
@@ -550,30 +650,51 @@ def test_checkouts_with_different_addressing(tmp_dir, dvc, run_copy):
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
(tmp_dir / "ipsum").unlink()
|
||||
assert set(dvc.checkout(PROJECT_FILE)["added"]) == {"bar", "ipsum"}
|
||||
assert dvc.checkout(PROJECT_FILE) == empty_checkout | {
|
||||
"added": ["bar", "ipsum"],
|
||||
"stats": empty_stats | {"added": 2},
|
||||
}
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
(tmp_dir / "ipsum").unlink()
|
||||
assert set(dvc.checkout(":")["added"]) == {"bar", "ipsum"}
|
||||
assert dvc.checkout(":") == empty_checkout | {
|
||||
"added": ["bar", "ipsum"],
|
||||
"stats": empty_stats | {"added": 2},
|
||||
}
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
assert dvc.checkout("copy-foo-bar")["added"] == ["bar"]
|
||||
assert dvc.checkout("copy-foo-bar") == empty_checkout | {
|
||||
"added": ["bar"],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
assert dvc.checkout("dvc.yaml:copy-foo-bar")["added"] == ["bar"]
|
||||
assert dvc.checkout("dvc.yaml:copy-foo-bar") == empty_checkout | {
|
||||
"added": ["bar"],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
assert dvc.checkout(":copy-foo-bar")["added"] == ["bar"]
|
||||
assert dvc.checkout(":copy-foo-bar") == empty_checkout | {
|
||||
"added": ["bar"],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
(tmp_dir / "data").mkdir()
|
||||
with (tmp_dir / "data").chdir():
|
||||
assert dvc.checkout(relpath(tmp_dir / "dvc.yaml") + ":copy-foo-bar")[
|
||||
"added"
|
||||
] == [relpath(tmp_dir / "bar")]
|
||||
assert dvc.checkout(
|
||||
relpath(tmp_dir / "dvc.yaml") + ":copy-foo-bar"
|
||||
) == empty_checkout | {
|
||||
"added": [relpath(tmp_dir / "bar")],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
assert dvc.checkout("bar")["added"] == ["bar"]
|
||||
assert dvc.checkout("bar") == empty_checkout | {
|
||||
"added": ["bar"],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
|
||||
|
||||
def test_checkouts_on_same_stage_name_and_output_name(tmp_dir, dvc, run_copy):
|
||||
@@ -583,8 +704,14 @@ def test_checkouts_on_same_stage_name_and_output_name(tmp_dir, dvc, run_copy):
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
(tmp_dir / "copy-foo-bar").unlink()
|
||||
assert dvc.checkout("copy-foo-bar")["added"] == ["bar"]
|
||||
assert dvc.checkout("./copy-foo-bar")["added"] == ["copy-foo-bar"]
|
||||
assert dvc.checkout("copy-foo-bar") == empty_checkout | {
|
||||
"added": ["bar"],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
assert dvc.checkout("./copy-foo-bar") == empty_checkout | {
|
||||
"added": ["copy-foo-bar"],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
|
||||
|
||||
def test_checkouts_for_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
|
||||
@@ -595,20 +722,35 @@ def test_checkouts_for_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
|
||||
|
||||
for out in ["bar", "ipsum"]:
|
||||
(tmp_dir / out).unlink()
|
||||
assert dvc.checkout(["bar"])["added"] == ["bar"]
|
||||
assert dvc.checkout(["bar"]) == empty_checkout | {
|
||||
"added": ["bar"],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
assert set(dvc.checkout([PROJECT_FILE])["added"]) == {"bar", "ipsum"}
|
||||
assert dvc.checkout([PROJECT_FILE]) == empty_checkout | {
|
||||
"added": ["bar", "ipsum"],
|
||||
"stats": empty_stats | {"added": 2},
|
||||
}
|
||||
|
||||
for out in ["bar", "ipsum"]:
|
||||
(tmp_dir / out).unlink()
|
||||
assert set(dvc.checkout([stage1.addressing])["added"]) == {"bar"}
|
||||
assert dvc.checkout([stage1.addressing]) == empty_checkout | {
|
||||
"added": ["bar"],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
assert set(dvc.checkout([stage2.addressing])["added"]) == {"ipsum"}
|
||||
assert dvc.checkout([stage2.addressing]) == empty_checkout | {
|
||||
"added": ["ipsum"],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
|
||||
(tmp_dir / "ipsum").unlink()
|
||||
assert set(dvc.checkout()["added"]) == {"bar", "ipsum"}
|
||||
assert dvc.checkout() == empty_checkout | {
|
||||
"added": ["bar", "ipsum"],
|
||||
"stats": empty_stats | {"added": 2},
|
||||
}
|
||||
|
||||
|
||||
def test_checkout_executable(tmp_dir, dvc):
|
||||
@@ -618,7 +760,10 @@ def test_checkout_executable(tmp_dir, dvc):
|
||||
contents["outs"][0]["isexec"] = True
|
||||
(tmp_dir / "foo.dvc").dump(contents)
|
||||
|
||||
dvc.checkout("foo")
|
||||
assert dvc.checkout("foo") == empty_checkout | {
|
||||
"modified": ["foo"],
|
||||
"stats": empty_stats | {"modified": 1},
|
||||
}
|
||||
|
||||
isexec = os.stat("foo").st_mode & stat.S_IEXEC
|
||||
if os.name == "nt":
|
||||
@@ -634,13 +779,22 @@ def test_checkout_partial(tmp_dir, dvc):
|
||||
data_dir = tmp_dir / "data"
|
||||
shutil.rmtree(data_dir)
|
||||
|
||||
dvc.checkout(str(data_dir / "foo"))
|
||||
assert dvc.checkout(str(data_dir / "foo")) == empty_checkout | {
|
||||
"added": ["data" + os.sep],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
assert data_dir.read_text() == {"foo": "foo"}
|
||||
|
||||
dvc.checkout(str(data_dir / "sub_dir" / "baz"))
|
||||
assert dvc.checkout(str(data_dir / "sub_dir" / "baz")) == empty_checkout | {
|
||||
"modified": ["data" + os.sep],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
assert data_dir.read_text() == {"foo": "foo", "sub_dir": {"baz": "baz"}}
|
||||
|
||||
dvc.checkout(str(data_dir / "bar"))
|
||||
assert dvc.checkout(str(data_dir / "bar")) == empty_checkout | {
|
||||
"modified": ["data" + os.sep],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
assert data_dir.read_text() == {
|
||||
"foo": "foo",
|
||||
"bar": "bar",
|
||||
@@ -664,36 +818,45 @@ def test_checkout_partial_unchanged(tmp_dir, dvc):
|
||||
sub_dir_file = sub_dir / "baz"
|
||||
|
||||
# Nothing changed, nothing added/deleted/modified
|
||||
stats = dvc.checkout(str(bar))
|
||||
assert not any(stats.values())
|
||||
assert dvc.checkout(str(bar)) == empty_checkout
|
||||
|
||||
# Irrelevant file changed, still nothing added/deleted/modified
|
||||
foo.unlink()
|
||||
stats = dvc.checkout(str(bar))
|
||||
assert not any(stats.values())
|
||||
assert dvc.checkout(str(bar)) == empty_checkout
|
||||
|
||||
# Relevant change, one modified
|
||||
bar.unlink()
|
||||
stats = dvc.checkout(str(bar))
|
||||
assert len(stats["modified"]) == 1
|
||||
assert stats == empty_checkout | {
|
||||
"modified": ["data" + os.sep],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
|
||||
# No changes inside data/sub
|
||||
stats = dvc.checkout(str(sub_dir))
|
||||
assert not any(stats.values())
|
||||
assert dvc.checkout(str(sub_dir)) == empty_checkout
|
||||
|
||||
# Relevant change, one modified
|
||||
sub_dir_file.unlink()
|
||||
stats = dvc.checkout(str(sub_dir))
|
||||
assert stats == empty_checkout | {
|
||||
"modified": ["data" + os.sep],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
assert len(stats["modified"]) == 1
|
||||
|
||||
stats = dvc.checkout(str(data_dir / "empty_sub_dir"))
|
||||
assert stats == {**empty_checkout, "modified": ["data" + os.sep]}
|
||||
assert stats == empty_checkout | {
|
||||
"modified": ["data" + os.sep],
|
||||
"stats": empty_stats | {"deleted": 1},
|
||||
}
|
||||
|
||||
dvc.checkout(str(data_dir))
|
||||
assert dvc.checkout(str(data_dir)) == empty_checkout | {
|
||||
"modified": ["data" + os.sep],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
|
||||
# Everything is in place, no action taken
|
||||
stats = dvc.checkout(str(data_dir))
|
||||
assert not any(stats.values())
|
||||
assert dvc.checkout(str(data_dir)) == empty_checkout
|
||||
|
||||
|
||||
def test_checkout_partial_subdir(tmp_dir, dvc):
|
||||
@@ -704,14 +867,20 @@ def test_checkout_partial_subdir(tmp_dir, dvc):
|
||||
sub_dir_bar = sub_dir / "baz"
|
||||
|
||||
shutil.rmtree(sub_dir)
|
||||
dvc.checkout(str(sub_dir))
|
||||
assert dvc.checkout(str(sub_dir)) == empty_checkout | {
|
||||
"modified": ["data" + os.sep],
|
||||
"stats": empty_stats | {"added": 2},
|
||||
}
|
||||
assert data_dir.read_text() == {
|
||||
"foo": "foo",
|
||||
"sub_dir": {"bar": "bar", "baz": "baz"},
|
||||
}
|
||||
|
||||
sub_dir_bar.unlink()
|
||||
dvc.checkout(str(sub_dir_bar))
|
||||
assert dvc.checkout(str(sub_dir_bar)) == empty_checkout | {
|
||||
"modified": ["data" + os.sep],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
assert data_dir.read_text() == {
|
||||
"foo": "foo",
|
||||
"sub_dir": {"bar": "bar", "baz": "baz"},
|
||||
@@ -721,12 +890,14 @@ def test_checkout_partial_subdir(tmp_dir, dvc):
|
||||
def test_checkout_file(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
|
||||
stats = dvc.checkout("foo")
|
||||
assert not any(stats.values())
|
||||
assert dvc.checkout("foo") == empty_checkout
|
||||
|
||||
os.unlink("foo")
|
||||
stats = dvc.checkout("foo")
|
||||
assert stats["added"] == ["foo"]
|
||||
assert stats == empty_checkout | {
|
||||
"added": ["foo"],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
|
||||
|
||||
def test_checkout_dir_compat(tmp_dir, dvc):
|
||||
@@ -743,7 +914,10 @@ def test_checkout_dir_compat(tmp_dir, dvc):
|
||||
),
|
||||
)
|
||||
remove("data")
|
||||
dvc.checkout()
|
||||
assert dvc.checkout() == empty_checkout | {
|
||||
"added": ["data" + os.sep],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
assert (tmp_dir / "data").read_text() == {"foo": "foo"}
|
||||
|
||||
|
||||
@@ -751,7 +925,10 @@ def test_checkout_cleanup_properly_on_untracked_nested_directories(tmp_dir, scm,
|
||||
tmp_dir.dvc_gen({"datasets": {"dir1": {"file1": "file1"}}})
|
||||
tmp_dir.gen({"datasets": {"dir2": {"dir3": {"file2": "file2"}}}})
|
||||
|
||||
dvc.checkout(force=True)
|
||||
assert dvc.checkout(force=True) == empty_checkout | {
|
||||
"modified": ["datasets" + os.sep],
|
||||
"stats": empty_stats | {"deleted": 3},
|
||||
}
|
||||
|
||||
assert (tmp_dir / "datasets").read_text() == {"dir1": {"file1": "file1"}}
|
||||
|
||||
@@ -766,7 +943,10 @@ def test_checkout_loads_specific_file(tmp_dir, dvc, mocker):
|
||||
f = SingleStageFile(dvc, "foo.dvc")
|
||||
|
||||
spy = mocker.spy(FileMixin, "_load")
|
||||
assert dvc.checkout("foo.dvc") == {"added": ["foo"], "deleted": [], "modified": []}
|
||||
assert dvc.checkout("foo.dvc") == empty_checkout | {
|
||||
"added": ["foo"],
|
||||
"stats": empty_stats | {"added": 1},
|
||||
}
|
||||
|
||||
spy.assert_called_with(f)
|
||||
assert (tmp_dir / "foo").exists()
|
||||
|
||||
+128
-52
@@ -17,9 +17,13 @@ from dvc.utils.fs import remove
|
||||
from dvc_data.hashfile.db import HashFileDB
|
||||
from dvc_data.hashfile.db.local import LocalHashFileDB
|
||||
from dvc_data.hashfile.hash_info import HashInfo
|
||||
from tests.func.test_checkout import empty_checkout, empty_stats
|
||||
|
||||
empty_stats = empty_stats | {"fetched": 0}
|
||||
empty_pull = empty_checkout | {"stats": empty_stats}
|
||||
|
||||
|
||||
def test_cloud_cli(tmp_dir, dvc, remote, mocker): # noqa: PLR0915
|
||||
def test_cloud_cli(tmp_dir, dvc, capsys, remote, mocker): # noqa: PLR0915
|
||||
jobs = 2
|
||||
args = ["-v", "-j", str(jobs)]
|
||||
|
||||
@@ -42,6 +46,7 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker): # noqa: PLR0915
|
||||
oids_exist = mocker.spy(LocalHashFileDB, "oids_exist")
|
||||
|
||||
assert main(["push", *args]) == 0
|
||||
assert capsys.readouterr().out == "5 files pushed\n"
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
assert os.path.isfile(cache_dir)
|
||||
@@ -54,6 +59,7 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker): # noqa: PLR0915
|
||||
oids_exist.reset_mock()
|
||||
|
||||
assert main(["fetch", *args]) == 0
|
||||
assert capsys.readouterr().out == "5 files fetched\n"
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
assert os.path.isfile(cache_dir)
|
||||
@@ -65,6 +71,7 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker): # noqa: PLR0915
|
||||
oids_exist.reset_mock()
|
||||
|
||||
assert main(["pull", *args]) == 0
|
||||
assert capsys.readouterr().out == "Everything is up to date.\n"
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
assert os.path.isfile(cache_dir)
|
||||
@@ -93,6 +100,7 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker): # noqa: PLR0915
|
||||
shutil.move(dvc.cache.local.path, dvc.cache.local.path + ".back")
|
||||
|
||||
assert main(["fetch", *args]) == 0
|
||||
assert capsys.readouterr().out == "5 files fetched\n"
|
||||
|
||||
assert oids_exist.called
|
||||
assert all(
|
||||
@@ -101,6 +109,7 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker): # noqa: PLR0915
|
||||
|
||||
oids_exist.reset_mock()
|
||||
assert main(["pull", "-f", *args]) == 0
|
||||
assert capsys.readouterr().out == "Everything is up to date.\n"
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
assert os.path.isfile(cache_dir)
|
||||
@@ -195,7 +204,11 @@ def test_verify_hashes(tmp_dir, scm, dvc, mocker, tmp_path_factory, local_remote
|
||||
|
||||
hash_spy = mocker.spy(dvc_data.hashfile.hash, "file_md5")
|
||||
|
||||
dvc.pull()
|
||||
assert dvc.pull() == empty_pull | {
|
||||
"added": ["dir" + os.sep, "file"],
|
||||
"stats": empty_stats | {"fetched": 3, "added": 2},
|
||||
}
|
||||
|
||||
# NOTE: 2 are for index.data_tree building
|
||||
assert hash_spy.call_count == 3
|
||||
|
||||
@@ -205,11 +218,11 @@ def test_verify_hashes(tmp_dir, scm, dvc, mocker, tmp_path_factory, local_remote
|
||||
with dvc.config.edit() as conf:
|
||||
conf["remote"]["upstream"]["verify"] = True
|
||||
|
||||
dvc.pull()
|
||||
assert dvc.pull() == empty_pull | {"stats": empty_stats | {"fetched": 3}}
|
||||
assert hash_spy.call_count == 10
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=3)
|
||||
# @pytest.mark.flaky(reruns=3)
|
||||
@pytest.mark.parametrize("erepo_type", ["git_dir", "erepo_dir"])
|
||||
def test_pull_git_imports(request, tmp_dir, dvc, scm, erepo_type):
|
||||
erepo = request.getfixturevalue(erepo_type)
|
||||
@@ -220,7 +233,7 @@ def test_pull_git_imports(request, tmp_dir, dvc, scm, erepo_type):
|
||||
dvc.imp(os.fspath(erepo), "foo")
|
||||
dvc.imp(os.fspath(erepo), "dir", out="new_dir", rev="HEAD~")
|
||||
|
||||
assert dvc.pull()["fetched"] == 0
|
||||
assert dvc.pull() == empty_pull
|
||||
|
||||
for item in ["foo", "new_dir"]:
|
||||
remove(item)
|
||||
@@ -228,7 +241,10 @@ def test_pull_git_imports(request, tmp_dir, dvc, scm, erepo_type):
|
||||
os.makedirs(dvc.cache.local.path, exist_ok=True)
|
||||
clean_repos()
|
||||
|
||||
assert dvc.pull(force=True)["fetched"] == 2
|
||||
assert dvc.pull(force=True) == empty_pull | {
|
||||
"added": ["new_dir" + os.sep, "foo"],
|
||||
"stats": empty_stats | {"fetched": 2, "added": 2},
|
||||
}
|
||||
|
||||
assert (tmp_dir / "foo").exists()
|
||||
assert (tmp_dir / "foo").read_text() == "foo"
|
||||
@@ -248,11 +264,14 @@ def test_pull_external_dvc_imports(tmp_dir, dvc, scm, erepo_dir):
|
||||
dvc.imp(os.fspath(erepo_dir), "foo")
|
||||
dvc.imp(os.fspath(erepo_dir), "dir", out="new_dir", rev="HEAD~")
|
||||
|
||||
assert dvc.pull()["fetched"] == 0
|
||||
assert dvc.pull() == empty_pull
|
||||
|
||||
clean(["foo", "new_dir"], dvc)
|
||||
|
||||
assert dvc.pull(force=True)["fetched"] == 2
|
||||
assert dvc.pull(force=True) == empty_pull | {
|
||||
"added": ["new_dir" + os.sep, "foo"],
|
||||
"stats": empty_stats | {"fetched": 2, "added": 2},
|
||||
}
|
||||
|
||||
assert (tmp_dir / "foo").exists()
|
||||
assert (tmp_dir / "foo").read_text() == "foo"
|
||||
@@ -267,7 +286,10 @@ def test_pull_partial_import(tmp_dir, dvc, local_workspace):
|
||||
stage = dvc.imp_url("remote://workspace/file", os.fspath(dst), no_download=True)
|
||||
|
||||
result = dvc.pull("file")
|
||||
assert result["fetched"] == 1
|
||||
assert result == empty_checkout | {
|
||||
"added": ["file"],
|
||||
"stats": empty_stats | {"fetched": 1, "added": 1},
|
||||
}
|
||||
assert dst.exists()
|
||||
|
||||
assert stage.outs[0].get_hash().value == "d10b4c3ff123b26dc068d43a8bef2d23"
|
||||
@@ -279,8 +301,9 @@ def test_pull_partial_import_missing(tmp_dir, dvc, local_workspace):
|
||||
dvc.imp_url("remote://workspace/file", os.fspath(dst), no_download=True)
|
||||
|
||||
(local_workspace / "file").unlink()
|
||||
with pytest.raises(CheckoutError):
|
||||
with pytest.raises(CheckoutError) as exc:
|
||||
dvc.pull("file")
|
||||
assert exc.value.result == empty_pull | {"failed": ["file"]}
|
||||
assert not dst.exists()
|
||||
|
||||
|
||||
@@ -290,8 +313,9 @@ def test_pull_partial_import_modified(tmp_dir, dvc, local_workspace):
|
||||
dvc.imp_url("remote://workspace/file", os.fspath(dst), no_download=True)
|
||||
|
||||
local_workspace.gen("file", "updated file content")
|
||||
with pytest.raises(CheckoutError):
|
||||
with pytest.raises(CheckoutError) as exc:
|
||||
dvc.pull("file")
|
||||
assert exc.value.result == empty_pull | {"failed": ["file"]}
|
||||
assert not dst.exists()
|
||||
|
||||
|
||||
@@ -309,7 +333,10 @@ def test_pull_external_dvc_imports_mixed(tmp_dir, dvc, scm, erepo_dir, local_rem
|
||||
|
||||
clean(["foo", "bar"], dvc)
|
||||
|
||||
assert dvc.pull()["fetched"] == 2
|
||||
assert dvc.pull() == empty_pull | {
|
||||
"added": ["bar", "foo"],
|
||||
"stats": empty_stats | {"fetched": 2, "added": 2},
|
||||
}
|
||||
assert (tmp_dir / "foo").read_text() == "foo"
|
||||
assert (tmp_dir / "bar").read_text() == "bar"
|
||||
|
||||
@@ -330,34 +357,44 @@ def recurse_list_dir(d):
|
||||
|
||||
|
||||
def test_dvc_pull_pipeline_stages(tmp_dir, dvc, run_copy, local_remote):
|
||||
(stage0,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
stage1 = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
stage2 = run_copy("bar", "foobar", name="copy-bar-foobar")
|
||||
(stage0,) = tmp_dir.dvc_gen("bar", "bar")
|
||||
stage1 = run_copy("bar", "foo", name="copy-bar-foo")
|
||||
stage2 = run_copy("foo", "foobar", name="copy-foo-foobar")
|
||||
dvc.push()
|
||||
|
||||
outs = ["foo", "bar", "foobar"]
|
||||
outs = ["bar", "foo", "foobar"]
|
||||
|
||||
clean(outs, dvc)
|
||||
dvc.pull()
|
||||
assert dvc.pull() == empty_pull | {
|
||||
"added": outs,
|
||||
"stats": empty_stats | {"fetched": 1, "added": 3},
|
||||
}
|
||||
assert all((tmp_dir / file).exists() for file in outs)
|
||||
|
||||
for out, stage in zip(outs, [stage0, stage1, stage2]):
|
||||
for target in [stage.addressing, out]:
|
||||
clean(outs, dvc)
|
||||
stats = dvc.pull([target])
|
||||
assert stats["fetched"] == 1
|
||||
assert stats["added"] == [out]
|
||||
assert stats == empty_pull | {
|
||||
"added": [out],
|
||||
"stats": empty_stats | {"fetched": 1, "added": 1},
|
||||
}
|
||||
assert os.path.exists(out)
|
||||
assert not any(os.path.exists(out) for out in set(outs) - {out})
|
||||
|
||||
clean(outs, dvc)
|
||||
stats = dvc.pull([stage2.addressing], with_deps=True)
|
||||
assert len(stats["added"]) == 3
|
||||
assert set(stats["added"]) == set(outs)
|
||||
assert stats == empty_pull | {
|
||||
"added": outs,
|
||||
"stats": empty_stats | {"fetched": 1, "added": 3},
|
||||
}
|
||||
|
||||
clean(outs, dvc)
|
||||
stats = dvc.pull([os.curdir], recursive=True)
|
||||
assert set(stats["added"]) == set(outs)
|
||||
assert stats == empty_pull | {
|
||||
"added": outs,
|
||||
"stats": empty_stats | {"fetched": 1, "added": 3},
|
||||
}
|
||||
|
||||
|
||||
def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, local_remote):
|
||||
@@ -374,7 +411,7 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, local_remote):
|
||||
|
||||
remove(dvc.stage_cache.cache_dir)
|
||||
|
||||
dvc.push()
|
||||
assert dvc.push() == 3
|
||||
|
||||
outs = ["foo", "lorem", "ipsum", "baz", "lorem2"]
|
||||
|
||||
@@ -382,20 +419,26 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, local_remote):
|
||||
assert len(recurse_list_dir(path)) == 3
|
||||
|
||||
clean(outs, dvc)
|
||||
assert set(dvc.pull(["dvc.yaml"])["added"]) == {"lorem2", "baz"}
|
||||
assert dvc.pull(["dvc.yaml"]) == empty_pull | {
|
||||
"added": ["baz", "lorem2"],
|
||||
"stats": empty_stats | {"fetched": 2, "added": 2},
|
||||
}
|
||||
|
||||
clean(outs, dvc)
|
||||
assert set(dvc.pull()["added"]) == set(outs)
|
||||
assert dvc.pull() == empty_pull | {
|
||||
"added": ["baz", "foo", "ipsum", "lorem", "lorem2"],
|
||||
"stats": empty_stats | {"fetched": 3, "added": 5},
|
||||
}
|
||||
|
||||
# clean everything in remote and push
|
||||
from dvc.testing.tmp_dir import TmpDir
|
||||
|
||||
clean(TmpDir(path).iterdir())
|
||||
dvc.push(["dvc.yaml:copy-ipsum-baz"])
|
||||
assert dvc.push(["dvc.yaml:copy-ipsum-baz"]) == 1
|
||||
assert len(recurse_list_dir(path)) == 1
|
||||
|
||||
clean(TmpDir(path).iterdir())
|
||||
dvc.push(["dvc.yaml"])
|
||||
assert dvc.push(["dvc.yaml"]) == 2
|
||||
assert len(recurse_list_dir(path)) == 2
|
||||
|
||||
with pytest.raises(StageNotFound):
|
||||
@@ -440,23 +483,36 @@ def test_fetch_stats(tmp_dir, dvc, fs, msg, capsys, local_remote):
|
||||
|
||||
|
||||
def test_pull_stats(tmp_dir, dvc, capsys, local_remote):
|
||||
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
|
||||
tmp_dir.dvc_gen(
|
||||
{
|
||||
"foo": "foo",
|
||||
"bar": "bar",
|
||||
"lorem": "lorem",
|
||||
"dir": {"file": "file"},
|
||||
"ipsum": "ipsum",
|
||||
"dolor": "dolor",
|
||||
}
|
||||
)
|
||||
dvc.push()
|
||||
clean(["foo", "bar"], dvc)
|
||||
clean(["foo", "bar", "dir", "lorem"], dvc)
|
||||
|
||||
(tmp_dir / "ipsum.dvc").unlink()
|
||||
(tmp_dir / "bar").write_text("foobar")
|
||||
|
||||
assert main(["pull", "--force"]) == 0
|
||||
|
||||
out, _ = capsys.readouterr()
|
||||
assert "M\tbar".expandtabs() in out
|
||||
assert "A\tfoo".expandtabs() in out
|
||||
assert "2 files fetched" in out
|
||||
assert "1 file added" in out
|
||||
assert "1 file modified" in out
|
||||
assert out.splitlines() == [
|
||||
"M\tbar".expandtabs(),
|
||||
"A\tdir".expandtabs() + os.sep,
|
||||
"A\tfoo".expandtabs(),
|
||||
"A\tlorem".expandtabs(),
|
||||
"D\tipsum".expandtabs(),
|
||||
"6 files fetched, 1 file modified, 3 files added and 1 file deleted",
|
||||
]
|
||||
|
||||
main(["pull"])
|
||||
out, _ = capsys.readouterr()
|
||||
assert "Everything is up to date." in out
|
||||
assert out == "Everything is up to date.\n"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -475,26 +531,32 @@ def test_push_pull_all(tmp_dir, scm, dvc, local_remote, key, expected):
|
||||
assert dvc.push(**{key: True}) == expected
|
||||
|
||||
clean(["foo", "bar", "baz"], dvc)
|
||||
assert dvc.pull(**{key: True})["fetched"] == expected
|
||||
assert dvc.pull(**{key: True}) == empty_pull | {
|
||||
"added": ["bar", "foo"],
|
||||
"stats": empty_stats | {"fetched": expected, "added": 2},
|
||||
}
|
||||
|
||||
|
||||
def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, local_remote):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
|
||||
dvc.push("copy-foo-bar")
|
||||
assert dvc.push("copy-foo-bar") == 1
|
||||
assert len(recurse_list_dir(local_remote.url)) == 1
|
||||
# pushing everything so as we can check pull/fetch only downloads
|
||||
# from specified targets
|
||||
dvc.push()
|
||||
assert dvc.push() == 0
|
||||
clean(["foo", "bar"], dvc)
|
||||
|
||||
dvc.pull("copy-foo-bar")
|
||||
assert dvc.pull("copy-foo-bar") == empty_pull | {
|
||||
"added": ["bar"],
|
||||
"stats": empty_stats | {"fetched": 1, "added": 1},
|
||||
}
|
||||
assert (tmp_dir / "bar").exists()
|
||||
assert len(recurse_list_dir(dvc.cache.local.path)) == 1
|
||||
clean(["bar"], dvc)
|
||||
|
||||
dvc.fetch("copy-foo-bar")
|
||||
assert dvc.fetch("copy-foo-bar") == 1
|
||||
assert len(recurse_list_dir(dvc.cache.local.path)) == 1
|
||||
|
||||
|
||||
@@ -505,7 +567,10 @@ def test_pull_partial(tmp_dir, dvc, local_remote):
|
||||
clean(["foo"], dvc)
|
||||
|
||||
stats = dvc.pull(os.path.join("foo", "bar"))
|
||||
assert stats["fetched"] == 2
|
||||
assert stats == empty_pull | {
|
||||
"added": [os.path.join("foo", "")],
|
||||
"stats": empty_stats | {"fetched": 2, "added": 1},
|
||||
}
|
||||
assert (tmp_dir / "foo").read_text() == {"bar": {"baz": "baz"}}
|
||||
|
||||
|
||||
@@ -540,7 +605,10 @@ def test_output_remote(tmp_dir, dvc, make_remote):
|
||||
|
||||
clean(["foo", "bar", "data"], dvc)
|
||||
|
||||
dvc.pull()
|
||||
assert dvc.pull() == empty_pull | {
|
||||
"added": ["data" + os.sep, "bar", "foo"],
|
||||
"stats": empty_stats | {"fetched": 5, "added": 4},
|
||||
}
|
||||
|
||||
assert set(dvc.cache.local.all()) == {
|
||||
"37b51d194a7513e45b56f6524f2d51f2",
|
||||
@@ -573,7 +641,10 @@ def test_target_remote(tmp_dir, dvc, make_remote):
|
||||
|
||||
clean(["foo", "data"], dvc)
|
||||
|
||||
dvc.pull(remote="myremote")
|
||||
assert dvc.pull(remote="myremote") == empty_pull | {
|
||||
"added": ["data" + os.sep, "foo"],
|
||||
"stats": empty_stats | {"fetched": 4, "added": 3},
|
||||
}
|
||||
|
||||
assert set(dvc.cache.local.all()) == {
|
||||
"acbd18db4cc2f85cedef654fccc4a4d8",
|
||||
@@ -630,7 +701,10 @@ def test_output_target_remote(tmp_dir, dvc, make_remote):
|
||||
clean(["foo", "bar", "data"], dvc)
|
||||
|
||||
# pull foo and data from for_foo remote
|
||||
dvc.pull(remote="for_foo", allow_missing=True)
|
||||
assert dvc.pull(remote="for_foo", allow_missing=True) == empty_pull | {
|
||||
"added": ["data" + os.sep, "foo"],
|
||||
"stats": empty_stats | {"fetched": 4, "added": 3},
|
||||
}
|
||||
|
||||
assert set(dvc.cache.local.all()) == expected
|
||||
|
||||
@@ -638,15 +712,19 @@ def test_output_target_remote(tmp_dir, dvc, make_remote):
|
||||
def test_pull_allow_missing(tmp_dir, dvc, local_remote):
|
||||
dvc.stage.add(name="bar", outs=["bar"], cmd="echo bar > bar")
|
||||
|
||||
with pytest.raises(CheckoutError):
|
||||
with pytest.raises(CheckoutError) as exc:
|
||||
dvc.pull()
|
||||
assert exc.value.result == empty_pull | {"failed": ["bar"]}
|
||||
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
dvc.push()
|
||||
clean(["foo"], dvc)
|
||||
|
||||
stats = dvc.pull(allow_missing=True)
|
||||
assert stats["fetched"] == 1
|
||||
assert stats == empty_pull | {
|
||||
"added": ["foo"],
|
||||
"stats": empty_stats | {"fetched": 1, "added": 1},
|
||||
}
|
||||
|
||||
|
||||
def test_pull_granular_excluding_import_that_cannot_be_pulled(
|
||||
@@ -668,11 +746,9 @@ def test_pull_granular_excluding_import_that_cannot_be_pulled(
|
||||
shutil.rmtree("dir")
|
||||
dvc.cache.local.clear()
|
||||
|
||||
assert dvc.pull(stage.addressing) == {
|
||||
assert dvc.pull(stage.addressing) == empty_pull | {
|
||||
"added": [join("dir", "")],
|
||||
"deleted": [],
|
||||
"modified": [],
|
||||
"fetched": 3,
|
||||
"stats": empty_stats | {"added": 2, "fetched": 3},
|
||||
}
|
||||
|
||||
with pytest.raises(CloneError, match="SCM error"):
|
||||
@@ -704,8 +780,8 @@ def test_loads_single_file(tmp_dir, dvc, local_remote, mocker):
|
||||
assert dvc.pull("foo.dvc") == {
|
||||
"added": ["foo"],
|
||||
"deleted": [],
|
||||
"fetched": 1,
|
||||
"modified": [],
|
||||
"stats": {"added": 1, "deleted": 0, "modified": 0, "fetched": 1},
|
||||
}
|
||||
spy.assert_called_with(foo_dvcfile)
|
||||
assert (tmp_dir / "foo").exists()
|
||||
|
||||
@@ -20,7 +20,12 @@ def test_push_pull(tmp_dir, dvc, erepo_dir, run_copy, local_remote):
|
||||
erepo_dir.add_remote(config=local_remote.config)
|
||||
with erepo_dir.chdir():
|
||||
assert not os.path.exists(erepo_dir.dvc.stage_cache.cache_dir)
|
||||
assert erepo_dir.dvc.pull(run_cache=True)["fetched"] == 0
|
||||
assert erepo_dir.dvc.pull(run_cache=True) == {
|
||||
"added": [],
|
||||
"deleted": [],
|
||||
"modified": [],
|
||||
"stats": {"fetched": 0, "added": 0, "deleted": 0, "modified": 0},
|
||||
}
|
||||
assert os.listdir(erepo_dir.dvc.stage_cache.cache_dir)
|
||||
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import os
|
||||
import shutil
|
||||
from os.path import join
|
||||
|
||||
@@ -180,7 +181,12 @@ def test_partial_checkout_and_update(M, tmp_dir, dvc, remote):
|
||||
dvc.cache.local.clear()
|
||||
shutil.rmtree("dir")
|
||||
|
||||
assert dvc.pull("dir/subdir") == M.dict(added=[join("dir", "")], fetched=11)
|
||||
assert dvc.pull("dir/subdir") == {
|
||||
"added": ["dir" + os.sep],
|
||||
"deleted": [],
|
||||
"modified": [],
|
||||
"stats": {"added": 10, "deleted": 0, "fetched": 11, "modified": 0},
|
||||
}
|
||||
assert (tmp_dir / "dir").read_text() == {"subdir": dir1}
|
||||
|
||||
tmp_dir.gen({"dir": {"subdir": {"file": "file"}}})
|
||||
|
||||
Reference in New Issue
Block a user