checkout: fix partial file checkouts in the same directory (#5163)

* Fix partial file checkouts in the same directory

* unittest => pytest

* Try to rationalize changed() to take filter_info

* Use path.read_text() to verify the directory structure

* Try to use specialized hash, fall back to general one if doesn't exist

* Cover subdirectories in hash specialization

* coverage: add a test for empty subdirectories

* Move out the dir_info filtering block

* hash_origin => path_info

* cleanup

* Move hashing functionality after the existence checks

* Simplify it a bit

* Update dvc/cache/base.py

Co-authored-by: Ruslan Kuprieiev <[email protected]>

* Apply suggestions from code review

Co-authored-by: Ruslan Kuprieiev <[email protected]>

* Apply suggestions

* simplify tests

* change filenames

* Some visual changes for consistency

Co-authored-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Batuhan Taskaya
2021-01-06 14:21:00 +02:00
committed by GitHub
co-authored by Ruslan Kuprieiev
parent 9b73a745b4
commit 6fb84f641c
2 changed files with 134 additions and 15 deletions
+40 -15
View File
@@ -88,7 +88,29 @@ class CloudCache:
return DirInfo.from_list(d)
def changed(self, path_info, hash_info):
def _filter_hash_info(self, hash_info, path_info, filter_info):
if not filter_info:
return hash_info
dir_info = self.get_dir_cache(hash_info)
hash_key = filter_info.relative_to(path_info).parts
# Check whether it is a file that exists on the trie
hi = dir_info.trie.get(hash_key)
if hi:
return hi
depth = len(hash_key)
filtered_dir_info = DirInfo()
try:
for key, value in dir_info.trie.items(hash_key):
filtered_dir_info.trie[key[depth:]] = value
except KeyError:
return None
return self._get_dir_info_hash(filtered_dir_info)[0]
def changed(self, path_info, hash_info, filter_info=None):
"""Checks if data has changed.
A file is considered changed if:
@@ -100,40 +122,41 @@ class CloudCache:
Args:
path_info: dict with path information.
hash: expected hash value for this data.
filter_info: an optional argument to target a specific path.
Returns:
bool: True if data has changed, False otherwise.
"""
path = filter_info or path_info
logger.trace(
"checking if '%s'('%s') has changed.", path_info, hash_info
)
if not self.tree.exists(path_info):
logger.debug("'%s' doesn't exist.", path_info)
if not self.tree.exists(path):
logger.debug("'%s' doesn't exist.", path)
return True
if not hash_info:
logger.debug("hash value for '%s' is missing.", path_info)
hi = self._filter_hash_info(hash_info, path_info, filter_info)
if not hi:
logger.debug("hash value for '%s' is missing.", path)
return True
if self.changed_cache(hash_info):
logger.debug(
"cache for '%s'('%s') has changed.", path_info, hash_info
)
if self.changed_cache(hi):
logger.debug("cache for '%s'('%s') has changed.", path, hi)
return True
actual = self.tree.get_hash(path_info)
if hash_info != actual:
actual = self.tree.get_hash(path)
if hi != actual:
logger.debug(
"hash value '%s' for '%s' has changed (actual '%s').",
hash_info,
hi,
actual,
path_info,
path,
)
return True
logger.trace("'%s' hasn't changed.", path_info)
logger.trace("'%s' hasn't changed.", path)
return False
def link(self, from_info, to_info):
@@ -519,7 +542,9 @@ class CloudCache:
self.safe_remove(path_info, force=force)
failed = path_info
elif not relink and not self.changed(path_info, hash_info):
elif not relink and not self.changed(
path_info, hash_info, filter_info=filter_info
):
logger.trace("Data '%s' didn't change.", path_info)
skip = True
+94
View File
@@ -880,3 +880,97 @@ def test_checkout_executable(tmp_dir, dvc):
assert not isexec
else:
assert isexec
def test_checkout_partial(tmp_dir, dvc):
tmp_dir.dvc_gen(
{"data": {"foo": "foo", "bar": "bar", "sub_dir": {"baz": "baz"}}}
)
data_dir = tmp_dir / "data"
shutil.rmtree(data_dir)
dvc.checkout(str(data_dir / "foo"))
assert data_dir.read_text() == {"foo": "foo"}
dvc.checkout(str(data_dir / "sub_dir" / "baz"))
assert data_dir.read_text() == {"foo": "foo", "sub_dir": {"baz": "baz"}}
dvc.checkout(str(data_dir / "bar"))
assert data_dir.read_text() == {
"foo": "foo",
"bar": "bar",
"sub_dir": {"baz": "baz"},
}
def test_checkout_partial_unchanged(tmp_dir, dvc):
original_dir_shape = {
"foo": "foo",
"bar": "bar",
"sub_dir": {"baz": "baz"},
"empty_sub_dir": {},
}
tmp_dir.dvc_gen({"data": original_dir_shape})
data_dir = tmp_dir / "data"
sub_dir = data_dir / "sub_dir"
foo = data_dir / "foo"
bar = data_dir / "bar"
sub_dir_file = sub_dir / "baz"
# Nothing changed, nothing added/deleted/modified
stats = dvc.checkout(str(bar))
assert not any(stats.values())
# Irrelevant file changed, still nothing added/deleted/modified
foo.unlink()
stats = dvc.checkout(str(bar))
assert not any(stats.values())
# Relevant change, one modified
bar.unlink()
stats = dvc.checkout(str(bar))
assert len(stats["modified"]) == 1
# No changes inside data/sub
stats = dvc.checkout(str(sub_dir))
assert not any(stats.values())
# Relevant change, one modified
sub_dir_file.unlink()
stats = dvc.checkout(str(sub_dir))
assert len(stats["modified"]) == 1
stats = dvc.checkout(str(data_dir / "empty_sub_dir"))
assert not any(stats.values())
dvc.checkout(str(data_dir))
# Everything is in place, no action taken
stats = dvc.checkout(str(data_dir))
assert not any(stats.values())
def test_checkout_partial_subdir(tmp_dir, dvc):
tmp_dir.dvc_gen(
{"data": {"foo": "foo", "sub_dir": {"bar": "bar", "baz": "baz"}}}
)
data_dir = tmp_dir / "data"
sub_dir = data_dir / "sub_dir"
sub_dir_bar = sub_dir / "baz"
shutil.rmtree(sub_dir)
dvc.checkout(str(sub_dir))
assert data_dir.read_text() == {
"foo": "foo",
"sub_dir": {"bar": "bar", "baz": "baz"},
}
sub_dir_bar.unlink()
dvc.checkout(str(sub_dir_bar))
assert data_dir.read_text() == {
"foo": "foo",
"sub_dir": {"bar": "bar", "baz": "baz"},
}