mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
checkout: fix partial file checkouts in the same directory (#5163)
* Fix partial file checkouts in the same directory * unittest => pytest * Try to rationalize changed() to take filter_info * Use path.read_text() to verify the directory structure * Try to use specialized hash, fall back to general one if doesn't exist * Cover subdirectories in hash specialization * coverage: add a test for empty subdirectories * Move out the dir_info filtering block * hash_origin => path_info * cleanup * Move hashing functionality after the existence checks * Simplify it a bit * Update dvc/cache/base.py Co-authored-by: Ruslan Kuprieiev <[email protected]> * Apply suggestions from code review Co-authored-by: Ruslan Kuprieiev <[email protected]> * Apply suggestions * simplify tests * change filenames * Some visual changes for consistency Co-authored-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
co-authored by
Ruslan Kuprieiev
parent
9b73a745b4
commit
6fb84f641c
Vendored
+40
-15
@@ -88,7 +88,29 @@ class CloudCache:
|
||||
|
||||
return DirInfo.from_list(d)
|
||||
|
||||
def changed(self, path_info, hash_info):
|
||||
def _filter_hash_info(self, hash_info, path_info, filter_info):
|
||||
if not filter_info:
|
||||
return hash_info
|
||||
|
||||
dir_info = self.get_dir_cache(hash_info)
|
||||
hash_key = filter_info.relative_to(path_info).parts
|
||||
|
||||
# Check whether it is a file that exists on the trie
|
||||
hi = dir_info.trie.get(hash_key)
|
||||
if hi:
|
||||
return hi
|
||||
|
||||
depth = len(hash_key)
|
||||
filtered_dir_info = DirInfo()
|
||||
try:
|
||||
for key, value in dir_info.trie.items(hash_key):
|
||||
filtered_dir_info.trie[key[depth:]] = value
|
||||
except KeyError:
|
||||
return None
|
||||
|
||||
return self._get_dir_info_hash(filtered_dir_info)[0]
|
||||
|
||||
def changed(self, path_info, hash_info, filter_info=None):
|
||||
"""Checks if data has changed.
|
||||
|
||||
A file is considered changed if:
|
||||
@@ -100,40 +122,41 @@ class CloudCache:
|
||||
Args:
|
||||
path_info: dict with path information.
|
||||
hash: expected hash value for this data.
|
||||
filter_info: an optional argument to target a specific path.
|
||||
|
||||
Returns:
|
||||
bool: True if data has changed, False otherwise.
|
||||
"""
|
||||
|
||||
path = filter_info or path_info
|
||||
logger.trace(
|
||||
"checking if '%s'('%s') has changed.", path_info, hash_info
|
||||
)
|
||||
|
||||
if not self.tree.exists(path_info):
|
||||
logger.debug("'%s' doesn't exist.", path_info)
|
||||
if not self.tree.exists(path):
|
||||
logger.debug("'%s' doesn't exist.", path)
|
||||
return True
|
||||
|
||||
if not hash_info:
|
||||
logger.debug("hash value for '%s' is missing.", path_info)
|
||||
hi = self._filter_hash_info(hash_info, path_info, filter_info)
|
||||
if not hi:
|
||||
logger.debug("hash value for '%s' is missing.", path)
|
||||
return True
|
||||
|
||||
if self.changed_cache(hash_info):
|
||||
logger.debug(
|
||||
"cache for '%s'('%s') has changed.", path_info, hash_info
|
||||
)
|
||||
if self.changed_cache(hi):
|
||||
logger.debug("cache for '%s'('%s') has changed.", path, hi)
|
||||
return True
|
||||
|
||||
actual = self.tree.get_hash(path_info)
|
||||
if hash_info != actual:
|
||||
actual = self.tree.get_hash(path)
|
||||
if hi != actual:
|
||||
logger.debug(
|
||||
"hash value '%s' for '%s' has changed (actual '%s').",
|
||||
hash_info,
|
||||
hi,
|
||||
actual,
|
||||
path_info,
|
||||
path,
|
||||
)
|
||||
return True
|
||||
|
||||
logger.trace("'%s' hasn't changed.", path_info)
|
||||
logger.trace("'%s' hasn't changed.", path)
|
||||
return False
|
||||
|
||||
def link(self, from_info, to_info):
|
||||
@@ -519,7 +542,9 @@ class CloudCache:
|
||||
self.safe_remove(path_info, force=force)
|
||||
failed = path_info
|
||||
|
||||
elif not relink and not self.changed(path_info, hash_info):
|
||||
elif not relink and not self.changed(
|
||||
path_info, hash_info, filter_info=filter_info
|
||||
):
|
||||
logger.trace("Data '%s' didn't change.", path_info)
|
||||
skip = True
|
||||
|
||||
|
||||
@@ -880,3 +880,97 @@ def test_checkout_executable(tmp_dir, dvc):
|
||||
assert not isexec
|
||||
else:
|
||||
assert isexec
|
||||
|
||||
|
||||
def test_checkout_partial(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen(
|
||||
{"data": {"foo": "foo", "bar": "bar", "sub_dir": {"baz": "baz"}}}
|
||||
)
|
||||
|
||||
data_dir = tmp_dir / "data"
|
||||
shutil.rmtree(data_dir)
|
||||
|
||||
dvc.checkout(str(data_dir / "foo"))
|
||||
assert data_dir.read_text() == {"foo": "foo"}
|
||||
|
||||
dvc.checkout(str(data_dir / "sub_dir" / "baz"))
|
||||
assert data_dir.read_text() == {"foo": "foo", "sub_dir": {"baz": "baz"}}
|
||||
|
||||
dvc.checkout(str(data_dir / "bar"))
|
||||
assert data_dir.read_text() == {
|
||||
"foo": "foo",
|
||||
"bar": "bar",
|
||||
"sub_dir": {"baz": "baz"},
|
||||
}
|
||||
|
||||
|
||||
def test_checkout_partial_unchanged(tmp_dir, dvc):
|
||||
original_dir_shape = {
|
||||
"foo": "foo",
|
||||
"bar": "bar",
|
||||
"sub_dir": {"baz": "baz"},
|
||||
"empty_sub_dir": {},
|
||||
}
|
||||
tmp_dir.dvc_gen({"data": original_dir_shape})
|
||||
|
||||
data_dir = tmp_dir / "data"
|
||||
sub_dir = data_dir / "sub_dir"
|
||||
foo = data_dir / "foo"
|
||||
bar = data_dir / "bar"
|
||||
sub_dir_file = sub_dir / "baz"
|
||||
|
||||
# Nothing changed, nothing added/deleted/modified
|
||||
stats = dvc.checkout(str(bar))
|
||||
assert not any(stats.values())
|
||||
|
||||
# Irrelevant file changed, still nothing added/deleted/modified
|
||||
foo.unlink()
|
||||
stats = dvc.checkout(str(bar))
|
||||
assert not any(stats.values())
|
||||
|
||||
# Relevant change, one modified
|
||||
bar.unlink()
|
||||
stats = dvc.checkout(str(bar))
|
||||
assert len(stats["modified"]) == 1
|
||||
|
||||
# No changes inside data/sub
|
||||
stats = dvc.checkout(str(sub_dir))
|
||||
assert not any(stats.values())
|
||||
|
||||
# Relevant change, one modified
|
||||
sub_dir_file.unlink()
|
||||
stats = dvc.checkout(str(sub_dir))
|
||||
assert len(stats["modified"]) == 1
|
||||
|
||||
stats = dvc.checkout(str(data_dir / "empty_sub_dir"))
|
||||
assert not any(stats.values())
|
||||
|
||||
dvc.checkout(str(data_dir))
|
||||
|
||||
# Everything is in place, no action taken
|
||||
stats = dvc.checkout(str(data_dir))
|
||||
assert not any(stats.values())
|
||||
|
||||
|
||||
def test_checkout_partial_subdir(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen(
|
||||
{"data": {"foo": "foo", "sub_dir": {"bar": "bar", "baz": "baz"}}}
|
||||
)
|
||||
|
||||
data_dir = tmp_dir / "data"
|
||||
sub_dir = data_dir / "sub_dir"
|
||||
sub_dir_bar = sub_dir / "baz"
|
||||
|
||||
shutil.rmtree(sub_dir)
|
||||
dvc.checkout(str(sub_dir))
|
||||
assert data_dir.read_text() == {
|
||||
"foo": "foo",
|
||||
"sub_dir": {"bar": "bar", "baz": "baz"},
|
||||
}
|
||||
|
||||
sub_dir_bar.unlink()
|
||||
dvc.checkout(str(sub_dir_bar))
|
||||
assert data_dir.read_text() == {
|
||||
"foo": "foo",
|
||||
"sub_dir": {"bar": "bar", "baz": "baz"},
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user