From 6fb84f641c94fe1c2cbb9a75d8c880f7d67c5029 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Wed, 6 Jan 2021 15:21:00 +0300 Subject: [PATCH] checkout: fix partial file checkouts in the same directory (#5163) * Fix partial file checkouts in the same directory * unittest => pytest * Try to rationalize changed() to take filter_info * Use path.read_text() to verify the directory structure * Try to use specialized hash, fall back to general one if doesn't exist * Cover subdirectories in hash specialization * coverage: add a test for empty subdirectories * Move out the dir_info filtering block * hash_origin => path_info * cleanup * Move hashing functionality after the existence checks * Simplify it a bit * Update dvc/cache/base.py Co-authored-by: Ruslan Kuprieiev * Apply suggestions from code review Co-authored-by: Ruslan Kuprieiev * Apply suggestions * simplify tests * change filenames * Some visual changes for consistency Co-authored-by: Ruslan Kuprieiev --- dvc/cache/base.py | 55 ++++++++++++++++------ tests/func/test_checkout.py | 94 +++++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 15 deletions(-) diff --git a/dvc/cache/base.py b/dvc/cache/base.py index bf1a37d6c..e0d24c070 100644 --- a/dvc/cache/base.py +++ b/dvc/cache/base.py @@ -88,7 +88,29 @@ class CloudCache: return DirInfo.from_list(d) - def changed(self, path_info, hash_info): + def _filter_hash_info(self, hash_info, path_info, filter_info): + if not filter_info: + return hash_info + + dir_info = self.get_dir_cache(hash_info) + hash_key = filter_info.relative_to(path_info).parts + + # Check whether it is a file that exists on the trie + hi = dir_info.trie.get(hash_key) + if hi: + return hi + + depth = len(hash_key) + filtered_dir_info = DirInfo() + try: + for key, value in dir_info.trie.items(hash_key): + filtered_dir_info.trie[key[depth:]] = value + except KeyError: + return None + + return self._get_dir_info_hash(filtered_dir_info)[0] + + def changed(self, path_info, hash_info, filter_info=None): """Checks if data has changed. A file is considered changed if: @@ -100,40 +122,41 @@ class CloudCache: Args: path_info: dict with path information. hash: expected hash value for this data. + filter_info: an optional argument to target a specific path. Returns: bool: True if data has changed, False otherwise. """ + path = filter_info or path_info logger.trace( "checking if '%s'('%s') has changed.", path_info, hash_info ) - if not self.tree.exists(path_info): - logger.debug("'%s' doesn't exist.", path_info) + if not self.tree.exists(path): + logger.debug("'%s' doesn't exist.", path) return True - if not hash_info: - logger.debug("hash value for '%s' is missing.", path_info) + hi = self._filter_hash_info(hash_info, path_info, filter_info) + if not hi: + logger.debug("hash value for '%s' is missing.", path) return True - if self.changed_cache(hash_info): - logger.debug( - "cache for '%s'('%s') has changed.", path_info, hash_info - ) + if self.changed_cache(hi): + logger.debug("cache for '%s'('%s') has changed.", path, hi) return True - actual = self.tree.get_hash(path_info) - if hash_info != actual: + actual = self.tree.get_hash(path) + if hi != actual: logger.debug( "hash value '%s' for '%s' has changed (actual '%s').", - hash_info, + hi, actual, - path_info, + path, ) return True - logger.trace("'%s' hasn't changed.", path_info) + logger.trace("'%s' hasn't changed.", path) return False def link(self, from_info, to_info): @@ -519,7 +542,9 @@ class CloudCache: self.safe_remove(path_info, force=force) failed = path_info - elif not relink and not self.changed(path_info, hash_info): + elif not relink and not self.changed( + path_info, hash_info, filter_info=filter_info + ): logger.trace("Data '%s' didn't change.", path_info) skip = True diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index 712b26df6..95dcfff4d 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -880,3 +880,97 @@ def test_checkout_executable(tmp_dir, dvc): assert not isexec else: assert isexec + + +def test_checkout_partial(tmp_dir, dvc): + tmp_dir.dvc_gen( + {"data": {"foo": "foo", "bar": "bar", "sub_dir": {"baz": "baz"}}} + ) + + data_dir = tmp_dir / "data" + shutil.rmtree(data_dir) + + dvc.checkout(str(data_dir / "foo")) + assert data_dir.read_text() == {"foo": "foo"} + + dvc.checkout(str(data_dir / "sub_dir" / "baz")) + assert data_dir.read_text() == {"foo": "foo", "sub_dir": {"baz": "baz"}} + + dvc.checkout(str(data_dir / "bar")) + assert data_dir.read_text() == { + "foo": "foo", + "bar": "bar", + "sub_dir": {"baz": "baz"}, + } + + +def test_checkout_partial_unchanged(tmp_dir, dvc): + original_dir_shape = { + "foo": "foo", + "bar": "bar", + "sub_dir": {"baz": "baz"}, + "empty_sub_dir": {}, + } + tmp_dir.dvc_gen({"data": original_dir_shape}) + + data_dir = tmp_dir / "data" + sub_dir = data_dir / "sub_dir" + foo = data_dir / "foo" + bar = data_dir / "bar" + sub_dir_file = sub_dir / "baz" + + # Nothing changed, nothing added/deleted/modified + stats = dvc.checkout(str(bar)) + assert not any(stats.values()) + + # Irrelevant file changed, still nothing added/deleted/modified + foo.unlink() + stats = dvc.checkout(str(bar)) + assert not any(stats.values()) + + # Relevant change, one modified + bar.unlink() + stats = dvc.checkout(str(bar)) + assert len(stats["modified"]) == 1 + + # No changes inside data/sub + stats = dvc.checkout(str(sub_dir)) + assert not any(stats.values()) + + # Relevant change, one modified + sub_dir_file.unlink() + stats = dvc.checkout(str(sub_dir)) + assert len(stats["modified"]) == 1 + + stats = dvc.checkout(str(data_dir / "empty_sub_dir")) + assert not any(stats.values()) + + dvc.checkout(str(data_dir)) + + # Everything is in place, no action taken + stats = dvc.checkout(str(data_dir)) + assert not any(stats.values()) + + +def test_checkout_partial_subdir(tmp_dir, dvc): + tmp_dir.dvc_gen( + {"data": {"foo": "foo", "sub_dir": {"bar": "bar", "baz": "baz"}}} + ) + + data_dir = tmp_dir / "data" + sub_dir = data_dir / "sub_dir" + sub_dir_bar = sub_dir / "baz" + + shutil.rmtree(sub_dir) + dvc.checkout(str(sub_dir)) + assert data_dir.read_text() == { + "foo": "foo", + "sub_dir": {"bar": "bar", "baz": "baz"}, + } + + sub_dir_bar.unlink() + dvc.checkout(str(sub_dir_bar)) + assert data_dir.read_text() == { + "foo": "foo", + "sub_dir": {"bar": "bar", "baz": "baz"}, + }