diff --git a/dvc/data/gc.py b/dvc/data/gc.py new file mode 100644 index 000000000..338f3ccfa --- /dev/null +++ b/dvc/data/gc.py @@ -0,0 +1,40 @@ +def gc(odb, used, jobs=None, cache_odb=None, shallow=True): + from dvc.data.tree import Tree + from dvc.objects.errors import ObjectDBPermissionError + + if odb.read_only: + raise ObjectDBPermissionError("Cannot gc read-only ODB") + if not cache_odb: + cache_odb = odb + used_hashes = set() + for hash_info in used: + used_hashes.add(hash_info.value) + if hash_info.isdir and not shallow: + tree = Tree.load(cache_odb, hash_info) + used_hashes.update( + entry_obj.hash_info.value for _, entry_obj in tree + ) + + def _is_dir_hash(_hash): + from dvc.hash_info import HASH_DIR_SUFFIX + + return _hash.endswith(HASH_DIR_SUFFIX) + + removed = False + # hashes must be sorted to ensure we always remove .dir files first + for hash_ in sorted( + odb.all(jobs, odb.fs_path), + key=_is_dir_hash, + reverse=True, + ): + if hash_ in used_hashes: + continue + fs_path = odb.hash_to_path(hash_) + if _is_dir_hash(hash_): + # backward compatibility + # pylint: disable=protected-access + odb._remove_unpacked_dir(hash_) + odb.fs.remove(fs_path) + removed = True + + return removed diff --git a/dvc/objects/db.py b/dvc/objects/db.py index 97b2dc7a2..1d5eee565 100644 --- a/dvc/objects/db.py +++ b/dvc/objects/db.py @@ -356,46 +356,6 @@ class ObjectDB: def _remove_unpacked_dir(self, hash_): pass - def gc(self, used, jobs=None, cache_odb=None, shallow=True): - from dvc.data.tree import Tree - - if self.read_only: - raise ObjectDBPermissionError("Cannot gc read-only ODB") - if not cache_odb: - cache_odb = self - used_hashes = set() - for hash_info in used: - used_hashes.add(hash_info.value) - if hash_info.isdir and not shallow: - tree = Tree.load(cache_odb, hash_info) - used_hashes.update( - entry_obj.hash_info.value for _, entry_obj in tree - ) - - def _is_dir_hash(_hash): - from dvc.hash_info import HASH_DIR_SUFFIX - - return _hash.endswith(HASH_DIR_SUFFIX) - - removed = False - # hashes must be sorted to ensure we always remove .dir files first - for hash_ in sorted( - self.all(jobs, self.fs_path), - key=_is_dir_hash, - reverse=True, - ): - if hash_ in used_hashes: - continue - fs_path = self.hash_to_path(hash_) - if _is_dir_hash(hash_): - # backward compatibility - # pylint: disable=protected-access - self._remove_unpacked_dir(hash_) - self.fs.remove(fs_path) - removed = True - - return removed - def list_hashes_exists(self, hashes, jobs=None, name=None): """Return list of the specified hashes which exist in this fs. Hashes will be queried individually. diff --git a/dvc/repo/gc.py b/dvc/repo/gc.py index 549f315bb..6aa98ac22 100644 --- a/dvc/repo/gc.py +++ b/dvc/repo/gc.py @@ -45,6 +45,7 @@ def gc( from contextlib import ExitStack from dvc.data.db import get_index + from dvc.data.gc import gc as ogc from dvc.repo import Repo if not repos: @@ -73,7 +74,7 @@ def gc( if not odb: continue - removed = odb.gc(used_obj_ids, jobs=jobs) + removed = ogc(odb, used_obj_ids, jobs=jobs) if not removed: logger.info(f"No unused '{scheme}' cache to remove.") @@ -81,7 +82,7 @@ def gc( return odb = self.cloud.get_remote_odb(remote, "gc -c") - removed = odb.gc(used_obj_ids) + removed = ogc(odb, used_obj_ids) if removed: get_index(odb).clear() else: