odb: move gc into a separate function

This commit is contained in:
Ruslan Kuprieiev
2022-01-11 01:52:34 +02:00
parent e190433249
commit 9a01d98876
3 changed files with 43 additions and 42 deletions
+40
View File
@@ -0,0 +1,40 @@
def gc(odb, used, jobs=None, cache_odb=None, shallow=True):
from dvc.data.tree import Tree
from dvc.objects.errors import ObjectDBPermissionError
if odb.read_only:
raise ObjectDBPermissionError("Cannot gc read-only ODB")
if not cache_odb:
cache_odb = odb
used_hashes = set()
for hash_info in used:
used_hashes.add(hash_info.value)
if hash_info.isdir and not shallow:
tree = Tree.load(cache_odb, hash_info)
used_hashes.update(
entry_obj.hash_info.value for _, entry_obj in tree
)
def _is_dir_hash(_hash):
from dvc.hash_info import HASH_DIR_SUFFIX
return _hash.endswith(HASH_DIR_SUFFIX)
removed = False
# hashes must be sorted to ensure we always remove .dir files first
for hash_ in sorted(
odb.all(jobs, odb.fs_path),
key=_is_dir_hash,
reverse=True,
):
if hash_ in used_hashes:
continue
fs_path = odb.hash_to_path(hash_)
if _is_dir_hash(hash_):
# backward compatibility
# pylint: disable=protected-access
odb._remove_unpacked_dir(hash_)
odb.fs.remove(fs_path)
removed = True
return removed
-40
View File
@@ -356,46 +356,6 @@ class ObjectDB:
def _remove_unpacked_dir(self, hash_):
pass
def gc(self, used, jobs=None, cache_odb=None, shallow=True):
from dvc.data.tree import Tree
if self.read_only:
raise ObjectDBPermissionError("Cannot gc read-only ODB")
if not cache_odb:
cache_odb = self
used_hashes = set()
for hash_info in used:
used_hashes.add(hash_info.value)
if hash_info.isdir and not shallow:
tree = Tree.load(cache_odb, hash_info)
used_hashes.update(
entry_obj.hash_info.value for _, entry_obj in tree
)
def _is_dir_hash(_hash):
from dvc.hash_info import HASH_DIR_SUFFIX
return _hash.endswith(HASH_DIR_SUFFIX)
removed = False
# hashes must be sorted to ensure we always remove .dir files first
for hash_ in sorted(
self.all(jobs, self.fs_path),
key=_is_dir_hash,
reverse=True,
):
if hash_ in used_hashes:
continue
fs_path = self.hash_to_path(hash_)
if _is_dir_hash(hash_):
# backward compatibility
# pylint: disable=protected-access
self._remove_unpacked_dir(hash_)
self.fs.remove(fs_path)
removed = True
return removed
def list_hashes_exists(self, hashes, jobs=None, name=None):
"""Return list of the specified hashes which exist in this fs.
Hashes will be queried individually.
+3 -2
View File
@@ -45,6 +45,7 @@ def gc(
from contextlib import ExitStack
from dvc.data.db import get_index
from dvc.data.gc import gc as ogc
from dvc.repo import Repo
if not repos:
@@ -73,7 +74,7 @@ def gc(
if not odb:
continue
removed = odb.gc(used_obj_ids, jobs=jobs)
removed = ogc(odb, used_obj_ids, jobs=jobs)
if not removed:
logger.info(f"No unused '{scheme}' cache to remove.")
@@ -81,7 +82,7 @@ def gc(
return
odb = self.cloud.get_remote_odb(remote, "gc -c")
removed = odb.gc(used_obj_ids)
removed = ogc(odb, used_obj_ids)
if removed:
get_index(odb).clear()
else: