fs: get rid of CHECKSUM_DIR_SUFFIX and is_dir_hash

These are leftovers from old remotes, that clearly have nothing
to do with filesystems.
This commit is contained in:
Ruslan Kuprieiev
2021-12-10 10:53:10 +02:00
parent 82c5caee27
commit 81eda7e2b7
6 changed files with 17 additions and 34 deletions
-7
View File
@@ -43,7 +43,6 @@ class FileSystem:
REQUIRES: ClassVar[Dict[str, str]] = {}
_JOBS = 4 * cpu_count()
CHECKSUM_DIR_SUFFIX = ".dir"
HASH_JOBS = max(1, min(4, cpu_count() // 2))
LIST_OBJECT_PAGE_SIZE = 1000
TRAVERSE_WEIGHT_MULTIPLIER = 5
@@ -218,12 +217,6 @@ class FileSystem:
# pylint: enable=unused-argument
@classmethod
def is_dir_hash(cls, hash_):
if not hash_:
return False
return hash_.endswith(cls.CHECKSUM_DIR_SUFFIX)
def upload(
self,
from_info,
-1
View File
@@ -46,7 +46,6 @@ def get_index(odb) -> "ObjectDBIndexBase":
hashlib.sha256(
odb.fs.unstrip_protocol(odb.fs_path).encode("utf-8")
).hexdigest(),
odb.fs.CHECKSUM_DIR_SUFFIX,
)
+7 -2
View File
@@ -372,17 +372,22 @@ class ObjectDB:
entry_obj.hash_info.value for _, entry_obj in tree
)
def _is_dir_hash(_hash):
from dvc.hash_info import HASH_DIR_SUFFIX
return _hash.endswith(HASH_DIR_SUFFIX)
removed = False
# hashes must be sorted to ensure we always remove .dir files first
for hash_ in sorted(
self.all(jobs, self.fs_path),
key=self.fs.is_dir_hash,
key=_is_dir_hash,
reverse=True,
):
if hash_ in used_hashes:
continue
fs_path = self.hash_to_path(hash_)
if self.fs.is_dir_hash(hash_):
if _is_dir_hash(hash_):
# backward compatibility
# pylint: disable=protected-access
self._remove_unpacked_dir(hash_)
+10 -11
View File
@@ -1,7 +1,7 @@
import logging
import os
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Iterable, Optional, Set
from typing import TYPE_CHECKING, Iterable, Set
from ..errors import ObjectDBError
@@ -14,7 +14,9 @@ logger = logging.getLogger(__name__)
class ObjectDBIndexBase(ABC):
@abstractmethod
def __init__(
self, tmp_dir: "StrPath", name: str, dir_suffix: Optional[str] = None
self,
tmp_dir: "StrPath",
name: str,
):
pass
@@ -50,7 +52,9 @@ class ObjectDBIndexNoop(ObjectDBIndexBase):
"""No-op class for ODBs which are not indexed."""
def __init__(
self, tmp_dir: "StrPath", name: str, dir_suffix: Optional[str] = None
self,
tmp_dir: "StrPath",
name: str,
): # pylint: disable=super-init-not-called
pass
@@ -80,7 +84,9 @@ class ObjectDBIndex(ObjectDBIndexBase):
INDEX_DIR = "index"
def __init__(
self, tmp_dir: "StrPath", name: str, dir_suffix: Optional[str] = None
self,
tmp_dir: "StrPath",
name: str,
): # pylint: disable=super-init-not-called
from diskcache import Index
@@ -92,10 +98,6 @@ class ObjectDBIndex(ObjectDBIndexBase):
self.fs = LocalFileSystem()
self.index = Index(self.index_dir)
if not dir_suffix:
dir_suffix = self.fs.CHECKSUM_DIR_SUFFIX
self.dir_suffix = dir_suffix
def __iter__(self):
return iter(self.index)
@@ -106,9 +108,6 @@ class ObjectDBIndex(ObjectDBIndexBase):
"""Iterate over .dir hashes stored in the index."""
yield from (hash_ for hash_, is_dir in self.index.items() if is_dir)
def is_dir_hash(self, hash_: str):
return hash_.endswith(self.dir_suffix)
def clear(self):
"""Clear this index (to force re-indexing later)."""
from diskcache import Timeout
-8
View File
@@ -135,11 +135,3 @@ def test_list_paths(dvc):
walk_mock.assert_called_with(
posixpath.join(path, "00", "0"), prefix=True
)
@pytest.mark.parametrize(
"hash_, result",
[(None, False), ("", False), ("3456.dir", True), ("3456", False)],
)
def test_is_dir_hash(hash_, result):
assert FileSystem.is_dir_hash(hash_) == result
-5
View File
@@ -16,11 +16,6 @@ def test_init(dvc, index):
assert str(index.index_dir) == os.path.join(dvc.tmp_dir, "index", "foo")
def test_is_dir_hash(dvc, index):
assert index.is_dir_hash("foo.dir")
assert not index.is_dir_hash("foo")
def test_roundtrip(dvc, index):
expected_dir = {"1234.dir"}
expected_file = {"5678"}