dvc.objects.cache: set type in a more compatible way

It turns out that diskcache.Disk saves passed arguments to the database
and uses that all the time. So this means that we are not forward
compatible as it breaks moving from newer version of DVC to an older
one. We don't promise this, but this breaks our benchmarks which
are probably not isolated enough.

Regarding the use of cache, it is a bit iffy to extend it this way.
We only need this type for error messages. We could consider removing
this, but since the error message is documented and it's not too hard
to extend this :), I decided to preserve the message.
This commit is contained in:
Saugat Pachhai (सौगात)
2022-05-13 11:00:31 +05:45
committed by skshetry
parent 3f5757a0a5
commit dc1d3e880a
3 changed files with 9 additions and 7 deletions
+1 -3
View File
@@ -95,9 +95,7 @@ class ObjectDBIndex(ObjectDBIndexBase):
self.index_dir = os.path.join(tmp_dir, self.INDEX_DIR, name)
makedirs(self.index_dir, exist_ok=True)
self.fs = LocalFileSystem()
cache = Cache(
self.index_dir, eviction_policy="none", disk_type="index"
)
cache = Cache(self.index_dir, eviction_policy="none", type="index")
self.index = Index.fromcache(cache)
def __iter__(self):
+7 -3
View File
@@ -35,9 +35,8 @@ def translate_pickle_error(fn):
class Disk(disk):
"""Reraise pickle-related errors as DiskError."""
def __init__(self, *args, type=None, **kwargs):
super().__init__(*args, **kwargs)
self._type = type or os.path.basename(self._directory)
# we need type to differentiate cache for better error messages
_type: str
put = translate_pickle_error(disk.put)
get = translate_pickle_error(disk.get)
@@ -53,9 +52,14 @@ class Cache(diskcache.Cache):
directory: str = None,
timeout: int = 60,
disk: disk = Disk, # pylint: disable=redefined-outer-name
type: str = None,
**settings: Any,
) -> None:
settings.setdefault("disk_pickle_protocol", 4)
super().__init__(
directory=directory, timeout=timeout, disk=disk, **settings
)
self.disk._type = self._type = type or os.path.basename(self.directory)
def __getstate__(self):
return (*super().__getstate__(), self._type)
+1 -1
View File
@@ -17,7 +17,7 @@ def test_pickle_protocol_error(tmp_dir, disk_type):
cache = Cache(
directory,
disk_pickle_protocol=pickle.HIGHEST_PROTOCOL + 1,
disk_type=disk_type,
type=disk_type,
)
with pytest.raises(DiskError) as exc, cache as cache:
set_value(cache, "key", ("value1", "value2"))