utils: add decorator to wrap diskcache pickle errors

This commit is contained in:
Peter Rowlands
2022-01-25 16:21:53 +09:00
committed by Peter Rowlands (변기호)
parent 822b501069
commit 47d10c4a8b
4 changed files with 89 additions and 0 deletions
+7
View File
@@ -4,6 +4,7 @@ from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Iterable, Set
from dvc.objects.errors import ObjectDBError
from dvc.utils.decorators import with_diskcache
if TYPE_CHECKING:
from dvc.types import StrPath
@@ -104,16 +105,20 @@ class ObjectDBIndex(ObjectDBIndexBase):
)
)
@with_diskcache(name="index")
def __iter__(self):
return iter(self.index)
@with_diskcache(name="index")
def __contains__(self, hash_):
return hash_ in self.index
@with_diskcache(name="index")
def dir_hashes(self):
"""Iterate over .dir hashes stored in the index."""
yield from (hash_ for hash_, is_dir in self.index.items() if is_dir)
@with_diskcache(name="index")
def clear(self):
"""Clear this index (to force re-indexing later)."""
from diskcache import Timeout
@@ -123,6 +128,7 @@ class ObjectDBIndex(ObjectDBIndexBase):
except Timeout as exc:
raise ObjectDBError("Failed to clear ODB index") from exc
@with_diskcache(name="index")
def update(self, dir_hashes: Iterable[str], file_hashes: Iterable[str]):
"""Update this index, adding the specified hashes."""
from diskcache import Timeout
@@ -137,6 +143,7 @@ class ObjectDBIndex(ObjectDBIndexBase):
except Timeout as exc:
raise ObjectDBError("Failed to update ODB index") from exc
@with_diskcache(name="index")
def intersection(self, hashes: Set[str]):
"""Iterate over values from `hashes` which exist in the index."""
yield from hashes.intersection(self.index.keys())
+6
View File
@@ -7,6 +7,7 @@ from abc import ABC, abstractmethod
from dvc.fs.local import LocalFileSystem
from dvc.hash_info import HashInfo
from dvc.utils import relpath
from dvc.utils.decorators import with_diskcache
from dvc.utils.fs import get_inode, get_mtime_and_size, remove
logger = logging.getLogger(__name__)
@@ -68,6 +69,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
self.md5s.close()
self.links.close()
@with_diskcache(name="md5s")
def save(self, path, fs, hash_info):
"""Save hash for the specified path info.
@@ -92,6 +94,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
self.md5s[inode] = (mtime, str(size), hash_info.value)
@with_diskcache(name="md5s")
def get(self, path, fs):
"""Gets the hash for the specified path info. Hash will be
retrieved from the state database if available.
@@ -122,6 +125,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
return Meta(size=size), HashInfo("md5", value[2])
@with_diskcache(name="links")
def save_link(self, path, fs):
"""Adds the specified path to the list of links created by dvc. This
list is later used on `dvc checkout` to cleanup old links.
@@ -143,6 +147,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
with self.links as ref:
ref[relative_path] = (inode, mtime)
@with_diskcache(name="links")
def get_unused_links(self, used, fs):
"""Removes all saved links except the ones that are used.
@@ -170,6 +175,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
return unused
@with_diskcache(name="links")
def remove_links(self, unused, fs):
if not isinstance(fs, LocalFileSystem):
return
+30
View File
@@ -0,0 +1,30 @@
import pickle
from typing import Callable, TypeVar
from funcy import decorator
from dvc.exceptions import DvcException
from . import format_link
_R = TypeVar("_R")
@decorator
def with_diskcache(call: Callable[..., _R], name: str) -> _R:
try:
return call()
except (pickle.PickleError, ValueError) as exc:
if isinstance(exc, ValueError) and not str(exc).startswith(
"pickle protocol"
):
raise
link = format_link(
"https://dvc.org/doc/user-guide/troubleshooting#pickle"
)
msg = (
f"Could not open pickled '{name}' cache. Remove the "
f"'.dvc/tmp/{name}' directory and then retry this command. "
f"See {link} for more information."
)
raise DvcException(msg) from exc
+46
View File
@@ -0,0 +1,46 @@
import pickle
from typing import Any
import diskcache
import pytest
from dvc.exceptions import DvcException
from dvc.utils.decorators import with_diskcache
@with_diskcache(name="test")
def set_value(cache: diskcache.Cache, key: str, value: Any) -> Any:
cache[key] = value
return cache[key]
def test_pickle_protocol_error(tmp_dir):
with pytest.raises(DvcException) as exc:
with diskcache.Cache(
directory=(tmp_dir / "test"),
disk_pickle_protocol=pickle.HIGHEST_PROTOCOL + 1,
) as cache:
set_value(cache, "key", ("value1", "value2"))
assert "troubleshooting#pickle" in str(exc)
@pytest.mark.parametrize(
"proto_a, proto_b",
[
(pickle.HIGHEST_PROTOCOL - 1, pickle.HIGHEST_PROTOCOL),
(pickle.HIGHEST_PROTOCOL, pickle.HIGHEST_PROTOCOL - 1),
],
)
def test_pickle_backwards_compat(tmp_dir, proto_a, proto_b):
with diskcache.Cache(
directory=(tmp_dir / "test"),
disk_pickle_protocol=proto_a,
) as cache:
set_value(cache, "key", ("value1", "value2"))
with diskcache.Cache(
directory=(tmp_dir / "test"),
disk_pickle_protocol=proto_b,
) as cache:
assert ("value1", "value2") == cache["key"]
set_value(cache, "key", ("value3", "value4"))
assert ("value3", "value4") == cache["key"]