From 47d10c4a8b9bdc18dec17a9380f0593e43383c4c Mon Sep 17 00:00:00 2001 From: Peter Rowlands Date: Fri, 21 Jan 2022 17:01:21 +0900 Subject: [PATCH] utils: add decorator to wrap diskcache pickle errors --- dvc/data/db/index.py | 7 +++++ dvc/state.py | 6 ++++ dvc/utils/decorators.py | 30 +++++++++++++++++++ tests/unit/utils/test_decorators.py | 46 +++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+) create mode 100644 dvc/utils/decorators.py create mode 100644 tests/unit/utils/test_decorators.py diff --git a/dvc/data/db/index.py b/dvc/data/db/index.py index ee02746b2..25f42285c 100644 --- a/dvc/data/db/index.py +++ b/dvc/data/db/index.py @@ -4,6 +4,7 @@ from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Iterable, Set from dvc.objects.errors import ObjectDBError +from dvc.utils.decorators import with_diskcache if TYPE_CHECKING: from dvc.types import StrPath @@ -104,16 +105,20 @@ class ObjectDBIndex(ObjectDBIndexBase): ) ) + @with_diskcache(name="index") def __iter__(self): return iter(self.index) + @with_diskcache(name="index") def __contains__(self, hash_): return hash_ in self.index + @with_diskcache(name="index") def dir_hashes(self): """Iterate over .dir hashes stored in the index.""" yield from (hash_ for hash_, is_dir in self.index.items() if is_dir) + @with_diskcache(name="index") def clear(self): """Clear this index (to force re-indexing later).""" from diskcache import Timeout @@ -123,6 +128,7 @@ class ObjectDBIndex(ObjectDBIndexBase): except Timeout as exc: raise ObjectDBError("Failed to clear ODB index") from exc + @with_diskcache(name="index") def update(self, dir_hashes: Iterable[str], file_hashes: Iterable[str]): """Update this index, adding the specified hashes.""" from diskcache import Timeout @@ -137,6 +143,7 @@ class ObjectDBIndex(ObjectDBIndexBase): except Timeout as exc: raise ObjectDBError("Failed to update ODB index") from exc + @with_diskcache(name="index") def intersection(self, hashes: Set[str]): """Iterate over values from `hashes` which exist in the index.""" yield from hashes.intersection(self.index.keys()) diff --git a/dvc/state.py b/dvc/state.py index 6f8c86d90..a1463a232 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -7,6 +7,7 @@ from abc import ABC, abstractmethod from dvc.fs.local import LocalFileSystem from dvc.hash_info import HashInfo from dvc.utils import relpath +from dvc.utils.decorators import with_diskcache from dvc.utils.fs import get_inode, get_mtime_and_size, remove logger = logging.getLogger(__name__) @@ -68,6 +69,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes self.md5s.close() self.links.close() + @with_diskcache(name="md5s") def save(self, path, fs, hash_info): """Save hash for the specified path info. @@ -92,6 +94,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes self.md5s[inode] = (mtime, str(size), hash_info.value) + @with_diskcache(name="md5s") def get(self, path, fs): """Gets the hash for the specified path info. Hash will be retrieved from the state database if available. @@ -122,6 +125,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes return Meta(size=size), HashInfo("md5", value[2]) + @with_diskcache(name="links") def save_link(self, path, fs): """Adds the specified path to the list of links created by dvc. This list is later used on `dvc checkout` to cleanup old links. @@ -143,6 +147,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes with self.links as ref: ref[relative_path] = (inode, mtime) + @with_diskcache(name="links") def get_unused_links(self, used, fs): """Removes all saved links except the ones that are used. @@ -170,6 +175,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes return unused + @with_diskcache(name="links") def remove_links(self, unused, fs): if not isinstance(fs, LocalFileSystem): return diff --git a/dvc/utils/decorators.py b/dvc/utils/decorators.py new file mode 100644 index 000000000..b9ea532be --- /dev/null +++ b/dvc/utils/decorators.py @@ -0,0 +1,30 @@ +import pickle +from typing import Callable, TypeVar + +from funcy import decorator + +from dvc.exceptions import DvcException + +from . import format_link + +_R = TypeVar("_R") + + +@decorator +def with_diskcache(call: Callable[..., _R], name: str) -> _R: + try: + return call() + except (pickle.PickleError, ValueError) as exc: + if isinstance(exc, ValueError) and not str(exc).startswith( + "pickle protocol" + ): + raise + link = format_link( + "https://dvc.org/doc/user-guide/troubleshooting#pickle" + ) + msg = ( + f"Could not open pickled '{name}' cache. Remove the " + f"'.dvc/tmp/{name}' directory and then retry this command. " + f"See {link} for more information." + ) + raise DvcException(msg) from exc diff --git a/tests/unit/utils/test_decorators.py b/tests/unit/utils/test_decorators.py new file mode 100644 index 000000000..5998b7739 --- /dev/null +++ b/tests/unit/utils/test_decorators.py @@ -0,0 +1,46 @@ +import pickle +from typing import Any + +import diskcache +import pytest + +from dvc.exceptions import DvcException +from dvc.utils.decorators import with_diskcache + + +@with_diskcache(name="test") +def set_value(cache: diskcache.Cache, key: str, value: Any) -> Any: + cache[key] = value + return cache[key] + + +def test_pickle_protocol_error(tmp_dir): + with pytest.raises(DvcException) as exc: + with diskcache.Cache( + directory=(tmp_dir / "test"), + disk_pickle_protocol=pickle.HIGHEST_PROTOCOL + 1, + ) as cache: + set_value(cache, "key", ("value1", "value2")) + assert "troubleshooting#pickle" in str(exc) + + +@pytest.mark.parametrize( + "proto_a, proto_b", + [ + (pickle.HIGHEST_PROTOCOL - 1, pickle.HIGHEST_PROTOCOL), + (pickle.HIGHEST_PROTOCOL, pickle.HIGHEST_PROTOCOL - 1), + ], +) +def test_pickle_backwards_compat(tmp_dir, proto_a, proto_b): + with diskcache.Cache( + directory=(tmp_dir / "test"), + disk_pickle_protocol=proto_a, + ) as cache: + set_value(cache, "key", ("value1", "value2")) + with diskcache.Cache( + directory=(tmp_dir / "test"), + disk_pickle_protocol=proto_b, + ) as cache: + assert ("value1", "value2") == cache["key"] + set_value(cache, "key", ("value3", "value4")) + assert ("value3", "value4") == cache["key"]