mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 23:10:14 +01:00
utils: add decorator to wrap diskcache pickle errors
This commit is contained in:
committed by
Peter Rowlands (변기호)
parent
822b501069
commit
47d10c4a8b
@@ -4,6 +4,7 @@ from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING, Iterable, Set
|
||||
|
||||
from dvc.objects.errors import ObjectDBError
|
||||
from dvc.utils.decorators import with_diskcache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.types import StrPath
|
||||
@@ -104,16 +105,20 @@ class ObjectDBIndex(ObjectDBIndexBase):
|
||||
)
|
||||
)
|
||||
|
||||
@with_diskcache(name="index")
|
||||
def __iter__(self):
|
||||
return iter(self.index)
|
||||
|
||||
@with_diskcache(name="index")
|
||||
def __contains__(self, hash_):
|
||||
return hash_ in self.index
|
||||
|
||||
@with_diskcache(name="index")
|
||||
def dir_hashes(self):
|
||||
"""Iterate over .dir hashes stored in the index."""
|
||||
yield from (hash_ for hash_, is_dir in self.index.items() if is_dir)
|
||||
|
||||
@with_diskcache(name="index")
|
||||
def clear(self):
|
||||
"""Clear this index (to force re-indexing later)."""
|
||||
from diskcache import Timeout
|
||||
@@ -123,6 +128,7 @@ class ObjectDBIndex(ObjectDBIndexBase):
|
||||
except Timeout as exc:
|
||||
raise ObjectDBError("Failed to clear ODB index") from exc
|
||||
|
||||
@with_diskcache(name="index")
|
||||
def update(self, dir_hashes: Iterable[str], file_hashes: Iterable[str]):
|
||||
"""Update this index, adding the specified hashes."""
|
||||
from diskcache import Timeout
|
||||
@@ -137,6 +143,7 @@ class ObjectDBIndex(ObjectDBIndexBase):
|
||||
except Timeout as exc:
|
||||
raise ObjectDBError("Failed to update ODB index") from exc
|
||||
|
||||
@with_diskcache(name="index")
|
||||
def intersection(self, hashes: Set[str]):
|
||||
"""Iterate over values from `hashes` which exist in the index."""
|
||||
yield from hashes.intersection(self.index.keys())
|
||||
|
||||
@@ -7,6 +7,7 @@ from abc import ABC, abstractmethod
|
||||
from dvc.fs.local import LocalFileSystem
|
||||
from dvc.hash_info import HashInfo
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.decorators import with_diskcache
|
||||
from dvc.utils.fs import get_inode, get_mtime_and_size, remove
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -68,6 +69,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
self.md5s.close()
|
||||
self.links.close()
|
||||
|
||||
@with_diskcache(name="md5s")
|
||||
def save(self, path, fs, hash_info):
|
||||
"""Save hash for the specified path info.
|
||||
|
||||
@@ -92,6 +94,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
|
||||
self.md5s[inode] = (mtime, str(size), hash_info.value)
|
||||
|
||||
@with_diskcache(name="md5s")
|
||||
def get(self, path, fs):
|
||||
"""Gets the hash for the specified path info. Hash will be
|
||||
retrieved from the state database if available.
|
||||
@@ -122,6 +125,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
|
||||
return Meta(size=size), HashInfo("md5", value[2])
|
||||
|
||||
@with_diskcache(name="links")
|
||||
def save_link(self, path, fs):
|
||||
"""Adds the specified path to the list of links created by dvc. This
|
||||
list is later used on `dvc checkout` to cleanup old links.
|
||||
@@ -143,6 +147,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
with self.links as ref:
|
||||
ref[relative_path] = (inode, mtime)
|
||||
|
||||
@with_diskcache(name="links")
|
||||
def get_unused_links(self, used, fs):
|
||||
"""Removes all saved links except the ones that are used.
|
||||
|
||||
@@ -170,6 +175,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
|
||||
|
||||
return unused
|
||||
|
||||
@with_diskcache(name="links")
|
||||
def remove_links(self, unused, fs):
|
||||
if not isinstance(fs, LocalFileSystem):
|
||||
return
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import pickle
|
||||
from typing import Callable, TypeVar
|
||||
|
||||
from funcy import decorator
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
from . import format_link
|
||||
|
||||
_R = TypeVar("_R")
|
||||
|
||||
|
||||
@decorator
|
||||
def with_diskcache(call: Callable[..., _R], name: str) -> _R:
|
||||
try:
|
||||
return call()
|
||||
except (pickle.PickleError, ValueError) as exc:
|
||||
if isinstance(exc, ValueError) and not str(exc).startswith(
|
||||
"pickle protocol"
|
||||
):
|
||||
raise
|
||||
link = format_link(
|
||||
"https://dvc.org/doc/user-guide/troubleshooting#pickle"
|
||||
)
|
||||
msg = (
|
||||
f"Could not open pickled '{name}' cache. Remove the "
|
||||
f"'.dvc/tmp/{name}' directory and then retry this command. "
|
||||
f"See {link} for more information."
|
||||
)
|
||||
raise DvcException(msg) from exc
|
||||
@@ -0,0 +1,46 @@
|
||||
import pickle
|
||||
from typing import Any
|
||||
|
||||
import diskcache
|
||||
import pytest
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.utils.decorators import with_diskcache
|
||||
|
||||
|
||||
@with_diskcache(name="test")
|
||||
def set_value(cache: diskcache.Cache, key: str, value: Any) -> Any:
|
||||
cache[key] = value
|
||||
return cache[key]
|
||||
|
||||
|
||||
def test_pickle_protocol_error(tmp_dir):
|
||||
with pytest.raises(DvcException) as exc:
|
||||
with diskcache.Cache(
|
||||
directory=(tmp_dir / "test"),
|
||||
disk_pickle_protocol=pickle.HIGHEST_PROTOCOL + 1,
|
||||
) as cache:
|
||||
set_value(cache, "key", ("value1", "value2"))
|
||||
assert "troubleshooting#pickle" in str(exc)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"proto_a, proto_b",
|
||||
[
|
||||
(pickle.HIGHEST_PROTOCOL - 1, pickle.HIGHEST_PROTOCOL),
|
||||
(pickle.HIGHEST_PROTOCOL, pickle.HIGHEST_PROTOCOL - 1),
|
||||
],
|
||||
)
|
||||
def test_pickle_backwards_compat(tmp_dir, proto_a, proto_b):
|
||||
with diskcache.Cache(
|
||||
directory=(tmp_dir / "test"),
|
||||
disk_pickle_protocol=proto_a,
|
||||
) as cache:
|
||||
set_value(cache, "key", ("value1", "value2"))
|
||||
with diskcache.Cache(
|
||||
directory=(tmp_dir / "test"),
|
||||
disk_pickle_protocol=proto_b,
|
||||
) as cache:
|
||||
assert ("value1", "value2") == cache["key"]
|
||||
set_value(cache, "key", ("value3", "value4"))
|
||||
assert ("value3", "value4") == cache["key"]
|
||||
Reference in New Issue
Block a user