Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-11 07:18:28 +01:00
Code Issues Packages Projects Releases Wiki Activity

dvc: use HashInfo to interact with cache (#4506)

Browse Source
This commit is contained in:
Ruslan Kuprieiev
2020-09-01 03:32:51 +03:00
committed by GitHub
parent 56500601bd
commit fc321683d8
23 changed files with 182 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/api.py
+1 -1
View File
@@ -30,7 +30,7 @@ def get_url(path, repo=None, rev=None, remote=None):
raise UrlNotDvcRepoError(_repo.url) # pylint: disable=no-member
out = _repo.find_out_by_relpath(path)
remote_obj = _repo.cloud.get_remote(remote)
return str(remote_obj.tree.hash_to_path_info(out.checksum))
return str(remote_obj.tree.hash_to_path_info(out.hash_info.value))
def open( # noqa, pylint: disable=redefined-builtin
dvc/cache/base.py
Vendored
+87 -78
View File
@@ -11,6 +11,7 @@ from dvc.exceptions import (
DvcException,
MergeError,
)
from dvc.hash_info import HashInfo
from dvc.path_info import WindowsPathInfo
from dvc.progress import Tqdm
from dvc.remote.slow_link_detection import slow_link_guard
@@ -56,29 +57,29 @@ class CloudCache:
self.cache_type_confirmed = False
self._dir_info = {}
def get_dir_cache(self, hash_):
assert hash_
def get_dir_cache(self, hash_info):
assert hash_info
dir_info = self._dir_info.get(hash_)
dir_info = self._dir_info.get(hash_info.value)
if dir_info:
return dir_info
try:
dir_info = self.load_dir_cache(hash_)
dir_info = self.load_dir_cache(hash_info)
except DirCacheError:
dir_info = []
self._dir_info[hash_] = dir_info
self._dir_info[hash_info.value] = dir_info
return dir_info
def load_dir_cache(self, hash_):
path_info = self.tree.hash_to_path_info(hash_)
def load_dir_cache(self, hash_info):
path_info = self.tree.hash_to_path_info(hash_info.value)
try:
with self.tree.open(path_info, "r") as fobj:
d = json.load(fobj)
except (ValueError, FileNotFoundError) as exc:
raise DirCacheError(hash_) from exc
raise DirCacheError(hash_info) from exc
if not isinstance(d, list):
logger.error(
@@ -123,20 +124,21 @@ class CloudCache:
logger.debug("'%s' doesn't exist.", path_info)
return True
hash_ = hash_info.get(self.tree.PARAM_CHECKSUM)
if hash_ is None:
if not hash_info:
logger.debug("hash value for '%s' is missing.", path_info)
return True
if self.changed_cache(hash_):
logger.debug("cache for '%s'('%s') has changed.", path_info, hash_)
if self.changed_cache(hash_info):
logger.debug(
"cache for '%s'('%s') has changed.", path_info, hash_info
)
return True
actual = self.tree.get_hash(path_info)
if hash_ != actual.value:
if hash_info != actual:
logger.debug(
"hash value '%s' for '%s' has changed (actual '%s').",
hash_,
hash_info,
actual,
path_info,
)
@@ -193,12 +195,12 @@ class CloudCache:
"Created '%s': %s -> %s", self.cache_types[0], from_info, to_info,
)
def _save_file(self, path_info, tree, hash_, save_link=True, **kwargs):
assert hash_
def _save_file(self, path_info, tree, hash_info, save_link=True, **kwargs):
assert hash_info
cache_info = self.tree.hash_to_path_info(hash_)
cache_info = self.tree.hash_to_path_info(hash_info.value)
if tree == self.tree:
if self.changed_cache(hash_):
if self.changed_cache(hash_info):
self.tree.move(path_info, cache_info, mode=self.CACHE_MODE)
self.link(cache_info, path_info)
elif self.tree.iscopy(path_info) and self._cache_is_copy(
@@ -215,9 +217,9 @@ class CloudCache:
# we need to update path and cache, since in case of reflink,
# or copy cache type moving original file results in updates on
# next executed command, which causes md5 recalculation
self.tree.state.save(path_info, hash_)
self.tree.state.save(path_info, hash_info.value)
else:
if self.changed_cache(hash_):
if self.changed_cache(hash_info):
with tree.open(path_info, mode="rb") as fobj:
# if tree has fetch enabled, DVC out will be fetched on
# open and we do not need to read/copy any data
@@ -229,8 +231,7 @@ class CloudCache:
if callback:
callback(1)
self.tree.state.save(cache_info, hash_)
return {self.tree.PARAM_CHECKSUM: hash_}
self.tree.state.save(cache_info, hash_info.value)
def _cache_is_copy(self, path_info):
"""Checks whether cache uses copies."""
@@ -254,13 +255,15 @@ class CloudCache:
self.cache_type_confirmed = True
return self.cache_types[0] == "copy"
def _save_dir(self, path_info, tree, hash_, save_link=True, **kwargs):
dir_info = self.get_dir_cache(hash_)
def _save_dir(self, path_info, tree, hash_info, save_link=True, **kwargs):
dir_info = self.get_dir_cache(hash_info)
for entry in Tqdm(
dir_info, desc="Saving " + path_info.name, unit="file"
):
entry_info = path_info / entry[self.tree.PARAM_RELPATH]
entry_hash = entry[self.tree.PARAM_CHECKSUM]
entry_hash = HashInfo(
self.tree.PARAM_CHECKSUM, entry[self.tree.PARAM_CHECKSUM]
)
self._save_file(
entry_info, tree, entry_hash, save_link=False, **kwargs
)
@@ -268,11 +271,10 @@ class CloudCache:
if save_link:
self.tree.state.save_link(path_info)
if self.tree.exists(path_info):
self.tree.state.save(path_info, hash_)
self.tree.state.save(path_info, hash_info.value)
cache_info = self.tree.hash_to_path_info(hash_)
self.tree.state.save(cache_info, hash_)
return {self.tree.PARAM_CHECKSUM: hash_}
cache_info = self.tree.hash_to_path_info(hash_info.value)
self.tree.state.save(cache_info, hash_info.value)
def save(self, path_info, tree, hash_info, save_link=True, **kwargs):
if path_info.scheme != self.tree.scheme:
@@ -281,23 +283,23 @@ class CloudCache:
self.tree.scheme,
)
hash_ = hash_info[self.tree.PARAM_CHECKSUM]
return self._save(path_info, tree, hash_, save_link, **kwargs)
self._save(path_info, tree, hash_info, save_link, **kwargs)
def _save(self, path_info, tree, hash_, save_link=True, **kwargs):
to_info = self.tree.hash_to_path_info(hash_)
def _save(self, path_info, tree, hash_info, save_link=True, **kwargs):
to_info = self.tree.hash_to_path_info(hash_info.value)
logger.debug("Saving '%s' to '%s'.", path_info, to_info)
if tree.isdir(path_info):
return self._save_dir(path_info, tree, hash_, save_link, **kwargs)
return self._save_file(path_info, tree, hash_, save_link, **kwargs)
self._save_dir(path_info, tree, hash_info, save_link, **kwargs)
else:
self._save_file(path_info, tree, hash_info, save_link, **kwargs)
# Override to return path as a string instead of PathInfo for clouds
# which support string paths (see local)
def hash_to_path(self, hash_):
return self.tree.hash_to_path_info(hash_)
def changed_cache_file(self, hash_):
def changed_cache_file(self, hash_info):
"""Compare the given hash with the (corresponding) actual one.
- Use `State` as a cache for computed hashes
@@ -310,7 +312,7 @@ class CloudCache:
- Remove the file from cache if it doesn't match the actual hash
"""
# Prefer string path over PathInfo when possible due to performance
cache_info = self.hash_to_path(hash_)
cache_info = self.hash_to_path(hash_info.value)
if self.tree.is_protected(cache_info):
logger.debug(
"Assuming '%s' is unchanged since it is read-only", cache_info
@@ -320,13 +322,16 @@ class CloudCache:
actual = self.tree.get_hash(cache_info)
logger.debug(
"cache '%s' expected '%s' actual '%s'", cache_info, hash_, actual,
"cache '%s' expected '%s' actual '%s'",
cache_info,
hash_info,
actual,
)
if not hash_ or not actual:
if not hash_info or not actual:
return True
if actual.value.split(".")[0] == hash_.split(".")[0]:
if actual.value.split(".")[0] == hash_info.value.split(".")[0]:
# making cache file read-only so we don't need to check it
# next time
self.tree.protect(cache_info)
@@ -338,12 +343,14 @@ class CloudCache:
return True
def _changed_dir_cache(self, hash_, path_info=None, filter_info=None):
if self.changed_cache_file(hash_):
def _changed_dir_cache(self, hash_info, path_info=None, filter_info=None):
if self.changed_cache_file(hash_info):
return True
for entry in self.get_dir_cache(hash_):
entry_hash = entry[self.tree.PARAM_CHECKSUM]
for entry in self.get_dir_cache(hash_info):
entry_hash = HashInfo(
self.tree.PARAM_CHECKSUM, entry[self.tree.PARAM_CHECKSUM]
)
if path_info and filter_info:
entry_info = path_info / entry[self.tree.PARAM_RELPATH]
@@ -355,12 +362,12 @@ class CloudCache:
return False
def changed_cache(self, hash_, path_info=None, filter_info=None):
if self.tree.is_dir_hash(hash_):
def changed_cache(self, hash_info, path_info=None, filter_info=None):
if hash_info.isdir:
return self._changed_dir_cache(
hash_, path_info=path_info, filter_info=filter_info
hash_info, path_info=path_info, filter_info=filter_info
)
return self.changed_cache_file(hash_)
return self.changed_cache_file(hash_info)
def already_cached(self, path_info):
_, current = self.tree.get_hash(path_info)
@@ -386,11 +393,11 @@ class CloudCache:
self.tree.remove(path_info)
def _checkout_file(
self, path_info, hash_, force, progress_callback=None, relink=False
self, path_info, hash_info, force, progress_callback=None, relink=False
):
"""The file is changed we need to checkout a new copy"""
added, modified = True, False
cache_info = self.tree.hash_to_path_info(hash_)
cache_info = self.tree.hash_to_path_info(hash_info.value)
if self.tree.exists(path_info):
logger.debug("data '%s' will be replaced.", path_info)
self.safe_remove(path_info, force=force)
@@ -398,7 +405,7 @@ class CloudCache:
self.link(cache_info, path_info)
self.tree.state.save_link(path_info)
self.tree.state.save(path_info, hash_)
self.tree.state.save(path_info, hash_info.value)
if progress_callback:
progress_callback(str(path_info))
@@ -407,7 +414,7 @@ class CloudCache:
def _checkout_dir(
self,
path_info,
hash_,
hash_info,
force,
progress_callback=None,
relink=False,
@@ -420,7 +427,7 @@ class CloudCache:
added = True
self.tree.makedirs(path_info)
dir_info = self.get_dir_cache(hash_)
dir_info = self.get_dir_cache(hash_info)
logger.debug("Linking directory '%s'.", path_info)
@@ -433,7 +440,7 @@ class CloudCache:
if filter_info and not entry_info.isin_or_eq(filter_info):
continue
entry_hash_info = {self.tree.PARAM_CHECKSUM: entry_hash}
entry_hash_info = HashInfo(self.tree.PARAM_CHECKSUM, entry_hash)
if relink or self.changed(entry_info, entry_hash_info):
modified = True
self.safe_remove(entry_info, force=force)
@@ -448,7 +455,7 @@ class CloudCache:
)
self.tree.state.save_link(path_info)
self.tree.state.save(path_info, hash_)
self.tree.state.save(path_info, hash_info.value)
# relink is not modified, assume it as nochange
return added, not added and modified and not relink
@@ -477,10 +484,9 @@ class CloudCache:
if path_info.scheme not in ["local", self.tree.scheme]:
raise NotImplementedError
hash_ = hash_info.get(self.tree.PARAM_CHECKSUM)
failed = None
skip = False
if not hash_:
if not hash_info:
logger.warning(
"No file hash info found for '%s'. " "It won't be created.",
path_info,
@@ -493,11 +499,11 @@ class CloudCache:
skip = True
elif self.changed_cache(
hash_, path_info=path_info, filter_info=filter_info
hash_info, path_info=path_info, filter_info=filter_info
):
logger.warning(
"Cache '%s' not found. File '%s' won't be created.",
hash_,
hash_info,
path_info,
)
self.safe_remove(path_info, force=force)
@@ -508,52 +514,59 @@ class CloudCache:
progress_callback(
str(path_info),
self.get_files_number(
self.tree.path_info, hash_, filter_info
self.tree.path_info, hash_info, filter_info
),
)
if failed:
raise CheckoutError([failed])
return
logger.debug("Checking out '%s' with cache '%s'.", path_info, hash_)
logger.debug(
"Checking out '%s' with cache '%s'.", path_info, hash_info
)
return self._checkout(
path_info, hash_, force, progress_callback, relink, filter_info,
path_info,
hash_info,
force,
progress_callback,
relink,
filter_info,
)
def _checkout(
self,
path_info,
hash_,
hash_info,
force=False,
progress_callback=None,
relink=False,
filter_info=None,
):
if not self.tree.is_dir_hash(hash_):
if not hash_info.isdir:
return self._checkout_file(
path_info, hash_, force, progress_callback, relink
path_info, hash_info, force, progress_callback, relink
)
return self._checkout_dir(
path_info, hash_, force, progress_callback, relink, filter_info
path_info, hash_info, force, progress_callback, relink, filter_info
)
def get_files_number(self, path_info, hash_, filter_info):
def get_files_number(self, path_info, hash_info, filter_info):
from funcy.py3 import ilen
if not hash_:
if not hash_info:
return 0
if not self.tree.is_dir_hash(hash_):
if not hash_info.isdir:
return 1
if not filter_info:
return len(self.get_dir_cache(hash_))
return len(self.get_dir_cache(hash_info))
return ilen(
filter_info.isin_or_eq(path_info / entry[self.tree.PARAM_CHECKSUM])
for entry in self.get_dir_cache(hash_)
for entry in self.get_dir_cache(hash_info)
)
def _to_dict(self, dir_info):
@@ -620,16 +633,12 @@ class CloudCache:
assert their_info
if ancestor_info:
ancestor_hash = ancestor_info.value
ancestor = self.get_dir_cache(ancestor_hash)
ancestor = self.get_dir_cache(ancestor_info)
else:
ancestor = []
our_hash = our_info.value
our = self.get_dir_cache(our_hash)
their_hash = their_info.value
their = self.get_dir_cache(their_hash)
our = self.get_dir_cache(our_info)
their = self.get_dir_cache(their_info)
merged = self._merge_dirs(ancestor, our, their)
return self.tree.save_dir_info(merged)
dvc/cache/local.py
Vendored
+5 -5
View File
@@ -7,6 +7,7 @@ from functools import partial, wraps
from funcy import cached_property, concat
from dvc.exceptions import DownloadError, UploadError
from dvc.hash_info import HashInfo
from dvc.path_info import PathInfo
from dvc.progress import Tqdm
@@ -85,20 +86,19 @@ class LocalCache(CloudCache):
desc="Querying "
+ ("cache in " + name if name else "local cache"),
)
if not self.changed_cache_file(hash_)
if not self.changed_cache_file(
HashInfo(self.tree.PARAM_CHECKSUM, hash_)
)
]
def already_cached(self, path_info):
assert path_info.scheme in ["", "local"]
current = self.tree.get_hash(path_info)
assert current.name == "md5"
if not current:
return False
return not self.changed_cache(current.value)
return not self.changed_cache(current)
def _verify_link(self, path_info, link_type):
if link_type == "hardlink" and self.tree.getsize(path_info) == 0:
dvc/external_repo.py
+1 -1
View File
@@ -144,7 +144,7 @@ class BaseExternalRepo:
raise PathMissingError(path, self.url)
hash_info = self.repo_tree.get_hash(
path, download_callback=download_update
).to_dict()
)
self.local_cache.save(
path,
self.repo_tree,
dvc/hash_info.py
+8
View File
@@ -1,5 +1,7 @@
from dataclasses import dataclass
HASH_DIR_SUFFIX = ".dir"
@dataclass
class HashInfo:
@@ -18,3 +20,9 @@ class HashInfo:
def to_dict(self):
return {self.name: self.value} if self else {}
@property
def isdir(self):
if not self:
return False
return self.value.endswith(HASH_DIR_SUFFIX)
dvc/output/base.py
+19 -26
View File
@@ -163,7 +163,7 @@ class BaseOutput:
@property
def dir_cache(self):
return self.cache.get_dir_cache(self.checksum)
return self.cache.get_dir_cache(self.hash_info)
@classmethod
def supported(cls, url):
@@ -171,34 +171,29 @@ class BaseOutput:
@property
def cache_path(self):
return self.cache.tree.hash_to_path_info(self.checksum).url
@property
def checksum(self):
return self.hash_info.value
def get_checksum(self):
return self.get_hash().value
return self.cache.tree.hash_to_path_info(self.hash_info.value).url
def get_hash(self):
return self.tree.get_hash(self.path_info)
@property
def is_dir_checksum(self):
return self.tree.is_dir_hash(self.checksum)
return self.hash_info.isdir
@property
def exists(self):
return self.tree.exists(self.path_info)
def changed_checksum(self):
return self.checksum != self.get_checksum()
return self.hash_info != self.get_hash()
def changed_cache(self, filter_info=None):
if not self.use_cache or not self.checksum:
if not self.use_cache or not self.hash_info:
return True
return self.cache.changed_cache(self.checksum, filter_info=filter_info)
return self.cache.changed_cache(
self.hash_info, filter_info=filter_info
)
def workspace_status(self):
if not self.exists:
@@ -207,13 +202,13 @@ class BaseOutput:
if self.changed_checksum():
return {str(self): "modified"}
if not self.checksum:
if not self.hash_info:
return {str(self): "new"}
return {}
def status(self):
if self.checksum and self.use_cache and self.changed_cache():
if self.hash_info and self.use_cache and self.changed_cache():
return {str(self): "not in cache"}
return self.workspace_status()
@@ -286,9 +281,7 @@ class BaseOutput:
def commit(self):
assert self.hash_info
if self.use_cache:
self.cache.save(
self.path_info, self.cache.tree, self.hash_info.to_dict()
)
self.cache.save(self.path_info, self.cache.tree, self.hash_info)
def dumpd(self):
ret = copy(self.hash_info.to_dict())
@@ -340,7 +333,7 @@ class BaseOutput:
return self.cache.checkout(
self.path_info,
self.hash_info.to_dict(),
self.hash_info,
force=force,
progress_callback=progress_callback,
relink=relink,
@@ -374,7 +367,7 @@ class BaseOutput:
return 0
return self.cache.get_files_number(
self.path_info, self.checksum, filter_info
self.path_info, self.hash_info, filter_info
)
def unprotect(self):
@@ -384,9 +377,9 @@ class BaseOutput:
def get_dir_cache(self, **kwargs):
if not self.is_dir_checksum:
raise DvcException("cannot get dir cache for file checksum")
if self.cache.changed_cache_file(self.checksum):
if self.cache.changed_cache_file(self.hash_info):
self.repo.cloud.pull(
NamedCache.make("local", self.checksum, str(self)),
NamedCache.make("local", self.hash_info.value, str(self)),
show_checksums=False,
**kwargs,
)
@@ -421,7 +414,7 @@ class BaseOutput:
except DvcException:
logger.debug(f"failed to pull cache for '{self}'")
if self.cache.changed_cache_file(self.checksum):
if self.cache.changed_cache_file(self.hash_info):
msg = (
"Missing cache for directory '{}'. "
"Cache for files inside will be lost. "
@@ -471,7 +464,7 @@ class BaseOutput:
cache.external[dep.repo_pair].add(dep.def_path)
return cache
if not self.checksum:
if not self.hash_info:
msg = (
"Output '{}'({}) is missing version info. "
"Cache for it will not be collected. "
@@ -490,13 +483,13 @@ class BaseOutput:
logger.warning(msg)
return NamedCache()
ret = NamedCache.make(self.scheme, self.checksum, str(self))
ret = NamedCache.make(self.scheme, self.hash_info.value, str(self))
if not self.is_dir_checksum:
return ret
ret.add_child_cache(
self.checksum, self.collect_used_dir_cache(**kwargs),
self.hash_info.value, self.collect_used_dir_cache(**kwargs),
)
return ret
dvc/repo/diff.py
+1 -1
View File
@@ -38,7 +38,7 @@ def diff(self, a_rev="HEAD", b_rev=None):
def _to_checksum(output):
if on_working_tree:
return self.cache.local.tree.get_hash(output.path_info).value
return output.checksum
return output.hash_info.value
def _exists(output):
if on_working_tree:
dvc/stage/cache.py
+1 -1
View File
@@ -28,7 +28,7 @@ def _get_stage_hash(stage):
return None
for dep in stage.deps:
if not (dep.scheme == "local" and dep.def_path and dep.get_checksum()):
if not (dep.scheme == "local" and dep.def_path and dep.get_hash()):
return None
for out in stage.outs:
dvc/tree/base.py
+1 -1
View File
@@ -343,7 +343,7 @@ class BaseTree:
def save_dir_info(self, dir_info):
hash_info, tmp_info = self._get_dir_info_hash(dir_info)
new_info = self.cache.tree.hash_to_path_info(hash_info.value)
if self.cache.changed_cache_file(hash_info.value):
if self.cache.changed_cache_file(hash_info):
self.cache.tree.makedirs(new_info.parent)
self.cache.tree.move(
tmp_info, new_info, mode=self.cache.CACHE_MODE
dvc/tree/dvc.py
+3 -3
View File
@@ -85,7 +85,7 @@ class DvcTree(BaseTree): # pylint:disable=abstract-method
if out.is_dir_checksum:
checksum = self._get_granular_checksum(path, out)
else:
checksum = out.checksum
checksum = out.hash_info.value
try:
remote_info = remote_obj.tree.hash_to_path_info(checksum)
return remote_obj.tree.open(
@@ -246,7 +246,7 @@ class DvcTree(BaseTree): # pylint:disable=abstract-method
out = outs[0]
# other code expects us to fetch the dir at this point
self._fetch_dir(out, **kwargs)
return HashInfo(out.tree.PARAM_CHECKSUM, out.checksum)
return out.hash_info
except OutputNotFoundError:
pass
@@ -262,7 +262,7 @@ class DvcTree(BaseTree): # pylint:disable=abstract-method
out.tree.PARAM_CHECKSUM,
self._get_granular_checksum(path_info, out),
)
return HashInfo(out.tree.PARAM_CHECKSUM, out.checksum)
return out.hash_info
def metadata(self, path_info):
path_info = PathInfo(os.path.abspath(path_info))
tests/func/remote/test_index.py
+7 -7
View File
@@ -39,9 +39,9 @@ def test_indexed_on_status(tmp_dir, dvc, tmp_path_factory, remote):
dvc.status(cloud=True)
with remote.index:
assert {bar.checksum, baz["md5"]} == set(remote.index.hashes())
assert [bar.checksum] == list(remote.index.dir_hashes())
assert foo.checksum not in remote.index.hashes()
assert {bar.hash_info.value, baz["md5"]} == set(remote.index.hashes())
assert [bar.hash_info.value] == list(remote.index.dir_hashes())
assert foo.hash_info.value not in remote.index.hashes()
def test_indexed_on_push(tmp_dir, dvc, tmp_path_factory, remote):
@@ -51,15 +51,15 @@ def test_indexed_on_push(tmp_dir, dvc, tmp_path_factory, remote):
dvc.push()
with remote.index:
assert {bar.checksum, baz["md5"]} == set(remote.index.hashes())
assert [bar.checksum] == list(remote.index.dir_hashes())
assert foo.checksum not in remote.index.hashes()
assert {bar.hash_info.value, baz["md5"]} == set(remote.index.hashes())
assert [bar.hash_info.value] == list(remote.index.dir_hashes())
assert foo.hash_info.value not in remote.index.hashes()
def test_indexed_dir_missing(tmp_dir, dvc, tmp_path_factory, remote):
bar = tmp_dir.dvc_gen({"bar": {"baz": "baz content"}})[0].outs[0]
with remote.index:
remote.index.update([bar.checksum], [])
remote.index.update([bar.hash_info.value], [])
dvc.status(cloud=True)
with remote.index:
assert not list(remote.index.hashes())
tests/func/test_add.py
+2 -2
View File
@@ -72,9 +72,9 @@ def test_add_directory(tmp_dir, dvc):
assert len(stage.deps) == 0
assert len(stage.outs) == 1
md5 = stage.outs[0].hash_info.value
hash_info = stage.outs[0].hash_info
dir_info = dvc.cache.local.load_dir_cache(md5)
dir_info = dvc.cache.local.load_dir_cache(hash_info)
for info in dir_info:
assert "\\" not in info["relpath"]
tests/func/test_cache.py
+5 -2
View File
@@ -6,6 +6,7 @@ import pytest
from dvc.cache import Cache
from dvc.cache.base import DirCacheError
from dvc.hash_info import HashInfo
from dvc.main import main
from dvc.utils import relpath
from tests.basic_env import TestDir, TestDvc
@@ -52,14 +53,16 @@ class TestCacheLoadBadDirCache(TestDvc):
)
self.create(fname, "<clearly>not,json")
with pytest.raises(DirCacheError):
self.dvc.cache.local.load_dir_cache(dir_hash)
self.dvc.cache.local.load_dir_cache(HashInfo("md5", dir_hash))
dir_hash = "234.dir"
fname = os.fspath(
self.dvc.cache.local.tree.hash_to_path_info(dir_hash)
)
self.create(fname, '{"a": "b"}')
self._do_test(self.dvc.cache.local.load_dir_cache(dir_hash))
self._do_test(
self.dvc.cache.local.load_dir_cache(HashInfo("md5", dir_hash))
)
class TestExternalCacheDir(TestDvc):
tests/func/test_checkout.py
+1 -1
View File
@@ -100,7 +100,7 @@ class TestCheckoutCorruptedCacheDir(TestDvc):
# NOTE: modifying cache file for one of the files inside the directory
# to check if dvc will detect that the cache is corrupted.
entry = self.dvc.cache.local.load_dir_cache(out.checksum)[0]
entry = self.dvc.cache.local.load_dir_cache(out.hash_info)[0]
entry_hash = entry[self.dvc.cache.local.tree.PARAM_CHECKSUM]
cache = os.fspath(
self.dvc.cache.local.tree.hash_to_path_info(entry_hash)
tests/func/test_data_cloud.py
+2 -2
View File
@@ -27,7 +27,7 @@ def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument
(stage,) = tmp_dir.dvc_gen("foo", "foo")
out = stage.outs[0]
cache = out.cache_path
md5 = out.checksum
md5 = out.hash_info.value
info = out.get_used_cache()
(stage_dir,) = tmp_dir.dvc_gen(
@@ -41,7 +41,7 @@ def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument
out_dir = stage_dir.outs[0]
cache_dir = out_dir.cache_path
name_dir = str(out_dir)
md5_dir = out_dir.checksum
md5_dir = out_dir.hash_info.value
info_dir = NamedCache.make(out_dir.scheme, md5_dir, name_dir)
with dvc.state:
tests/func/test_gc.py
+2 -2
View File
@@ -370,8 +370,8 @@ def test_gc_external_output(tmp_dir, dvc, workspace):
(foo_stage,) = dvc.add("remote://workspace/foo")
(bar_stage,) = dvc.add("remote://workspace/bar")
foo_hash = foo_stage.outs[0].checksum
bar_hash = bar_stage.outs[0].checksum
foo_hash = foo_stage.outs[0].hash_info.value
bar_hash = bar_stage.outs[0].hash_info.value
assert (
workspace / "cache" / foo_hash[:2] / foo_hash[2:]
tests/func/test_remote.py
+9 -5
View File
@@ -193,11 +193,13 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote):
remote = dvc.cloud.get_remote("upstream")
assert not remote.tree.exists(
remote.tree.hash_to_path_info(foo.checksum)
remote.tree.hash_to_path_info(foo.hash_info.value)
)
assert remote.tree.exists(
remote.tree.hash_to_path_info(bar.hash_info.value)
)
assert remote.tree.exists(remote.tree.hash_to_path_info(bar.checksum))
assert not remote.tree.exists(
remote.tree.hash_to_path_info(baz.checksum)
remote.tree.hash_to_path_info(baz.hash_info.value)
)
# Push everything and delete local cache
@@ -408,7 +410,9 @@ def test_protect_local_remote(tmp_dir, dvc, local_remote):
dvc.push()
remote = dvc.cloud.get_remote("upstream")
remote_cache_file = remote.tree.hash_to_path_info(stage.outs[0].checksum)
remote_cache_file = remote.tree.hash_to_path_info(
stage.outs[0].hash_info.value
)
assert os.path.exists(remote_cache_file)
assert stat.S_IMODE(os.stat(remote_cache_file).st_mode) == 0o444
@@ -419,7 +423,7 @@ def test_push_incomplete_dir(tmp_dir, dvc, mocker, local_remote):
remote = dvc.cloud.get_remote("upstream")
cache = dvc.cache.local
dir_hash = stage.outs[0].checksum
dir_hash = stage.outs[0].hash_info.value
used = stage.get_used_cache(remote=remote)
# remove one of the cache files for directory
tests/func/test_repro.py
+5 -3
View File
@@ -674,7 +674,9 @@ class TestReproDataSource(TestReproChangedData):
stages = self.dvc.reproduce(self.foo_stage.path)
self.assertTrue(filecmp.cmp(self.FOO, self.BAR, shallow=False))
self.assertEqual(stages[0].outs[0].checksum, file_md5(self.BAR)[0])
self.assertEqual(
stages[0].outs[0].hash_info.value, file_md5(self.BAR)[0]
)
class TestReproChangedDir(SingleStageRun, TestDvc):
@@ -911,7 +913,7 @@ class TestReproAlreadyCached(TestRepro):
0
]
self.assertNotEqual(run_out.checksum, repro_out.checksum)
self.assertNotEqual(run_out.hash_info, repro_out.hash_info)
def test_force_with_dependencies(self):
run_out = self.dvc.run(
@@ -927,7 +929,7 @@ class TestReproAlreadyCached(TestRepro):
repro_out = Dvcfile(self.dvc, "datetime.dvc").stage.outs[0]
self.assertNotEqual(run_out.checksum, repro_out.checksum)
self.assertNotEqual(run_out.hash_info, repro_out.hash_info)
def test_force_import(self):
ret = main(["import-url", self.FOO, self.BAR])
tests/func/test_run_single_stage.py
+1 -1
View File
@@ -60,7 +60,7 @@ class TestRun(TestDvc):
self.assertEqual(len(stage.deps), len(deps))
self.assertEqual(len(stage.outs), len(outs + outs_no_cache))
self.assertEqual(stage.outs[0].fspath, outs[0])
self.assertEqual(stage.outs[0].checksum, file_md5(self.FOO)[0])
self.assertEqual(stage.outs[0].hash_info.value, file_md5(self.FOO)[0])
self.assertTrue(stage.path, fname)
with self.assertRaises(OutputDuplicationError):
tests/func/test_tree.py
+1 -1
View File
@@ -226,7 +226,7 @@ def test_repotree_cache_save(tmp_dir, dvc, scm, erepo_dir, local_cloud):
cache = dvc.cache.local
with cache.tree.state:
path_info = PathInfo(erepo_dir / "dir")
hash_info = cache.tree.get_hash(path_info).to_dict()
hash_info = cache.tree.get_hash(path_info)
cache.save(path_info, tree, hash_info)
for hash_ in expected:
tests/unit/output/test_local.py
+3 -4
View File
@@ -3,6 +3,7 @@ import os
from mock import patch
from dvc.cache.local import LocalCache
from dvc.hash_info import HashInfo
from dvc.output import LocalOutput
from dvc.stage import Stage
from dvc.utils import relpath
@@ -77,7 +78,6 @@ class TestGetFilesNumber(TestDvc):
o.use_cache = False
self.assertEqual(0, o.get_files_number())
@patch.object(LocalOutput, "checksum", "12345678.dir")
@patch.object(
LocalCache,
"get_dir_cache",
@@ -85,12 +85,11 @@ class TestGetFilesNumber(TestDvc):
)
def test_return_multiple_for_dir(self, _mock_get_dir_cache):
o = self._get_output()
o.hash_info = HashInfo("md5", "12345678.dir")
self.assertEqual(2, o.get_files_number())
@patch.object(LocalOutput, "checksum", "12345678")
@patch.object(LocalOutput, "is_dir_checksum", False)
def test_return_1_on_single_file_cache(self):
o = self._get_output()
o.hash_info = HashInfo("md5", "12345678")
self.assertEqual(1, o.get_files_number())
tests/unit/output/test_output.py
-3
View File
@@ -83,9 +83,6 @@ def test_get_used_cache(exists, expected_message, mocker, caplog):
mocker.patch.object(output, "use_cache", True)
mocker.patch.object(stage, "is_repo_import", False)
mocker.patch.object(
BaseOutput, "checksum", new_callable=mocker.PropertyMock
).return_value = None
mocker.patch.object(
BaseOutput, "exists", new_callable=mocker.PropertyMock
).return_value = exists
tests/unit/stage/test_loader_pipeline_file.py
+17 -17
View File
@@ -31,7 +31,7 @@ def test_fill_from_lock_deps_outs(dvc, lock_data):
)
for item in chain(stage.deps, stage.outs):
assert not item.checksum and not item.hash_info
assert not item.hash_info
StageLoader.fill_from_lock(stage, lock_data)
@@ -102,7 +102,7 @@ def test_fill_from_lock_missing_checksums(dvc, lock_data):
assert stage.deps[0].hash_info == HashInfo("md5", "foo_checksum")
assert stage.outs[0].hash_info == HashInfo("md5", "bar_checksum")
assert not stage.deps[1].checksum and not stage.outs[1].checksum
assert not stage.deps[1].hash_info and not stage.outs[1].hash_info
def test_fill_from_lock_use_appropriate_checksum(dvc, lock_data):
@@ -115,8 +115,8 @@ def test_fill_from_lock_use_appropriate_checksum(dvc, lock_data):
)
lock_data["deps"] = [{"path": "s3://dvc-temp/foo", "etag": "e-tag"}]
StageLoader.fill_from_lock(stage, lock_data)
assert stage.deps[0].checksum == "e-tag"
assert stage.outs[0].checksum == "bar_checksum"
assert stage.deps[0].hash_info == HashInfo("etag", "e-tag")
assert stage.outs[0].hash_info == HashInfo("md5", "bar_checksum")
def test_fill_from_lock_with_missing_sections(dvc, lock_data):
@@ -126,14 +126,14 @@ def test_fill_from_lock_with_missing_sections(dvc, lock_data):
lock = deepcopy(lock_data)
del lock["deps"]
StageLoader.fill_from_lock(stage, lock)
assert not stage.deps[0].checksum
assert stage.outs[0].checksum == "bar_checksum"
assert not stage.deps[0].hash_info
assert stage.outs[0].hash_info == HashInfo("md5", "bar_checksum")
lock = deepcopy(lock_data)
del lock["outs"]
StageLoader.fill_from_lock(stage, lock)
assert stage.deps[0].checksum == "foo_checksum"
assert not stage.outs[0].checksum
assert stage.deps[0].hash_info == HashInfo("md5", "foo_checksum")
assert not stage.outs[0].hash_info
def test_fill_from_lock_empty_data(dvc):
@@ -141,9 +141,9 @@ def test_fill_from_lock_empty_data(dvc):
PipelineStage, dvc, PIPELINE_FILE, deps=["foo"], outs=["bar"]
)
StageLoader.fill_from_lock(stage, None)
assert not stage.deps[0].checksum and not stage.outs[0].checksum
assert not stage.deps[0].hash_info and not stage.outs[0].hash_info
StageLoader.fill_from_lock(stage, {})
assert not stage.deps[0].checksum and not stage.outs[0].checksum
assert not stage.deps[0].hash_info and not stage.outs[0].hash_info
def test_load_stage(dvc, stage_data, lock_data):
@@ -155,9 +155,9 @@ def test_load_stage(dvc, stage_data, lock_data):
assert stage.cmd == "command"
assert stage.path == os.path.abspath(PIPELINE_FILE)
assert stage.deps[0].def_path == "foo"
assert stage.deps[0].checksum == "foo_checksum"
assert stage.deps[0].hash_info == HashInfo("md5", "foo_checksum")
assert stage.outs[0].def_path == "bar"
assert stage.outs[0].checksum == "bar_checksum"
assert stage.outs[0].hash_info == HashInfo("md5", "bar_checksum")
def test_load_stage_outs_with_flags(dvc, stage_data, lock_data):
@@ -171,8 +171,8 @@ def test_load_stage_no_lock(dvc, stage_data):
dvcfile = Dvcfile(dvc, PIPELINE_FILE)
stage = StageLoader.load_stage(dvcfile, "stage-1", stage_data)
assert stage.deps[0].def_path == "foo" and stage.outs[0].def_path == "bar"
assert not stage.deps[0].checksum
assert not stage.outs[0].checksum
assert not stage.deps[0].hash_info
assert not stage.outs[0].hash_info
def test_load_stage_with_params(dvc, stage_data, lock_data):
@@ -185,8 +185,8 @@ def test_load_stage_with_params(dvc, stage_data, lock_data):
assert deps[0].def_path == "foo" and stage.outs[0].def_path == "bar"
assert params[0].def_path == "params.yaml"
assert params[0].hash_info == HashInfo("params", {"lorem": "ipsum"})
assert deps[0].checksum == "foo_checksum"
assert stage.outs[0].checksum == "bar_checksum"
assert deps[0].hash_info == HashInfo("md5", "foo_checksum")
assert stage.outs[0].hash_info == HashInfo("md5", "bar_checksum")
@pytest.mark.parametrize("typ", ["metrics", "plots"])
@@ -196,7 +196,7 @@ def test_load_stage_with_metrics_and_plots(dvc, stage_data, lock_data, typ):
stage = StageLoader.load_stage(dvcfile, "stage-1", stage_data, lock_data)
assert stage.outs[0].def_path == "bar"
assert stage.outs[0].checksum == "bar_checksum"
assert stage.outs[0].hash_info == HashInfo("md5", "bar_checksum")
def test_load_changed_command(dvc, stage_data, lock_data):