Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

tree: move get_hash to object id logic (#5460)

Browse Source
This commit is contained in:
Ruslan Kuprieiev
2021-02-13 04:22:29 +02:00
committed by GitHub
parent f4b14c2d4a
commit 9ce6bce9b6
39 changed files with 385 additions and 416 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/api/__init__.py
+2 -1
View File
@@ -4,6 +4,7 @@ from contextlib import _GeneratorContextManager as GCM
from funcy import reraise
from dvc.exceptions import OutputNotFoundError, PathMissingError
from dvc.oid import get_hash
from dvc.path_info import PathInfo
from dvc.repo import Repo
@@ -28,7 +29,7 @@ def get_url(path, repo=None, rev=None, remote=None):
raise OutputNotFoundError(path, repo)
cloud = metadata.repo.cloud
hash_info = _repo.repo_tree.get_hash(path_info, "md5")
hash_info = get_hash(path_info, _repo.repo_tree, "md5")
return cloud.get_url_for(remote, checksum=hash_info.value)
dvc/checkout.py
+3 -2
View File
@@ -10,6 +10,7 @@ from dvc.exceptions import (
DvcException,
)
from dvc.objects import ObjectFormatError, check, load
from dvc.oid import get_hash
from dvc.remote.slow_link_detection import ( # type: ignore[attr-defined]
slow_link_guard,
)
@@ -29,7 +30,7 @@ def _changed(path_info, tree, obj, cache):
return True
try:
actual = tree.get_hash(path_info, obj.hash_info.name)
actual = get_hash(path_info, tree, obj.hash_info.name)
except FileNotFoundError:
logger.debug("'%s' doesn't exist.", path_info)
return True
@@ -55,7 +56,7 @@ def _remove(path_info, tree, cache, force=False):
tree.remove(path_info)
return
current = tree.get_hash(path_info, tree.PARAM_CHECKSUM)
current = get_hash(path_info, tree, tree.PARAM_CHECKSUM)
try:
obj = load(cache, current)
check(cache, obj)
dvc/dependency/repo.py
+4 -2
View File
@@ -47,10 +47,12 @@ class RepoDependency(LocalDependency):
return external_repo(d["url"], rev=rev, **kwargs)
def _get_hash(self, locked=True):
from dvc.oid import get_hash
with self._make_repo(locked=locked) as repo:
path_info = PathInfo(repo.root_dir) / self.def_path
return repo.repo_tree.get_hash(
path_info, "md5", follow_subrepos=False
return get_hash(
path_info, repo.repo_tree, "md5", follow_subrepos=False
)
def workspace_status(self):
dvc/objects.py
+9 -4
View File
@@ -7,6 +7,7 @@ from dvc.progress import Tqdm
from .dir_info import DirInfo
from .exceptions import DvcException
from .oid import get_hash
logger = logging.getLogger(__name__)
@@ -35,7 +36,7 @@ class HashFile:
return bool(self.hash_info)
def check(self, odb):
actual = self.tree.get_hash(self.path_info, odb.tree.PARAM_CHECKSUM)
actual = get_hash(self.path_info, self.tree, odb.tree.PARAM_CHECKSUM)
logger.trace(
"cache '%s' expected '%s' actual '%s'",
@@ -63,7 +64,9 @@ class File(HashFile):
@classmethod
def stage(cls, odb, path_info, tree, **kwargs):
hash_info = tree.get_hash(path_info, odb.tree.PARAM_CHECKSUM, **kwargs)
hash_info = get_hash(
path_info, tree, odb.tree.PARAM_CHECKSUM, **kwargs
)
raw = odb.get(hash_info)
obj = cls(raw.path_info, raw.tree, hash_info)
obj.src = HashFile(path_info, tree, hash_info)
@@ -113,7 +116,7 @@ class Tree(HashFile):
with tree.open(path_info, "rb") as fobj:
odb.tree.upload_fobj(fobj, tmp_info)
hash_info = odb.tree.get_hash(tmp_info, odb.tree.PARAM_CHECKSUM)
hash_info = get_hash(tmp_info, odb.tree, odb.tree.PARAM_CHECKSUM)
hash_info.value += odb.tree.CHECKSUM_DIR_SUFFIX
hash_info.dir_info = dir_info
hash_info.nfiles = dir_info.nfiles
@@ -124,7 +127,9 @@ class Tree(HashFile):
@classmethod
def stage(cls, odb, path_info, tree, **kwargs):
hash_info = tree.get_hash(path_info, odb.tree.PARAM_CHECKSUM, **kwargs)
hash_info = get_hash(
path_info, tree, odb.tree.PARAM_CHECKSUM, **kwargs
)
hi = cls.save_dir_info(odb, hash_info.dir_info, hash_info)
hi.size = hash_info.size
raw = odb.get(hi)
dvc/oid.py
+144
View File
@@ -0,0 +1,144 @@
import errno
import os
from concurrent.futures import ThreadPoolExecutor
from dvc.exceptions import DvcIgnoreInCollectedDirError
from dvc.hash_info import HashInfo
from dvc.ignore import DvcIgnore
from dvc.progress import Tqdm
from dvc.utils import file_md5
def get_file_hash(path_info, tree, name):
info = tree.info(path_info)
if name in info:
assert not info[name].endswith(".dir")
return HashInfo(name, info[name], size=info["size"])
func = getattr(tree, name, None)
if func:
return func(path_info)
if name == "md5":
return HashInfo(
name, file_md5(path_info, tree), size=tree.getsize(path_info)
)
raise NotImplementedError
def _calculate_hashes(file_infos, tree, name):
def _get_file_hash(path_info):
return get_file_hash(path_info, tree, name)
with Tqdm(
total=len(file_infos),
unit="md5",
desc="Computing file/dir hashes (only done once)",
) as pbar:
worker = pbar.wrap_fn(_get_file_hash)
with ThreadPoolExecutor(max_workers=tree.hash_jobs) as executor:
hash_infos = executor.map(worker, file_infos)
return dict(zip(file_infos, hash_infos))
def _iter_hashes(path_info, tree, name, **kwargs):
if name in tree.DETAIL_FIELDS:
for details in tree.ls(path_info, recursive=True, detail=True):
file_info = path_info.replace(path=details["name"])
hash_info = HashInfo(
name, details[name], size=details.get("size"),
)
yield file_info, hash_info
return None
file_infos = []
for file_info in tree.walk_files(path_info, **kwargs):
hash_info = tree.state.get( # pylint: disable=assignment-from-none
file_info
)
if not hash_info:
file_infos.append(file_info)
continue
assert hash_info.name == name
yield file_info, hash_info
yield from _calculate_hashes(file_infos, tree, name).items()
def _collect_dir(path_info, tree, name, **kwargs):
from dvc.dir_info import DirInfo
dir_info = DirInfo()
for fi, hi in _iter_hashes(path_info, tree, name, **kwargs):
if DvcIgnore.DVCIGNORE_FILE == fi.name:
raise DvcIgnoreInCollectedDirError(fi.parent)
# NOTE: this is lossy transformation:
# "hey\there" -> "hey/there"
# "hey/there" -> "hey/there"
# The latter is fine filename on Windows, which
# will transform to dir/file on back transform.
#
# Yes, this is a BUG, as long as we permit "/" in
# filenames on Windows and "\" on Unix
dir_info.trie[fi.relative_to(path_info).parts] = hi
return dir_info
def get_dir_hash(path_info, tree, name, **kwargs):
from dvc.objects import Tree
dir_info = _collect_dir(path_info, tree, name, **kwargs)
hash_info = Tree.save_dir_info(tree.repo.cache.local, dir_info)
hash_info.size = dir_info.size
hash_info.dir_info = dir_info
return hash_info
def get_hash(path_info, tree, name, **kwargs):
assert path_info and (
isinstance(path_info, str) or path_info.scheme == tree.scheme
)
if not tree.exists(path_info):
raise FileNotFoundError(
errno.ENOENT, os.strerror(errno.ENOENT), path_info
)
with tree.state:
# pylint: disable=assignment-from-none
hash_info = tree.state.get(path_info)
# If we have dir hash in state db, but dir cache file is lost,
# then we need to recollect the dir via .get_dir_hash() call below,
# see https://github.com/iterative/dvc/issues/2219 for context
if (
hash_info
and hash_info.isdir
and not tree.cache.tree.exists(
tree.cache.hash_to_path_info(hash_info.value)
)
):
hash_info = None
if hash_info:
if hash_info.isdir:
from dvc.objects import Tree
# NOTE: loading the tree will restore hash_info.dir_info
Tree.load(tree.cache, hash_info)
assert hash_info.name == name
return hash_info
if tree.isdir(path_info):
hash_info = get_dir_hash(path_info, tree, name, **kwargs)
else:
hash_info = get_file_hash(path_info, tree, name)
if hash_info and tree.exists(path_info):
tree.state.save(path_info, hash_info)
return hash_info
dvc/output/base.py
+4 -1
View File
@@ -18,6 +18,7 @@ from dvc.exceptions import (
RemoteCacheRequiredError,
)
from dvc.hash_info import HashInfo
from dvc.oid import get_hash
from ..tree.base import BaseTree
@@ -195,7 +196,9 @@ class BaseOutput:
def get_hash(self):
if not self.use_cache:
return self.tree.get_hash(self.path_info, self.tree.PARAM_CHECKSUM)
return get_hash(
self.path_info, self.tree, self.tree.PARAM_CHECKSUM
)
return objects.stage(self.cache, self.path_info, self.tree).hash_info
@property
dvc/repo/diff.py
+4 -3
View File
@@ -2,6 +2,7 @@ import logging
import os
from dvc.exceptions import PathMissingError
from dvc.oid import get_file_hash, get_hash
from dvc.repo import locked
logger = logging.getLogger(__name__)
@@ -127,8 +128,8 @@ def _output_paths(repo, repo_tree, targets):
def _to_checksum(output):
if on_working_tree:
return repo.cache.local.tree.get_hash(
output.path_info, "md5"
return get_hash(
output.path_info, repo.cache.local.tree, "md5"
).value
return output.hash_info.value
@@ -157,7 +158,7 @@ def _dir_output_paths(repo_tree, output, targets=None):
if targets is None or any(
fname.isin_or_eq(target) for target in targets
):
yield str(fname), repo_tree.get_file_hash(fname, "md5").value
yield str(fname), get_file_hash(fname, repo_tree, "md5").value
except NoRemoteError:
logger.warning("dir cache entry for '%s' is missing", output)
dvc/tree/azure.py
+7 -8
View File
@@ -5,7 +5,6 @@ from datetime import datetime, timedelta
from funcy import cached_property, wrap_prop
from dvc.hash_info import HashInfo
from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.scheme import Schemes
@@ -167,7 +166,7 @@ class AzureTree(BaseTree):
"type": "file",
"name": blob.name,
"size": blob.size,
"etag": blob.etag,
"etag": blob.etag.strip('"'),
}
else:
yield blob.name
@@ -181,16 +180,16 @@ class AzureTree(BaseTree):
path_info.bucket, path_info.path
).delete_blob()
def getsize(self, path_info):
def info(self, path_info):
blob_client = self.blob_service.get_blob_client(
path_info.bucket, path_info.path
)
properties = blob_client.get_blob_properties()
return properties.size
def get_file_hash(self, path_info, name):
assert name == self.PARAM_CHECKSUM
return HashInfo(self.PARAM_CHECKSUM, self.get_etag(path_info))
return {
"type": "file",
"size": properties.size,
"etag": properties.etag.strip('"'),
}
def _upload_fobj(self, fobj, to_info):
blob_client = self.blob_service.get_blob_client(
dvc/tree/base.py
+5 -122
View File
@@ -1,6 +1,4 @@
import errno
import logging
import os
from concurrent.futures import ThreadPoolExecutor, as_completed
from functools import partial
from multiprocessing import cpu_count
@@ -9,10 +7,7 @@ from urllib.parse import urlparse
from funcy import cached_property, decorator
from dvc.dir_info import DirInfo
from dvc.exceptions import DvcException, DvcIgnoreInCollectedDirError
from dvc.hash_info import HashInfo
from dvc.ignore import DvcIgnore
from dvc.exceptions import DvcException
from dvc.path_info import URLInfo
from dvc.progress import Tqdm
from dvc.state import StateNoop
@@ -212,8 +207,11 @@ class BaseTree:
def is_empty(self, path_info):
return False
def info(self, path_info):
raise NotImplementedError
def getsize(self, path_info):
return None
return self.info(path_info).get("size")
def remove(self, path_info):
raise RemoteActionNotImplemented("remove", self.scheme)
@@ -247,121 +245,6 @@ class BaseTree:
return False
return hash_.endswith(cls.CHECKSUM_DIR_SUFFIX)
@use_state
def get_hash(self, path_info, name, **kwargs):
assert path_info and (
isinstance(path_info, str) or path_info.scheme == self.scheme
)
if not self.exists(path_info):
raise FileNotFoundError(
errno.ENOENT, os.strerror(errno.ENOENT), path_info
)
# pylint: disable=assignment-from-none
hash_info = self.state.get(path_info)
# If we have dir hash in state db, but dir cache file is lost,
# then we need to recollect the dir via .get_dir_hash() call below,
# see https://github.com/iterative/dvc/issues/2219 for context
if (
hash_info
and hash_info.isdir
and not self.cache.tree.exists(
self.cache.hash_to_path_info(hash_info.value)
)
):
hash_info = None
if hash_info:
assert hash_info.name == self.PARAM_CHECKSUM
if hash_info.isdir:
from dvc.objects import Tree
# NOTE: loading the tree will restore hash_info.dir_info
Tree.load(self.cache, hash_info)
assert hash_info.name == name
return hash_info
if self.isdir(path_info):
hash_info = self.get_dir_hash(path_info, name, **kwargs)
else:
hash_info = self.get_file_hash(path_info, name)
if hash_info and self.exists(path_info):
self.state.save(path_info, hash_info)
return hash_info
def get_file_hash(self, path_info, name):
raise NotImplementedError
def _calculate_hashes(self, file_infos, name):
def _get_file_hash(path_info):
return self.get_file_hash(path_info, name)
with Tqdm(
total=len(file_infos),
unit="md5",
desc="Computing file/dir hashes (only done once)",
) as pbar:
worker = pbar.wrap_fn(_get_file_hash)
with ThreadPoolExecutor(max_workers=self.hash_jobs) as executor:
hash_infos = executor.map(worker, file_infos)
return dict(zip(file_infos, hash_infos))
def _iter_hashes(self, path_info, name, **kwargs):
if name in self.DETAIL_FIELDS:
for details in self.ls(path_info, recursive=True, detail=True):
file_info = path_info.replace(path=details["name"])
hash_info = HashInfo(
name, details[name], size=details.get("size"),
)
yield file_info, hash_info
return None
file_infos = []
for file_info in self.walk_files(path_info, **kwargs):
hash_info = self.state.get( # pylint: disable=assignment-from-none
file_info
)
if not hash_info:
file_infos.append(file_info)
continue
assert hash_info.name == name
yield file_info, hash_info
yield from self._calculate_hashes(file_infos, name).items()
def _collect_dir(self, path_info, name, **kwargs):
dir_info = DirInfo()
for fi, hi in self._iter_hashes(path_info, name, **kwargs):
if DvcIgnore.DVCIGNORE_FILE == fi.name:
raise DvcIgnoreInCollectedDirError(fi.parent)
# NOTE: this is lossy transformation:
# "hey\there" -> "hey/there"
# "hey/there" -> "hey/there"
# The latter is fine filename on Windows, which
# will transform to dir/file on back transform.
#
# Yes, this is a BUG, as long as we permit "/" in
# filenames on Windows and "\" on Unix
dir_info.trie[fi.relative_to(path_info).parts] = hi
return dir_info
@use_state
def get_dir_hash(self, path_info, name, **kwargs):
from dvc.objects import Tree
dir_info = self._collect_dir(path_info, name, **kwargs)
hash_info = Tree.save_dir_info(self.repo.cache.local, dir_info)
hash_info.size = dir_info.size
hash_info.dir_info = dir_info
return hash_info
def upload(
self, from_info, to_info, name=None, no_progress_bar=False,
):
dvc/tree/dvc.py
+18 -25
View File
@@ -54,7 +54,7 @@ class DvcTree(BaseTree): # pylint:disable=abstract-method
raise FileNotFoundError
def open( # type: ignore
self, path: PathInfo, mode="r", encoding="utf-8", remote=None, **kwargs
self, path: PathInfo, mode="r", encoding=None, remote=None, **kwargs
): # pylint: disable=arguments-differ
try:
outs = self._find_outs(path, strict=False)
@@ -219,30 +219,6 @@ class DvcTree(BaseTree): # pylint:disable=abstract-method
def isexec(self, path_info): # pylint: disable=unused-argument
return False
def get_dir_hash(self, path_info, name, **kwargs):
try:
outs = self._find_outs(path_info, strict=True)
if len(outs) == 1 and outs[0].is_dir_checksum:
out = outs[0]
# other code expects us to fetch the dir at this point
self._fetch_dir(out, **kwargs)
assert out.hash_info.name == name
return out.hash_info
except OutputNotFoundError:
pass
return super().get_dir_hash(path_info, name, **kwargs)
def get_file_hash(self, path_info, name):
assert name == "md5"
outs = self._find_outs(path_info, strict=False)
if len(outs) != 1:
raise OutputNotFoundError
out = outs[0]
if out.is_dir_checksum:
return self._get_granular_hash(path_info, out)
return out.hash_info
def metadata(self, path_info):
path_info = PathInfo(os.path.abspath(path_info))
@@ -254,3 +230,20 @@ class DvcTree(BaseTree): # pylint:disable=abstract-method
meta = Metadata(path_info=path_info, outs=outs, repo=self.repo)
meta.isdir = meta.isdir or self.check_isdir(meta.path_info, meta.outs)
return meta
def info(self, path_info):
meta = self.metadata(path_info)
ret = {"type": "dir" if meta.isdir else "file"}
if meta.is_output and len(meta.outs) == 1 and meta.outs[0].hash_info:
hash_info = meta.outs[0].hash_info
ret["size"] = hash_info.size
ret[hash_info.name] = hash_info.value
elif meta.part_of_output:
(out,) = meta.outs
key = path_info.relative_to(out.path_info).parts
hash_info = out.hash_info.dir_info.trie.get(key)
if hash_info:
ret["size"] = hash_info.size
ret[hash_info.name] = hash_info.value
return ret
dvc/tree/gdrive.py
+2 -5
View File
@@ -599,14 +599,11 @@ class GDriveTree(BaseTree):
item_id = self._get_item_id(path_info)
self.gdrive_delete_file(item_id)
def get_file_hash(self, path_info, name):
raise NotImplementedError
def getsize(self, path_info):
def info(self, path_info):
item_id = self._get_item_id(path_info)
gdrive_file = self._drive.CreateFile({"id": item_id})
gdrive_file.FetchMetadata(fields="fileSize")
return gdrive_file.get("fileSize")
return {"size": gdrive_file.get("fileSize")}
def _upload_fobj(self, fobj, to_info):
dirname = to_info.parent
dvc/tree/git.py
+10
View File
@@ -138,6 +138,16 @@ class GitTree(BaseTree): # pylint:disable=abstract-method
mode = self.stat(path_info).st_mode
return is_exec(mode)
def info(self, path_info):
key = self._get_key(path_info)
try:
st = self.trie.stat(key)
except KeyError:
raise FileNotFoundError(
errno.ENOENT, os.strerror(errno.ENOENT), path_info
)
return {"size": st.st_size}
def stat(self, path):
key = self._get_key(path)
try:
dvc/tree/gs.py
+2 -22
View File
@@ -7,7 +7,6 @@ from functools import wraps
from funcy import cached_property, wrap_prop
from dvc.exceptions import DvcException
from dvc.hash_info import HashInfo
from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.scheme import Schemes
@@ -124,10 +123,10 @@ class GSTree(BaseTree):
blob = self.gs.bucket(path_info.bucket).blob(path_info.path)
return blob.exists()
def getsize(self, path_info):
def info(self, path_info):
bucket = self.gs.bucket(path_info.bucket)
blob = bucket.get_blob(path_info.path)
return blob.size
return {"type": "file", "size": blob.size}
def _list_paths(self, path_info, max_items=None):
for blob in self.gs.bucket(path_info.bucket).list_blobs(
@@ -194,25 +193,6 @@ class GSTree(BaseTree):
to_bucket = self.gs.bucket(to_info.bucket)
from_bucket.copy_blob(blob, to_bucket, new_name=to_info.path)
def get_file_hash(self, path_info, name):
import base64
import codecs
assert name == self.PARAM_CHECKSUM
bucket = path_info.bucket
path = path_info.path
blob = self.gs.bucket(bucket).get_blob(path)
if not blob:
return HashInfo(self.PARAM_CHECKSUM, None)
b64_md5 = blob.md5_hash
md5 = base64.b64decode(b64_md5)
return HashInfo(
self.PARAM_CHECKSUM,
codecs.getencoder("hex")(md5)[0].decode("utf-8"),
size=blob.size,
)
def _upload_fobj(self, fobj, to_info):
bucket = self.gs.bucket(to_info.bucket)
# With other references being given in the @dynamic_chunk_size
dvc/tree/hdfs.py
+5 -7
View File
@@ -222,16 +222,14 @@ class HDFSTree(BaseTree):
file_info = hdfs.get_file_info(path_info.path)
return file_info.type == pyarrow.fs.FileType.Directory
def getsize(self, path_info):
def info(self, path_info):
with self.hdfs(path_info) as hdfs:
file_info = hdfs.get_file_info(path_info.path)
return file_info.size
finfo = hdfs.get_file_info(path_info.path)
return {"size": finfo.size}
def get_file_hash(self, path_info, name):
assert name == self.PARAM_CHECKSUM
# NOTE: pyarrow doesn't support checksum, so we need to use hadoop
def checksum(self, path_info):
return HashInfo(
self.PARAM_CHECKSUM,
"checksum",
_hadoop_fs_checksum(path_info),
size=self.getsize(path_info),
)
dvc/tree/http.py
+3 -20
View File
@@ -6,7 +6,6 @@ from funcy import cached_property, memoize, wrap_prop, wrap_with
import dvc.prompt as prompt
from dvc.exceptions import DvcException, HTTPError
from dvc.hash_info import HashInfo
from dvc.path_info import HTTPURLInfo
from dvc.progress import Tqdm
from dvc.scheme import Schemes
@@ -146,28 +145,12 @@ class HTTPTree(BaseTree): # pylint:disable=abstract-method
return True
raise HTTPError(res.status_code, res.reason)
def getsize(self, path_info):
response = self.request("GET", path_info.url, stream=True)
if response.status_code != 200:
raise HTTPError(response.status_code, response.reason)
return self._content_length(response)
def get_file_hash(self, path_info, name):
assert name == self.PARAM_CHECKSUM
def info(self, path_info):
url = path_info.url
headers = self._head(url).headers
etag = headers.get("ETag") or headers.get("Content-MD5")
if not etag:
raise DvcException(
"could not find an ETag or "
"Content-MD5 header for '{url}'".format(url=url)
)
return HashInfo(self.PARAM_CHECKSUM, etag)
size = int(headers.get("Content-Length"))
return {"etag": etag, "size": size}
def _upload_fobj(self, fobj, to_info):
def chunks(fobj):
dvc/tree/local.py
+8 -13
View File
@@ -1,13 +1,13 @@
import logging
import os
import stat
from funcy import cached_property
from dvc.hash_info import HashInfo
from dvc.path_info import PathInfo
from dvc.scheme import Schemes
from dvc.system import System
from dvc.utils import file_md5, is_exec, tmp_fname
from dvc.utils import is_exec, tmp_fname
from dvc.utils.fs import copy_fobj_to_file, copyfile, makedirs, move, remove
from .base import BaseTree
@@ -207,18 +207,13 @@ class LocalTree(BaseTree):
def reflink(self, from_info, to_info):
System.reflink(from_info, to_info)
def get_file_hash(self, path_info, name):
assert name == "md5"
hash_info = HashInfo(name, file_md5(path_info))
if hash_info:
hash_info.size = os.path.getsize(path_info)
return hash_info
@staticmethod
def getsize(path_info):
return os.path.getsize(path_info)
def info(path_info):
st = os.stat(path_info)
return {
"size": st.st_size,
"type": "dir" if stat.S_ISDIR(st.st_mode) else "file",
}
def _upload(
self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs,
dvc/tree/memory.py
-7
View File
@@ -1,6 +1,3 @@
from dvc.hash_info import HashInfo
from dvc.utils import file_md5
from .base import BaseTree
@@ -30,9 +27,5 @@ class MemoryTree(BaseTree):
return os.stat_result((0, 0, 0, 0, 0, 0, info["size"], 0, 0, 0))
def get_file_hash(self, path_info, name):
assert name == self.PARAM_CHECKSUM
return HashInfo(self.PARAM_CHECKSUM, file_md5(path_info, self))
def walk_files(self, path_info, **kwargs):
raise NotImplementedError
dvc/tree/repo.py
+8 -33
View File
@@ -8,7 +8,6 @@ from typing import TYPE_CHECKING, Callable, Optional, Tuple, Type, Union
from funcy import lfilter, wrap_with
from dvc.hash_info import HashInfo
from dvc.path_info import PathInfo
from .base import BaseTree
@@ -370,38 +369,6 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
for fname in files:
yield PathInfo(root) / fname
def get_dir_hash(
self, path_info, name, follow_subrepos=None, **kwargs
): # pylint: disable=arguments-differ
tree, dvc_tree = self._get_tree_pair(path_info)
if tree.exists(path_info):
return super().get_dir_hash(
path_info, name, follow_subrepos=follow_subrepos, **kwargs
)
if not dvc_tree:
raise FileNotFoundError
return dvc_tree.get_dir_hash(path_info, name, **kwargs)
def get_file_hash(self, path_info, name):
"""Return file checksum for specified path.
If path_info is a DVC out, the pre-computed checksum for the file
will be used. If path_info is a git file, MD5 will be computed for
the git object.
"""
from dvc.utils import file_md5
assert name == self.PARAM_CHECKSUM
if not self.exists(path_info):
raise FileNotFoundError
_, dvc_tree = self._get_tree_pair(path_info)
if dvc_tree and dvc_tree.exists(path_info):
try:
return dvc_tree.get_file_hash(path_info, name)
except FileNotFoundError:
pass
return HashInfo(self.PARAM_CHECKSUM, file_md5(path_info, self))
def _download(
self, from_info, to_file, name=None, no_progress_bar=False, **kwargs
):
@@ -452,3 +419,11 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method
meta.is_exec = bool(stat_result) and is_exec(stat_result.st_mode)
return meta
def info(self, path_info):
tree, dvc_tree = self._get_tree_pair(path_info)
try:
return tree.info(path_info)
except FileNotFoundError:
return dvc_tree.info(path_info)
dvc/tree/s3.py
+7 -9
View File
@@ -260,9 +260,12 @@ class S3Tree(BaseTree):
return path_info.path in self._list_paths(path_info)
def getsize(self, path_info):
def info(self, path_info):
with self._get_obj(path_info) as obj:
return obj.content_length
return {
"etag": obj.e_tag.strip('"'),
"size": obj.content_length,
}
def _list_paths(self, path_info, max_items=None):
with self._get_bucket(path_info.bucket) as bucket:
@@ -418,14 +421,9 @@ class S3Tree(BaseTree):
if etag != cached_etag:
raise ETagMismatchError(etag, cached_etag)
def get_file_hash(self, path_info, name):
assert name == self.PARAM_CHECKSUM
def etag(self, path_info):
with self._get_obj(path_info) as obj:
return HashInfo(
self.PARAM_CHECKSUM,
obj.e_tag.strip('"'),
size=obj.content_length,
)
return HashInfo("etag", size=obj.content_length,)
def _upload_fobj(self, fobj, to_info):
with self._get_obj(to_info) as obj:
dvc/tree/ssh/__init__.py
+8 -13
View File
@@ -235,22 +235,17 @@ class SSHTree(BaseTree):
with self.ssh(from_info) as ssh:
ssh.reflink(from_info.path, to_info.path)
def get_file_hash(self, path_info, name):
if path_info.scheme != self.scheme:
raise NotImplementedError
assert name == self.PARAM_CHECKSUM
def md5(self, path_info):
with self.ssh(path_info) as ssh:
hash_info = HashInfo(self.PARAM_CHECKSUM, ssh.md5(path_info.path),)
return HashInfo(
"md5",
ssh.md5(path_info.path),
size=ssh.getsize(path_info.path),
)
if hash_info:
hash_info.size = ssh.getsize(path_info.path)
return hash_info
def getsize(self, path_info):
def info(self, path_info):
with self.ssh(path_info) as ssh:
return ssh.getsize(path_info.path)
return ssh.info(path_info.path)
def _upload_fobj(self, fobj, to_info):
self.makedirs(to_info.parent)
dvc/tree/ssh/connection.py
+8 -5
View File
@@ -76,12 +76,15 @@ class SSHConnection:
return lstat.st_mode if lstat else 0
def getsize(self, path):
lstat = None
with suppress(FileNotFoundError):
lstat = self.sftp.lstat(path)
def info(self, path):
st = self.sftp.stat(path)
return {
"size": st.st_size,
"type": "dir" if stat.S_ISDIR(st.st_mode) else "file",
}
return lstat.st_size if lstat else 0
def getsize(self, path):
return self.info(path)["size"]
def exists(self, path):
return bool(self.st_mode(path))
dvc/tree/webdav.py
+6 -21
View File
@@ -8,7 +8,6 @@ from funcy import cached_property, nullcontext, wrap_prop
from dvc.config import ConfigError
from dvc.exceptions import DvcException
from dvc.hash_info import HashInfo
from dvc.path_info import HTTPURLInfo, WebDAVURLInfo
from dvc.progress import Tqdm
from dvc.scheme import Schemes
@@ -147,22 +146,6 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method
# Use webdav check to test for file existence
return self._client.check(path_info.path)
# Gets file hash 'etag'
def get_file_hash(self, path_info, name):
assert name == self.PARAM_CHECKSUM
# Use webdav client info method to get etag
etag = self._client.info(path_info.path)["etag"].strip('"')
# From HTTPTree
if not etag:
raise DvcException(
"could not find an ETag or "
"Content-MD5 header for '{url}'".format(url=path_info.url)
)
return HashInfo(self.PARAM_CHECKSUM, etag)
# Checks whether path points to directory
def isdir(self, path_info):
# Use webdav is_dir to test whether path points to a directory
@@ -293,7 +276,9 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method
buff=fd_wrapped, remote_path=to_info.path
)
# Queries size of file at remote
def getsize(self, path_info):
# Get file size from info dictionary and convert to int (from str)
return int(self._client.info(path_info.path)["size"])
def info(self, path_info):
info = self._client.info(path_info.path)
return {
"size": int(info["size"]),
"etag": info["etag"],
}
dvc/tree/webhdfs.py
+9 -10
View File
@@ -121,17 +121,16 @@ class WebHDFSTree(BaseTree):
status = self.hdfs_client.status(path_info.path, strict=False)
return status is not None
def getsize(self, path_info):
return self.hdfs_client.status(path_info.path)["length"]
def info(self, path_info):
st = self.hdfs_client.status(path_info.path)
return {"size": st["length"]}
def get_file_hash(self, path_info, name):
assert name == self.PARAM_CHECKSUM
checksum = self.hdfs_client.checksum(path_info.path)
hash_info = HashInfo(self.PARAM_CHECKSUM, checksum["bytes"])
hash_info.size = self.hdfs_client.status(path_info.path)["length"]
return hash_info
def checksum(self, path_info):
return HashInfo(
"checksum",
self.hdfs_client.checksum(path_info.path)["bytes"],
size=self.hdfs_client.status(path_info.path)["length"],
)
def copy(self, from_info, to_info, **_kwargs):
with self.hdfs_client.read(from_info.path) as reader:
dvc/utils/__init__.py
+9 -14
View File
@@ -41,36 +41,31 @@ def _fobj_md5(fobj, hash_md5, binary, progress_func=None):
progress_func(len(data))
def file_md5(fname, tree=None):
def file_md5(fname, tree):
""" get the (md5 hexdigest, md5 digest) of a file """
from dvc.istextfile import istextfile
from dvc.progress import Tqdm
if tree:
stat_func = tree.stat
open_func = tree.open
else:
stat_func = os.stat
open_func = open
hash_md5 = hashlib.md5()
binary = not istextfile(fname, tree=tree)
size = stat_func(fname).st_size
size = tree.getsize(fname) or 0
no_progress_bar = True
if size >= LARGE_FILE_SIZE:
no_progress_bar = False
msg = "Computing md5 for a large file '{}'. This is only done once."
logger.info(msg.format(relpath(fname)))
name = relpath(fname)
msg = (
f"Computing md5 for a large file '{fname}'. "
"This is only done once."
)
logger.info(msg)
with Tqdm(
desc=name,
desc=str(fname),
disable=no_progress_bar,
total=size,
bytes=True,
leave=False,
) as pbar:
with open_func(fname, "rb") as fobj:
with tree.open(fname, "rb") as fobj:
_fobj_md5(fobj, hash_md5, binary, pbar.update)
return hash_md5.hexdigest()
tests/func/test_add.py
+4 -4
View File
@@ -38,7 +38,7 @@ from tests.utils import get_gitignore_content
def test_add(tmp_dir, dvc):
(stage,) = tmp_dir.dvc_gen({"foo": "foo"})
md5 = file_md5("foo")
md5 = file_md5("foo", dvc.tree)
assert stage is not None
@@ -443,7 +443,7 @@ class TestDoubleAddUnchanged(TestDvc):
def test_should_update_state_entry_for_file_after_add(mocker, dvc, tmp_dir):
file_md5_counter = mocker.spy(dvc_module.tree.local, "file_md5")
file_md5_counter = mocker.spy(dvc_module.oid, "file_md5")
tmp_dir.gen("foo", "foo")
ret = main(["config", "cache.type", "copy"])
@@ -474,7 +474,7 @@ def test_should_update_state_entry_for_file_after_add(mocker, dvc, tmp_dir):
def test_should_update_state_entry_for_directory_after_add(
mocker, dvc, tmp_dir
):
file_md5_counter = mocker.spy(dvc_module.tree.local, "file_md5")
file_md5_counter = mocker.spy(dvc_module.oid, "file_md5")
tmp_dir.gen({"data/data": "foo", "data/data_sub/sub_data": "foo"})
@@ -521,7 +521,7 @@ class TestAddCommit(TestDvc):
def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
tmp_dir.gen({"data/data": "foo"})
get_dir_hash_counter = mocker.spy(LocalTree, "get_dir_hash")
get_dir_hash_counter = mocker.spy(dvc_module.oid, "get_dir_hash")
ret = main(["add", "data"])
assert ret == 0
tests/func/test_data_cloud.py
+3 -3
View File
@@ -5,6 +5,7 @@ import shutil
import pytest
from flaky.flaky_decorator import flaky
import dvc as dvc_module
from dvc.cache import NamedCache
from dvc.external_repo import clean_repos
from dvc.main import main
@@ -15,7 +16,6 @@ from dvc.remote.base import (
STATUS_OK,
)
from dvc.stage.exceptions import StageNotFound
from dvc.tree.local import LocalTree
from dvc.utils.fs import move, remove
from dvc.utils.serialize import dump_yaml, load_yaml
@@ -231,7 +231,7 @@ def test_warn_on_outdated_stage(tmp_dir, dvc, local_remote, caplog):
def test_hash_recalculation(mocker, dvc, tmp_dir, local_remote):
tmp_dir.gen({"foo": "foo"})
test_get_file_hash = mocker.spy(LocalTree, "get_file_hash")
test_get_file_hash = mocker.spy(dvc_module.oid, "get_file_hash")
ret = main(["config", "cache.type", "hardlink"])
assert ret == 0
ret = main(["add", "foo"])
@@ -287,7 +287,7 @@ def test_verify_hashes(
remove("dir")
remove(dvc.cache.local.cache_dir)
hash_spy = mocker.spy(dvc.cache.local.tree, "get_file_hash")
hash_spy = mocker.spy(dvc_module.oid, "get_file_hash")
dvc.pull()
assert hash_spy.call_count == 0
tests/func/test_install.py
+1 -1
View File
@@ -64,7 +64,7 @@ class TestInstall:
conf["core"]["remote"] = "store"
tmp_dir.dvc_gen("file", "file_content", "commit message")
file_checksum = file_md5("file")
file_checksum = file_md5("file", dvc.tree)
expected_storage_path = (
storage_path / file_checksum[:2] / file_checksum[2:]
)
tests/func/test_remote.py
+8 -7
View File
@@ -11,7 +11,6 @@ from dvc.dir_info import DirInfo
from dvc.exceptions import DownloadError, RemoteCacheRequiredError, UploadError
from dvc.main import main
from dvc.path_info import PathInfo
from dvc.tree.base import BaseTree
from dvc.tree.local import LocalTree
from dvc.utils.fs import remove
from tests.basic_env import TestDvc
@@ -147,6 +146,8 @@ class TestRemoteShouldHandleUppercaseRemoteName(TestDvc):
def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
from dvc.oid import get_hash
tmp_dir.gen({"data": {"1": "1 content", "2": "2 content"}})
path_info = PathInfo("data")
@@ -154,18 +155,18 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
dir_info = DirInfo.from_list(
[{"relpath": "1", "md5": "1"}, {"relpath": "2", "md5": "2"}]
)
with patch.object(
BaseTree, "_collect_dir", return_value=dir_info,
with patch(
"dvc.oid._collect_dir", return_value=dir_info,
):
hash1 = dvc.cache.local.tree.get_hash(path_info, "md5")
hash1 = get_hash(path_info, dvc.cache.local.tree, "md5")
dir_info = DirInfo.from_list(
[{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}]
)
with patch.object(
BaseTree, "_collect_dir", return_value=dir_info,
with patch(
"dvc.oid._collect_dir", return_value=dir_info,
):
hash2 = dvc.cache.local.tree.get_hash(path_info, "md5")
hash2 = get_hash(path_info, dvc.cache.local.tree, "md5")
assert hash1 == hash2
tests/func/test_repro.py
+4 -1
View File
@@ -651,7 +651,10 @@ class TestReproDataSource(TestReproChangedData):
stages = self.dvc.reproduce(self.foo_stage.path)
self.assertTrue(filecmp.cmp(self.FOO, self.BAR, shallow=False))
self.assertEqual(stages[0].outs[0].hash_info.value, file_md5(self.BAR))
self.assertEqual(
stages[0].outs[0].hash_info.value,
file_md5(self.BAR, self.dvc.tree),
)
class TestReproChangedDir(SingleStageRun, TestDvc):
tests/func/test_run_single_stage.py
+3 -1
View File
@@ -60,7 +60,9 @@ class TestRun(TestDvc):
self.assertEqual(len(stage.deps), len(deps))
self.assertEqual(len(stage.outs), len(outs + outs_no_cache))
self.assertEqual(stage.outs[0].fspath, outs[0])
self.assertEqual(stage.outs[0].hash_info.value, file_md5(self.FOO))
self.assertEqual(
stage.outs[0].hash_info.value, file_md5(self.FOO, self.dvc.tree)
)
self.assertTrue(stage.path, fname)
with self.assertRaises(OutputDuplicationError):
tests/func/test_state.py
+3 -3
View File
@@ -12,7 +12,7 @@ def test_state(tmp_dir, dvc):
tmp_dir.gen("foo", "foo content")
path = tmp_dir / "foo"
path_info = PathInfo(path)
hash_info = HashInfo("md5", file_md5(path))
hash_info = HashInfo("md5", file_md5(path, dvc.tree))
state = State(dvc)
@@ -25,7 +25,7 @@ def test_state(tmp_dir, dvc):
assert state.get(path_info) is None
hash_info = HashInfo("md5", file_md5(path))
hash_info = HashInfo("md5", file_md5(path, dvc.tree))
state.save(path_info, hash_info)
assert state.get(path_info) == hash_info
@@ -60,7 +60,7 @@ def test_get_state_record_for_inode(get_inode_mock, tmp_dir, dvc):
assert inode != state._to_sqlite(inode)
foo = tmp_dir / "foo"
md5 = file_md5(foo)
md5 = file_md5(foo, dvc.tree)
get_inode_mock.side_effect = mock_get_inode(inode)
with state:
tests/func/test_tree.py
+2 -4
View File
@@ -346,8 +346,6 @@ def test_tree_ls_with_etag(dvc, cloud):
for details in tree.ls(path_info / "data", recursive=True, detail=True):
assert (
tree.get_file_hash(
path_info.replace(path=details["name"]), tree.PARAM_CHECKSUM
).value
== details[tree.PARAM_CHECKSUM]
tree.info(path_info.replace(path=details["name"]))["etag"]
== details["etag"]
)
tests/func/test_utils.py
+3 -1
View File
@@ -1,10 +1,12 @@
from dvc import utils
from dvc.tree.local import LocalTree
def test_file_md5_crlf(tmp_dir):
tree = LocalTree(None, {})
tmp_dir.gen("cr", b"a\nb\nc")
tmp_dir.gen("crlf", b"a\r\nb\r\nc")
assert utils.file_md5("cr") == utils.file_md5("crlf")
assert utils.file_md5("cr", tree) == utils.file_md5("crlf", tree)
def test_dict_md5():
tests/unit/remote/test_azure.py
+2 -5
View File
@@ -29,16 +29,13 @@ def test_init(dvc):
assert tree._conn_str == connection_string
def test_get_file_hash(tmp_dir, azure):
def test_info(tmp_dir, azure):
tmp_dir.gen("foo", "foo")
tree = AzureTree(None, azure.config)
to_info = azure
tree.upload(PathInfo("foo"), to_info)
assert tree.exists(to_info)
hash_info = tree.get_file_hash(to_info, "etag")
assert hash_info.name == "etag"
hash_ = hash_info.value
assert hash_
hash_ = tree.info(to_info)["etag"]
assert isinstance(hash_, str)
assert hash_.strip("'").strip('"') == hash_
tests/unit/remote/test_remote_tree.py
+2 -2
View File
@@ -98,8 +98,8 @@ def test_copy_preserve_etag_across_buckets(remote, dvc):
remote.tree.copy(from_info, to_info)
from_hash = remote.tree.get_hash(from_info, remote.tree.PARAM_CHECKSUM)
to_hash = another.get_hash(to_info, remote.tree.PARAM_CHECKSUM)
from_hash = remote.tree.info(from_info)["etag"]
to_hash = another.info(to_info)["etag"]
assert from_hash == to_hash
tests/unit/tree/test_dvc.py
+26 -14
View File
@@ -4,6 +4,7 @@ import shutil
import pytest
from dvc.hash_info import HashInfo
from dvc.oid import get_hash
from dvc.path_info import PathInfo
from dvc.tree.dvc import DvcTree
@@ -200,19 +201,23 @@ def test_isdvc(tmp_dir, dvc):
def test_get_hash_file(tmp_dir, dvc):
tmp_dir.dvc_gen({"foo": "foo"})
tree = DvcTree(dvc)
assert tree.get_hash(PathInfo(tmp_dir) / "foo", "md5") == HashInfo(
"md5", "acbd18db4cc2f85cedef654fccc4a4d8",
assert (
tree.info(PathInfo(tmp_dir) / "foo")["md5"]
== "acbd18db4cc2f85cedef654fccc4a4d8"
)
def test_get_hash_dir(tmp_dir, dvc, mocker):
import dvc as dvc_module
tmp_dir.dvc_gen(
{"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}}
)
tree = DvcTree(dvc)
get_file_hash_spy = mocker.spy(tree, "get_file_hash")
assert tree.get_hash(PathInfo(tmp_dir) / "dir", "md5") == HashInfo(
"md5", "8761c4e9acad696bee718615e23e22db.dir",
get_file_hash_spy = mocker.spy(dvc_module.oid, "get_file_hash")
assert (
tree.info(PathInfo(tmp_dir) / "dir")["md5"]
== "8761c4e9acad696bee718615e23e22db.dir"
)
assert not get_file_hash_spy.called
@@ -223,10 +228,14 @@ def test_get_hash_granular(tmp_dir, dvc):
)
tree = DvcTree(dvc)
subdir = PathInfo(tmp_dir) / "dir" / "subdir"
assert tree.get_hash(subdir, "md5") == HashInfo(
assert tree.info(subdir).get("md5") is None
assert get_hash(subdir, tree, "md5") == HashInfo(
"md5", "af314506f1622d107e0ed3f14ec1a3b5.dir",
)
assert tree.get_hash(subdir / "data", "md5") == HashInfo(
assert (
tree.info(subdir / "data")["md5"] == "8d777f385d3dfec8815d20f7496026dc"
)
assert get_hash(subdir / "data", tree, "md5") == HashInfo(
"md5", "8d777f385d3dfec8815d20f7496026dc",
)
@@ -236,9 +245,11 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
(tmp_dir / "file").write_text("something")
tree = DvcTree(dvc)
actual = tree.get_hash(PathInfo(tmp_dir) / "file", "md5")
expected = HashInfo("md5", "8c7dd922ad47494fc02c388e12c00eac")
assert actual == expected
expected = "8c7dd922ad47494fc02c388e12c00eac"
assert tree.info(PathInfo(tmp_dir) / "file").get("md5") == expected
assert get_hash(PathInfo(tmp_dir) / "file", tree, "md5") == HashInfo(
"md5", expected
)
def test_get_hash_dirty_dir(tmp_dir, dvc):
@@ -246,7 +257,8 @@ def test_get_hash_dirty_dir(tmp_dir, dvc):
(tmp_dir / "dir" / "baz").write_text("baz")
tree = DvcTree(dvc)
actual = tree.get_hash(PathInfo(tmp_dir) / "dir", "md5")
expected = HashInfo("md5", "5ea40360f5b4ec688df672a4db9c17d1.dir")
assert actual == expected
expected = "5ea40360f5b4ec688df672a4db9c17d1.dir"
assert tree.info(PathInfo(tmp_dir) / "dir").get("md5") == expected
assert get_hash(PathInfo(tmp_dir) / "dir", tree, "md5") == HashInfo(
"md5", expected
)
tests/unit/tree/test_repo.py
+33 -20
View File
@@ -5,6 +5,7 @@ from unittest import mock
import pytest
from dvc.hash_info import HashInfo
from dvc.oid import get_hash
from dvc.path_info import PathInfo
from dvc.tree.repo import RepoTree
@@ -483,11 +484,13 @@ def test_repo_tree_no_subrepos(tmp_dir, dvc, scm):
def test_get_hash_cached_file(tmp_dir, dvc, mocker):
tmp_dir.dvc_gen({"foo": "foo"})
tree = RepoTree(dvc)
dvc_tree_spy = mocker.spy(tree._dvctrees[dvc.root_dir], "get_file_hash")
assert tree.get_hash(PathInfo(tmp_dir) / "foo", "md5") == HashInfo(
"md5", "acbd18db4cc2f85cedef654fccc4a4d8",
expected = "acbd18db4cc2f85cedef654fccc4a4d8"
assert tree.info(PathInfo(tmp_dir) / "foo").get("md5") is None
assert get_hash(PathInfo(tmp_dir) / "foo", tree, "md5") == HashInfo(
"md5", expected,
)
assert dvc_tree_spy.called
(tmp_dir / "foo").unlink()
assert tree.info(PathInfo(tmp_dir) / "foo")["md5"] == expected
def test_get_hash_cached_dir(tmp_dir, dvc, mocker):
@@ -495,21 +498,17 @@ def test_get_hash_cached_dir(tmp_dir, dvc, mocker):
{"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}}
)
tree = RepoTree(dvc)
get_file_hash_spy = mocker.spy(tree, "get_file_hash")
dvc_tree_spy = mocker.spy(tree._dvctrees[dvc.root_dir], "get_dir_hash")
assert tree.get_hash(PathInfo(tmp_dir) / "dir", "md5") == HashInfo(
expected = "8761c4e9acad696bee718615e23e22db.dir"
assert tree.info(PathInfo(tmp_dir) / "dir").get("md5") is None
assert get_hash(PathInfo(tmp_dir) / "dir", tree, "md5") == HashInfo(
"md5", "8761c4e9acad696bee718615e23e22db.dir",
)
assert get_file_hash_spy.called
assert not dvc_tree_spy.called
get_file_hash_spy.reset_mock()
shutil.rmtree(tmp_dir / "dir")
assert tree.get_hash(PathInfo(tmp_dir) / "dir", "md5") == HashInfo(
assert tree.info(PathInfo(tmp_dir) / "dir")["md5"] == expected
assert get_hash(PathInfo(tmp_dir) / "dir", tree, "md5") == HashInfo(
"md5", "8761c4e9acad696bee718615e23e22db.dir",
)
assert not get_file_hash_spy.called
assert dvc_tree_spy.called
def test_get_hash_cached_granular(tmp_dir, dvc, mocker):
@@ -517,15 +516,19 @@ def test_get_hash_cached_granular(tmp_dir, dvc, mocker):
{"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}}
)
tree = RepoTree(dvc)
dvc_tree_spy = mocker.spy(tree._dvctrees[dvc.root_dir], "get_file_hash")
subdir = PathInfo(tmp_dir) / "dir" / "subdir"
assert tree.get_hash(subdir, "md5") == HashInfo(
assert tree.info(subdir).get("md5") is None
assert get_hash(subdir, tree, "md5") == HashInfo(
"md5", "af314506f1622d107e0ed3f14ec1a3b5.dir",
)
assert tree.get_hash(subdir / "data", "md5") == HashInfo(
assert tree.info(subdir / "data").get("md5") is None
assert get_hash(subdir / "data", tree, "md5") == HashInfo(
"md5", "8d777f385d3dfec8815d20f7496026dc",
)
assert dvc_tree_spy.called
(tmp_dir / "dir" / "subdir" / "data").unlink()
assert (
tree.info(subdir / "data")["md5"] == "8d777f385d3dfec8815d20f7496026dc"
)
def test_get_hash_mixed_dir(tmp_dir, scm, dvc):
@@ -541,7 +544,7 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc):
tmp_dir.scm.commit("add dir")
tree = RepoTree(dvc)
actual = tree.get_hash(PathInfo(tmp_dir) / "dir", "md5")
actual = get_hash(PathInfo(tmp_dir) / "dir", tree, "md5")
expected = HashInfo("md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir")
assert actual == expected
@@ -551,7 +554,17 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
(tmp_dir / "file").write_text("something")
tree = RepoTree(dvc)
actual = tree.get_hash(PathInfo(tmp_dir) / "file", "md5")
assert tree.info(PathInfo(tmp_dir) / "file").get("md5") is None
actual = get_hash(PathInfo(tmp_dir) / "file", tree, "md5")
expected = HashInfo("md5", "437b930db84b8079c2dd804a71936b5f")
assert actual == expected
(tmp_dir / "file").unlink()
assert (
tree.info(PathInfo(tmp_dir) / "file")["md5"]
== "8c7dd922ad47494fc02c388e12c00eac"
)
actual = get_hash(PathInfo(tmp_dir) / "file", tree, "md5")
expected = HashInfo("md5", "8c7dd922ad47494fc02c388e12c00eac")
assert actual == expected
@@ -561,7 +574,7 @@ def test_get_hash_dirty_dir(tmp_dir, dvc):
(tmp_dir / "dir" / "baz").write_text("baz")
tree = RepoTree(dvc)
actual = tree.get_hash(PathInfo(tmp_dir) / "dir", "md5")
actual = get_hash(PathInfo(tmp_dir) / "dir", tree, "md5")
expected = HashInfo("md5", "ba75a2162ca9c29acecb7957105a0bc2.dir")
assert actual == expected
assert actual.dir_info.nfiles == 3
tests/unit/utils/test_stream.py
+3 -2
View File
@@ -1,3 +1,4 @@
from dvc.tree.local import LocalTree
from dvc.utils import file_md5
from dvc.utils.stream import HashedStreamReader
@@ -10,7 +11,7 @@ def test_hashed_stream_reader(tmp_dir):
stream_reader = HashedStreamReader(fobj)
assert stream_reader.read(3) == b"foo"
hex_digest = file_md5(foo)
hex_digest = file_md5(foo, LocalTree(None, {}))
assert stream_reader.is_text_file
assert hex_digest == stream_reader.hash_info.value
@@ -26,6 +27,6 @@ def test_hashed_stream_reader_as_chunks(tmp_dir):
if not chunk:
break
hex_digest = file_md5(foo)
hex_digest = file_md5(foo, LocalTree(None, {}))
assert not stream_reader.is_text_file
assert hex_digest == stream_reader.hash_info.value
tests/unit/utils/test_utils.py
+3 -1
View File
@@ -4,6 +4,7 @@ import re
import pytest
from dvc.path_info import PathInfo
from dvc.tree.local import LocalTree
from dvc.utils import (
dict_sha256,
file_md5,
@@ -86,7 +87,8 @@ def test_fix_env_pyenv(path, orig):
def test_file_md5(tmp_dir):
tmp_dir.gen("foo", "foo content")
assert file_md5("foo") == file_md5(PathInfo("foo"))
tree = LocalTree(None, {})
assert file_md5("foo", tree) == file_md5(PathInfo("foo"), tree)
def test_tmp_fname():