diff --git a/dvc/api/__init__.py b/dvc/api/__init__.py index 80962d00d..53ce86ccd 100644 --- a/dvc/api/__init__.py +++ b/dvc/api/__init__.py @@ -4,6 +4,7 @@ from contextlib import _GeneratorContextManager as GCM from funcy import reraise from dvc.exceptions import OutputNotFoundError, PathMissingError +from dvc.oid import get_hash from dvc.path_info import PathInfo from dvc.repo import Repo @@ -28,7 +29,7 @@ def get_url(path, repo=None, rev=None, remote=None): raise OutputNotFoundError(path, repo) cloud = metadata.repo.cloud - hash_info = _repo.repo_tree.get_hash(path_info, "md5") + hash_info = get_hash(path_info, _repo.repo_tree, "md5") return cloud.get_url_for(remote, checksum=hash_info.value) diff --git a/dvc/checkout.py b/dvc/checkout.py index a28140dd8..47826c239 100644 --- a/dvc/checkout.py +++ b/dvc/checkout.py @@ -10,6 +10,7 @@ from dvc.exceptions import ( DvcException, ) from dvc.objects import ObjectFormatError, check, load +from dvc.oid import get_hash from dvc.remote.slow_link_detection import ( # type: ignore[attr-defined] slow_link_guard, ) @@ -29,7 +30,7 @@ def _changed(path_info, tree, obj, cache): return True try: - actual = tree.get_hash(path_info, obj.hash_info.name) + actual = get_hash(path_info, tree, obj.hash_info.name) except FileNotFoundError: logger.debug("'%s' doesn't exist.", path_info) return True @@ -55,7 +56,7 @@ def _remove(path_info, tree, cache, force=False): tree.remove(path_info) return - current = tree.get_hash(path_info, tree.PARAM_CHECKSUM) + current = get_hash(path_info, tree, tree.PARAM_CHECKSUM) try: obj = load(cache, current) check(cache, obj) diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index d5611add9..fbb11fbe5 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -47,10 +47,12 @@ class RepoDependency(LocalDependency): return external_repo(d["url"], rev=rev, **kwargs) def _get_hash(self, locked=True): + from dvc.oid import get_hash + with self._make_repo(locked=locked) as repo: path_info = PathInfo(repo.root_dir) / self.def_path - return repo.repo_tree.get_hash( - path_info, "md5", follow_subrepos=False + return get_hash( + path_info, repo.repo_tree, "md5", follow_subrepos=False ) def workspace_status(self): diff --git a/dvc/objects.py b/dvc/objects.py index 4f16e0a0e..02a2b652d 100644 --- a/dvc/objects.py +++ b/dvc/objects.py @@ -7,6 +7,7 @@ from dvc.progress import Tqdm from .dir_info import DirInfo from .exceptions import DvcException +from .oid import get_hash logger = logging.getLogger(__name__) @@ -35,7 +36,7 @@ class HashFile: return bool(self.hash_info) def check(self, odb): - actual = self.tree.get_hash(self.path_info, odb.tree.PARAM_CHECKSUM) + actual = get_hash(self.path_info, self.tree, odb.tree.PARAM_CHECKSUM) logger.trace( "cache '%s' expected '%s' actual '%s'", @@ -63,7 +64,9 @@ class File(HashFile): @classmethod def stage(cls, odb, path_info, tree, **kwargs): - hash_info = tree.get_hash(path_info, odb.tree.PARAM_CHECKSUM, **kwargs) + hash_info = get_hash( + path_info, tree, odb.tree.PARAM_CHECKSUM, **kwargs + ) raw = odb.get(hash_info) obj = cls(raw.path_info, raw.tree, hash_info) obj.src = HashFile(path_info, tree, hash_info) @@ -113,7 +116,7 @@ class Tree(HashFile): with tree.open(path_info, "rb") as fobj: odb.tree.upload_fobj(fobj, tmp_info) - hash_info = odb.tree.get_hash(tmp_info, odb.tree.PARAM_CHECKSUM) + hash_info = get_hash(tmp_info, odb.tree, odb.tree.PARAM_CHECKSUM) hash_info.value += odb.tree.CHECKSUM_DIR_SUFFIX hash_info.dir_info = dir_info hash_info.nfiles = dir_info.nfiles @@ -124,7 +127,9 @@ class Tree(HashFile): @classmethod def stage(cls, odb, path_info, tree, **kwargs): - hash_info = tree.get_hash(path_info, odb.tree.PARAM_CHECKSUM, **kwargs) + hash_info = get_hash( + path_info, tree, odb.tree.PARAM_CHECKSUM, **kwargs + ) hi = cls.save_dir_info(odb, hash_info.dir_info, hash_info) hi.size = hash_info.size raw = odb.get(hi) diff --git a/dvc/oid.py b/dvc/oid.py new file mode 100644 index 000000000..716456e32 --- /dev/null +++ b/dvc/oid.py @@ -0,0 +1,144 @@ +import errno +import os +from concurrent.futures import ThreadPoolExecutor + +from dvc.exceptions import DvcIgnoreInCollectedDirError +from dvc.hash_info import HashInfo +from dvc.ignore import DvcIgnore +from dvc.progress import Tqdm +from dvc.utils import file_md5 + + +def get_file_hash(path_info, tree, name): + info = tree.info(path_info) + if name in info: + assert not info[name].endswith(".dir") + return HashInfo(name, info[name], size=info["size"]) + + func = getattr(tree, name, None) + if func: + return func(path_info) + + if name == "md5": + return HashInfo( + name, file_md5(path_info, tree), size=tree.getsize(path_info) + ) + + raise NotImplementedError + + +def _calculate_hashes(file_infos, tree, name): + def _get_file_hash(path_info): + return get_file_hash(path_info, tree, name) + + with Tqdm( + total=len(file_infos), + unit="md5", + desc="Computing file/dir hashes (only done once)", + ) as pbar: + worker = pbar.wrap_fn(_get_file_hash) + with ThreadPoolExecutor(max_workers=tree.hash_jobs) as executor: + hash_infos = executor.map(worker, file_infos) + return dict(zip(file_infos, hash_infos)) + + +def _iter_hashes(path_info, tree, name, **kwargs): + if name in tree.DETAIL_FIELDS: + for details in tree.ls(path_info, recursive=True, detail=True): + file_info = path_info.replace(path=details["name"]) + hash_info = HashInfo( + name, details[name], size=details.get("size"), + ) + yield file_info, hash_info + + return None + + file_infos = [] + for file_info in tree.walk_files(path_info, **kwargs): + hash_info = tree.state.get( # pylint: disable=assignment-from-none + file_info + ) + if not hash_info: + file_infos.append(file_info) + continue + assert hash_info.name == name + yield file_info, hash_info + + yield from _calculate_hashes(file_infos, tree, name).items() + + +def _collect_dir(path_info, tree, name, **kwargs): + from dvc.dir_info import DirInfo + + dir_info = DirInfo() + for fi, hi in _iter_hashes(path_info, tree, name, **kwargs): + if DvcIgnore.DVCIGNORE_FILE == fi.name: + raise DvcIgnoreInCollectedDirError(fi.parent) + + # NOTE: this is lossy transformation: + # "hey\there" -> "hey/there" + # "hey/there" -> "hey/there" + # The latter is fine filename on Windows, which + # will transform to dir/file on back transform. + # + # Yes, this is a BUG, as long as we permit "/" in + # filenames on Windows and "\" on Unix + dir_info.trie[fi.relative_to(path_info).parts] = hi + + return dir_info + + +def get_dir_hash(path_info, tree, name, **kwargs): + from dvc.objects import Tree + + dir_info = _collect_dir(path_info, tree, name, **kwargs) + hash_info = Tree.save_dir_info(tree.repo.cache.local, dir_info) + hash_info.size = dir_info.size + hash_info.dir_info = dir_info + return hash_info + + +def get_hash(path_info, tree, name, **kwargs): + assert path_info and ( + isinstance(path_info, str) or path_info.scheme == tree.scheme + ) + + if not tree.exists(path_info): + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), path_info + ) + + with tree.state: + # pylint: disable=assignment-from-none + hash_info = tree.state.get(path_info) + + # If we have dir hash in state db, but dir cache file is lost, + # then we need to recollect the dir via .get_dir_hash() call below, + # see https://github.com/iterative/dvc/issues/2219 for context + if ( + hash_info + and hash_info.isdir + and not tree.cache.tree.exists( + tree.cache.hash_to_path_info(hash_info.value) + ) + ): + hash_info = None + + if hash_info: + if hash_info.isdir: + from dvc.objects import Tree + + # NOTE: loading the tree will restore hash_info.dir_info + Tree.load(tree.cache, hash_info) + assert hash_info.name == name + return hash_info + + if tree.isdir(path_info): + hash_info = get_dir_hash(path_info, tree, name, **kwargs) + else: + hash_info = get_file_hash(path_info, tree, name) + + if hash_info and tree.exists(path_info): + tree.state.save(path_info, hash_info) + + return hash_info diff --git a/dvc/output/base.py b/dvc/output/base.py index 3755a5984..91ecc79c0 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -18,6 +18,7 @@ from dvc.exceptions import ( RemoteCacheRequiredError, ) from dvc.hash_info import HashInfo +from dvc.oid import get_hash from ..tree.base import BaseTree @@ -195,7 +196,9 @@ class BaseOutput: def get_hash(self): if not self.use_cache: - return self.tree.get_hash(self.path_info, self.tree.PARAM_CHECKSUM) + return get_hash( + self.path_info, self.tree, self.tree.PARAM_CHECKSUM + ) return objects.stage(self.cache, self.path_info, self.tree).hash_info @property diff --git a/dvc/repo/diff.py b/dvc/repo/diff.py index 8f3c0d1d2..22734c1d8 100644 --- a/dvc/repo/diff.py +++ b/dvc/repo/diff.py @@ -2,6 +2,7 @@ import logging import os from dvc.exceptions import PathMissingError +from dvc.oid import get_file_hash, get_hash from dvc.repo import locked logger = logging.getLogger(__name__) @@ -127,8 +128,8 @@ def _output_paths(repo, repo_tree, targets): def _to_checksum(output): if on_working_tree: - return repo.cache.local.tree.get_hash( - output.path_info, "md5" + return get_hash( + output.path_info, repo.cache.local.tree, "md5" ).value return output.hash_info.value @@ -157,7 +158,7 @@ def _dir_output_paths(repo_tree, output, targets=None): if targets is None or any( fname.isin_or_eq(target) for target in targets ): - yield str(fname), repo_tree.get_file_hash(fname, "md5").value + yield str(fname), get_file_hash(fname, repo_tree, "md5").value except NoRemoteError: logger.warning("dir cache entry for '%s' is missing", output) diff --git a/dvc/tree/azure.py b/dvc/tree/azure.py index d589734a4..2cd02ede8 100644 --- a/dvc/tree/azure.py +++ b/dvc/tree/azure.py @@ -5,7 +5,6 @@ from datetime import datetime, timedelta from funcy import cached_property, wrap_prop -from dvc.hash_info import HashInfo from dvc.path_info import CloudURLInfo from dvc.progress import Tqdm from dvc.scheme import Schemes @@ -167,7 +166,7 @@ class AzureTree(BaseTree): "type": "file", "name": blob.name, "size": blob.size, - "etag": blob.etag, + "etag": blob.etag.strip('"'), } else: yield blob.name @@ -181,16 +180,16 @@ class AzureTree(BaseTree): path_info.bucket, path_info.path ).delete_blob() - def getsize(self, path_info): + def info(self, path_info): blob_client = self.blob_service.get_blob_client( path_info.bucket, path_info.path ) properties = blob_client.get_blob_properties() - return properties.size - - def get_file_hash(self, path_info, name): - assert name == self.PARAM_CHECKSUM - return HashInfo(self.PARAM_CHECKSUM, self.get_etag(path_info)) + return { + "type": "file", + "size": properties.size, + "etag": properties.etag.strip('"'), + } def _upload_fobj(self, fobj, to_info): blob_client = self.blob_service.get_blob_client( diff --git a/dvc/tree/base.py b/dvc/tree/base.py index b99ca4892..bc12ebaee 100644 --- a/dvc/tree/base.py +++ b/dvc/tree/base.py @@ -1,6 +1,4 @@ -import errno import logging -import os from concurrent.futures import ThreadPoolExecutor, as_completed from functools import partial from multiprocessing import cpu_count @@ -9,10 +7,7 @@ from urllib.parse import urlparse from funcy import cached_property, decorator -from dvc.dir_info import DirInfo -from dvc.exceptions import DvcException, DvcIgnoreInCollectedDirError -from dvc.hash_info import HashInfo -from dvc.ignore import DvcIgnore +from dvc.exceptions import DvcException from dvc.path_info import URLInfo from dvc.progress import Tqdm from dvc.state import StateNoop @@ -212,8 +207,11 @@ class BaseTree: def is_empty(self, path_info): return False + def info(self, path_info): + raise NotImplementedError + def getsize(self, path_info): - return None + return self.info(path_info).get("size") def remove(self, path_info): raise RemoteActionNotImplemented("remove", self.scheme) @@ -247,121 +245,6 @@ class BaseTree: return False return hash_.endswith(cls.CHECKSUM_DIR_SUFFIX) - @use_state - def get_hash(self, path_info, name, **kwargs): - assert path_info and ( - isinstance(path_info, str) or path_info.scheme == self.scheme - ) - - if not self.exists(path_info): - raise FileNotFoundError( - errno.ENOENT, os.strerror(errno.ENOENT), path_info - ) - - # pylint: disable=assignment-from-none - hash_info = self.state.get(path_info) - - # If we have dir hash in state db, but dir cache file is lost, - # then we need to recollect the dir via .get_dir_hash() call below, - # see https://github.com/iterative/dvc/issues/2219 for context - if ( - hash_info - and hash_info.isdir - and not self.cache.tree.exists( - self.cache.hash_to_path_info(hash_info.value) - ) - ): - hash_info = None - - if hash_info: - assert hash_info.name == self.PARAM_CHECKSUM - if hash_info.isdir: - from dvc.objects import Tree - - # NOTE: loading the tree will restore hash_info.dir_info - Tree.load(self.cache, hash_info) - assert hash_info.name == name - return hash_info - - if self.isdir(path_info): - hash_info = self.get_dir_hash(path_info, name, **kwargs) - else: - hash_info = self.get_file_hash(path_info, name) - - if hash_info and self.exists(path_info): - self.state.save(path_info, hash_info) - - return hash_info - - def get_file_hash(self, path_info, name): - raise NotImplementedError - - def _calculate_hashes(self, file_infos, name): - def _get_file_hash(path_info): - return self.get_file_hash(path_info, name) - - with Tqdm( - total=len(file_infos), - unit="md5", - desc="Computing file/dir hashes (only done once)", - ) as pbar: - worker = pbar.wrap_fn(_get_file_hash) - with ThreadPoolExecutor(max_workers=self.hash_jobs) as executor: - hash_infos = executor.map(worker, file_infos) - return dict(zip(file_infos, hash_infos)) - - def _iter_hashes(self, path_info, name, **kwargs): - if name in self.DETAIL_FIELDS: - for details in self.ls(path_info, recursive=True, detail=True): - file_info = path_info.replace(path=details["name"]) - hash_info = HashInfo( - name, details[name], size=details.get("size"), - ) - yield file_info, hash_info - - return None - - file_infos = [] - for file_info in self.walk_files(path_info, **kwargs): - hash_info = self.state.get( # pylint: disable=assignment-from-none - file_info - ) - if not hash_info: - file_infos.append(file_info) - continue - assert hash_info.name == name - yield file_info, hash_info - - yield from self._calculate_hashes(file_infos, name).items() - - def _collect_dir(self, path_info, name, **kwargs): - dir_info = DirInfo() - for fi, hi in self._iter_hashes(path_info, name, **kwargs): - if DvcIgnore.DVCIGNORE_FILE == fi.name: - raise DvcIgnoreInCollectedDirError(fi.parent) - - # NOTE: this is lossy transformation: - # "hey\there" -> "hey/there" - # "hey/there" -> "hey/there" - # The latter is fine filename on Windows, which - # will transform to dir/file on back transform. - # - # Yes, this is a BUG, as long as we permit "/" in - # filenames on Windows and "\" on Unix - dir_info.trie[fi.relative_to(path_info).parts] = hi - - return dir_info - - @use_state - def get_dir_hash(self, path_info, name, **kwargs): - from dvc.objects import Tree - - dir_info = self._collect_dir(path_info, name, **kwargs) - hash_info = Tree.save_dir_info(self.repo.cache.local, dir_info) - hash_info.size = dir_info.size - hash_info.dir_info = dir_info - return hash_info - def upload( self, from_info, to_info, name=None, no_progress_bar=False, ): diff --git a/dvc/tree/dvc.py b/dvc/tree/dvc.py index 82fe257d8..5d1862a41 100644 --- a/dvc/tree/dvc.py +++ b/dvc/tree/dvc.py @@ -54,7 +54,7 @@ class DvcTree(BaseTree): # pylint:disable=abstract-method raise FileNotFoundError def open( # type: ignore - self, path: PathInfo, mode="r", encoding="utf-8", remote=None, **kwargs + self, path: PathInfo, mode="r", encoding=None, remote=None, **kwargs ): # pylint: disable=arguments-differ try: outs = self._find_outs(path, strict=False) @@ -219,30 +219,6 @@ class DvcTree(BaseTree): # pylint:disable=abstract-method def isexec(self, path_info): # pylint: disable=unused-argument return False - def get_dir_hash(self, path_info, name, **kwargs): - try: - outs = self._find_outs(path_info, strict=True) - if len(outs) == 1 and outs[0].is_dir_checksum: - out = outs[0] - # other code expects us to fetch the dir at this point - self._fetch_dir(out, **kwargs) - assert out.hash_info.name == name - return out.hash_info - except OutputNotFoundError: - pass - - return super().get_dir_hash(path_info, name, **kwargs) - - def get_file_hash(self, path_info, name): - assert name == "md5" - outs = self._find_outs(path_info, strict=False) - if len(outs) != 1: - raise OutputNotFoundError - out = outs[0] - if out.is_dir_checksum: - return self._get_granular_hash(path_info, out) - return out.hash_info - def metadata(self, path_info): path_info = PathInfo(os.path.abspath(path_info)) @@ -254,3 +230,20 @@ class DvcTree(BaseTree): # pylint:disable=abstract-method meta = Metadata(path_info=path_info, outs=outs, repo=self.repo) meta.isdir = meta.isdir or self.check_isdir(meta.path_info, meta.outs) return meta + + def info(self, path_info): + meta = self.metadata(path_info) + ret = {"type": "dir" if meta.isdir else "file"} + if meta.is_output and len(meta.outs) == 1 and meta.outs[0].hash_info: + hash_info = meta.outs[0].hash_info + ret["size"] = hash_info.size + ret[hash_info.name] = hash_info.value + elif meta.part_of_output: + (out,) = meta.outs + key = path_info.relative_to(out.path_info).parts + hash_info = out.hash_info.dir_info.trie.get(key) + if hash_info: + ret["size"] = hash_info.size + ret[hash_info.name] = hash_info.value + + return ret diff --git a/dvc/tree/gdrive.py b/dvc/tree/gdrive.py index 86e8cc33c..09b264afe 100644 --- a/dvc/tree/gdrive.py +++ b/dvc/tree/gdrive.py @@ -599,14 +599,11 @@ class GDriveTree(BaseTree): item_id = self._get_item_id(path_info) self.gdrive_delete_file(item_id) - def get_file_hash(self, path_info, name): - raise NotImplementedError - - def getsize(self, path_info): + def info(self, path_info): item_id = self._get_item_id(path_info) gdrive_file = self._drive.CreateFile({"id": item_id}) gdrive_file.FetchMetadata(fields="fileSize") - return gdrive_file.get("fileSize") + return {"size": gdrive_file.get("fileSize")} def _upload_fobj(self, fobj, to_info): dirname = to_info.parent diff --git a/dvc/tree/git.py b/dvc/tree/git.py index 29da654b7..428566388 100644 --- a/dvc/tree/git.py +++ b/dvc/tree/git.py @@ -138,6 +138,16 @@ class GitTree(BaseTree): # pylint:disable=abstract-method mode = self.stat(path_info).st_mode return is_exec(mode) + def info(self, path_info): + key = self._get_key(path_info) + try: + st = self.trie.stat(key) + except KeyError: + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), path_info + ) + return {"size": st.st_size} + def stat(self, path): key = self._get_key(path) try: diff --git a/dvc/tree/gs.py b/dvc/tree/gs.py index 3f5a47a96..9b48396a4 100644 --- a/dvc/tree/gs.py +++ b/dvc/tree/gs.py @@ -7,7 +7,6 @@ from functools import wraps from funcy import cached_property, wrap_prop from dvc.exceptions import DvcException -from dvc.hash_info import HashInfo from dvc.path_info import CloudURLInfo from dvc.progress import Tqdm from dvc.scheme import Schemes @@ -124,10 +123,10 @@ class GSTree(BaseTree): blob = self.gs.bucket(path_info.bucket).blob(path_info.path) return blob.exists() - def getsize(self, path_info): + def info(self, path_info): bucket = self.gs.bucket(path_info.bucket) blob = bucket.get_blob(path_info.path) - return blob.size + return {"type": "file", "size": blob.size} def _list_paths(self, path_info, max_items=None): for blob in self.gs.bucket(path_info.bucket).list_blobs( @@ -194,25 +193,6 @@ class GSTree(BaseTree): to_bucket = self.gs.bucket(to_info.bucket) from_bucket.copy_blob(blob, to_bucket, new_name=to_info.path) - def get_file_hash(self, path_info, name): - import base64 - import codecs - - assert name == self.PARAM_CHECKSUM - bucket = path_info.bucket - path = path_info.path - blob = self.gs.bucket(bucket).get_blob(path) - if not blob: - return HashInfo(self.PARAM_CHECKSUM, None) - - b64_md5 = blob.md5_hash - md5 = base64.b64decode(b64_md5) - return HashInfo( - self.PARAM_CHECKSUM, - codecs.getencoder("hex")(md5)[0].decode("utf-8"), - size=blob.size, - ) - def _upload_fobj(self, fobj, to_info): bucket = self.gs.bucket(to_info.bucket) # With other references being given in the @dynamic_chunk_size diff --git a/dvc/tree/hdfs.py b/dvc/tree/hdfs.py index 72ac2b946..f4919759e 100644 --- a/dvc/tree/hdfs.py +++ b/dvc/tree/hdfs.py @@ -222,16 +222,14 @@ class HDFSTree(BaseTree): file_info = hdfs.get_file_info(path_info.path) return file_info.type == pyarrow.fs.FileType.Directory - def getsize(self, path_info): + def info(self, path_info): with self.hdfs(path_info) as hdfs: - file_info = hdfs.get_file_info(path_info.path) - return file_info.size + finfo = hdfs.get_file_info(path_info.path) + return {"size": finfo.size} - def get_file_hash(self, path_info, name): - assert name == self.PARAM_CHECKSUM - # NOTE: pyarrow doesn't support checksum, so we need to use hadoop + def checksum(self, path_info): return HashInfo( - self.PARAM_CHECKSUM, + "checksum", _hadoop_fs_checksum(path_info), size=self.getsize(path_info), ) diff --git a/dvc/tree/http.py b/dvc/tree/http.py index baf35e49e..cf8810166 100644 --- a/dvc/tree/http.py +++ b/dvc/tree/http.py @@ -6,7 +6,6 @@ from funcy import cached_property, memoize, wrap_prop, wrap_with import dvc.prompt as prompt from dvc.exceptions import DvcException, HTTPError -from dvc.hash_info import HashInfo from dvc.path_info import HTTPURLInfo from dvc.progress import Tqdm from dvc.scheme import Schemes @@ -146,28 +145,12 @@ class HTTPTree(BaseTree): # pylint:disable=abstract-method return True raise HTTPError(res.status_code, res.reason) - def getsize(self, path_info): - response = self.request("GET", path_info.url, stream=True) - if response.status_code != 200: - raise HTTPError(response.status_code, response.reason) - return self._content_length(response) - - def get_file_hash(self, path_info, name): - assert name == self.PARAM_CHECKSUM - + def info(self, path_info): url = path_info.url - headers = self._head(url).headers - etag = headers.get("ETag") or headers.get("Content-MD5") - - if not etag: - raise DvcException( - "could not find an ETag or " - "Content-MD5 header for '{url}'".format(url=url) - ) - - return HashInfo(self.PARAM_CHECKSUM, etag) + size = int(headers.get("Content-Length")) + return {"etag": etag, "size": size} def _upload_fobj(self, fobj, to_info): def chunks(fobj): diff --git a/dvc/tree/local.py b/dvc/tree/local.py index 4df7deb1a..f718fbff3 100644 --- a/dvc/tree/local.py +++ b/dvc/tree/local.py @@ -1,13 +1,13 @@ import logging import os +import stat from funcy import cached_property -from dvc.hash_info import HashInfo from dvc.path_info import PathInfo from dvc.scheme import Schemes from dvc.system import System -from dvc.utils import file_md5, is_exec, tmp_fname +from dvc.utils import is_exec, tmp_fname from dvc.utils.fs import copy_fobj_to_file, copyfile, makedirs, move, remove from .base import BaseTree @@ -207,18 +207,13 @@ class LocalTree(BaseTree): def reflink(self, from_info, to_info): System.reflink(from_info, to_info) - def get_file_hash(self, path_info, name): - assert name == "md5" - hash_info = HashInfo(name, file_md5(path_info)) - - if hash_info: - hash_info.size = os.path.getsize(path_info) - - return hash_info - @staticmethod - def getsize(path_info): - return os.path.getsize(path_info) + def info(path_info): + st = os.stat(path_info) + return { + "size": st.st_size, + "type": "dir" if stat.S_ISDIR(st.st_mode) else "file", + } def _upload( self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs, diff --git a/dvc/tree/memory.py b/dvc/tree/memory.py index 57229e26a..27fa9d7f6 100644 --- a/dvc/tree/memory.py +++ b/dvc/tree/memory.py @@ -1,6 +1,3 @@ -from dvc.hash_info import HashInfo -from dvc.utils import file_md5 - from .base import BaseTree @@ -30,9 +27,5 @@ class MemoryTree(BaseTree): return os.stat_result((0, 0, 0, 0, 0, 0, info["size"], 0, 0, 0)) - def get_file_hash(self, path_info, name): - assert name == self.PARAM_CHECKSUM - return HashInfo(self.PARAM_CHECKSUM, file_md5(path_info, self)) - def walk_files(self, path_info, **kwargs): raise NotImplementedError diff --git a/dvc/tree/repo.py b/dvc/tree/repo.py index 5b201d246..b513873eb 100644 --- a/dvc/tree/repo.py +++ b/dvc/tree/repo.py @@ -8,7 +8,6 @@ from typing import TYPE_CHECKING, Callable, Optional, Tuple, Type, Union from funcy import lfilter, wrap_with -from dvc.hash_info import HashInfo from dvc.path_info import PathInfo from .base import BaseTree @@ -370,38 +369,6 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method for fname in files: yield PathInfo(root) / fname - def get_dir_hash( - self, path_info, name, follow_subrepos=None, **kwargs - ): # pylint: disable=arguments-differ - tree, dvc_tree = self._get_tree_pair(path_info) - if tree.exists(path_info): - return super().get_dir_hash( - path_info, name, follow_subrepos=follow_subrepos, **kwargs - ) - if not dvc_tree: - raise FileNotFoundError - return dvc_tree.get_dir_hash(path_info, name, **kwargs) - - def get_file_hash(self, path_info, name): - """Return file checksum for specified path. - - If path_info is a DVC out, the pre-computed checksum for the file - will be used. If path_info is a git file, MD5 will be computed for - the git object. - """ - from dvc.utils import file_md5 - - assert name == self.PARAM_CHECKSUM - if not self.exists(path_info): - raise FileNotFoundError - _, dvc_tree = self._get_tree_pair(path_info) - if dvc_tree and dvc_tree.exists(path_info): - try: - return dvc_tree.get_file_hash(path_info, name) - except FileNotFoundError: - pass - return HashInfo(self.PARAM_CHECKSUM, file_md5(path_info, self)) - def _download( self, from_info, to_file, name=None, no_progress_bar=False, **kwargs ): @@ -452,3 +419,11 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method meta.is_exec = bool(stat_result) and is_exec(stat_result.st_mode) return meta + + def info(self, path_info): + tree, dvc_tree = self._get_tree_pair(path_info) + + try: + return tree.info(path_info) + except FileNotFoundError: + return dvc_tree.info(path_info) diff --git a/dvc/tree/s3.py b/dvc/tree/s3.py index 0ac48e0f6..715e0cbe6 100644 --- a/dvc/tree/s3.py +++ b/dvc/tree/s3.py @@ -260,9 +260,12 @@ class S3Tree(BaseTree): return path_info.path in self._list_paths(path_info) - def getsize(self, path_info): + def info(self, path_info): with self._get_obj(path_info) as obj: - return obj.content_length + return { + "etag": obj.e_tag.strip('"'), + "size": obj.content_length, + } def _list_paths(self, path_info, max_items=None): with self._get_bucket(path_info.bucket) as bucket: @@ -418,14 +421,9 @@ class S3Tree(BaseTree): if etag != cached_etag: raise ETagMismatchError(etag, cached_etag) - def get_file_hash(self, path_info, name): - assert name == self.PARAM_CHECKSUM + def etag(self, path_info): with self._get_obj(path_info) as obj: - return HashInfo( - self.PARAM_CHECKSUM, - obj.e_tag.strip('"'), - size=obj.content_length, - ) + return HashInfo("etag", size=obj.content_length,) def _upload_fobj(self, fobj, to_info): with self._get_obj(to_info) as obj: diff --git a/dvc/tree/ssh/__init__.py b/dvc/tree/ssh/__init__.py index c3e102e32..8e730ae78 100644 --- a/dvc/tree/ssh/__init__.py +++ b/dvc/tree/ssh/__init__.py @@ -235,22 +235,17 @@ class SSHTree(BaseTree): with self.ssh(from_info) as ssh: ssh.reflink(from_info.path, to_info.path) - def get_file_hash(self, path_info, name): - if path_info.scheme != self.scheme: - raise NotImplementedError - - assert name == self.PARAM_CHECKSUM + def md5(self, path_info): with self.ssh(path_info) as ssh: - hash_info = HashInfo(self.PARAM_CHECKSUM, ssh.md5(path_info.path),) + return HashInfo( + "md5", + ssh.md5(path_info.path), + size=ssh.getsize(path_info.path), + ) - if hash_info: - hash_info.size = ssh.getsize(path_info.path) - - return hash_info - - def getsize(self, path_info): + def info(self, path_info): with self.ssh(path_info) as ssh: - return ssh.getsize(path_info.path) + return ssh.info(path_info.path) def _upload_fobj(self, fobj, to_info): self.makedirs(to_info.parent) diff --git a/dvc/tree/ssh/connection.py b/dvc/tree/ssh/connection.py index e3b236f5b..55ec70fe4 100644 --- a/dvc/tree/ssh/connection.py +++ b/dvc/tree/ssh/connection.py @@ -76,12 +76,15 @@ class SSHConnection: return lstat.st_mode if lstat else 0 - def getsize(self, path): - lstat = None - with suppress(FileNotFoundError): - lstat = self.sftp.lstat(path) + def info(self, path): + st = self.sftp.stat(path) + return { + "size": st.st_size, + "type": "dir" if stat.S_ISDIR(st.st_mode) else "file", + } - return lstat.st_size if lstat else 0 + def getsize(self, path): + return self.info(path)["size"] def exists(self, path): return bool(self.st_mode(path)) diff --git a/dvc/tree/webdav.py b/dvc/tree/webdav.py index f24840266..40fafd5fd 100644 --- a/dvc/tree/webdav.py +++ b/dvc/tree/webdav.py @@ -8,7 +8,6 @@ from funcy import cached_property, nullcontext, wrap_prop from dvc.config import ConfigError from dvc.exceptions import DvcException -from dvc.hash_info import HashInfo from dvc.path_info import HTTPURLInfo, WebDAVURLInfo from dvc.progress import Tqdm from dvc.scheme import Schemes @@ -147,22 +146,6 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method # Use webdav check to test for file existence return self._client.check(path_info.path) - # Gets file hash 'etag' - def get_file_hash(self, path_info, name): - assert name == self.PARAM_CHECKSUM - - # Use webdav client info method to get etag - etag = self._client.info(path_info.path)["etag"].strip('"') - - # From HTTPTree - if not etag: - raise DvcException( - "could not find an ETag or " - "Content-MD5 header for '{url}'".format(url=path_info.url) - ) - - return HashInfo(self.PARAM_CHECKSUM, etag) - # Checks whether path points to directory def isdir(self, path_info): # Use webdav is_dir to test whether path points to a directory @@ -293,7 +276,9 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method buff=fd_wrapped, remote_path=to_info.path ) - # Queries size of file at remote - def getsize(self, path_info): - # Get file size from info dictionary and convert to int (from str) - return int(self._client.info(path_info.path)["size"]) + def info(self, path_info): + info = self._client.info(path_info.path) + return { + "size": int(info["size"]), + "etag": info["etag"], + } diff --git a/dvc/tree/webhdfs.py b/dvc/tree/webhdfs.py index 6596e7b55..9fa3f71e0 100644 --- a/dvc/tree/webhdfs.py +++ b/dvc/tree/webhdfs.py @@ -121,17 +121,16 @@ class WebHDFSTree(BaseTree): status = self.hdfs_client.status(path_info.path, strict=False) return status is not None - def getsize(self, path_info): - return self.hdfs_client.status(path_info.path)["length"] + def info(self, path_info): + st = self.hdfs_client.status(path_info.path) + return {"size": st["length"]} - def get_file_hash(self, path_info, name): - assert name == self.PARAM_CHECKSUM - - checksum = self.hdfs_client.checksum(path_info.path) - hash_info = HashInfo(self.PARAM_CHECKSUM, checksum["bytes"]) - - hash_info.size = self.hdfs_client.status(path_info.path)["length"] - return hash_info + def checksum(self, path_info): + return HashInfo( + "checksum", + self.hdfs_client.checksum(path_info.path)["bytes"], + size=self.hdfs_client.status(path_info.path)["length"], + ) def copy(self, from_info, to_info, **_kwargs): with self.hdfs_client.read(from_info.path) as reader: diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 76e6ce0f0..a89034971 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -41,36 +41,31 @@ def _fobj_md5(fobj, hash_md5, binary, progress_func=None): progress_func(len(data)) -def file_md5(fname, tree=None): +def file_md5(fname, tree): """ get the (md5 hexdigest, md5 digest) of a file """ from dvc.istextfile import istextfile from dvc.progress import Tqdm - if tree: - stat_func = tree.stat - open_func = tree.open - else: - stat_func = os.stat - open_func = open - hash_md5 = hashlib.md5() binary = not istextfile(fname, tree=tree) - size = stat_func(fname).st_size + size = tree.getsize(fname) or 0 no_progress_bar = True if size >= LARGE_FILE_SIZE: no_progress_bar = False - msg = "Computing md5 for a large file '{}'. This is only done once." - logger.info(msg.format(relpath(fname))) - name = relpath(fname) + msg = ( + f"Computing md5 for a large file '{fname}'. " + "This is only done once." + ) + logger.info(msg) with Tqdm( - desc=name, + desc=str(fname), disable=no_progress_bar, total=size, bytes=True, leave=False, ) as pbar: - with open_func(fname, "rb") as fobj: + with tree.open(fname, "rb") as fobj: _fobj_md5(fobj, hash_md5, binary, pbar.update) return hash_md5.hexdigest() diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 971c2f28d..c1bc89dcb 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -38,7 +38,7 @@ from tests.utils import get_gitignore_content def test_add(tmp_dir, dvc): (stage,) = tmp_dir.dvc_gen({"foo": "foo"}) - md5 = file_md5("foo") + md5 = file_md5("foo", dvc.tree) assert stage is not None @@ -443,7 +443,7 @@ class TestDoubleAddUnchanged(TestDvc): def test_should_update_state_entry_for_file_after_add(mocker, dvc, tmp_dir): - file_md5_counter = mocker.spy(dvc_module.tree.local, "file_md5") + file_md5_counter = mocker.spy(dvc_module.oid, "file_md5") tmp_dir.gen("foo", "foo") ret = main(["config", "cache.type", "copy"]) @@ -474,7 +474,7 @@ def test_should_update_state_entry_for_file_after_add(mocker, dvc, tmp_dir): def test_should_update_state_entry_for_directory_after_add( mocker, dvc, tmp_dir ): - file_md5_counter = mocker.spy(dvc_module.tree.local, "file_md5") + file_md5_counter = mocker.spy(dvc_module.oid, "file_md5") tmp_dir.gen({"data/data": "foo", "data/data_sub/sub_data": "foo"}) @@ -521,7 +521,7 @@ class TestAddCommit(TestDvc): def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc): tmp_dir.gen({"data/data": "foo"}) - get_dir_hash_counter = mocker.spy(LocalTree, "get_dir_hash") + get_dir_hash_counter = mocker.spy(dvc_module.oid, "get_dir_hash") ret = main(["add", "data"]) assert ret == 0 diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index 888546030..2e5cef473 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -5,6 +5,7 @@ import shutil import pytest from flaky.flaky_decorator import flaky +import dvc as dvc_module from dvc.cache import NamedCache from dvc.external_repo import clean_repos from dvc.main import main @@ -15,7 +16,6 @@ from dvc.remote.base import ( STATUS_OK, ) from dvc.stage.exceptions import StageNotFound -from dvc.tree.local import LocalTree from dvc.utils.fs import move, remove from dvc.utils.serialize import dump_yaml, load_yaml @@ -231,7 +231,7 @@ def test_warn_on_outdated_stage(tmp_dir, dvc, local_remote, caplog): def test_hash_recalculation(mocker, dvc, tmp_dir, local_remote): tmp_dir.gen({"foo": "foo"}) - test_get_file_hash = mocker.spy(LocalTree, "get_file_hash") + test_get_file_hash = mocker.spy(dvc_module.oid, "get_file_hash") ret = main(["config", "cache.type", "hardlink"]) assert ret == 0 ret = main(["add", "foo"]) @@ -287,7 +287,7 @@ def test_verify_hashes( remove("dir") remove(dvc.cache.local.cache_dir) - hash_spy = mocker.spy(dvc.cache.local.tree, "get_file_hash") + hash_spy = mocker.spy(dvc_module.oid, "get_file_hash") dvc.pull() assert hash_spy.call_count == 0 diff --git a/tests/func/test_install.py b/tests/func/test_install.py index c3f480d80..be2cc4c82 100644 --- a/tests/func/test_install.py +++ b/tests/func/test_install.py @@ -64,7 +64,7 @@ class TestInstall: conf["core"]["remote"] = "store" tmp_dir.dvc_gen("file", "file_content", "commit message") - file_checksum = file_md5("file") + file_checksum = file_md5("file", dvc.tree) expected_storage_path = ( storage_path / file_checksum[:2] / file_checksum[2:] ) diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index e5fc1ecdf..970f6e3aa 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -11,7 +11,6 @@ from dvc.dir_info import DirInfo from dvc.exceptions import DownloadError, RemoteCacheRequiredError, UploadError from dvc.main import main from dvc.path_info import PathInfo -from dvc.tree.base import BaseTree from dvc.tree.local import LocalTree from dvc.utils.fs import remove from tests.basic_env import TestDvc @@ -147,6 +146,8 @@ class TestRemoteShouldHandleUppercaseRemoteName(TestDvc): def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): + from dvc.oid import get_hash + tmp_dir.gen({"data": {"1": "1 content", "2": "2 content"}}) path_info = PathInfo("data") @@ -154,18 +155,18 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): dir_info = DirInfo.from_list( [{"relpath": "1", "md5": "1"}, {"relpath": "2", "md5": "2"}] ) - with patch.object( - BaseTree, "_collect_dir", return_value=dir_info, + with patch( + "dvc.oid._collect_dir", return_value=dir_info, ): - hash1 = dvc.cache.local.tree.get_hash(path_info, "md5") + hash1 = get_hash(path_info, dvc.cache.local.tree, "md5") dir_info = DirInfo.from_list( [{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}] ) - with patch.object( - BaseTree, "_collect_dir", return_value=dir_info, + with patch( + "dvc.oid._collect_dir", return_value=dir_info, ): - hash2 = dvc.cache.local.tree.get_hash(path_info, "md5") + hash2 = get_hash(path_info, dvc.cache.local.tree, "md5") assert hash1 == hash2 diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py index d7fab0f32..602d4f0cd 100644 --- a/tests/func/test_repro.py +++ b/tests/func/test_repro.py @@ -651,7 +651,10 @@ class TestReproDataSource(TestReproChangedData): stages = self.dvc.reproduce(self.foo_stage.path) self.assertTrue(filecmp.cmp(self.FOO, self.BAR, shallow=False)) - self.assertEqual(stages[0].outs[0].hash_info.value, file_md5(self.BAR)) + self.assertEqual( + stages[0].outs[0].hash_info.value, + file_md5(self.BAR, self.dvc.tree), + ) class TestReproChangedDir(SingleStageRun, TestDvc): diff --git a/tests/func/test_run_single_stage.py b/tests/func/test_run_single_stage.py index 2d652cd6f..037e5132a 100644 --- a/tests/func/test_run_single_stage.py +++ b/tests/func/test_run_single_stage.py @@ -60,7 +60,9 @@ class TestRun(TestDvc): self.assertEqual(len(stage.deps), len(deps)) self.assertEqual(len(stage.outs), len(outs + outs_no_cache)) self.assertEqual(stage.outs[0].fspath, outs[0]) - self.assertEqual(stage.outs[0].hash_info.value, file_md5(self.FOO)) + self.assertEqual( + stage.outs[0].hash_info.value, file_md5(self.FOO, self.dvc.tree) + ) self.assertTrue(stage.path, fname) with self.assertRaises(OutputDuplicationError): diff --git a/tests/func/test_state.py b/tests/func/test_state.py index 2fbf4fe00..2ed075192 100644 --- a/tests/func/test_state.py +++ b/tests/func/test_state.py @@ -12,7 +12,7 @@ def test_state(tmp_dir, dvc): tmp_dir.gen("foo", "foo content") path = tmp_dir / "foo" path_info = PathInfo(path) - hash_info = HashInfo("md5", file_md5(path)) + hash_info = HashInfo("md5", file_md5(path, dvc.tree)) state = State(dvc) @@ -25,7 +25,7 @@ def test_state(tmp_dir, dvc): assert state.get(path_info) is None - hash_info = HashInfo("md5", file_md5(path)) + hash_info = HashInfo("md5", file_md5(path, dvc.tree)) state.save(path_info, hash_info) assert state.get(path_info) == hash_info @@ -60,7 +60,7 @@ def test_get_state_record_for_inode(get_inode_mock, tmp_dir, dvc): assert inode != state._to_sqlite(inode) foo = tmp_dir / "foo" - md5 = file_md5(foo) + md5 = file_md5(foo, dvc.tree) get_inode_mock.side_effect = mock_get_inode(inode) with state: diff --git a/tests/func/test_tree.py b/tests/func/test_tree.py index 88471761d..9176f7832 100644 --- a/tests/func/test_tree.py +++ b/tests/func/test_tree.py @@ -346,8 +346,6 @@ def test_tree_ls_with_etag(dvc, cloud): for details in tree.ls(path_info / "data", recursive=True, detail=True): assert ( - tree.get_file_hash( - path_info.replace(path=details["name"]), tree.PARAM_CHECKSUM - ).value - == details[tree.PARAM_CHECKSUM] + tree.info(path_info.replace(path=details["name"]))["etag"] + == details["etag"] ) diff --git a/tests/func/test_utils.py b/tests/func/test_utils.py index 2c1166f04..47a2dc71c 100644 --- a/tests/func/test_utils.py +++ b/tests/func/test_utils.py @@ -1,10 +1,12 @@ from dvc import utils +from dvc.tree.local import LocalTree def test_file_md5_crlf(tmp_dir): + tree = LocalTree(None, {}) tmp_dir.gen("cr", b"a\nb\nc") tmp_dir.gen("crlf", b"a\r\nb\r\nc") - assert utils.file_md5("cr") == utils.file_md5("crlf") + assert utils.file_md5("cr", tree) == utils.file_md5("crlf", tree) def test_dict_md5(): diff --git a/tests/unit/remote/test_azure.py b/tests/unit/remote/test_azure.py index 507f40898..f3b582f57 100644 --- a/tests/unit/remote/test_azure.py +++ b/tests/unit/remote/test_azure.py @@ -29,16 +29,13 @@ def test_init(dvc): assert tree._conn_str == connection_string -def test_get_file_hash(tmp_dir, azure): +def test_info(tmp_dir, azure): tmp_dir.gen("foo", "foo") tree = AzureTree(None, azure.config) to_info = azure tree.upload(PathInfo("foo"), to_info) assert tree.exists(to_info) - hash_info = tree.get_file_hash(to_info, "etag") - assert hash_info.name == "etag" - hash_ = hash_info.value - assert hash_ + hash_ = tree.info(to_info)["etag"] assert isinstance(hash_, str) assert hash_.strip("'").strip('"') == hash_ diff --git a/tests/unit/remote/test_remote_tree.py b/tests/unit/remote/test_remote_tree.py index 083925ccc..319985ec1 100644 --- a/tests/unit/remote/test_remote_tree.py +++ b/tests/unit/remote/test_remote_tree.py @@ -98,8 +98,8 @@ def test_copy_preserve_etag_across_buckets(remote, dvc): remote.tree.copy(from_info, to_info) - from_hash = remote.tree.get_hash(from_info, remote.tree.PARAM_CHECKSUM) - to_hash = another.get_hash(to_info, remote.tree.PARAM_CHECKSUM) + from_hash = remote.tree.info(from_info)["etag"] + to_hash = another.info(to_info)["etag"] assert from_hash == to_hash diff --git a/tests/unit/tree/test_dvc.py b/tests/unit/tree/test_dvc.py index 9ee63044b..840cdb4fa 100644 --- a/tests/unit/tree/test_dvc.py +++ b/tests/unit/tree/test_dvc.py @@ -4,6 +4,7 @@ import shutil import pytest from dvc.hash_info import HashInfo +from dvc.oid import get_hash from dvc.path_info import PathInfo from dvc.tree.dvc import DvcTree @@ -200,19 +201,23 @@ def test_isdvc(tmp_dir, dvc): def test_get_hash_file(tmp_dir, dvc): tmp_dir.dvc_gen({"foo": "foo"}) tree = DvcTree(dvc) - assert tree.get_hash(PathInfo(tmp_dir) / "foo", "md5") == HashInfo( - "md5", "acbd18db4cc2f85cedef654fccc4a4d8", + assert ( + tree.info(PathInfo(tmp_dir) / "foo")["md5"] + == "acbd18db4cc2f85cedef654fccc4a4d8" ) def test_get_hash_dir(tmp_dir, dvc, mocker): + import dvc as dvc_module + tmp_dir.dvc_gen( {"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}} ) tree = DvcTree(dvc) - get_file_hash_spy = mocker.spy(tree, "get_file_hash") - assert tree.get_hash(PathInfo(tmp_dir) / "dir", "md5") == HashInfo( - "md5", "8761c4e9acad696bee718615e23e22db.dir", + get_file_hash_spy = mocker.spy(dvc_module.oid, "get_file_hash") + assert ( + tree.info(PathInfo(tmp_dir) / "dir")["md5"] + == "8761c4e9acad696bee718615e23e22db.dir" ) assert not get_file_hash_spy.called @@ -223,10 +228,14 @@ def test_get_hash_granular(tmp_dir, dvc): ) tree = DvcTree(dvc) subdir = PathInfo(tmp_dir) / "dir" / "subdir" - assert tree.get_hash(subdir, "md5") == HashInfo( + assert tree.info(subdir).get("md5") is None + assert get_hash(subdir, tree, "md5") == HashInfo( "md5", "af314506f1622d107e0ed3f14ec1a3b5.dir", ) - assert tree.get_hash(subdir / "data", "md5") == HashInfo( + assert ( + tree.info(subdir / "data")["md5"] == "8d777f385d3dfec8815d20f7496026dc" + ) + assert get_hash(subdir / "data", tree, "md5") == HashInfo( "md5", "8d777f385d3dfec8815d20f7496026dc", ) @@ -236,9 +245,11 @@ def test_get_hash_dirty_file(tmp_dir, dvc): (tmp_dir / "file").write_text("something") tree = DvcTree(dvc) - actual = tree.get_hash(PathInfo(tmp_dir) / "file", "md5") - expected = HashInfo("md5", "8c7dd922ad47494fc02c388e12c00eac") - assert actual == expected + expected = "8c7dd922ad47494fc02c388e12c00eac" + assert tree.info(PathInfo(tmp_dir) / "file").get("md5") == expected + assert get_hash(PathInfo(tmp_dir) / "file", tree, "md5") == HashInfo( + "md5", expected + ) def test_get_hash_dirty_dir(tmp_dir, dvc): @@ -246,7 +257,8 @@ def test_get_hash_dirty_dir(tmp_dir, dvc): (tmp_dir / "dir" / "baz").write_text("baz") tree = DvcTree(dvc) - actual = tree.get_hash(PathInfo(tmp_dir) / "dir", "md5") - expected = HashInfo("md5", "5ea40360f5b4ec688df672a4db9c17d1.dir") - - assert actual == expected + expected = "5ea40360f5b4ec688df672a4db9c17d1.dir" + assert tree.info(PathInfo(tmp_dir) / "dir").get("md5") == expected + assert get_hash(PathInfo(tmp_dir) / "dir", tree, "md5") == HashInfo( + "md5", expected + ) diff --git a/tests/unit/tree/test_repo.py b/tests/unit/tree/test_repo.py index e9a40f61f..bb84f6500 100644 --- a/tests/unit/tree/test_repo.py +++ b/tests/unit/tree/test_repo.py @@ -5,6 +5,7 @@ from unittest import mock import pytest from dvc.hash_info import HashInfo +from dvc.oid import get_hash from dvc.path_info import PathInfo from dvc.tree.repo import RepoTree @@ -483,11 +484,13 @@ def test_repo_tree_no_subrepos(tmp_dir, dvc, scm): def test_get_hash_cached_file(tmp_dir, dvc, mocker): tmp_dir.dvc_gen({"foo": "foo"}) tree = RepoTree(dvc) - dvc_tree_spy = mocker.spy(tree._dvctrees[dvc.root_dir], "get_file_hash") - assert tree.get_hash(PathInfo(tmp_dir) / "foo", "md5") == HashInfo( - "md5", "acbd18db4cc2f85cedef654fccc4a4d8", + expected = "acbd18db4cc2f85cedef654fccc4a4d8" + assert tree.info(PathInfo(tmp_dir) / "foo").get("md5") is None + assert get_hash(PathInfo(tmp_dir) / "foo", tree, "md5") == HashInfo( + "md5", expected, ) - assert dvc_tree_spy.called + (tmp_dir / "foo").unlink() + assert tree.info(PathInfo(tmp_dir) / "foo")["md5"] == expected def test_get_hash_cached_dir(tmp_dir, dvc, mocker): @@ -495,21 +498,17 @@ def test_get_hash_cached_dir(tmp_dir, dvc, mocker): {"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}} ) tree = RepoTree(dvc) - get_file_hash_spy = mocker.spy(tree, "get_file_hash") - dvc_tree_spy = mocker.spy(tree._dvctrees[dvc.root_dir], "get_dir_hash") - assert tree.get_hash(PathInfo(tmp_dir) / "dir", "md5") == HashInfo( + expected = "8761c4e9acad696bee718615e23e22db.dir" + assert tree.info(PathInfo(tmp_dir) / "dir").get("md5") is None + assert get_hash(PathInfo(tmp_dir) / "dir", tree, "md5") == HashInfo( "md5", "8761c4e9acad696bee718615e23e22db.dir", ) - assert get_file_hash_spy.called - assert not dvc_tree_spy.called - get_file_hash_spy.reset_mock() shutil.rmtree(tmp_dir / "dir") - assert tree.get_hash(PathInfo(tmp_dir) / "dir", "md5") == HashInfo( + assert tree.info(PathInfo(tmp_dir) / "dir")["md5"] == expected + assert get_hash(PathInfo(tmp_dir) / "dir", tree, "md5") == HashInfo( "md5", "8761c4e9acad696bee718615e23e22db.dir", ) - assert not get_file_hash_spy.called - assert dvc_tree_spy.called def test_get_hash_cached_granular(tmp_dir, dvc, mocker): @@ -517,15 +516,19 @@ def test_get_hash_cached_granular(tmp_dir, dvc, mocker): {"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}} ) tree = RepoTree(dvc) - dvc_tree_spy = mocker.spy(tree._dvctrees[dvc.root_dir], "get_file_hash") subdir = PathInfo(tmp_dir) / "dir" / "subdir" - assert tree.get_hash(subdir, "md5") == HashInfo( + assert tree.info(subdir).get("md5") is None + assert get_hash(subdir, tree, "md5") == HashInfo( "md5", "af314506f1622d107e0ed3f14ec1a3b5.dir", ) - assert tree.get_hash(subdir / "data", "md5") == HashInfo( + assert tree.info(subdir / "data").get("md5") is None + assert get_hash(subdir / "data", tree, "md5") == HashInfo( "md5", "8d777f385d3dfec8815d20f7496026dc", ) - assert dvc_tree_spy.called + (tmp_dir / "dir" / "subdir" / "data").unlink() + assert ( + tree.info(subdir / "data")["md5"] == "8d777f385d3dfec8815d20f7496026dc" + ) def test_get_hash_mixed_dir(tmp_dir, scm, dvc): @@ -541,7 +544,7 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc): tmp_dir.scm.commit("add dir") tree = RepoTree(dvc) - actual = tree.get_hash(PathInfo(tmp_dir) / "dir", "md5") + actual = get_hash(PathInfo(tmp_dir) / "dir", tree, "md5") expected = HashInfo("md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir") assert actual == expected @@ -551,7 +554,17 @@ def test_get_hash_dirty_file(tmp_dir, dvc): (tmp_dir / "file").write_text("something") tree = RepoTree(dvc) - actual = tree.get_hash(PathInfo(tmp_dir) / "file", "md5") + assert tree.info(PathInfo(tmp_dir) / "file").get("md5") is None + actual = get_hash(PathInfo(tmp_dir) / "file", tree, "md5") + expected = HashInfo("md5", "437b930db84b8079c2dd804a71936b5f") + assert actual == expected + + (tmp_dir / "file").unlink() + assert ( + tree.info(PathInfo(tmp_dir) / "file")["md5"] + == "8c7dd922ad47494fc02c388e12c00eac" + ) + actual = get_hash(PathInfo(tmp_dir) / "file", tree, "md5") expected = HashInfo("md5", "8c7dd922ad47494fc02c388e12c00eac") assert actual == expected @@ -561,7 +574,7 @@ def test_get_hash_dirty_dir(tmp_dir, dvc): (tmp_dir / "dir" / "baz").write_text("baz") tree = RepoTree(dvc) - actual = tree.get_hash(PathInfo(tmp_dir) / "dir", "md5") + actual = get_hash(PathInfo(tmp_dir) / "dir", tree, "md5") expected = HashInfo("md5", "ba75a2162ca9c29acecb7957105a0bc2.dir") assert actual == expected assert actual.dir_info.nfiles == 3 diff --git a/tests/unit/utils/test_stream.py b/tests/unit/utils/test_stream.py index 39d6c0f2d..cd37152c8 100644 --- a/tests/unit/utils/test_stream.py +++ b/tests/unit/utils/test_stream.py @@ -1,3 +1,4 @@ +from dvc.tree.local import LocalTree from dvc.utils import file_md5 from dvc.utils.stream import HashedStreamReader @@ -10,7 +11,7 @@ def test_hashed_stream_reader(tmp_dir): stream_reader = HashedStreamReader(fobj) assert stream_reader.read(3) == b"foo" - hex_digest = file_md5(foo) + hex_digest = file_md5(foo, LocalTree(None, {})) assert stream_reader.is_text_file assert hex_digest == stream_reader.hash_info.value @@ -26,6 +27,6 @@ def test_hashed_stream_reader_as_chunks(tmp_dir): if not chunk: break - hex_digest = file_md5(foo) + hex_digest = file_md5(foo, LocalTree(None, {})) assert not stream_reader.is_text_file assert hex_digest == stream_reader.hash_info.value diff --git a/tests/unit/utils/test_utils.py b/tests/unit/utils/test_utils.py index c990d6a5a..11b2d8d71 100644 --- a/tests/unit/utils/test_utils.py +++ b/tests/unit/utils/test_utils.py @@ -4,6 +4,7 @@ import re import pytest from dvc.path_info import PathInfo +from dvc.tree.local import LocalTree from dvc.utils import ( dict_sha256, file_md5, @@ -86,7 +87,8 @@ def test_fix_env_pyenv(path, orig): def test_file_md5(tmp_dir): tmp_dir.gen("foo", "foo content") - assert file_md5("foo") == file_md5(PathInfo("foo")) + tree = LocalTree(None, {}) + assert file_md5("foo", tree) == file_md5(PathInfo("foo"), tree) def test_tmp_fname():