Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-07 23:09:21 +01:00
Code Issues Packages Projects Releases Wiki Activity

remote: rename "checksum" -> "hash" (#4051)

Browse Source 
* remote: s/get_checksum/get_hash/

* remote/tree: s/get_file_checksum/get_file_hash/

* remote: s/get_dir_checksum/get_dir_hash/

* remote: s/checksum_to_path/hash_to_path/

* remote: s/path_to_checksum/path_to_hash/

* remote: cleanup remaining checksum references

* remote.index: s/checksum/hash/

* note: does not rename "checksum" field in the index DB schema
This commit is contained in:
Peter Rowlands (변기호)
2020-06-16 12:16:51 +03:00
committed by GitHub
parent 73e60a8c13
commit 0fb9870de6
29 changed files with 476 additions and 519 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/api.py
+1 -1
View File
@@ -30,7 +30,7 @@ def get_url(path, repo=None, rev=None, remote=None):
raise UrlNotDvcRepoError(_repo.url)
out = _repo.find_out_by_relpath(path)
remote_obj = _repo.cloud.get_remote(remote)
return str(remote_obj.checksum_to_path_info(out.checksum))
return str(remote_obj.hash_to_path_info(out.checksum))
def open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
dvc/data_cloud.py
+1 -1
View File
@@ -92,7 +92,7 @@ class DataCloud:
def _save_pulled_checksums(self, cache):
for checksum in cache.scheme_keys("local"):
cache_file = self.repo.cache.local.checksum_to_path_info(checksum)
cache_file = self.repo.cache.local.hash_to_path_info(checksum)
if self.repo.cache.local.tree.exists(cache_file):
# We can safely save here, as existing corrupted files will
# be removed upon status, while files corrupted during
dvc/dependency/repo.py
+2 -2
View File
@@ -64,8 +64,8 @@ class RepoDependency(LocalDependency):
# We are polluting our repo cache with some dir listing here
if tree.isdir(path):
return self.repo.cache.local.get_checksum(path, tree)
return tree.get_file_checksum(path)
return self.repo.cache.local.get_hash(path, tree)
return tree.get_file_hash(path)
def status(self):
current_checksum = self._get_checksum(locked=True)
dvc/output/base.py
+3 -3
View File
@@ -163,7 +163,7 @@ class BaseOutput:
@property
def cache_path(self):
return self.cache.checksum_to_path_info(self.checksum).url
return self.cache.hash_to_path_info(self.checksum).url
@property
def checksum_type(self):
@@ -178,11 +178,11 @@ class BaseOutput:
self.info[self.remote.tree.PARAM_CHECKSUM] = checksum
def get_checksum(self):
return self.remote.get_checksum(self.path_info)
return self.remote.get_hash(self.path_info)
@property
def is_dir_checksum(self):
return self.remote.is_dir_checksum(self.checksum)
return self.remote.is_dir_hash(self.checksum)
@property
def exists(self):
dvc/remote/azure.py
+1 -1
View File
@@ -113,7 +113,7 @@ class AzureRemoteTree(BaseRemoteTree):
logger.debug(f"Removing {path_info}")
self.blob_service.delete_blob(path_info.bucket, path_info.path)
def get_file_checksum(self, path_info):
def get_file_hash(self, path_info):
return self.get_etag(path_info)
def _upload(
dvc/remote/base.py
+249 -274
View File
File diff suppressed because it is too large Load Diff
dvc/remote/gdrive.py
+1 -1
View File
@@ -564,7 +564,7 @@ class GDriveRemoteTree(BaseRemoteTree):
item_id = self._get_item_id(path_info)
self.gdrive_delete_file(item_id)
def get_file_checksum(self, path_info):
def get_file_hash(self, path_info):
raise NotImplementedError
def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
dvc/remote/gs.py
+1 -1
View File
@@ -160,7 +160,7 @@ class GSRemoteTree(BaseRemoteTree):
to_bucket = self.gs.bucket(to_info.bucket)
from_bucket.copy_blob(blob, to_bucket, new_name=to_info.path)
def get_file_checksum(self, path_info):
def get_file_hash(self, path_info):
import base64
import codecs
dvc/remote/hdfs.py
+1 -1
View File
@@ -159,7 +159,7 @@ class HDFSRemoteTree(BaseRemoteTree):
assert match is not None
return match.group(gname)
def get_file_checksum(self, path_info):
def get_file_hash(self, path_info):
# NOTE: pyarrow doesn't support checksum, so we need to use hadoop
regex = r".*\t.*\t(?P<checksum>.*)"
stdout = self.hadoop_fs(
dvc/remote/http.py
+1 -1
View File
@@ -124,7 +124,7 @@ class HTTPRemoteTree(BaseRemoteTree):
def exists(self, path_info):
return bool(self.request("HEAD", path_info.url))
def get_file_checksum(self, path_info):
def get_file_hash(self, path_info):
url = path_info.url
headers = self.request("HEAD", url).headers
etag = headers.get("ETag") or headers.get("Content-MD5")
dvc/remote/index.py
+27 -31
View File
@@ -25,15 +25,15 @@ class RemoteIndexNoop:
def __iter__(self):
return iter([])
def __contains__(self, checksum):
def __contains__(self, hash_):
return False
@staticmethod
def checksums():
def hashes():
return []
@staticmethod
def dir_checksums():
def dir_hashes():
return []
def load(self):
@@ -54,13 +54,13 @@ class RemoteIndexNoop:
class RemoteIndex:
"""Class for indexing remote checksums in a sqlite3 database.
"""Class for indexing remote hashes in a sqlite3 database.
Args:
repo: repo for this remote index.
name: name for this index. Index db will be loaded from and saved to
``.dvc/tmp/index/{name}.idx``.
dir_suffix: suffix used for naming directory checksums
dir_suffix: suffix used for naming directory hashes
"""
INDEX_SUFFIX = ".idx"
@@ -79,8 +79,8 @@ class RemoteIndex:
def __iter__(self):
cmd = f"SELECT checksum FROM {self.INDEX_TABLE}"
for (checksum,) in self._execute(cmd):
yield checksum
for (hash_,) in self._execute(cmd):
yield hash_
def __enter__(self):
self.lock.acquire()
@@ -90,25 +90,25 @@ class RemoteIndex:
self.dump()
self.lock.release()
def __contains__(self, checksum):
def __contains__(self, hash_):
cmd = "SELECT checksum FROM {} WHERE checksum = (?)".format(
self.INDEX_TABLE
)
self._execute(cmd, (checksum,))
self._execute(cmd, (hash_,))
return self.cursor.fetchone() is not None
def checksums(self):
"""Iterate over checksums stored in the index."""
def hashes(self):
"""Iterate over hashes stored in the index."""
return iter(self)
def dir_checksums(self):
"""Iterate over .dir checksums stored in the index."""
def dir_hashes(self):
"""Iterate over .dir hashes stored in the index."""
cmd = f"SELECT checksum FROM {self.INDEX_TABLE} WHERE dir = 1"
for (checksum,) in self._execute(cmd):
yield checksum
for (hash_,) in self._execute(cmd):
yield hash_
def is_dir_checksum(self, checksum):
return checksum.endswith(self.dir_suffix)
def is_dir_hash(self, hash_):
return hash_.endswith(self.dir_suffix)
def _execute(self, cmd, parameters=()):
return self.cursor.execute(cmd, parameters)
@@ -185,28 +185,24 @@ class RemoteIndex:
cmd = f"DELETE FROM {self.INDEX_TABLE}"
self._execute(cmd)
def update(self, dir_checksums, file_checksums):
"""Update this index, adding the specified checksums.
def update(self, dir_hashes, file_hashes):
"""Update this index, adding the specified hashes.
Changes to the index will not committed until dump() is called.
"""
cmd = "INSERT OR IGNORE INTO {} (checksum, dir) VALUES (?, ?)".format(
self.INDEX_TABLE
)
self._executemany(
cmd, ((checksum, True) for checksum in dir_checksums)
)
self._executemany(
cmd, ((checksum, False) for checksum in file_checksums)
)
self._executemany(cmd, ((hash_, True) for hash_ in dir_hashes))
self._executemany(cmd, ((hash_, False) for hash_ in file_hashes))
def intersection(self, checksums):
"""Iterate over values from `checksums` which exist in the index."""
def intersection(self, hashes):
"""Iterate over values from `hashes` which exist in the index."""
# sqlite has a compile time limit of 999, see:
# https://www.sqlite.org/c3ref/c_limit_attached.html#sqlitelimitvariablenumber
for chunk in lchunks(999, checksums):
for chunk in lchunks(999, hashes):
cmd = "SELECT checksum FROM {} WHERE checksum IN ({})".format(
self.INDEX_TABLE, ",".join("?" for checksum in chunk)
self.INDEX_TABLE, ",".join("?" for hash_ in chunk)
)
for (checksum,) in self._execute(cmd, chunk):
yield checksum
for (hash_,) in self._execute(cmd, chunk):
yield hash_
dvc/remote/local.py
+60 -68
View File
@@ -169,7 +169,7 @@ class LocalRemoteTree(BaseRemoteTree):
# and the cache type is `hardlink`, we might reach link limits and
# will get something like: `too many links error`
#
# This is because all those empty files will have the same checksum
# This is because all those empty files will have the same hash
# (i.e. 68b329da9893e34099c7d8ad5cb9c940), therefore, they will be
# linked to the same file in the cache.
#
@@ -268,7 +268,7 @@ class LocalRemoteTree(BaseRemoteTree):
return stat.S_IMODE(mode) == self.CACHE_MODE
def get_file_checksum(self, path_info):
def get_file_hash(self, path_info):
return file_md5(path_info)[0]
@staticmethod
@@ -313,8 +313,8 @@ class LocalRemoteTree(BaseRemoteTree):
else:
yield from walk_files(path_info)
def _remove_unpacked_dir(self, checksum):
info = self.checksum_to_path_info(checksum)
def _remove_unpacked_dir(self, hash_):
info = self.hash_to_path_info(hash_)
path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
self.remove(path_info)
@@ -367,30 +367,28 @@ class LocalCache(CloudCache):
def cache_path(self):
return os.path.abspath(self.cache_dir)
def checksum_to_path(self, checksum):
def hash_to_path(self, hash_):
# NOTE: `self.cache_path` is already normalized so we can simply use
# `os.sep` instead of `os.path.join`. This results in this helper
# being ~5.5 times faster.
return (
f"{self.cache_path}{os.sep}{checksum[0:2]}{os.sep}{checksum[2:]}"
)
return f"{self.cache_path}{os.sep}{hash_[0:2]}{os.sep}{hash_[2:]}"
def checksums_exist(self, checksums, jobs=None, name=None):
def hashes_exist(self, hashes, jobs=None, name=None):
return [
checksum
for checksum in Tqdm(
checksums,
hash_
for hash_ in Tqdm(
hashes,
unit="file",
desc="Querying "
+ ("cache in " + name if name else "local cache"),
)
if not self.changed_cache_file(checksum)
if not self.changed_cache_file(hash_)
]
def already_cached(self, path_info):
assert path_info.scheme in ["", "local"]
current_md5 = self.get_checksum(path_info)
current_md5 = self.get_hash(path_info)
if not current_md5:
return False
@@ -434,7 +432,7 @@ class LocalCache(CloudCache):
dir_status_info contains status for .dir files, file_status_info
contains status for all other files, and dir_contents is a dict of
{dir_checksum: set(file_checksum, ...)} which can be used to map
{dir_hash: set(file_hash, ...)} which can be used to map
a .dir file to its file contents.
"""
logger.debug(f"Preparing to collect status from {remote.path_info}")
@@ -442,7 +440,7 @@ class LocalCache(CloudCache):
logger.debug("Collecting information from local cache...")
local_exists = frozenset(
self.checksums_exist(md5s, jobs=jobs, name=self.cache_dir)
self.hashes_exist(md5s, jobs=jobs, name=self.cache_dir)
)
# This is a performance optimization. We can safely assume that,
@@ -457,12 +455,12 @@ class LocalCache(CloudCache):
dir_md5s = set(named_cache.dir_keys(self.scheme))
if dir_md5s:
remote_exists.update(
self._indexed_dir_checksums(named_cache, remote, dir_md5s)
self._indexed_dir_hashes(named_cache, remote, dir_md5s)
)
md5s.difference_update(remote_exists)
if md5s:
remote_exists.update(
remote.checksums_exist(
remote.hashes_exist(
md5s, jobs=jobs, name=str(remote.path_info)
)
)
@@ -473,23 +471,23 @@ class LocalCache(CloudCache):
def _make_status(
self, named_cache, show_checksums, local_exists, remote_exists
):
def make_names(checksum, names):
return {"name": checksum if show_checksums else " ".join(names)}
def make_names(hash_, names):
return {"name": hash_ if show_checksums else " ".join(names)}
dir_status = {}
file_status = {}
dir_contents = {}
for checksum, item in named_cache[self.scheme].items():
for hash_, item in named_cache[self.scheme].items():
if item.children:
dir_status[checksum] = make_names(checksum, item.names)
dir_contents[checksum] = set()
for child_checksum, child in item.children.items():
file_status[child_checksum] = make_names(
child_checksum, child.names
dir_status[hash_] = make_names(hash_, item.names)
dir_contents[hash_] = set()
for child_hash, child in item.children.items():
file_status[child_hash] = make_names(
child_hash, child.names
)
dir_contents[checksum].add(child_checksum)
dir_contents[hash_].add(child_hash)
else:
file_status[checksum] = make_names(checksum, item.names)
file_status[hash_] = make_names(hash_, item.names)
self._fill_statuses(dir_status, local_exists, remote_exists)
self._fill_statuses(file_status, local_exists, remote_exists)
@@ -498,52 +496,50 @@ class LocalCache(CloudCache):
return dir_status, file_status, dir_contents
def _indexed_dir_checksums(self, named_cache, remote, dir_md5s):
# Validate our index by verifying all indexed .dir checksums
def _indexed_dir_hashes(self, named_cache, remote, dir_md5s):
# Validate our index by verifying all indexed .dir hashes
# still exist on the remote
indexed_dirs = set(remote.index.dir_checksums())
indexed_dirs = set(remote.index.dir_hashes())
indexed_dir_exists = set()
if indexed_dirs:
indexed_dir_exists.update(
remote.tree.list_checksums_exists(indexed_dirs)
remote.tree.list_hashes_exists(indexed_dirs)
)
missing_dirs = indexed_dirs.difference(indexed_dir_exists)
if missing_dirs:
logger.debug(
"Remote cache missing indexed .dir checksums '{}', "
"Remote cache missing indexed .dir hashes '{}', "
"clearing remote index".format(", ".join(missing_dirs))
)
remote.index.clear()
# Check if non-indexed (new) dir checksums exist on remote
# Check if non-indexed (new) dir hashes exist on remote
dir_exists = dir_md5s.intersection(indexed_dir_exists)
dir_exists.update(
remote.tree.list_checksums_exists(dir_md5s - dir_exists)
remote.tree.list_hashes_exists(dir_md5s - dir_exists)
)
# If .dir checksum exists on the remote, assume directory contents
# If .dir hash exists on the remote, assume directory contents
# still exists on the remote
for dir_checksum in dir_exists:
file_checksums = list(
named_cache.child_keys(self.scheme, dir_checksum)
)
if dir_checksum not in remote.index:
for dir_hash in dir_exists:
file_hashes = list(named_cache.child_keys(self.scheme, dir_hash))
if dir_hash not in remote.index:
logger.debug(
"Indexing new .dir '{}' with '{}' nested files".format(
dir_checksum, len(file_checksums)
dir_hash, len(file_hashes)
)
)
remote.index.update([dir_checksum], file_checksums)
yield dir_checksum
yield from file_checksums
remote.index.update([dir_hash], file_hashes)
yield dir_hash
yield from file_hashes
@staticmethod
def _fill_statuses(checksum_info_dir, local_exists, remote_exists):
def _fill_statuses(hash_info_dir, local_exists, remote_exists):
# Using sets because they are way faster for lookups
local = set(local_exists)
remote = set(remote_exists)
for md5, info in checksum_info_dir.items():
for md5, info in hash_info_dir.items():
status = STATUS_MAP[(md5 in local, md5 in remote)]
info["status"] = status
@@ -551,15 +547,15 @@ class LocalCache(CloudCache):
cache = []
path_infos = []
names = []
checksums = []
hashes = []
for md5, info in Tqdm(
status_info.items(), desc="Analysing status", unit="file"
):
if info["status"] == status:
cache.append(self.checksum_to_path_info(md5))
path_infos.append(remote.checksum_to_path_info(md5))
cache.append(self.hash_to_path_info(md5))
path_infos.append(remote.hash_to_path_info(md5))
names.append(info["name"])
checksums.append(md5)
hashes.append(md5)
if download:
to_infos = cache
@@ -568,7 +564,7 @@ class LocalCache(CloudCache):
to_infos = path_infos
from_infos = cache
return from_infos, to_infos, names, checksums
return from_infos, to_infos, names, hashes
def _process(
self,
@@ -630,20 +626,18 @@ class LocalCache(CloudCache):
# for uploads, push files first, and any .dir files last
file_futures = {}
for from_info, to_info, name, checksum in zip(*file_plans):
file_futures[checksum] = executor.submit(
for from_info, to_info, name, hash_ in zip(*file_plans):
file_futures[hash_] = executor.submit(
func, from_info, to_info, name
)
dir_futures = {}
for from_info, to_info, name, dir_checksum in zip(
*dir_plans
):
for from_info, to_info, name, dir_hash in zip(*dir_plans):
wait_futures = {
future
for file_checksum, future in file_futures.items()
if file_checksum in dir_contents[dir_checksum]
for file_hash, future in file_futures.items()
if file_hash in dir_contents[dir_hash]
}
dir_futures[dir_checksum] = executor.submit(
dir_futures[dir_hash] = executor.submit(
self._dir_upload,
func,
wait_futures,
@@ -666,16 +660,14 @@ class LocalCache(CloudCache):
if not download:
# index successfully pushed dirs
for dir_checksum, future in dir_futures.items():
for dir_hash, future in dir_futures.items():
if future.result() == 0:
file_checksums = dir_contents[dir_checksum]
file_hashes = dir_contents[dir_hash]
logger.debug(
"Indexing pushed dir '{}' with "
"'{}' nested files".format(
dir_checksum, len(file_checksums)
)
"'{}' nested files".format(dir_hash, len(file_hashes))
)
remote.index.update([dir_checksum], file_checksums)
remote.index.update([dir_hash], file_hashes)
return len(dir_plans[0]) + len(file_plans[0])
@@ -714,10 +706,10 @@ class LocalCache(CloudCache):
)
@staticmethod
def _log_missing_caches(checksum_info_dict):
def _log_missing_caches(hash_info_dict):
missing_caches = [
(md5, info)
for md5, info in checksum_info_dict.items()
for md5, info in hash_info_dict.items()
if info["status"] == STATUS_MISSING
]
if missing_caches:
dvc/remote/s3.py
+1 -1
View File
@@ -308,7 +308,7 @@ class S3RemoteTree(BaseRemoteTree):
if etag != cached_etag:
raise ETagMismatchError(etag, cached_etag)
def get_file_checksum(self, path_info):
def get_file_hash(self, path_info):
return self.get_etag(self.s3, path_info.bucket, path_info.path)
def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
dvc/remote/ssh/__init__.py
+6 -6
View File
@@ -237,7 +237,7 @@ class SSHRemoteTree(BaseRemoteTree):
with self.ssh(from_info) as ssh:
ssh.reflink(from_info.path, to_info.path)
def get_file_checksum(self, path_info):
def get_file_hash(self, path_info):
if path_info.scheme != self.scheme:
raise NotImplementedError
@@ -314,14 +314,14 @@ class SSHRemote(Remote):
return results
def checksums_exist(self, checksums, jobs=None, name=None):
def hashes_exist(self, hashes, jobs=None, name=None):
"""This is older implementation used in remote/base.py
We are reusing it in RemoteSSH, because SSH's batch_exists proved to be
faster than current approach (relying on exists(path_info)) applied in
remote/base.
"""
if not self.tree.CAN_TRAVERSE:
return list(set(checksums) & set(self.tree.all()))
return list(set(hashes) & set(self.tree.all()))
# possibly prompt for credentials before "Querying" progress output
self.tree.ensure_credentials()
@@ -329,7 +329,7 @@ class SSHRemote(Remote):
with Tqdm(
desc="Querying "
+ ("cache in " + name if name else "remote cache"),
total=len(checksums),
total=len(hashes),
unit="file",
) as pbar:
@@ -339,9 +339,9 @@ class SSHRemote(Remote):
with ThreadPoolExecutor(
max_workers=jobs or self.tree.JOBS
) as executor:
path_infos = [self.checksum_to_path_info(x) for x in checksums]
path_infos = [self.hash_to_path_info(x) for x in hashes]
chunks = to_chunks(path_infos, num_chunks=self.tree.JOBS)
results = executor.map(exists_with_progress, chunks)
in_remote = itertools.chain.from_iterable(results)
ret = list(itertools.compress(checksums, in_remote))
ret = list(itertools.compress(hashes, in_remote))
return ret
dvc/repo/diff.py
+1 -1
View File
@@ -37,7 +37,7 @@ def diff(self, a_rev="HEAD", b_rev=None):
def _to_checksum(output):
if on_working_tree:
return self.cache.local.get_checksum(output.path_info)
return self.cache.local.get_hash(output.path_info)
return output.checksum
def _exists(output):
dvc/repo/tree.py
+5 -5
View File
@@ -82,7 +82,7 @@ class DvcTree(BaseTree):
else:
checksum = out.checksum
try:
remote_info = remote_obj.checksum_to_path_info(checksum)
remote_info = remote_obj.hash_to_path_info(checksum)
return remote_obj.open(
remote_info, mode=mode, encoding=encoding
)
@@ -93,7 +93,7 @@ class DvcTree(BaseTree):
if out.is_dir_checksum:
checksum = self._get_granular_checksum(path, out)
cache_path = out.cache.checksum_to_path_info(checksum).url
cache_path = out.cache.hash_to_path_info(checksum).url
else:
cache_path = out.cache_path
return open(cache_path, mode=mode, encoding=encoding)
@@ -212,7 +212,7 @@ class DvcTree(BaseTree):
def isexec(self, path):
return False
def get_file_checksum(self, path_info):
def get_file_hash(self, path_info):
outs = self._find_outs(path_info, strict=False)
if len(outs) != 1:
raise OutputNotFoundError
@@ -373,7 +373,7 @@ class RepoTree(BaseTree):
for fname in files:
yield PathInfo(root) / fname
def get_file_checksum(self, path_info):
def get_file_hash(self, path_info):
"""Return file checksum for specified path.
If path_info is a DVC out, the pre-computed checksum for the file
@@ -384,7 +384,7 @@ class RepoTree(BaseTree):
raise FileNotFoundError
if self.dvctree and self.dvctree.exists(path_info):
try:
return self.dvctree.get_file_checksum(path_info)
return self.dvctree.get_file_hash(path_info)
except OutputNotFoundError:
pass
return file_md5(path_info, self)[0]
tests/func/remote/test_index.py
+13 -13
View File
@@ -15,12 +15,12 @@ def remote(tmp_dir, dvc, tmp_path_factory, mocker):
dvc.config["remote"]["upstream"] = {"url": url}
dvc.config["core"]["remote"] = "upstream"
# patch checksums_exist since the LocalRemote normally overrides
# BaseRemoteTree.checksums_exist.
def checksums_exist(self, *args, **kwargs):
return Remote.checksums_exist(self, *args, **kwargs)
# patch hashes_exist since the LocalRemote normally overrides
# BaseRemoteTree.hashes_exist.
def hashes_exist(self, *args, **kwargs):
return Remote.hashes_exist(self, *args, **kwargs)
mocker.patch.object(LocalRemote, "checksums_exist", checksums_exist)
mocker.patch.object(LocalRemote, "hashes_exist", hashes_exist)
# patch index class since LocalRemote normally overrides index class
mocker.patch.object(LocalRemote, "INDEX_CLS", RemoteIndex)
@@ -38,9 +38,9 @@ def test_indexed_on_status(tmp_dir, dvc, tmp_path_factory, remote):
dvc.status(cloud=True)
with remote.index:
assert {bar.checksum, baz["md5"]} == set(remote.index.checksums())
assert [bar.checksum] == list(remote.index.dir_checksums())
assert foo.checksum not in remote.index.checksums()
assert {bar.checksum, baz["md5"]} == set(remote.index.hashes())
assert [bar.checksum] == list(remote.index.dir_hashes())
assert foo.checksum not in remote.index.hashes()
def test_indexed_on_push(tmp_dir, dvc, tmp_path_factory, remote):
@@ -50,9 +50,9 @@ def test_indexed_on_push(tmp_dir, dvc, tmp_path_factory, remote):
dvc.push()
with remote.index:
assert {bar.checksum, baz["md5"]} == set(remote.index.checksums())
assert [bar.checksum] == list(remote.index.dir_checksums())
assert foo.checksum not in remote.index.checksums()
assert {bar.checksum, baz["md5"]} == set(remote.index.hashes())
assert [bar.checksum] == list(remote.index.dir_hashes())
assert foo.checksum not in remote.index.hashes()
def test_indexed_dir_missing(tmp_dir, dvc, tmp_path_factory, remote):
@@ -61,7 +61,7 @@ def test_indexed_dir_missing(tmp_dir, dvc, tmp_path_factory, remote):
remote.index.update([bar.checksum], [])
dvc.status(cloud=True)
with remote.index:
assert not list(remote.index.checksums())
assert not list(remote.index.hashes())
def test_clear_on_gc(tmp_dir, dvc, tmp_path_factory, remote, mocker):
@@ -101,4 +101,4 @@ def test_partial_upload(tmp_dir, dvc, tmp_path_factory, remote, mocker):
with pytest.raises(UploadError):
dvc.push()
with remote.index:
assert not list(remote.index.checksums())
assert not list(remote.index.hashes())
tests/func/test_add.py
+2 -2
View File
@@ -340,7 +340,7 @@ class TestAddCommit(TestDvc):
def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
tmp_dir.gen({"data/data": "foo"})
get_dir_checksum_counter = mocker.spy(LocalRemoteTree, "get_dir_checksum")
get_dir_hash_counter = mocker.spy(LocalRemoteTree, "get_dir_hash")
ret = main(["add", "data"])
assert ret == 0
@@ -349,7 +349,7 @@ def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
ret = main(["status"])
assert ret == 0
assert get_dir_checksum_counter.mock.call_count == 1
assert get_dir_hash_counter.mock.call_count == 1
class SymlinkAddTestBase(TestDvc):
tests/func/test_cache.py
+7 -7
View File
@@ -36,7 +36,7 @@ class TestCache(TestDvc):
self.assertIn(self.cache2_md5, md5_list)
def test_get(self):
cache = Cache(self.dvc).local.checksum_to_path_info(self.cache1_md5)
cache = Cache(self.dvc).local.hash_to_path_info(self.cache1_md5)
self.assertEqual(os.fspath(cache), self.cache1)
@@ -46,16 +46,16 @@ class TestCacheLoadBadDirCache(TestDvc):
self.assertEqual(len(ret), 0)
def test(self):
checksum = "123.dir"
fname = os.fspath(self.dvc.cache.local.checksum_to_path_info(checksum))
dir_hash = "123.dir"
fname = os.fspath(self.dvc.cache.local.hash_to_path_info(dir_hash))
self.create(fname, "<clearly>not,json")
with pytest.raises(DirCacheError):
self.dvc.cache.local.load_dir_cache(checksum)
self.dvc.cache.local.load_dir_cache(dir_hash)
checksum = "234.dir"
fname = os.fspath(self.dvc.cache.local.checksum_to_path_info(checksum))
dir_hash = "234.dir"
fname = os.fspath(self.dvc.cache.local.hash_to_path_info(dir_hash))
self.create(fname, '{"a": "b"}')
self._do_test(self.dvc.cache.local.load_dir_cache(checksum))
self._do_test(self.dvc.cache.local.load_dir_cache(dir_hash))
class TestExternalCacheDir(TestDvc):
tests/func/test_checkout.py
+2 -2
View File
@@ -100,8 +100,8 @@ class TestCheckoutCorruptedCacheDir(TestDvc):
# NOTE: modifying cache file for one of the files inside the directory
# to check if dvc will detect that the cache is corrupted.
entry = self.dvc.cache.local.load_dir_cache(out.checksum)[0]
checksum = entry[self.dvc.cache.local.tree.PARAM_CHECKSUM]
cache = os.fspath(self.dvc.cache.local.checksum_to_path_info(checksum))
entry_hash = entry[self.dvc.cache.local.tree.PARAM_CHECKSUM]
cache = os.fspath(self.dvc.cache.local.hash_to_path_info(entry_hash))
os.chmod(cache, 0o644)
with open(cache, "w+") as fobj:
tests/func/test_data_cloud.py
+11 -13
View File
@@ -581,10 +581,8 @@ class TestRecursiveSyncOperations(Local, TestDataCloudBase):
self._clear_local_cache()
local_cache = self.dvc.cache.local
local_cache_data_path = local_cache.checksum_to_path_info(data_md5)
local_cache_data_sub_path = local_cache.checksum_to_path_info(
data_sub_md5
)
local_cache_data_path = local_cache.hash_to_path_info(data_md5)
local_cache_data_sub_path = local_cache.hash_to_path_info(data_sub_md5)
self.assertFalse(os.path.exists(local_cache_data_path))
self.assertFalse(os.path.exists(local_cache_data_sub_path))
@@ -596,8 +594,8 @@ class TestRecursiveSyncOperations(Local, TestDataCloudBase):
def _test_recursive_push(self, data_md5, data_sub_md5):
remote = self.cloud.get_remote()
cloud_data_path = remote.checksum_to_path_info(data_md5)
cloud_data_sub_path = remote.checksum_to_path_info(data_sub_md5)
cloud_data_path = remote.hash_to_path_info(data_md5)
cloud_data_sub_path = remote.hash_to_path_info(data_sub_md5)
self.assertFalse(os.path.exists(cloud_data_path))
self.assertFalse(os.path.exists(cloud_data_sub_path))
@@ -621,9 +619,9 @@ class TestRecursiveSyncOperations(Local, TestDataCloudBase):
self._test_recursive_pull()
def test_checksum_recalculation(mocker, dvc, tmp_dir):
def test_hash_recalculation(mocker, dvc, tmp_dir):
tmp_dir.gen({"foo": "foo"})
test_get_file_checksum = mocker.spy(LocalRemoteTree, "get_file_checksum")
test_get_file_hash = mocker.spy(LocalRemoteTree, "get_file_hash")
url = Local.get_url()
ret = main(["remote", "add", "-d", TEST_REMOTE, url])
assert ret == 0
@@ -635,7 +633,7 @@ def test_checksum_recalculation(mocker, dvc, tmp_dir):
assert ret == 0
ret = main(["run", "--single-stage", "-d", "foo", "echo foo"])
assert ret == 0
assert test_get_file_checksum.mock.call_count == 1
assert test_get_file_hash.mock.call_count == 1
class TestShouldWarnOnNoChecksumInLocalAndRemoteCache(TestDvc):
@@ -688,7 +686,7 @@ class TestShouldWarnOnNoChecksumInLocalAndRemoteCache(TestDvc):
assert self.message_bar_part in self._caplog.text
def test_verify_checksums(
def test_verify_hashes(
tmp_dir, scm, dvc, mocker, tmp_path_factory, setup_remote
):
@@ -702,10 +700,10 @@ def test_verify_checksums(
remove("dir")
remove(dvc.cache.local.cache_dir)
checksum_spy = mocker.spy(dvc.cache.local.tree, "get_file_checksum")
hash_spy = mocker.spy(dvc.cache.local.tree, "get_file_hash")
dvc.pull()
assert checksum_spy.call_count == 0
assert hash_spy.call_count == 0
# Removing cache will invalidate existing state entries
remove(dvc.cache.local.cache_dir)
@@ -713,7 +711,7 @@ def test_verify_checksums(
dvc.config["remote"]["upstream"]["verify"] = True
dvc.pull()
assert checksum_spy.call_count == 3
assert hash_spy.call_count == 3
@flaky(max_runs=3, min_passes=1)
tests/func/test_gc.py
+1 -1
View File
@@ -21,7 +21,7 @@ class TestGC(TestDvcGit):
self.dvc.add(self.FOO)
self.dvc.add(self.DATA_DIR)
self.good_cache = [
self.dvc.cache.local.checksum_to_path_info(md5)
self.dvc.cache.local.hash_to_path_info(md5)
for md5 in self.dvc.cache.local.tree.all()
]
tests/func/test_remote.py
+8 -12
View File
@@ -145,7 +145,7 @@ class TestRemoteShouldHandleUppercaseRemoteName(TestDvc):
self.assertEqual(ret, 0)
def test_dir_checksum_should_be_key_order_agnostic(tmp_dir, dvc):
def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
tmp_dir.gen({"data": {"1": "1 content", "2": "2 content"}})
path_info = PathInfo("data")
@@ -158,7 +158,7 @@ def test_dir_checksum_should_be_key_order_agnostic(tmp_dir, dvc):
{"relpath": "2", "md5": "2"},
],
):
checksum1 = dvc.cache.local.get_checksum(path_info)
hash1 = dvc.cache.local.get_hash(path_info)
with patch.object(
BaseRemoteTree,
@@ -168,9 +168,9 @@ def test_dir_checksum_should_be_key_order_agnostic(tmp_dir, dvc):
{"md5": "2", "relpath": "2"},
],
):
checksum2 = dvc.cache.local.get_checksum(path_info)
hash2 = dvc.cache.local.get_hash(path_info)
assert checksum1 == checksum2
assert hash1 == hash2
def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, setup_remote):
@@ -194,13 +194,9 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, setup_remote):
assert upload_error_info.value.amount == 3
remote = dvc.cloud.get_remote("upstream")
assert not remote.tree.exists(
remote.checksum_to_path_info(foo.checksum)
)
assert remote.tree.exists(remote.checksum_to_path_info(bar.checksum))
assert not remote.tree.exists(
remote.checksum_to_path_info(baz.checksum)
)
assert not remote.tree.exists(remote.hash_to_path_info(foo.checksum))
assert remote.tree.exists(remote.hash_to_path_info(bar.checksum))
assert not remote.tree.exists(remote.hash_to_path_info(baz.checksum))
# Push everything and delete local cache
dvc.push()
@@ -397,7 +393,7 @@ def test_protect_local_remote(tmp_dir, dvc, setup_remote):
dvc.push()
remote = dvc.cloud.get_remote("upstream")
remote_cache_file = remote.checksum_to_path_info(stage.outs[0].checksum)
remote_cache_file = remote.hash_to_path_info(stage.outs[0].checksum)
assert os.path.exists(remote_cache_file)
assert stat.S_IMODE(os.stat(remote_cache_file).st_mode) == 0o444
tests/func/test_tree.py
+5 -5
View File
@@ -192,8 +192,8 @@ def test_repotree_walk_fetch(tmp_dir, dvc, scm, setup_remote):
assert os.path.exists(out.cache_path)
for entry in out.dir_cache:
checksum = entry[out.remote.tree.PARAM_CHECKSUM]
assert os.path.exists(dvc.cache.local.checksum_to_path_info(checksum))
hash_ = entry[out.remote.tree.PARAM_CHECKSUM]
assert os.path.exists(dvc.cache.local.hash_to_path_info(hash_))
def test_repotree_cache_save(tmp_dir, dvc, scm, erepo_dir, setup_remote):
@@ -211,7 +211,7 @@ def test_repotree_cache_save(tmp_dir, dvc, scm, erepo_dir, setup_remote):
# into dvc.cache, not fetched or streamed from a remote
tree = RepoTree(erepo_dir.dvc, stream=True)
expected = [
tree.get_file_checksum(erepo_dir / path)
tree.get_file_hash(erepo_dir / path)
for path in ("dir/bar", "dir/subdir/foo")
]
@@ -219,8 +219,8 @@ def test_repotree_cache_save(tmp_dir, dvc, scm, erepo_dir, setup_remote):
cache = dvc.cache.local
with cache.state:
cache.save(PathInfo(erepo_dir / "dir"), tree, None)
for checksum in expected:
assert os.path.exists(cache.checksum_to_path_info(checksum))
for hash_ in expected:
assert os.path.exists(cache.hash_to_path_info(hash_))
def test_cleantree_subrepo(tmp_dir, dvc, scm, monkeypatch):
tests/unit/remote/test_azure.py
+5 -5
View File
@@ -32,7 +32,7 @@ def test_init(dvc):
assert tree.connection_string == connection_string
def test_get_file_checksum(tmp_dir):
def test_get_file_hash(tmp_dir):
if not Azure.should_test():
pytest.skip("no azurite running")
@@ -42,7 +42,7 @@ def test_get_file_checksum(tmp_dir):
to_info = tree.PATH_CLS(Azure.get_url())
tree.upload(PathInfo("foo"), to_info)
assert tree.exists(to_info)
checksum = tree.get_file_checksum(to_info)
assert checksum
assert isinstance(checksum, str)
assert checksum.strip("'").strip('"') == checksum
hash_ = tree.get_file_hash(to_info)
assert hash_
assert isinstance(hash_, str)
assert hash_.strip("'").strip('"') == hash_
tests/unit/remote/test_base.py
+33 -33
View File
@@ -45,19 +45,19 @@ def test_cmd_error(dvc):
BaseRemoteTree(dvc, config).remove("file")
@mock.patch.object(BaseRemoteTree, "list_checksums_traverse")
@mock.patch.object(BaseRemoteTree, "list_checksums_exists")
def test_checksums_exist(object_exists, traverse, dvc):
@mock.patch.object(BaseRemoteTree, "list_hashes_traverse")
@mock.patch.object(BaseRemoteTree, "list_hashes_exists")
def test_hashes_exist(object_exists, traverse, dvc):
remote = Remote(BaseRemoteTree(dvc, {}))
# remote does not support traverse
remote.tree.CAN_TRAVERSE = False
with mock.patch.object(
remote.tree, "list_checksums", return_value=list(range(256))
remote.tree, "list_hashes", return_value=list(range(256))
):
checksums = set(range(1000))
remote.checksums_exist(checksums)
object_exists.assert_called_with(checksums, None, None)
hashes = set(range(1000))
remote.hashes_exist(hashes)
object_exists.assert_called_with(hashes, None, None)
traverse.assert_not_called()
remote.tree.CAN_TRAVERSE = True
@@ -66,19 +66,19 @@ def test_checksums_exist(object_exists, traverse, dvc):
object_exists.reset_mock()
traverse.reset_mock()
with mock.patch.object(
remote.tree, "list_checksums", return_value=list(range(256))
remote.tree, "list_hashes", return_value=list(range(256))
):
checksums = list(range(1000))
remote.checksums_exist(checksums)
hashes = list(range(1000))
remote.hashes_exist(hashes)
# verify that _cache_paths_with_max() short circuits
# before returning all 256 remote checksums
max_checksums = math.ceil(
remote.tree._max_estimation_size(checksums)
# before returning all 256 remote hashes
max_hashes = math.ceil(
remote.tree._max_estimation_size(hashes)
/ pow(16, remote.tree.TRAVERSE_PREFIX_LEN)
)
assert max_checksums < 256
assert max_hashes < 256
object_exists.assert_called_with(
frozenset(range(max_checksums, 1000)), None, None
frozenset(range(max_hashes, 1000)), None, None
)
traverse.assert_not_called()
@@ -87,10 +87,10 @@ def test_checksums_exist(object_exists, traverse, dvc):
traverse.reset_mock()
remote.tree.JOBS = 16
with mock.patch.object(
remote.tree, "list_checksums", return_value=list(range(256))
remote.tree, "list_hashes", return_value=list(range(256))
):
checksums = list(range(1000000))
remote.checksums_exist(checksums)
hashes = list(range(1000000))
remote.hashes_exist(hashes)
object_exists.assert_not_called()
traverse.assert_called_with(
256 * pow(16, remote.tree.TRAVERSE_PREFIX_LEN),
@@ -101,50 +101,50 @@ def test_checksums_exist(object_exists, traverse, dvc):
@mock.patch.object(
BaseRemoteTree, "list_checksums", return_value=[],
BaseRemoteTree, "list_hashes", return_value=[],
)
@mock.patch.object(
BaseRemoteTree, "path_to_checksum", side_effect=lambda x: x,
BaseRemoteTree, "path_to_hash", side_effect=lambda x: x,
)
def test_list_checksums_traverse(path_to_checksum, list_checksums, dvc):
def test_list_hashes_traverse(path_to_hash, list_hashes, dvc):
tree = BaseRemoteTree(dvc, {})
tree.path_info = PathInfo("foo")
# parallel traverse
size = 256 / tree.JOBS * tree.LIST_OBJECT_PAGE_SIZE
list(tree.list_checksums_traverse(size, {0}))
list(tree.list_hashes_traverse(size, {0}))
for i in range(1, 16):
list_checksums.assert_any_call(
list_hashes.assert_any_call(
prefix=f"{i:03x}", progress_callback=CallableOrNone
)
for i in range(1, 256):
list_checksums.assert_any_call(
list_hashes.assert_any_call(
prefix=f"{i:02x}", progress_callback=CallableOrNone
)
# default traverse (small remote)
size -= 1
list_checksums.reset_mock()
list(tree.list_checksums_traverse(size - 1, {0}))
list_checksums.assert_called_with(
list_hashes.reset_mock()
list(tree.list_hashes_traverse(size - 1, {0}))
list_hashes.assert_called_with(
prefix=None, progress_callback=CallableOrNone
)
def test_list_checksums(dvc):
def test_list_hashes(dvc):
tree = BaseRemoteTree(dvc, {})
tree.path_info = PathInfo("foo")
with mock.patch.object(
tree, "list_paths", return_value=["12/3456", "bar"]
):
checksums = list(tree.list_checksums())
assert checksums == ["123456"]
hashes = list(tree.list_hashes())
assert hashes == ["123456"]
@pytest.mark.parametrize(
"checksum, result",
"hash_, result",
[(None, False), ("", False), ("3456.dir", True), ("3456", False)],
)
def test_is_dir_checksum(checksum, result):
assert BaseRemoteTree.is_dir_checksum(checksum) == result
def test_is_dir_hash(hash_, result):
assert BaseRemoteTree.is_dir_hash(hash_) == result
tests/unit/remote/test_index.py
+10 -10
View File
@@ -19,9 +19,9 @@ def test_init(dvc, index):
assert str(index.path) == os.path.join(dvc.index_dir, "foo.idx")
def test_is_dir_checksum(dvc, index):
assert index.is_dir_checksum("foo.dir")
assert not index.is_dir_checksum("foo")
def test_is_dir_hash(dvc, index):
assert index.is_dir_hash("foo.dir")
assert not index.is_dir_hash("foo")
def test_roundtrip(dvc, index):
@@ -30,8 +30,8 @@ def test_roundtrip(dvc, index):
index.update(expected_dir, expected_file)
index.dump()
index.load()
assert set(index.dir_checksums()) == expected_dir
assert set(index.checksums()) == expected_dir | expected_file
assert set(index.dir_hashes()) == expected_dir
assert set(index.hashes()) == expected_dir | expected_file
def test_clear(dvc, index):
@@ -39,19 +39,19 @@ def test_clear(dvc, index):
["1234.dir"], ["5678"],
)
index.clear()
assert first(index.checksums()) is None
assert first(index.hashes()) is None
def test_update(dvc, index):
expected_dir = {"1234.dir"}
expected_file = {"5678"}
index.update(expected_dir, expected_file)
assert set(index.dir_checksums()) == expected_dir
assert set(index.checksums()) == expected_dir | expected_file
assert set(index.dir_hashes()) == expected_dir
assert set(index.hashes()) == expected_dir | expected_file
def test_intersection(dvc, index):
checksums = (str(i) for i in range(2000))
hashes = (str(i) for i in range(2000))
expected = {str(i) for i in range(1000)}
index.update([], checksums)
index.update([], hashes)
assert set(index.intersection(expected)) == expected
tests/unit/remote/test_local.py
+3 -3
View File
@@ -21,16 +21,16 @@ def test_status_download_optimization(mocker, dvc):
infos.add("local", "37b51d194a7513e45b56f6524f2d51f2", "bar")
local_exists = list(infos["local"])
mocker.patch.object(cache, "checksums_exist", return_value=local_exists)
mocker.patch.object(cache, "hashes_exist", return_value=local_exists)
other_remote = mocker.Mock()
other_remote.url = "other_remote"
other_remote.checksums_exist.return_value = []
other_remote.hashes_exist.return_value = []
other_remote.index = RemoteIndexNoop()
cache.status(infos, other_remote, download=True)
assert other_remote.checksums_exist.call_count == 0
assert other_remote.hashes_exist.call_count == 0
@pytest.mark.parametrize("link_name", ["hardlink", "symlink"])
tests/unit/remote/test_remote.py
+15 -15
View File
@@ -5,30 +5,30 @@ from dvc.remote.gs import GSRemoteTree
from dvc.remote.s3 import S3RemoteTree
def test_remote_with_checksum_jobs(dvc):
dvc.config["remote"]["with_checksum_jobs"] = {
def test_remote_with_hash_jobs(dvc):
dvc.config["remote"]["with_hash_jobs"] = {
"url": "s3://bucket/name",
"checksum_jobs": 100,
"hash_jobs": 100,
}
dvc.config["core"]["checksum_jobs"] = 200
dvc.config["core"]["hash_jobs"] = 200
tree = get_cloud_tree(dvc, name="with_checksum_jobs")
assert tree.checksum_jobs == 100
tree = get_cloud_tree(dvc, name="with_hash_jobs")
assert tree.hash_jobs == 100
def test_remote_without_checksum_jobs(dvc):
dvc.config["remote"]["without_checksum_jobs"] = {"url": "s3://bucket/name"}
dvc.config["core"]["checksum_jobs"] = 200
def test_remote_without_hash_jobs(dvc):
dvc.config["remote"]["without_hash_jobs"] = {"url": "s3://bucket/name"}
dvc.config["core"]["hash_jobs"] = 200
tree = get_cloud_tree(dvc, name="without_checksum_jobs")
assert tree.checksum_jobs == 200
tree = get_cloud_tree(dvc, name="without_hash_jobs")
assert tree.hash_jobs == 200
def test_remote_without_checksum_jobs_default(dvc):
dvc.config["remote"]["without_checksum_jobs"] = {"url": "s3://bucket/name"}
def test_remote_without_hash_jobs_default(dvc):
dvc.config["remote"]["without_hash_jobs"] = {"url": "s3://bucket/name"}
tree = get_cloud_tree(dvc, name="without_checksum_jobs")
assert tree.checksum_jobs == tree.CHECKSUM_JOBS
tree = get_cloud_tree(dvc, name="without_hash_jobs")
assert tree.hash_jobs == tree.HASH_JOBS
@pytest.mark.parametrize("tree_cls", [GSRemoteTree, S3RemoteTree])