add cache_dir to progress output

This commit is contained in:
Casper da Costa-Luis
2019-08-28 17:44:12 +01:00
parent 5413576e27
commit 5f81352f90
3 changed files with 28 additions and 13 deletions
+14 -3
View File
@@ -140,6 +140,14 @@ class RemoteBASE(object):
def cache(self):
return getattr(self.repo.cache, self.scheme)
@property
def cache_dir(self):
return (
getattr(self.path_info, "fspath", self.path_info.url)
if self.path_info
else None
)
def get_file_checksum(self, path_info):
raise NotImplementedError
@@ -612,7 +620,7 @@ class RemoteBASE(object):
return self._changed_dir_cache(checksum)
return self.changed_cache_file(checksum)
def cache_exists(self, checksums, jobs=None):
def cache_exists(self, checksums, jobs=None, name=None):
"""Check if the given checksums are stored in the remote.
There are two ways of performing this check:
@@ -630,7 +638,7 @@ class RemoteBASE(object):
take much shorter time to just retrieve everything they have under
a certain prefix (e.g. s3, gs, ssh, hdfs). Other remotes that can
check if particular file exists much quicker, use their own
implementation of cache_exists (see http, local).
implementation of cache_exists (see ssh, local).
Returns:
A list with checksums that were found in the remote
@@ -639,7 +647,10 @@ class RemoteBASE(object):
return list(set(checksums) & set(self.all()))
with Tqdm(
desc="Querying remote cache", total=len(checksums), unit="md5"
desc="Querying "
+ ("cache in " + name if name else "remote cache"),
total=len(checksums),
unit="md5",
) as pbar:
def exists_with_progress(path_info):
+9 -8
View File
@@ -97,10 +97,6 @@ class RemoteLOCAL(RemoteBASE):
def state(self):
return self.repo.state
@property
def cache_dir(self):
return self.path_info.fspath if self.path_info else None
@classmethod
def supported(cls, config):
return True
@@ -251,11 +247,14 @@ class RemoteLOCAL(RemoteBASE):
move(from_info, to_info, mode=mode)
def cache_exists(self, checksums, jobs=None):
def cache_exists(self, checksums, jobs=None, name=None):
return [
checksum
for checksum in Tqdm(
checksums, unit="md5", desc="Querying local cache"
checksums,
unit="md5",
desc="Querying "
+ ("cache in " + name if name else "local cache"),
)
if not self.changed_cache_file(checksum)
]
@@ -322,7 +321,7 @@ class RemoteLOCAL(RemoteBASE):
md5s = list(ret)
logger.debug("Collecting information from local cache...")
local_exists = self.cache_exists(md5s, jobs=jobs)
local_exists = self.cache_exists(md5s, jobs=jobs, name=self.cache_dir)
# This is a performance optimization. We can safely assume that,
# if the resources that we want to fetch are already cached,
@@ -332,7 +331,9 @@ class RemoteLOCAL(RemoteBASE):
remote_exists = local_exists
else:
logger.debug("Collecting information from remote cache...")
remote_exists = list(remote.cache_exists(md5s, jobs=jobs))
remote_exists = list(
remote.cache_exists(md5s, jobs=jobs, name=remote.cache_dir)
)
self._fill_statuses(ret, local_exists, remote_exists)
+5 -2
View File
@@ -258,7 +258,7 @@ class RemoteSSH(RemoteBASE):
return results
def cache_exists(self, checksums, jobs=None):
def cache_exists(self, checksums, jobs=None, name=None):
"""This is older implementation used in remote/base.py
We are reusing it in RemoteSSH, because SSH's batch_exists proved to be
faster than current approach (relying on exists(path_info)) applied in
@@ -268,7 +268,10 @@ class RemoteSSH(RemoteBASE):
return list(set(checksums) & set(self.all()))
with Tqdm(
desc="Querying remote cache", total=len(checksums), unit="md5"
desc="Querying "
+ ("cache in " + name if name else "remote cache"),
total=len(checksums),
unit="md5",
) as pbar:
def exists_with_progress(chunks):