add cache_dir to progress output

This commit is contained in:
Casper da Costa-Luis
2019-08-28 17:44:12 +01:00
parent 5413576e27
commit 5f81352f90
3 changed files with 28 additions and 13 deletions
+14 -3
View File
@@ -140,6 +140,14 @@ class RemoteBASE(object):
def cache(self): def cache(self):
return getattr(self.repo.cache, self.scheme) return getattr(self.repo.cache, self.scheme)
@property
def cache_dir(self):
return (
getattr(self.path_info, "fspath", self.path_info.url)
if self.path_info
else None
)
def get_file_checksum(self, path_info): def get_file_checksum(self, path_info):
raise NotImplementedError raise NotImplementedError
@@ -612,7 +620,7 @@ class RemoteBASE(object):
return self._changed_dir_cache(checksum) return self._changed_dir_cache(checksum)
return self.changed_cache_file(checksum) return self.changed_cache_file(checksum)
def cache_exists(self, checksums, jobs=None): def cache_exists(self, checksums, jobs=None, name=None):
"""Check if the given checksums are stored in the remote. """Check if the given checksums are stored in the remote.
There are two ways of performing this check: There are two ways of performing this check:
@@ -630,7 +638,7 @@ class RemoteBASE(object):
take much shorter time to just retrieve everything they have under take much shorter time to just retrieve everything they have under
a certain prefix (e.g. s3, gs, ssh, hdfs). Other remotes that can a certain prefix (e.g. s3, gs, ssh, hdfs). Other remotes that can
check if particular file exists much quicker, use their own check if particular file exists much quicker, use their own
implementation of cache_exists (see http, local). implementation of cache_exists (see ssh, local).
Returns: Returns:
A list with checksums that were found in the remote A list with checksums that were found in the remote
@@ -639,7 +647,10 @@ class RemoteBASE(object):
return list(set(checksums) & set(self.all())) return list(set(checksums) & set(self.all()))
with Tqdm( with Tqdm(
desc="Querying remote cache", total=len(checksums), unit="md5" desc="Querying "
+ ("cache in " + name if name else "remote cache"),
total=len(checksums),
unit="md5",
) as pbar: ) as pbar:
def exists_with_progress(path_info): def exists_with_progress(path_info):
+9 -8
View File
@@ -97,10 +97,6 @@ class RemoteLOCAL(RemoteBASE):
def state(self): def state(self):
return self.repo.state return self.repo.state
@property
def cache_dir(self):
return self.path_info.fspath if self.path_info else None
@classmethod @classmethod
def supported(cls, config): def supported(cls, config):
return True return True
@@ -251,11 +247,14 @@ class RemoteLOCAL(RemoteBASE):
move(from_info, to_info, mode=mode) move(from_info, to_info, mode=mode)
def cache_exists(self, checksums, jobs=None): def cache_exists(self, checksums, jobs=None, name=None):
return [ return [
checksum checksum
for checksum in Tqdm( for checksum in Tqdm(
checksums, unit="md5", desc="Querying local cache" checksums,
unit="md5",
desc="Querying "
+ ("cache in " + name if name else "local cache"),
) )
if not self.changed_cache_file(checksum) if not self.changed_cache_file(checksum)
] ]
@@ -322,7 +321,7 @@ class RemoteLOCAL(RemoteBASE):
md5s = list(ret) md5s = list(ret)
logger.debug("Collecting information from local cache...") logger.debug("Collecting information from local cache...")
local_exists = self.cache_exists(md5s, jobs=jobs) local_exists = self.cache_exists(md5s, jobs=jobs, name=self.cache_dir)
# This is a performance optimization. We can safely assume that, # This is a performance optimization. We can safely assume that,
# if the resources that we want to fetch are already cached, # if the resources that we want to fetch are already cached,
@@ -332,7 +331,9 @@ class RemoteLOCAL(RemoteBASE):
remote_exists = local_exists remote_exists = local_exists
else: else:
logger.debug("Collecting information from remote cache...") logger.debug("Collecting information from remote cache...")
remote_exists = list(remote.cache_exists(md5s, jobs=jobs)) remote_exists = list(
remote.cache_exists(md5s, jobs=jobs, name=remote.cache_dir)
)
self._fill_statuses(ret, local_exists, remote_exists) self._fill_statuses(ret, local_exists, remote_exists)
+5 -2
View File
@@ -258,7 +258,7 @@ class RemoteSSH(RemoteBASE):
return results return results
def cache_exists(self, checksums, jobs=None): def cache_exists(self, checksums, jobs=None, name=None):
"""This is older implementation used in remote/base.py """This is older implementation used in remote/base.py
We are reusing it in RemoteSSH, because SSH's batch_exists proved to be We are reusing it in RemoteSSH, because SSH's batch_exists proved to be
faster than current approach (relying on exists(path_info)) applied in faster than current approach (relying on exists(path_info)) applied in
@@ -268,7 +268,10 @@ class RemoteSSH(RemoteBASE):
return list(set(checksums) & set(self.all())) return list(set(checksums) & set(self.all()))
with Tqdm( with Tqdm(
desc="Querying remote cache", total=len(checksums), unit="md5" desc="Querying "
+ ("cache in " + name if name else "remote cache"),
total=len(checksums),
unit="md5",
) as pbar: ) as pbar:
def exists_with_progress(chunks): def exists_with_progress(chunks):