mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
add cache_dir to progress output
This commit is contained in:
+14
-3
@@ -140,6 +140,14 @@ class RemoteBASE(object):
|
||||
def cache(self):
|
||||
return getattr(self.repo.cache, self.scheme)
|
||||
|
||||
@property
|
||||
def cache_dir(self):
|
||||
return (
|
||||
getattr(self.path_info, "fspath", self.path_info.url)
|
||||
if self.path_info
|
||||
else None
|
||||
)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -612,7 +620,7 @@ class RemoteBASE(object):
|
||||
return self._changed_dir_cache(checksum)
|
||||
return self.changed_cache_file(checksum)
|
||||
|
||||
def cache_exists(self, checksums, jobs=None):
|
||||
def cache_exists(self, checksums, jobs=None, name=None):
|
||||
"""Check if the given checksums are stored in the remote.
|
||||
|
||||
There are two ways of performing this check:
|
||||
@@ -630,7 +638,7 @@ class RemoteBASE(object):
|
||||
take much shorter time to just retrieve everything they have under
|
||||
a certain prefix (e.g. s3, gs, ssh, hdfs). Other remotes that can
|
||||
check if particular file exists much quicker, use their own
|
||||
implementation of cache_exists (see http, local).
|
||||
implementation of cache_exists (see ssh, local).
|
||||
|
||||
Returns:
|
||||
A list with checksums that were found in the remote
|
||||
@@ -639,7 +647,10 @@ class RemoteBASE(object):
|
||||
return list(set(checksums) & set(self.all()))
|
||||
|
||||
with Tqdm(
|
||||
desc="Querying remote cache", total=len(checksums), unit="md5"
|
||||
desc="Querying "
|
||||
+ ("cache in " + name if name else "remote cache"),
|
||||
total=len(checksums),
|
||||
unit="md5",
|
||||
) as pbar:
|
||||
|
||||
def exists_with_progress(path_info):
|
||||
|
||||
@@ -97,10 +97,6 @@ class RemoteLOCAL(RemoteBASE):
|
||||
def state(self):
|
||||
return self.repo.state
|
||||
|
||||
@property
|
||||
def cache_dir(self):
|
||||
return self.path_info.fspath if self.path_info else None
|
||||
|
||||
@classmethod
|
||||
def supported(cls, config):
|
||||
return True
|
||||
@@ -251,11 +247,14 @@ class RemoteLOCAL(RemoteBASE):
|
||||
|
||||
move(from_info, to_info, mode=mode)
|
||||
|
||||
def cache_exists(self, checksums, jobs=None):
|
||||
def cache_exists(self, checksums, jobs=None, name=None):
|
||||
return [
|
||||
checksum
|
||||
for checksum in Tqdm(
|
||||
checksums, unit="md5", desc="Querying local cache"
|
||||
checksums,
|
||||
unit="md5",
|
||||
desc="Querying "
|
||||
+ ("cache in " + name if name else "local cache"),
|
||||
)
|
||||
if not self.changed_cache_file(checksum)
|
||||
]
|
||||
@@ -322,7 +321,7 @@ class RemoteLOCAL(RemoteBASE):
|
||||
md5s = list(ret)
|
||||
|
||||
logger.debug("Collecting information from local cache...")
|
||||
local_exists = self.cache_exists(md5s, jobs=jobs)
|
||||
local_exists = self.cache_exists(md5s, jobs=jobs, name=self.cache_dir)
|
||||
|
||||
# This is a performance optimization. We can safely assume that,
|
||||
# if the resources that we want to fetch are already cached,
|
||||
@@ -332,7 +331,9 @@ class RemoteLOCAL(RemoteBASE):
|
||||
remote_exists = local_exists
|
||||
else:
|
||||
logger.debug("Collecting information from remote cache...")
|
||||
remote_exists = list(remote.cache_exists(md5s, jobs=jobs))
|
||||
remote_exists = list(
|
||||
remote.cache_exists(md5s, jobs=jobs, name=remote.cache_dir)
|
||||
)
|
||||
|
||||
self._fill_statuses(ret, local_exists, remote_exists)
|
||||
|
||||
|
||||
@@ -258,7 +258,7 @@ class RemoteSSH(RemoteBASE):
|
||||
|
||||
return results
|
||||
|
||||
def cache_exists(self, checksums, jobs=None):
|
||||
def cache_exists(self, checksums, jobs=None, name=None):
|
||||
"""This is older implementation used in remote/base.py
|
||||
We are reusing it in RemoteSSH, because SSH's batch_exists proved to be
|
||||
faster than current approach (relying on exists(path_info)) applied in
|
||||
@@ -268,7 +268,10 @@ class RemoteSSH(RemoteBASE):
|
||||
return list(set(checksums) & set(self.all()))
|
||||
|
||||
with Tqdm(
|
||||
desc="Querying remote cache", total=len(checksums), unit="md5"
|
||||
desc="Querying "
|
||||
+ ("cache in " + name if name else "remote cache"),
|
||||
total=len(checksums),
|
||||
unit="md5",
|
||||
) as pbar:
|
||||
|
||||
def exists_with_progress(chunks):
|
||||
|
||||
Reference in New Issue
Block a user