mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-10 23:10:42 +01:00
add cache_dir to progress output
This commit is contained in:
+14
-3
@@ -140,6 +140,14 @@ class RemoteBASE(object):
|
|||||||
def cache(self):
|
def cache(self):
|
||||||
return getattr(self.repo.cache, self.scheme)
|
return getattr(self.repo.cache, self.scheme)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def cache_dir(self):
|
||||||
|
return (
|
||||||
|
getattr(self.path_info, "fspath", self.path_info.url)
|
||||||
|
if self.path_info
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
|
||||||
def get_file_checksum(self, path_info):
|
def get_file_checksum(self, path_info):
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
@@ -612,7 +620,7 @@ class RemoteBASE(object):
|
|||||||
return self._changed_dir_cache(checksum)
|
return self._changed_dir_cache(checksum)
|
||||||
return self.changed_cache_file(checksum)
|
return self.changed_cache_file(checksum)
|
||||||
|
|
||||||
def cache_exists(self, checksums, jobs=None):
|
def cache_exists(self, checksums, jobs=None, name=None):
|
||||||
"""Check if the given checksums are stored in the remote.
|
"""Check if the given checksums are stored in the remote.
|
||||||
|
|
||||||
There are two ways of performing this check:
|
There are two ways of performing this check:
|
||||||
@@ -630,7 +638,7 @@ class RemoteBASE(object):
|
|||||||
take much shorter time to just retrieve everything they have under
|
take much shorter time to just retrieve everything they have under
|
||||||
a certain prefix (e.g. s3, gs, ssh, hdfs). Other remotes that can
|
a certain prefix (e.g. s3, gs, ssh, hdfs). Other remotes that can
|
||||||
check if particular file exists much quicker, use their own
|
check if particular file exists much quicker, use their own
|
||||||
implementation of cache_exists (see http, local).
|
implementation of cache_exists (see ssh, local).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
A list with checksums that were found in the remote
|
A list with checksums that were found in the remote
|
||||||
@@ -639,7 +647,10 @@ class RemoteBASE(object):
|
|||||||
return list(set(checksums) & set(self.all()))
|
return list(set(checksums) & set(self.all()))
|
||||||
|
|
||||||
with Tqdm(
|
with Tqdm(
|
||||||
desc="Querying remote cache", total=len(checksums), unit="md5"
|
desc="Querying "
|
||||||
|
+ ("cache in " + name if name else "remote cache"),
|
||||||
|
total=len(checksums),
|
||||||
|
unit="md5",
|
||||||
) as pbar:
|
) as pbar:
|
||||||
|
|
||||||
def exists_with_progress(path_info):
|
def exists_with_progress(path_info):
|
||||||
|
|||||||
@@ -97,10 +97,6 @@ class RemoteLOCAL(RemoteBASE):
|
|||||||
def state(self):
|
def state(self):
|
||||||
return self.repo.state
|
return self.repo.state
|
||||||
|
|
||||||
@property
|
|
||||||
def cache_dir(self):
|
|
||||||
return self.path_info.fspath if self.path_info else None
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def supported(cls, config):
|
def supported(cls, config):
|
||||||
return True
|
return True
|
||||||
@@ -251,11 +247,14 @@ class RemoteLOCAL(RemoteBASE):
|
|||||||
|
|
||||||
move(from_info, to_info, mode=mode)
|
move(from_info, to_info, mode=mode)
|
||||||
|
|
||||||
def cache_exists(self, checksums, jobs=None):
|
def cache_exists(self, checksums, jobs=None, name=None):
|
||||||
return [
|
return [
|
||||||
checksum
|
checksum
|
||||||
for checksum in Tqdm(
|
for checksum in Tqdm(
|
||||||
checksums, unit="md5", desc="Querying local cache"
|
checksums,
|
||||||
|
unit="md5",
|
||||||
|
desc="Querying "
|
||||||
|
+ ("cache in " + name if name else "local cache"),
|
||||||
)
|
)
|
||||||
if not self.changed_cache_file(checksum)
|
if not self.changed_cache_file(checksum)
|
||||||
]
|
]
|
||||||
@@ -322,7 +321,7 @@ class RemoteLOCAL(RemoteBASE):
|
|||||||
md5s = list(ret)
|
md5s = list(ret)
|
||||||
|
|
||||||
logger.debug("Collecting information from local cache...")
|
logger.debug("Collecting information from local cache...")
|
||||||
local_exists = self.cache_exists(md5s, jobs=jobs)
|
local_exists = self.cache_exists(md5s, jobs=jobs, name=self.cache_dir)
|
||||||
|
|
||||||
# This is a performance optimization. We can safely assume that,
|
# This is a performance optimization. We can safely assume that,
|
||||||
# if the resources that we want to fetch are already cached,
|
# if the resources that we want to fetch are already cached,
|
||||||
@@ -332,7 +331,9 @@ class RemoteLOCAL(RemoteBASE):
|
|||||||
remote_exists = local_exists
|
remote_exists = local_exists
|
||||||
else:
|
else:
|
||||||
logger.debug("Collecting information from remote cache...")
|
logger.debug("Collecting information from remote cache...")
|
||||||
remote_exists = list(remote.cache_exists(md5s, jobs=jobs))
|
remote_exists = list(
|
||||||
|
remote.cache_exists(md5s, jobs=jobs, name=remote.cache_dir)
|
||||||
|
)
|
||||||
|
|
||||||
self._fill_statuses(ret, local_exists, remote_exists)
|
self._fill_statuses(ret, local_exists, remote_exists)
|
||||||
|
|
||||||
|
|||||||
@@ -258,7 +258,7 @@ class RemoteSSH(RemoteBASE):
|
|||||||
|
|
||||||
return results
|
return results
|
||||||
|
|
||||||
def cache_exists(self, checksums, jobs=None):
|
def cache_exists(self, checksums, jobs=None, name=None):
|
||||||
"""This is older implementation used in remote/base.py
|
"""This is older implementation used in remote/base.py
|
||||||
We are reusing it in RemoteSSH, because SSH's batch_exists proved to be
|
We are reusing it in RemoteSSH, because SSH's batch_exists proved to be
|
||||||
faster than current approach (relying on exists(path_info)) applied in
|
faster than current approach (relying on exists(path_info)) applied in
|
||||||
@@ -268,7 +268,10 @@ class RemoteSSH(RemoteBASE):
|
|||||||
return list(set(checksums) & set(self.all()))
|
return list(set(checksums) & set(self.all()))
|
||||||
|
|
||||||
with Tqdm(
|
with Tqdm(
|
||||||
desc="Querying remote cache", total=len(checksums), unit="md5"
|
desc="Querying "
|
||||||
|
+ ("cache in " + name if name else "remote cache"),
|
||||||
|
total=len(checksums),
|
||||||
|
unit="md5",
|
||||||
) as pbar:
|
) as pbar:
|
||||||
|
|
||||||
def exists_with_progress(chunks):
|
def exists_with_progress(chunks):
|
||||||
|
|||||||
Reference in New Issue
Block a user