remote: honor --jobs for status collection

This commit is contained in:
Alexander Schepanovski
2019-07-21 12:50:35 +07:00
parent b6bc65fcbf
commit 624ccfc744
2 changed files with 5 additions and 5 deletions
+2 -2
View File
@@ -589,7 +589,7 @@ class RemoteBASE(object):
return self._changed_dir_cache(checksum)
return self.changed_cache_file(checksum)
def cache_exists(self, checksums):
def cache_exists(self, checksums, jobs=None):
"""Check if the given checksums are stored in the remote.
There are two ways of performing this check:
@@ -618,7 +618,7 @@ class RemoteBASE(object):
return self.batch_exists(chunks, callback=progress_callback)
if self.no_traverse and hasattr(self, "batch_exists"):
with ThreadPoolExecutor(max_workers=self.JOBS) as executor:
with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor:
path_infos = [self.checksum_to_path_info(x) for x in checksums]
chunks = to_chunks(path_infos, num_chunks=self.JOBS)
results = executor.map(exists_with_progress, chunks)
+3 -3
View File
@@ -241,7 +241,7 @@ class RemoteLOCAL(RemoteBASE):
move(inp, tmp)
move(tmp, outp)
def cache_exists(self, md5s):
def cache_exists(self, md5s, jobs=None):
return [
checksum
for checksum in progress(md5s)
@@ -306,7 +306,7 @@ class RemoteLOCAL(RemoteBASE):
md5s = list(ret)
logger.info("Collecting information from local cache...")
local_exists = self.cache_exists(md5s)
local_exists = self.cache_exists(md5s, jobs=jobs)
# This is a performance optimization. We can safely assume that,
# if the resources that we want to fetch are already cached,
@@ -316,7 +316,7 @@ class RemoteLOCAL(RemoteBASE):
remote_exists = local_exists
else:
logger.info("Collecting information from remote cache...")
remote_exists = list(remote.cache_exists(md5s))
remote_exists = list(remote.cache_exists(md5s, jobs=jobs))
self._fill_statuses(ret, local_exists, remote_exists)