diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 8e35ffe41..7b7198d80 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -589,7 +589,7 @@ class RemoteBASE(object): return self._changed_dir_cache(checksum) return self.changed_cache_file(checksum) - def cache_exists(self, checksums): + def cache_exists(self, checksums, jobs=None): """Check if the given checksums are stored in the remote. There are two ways of performing this check: @@ -618,7 +618,7 @@ class RemoteBASE(object): return self.batch_exists(chunks, callback=progress_callback) if self.no_traverse and hasattr(self, "batch_exists"): - with ThreadPoolExecutor(max_workers=self.JOBS) as executor: + with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor: path_infos = [self.checksum_to_path_info(x) for x in checksums] chunks = to_chunks(path_infos, num_chunks=self.JOBS) results = executor.map(exists_with_progress, chunks) diff --git a/dvc/remote/local/__init__.py b/dvc/remote/local/__init__.py index 564cda289..c35502df7 100644 --- a/dvc/remote/local/__init__.py +++ b/dvc/remote/local/__init__.py @@ -241,7 +241,7 @@ class RemoteLOCAL(RemoteBASE): move(inp, tmp) move(tmp, outp) - def cache_exists(self, md5s): + def cache_exists(self, md5s, jobs=None): return [ checksum for checksum in progress(md5s) @@ -306,7 +306,7 @@ class RemoteLOCAL(RemoteBASE): md5s = list(ret) logger.info("Collecting information from local cache...") - local_exists = self.cache_exists(md5s) + local_exists = self.cache_exists(md5s, jobs=jobs) # This is a performance optimization. We can safely assume that, # if the resources that we want to fetch are already cached, @@ -316,7 +316,7 @@ class RemoteLOCAL(RemoteBASE): remote_exists = local_exists else: logger.info("Collecting information from remote cache...") - remote_exists = list(remote.cache_exists(md5s)) + remote_exists = list(remote.cache_exists(md5s, jobs=jobs)) self._fill_statuses(ret, local_exists, remote_exists)