Merge pull request #2278 from Suor/ssh-md5-jobs

remote: speed up ssh dir checksum

dvc/remote/base.py

@@ -43,9 +43,6 @@ STATUS_MAP = {
 }
 
 
-CHECKSUM_JOBS = max(1, min(4, cpu_count() // 2))
-
-
 class DataCloudError(DvcException):
     """ Data Cloud exception """
 
@@ -79,6 +76,7 @@ class RemoteBASE(object):
 
     PARAM_RELPATH = "relpath"
     CHECKSUM_DIR_SUFFIX = ".dir"
+    CHECKSUM_JOBS = max(1, min(4, cpu_count() // 2))
 
     def __init__(self, repo, config):
         self.repo = repo
@@ -110,7 +108,7 @@ class RemoteBASE(object):
 
         core = config.get(Config.SECTION_CORE, {})
         self.checksum_jobs = core.get(
-            Config.SECTION_CORE_CHECKSUM_JOBS, CHECKSUM_JOBS
+            Config.SECTION_CORE_CHECKSUM_JOBS, self.CHECKSUM_JOBS
         )
 
         self.protected = False

dvc/remote/ssh/__init__.py

@@ -32,6 +32,10 @@ class RemoteSSH(RemoteBASE):
     PARAM_CHECKSUM = "md5"
     DEFAULT_PORT = 22
     TIMEOUT = 1800
+    # At any given time some of the connections will go over network and
+    # paramiko stuff, so we would ideally have it double of server processors.
+    # We use conservative setting of 4 instead to not exhaust max sessions.
+    CHECKSUM_JOBS = 4
 
     def __init__(self, repo, config):
         super(RemoteSSH, self).__init__(repo, config)