diff --git a/dvc/remote/base.py b/dvc/remote/base.py index ae6b584fe..8e35ffe41 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -43,9 +43,6 @@ STATUS_MAP = { } -CHECKSUM_JOBS = max(1, min(4, cpu_count() // 2)) - - class DataCloudError(DvcException): """ Data Cloud exception """ @@ -79,6 +76,7 @@ class RemoteBASE(object): PARAM_RELPATH = "relpath" CHECKSUM_DIR_SUFFIX = ".dir" + CHECKSUM_JOBS = max(1, min(4, cpu_count() // 2)) def __init__(self, repo, config): self.repo = repo @@ -110,7 +108,7 @@ class RemoteBASE(object): core = config.get(Config.SECTION_CORE, {}) self.checksum_jobs = core.get( - Config.SECTION_CORE_CHECKSUM_JOBS, CHECKSUM_JOBS + Config.SECTION_CORE_CHECKSUM_JOBS, self.CHECKSUM_JOBS ) self.protected = False diff --git a/dvc/remote/ssh/__init__.py b/dvc/remote/ssh/__init__.py index ed5e5889b..bd25f954c 100644 --- a/dvc/remote/ssh/__init__.py +++ b/dvc/remote/ssh/__init__.py @@ -32,6 +32,10 @@ class RemoteSSH(RemoteBASE): PARAM_CHECKSUM = "md5" DEFAULT_PORT = 22 TIMEOUT = 1800 + # At any given time some of the connections will go over network and + # paramiko stuff, so we would ideally have it double of server processors. + # We use conservative setting of 4 instead to not exhaust max sessions. + CHECKSUM_JOBS = 4 def __init__(self, repo, config): super(RemoteSSH, self).__init__(repo, config)