Merge pull request #810 from villasv/master

Optional dependencies for cache strategies

dvc/remote/base.py

@@ -33,14 +33,18 @@ class DataCloudError(DvcException):
 
 class RemoteBase(object):
     REGEX = None
+    REQUIRES = []
 
     def __init__(self, project, config):
         pass
 
     @classmethod
     def supported(cls, config):
-        url = config[Config.SECTION_REMOTE_URL]
-        return cls.match(url) != None
+        url_ok = cls.match(config[Config.SECTION_REMOTE_URL])
+        deps_ok = all(cls.REQUIRES)
+        if url_ok and not deps_ok:
+            Logger.warning("URL matches but requires missing dependencies")
+        return url_ok and deps_ok
 
     @classmethod
     def match(cls, url):

dvc/remote/gs.py

@@ -1,6 +1,10 @@
 import os
 import posixpath
-from google.cloud import storage
+
+try:
+    from google.cloud import storage
+except ImportError:
+    storage = None
 
 try:
     from urlparse import urlparse
@@ -16,6 +20,7 @@ from dvc.progress import progress
 class RemoteGS(RemoteBase):
     scheme = 'gs'
     REGEX = r'^gs://(?P<path>.*)$'
+    REQUIRES = RemoteBase.REQUIRES + [storage]
     PARAM_ETAG = 'etag'
 
     def __init__(self, project, config):

dvc/remote/s3.py

@@ -1,9 +1,13 @@
 import os
 import math
-import boto3
 import threading
 import posixpath
 
+try:
+    import boto3
+except ImportError:
+    boto3 = None
+
 try:
     from urlparse import urlparse
 except ImportError:
@@ -31,6 +35,7 @@ class Callback(object):
 class RemoteS3(RemoteBase):
     scheme = 's3'
     REGEX = r'^s3://(?P<path>.*)$'
+    REQUIRES = RemoteBase.REQUIRES + [boto3]
     PARAM_ETAG = 'etag'
 
     def __init__(self, project, config):

dvc/remote/ssh.py

@@ -3,9 +3,13 @@ import re
 import errno
 import getpass
 import filecmp
-import paramiko
 import posixpath
 
+try:
+    import paramiko
+except ImportError:
+    paramiko = None
+
 from dvc.logger import Logger
 from dvc.utils import copyfile, file_md5
 from dvc.progress import progress
@@ -41,6 +45,7 @@ class RemoteSSH(RemoteBase):
     #NOTE: temporarily only absolute paths are allowed
     REGEX = r'^ssh://((?P<user>.*)@)?(?P<host>[^/]*):(?P<path>/.*)$'
 
+    REQUIRES = RemoteBase.REQUIRES + [paramiko]
     PARAM_MD5 = 'md5'
 
     def __init__(self, project, config):

setup.py

@@ -5,12 +5,10 @@ from dvc import VERSION
 
 
 install_requires = [
-    "boto3==1.7.4",
     "ply==3.8", # required by google-gax
     "configparser>=3.5.0",
     "zc.lockfile>=1.2.1",
     "future>=0.16.0",
-    "google-cloud==0.32.0",
     "colorama>=0.3.9",
     "configobj>=5.0.6",
     "networkx>=1.11",
@@ -22,13 +20,24 @@ install_requires = [
     "nanotime>=0.5.2",
     "pyasn1>=0.4.1",
     "schema>=0.6.7",
-    "paramiko>=2.4.1",
     "binaryornot==0.4.4",
     "jsonpath-rw==1.4.0",
     "reflink==0.2.0",
     "requests>=2.18.4",
 ]
 
+# Extra dependencies for remote integrations
+gs = [
+    "google-cloud==0.32.0",
+]
+s3 = [
+    "boto3==1.7.4",
+]
+ssh = [
+    "paramiko>=2.4.1",
+]
+all_remotes = gs + s3 + ssh
+
 setup(
     name='dvc',
     version=VERSION,
@@ -39,6 +48,12 @@ setup(
     download_url='https://github.com/iterative/dvc',
     license='Apache License 2.0',
     install_requires=install_requires,
+    extras_require={
+        'all': all_remotes,
+        'gs': gs,
+        's3': s3,
+        'ssh': ssh,
+    },
     keywords='data science, data version control, machine learning',
     classifiers=[
         'Development Status :: 4 - Beta',