From 0831414d1cd0b7f107713adf8538fbb18f6f428e Mon Sep 17 00:00:00 2001 From: Victor Villas Date: Tue, 26 Jun 2018 18:56:17 +0000 Subject: [PATCH 1/3] Move remote dependencies to extras_require section --- setup.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index cdbf37475..b7b6ed81c 100644 --- a/setup.py +++ b/setup.py @@ -5,12 +5,9 @@ from dvc import VERSION install_requires = [ - "boto3==1.7.4", - "ply==3.8", # required by google-gax "configparser>=3.5.0", "zc.lockfile>=1.2.1", "future>=0.16.0", - "google-cloud==0.32.0", "colorama>=0.3.9", "configobj>=5.0.6", "networkx>=1.11", @@ -22,13 +19,25 @@ install_requires = [ "nanotime>=0.5.2", "pyasn1>=0.4.1", "schema>=0.6.7", - "paramiko>=2.4.1", "binaryornot==0.4.4", "jsonpath-rw==1.4.0", "reflink==0.2.0", "requests>=2.18.4", ] +# Extra dependencies for remote integrations +gs = [ + "ply==3.8", + "google-cloud==0.32.0", +] +s3 = [ + "boto3==1.7.4", +] +ssh = [ + "paramiko>=2.4.1", +] +all_remotes = gs + s3 + ssh + setup( name='dvc', version=VERSION, @@ -39,6 +48,12 @@ setup( download_url='https://github.com/iterative/dvc', license='Apache License 2.0', install_requires=install_requires, + extras_require={ + 'all': all_remotes, + 'gs': gs, + 's3': s3, + 'ssh': ssh, + }, keywords='data science, data version control, machine learning', classifiers=[ 'Development Status :: 4 - Beta', From ebee3ceae1aa01ef51fb3a85aeaf915cf67480b8 Mon Sep 17 00:00:00 2001 From: Victor Villas Date: Tue, 26 Jun 2018 21:27:49 +0000 Subject: [PATCH 2/3] Add dependency checking as config validation --- dvc/remote/__init__.py | 2 +- dvc/remote/base.py | 6 ++++-- dvc/remote/gs.py | 7 ++++++- dvc/remote/s3.py | 7 ++++++- dvc/remote/ssh.py | 7 ++++++- setup.py | 2 +- 6 files changed, 24 insertions(+), 7 deletions(-) diff --git a/dvc/remote/__init__.py b/dvc/remote/__init__.py index 40a44b5d3..1648e1ec8 100644 --- a/dvc/remote/__init__.py +++ b/dvc/remote/__init__.py @@ -21,6 +21,6 @@ def supported_url(url): def Remote(project, config): for r in REMOTES: - if r.supported(config): + if r.supported(config, check_dependencies=True): return r(project, config) raise DvcException('Remote \'{}\' is not supported.'.format(config)) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 6fb0a791f..f526ae1c3 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -33,14 +33,16 @@ class DataCloudError(DvcException): class RemoteBase(object): REGEX = None + REQUIRES = [] def __init__(self, project, config): pass @classmethod - def supported(cls, config): + def supported(cls, config, check_dependencies=False): url = config[Config.SECTION_REMOTE_URL] - return cls.match(url) != None + deps_ok = not check_dependencies or all(cls.REQUIRES) + return cls.match(url) != None and deps_ok @classmethod def match(cls, url): diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index 4683fb648..0ae7ee3b7 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -1,6 +1,10 @@ import os import posixpath -from google.cloud import storage + +try: + from google.cloud import storage +except ImportError: + storage = None try: from urlparse import urlparse @@ -16,6 +20,7 @@ from dvc.progress import progress class RemoteGS(RemoteBase): scheme = 'gs' REGEX = r'^gs://(?P.*)$' + REQUIRES = RemoteBase.REQUIRES + [storage] PARAM_ETAG = 'etag' def __init__(self, project, config): diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index ad2c1a832..2d1416295 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -1,9 +1,13 @@ import os import math -import boto3 import threading import posixpath +try: + import boto3 +except ImportError: + boto3 = None + try: from urlparse import urlparse except ImportError: @@ -31,6 +35,7 @@ class Callback(object): class RemoteS3(RemoteBase): scheme = 's3' REGEX = r'^s3://(?P.*)$' + REQUIRES = RemoteBase.REQUIRES + [boto3] PARAM_ETAG = 'etag' def __init__(self, project, config): diff --git a/dvc/remote/ssh.py b/dvc/remote/ssh.py index 6becc191f..211aa54d1 100644 --- a/dvc/remote/ssh.py +++ b/dvc/remote/ssh.py @@ -3,9 +3,13 @@ import re import errno import getpass import filecmp -import paramiko import posixpath +try: + import paramiko +except ImportError: + paramiko = None + from dvc.logger import Logger from dvc.utils import copyfile, file_md5 from dvc.progress import progress @@ -41,6 +45,7 @@ class RemoteSSH(RemoteBase): #NOTE: temporarily only absolute paths are allowed REGEX = r'^ssh://((?P.*)@)?(?P[^/]*):(?P/.*)$' + REQUIRES = RemoteBase.REQUIRES + [paramiko] PARAM_MD5 = 'md5' def __init__(self, project, config): diff --git a/setup.py b/setup.py index b7b6ed81c..5bb4fb875 100644 --- a/setup.py +++ b/setup.py @@ -5,6 +5,7 @@ from dvc import VERSION install_requires = [ + "ply==3.8", # required by google-gax "configparser>=3.5.0", "zc.lockfile>=1.2.1", "future>=0.16.0", @@ -27,7 +28,6 @@ install_requires = [ # Extra dependencies for remote integrations gs = [ - "ply==3.8", "google-cloud==0.32.0", ] s3 = [ From 8da7bf663877e0ef0f84513d2568bfd118f29f2c Mon Sep 17 00:00:00 2001 From: Victor Villas Date: Wed, 27 Jun 2018 14:15:09 +0000 Subject: [PATCH 3/3] Remove check deps flag and add warning on req miss --- dvc/remote/__init__.py | 2 +- dvc/remote/base.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dvc/remote/__init__.py b/dvc/remote/__init__.py index 1648e1ec8..40a44b5d3 100644 --- a/dvc/remote/__init__.py +++ b/dvc/remote/__init__.py @@ -21,6 +21,6 @@ def supported_url(url): def Remote(project, config): for r in REMOTES: - if r.supported(config, check_dependencies=True): + if r.supported(config): return r(project, config) raise DvcException('Remote \'{}\' is not supported.'.format(config)) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index f526ae1c3..41c5e5637 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -39,10 +39,12 @@ class RemoteBase(object): pass @classmethod - def supported(cls, config, check_dependencies=False): - url = config[Config.SECTION_REMOTE_URL] - deps_ok = not check_dependencies or all(cls.REQUIRES) - return cls.match(url) != None and deps_ok + def supported(cls, config): + url_ok = cls.match(config[Config.SECTION_REMOTE_URL]) + deps_ok = all(cls.REQUIRES) + if url_ok and not deps_ok: + Logger.warning("URL matches but requires missing dependencies") + return url_ok and deps_ok @classmethod def match(cls, url):