Add dependency checking as config validation

This commit is contained in:
Victor Villas
2018-06-26 21:27:49 +00:00
parent 0831414d1c
commit ebee3ceae1
6 changed files with 24 additions and 7 deletions
+1 -1
View File
@@ -21,6 +21,6 @@ def supported_url(url):
def Remote(project, config):
for r in REMOTES:
if r.supported(config):
if r.supported(config, check_dependencies=True):
return r(project, config)
raise DvcException('Remote \'{}\' is not supported.'.format(config))
+4 -2
View File
@@ -33,14 +33,16 @@ class DataCloudError(DvcException):
class RemoteBase(object):
REGEX = None
REQUIRES = []
def __init__(self, project, config):
pass
@classmethod
def supported(cls, config):
def supported(cls, config, check_dependencies=False):
url = config[Config.SECTION_REMOTE_URL]
return cls.match(url) != None
deps_ok = not check_dependencies or all(cls.REQUIRES)
return cls.match(url) != None and deps_ok
@classmethod
def match(cls, url):
+6 -1
View File
@@ -1,6 +1,10 @@
import os
import posixpath
from google.cloud import storage
try:
from google.cloud import storage
except ImportError:
storage = None
try:
from urlparse import urlparse
@@ -16,6 +20,7 @@ from dvc.progress import progress
class RemoteGS(RemoteBase):
scheme = 'gs'
REGEX = r'^gs://(?P<path>.*)$'
REQUIRES = RemoteBase.REQUIRES + [storage]
PARAM_ETAG = 'etag'
def __init__(self, project, config):
+6 -1
View File
@@ -1,9 +1,13 @@
import os
import math
import boto3
import threading
import posixpath
try:
import boto3
except ImportError:
boto3 = None
try:
from urlparse import urlparse
except ImportError:
@@ -31,6 +35,7 @@ class Callback(object):
class RemoteS3(RemoteBase):
scheme = 's3'
REGEX = r'^s3://(?P<path>.*)$'
REQUIRES = RemoteBase.REQUIRES + [boto3]
PARAM_ETAG = 'etag'
def __init__(self, project, config):
+6 -1
View File
@@ -3,9 +3,13 @@ import re
import errno
import getpass
import filecmp
import paramiko
import posixpath
try:
import paramiko
except ImportError:
paramiko = None
from dvc.logger import Logger
from dvc.utils import copyfile, file_md5
from dvc.progress import progress
@@ -41,6 +45,7 @@ class RemoteSSH(RemoteBase):
#NOTE: temporarily only absolute paths are allowed
REGEX = r'^ssh://((?P<user>.*)@)?(?P<host>[^/]*):(?P<path>/.*)$'
REQUIRES = RemoteBase.REQUIRES + [paramiko]
PARAM_MD5 = 'md5'
def __init__(self, project, config):
+1 -1
View File
@@ -5,6 +5,7 @@ from dvc import VERSION
install_requires = [
"ply==3.8", # required by google-gax
"configparser>=3.5.0",
"zc.lockfile>=1.2.1",
"future>=0.16.0",
@@ -27,7 +28,6 @@ install_requires = [
# Extra dependencies for remote integrations
gs = [
"ply==3.8",
"google-cloud==0.32.0",
]
s3 = [