From d1be895b6115513c997464b37750f9f7237ceebf Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 4 Jan 2019 07:21:44 +0200 Subject: [PATCH 1/4] cli: extract --global/--system/--local into a common parser Signed-off-by: Ruslan Kuprieiev --- dvc/cli.py | 129 +++++++++++------------------------------------------ 1 file changed, 27 insertions(+), 102 deletions(-) diff --git a/dvc/cli.py b/dvc/cli.py index 08d726fc0..6b69d7ba8 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -643,11 +643,32 @@ def parse_args(argv=None): help='Collect garbage for all given projects.') gc_parser.set_defaults(func=CmdGC) + # Parent parser used in config/remote + parent_config_parser = argparse.ArgumentParser( + add_help=False, + parents=[parent_parser]) + parent_config_parser.add_argument( + '--global', + dest='glob', + action='store_true', + default=False, + help='Use global config.') + parent_config_parser.add_argument( + '--system', + action='store_true', + default=False, + help='Use system config.') + parent_config_parser.add_argument( + '--local', + action='store_true', + default=False, + help='Use local config.') + # Config CONFIG_HELP = 'Get or set config options.' config_parser = subparsers.add_parser( 'config', - parents=[parent_parser], + parents=[parent_config_parser], description=CONFIG_HELP, help=CONFIG_HELP) config_parser.add_argument( @@ -664,22 +685,6 @@ def parse_args(argv=None): nargs='?', default=None, help='Option value.') - config_parser.add_argument( - '--global', - dest='glob', - action='store_true', - default=False, - help='Use global config.') - config_parser.add_argument( - '--system', - action='store_true', - default=False, - help='Use system config.') - config_parser.add_argument( - '--local', - action='store_true', - default=False, - help='Use local config.') config_parser.set_defaults(func=CmdConfig) # Remote @@ -700,7 +705,7 @@ def parse_args(argv=None): REMOTE_ADD_HELP = 'Add remote.' remote_add_parser = remote_subparsers.add_parser( 'add', - parents=[parent_parser], + parents=[parent_config_parser], description=REMOTE_ADD_HELP, help=REMOTE_ADD_HELP) remote_add_parser.add_argument( @@ -709,22 +714,6 @@ def parse_args(argv=None): remote_add_parser.add_argument( 'url', help='URL.') - remote_add_parser.add_argument( - '--global', - dest='glob', - action='store_true', - default=False, - help='Use global config.') - remote_add_parser.add_argument( - '--system', - action='store_true', - default=False, - help='Use system config.') - remote_add_parser.add_argument( - '--local', - action='store_true', - default=False, - help='Use local config.') remote_add_parser.add_argument( '-d', '--default', @@ -736,7 +725,7 @@ def parse_args(argv=None): REMOTE_DEFAULT_HELP = 'Set/unset default remote.' remote_default_parser = remote_subparsers.add_parser( 'default', - parents=[parent_parser], + parents=[parent_config_parser], description=REMOTE_DEFAULT_HELP, help=REMOTE_DEFAULT_HELP) remote_default_parser.add_argument( @@ -749,55 +738,23 @@ def parse_args(argv=None): action='store_true', default=False, help='Unset default remote.') - remote_default_parser.add_argument( - '--global', - dest='glob', - action='store_true', - default=False, - help='Use global config.') - remote_default_parser.add_argument( - '--system', - action='store_true', - default=False, - help='Use system config.') - remote_default_parser.add_argument( - '--local', - action='store_true', - default=False, - help='Use local config.') remote_default_parser.set_defaults(func=CmdRemoteDefault) REMOTE_REMOVE_HELP = 'Remove remote.' remote_remove_parser = remote_subparsers.add_parser( 'remove', - parents=[parent_parser], + parents=[parent_config_parser], description=REMOTE_REMOVE_HELP, help=REMOTE_REMOVE_HELP) remote_remove_parser.add_argument( 'name', help='Name') - remote_remove_parser.add_argument( - '--global', - dest='glob', - action='store_true', - default=False, - help='Use global config.') - remote_remove_parser.add_argument( - '--system', - action='store_true', - default=False, - help='Use system config.') - remote_remove_parser.add_argument( - '--local', - action='store_true', - default=False, - help='Use local config.') remote_remove_parser.set_defaults(func=CmdRemoteRemove) REMOTE_MODIFY_HELP = 'Modify remote.' remote_modify_parser = remote_subparsers.add_parser( 'modify', - parents=[parent_parser], + parents=[parent_config_parser], description=REMOTE_MODIFY_HELP, help=REMOTE_MODIFY_HELP) remote_modify_parser.add_argument( @@ -816,46 +773,14 @@ def parse_args(argv=None): default=False, action='store_true', help='Unset option.') - remote_modify_parser.add_argument( - '--global', - dest='glob', - action='store_true', - default=False, - help='Use global config.') - remote_modify_parser.add_argument( - '--system', - action='store_true', - default=False, - help='Use system config.') - remote_modify_parser.add_argument( - '--local', - action='store_true', - default=False, - help='Use local config.') remote_modify_parser.set_defaults(func=CmdRemoteModify) REMOTE_LIST_HELP = 'List remotes.' remote_list_parser = remote_subparsers.add_parser( 'list', - parents=[parent_parser], + parents=[parent_config_parser], description=REMOTE_LIST_HELP, help=REMOTE_LIST_HELP) - remote_list_parser.add_argument( - '--global', - dest='glob', - action='store_true', - default=False, - help='Use global config.') - remote_list_parser.add_argument( - '--system', - action='store_true', - default=False, - help='Use system config.') - remote_list_parser.add_argument( - '--local', - action='store_true', - default=False, - help='Use local config.') remote_list_parser.set_defaults(func=CmdRemoteList) # Metrics From db6cd8e1bb87d1d707bcc15545fc85817703634c Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 4 Jan 2019 09:02:37 +0200 Subject: [PATCH 2/4] dvc: introduce `dvc cache dir` command Signed-off-by: Ruslan Kuprieiev --- dvc/cli.py | 41 +++++++++++++++++++++++++++++++++++++++++ dvc/command/cache.py | 16 ++++++++++++++++ tests/test_cache.py | 22 ++++++++++++++++++++++ 3 files changed, 79 insertions(+) create mode 100644 dvc/command/cache.py diff --git a/dvc/cli.py b/dvc/cli.py index 6b69d7ba8..605fa8957 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -21,6 +21,7 @@ from dvc.command.config import CmdConfig from dvc.command.checkout import CmdCheckout from dvc.command.remote import CmdRemoteAdd, CmdRemoteRemove from dvc.command.remote import CmdRemoteModify, CmdRemoteList, CmdRemoteDefault +from dvc.command.cache import CmdCacheDir from dvc.command.metrics import CmdMetricsShow, CmdMetricsAdd from dvc.command.metrics import CmdMetricsRemove, CmdMetricsModify from dvc.command.install import CmdInstall @@ -783,6 +784,46 @@ def parse_args(argv=None): help=REMOTE_LIST_HELP) remote_list_parser.set_defaults(func=CmdRemoteList) + # Cache + CACHE_HELP = 'Manage cache settings.' + cache_parser = subparsers.add_parser( + 'cache', + parents=[parent_parser], + description=CACHE_HELP, + help=CACHE_HELP) + + cache_subparsers = cache_parser.add_subparsers( + dest='cmd', + help='Use dvc cache CMD --help for ' + 'command-specific help.') + + _fix_subparsers(cache_subparsers) + + parent_cache_config_parser = argparse.ArgumentParser( + add_help=False, + parents=[parent_config_parser]) + CACHE_DIR_HELP = 'Configure cache directory location.' + cache_dir_parser = cache_subparsers.add_parser( + 'dir', + parents=[parent_cache_config_parser], + description=CACHE_DIR_HELP, + help=CACHE_DIR_HELP) + cache_dir_parser.add_argument( + '-u', + '--unset', + default=False, + action='store_true', + help='Unset option.') + cache_dir_parser.add_argument( + 'value', + nargs='?', + default=None, + help='Path to cache directory. Relative paths are ' + 'resolved relative to the current directory and ' + 'saved to config relative to the config file ' + 'location.') + cache_dir_parser.set_defaults(func=CmdCacheDir) + # Metrics METRICS_HELP = 'Get metrics from all branches.' metrics_parser = subparsers.add_parser( diff --git a/dvc/command/cache.py b/dvc/command/cache.py new file mode 100644 index 000000000..599a1ed96 --- /dev/null +++ b/dvc/command/cache.py @@ -0,0 +1,16 @@ +import os + +from dvc.command.config import CmdConfig + + +class CmdCacheDir(CmdConfig): + def run(self): + self.args.name = 'cache.dir' + + path = self.args.value + if not os.path.isabs(path): + config_dir = os.path.dirname(self.configobj.filename) + path = os.path.relpath(path, config_dir) + self.args.value = path + + return super(CmdCacheDir, self).run() diff --git a/tests/test_cache.py b/tests/test_cache.py index a7a33a520..099fdd609 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -1,5 +1,6 @@ import os import shutil +import configobj from dvc.cache import Cache from dvc.main import main @@ -114,3 +115,24 @@ class TestCacheLinkType(TestDvc): ret = main(['add', self.FOO]) self.assertEqual(ret, 0) + + +class TestCmdCacheDir(TestDvc): + def test_abs_path(self): + dname = os.path.join(os.path.dirname(self._root_dir), 'dir') + ret = main(['cache', 'dir', dname]) + self.assertEqual(ret, 0) + + config = configobj.ConfigObj(self.dvc.config.config_file) + self.assertEqual(config['cache']['dir'], dname) + + def test_relative_path(self): + dname = os.path.join('..', 'path', 'to', 'dir') + ret = main(['cache', 'dir', dname]) + self.assertEqual(ret, 0) + + # NOTE: we are in the project's root and config is in .dvc/, so + # dir path written to config should be just one level above. + rel = os.path.join('..', dname) + config = configobj.ConfigObj(self.dvc.config.config_file) + self.assertEqual(config['cache']['dir'], rel) From 294b259579a4555c93690d6b0b496d3fee23a4df Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 4 Jan 2019 10:51:52 +0200 Subject: [PATCH 3/4] cache: support relative paths Fixes #1284 Signed-off-by: Ruslan Kuprieiev --- dvc/cache.py | 7 ++----- dvc/config.py | 33 +++++++++++++++++++++++++++------ tests/test_cache.py | 11 ++++++++++- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/dvc/cache.py b/dvc/cache.py index c492e32cc..89ca47eac 100644 --- a/dvc/cache.py +++ b/dvc/cache.py @@ -18,14 +18,11 @@ class Cache(object): name = Config.SECTION_REMOTE_FMT.format(local) sect = project.config._config[name] else: - cache_dir = config.get(Config.SECTION_CACHE_DIR, self.CACHE_DIR) + default_cache_dir = os.path.join(project.dvc_dir, self.CACHE_DIR) + cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir) cache_type = config.get(Config.SECTION_CACHE_TYPE) protected = config.get(Config.SECTION_CACHE_PROTECTED) - if not os.path.isabs(cache_dir): - cache_dir = os.path.join(project.dvc_dir, cache_dir) - cache_dir = os.path.abspath(os.path.realpath(cache_dir)) - sect = { Config.SECTION_REMOTE_URL: cache_dir, Config.SECTION_CACHE_TYPE: cache_type, diff --git a/dvc/config.py b/dvc/config.py index bdc6c08ac..25f5228c9 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -253,19 +253,39 @@ class Config(object): self._config = None + def _load_config(self, path): + config = configobj.ConfigObj(path) + config = self._lower(config) + self._resolve_paths(config, path) + return config + + def _resolve_paths(self, config, fname): + cache = config.get(self.SECTION_CACHE) + if cache is None: + return + + cache_dir = cache.get(self.SECTION_CACHE_DIR) + if cache_dir is None: + return + + if os.path.isabs(cache_dir): + return + + config_dir = os.path.dirname(fname) + cache_dir = os.path.abspath(os.path.join(config_dir, cache_dir)) + cache[self.SECTION_CACHE_DIR] = cache_dir + def load(self, validate=True): self._load() try: - self._config = configobj.ConfigObj(self.system_config_file) - user = configobj.ConfigObj(self.global_config_file) - config = configobj.ConfigObj(self.config_file) - local = configobj.ConfigObj(self.config_local_file) + self._config = self._load_config(self.system_config_file) + user = self._load_config(self.global_config_file) + config = self._load_config(self.config_file) + local = self._load_config(self.config_local_file) # NOTE: schema doesn't support ConfigObj.Section validation, so we # need to convert our config to dict before passing it to - self._config = self._lower(self._config) for c in [user, config, local]: - c = self._lower(c) self._config = self._merge(self._config, c) if validate: @@ -275,6 +295,7 @@ class Config(object): self._config = configobj.ConfigObj(self._config, write_empty_values=True) self._config.filename = self.config_file + self._resolve_paths(self._config, self.config_file) except Exception as ex: raise ConfigError(ex) diff --git a/tests/test_cache.py b/tests/test_cache.py index 099fdd609..7c39e35a8 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -127,7 +127,8 @@ class TestCmdCacheDir(TestDvc): self.assertEqual(config['cache']['dir'], dname) def test_relative_path(self): - dname = os.path.join('..', 'path', 'to', 'dir') + tmpdir = self.mkdtemp() + dname = os.path.relpath(tmpdir) ret = main(['cache', 'dir', dname]) self.assertEqual(ret, 0) @@ -136,3 +137,11 @@ class TestCmdCacheDir(TestDvc): rel = os.path.join('..', dname) config = configobj.ConfigObj(self.dvc.config.config_file) self.assertEqual(config['cache']['dir'], rel) + + ret = main(['add', self.FOO]) + self.assertEqual(ret, 0) + + subdirs = os.listdir(tmpdir) + self.assertEqual(len(subdirs), 1) + files = os.listdir(os.path.join(tmpdir, subdirs[0])) + self.assertEqual(len(files), 1) From 484f318c278e15699020641b5b5722abeb4d0a9a Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 4 Jan 2019 16:11:53 +0200 Subject: [PATCH 4/4] remote: local: support relative paths Fixes #1421 Signed-off-by: Ruslan Kuprieiev --- dvc/cache.py | 2 ++ dvc/command/cache.py | 11 +++------ dvc/command/remote.py | 25 ++++++++++++++++++++ dvc/config.py | 38 ++++++++++++++++++++----------- dvc/data_cloud.py | 14 ++---------- dvc/remote/__init__.py | 18 ++++++--------- dvc/remote/azure.py | 5 ++-- dvc/remote/base.py | 48 ++++++++++++++++++++++----------------- dvc/remote/gs.py | 2 +- dvc/remote/hdfs.py | 2 +- dvc/remote/http.py | 2 +- dvc/remote/local.py | 10 +++++--- dvc/remote/s3.py | 2 +- dvc/remote/ssh.py | 2 +- tests/__main__.py | 2 +- tests/test_data_cloud.py | 13 +---------- tests/test_remote.py | 15 ++++++++---- tests/unit/remote/base.py | 15 ++++++++++++ 18 files changed, 132 insertions(+), 94 deletions(-) create mode 100644 tests/unit/remote/base.py diff --git a/dvc/cache.py b/dvc/cache.py index 89ca47eac..285adf69a 100644 --- a/dvc/cache.py +++ b/dvc/cache.py @@ -24,6 +24,8 @@ class Cache(object): protected = config.get(Config.SECTION_CACHE_PROTECTED) sect = { + Config.PRIVATE_CWD: config.get(Config.PRIVATE_CWD, + project.dvc_dir), Config.SECTION_REMOTE_URL: cache_dir, Config.SECTION_CACHE_TYPE: cache_type, Config.SECTION_CACHE_PROTECTED: protected diff --git a/dvc/command/cache.py b/dvc/command/cache.py index 599a1ed96..1d31030b5 100644 --- a/dvc/command/cache.py +++ b/dvc/command/cache.py @@ -1,16 +1,11 @@ -import os - +from dvc.command.remote import CmdRemoteAdd from dvc.command.config import CmdConfig class CmdCacheDir(CmdConfig): def run(self): self.args.name = 'cache.dir' - - path = self.args.value - if not os.path.isabs(path): - config_dir = os.path.dirname(self.configobj.filename) - path = os.path.relpath(path, config_dir) - self.args.value = path + self.args.value = CmdRemoteAdd.resolve_path(self.args.value, + self.configobj.filename) return super(CmdCacheDir, self).run() diff --git a/dvc/command/remote.py b/dvc/command/remote.py index 682aef333..b0b50ec0a 100644 --- a/dvc/command/remote.py +++ b/dvc/command/remote.py @@ -1,12 +1,37 @@ +import os import re import dvc.logger as logger from dvc.config import Config +from dvc.remote import _get, RemoteLOCAL from dvc.command.config import CmdConfig class CmdRemoteAdd(CmdConfig): + @staticmethod + def resolve_path(path, config_file): + """Resolve path relative to config file location. + + Args: + path: Path to be resolved. + config_file: Path to config file, which `path` is specified + relative to. + + Returns: + Path relative to the `config_file` location. If `path` is an + absolute path then it will be returned without change. + + """ + if os.path.isabs(path): + return path + return os.path.relpath(path, os.path.dirname(config_file)) + def run(self): + remote = _get({Config.SECTION_REMOTE_URL: self.args.url}) + if remote == RemoteLOCAL: + self.args.url = self.resolve_path(self.args.url, + self.configobj.filename) + section = Config.SECTION_REMOTE_FMT.format(self.args.name) ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url) if ret != 0: diff --git a/dvc/config.py b/dvc/config.py index 25f5228c9..84463d1ee 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -17,11 +17,6 @@ class ConfigError(DvcException): ex) -def supported_url(url): - from dvc.remote import supported_url as supported - return supported(url) - - def supported_cache_type(types): if isinstance(types, str): types = [t.strip() for t in types.split(',')] @@ -59,6 +54,10 @@ class Config(object): APPNAME = 'dvc' APPAUTHOR = 'iterative' + # NOTE: used internally in RemoteLOCAL to know config + # location, that url should resolved relative to. + PRIVATE_CWD = '_cwd' + CONFIG = 'config' CONFIG_LOCAL = 'config.local' @@ -90,10 +89,12 @@ class Config(object): Optional(SECTION_CACHE_SSH): str, Optional(SECTION_CACHE_AZURE): str, - Optional(SECTION_CACHE_DIR, default='cache'): str, + Optional(SECTION_CACHE_DIR): str, Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA, Optional(SECTION_CACHE_PROTECTED, default=False): And(str, is_bool, Use(to_bool)), + + Optional(PRIVATE_CWD): str, } # backward compatibility @@ -162,7 +163,7 @@ class Config(object): SECTION_REMOTE_PASSWORD = 'password' SECTION_REMOTE_ASK_PASSWORD = 'ask_password' SECTION_REMOTE_SCHEMA = { - SECTION_REMOTE_URL: And(supported_url, error="Unsupported URL"), + SECTION_REMOTE_URL: str, Optional(SECTION_AWS_REGION): str, Optional(SECTION_AWS_PROFILE): str, Optional(SECTION_AWS_CREDENTIALPATH): str, @@ -180,6 +181,8 @@ class Config(object): Optional(SECTION_REMOTE_PASSWORD): str, Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)), Optional(SECTION_AZURE_CONNECTION_STRING): str, + + Optional(PRIVATE_CWD): str, } SECTION_STATE = 'state' @@ -259,7 +262,13 @@ class Config(object): self._resolve_paths(config, path) return config - def _resolve_paths(self, config, fname): + @staticmethod + def _resolve_path(path, config_file): + assert os.path.isabs(config_file) + config_dir = os.path.dirname(config_file) + return os.path.abspath(os.path.join(config_dir, path)) + + def _resolve_cache_path(self, config, fname): cache = config.get(self.SECTION_CACHE) if cache is None: return @@ -268,12 +277,15 @@ class Config(object): if cache_dir is None: return - if os.path.isabs(cache_dir): - return + cache[self.PRIVATE_CWD] = os.path.dirname(fname) - config_dir = os.path.dirname(fname) - cache_dir = os.path.abspath(os.path.join(config_dir, cache_dir)) - cache[self.SECTION_CACHE_DIR] = cache_dir + def _resolve_paths(self, config, fname): + self._resolve_cache_path(config, fname) + for name, section in config.items(): + if self.SECTION_REMOTE_URL not in section.keys(): + continue + + section[self.PRIVATE_CWD] = os.path.dirname(fname) def load(self, validate=True): self._load() diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 9461599d8..be78c584c 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -1,6 +1,7 @@ import dvc.logger as logger from dvc.config import Config, ConfigError +from dvc.remote import Remote from dvc.remote.s3 import RemoteS3 from dvc.remote.gs import RemoteGS from dvc.remote.azure import RemoteAzure @@ -41,13 +42,6 @@ class DataCloud(object): return None - @staticmethod - def supported(config): - for cloud in DataCloud.CLOUD_MAP.values(): - if cloud.supported(config): - return cloud - return None - def _init_remote(self, remote): section = Config.SECTION_REMOTE_FMT.format(remote) cloud_config = self._config.get(section, None) @@ -55,11 +49,7 @@ class DataCloud(object): msg = "can't find remote section '{}' in config" raise ConfigError(msg.format(section)) - cloud_type = self.supported(cloud_config) - if not cloud_type: - raise ConfigError("unsupported cloud '{}'".format(cloud_config)) - - return self._init_cloud(cloud_config, cloud_type) + return Remote(self.project, cloud_config) def _init_compat(self): name = self._core.get(Config.SECTION_CORE_CLOUD, '').strip().lower() diff --git a/dvc/remote/__init__.py b/dvc/remote/__init__.py index f385108e9..e57665fb7 100644 --- a/dvc/remote/__init__.py +++ b/dvc/remote/__init__.py @@ -6,28 +6,24 @@ from dvc.remote.s3 import RemoteS3 from dvc.remote.ssh import RemoteSSH from dvc.remote.http import RemoteHTTP -from dvc.config import Config -from dvc.exceptions import UnsupportedRemoteError - REMOTES = [ RemoteAzure, RemoteGS, RemoteHDFS, RemoteHTTP, - RemoteLOCAL, RemoteS3, RemoteSSH, + # NOTE: RemoteLOCAL is the default ] -def supported_url(url): - config = {Config.SECTION_REMOTE_URL: url} - return any(remote.supported(config) for remote in REMOTES) +def _get(config): + for remote in REMOTES: + if remote.supported(config): + return remote + return RemoteLOCAL def Remote(project, config): - for remote in REMOTES: - if remote.supported(config): - return remote(project, config) - raise UnsupportedRemoteError(str(config)) + return _get(config)(project, config) diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index ba6023f70..a13c8ae62 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -38,9 +38,8 @@ class RemoteAzure(RemoteBase): def __init__(self, project, config): super(RemoteAzure, self).__init__(project, config) - self.project = project - self.url = config.get(Config.SECTION_REMOTE_URL) + self.url = config.get(Config.SECTION_REMOTE_URL, 'azure://') match = re.match(self.REGEX, self.url) # backward compatibility path = match.group('path') @@ -49,7 +48,7 @@ class RemoteAzure(RemoteBase): or match.group('container_name') # backward compatibility or os.getenv('AZURE_STORAGE_CONTAINER_NAME')) - self.prefix = urlparse(self.url).path.lstrip('/') + self.prefix = urlparse(self.url).path.lstrip('/') if path else '' self.connection_string = ( config.get(Config.SECTION_AZURE_CONNECTION_STRING) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 4b76b4e08..8738c2d07 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -36,31 +36,24 @@ class RemoteBaseCmdError(DvcException): super(RemoteBaseCmdError, self).__init__(m.format(cmd, ret, err)) +class RemoteMissingDepsError(DvcException): + pass + + class RemoteBase(object): + scheme = None REGEX = None REQUIRES = {} JOBS = 4 * cpu_count() def __init__(self, project, config): - pass - - def __repr__(self): - return "{class_name}: '{url}'".format( - class_name=type(self).__name__, - url=(self.url or 'No url') - ) - - def compat_config(config): - return config.copy() - - @classmethod - def supported(cls, config): - url = config[Config.SECTION_REMOTE_URL] - url_ok = cls.match(url) is not None - deps_ok = all(cls.REQUIRES.values()) - if url_ok and not deps_ok: - missing = [k for k, v in cls.REQUIRES.items() if v is None] - logger.warning( + self.project = project + deps_ok = all(self.REQUIRES.values()) + if not deps_ok: + missing = [k for k, v in self.REQUIRES.items() if v is None] + url = config.get(Config.SECTION_REMOTE_URL, + '{}://'.format(self.scheme)) + msg = ( "URL '{}' is supported but requires these missing " "dependencies: {}. If you have installed dvc using pip, " "choose one of these options to proceed: \n" @@ -77,10 +70,23 @@ class RemoteBase(object): "If you have installed dvc from a binary package and you " "are still seeing this message, please report it to us " "using https://github.com/iterative/dvc/issues. Thank you!" - .format(url, missing, " ".join(missing), cls.scheme) + .format(url, missing, " ".join(missing), self.scheme) ) + raise RemoteMissingDepsError(msg) - return url_ok and deps_ok + def __repr__(self): + return "{class_name}: '{url}'".format( + class_name=type(self).__name__, + url=(self.url or 'No url') + ) + + def compat_config(config): + return config.copy() + + @classmethod + def supported(cls, config): + url = config[Config.SECTION_REMOTE_URL] + return cls.match(url) is not None @classmethod def match(cls, url): diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index 7fc8ee923..8d808b6d8 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -24,7 +24,7 @@ class RemoteGS(RemoteBase): PARAM_CHECKSUM = 'md5' def __init__(self, project, config): - self.project = project + super(RemoteGS, self).__init__(project, config) storagepath = 'gs://' storagepath += config.get(Config.SECTION_AWS_STORAGEPATH, '/') storagepath.lstrip('/') diff --git a/dvc/remote/hdfs.py b/dvc/remote/hdfs.py index 07aae9664..ec1fdf105 100644 --- a/dvc/remote/hdfs.py +++ b/dvc/remote/hdfs.py @@ -20,7 +20,7 @@ class RemoteHDFS(RemoteBase): PARAM_CHECKSUM = 'checksum' def __init__(self, project, config): - self.project = project + super(RemoteHDFS, self).__init__(project, config) self.url = config.get(Config.SECTION_REMOTE_URL, '/') self.prefix = self.url self.user = self.group('user') diff --git a/dvc/remote/http.py b/dvc/remote/http.py index d939121b9..a7b2e2262 100644 --- a/dvc/remote/http.py +++ b/dvc/remote/http.py @@ -30,7 +30,7 @@ class RemoteHTTP(RemoteBase): PARAM_ETAG = 'etag' def __init__(self, project, config): - self.project = project + super(RemoteHTTP, self).__init__(project, config) self.cache_dir = config.get(Config.SECTION_REMOTE_URL) self.url = self.cache_dir self.path_info = {'scheme': 'http'} diff --git a/dvc/remote/local.py b/dvc/remote/local.py index fedb8b863..df6c482da 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -20,7 +20,7 @@ from concurrent.futures import ThreadPoolExecutor class RemoteLOCAL(RemoteBase): scheme = '' - REGEX = r'^(?P(/+|.:\\+).*)$' + REGEX = r'^(?P.*)$' PARAM_CHECKSUM = 'md5' PARAM_PATH = 'path' PARAM_RELPATH = 'relpath' @@ -35,12 +35,16 @@ class RemoteLOCAL(RemoteBase): } def __init__(self, project, config): - self.project = project - self.state = self.project.state + super(RemoteLOCAL, self).__init__(project, config) + self.state = self.project.state if self.project else None self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False) storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None) self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath) + if self.cache_dir is not None and not os.path.isabs(self.cache_dir): + cwd = config[Config.PRIVATE_CWD] + self.cache_dir = os.path.abspath(os.path.join(cwd, self.cache_dir)) + types = config.get(Config.SECTION_CACHE_TYPE, None) if types: if isinstance(types, str): diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 578d50f31..621895894 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -38,7 +38,7 @@ class RemoteS3(RemoteBase): PARAM_CHECKSUM = 'etag' def __init__(self, project, config): - self.project = project + super(RemoteS3, self).__init__(project, config) storagepath = 's3://{}'.format( config.get(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/') diff --git a/dvc/remote/ssh.py b/dvc/remote/ssh.py index 7c98243e9..db349065b 100644 --- a/dvc/remote/ssh.py +++ b/dvc/remote/ssh.py @@ -57,7 +57,7 @@ class RemoteSSH(RemoteBase): TIMEOUT = 1800 def __init__(self, project, config): - self.project = project + super(RemoteSSH, self).__init__(project, config) self.url = config.get(Config.SECTION_REMOTE_URL, '/') self.host = self.group('host') self.user = self.group('user') diff --git a/tests/__main__.py b/tests/__main__.py index b49dfcb25..e44c95c4b 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -10,5 +10,5 @@ os.putenv('DVC_HOME', REPO_ROOT) os.putenv('DVC_TEST', 'true') cmd = 'nosetests -v --processes=-1 --process-timeout=500 --cover-inclusive ' \ - '--cover-erase --cover-package=dvc --with-coverage' + '--cover-erase --cover-package=dvc --with-coverage --all-modules' check_call(cmd, shell=True) diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index 66da97551..aba8e40cf 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -8,7 +8,7 @@ import getpass import platform from dvc.main import main -from dvc.config import Config, ConfigError +from dvc.config import Config from dvc.data_cloud import (DataCloud, RemoteS3, RemoteGS, RemoteAzure, RemoteLOCAL, RemoteSSH, RemoteHDFS, RemoteHTTP) from dvc.remote.base import STATUS_OK, STATUS_NEW, STATUS_DELETED @@ -177,17 +177,6 @@ class TestDataCloud(TestDvc): config[TEST_SECTION][Config.SECTION_REMOTE_URL] = remote_url self._test_cloud(config, cl) - def test_unsupported(self): - remote_url = 'notsupportedscheme://a/b' - config = TEST_CONFIG - config[TEST_SECTION][Config.SECTION_REMOTE_URL] = remote_url - - # NOTE: making sure that error doesn't raise too early - cloud = DataCloud(self.dvc, config=config) - - with self.assertRaises(ConfigError): - cloud._cloud - class TestDataCloudBase(TestDvc): def _get_cloud_class(self): diff --git a/tests/test_remote.py b/tests/test_remote.py index 3762fa736..2b7e148ce 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -3,8 +3,6 @@ import configobj from dvc.main import main from dvc.command.remote import CmdRemoteAdd -from dvc.exceptions import UnsupportedRemoteError -from dvc.remote import Remote from dvc.config import Config from tests.basic_env import TestDvc @@ -45,9 +43,16 @@ class TestRemote(TestDvc): ret = cmd.run() self.assertNotEqual(ret, 0) - def test_unsupported(self): - with self.assertRaises(UnsupportedRemoteError): - Remote(self.dvc, {Config.SECTION_REMOTE_URL: 'unsupported://url'}) + def test_relative_path(self): + dname = os.path.join('..', 'path', 'to', 'dir') + ret = main(['remote', 'add', 'mylocal', dname]) + self.assertEqual(ret, 0) + + # NOTE: we are in the project's root and config is in .dvc/, so + # dir path written to config should be just one level above. + rel = os.path.join('..', dname) + config = configobj.ConfigObj(self.dvc.config.config_file) + self.assertEqual(config['remote "mylocal"']['url'], rel) class TestRemoteRemoveDefault(TestDvc): diff --git a/tests/unit/remote/base.py b/tests/unit/remote/base.py new file mode 100644 index 000000000..68bc343b8 --- /dev/null +++ b/tests/unit/remote/base.py @@ -0,0 +1,15 @@ +import mock +from unittest import TestCase + +from dvc.remote.base import RemoteBase, RemoteMissingDepsError +from dvc.remote import REMOTES, RemoteLOCAL + + +class TestMissingDeps(TestCase): + def test(self): + remotes = REMOTES + [RemoteLOCAL, RemoteBase] + for remote_class in remotes: + REQUIRES = {'foo': None, 'bar': None, 'mock': mock} + with mock.patch.object(remote_class, "REQUIRES", REQUIRES): + with self.assertRaises(RemoteMissingDepsError): + remote_class(None, {})