diff --git a/dvc/cache.py b/dvc/cache.py index e1b9d7b5d..c834ed0ae 100644 --- a/dvc/cache.py +++ b/dvc/cache.py @@ -7,6 +7,20 @@ import os from dvc.config import Config +class CacheConfig(object): + def __init__(self, config): + self.config = config + + def set_dir(self, dname, level=None): + from dvc.remote.config import RemoteConfig + + configobj = self.config.get_configobj(level) + path = RemoteConfig.resolve_path(dname, configobj.filename) + self.config.set( + Config.SECTION_CACHE, Config.SECTION_CACHE_DIR, path, level=level + ) + + class Cache(object): """Class that manages cache locations of a dvc repo. @@ -42,7 +56,7 @@ class Cache(object): Config.SECTION_CACHE_PROTECTED: protected, } - self.local = Remote(repo, settings) + self.local = Remote(repo, **settings) self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3) self.gs = self._get_remote(config, Config.SECTION_CACHE_GS) self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH) @@ -87,5 +101,4 @@ class Cache(object): if not remote: return None - settings = self.repo.config.get_remote_settings(remote) - return Remote(self.repo, settings) + return Remote(self.repo, name=remote) diff --git a/dvc/command/cache.py b/dvc/command/cache.py index 80b189cae..b774aca8b 100644 --- a/dvc/command/cache.py +++ b/dvc/command/cache.py @@ -3,18 +3,18 @@ from __future__ import unicode_literals import argparse from dvc.command.base import fix_subparsers, append_doc_link -from dvc.command.remote import CmdRemoteAdd from dvc.command.config import CmdConfig +from dvc.cache import CacheConfig class CmdCacheDir(CmdConfig): - def run(self): - self.args.name = "cache.dir" - self.args.value = CmdRemoteAdd.resolve_path( - self.args.value, self.configobj.filename - ) + def __init__(self, args): + super(CmdCacheDir, self).__init__(args) + self.config = CacheConfig(self.config) - return super(CmdCacheDir, self).run() + def run(self): + self.config.set_dir(self.args.value, level=self.args.level) + return 0 def add_parser(subparsers, parent_parser): diff --git a/dvc/command/config.py b/dvc/command/config.py index f62ac5d7d..8e3bdba40 100644 --- a/dvc/command/config.py +++ b/dvc/command/config.py @@ -1,12 +1,10 @@ from __future__ import unicode_literals -import os import argparse import logging from dvc.command.base import CmdBaseNoRepo, append_doc_link from dvc.config import Config -from dvc.exceptions import DvcException logger = logging.getLogger(__name__) @@ -14,87 +12,48 @@ logger = logging.getLogger(__name__) class CmdConfig(CmdBaseNoRepo): def __init__(self, args): - from dvc.repo import Repo, NotDvcRepoError - super(CmdConfig, self).__init__(args) - try: - dvc_dir = os.path.join(Repo.find_root(), Repo.DVC_DIR) - saved_exc = None - except NotDvcRepoError as exc: - dvc_dir = None - saved_exc = exc - - self.config = Config(dvc_dir, validate=False) - if self.args.system: - self.configobj = self.config._system_config - elif self.args.glob: - self.configobj = self.config._global_config - elif self.args.local: - if dvc_dir is None: - raise saved_exc - self.configobj = self.config._local_config - else: - if dvc_dir is None: - raise saved_exc - self.configobj = self.config._repo_config - - def _unset(self, section, opt=None, configobj=None): - if configobj is None: - configobj = self.configobj - - try: - self.config.unset(configobj, section, opt) - self.config.save(configobj) - except DvcException: - logger.exception("failed to unset '{}'".format(self.args.name)) - return 1 - return 0 - - def _show(self, section, opt): - try: - self.config.show(self.configobj, section, opt) - except DvcException: - logger.exception("failed to show '{}'".format(self.args.name)) - return 1 - return 0 - - def _set(self, section, opt, value): - try: - self.config.set(self.configobj, section, opt, value) - self.config.save(self.configobj) - except DvcException: - logger.exception( - "failed to set '{}.{}' to '{}'".format(section, opt, value) - ) - return 1 - return 0 + self.config = Config(validate=False) def run(self): section, opt = self.args.name.lower().strip().split(".", 1) if self.args.unset: - return self._unset(section, opt) + self.config.unset(section, opt, level=self.args.level) elif self.args.value is None: - return self._show(section, opt) + logger.info(self.config.get(section, opt, level=self.args.level)) else: - return self._set(section, opt, self.args.value) + self.config.set( + section, opt, self.args.value, level=self.args.level + ) + + return 0 parent_config_parser = argparse.ArgumentParser(add_help=False) parent_config_parser.add_argument( "--global", - dest="glob", - action="store_true", - default=False, + dest="level", + action="store_const", + const=Config.LEVEL_GLOBAL, help="Use global config.", ) parent_config_parser.add_argument( - "--system", action="store_true", default=False, help="Use system config." + "--system", + dest="level", + action="store_const", + const=Config.LEVEL_SYSTEM, + help="Use system config.", ) parent_config_parser.add_argument( - "--local", action="store_true", default=False, help="Use local config." + "--local", + dest="level", + action="store_const", + const=Config.LEVEL_LOCAL, + help="Use local config.", ) +parent_config_parser.set_defaults(level=Config.LEVEL_REPO) def add_parser(subparsers, parent_parser): diff --git a/dvc/command/remote.py b/dvc/command/remote.py index 803592ef4..df1251d0e 100644 --- a/dvc/command/remote.py +++ b/dvc/command/remote.py @@ -1,130 +1,62 @@ from __future__ import unicode_literals import argparse -import os -import re import logging from dvc.command.base import append_doc_link, fix_subparsers from dvc.command.config import CmdConfig -from dvc.config import Config - +from dvc.remote.config import RemoteConfig logger = logging.getLogger(__name__) -class CmdRemoteAdd(CmdConfig): - @staticmethod - def resolve_path(path, config_file): - """Resolve path relative to config file location. +class CmdRemoteConfig(CmdConfig): + def __init__(self, args): + super(CmdRemoteConfig, self).__init__(args) + self.config = RemoteConfig(self.config) - Args: - path: Path to be resolved. - config_file: Path to config file, which `path` is specified - relative to. - - Returns: - Path relative to the `config_file` location. If `path` is an - absolute path then it will be returned without change. - - """ - if os.path.isabs(path): - return path - return os.path.relpath(path, os.path.dirname(config_file)) +class CmdRemoteAdd(CmdRemoteConfig): def run(self): - from dvc.remote import _get, RemoteLOCAL - - remote = _get({Config.SECTION_REMOTE_URL: self.args.url}) - if remote == RemoteLOCAL and not self.args.url.startswith("remote://"): - self.args.url = self.resolve_path( - self.args.url, self.configobj.filename - ) - - section = Config.SECTION_REMOTE_FMT.format(self.args.name) - if (section in self.configobj.keys()) and not self.args.force: - logger.error( - "Remote with name {} already exists. " - "Use -f (--force) to overwrite remote " - "with new value".format(self.args.name) - ) - return 1 - - ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url) - if ret != 0: - return ret - - if self.args.default: - msg = "Setting '{}' as a default remote.".format(self.args.name) - logger.info(msg) - ret = self._set( - Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, self.args.name - ) - - return ret - - -class CmdRemoteRemove(CmdConfig): - def _remove_default(self, config): - core = config.get(Config.SECTION_CORE, None) - if core is None: - return 0 - - default = core.get(Config.SECTION_CORE_REMOTE, None) - if default is None: - return 0 - - if default == self.args.name: - return self._unset( - Config.SECTION_CORE, - opt=Config.SECTION_CORE_REMOTE, - configobj=config, - ) - - def run(self): - section = Config.SECTION_REMOTE_FMT.format(self.args.name) - ret = self._unset(section) - if ret != 0: - return ret - - for configobj in [ - self.config._local_config, - self.config._repo_config, - self.config._global_config, - self.config._system_config, - ]: - self._remove_default(configobj) - self.config.save(configobj) - if configobj == self.configobj: - break - + self.config.add( + self.args.name, + self.args.url, + force=self.args.force, + default=self.args.default, + level=self.args.level, + ) return 0 -class CmdRemoteModify(CmdConfig): +class CmdRemoteRemove(CmdRemoteConfig): def run(self): - section = Config.SECTION_REMOTE_FMT.format(self.args.name) - self.args.name = "{}.{}".format(section, self.args.option) - return super(CmdRemoteModify, self).run() + self.config.remove(self.args.name, level=self.args.level) + return 0 -class CmdRemoteDefault(CmdConfig): +class CmdRemoteModify(CmdRemoteConfig): def run(self): - self.args.value = self.args.name - self.args.name = "core.remote" - return super(CmdRemoteDefault, self).run() + self.config.modify( + self.args.name, + self.args.option, + self.args.value, + level=self.args.level, + ) + return 0 -class CmdRemoteList(CmdConfig): +class CmdRemoteDefault(CmdRemoteConfig): def run(self): - for section in self.configobj.keys(): - r = re.match(Config.SECTION_REMOTE_REGEX, section) - if r: - name = r.group("name") - url = self.configobj[section].get( - Config.SECTION_REMOTE_URL, "" - ) - logger.info("{}\t{}".format(name, url)) + self.config.set_default( + self.args.name, unset=self.args.unset, level=self.args.level + ) + return 0 + + +class CmdRemoteList(CmdRemoteConfig): + def run(self): + for name, url in self.config.list(level=self.args.level).items(): + logger.info("{}\t{}".format(name, url)) return 0 diff --git a/dvc/config.py b/dvc/config.py index 40e705e27..8222bf093 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -2,15 +2,17 @@ from __future__ import unicode_literals -from dvc.utils.compat import str, open, urlparse +from dvc.utils.compat import str, open import os +import re +import copy import errno import configobj import logging -from schema import Schema, Optional, And, Use, Regex -from dvc.exceptions import DvcException +from schema import Schema, Optional, And, Use, Regex, SchemaError +from dvc.exceptions import DvcException, NotDvcRepoError logger = logging.getLogger(__name__) @@ -20,12 +22,12 @@ class ConfigError(DvcException): Args: msg (str): error message. - ex (Exception): optional exception that has caused this error. + cause (Exception): optional exception that has caused this error. """ - def __init__(self, msg, ex=None): + def __init__(self, msg, cause=None): super(ConfigError, self).__init__( - "config file error: {}".format(msg), ex + "config file error: {}".format(msg), cause=cause ) @@ -134,6 +136,11 @@ class Config(object): # pylint: disable=too-many-instance-attributes CONFIG = "config" CONFIG_LOCAL = "config.local" + LEVEL_LOCAL = 0 + LEVEL_REPO = 1 + LEVEL_GLOBAL = 2 + LEVEL_SYSTEM = 3 + BOOL_SCHEMA = And(str, is_bool, Use(to_bool)) SECTION_CORE = "core" @@ -285,30 +292,20 @@ class Config(object): # pylint: disable=too-many-instance-attributes } def __init__(self, dvc_dir=None, validate=True): - self.system_config_file = os.path.join( - self.get_system_config_dir(), self.CONFIG - ) - self.global_config_file = os.path.join( - self.get_global_config_dir(), self.CONFIG - ) + self.dvc_dir = dvc_dir + self.validate = validate - if dvc_dir is not None: - self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir)) - self.config_file = os.path.join(dvc_dir, self.CONFIG) - self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL) + if not dvc_dir: + try: + from dvc.repo import Repo + + self.dvc_dir = os.path.join(Repo.find_dvc_dir()) + except NotDvcRepoError: + self.dvc_dir = None else: - self.dvc_dir = None - self.config_file = None - self.config_local_file = None + self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir)) - self._system_config = None - self._global_config = None - self._repo_config = None - self._local_config = None - - self.config = None - - self.load(validate=validate) + self.load() @staticmethod def get_global_config_dir(): @@ -350,35 +347,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes open(config_file, "w+").close() return Config(dvc_dir) - def _load(self): - self._system_config = configobj.ConfigObj(self.system_config_file) - self._global_config = configobj.ConfigObj(self.global_config_file) - - if self.config_file is not None: - self._repo_config = configobj.ConfigObj(self.config_file) - else: - self._repo_config = configobj.ConfigObj() - - if self.config_local_file is not None: - self._local_config = configobj.ConfigObj(self.config_local_file) - else: - self._local_config = configobj.ConfigObj() - - self.config = None - - def _load_config(self, path): - config = configobj.ConfigObj(path) - config = self._lower(config) - self._resolve_paths(config, path) - return config - - @staticmethod - def _resolve_path(path, config_file): - assert os.path.isabs(config_file) - config_dir = os.path.dirname(config_file) - return os.path.abspath(os.path.join(config_dir, path)) - - def _resolve_cache_path(self, config, fname): + def _resolve_cache_path(self, config): cache = config.get(self.SECTION_CACHE) if cache is None: return @@ -387,72 +356,86 @@ class Config(object): # pylint: disable=too-many-instance-attributes if cache_dir is None: return - cache[self.PRIVATE_CWD] = os.path.dirname(fname) + cache[self.PRIVATE_CWD] = os.path.dirname(config.filename) - def _resolve_paths(self, config, fname): - if fname is None: - return + def _resolve_paths(self, config): + if config.filename is None: + return config - self._resolve_cache_path(config, fname) - for section in config.values(): + ret = copy.deepcopy(config) + self._resolve_cache_path(ret) + + for section in ret.values(): if self.SECTION_REMOTE_URL not in section.keys(): continue - section[self.PRIVATE_CWD] = os.path.dirname(fname) + section[self.PRIVATE_CWD] = os.path.dirname(ret.filename) - def load(self, validate=True): + return ret + + def _load_configs(self): + system_config_file = os.path.join( + self.get_system_config_dir(), self.CONFIG + ) + + global_config_file = os.path.join( + self.get_global_config_dir(), self.CONFIG + ) + + self._system_config = configobj.ConfigObj(system_config_file) + self._global_config = configobj.ConfigObj(global_config_file) + self._repo_config = configobj.ConfigObj() + self._local_config = configobj.ConfigObj() + + if not self.dvc_dir: + return + + config_file = os.path.join(self.dvc_dir, self.CONFIG) + config_local_file = os.path.join(self.dvc_dir, self.CONFIG_LOCAL) + + self._repo_config = configobj.ConfigObj(config_file) + self._local_config = configobj.ConfigObj(config_local_file) + + @property + def config_local_file(self): + return self._local_config.filename + + @property + def config_file(self): + return self._repo_config.filename + + def load(self): """Loads config from all the config files. - Args: - validate (bool): optional flag to tell dvc if it should validate - the config or just load it as is. 'True' by default. - - Raises: dvc.config.ConfigError: thrown if config has invalid format. """ - self._load() + self._load_configs() + + self.config = configobj.ConfigObj() + for c in [ + self._system_config, + self._global_config, + self._repo_config, + self._local_config, + ]: + c = self._resolve_paths(c) + c = self._lower(c) + self.config.merge(c) + + if not self.validate: + return + + d = self.config.dict() try: - self.config = self._load_config(self.system_config_file) - user = self._load_config(self.global_config_file) - config = self._load_config(self.config_file) - local = self._load_config(self.config_local_file) - - # NOTE: schema doesn't support ConfigObj.Section validation, so we - # need to convert our config to dict before passing it to - for conf in [user, config, local]: - self.config = self._merge(self.config, conf) - - if validate: - self.config = Schema(self.SCHEMA).validate(self.config) - - # NOTE: now converting back to ConfigObj - self.config = configobj.ConfigObj( - self.config, write_empty_values=True - ) - self.config.filename = self.config_file - self._resolve_paths(self.config, self.config_file) - except Exception as ex: - raise ConfigError(ex) - - @staticmethod - def _get_key(conf, name, add=False): - for k in conf.keys(): - if k.lower() == name.lower(): - return k - - if add: - conf[name] = {} - return name - - return None + d = Schema(self.SCHEMA).validate(d) + except SchemaError as exc: + raise ConfigError("config format error", cause=exc) + self.config = configobj.ConfigObj(d, write_empty_values=True) def save(self, config=None): """Saves config to config files. - Args: - config (configobj.ConfigObj): optional config object to save. - Raises: dvc.config.ConfigError: thrown if failed to write config file. """ @@ -467,129 +450,103 @@ class Config(object): # pylint: disable=too-many-instance-attributes ] for conf in clist: - if conf.filename is None: - continue + self._save(conf) - try: - logger.debug("Writing '{}'.".format(conf.filename)) - dname = os.path.dirname(os.path.abspath(conf.filename)) - try: - os.makedirs(dname) - except OSError as exc: - if exc.errno != errno.EEXIST: - raise - conf.write() - except Exception as exc: - msg = "failed to write config '{}'".format(conf.filename) - raise ConfigError(msg, exc) - - def get_remote_settings(self, name): - import posixpath - - """ - Args: - name (str): The name of the remote that we want to retrieve - - Returns: - dict: The content beneath the given remote name. - - Example: - >>> config = {'remote "server"': {'url': 'ssh://localhost/'}} - >>> get_remote_settings("server") - {'url': 'ssh://localhost/'} - """ - settings = self.config.get( - self.SECTION_REMOTE_FMT.format(name.lower()) - ) - - if settings is None: - raise ConfigError( - "unable to find remote section '{}'".format(name) - ) - - parsed = urlparse(settings["url"]) - - # Support for cross referenced remotes. - # This will merge the settings, giving priority to the outer reference. - # For example, having: - # - # dvc remote add server ssh://localhost - # dvc remote modify server user root - # dvc remote modify server ask_password true - # - # dvc remote add images remote://server/tmp/pictures - # dvc remote modify images user alice - # dvc remote modify images ask_password false - # dvc remote modify images password asdf1234 - # - # Results on a config dictionary like: - # - # { - # "url": "ssh://localhost/tmp/pictures", - # "user": "alice", - # "password": "asdf1234", - # "ask_password": False, - # } - # - if parsed.scheme == "remote": - reference = self.get_remote_settings(parsed.netloc) - url = posixpath.join(reference["url"], parsed.path.lstrip("/")) - merged = reference.copy() - merged.update(settings) - merged["url"] = url - return merged - - return settings + self.load() @staticmethod - def unset(config, section, opt=None): + def _save(config): + if config.filename is None: + return + + logger.debug("Writing '{}'.".format(config.filename)) + dname = os.path.dirname(os.path.abspath(config.filename)) + try: + os.makedirs(dname) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + config.write() + + def unset(self, section, opt=None, level=None, force=False): """Unsets specified option and/or section in the config. Args: - config (configobj.ConfigObj): config to work on. section (str): section name. opt (str): optional option name. + level (int): config level to use. + force (bool): don't error-out even if section doesn't exist. False + by default. + + Raises: + dvc.config.ConfigError: thrown if section doesn't exist and + `force != True`. """ + config = self.get_configobj(level) + if section not in config.keys(): + if force: + return raise ConfigError("section '{}' doesn't exist".format(section)) - if opt is None: - del config[section] - return + if opt: + if opt not in config[section].keys(): + if force: + return + raise ConfigError( + "option '{}.{}' doesn't exist".format(section, opt) + ) + del config[section][opt] - if opt not in config[section].keys(): - raise ConfigError( - "option '{}.{}' doesn't exist".format(section, opt) - ) - del config[section][opt] - - if not config[section]: + if not config[section]: + del config[section] + else: del config[section] - @staticmethod - def set(config, section, opt, value): + self.save(config) + + def set(self, section, opt, value, level=None, force=True): """Sets specified option in the config. Args: - config (configobj.ConfigObj): config to work on. section (str): section name. opt (str): option name. value: value to set option to. + level (int): config level to use. + force (bool): set option even if section already exists. True by + default. + + Raises: + dvc.config.ConfigError: thrown if section already exists and + `force != True`. + """ + config = self.get_configobj(level) + if section not in config.keys(): config[section] = {} + elif not force: + raise ConfigError( + "Section '{}' already exists. Use `-f|--force` to overwrite " + "section with new value.".format(section) + ) config[section][opt] = value + self.save(config) - @staticmethod - def show(config, section, opt): - """Prints option value from the config. + def get(self, section, opt=None, level=None): + """Return option value from the config. Args: - config (configobj.ConfigObj): config to work on. section (str): section name. opt (str): option name. + level (int): config level to use. + + Returns: + value (str, int): option value. """ + config = self.get_configobj(level) + if section not in config.keys(): raise ConfigError("section '{}' doesn't exist".format(section)) @@ -598,25 +555,35 @@ class Config(object): # pylint: disable=too-many-instance-attributes "option '{}.{}' doesn't exist".format(section, opt) ) - logger.info(config[section][opt]) - - @staticmethod - def _merge(first, second): - res = {} - sections = list(first.keys()) + list(second.keys()) - for section in sections: - first_copy = first.get(section, {}).copy() - second_copy = second.get(section, {}).copy() - first_copy.update(second_copy) - res[section] = first_copy - return res + return config[section][opt] @staticmethod def _lower(config): - new_config = {} + new_config = configobj.ConfigObj() for s_key, s_value in config.items(): new_s = {} for key, value in s_value.items(): new_s[key.lower()] = str(value) new_config[s_key.lower()] = new_s return new_config + + def get_configobj(self, level): + configs = { + self.LEVEL_LOCAL: self._local_config, + self.LEVEL_REPO: self._repo_config, + self.LEVEL_GLOBAL: self._global_config, + self.LEVEL_SYSTEM: self._system_config, + } + + return configs.get(level, self._repo_config) + + def list_options(self, section_regex, option, level=None): + ret = {} + config = self.get_configobj(level) + for section in config.keys(): + r = re.match(section_regex, section) + if r: + name = r.group("name") + value = config[section].get(option, "") + ret[name] = value + return ret diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 5802826af..d55eb86fd 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -25,7 +25,6 @@ class DataCloud(object): Args: repo (dvc.repo.Repo): repo instance that belongs to the repo that we are working on. - config (dict): config of the repo. Raises: config.ConfigError: thrown when config has invalid format. @@ -75,8 +74,7 @@ class DataCloud(object): ) def _init_remote(self, remote): - config = self.repo.config.get_remote_settings(remote) - return Remote(self.repo, config) + return Remote(self.repo, name=remote) def _init_compat(self): name = self._core.get(Config.SECTION_CORE_CLOUD, "").strip().lower() diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index 5c5ba67d4..78ac2da31 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -52,8 +52,7 @@ def _get(stage, p, info): parsed = urlparse(p) if parsed.scheme == "remote": - settings = stage.repo.config.get_remote_settings(parsed.netloc) - remote = Remote(stage.repo, settings) + remote = Remote(stage.repo, name=parsed.netloc) return DEP_MAP[remote.scheme](stage, p, info, remote=remote) for d in DEPS: diff --git a/dvc/main.py b/dvc/main.py index 12db9e112..f00dc0ced 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -5,6 +5,7 @@ from __future__ import unicode_literals import logging from dvc.cli import parse_args +from dvc.config import ConfigError from dvc.analytics import Analytics from dvc.exceptions import NotDvcRepoError, DvcParserError @@ -36,6 +37,9 @@ def main(argv=None): cmd = args.func(args) ret = cmd.run_cmd() + except ConfigError: + logger.exception("configuration error") + ret = 251 except KeyboardInterrupt: logger.exception("interrupted by the user") ret = 252 diff --git a/dvc/output/__init__.py b/dvc/output/__init__.py index 60db6a50c..ca8ad7777 100644 --- a/dvc/output/__init__.py +++ b/dvc/output/__init__.py @@ -61,8 +61,7 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None): parsed = urlparse(p) if parsed.scheme == "remote": - settings = stage.repo.config.get_remote_settings(parsed.netloc) - remote = Remote(stage.repo, settings) + remote = Remote(stage.repo, name=parsed.netloc) return OUTS_MAP[remote.scheme]( stage, p, diff --git a/dvc/remote/__init__.py b/dvc/remote/__init__.py index 4eb124199..e8ffe81f4 100644 --- a/dvc/remote/__init__.py +++ b/dvc/remote/__init__.py @@ -10,6 +10,8 @@ from dvc.remote.http import RemoteHTTP from dvc.remote.https import RemoteHTTPS from dvc.remote.oss import RemoteOSS +from .config import RemoteConfig + REMOTES = [ RemoteAZURE, @@ -31,5 +33,11 @@ def _get(config): return RemoteLOCAL -def Remote(repo, config): - return _get(config)(repo, config) +def Remote(repo, **kwargs): + name = kwargs.get("name") + if name: + remote_config = RemoteConfig(repo.config) + settings = remote_config.get_settings(name) + else: + settings = kwargs + return _get(settings)(repo, settings) diff --git a/dvc/remote/config.py b/dvc/remote/config.py new file mode 100644 index 000000000..54b33ed2b --- /dev/null +++ b/dvc/remote/config.py @@ -0,0 +1,151 @@ +import os +import logging +import posixpath + +from dvc.utils.compat import urlparse +from dvc.config import Config, ConfigError + + +logger = logging.getLogger(__name__) + + +class RemoteConfig(object): + def __init__(self, config): + self.config = config + + def get_settings(self, name): + """ + Args: + name (str): The name of the remote that we want to retrieve + + Returns: + dict: The content beneath the given remote name. + + Example: + >>> config = {'remote "server"': {'url': 'ssh://localhost/'}} + >>> get_settings("server") + {'url': 'ssh://localhost/'} + """ + settings = self.config.config.get( + Config.SECTION_REMOTE_FMT.format(name.lower()) + ) + + if settings is None: + raise ConfigError( + "unable to find remote section '{}'".format(name) + ) + + parsed = urlparse(settings["url"]) + + # Support for cross referenced remotes. + # This will merge the settings, giving priority to the outer reference. + # For example, having: + # + # dvc remote add server ssh://localhost + # dvc remote modify server user root + # dvc remote modify server ask_password true + # + # dvc remote add images remote://server/tmp/pictures + # dvc remote modify images user alice + # dvc remote modify images ask_password false + # dvc remote modify images password asdf1234 + # + # Results on a config dictionary like: + # + # { + # "url": "ssh://localhost/tmp/pictures", + # "user": "alice", + # "password": "asdf1234", + # "ask_password": False, + # } + # + if parsed.scheme == "remote": + reference = self.get_settings(parsed.netloc) + url = posixpath.join(reference["url"], parsed.path.lstrip("/")) + merged = reference.copy() + merged.update(settings) + merged["url"] = url + return merged + + return settings + + @staticmethod + def resolve_path(path, config_file): + """Resolve path relative to config file location. + + Args: + path: Path to be resolved. + config_file: Path to config file, which `path` is specified + relative to. + + Returns: + Path relative to the `config_file` location. If `path` is an + absolute path then it will be returned without change. + + """ + if os.path.isabs(path): + return path + return os.path.relpath(path, os.path.dirname(config_file)) + + def add(self, name, url, default=False, force=False, level=None): + from dvc.remote import _get, RemoteLOCAL + + configobj = self.config.get_configobj(level) + remote = _get({Config.SECTION_REMOTE_URL: url}) + if remote == RemoteLOCAL and not url.startswith("remote://"): + url = self.resolve_path(url, configobj.filename) + + self.config.set( + Config.SECTION_REMOTE_FMT.format(name), + Config.SECTION_REMOTE_URL, + url, + force=force, + level=level, + ) + if default: + logger.info("Setting '{}' as a default remote.".format(name)) + self.config.set( + Config.SECTION_CORE, + Config.SECTION_CORE_REMOTE, + name, + level=level, + ) + + def remove(self, name, level=None): + self.config.unset(Config.SECTION_REMOTE_FMT.format(name), level=level) + + if level is None: + level = Config.LEVEL_REPO + + for lev in [ + Config.LEVEL_LOCAL, + Config.LEVEL_REPO, + Config.LEVEL_GLOBAL, + Config.LEVEL_SYSTEM, + ]: + self.config.unset( + Config.SECTION_CORE, + Config.SECTION_CORE_REMOTE, + level=lev, + force=True, + ) + if lev == level: + break + + def modify(self, name, option, value, level=None): + self.config.set( + Config.SECTION_REMOTE_FMT.format(name), option, value, level=level + ) + + def list(self, level=None): + return self.config.list_options( + Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL, level=level + ) + + def set_default(self, name, unset=False, level=None): + if unset: + self.config.unset(Config.SECTION_CORE, Config.SECTION_CORE_REMOTE) + return + self.config.set( + Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, name, level=level + ) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 0f43957c1..101e9c300 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -5,6 +5,7 @@ import logging import dvc.prompt as prompt +from dvc.config import Config from dvc.exceptions import ( DvcException, NotDvcRepoError, @@ -41,7 +42,6 @@ class Repo(object): from dvc.repo.brancher import brancher def __init__(self, root_dir=None): - from dvc.config import Config from dvc.state import State from dvc.lock import Lock from dvc.scm import SCM @@ -89,15 +89,15 @@ class Repo(object): def __repr__(self): return "Repo: '{root_dir}'".format(root_dir=self.root_dir) - @staticmethod - def find_root(root=None): + @classmethod + def find_root(cls, root=None): if root is None: root = os.getcwd() else: root = os.path.abspath(os.path.realpath(root)) while True: - dvc_dir = os.path.join(root, Repo.DVC_DIR) + dvc_dir = os.path.join(root, cls.DVC_DIR) if os.path.isdir(dvc_dir): return root if os.path.ismount(root): @@ -105,10 +105,10 @@ class Repo(object): root = os.path.dirname(root) raise NotDvcRepoError(root) - @staticmethod - def find_dvc_dir(root=None): - root_dir = Repo.find_root(root) - return os.path.join(root_dir, Repo.DVC_DIR) + @classmethod + def find_dvc_dir(cls, root=None): + root_dir = cls.find_root(root) + return os.path.join(root_dir, cls.DVC_DIR) @staticmethod def init(root_dir=os.curdir, no_scm=False, force=False): diff --git a/tests/basic_env.py b/tests/basic_env.py index 0c0b7ab70..8fc20e08c 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -57,6 +57,10 @@ class TestDirFixture(object): root_dir = self.mkdtemp() self._root_dir = os.path.realpath(root_dir) + @property + def root_dir(self): + return self._root_dir + def _pushd(self, d): if not hasattr(self, "_saved_dir"): self._saved_dir = os.path.realpath(os.curdir) @@ -193,9 +197,8 @@ class TestDvcDataFileFixture(TestDvcGitInitializedFixture): shutil.copytree(self.dvc.cache.local.cache_dir, self.cache_dir) class MockConfig: - system = None - glob = None - local = None + level = 1 + force = False default = True name = self.REMOTE url = self.cache_dir diff --git a/tests/func/test_config.py b/tests/func/test_config.py index 0de3bafd9..f6cae78d3 100644 --- a/tests/func/test_config.py +++ b/tests/func/test_config.py @@ -1,7 +1,6 @@ import configobj from dvc.main import main -from dvc.command.config import CmdConfig from tests.basic_env import TestDvc @@ -67,38 +66,20 @@ class TestConfigCLI(TestDvc): self._do_test(True) def test_non_existing(self): - # FIXME check stdout/err - ret = main(["config", "non_existing_section.field"]) - self.assertEqual(ret, 1) + self.assertEqual(ret, 251) ret = main(["config", "global.non_existing_field"]) - self.assertEqual(ret, 1) + self.assertEqual(ret, 251) ret = main(["config", "non_existing_section.field", "-u"]) - self.assertEqual(ret, 1) + self.assertEqual(ret, 251) ret = main(["config", "global.non_existing_field", "-u"]) - self.assertEqual(ret, 1) + self.assertEqual(ret, 251) ret = main(["config", "core.remote", "myremote"]) self.assertEqual(ret, 0) ret = main(["config", "core.non_existing_field", "-u"]) - self.assertEqual(ret, 1) - - def test_failed_write(self): - class A(object): - system = False - glob = False - local = False - name = "core.remote" - value = "myremote" - unset = False - - args = A() - cmd = CmdConfig(args) - - cmd.configobj.write = None - ret = cmd.run() - self.assertNotEqual(ret, 0) + self.assertEqual(ret, 251) diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index 987b7029e..97daa1a91 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -39,6 +39,7 @@ from tests.utils import spy TEST_REMOTE = "upstream" TEST_SECTION = 'remote "{}"'.format(TEST_REMOTE) TEST_CONFIG = { + Config.SECTION_CACHE: {}, Config.SECTION_CORE: {Config.SECTION_CORE_REMOTE: TEST_REMOTE}, TEST_SECTION: {Config.SECTION_REMOTE_URL: ""}, } diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index fd403cc1b..9f1b07d59 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -5,7 +5,6 @@ from dvc.remote.base import RemoteBASE from mock import patch from dvc.main import main -from dvc.command.remote import CmdRemoteAdd from dvc.config import Config from tests.basic_env import TestDvc @@ -33,22 +32,6 @@ class TestRemote(TestDvc): self.assertEqual(main(["remote", "list"]), 0) - def test_failed_write(self): - class A(object): - system = False - glob = False - local = False - name = "myremote" - url = "s3://remote" - unset = False - - args = A() - cmd = CmdRemoteAdd(args) - - cmd.configobj.write = None - ret = cmd.run() - self.assertNotEqual(ret, 0) - def test_relative_path(self): dname = os.path.join("..", "path", "to", "dir") ret = main(["remote", "add", "mylocal", dname]) @@ -64,7 +47,7 @@ class TestRemote(TestDvc): remote_name = "a" remote_url = "s3://bucket/name" self.assertEqual(main(["remote", "add", remote_name, remote_url]), 0) - self.assertEqual(main(["remote", "add", remote_name, remote_url]), 1) + self.assertEqual(main(["remote", "add", remote_name, remote_url]), 251) self.assertEqual( main(["remote", "add", "-f", remote_name, remote_url]), 0 )