Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2098 from efiop/config

Browse Source 
refactoring: move config logic from CLI to API
This commit is contained in:
Ruslan Kuprieiev
2019-06-06 21:53:53 +03:00
committed by GitHub
parent f41dcea248 8ef9133194
commit 1e7e2aaed5
16 changed files with 446 additions and 448 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/cache.py
+16 -3
View File
@@ -7,6 +7,20 @@ import os
from dvc.config import Config
class CacheConfig(object):
def __init__(self, config):
self.config = config
def set_dir(self, dname, level=None):
from dvc.remote.config import RemoteConfig
configobj = self.config.get_configobj(level)
path = RemoteConfig.resolve_path(dname, configobj.filename)
self.config.set(
Config.SECTION_CACHE, Config.SECTION_CACHE_DIR, path, level=level
)
class Cache(object):
"""Class that manages cache locations of a dvc repo.
@@ -42,7 +56,7 @@ class Cache(object):
Config.SECTION_CACHE_PROTECTED: protected,
}
self.local = Remote(repo, settings)
self.local = Remote(repo, **settings)
self.s3 = self._get_remote(config, Config.SECTION_CACHE_S3)
self.gs = self._get_remote(config, Config.SECTION_CACHE_GS)
self.ssh = self._get_remote(config, Config.SECTION_CACHE_SSH)
@@ -87,5 +101,4 @@ class Cache(object):
if not remote:
return None
settings = self.repo.config.get_remote_settings(remote)
return Remote(self.repo, settings)
return Remote(self.repo, name=remote)
dvc/command/cache.py
+7 -7
View File
@@ -3,18 +3,18 @@ from __future__ import unicode_literals
import argparse
from dvc.command.base import fix_subparsers, append_doc_link
from dvc.command.remote import CmdRemoteAdd
from dvc.command.config import CmdConfig
from dvc.cache import CacheConfig
class CmdCacheDir(CmdConfig):
def run(self):
self.args.name = "cache.dir"
self.args.value = CmdRemoteAdd.resolve_path(
self.args.value, self.configobj.filename
)
def __init__(self, args):
super(CmdCacheDir, self).__init__(args)
self.config = CacheConfig(self.config)
return super(CmdCacheDir, self).run()
def run(self):
self.config.set_dir(self.args.value, level=self.args.level)
return 0
def add_parser(subparsers, parent_parser):
dvc/command/config.py
+22 -63
View File
@@ -1,12 +1,10 @@
from __future__ import unicode_literals
import os
import argparse
import logging
from dvc.command.base import CmdBaseNoRepo, append_doc_link
from dvc.config import Config
from dvc.exceptions import DvcException
logger = logging.getLogger(__name__)
@@ -14,87 +12,48 @@ logger = logging.getLogger(__name__)
class CmdConfig(CmdBaseNoRepo):
def __init__(self, args):
from dvc.repo import Repo, NotDvcRepoError
super(CmdConfig, self).__init__(args)
try:
dvc_dir = os.path.join(Repo.find_root(), Repo.DVC_DIR)
saved_exc = None
except NotDvcRepoError as exc:
dvc_dir = None
saved_exc = exc
self.config = Config(dvc_dir, validate=False)
if self.args.system:
self.configobj = self.config._system_config
elif self.args.glob:
self.configobj = self.config._global_config
elif self.args.local:
if dvc_dir is None:
raise saved_exc
self.configobj = self.config._local_config
else:
if dvc_dir is None:
raise saved_exc
self.configobj = self.config._repo_config
def _unset(self, section, opt=None, configobj=None):
if configobj is None:
configobj = self.configobj
try:
self.config.unset(configobj, section, opt)
self.config.save(configobj)
except DvcException:
logger.exception("failed to unset '{}'".format(self.args.name))
return 1
return 0
def _show(self, section, opt):
try:
self.config.show(self.configobj, section, opt)
except DvcException:
logger.exception("failed to show '{}'".format(self.args.name))
return 1
return 0
def _set(self, section, opt, value):
try:
self.config.set(self.configobj, section, opt, value)
self.config.save(self.configobj)
except DvcException:
logger.exception(
"failed to set '{}.{}' to '{}'".format(section, opt, value)
)
return 1
return 0
self.config = Config(validate=False)
def run(self):
section, opt = self.args.name.lower().strip().split(".", 1)
if self.args.unset:
return self._unset(section, opt)
self.config.unset(section, opt, level=self.args.level)
elif self.args.value is None:
return self._show(section, opt)
logger.info(self.config.get(section, opt, level=self.args.level))
else:
return self._set(section, opt, self.args.value)
self.config.set(
section, opt, self.args.value, level=self.args.level
)
return 0
parent_config_parser = argparse.ArgumentParser(add_help=False)
parent_config_parser.add_argument(
"--global",
dest="glob",
action="store_true",
default=False,
dest="level",
action="store_const",
const=Config.LEVEL_GLOBAL,
help="Use global config.",
)
parent_config_parser.add_argument(
"--system", action="store_true", default=False, help="Use system config."
"--system",
dest="level",
action="store_const",
const=Config.LEVEL_SYSTEM,
help="Use system config.",
)
parent_config_parser.add_argument(
"--local", action="store_true", default=False, help="Use local config."
"--local",
dest="level",
action="store_const",
const=Config.LEVEL_LOCAL,
help="Use local config.",
)
parent_config_parser.set_defaults(level=Config.LEVEL_REPO)
def add_parser(subparsers, parent_parser):
dvc/command/remote.py
+35 -103
View File
@@ -1,130 +1,62 @@
from __future__ import unicode_literals
import argparse
import os
import re
import logging
from dvc.command.base import append_doc_link, fix_subparsers
from dvc.command.config import CmdConfig
from dvc.config import Config
from dvc.remote.config import RemoteConfig
logger = logging.getLogger(__name__)
class CmdRemoteAdd(CmdConfig):
@staticmethod
def resolve_path(path, config_file):
"""Resolve path relative to config file location.
class CmdRemoteConfig(CmdConfig):
def __init__(self, args):
super(CmdRemoteConfig, self).__init__(args)
self.config = RemoteConfig(self.config)
Args:
path: Path to be resolved.
config_file: Path to config file, which `path` is specified
relative to.
Returns:
Path relative to the `config_file` location. If `path` is an
absolute path then it will be returned without change.
"""
if os.path.isabs(path):
return path
return os.path.relpath(path, os.path.dirname(config_file))
class CmdRemoteAdd(CmdRemoteConfig):
def run(self):
from dvc.remote import _get, RemoteLOCAL
remote = _get({Config.SECTION_REMOTE_URL: self.args.url})
if remote == RemoteLOCAL and not self.args.url.startswith("remote://"):
self.args.url = self.resolve_path(
self.args.url, self.configobj.filename
)
section = Config.SECTION_REMOTE_FMT.format(self.args.name)
if (section in self.configobj.keys()) and not self.args.force:
logger.error(
"Remote with name {} already exists. "
"Use -f (--force) to overwrite remote "
"with new value".format(self.args.name)
)
return 1
ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url)
if ret != 0:
return ret
if self.args.default:
msg = "Setting '{}' as a default remote.".format(self.args.name)
logger.info(msg)
ret = self._set(
Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, self.args.name
)
return ret
class CmdRemoteRemove(CmdConfig):
def _remove_default(self, config):
core = config.get(Config.SECTION_CORE, None)
if core is None:
return 0
default = core.get(Config.SECTION_CORE_REMOTE, None)
if default is None:
return 0
if default == self.args.name:
return self._unset(
Config.SECTION_CORE,
opt=Config.SECTION_CORE_REMOTE,
configobj=config,
)
def run(self):
section = Config.SECTION_REMOTE_FMT.format(self.args.name)
ret = self._unset(section)
if ret != 0:
return ret
for configobj in [
self.config._local_config,
self.config._repo_config,
self.config._global_config,
self.config._system_config,
]:
self._remove_default(configobj)
self.config.save(configobj)
if configobj == self.configobj:
break
self.config.add(
self.args.name,
self.args.url,
force=self.args.force,
default=self.args.default,
level=self.args.level,
)
return 0
class CmdRemoteModify(CmdConfig):
class CmdRemoteRemove(CmdRemoteConfig):
def run(self):
section = Config.SECTION_REMOTE_FMT.format(self.args.name)
self.args.name = "{}.{}".format(section, self.args.option)
return super(CmdRemoteModify, self).run()
self.config.remove(self.args.name, level=self.args.level)
return 0
class CmdRemoteDefault(CmdConfig):
class CmdRemoteModify(CmdRemoteConfig):
def run(self):
self.args.value = self.args.name
self.args.name = "core.remote"
return super(CmdRemoteDefault, self).run()
self.config.modify(
self.args.name,
self.args.option,
self.args.value,
level=self.args.level,
)
return 0
class CmdRemoteList(CmdConfig):
class CmdRemoteDefault(CmdRemoteConfig):
def run(self):
for section in self.configobj.keys():
r = re.match(Config.SECTION_REMOTE_REGEX, section)
if r:
name = r.group("name")
url = self.configobj[section].get(
Config.SECTION_REMOTE_URL, ""
)
logger.info("{}\t{}".format(name, url))
self.config.set_default(
self.args.name, unset=self.args.unset, level=self.args.level
)
return 0
class CmdRemoteList(CmdRemoteConfig):
def run(self):
for name, url in self.config.list(level=self.args.level).items():
logger.info("{}\t{}".format(name, url))
return 0
dvc/config.py
+177 -210
View File
@@ -2,15 +2,17 @@
from __future__ import unicode_literals
from dvc.utils.compat import str, open, urlparse
from dvc.utils.compat import str, open
import os
import re
import copy
import errno
import configobj
import logging
from schema import Schema, Optional, And, Use, Regex
from dvc.exceptions import DvcException
from schema import Schema, Optional, And, Use, Regex, SchemaError
from dvc.exceptions import DvcException, NotDvcRepoError
logger = logging.getLogger(__name__)
@@ -20,12 +22,12 @@ class ConfigError(DvcException):
Args:
msg (str): error message.
ex (Exception): optional exception that has caused this error.
cause (Exception): optional exception that has caused this error.
"""
def __init__(self, msg, ex=None):
def __init__(self, msg, cause=None):
super(ConfigError, self).__init__(
"config file error: {}".format(msg), ex
"config file error: {}".format(msg), cause=cause
)
@@ -134,6 +136,11 @@ class Config(object): # pylint: disable=too-many-instance-attributes
CONFIG = "config"
CONFIG_LOCAL = "config.local"
LEVEL_LOCAL = 0
LEVEL_REPO = 1
LEVEL_GLOBAL = 2
LEVEL_SYSTEM = 3
BOOL_SCHEMA = And(str, is_bool, Use(to_bool))
SECTION_CORE = "core"
@@ -285,30 +292,20 @@ class Config(object): # pylint: disable=too-many-instance-attributes
}
def __init__(self, dvc_dir=None, validate=True):
self.system_config_file = os.path.join(
self.get_system_config_dir(), self.CONFIG
)
self.global_config_file = os.path.join(
self.get_global_config_dir(), self.CONFIG
)
self.dvc_dir = dvc_dir
self.validate = validate
if dvc_dir is not None:
self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
self.config_file = os.path.join(dvc_dir, self.CONFIG)
self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL)
if not dvc_dir:
try:
from dvc.repo import Repo
self.dvc_dir = os.path.join(Repo.find_dvc_dir())
except NotDvcRepoError:
self.dvc_dir = None
else:
self.dvc_dir = None
self.config_file = None
self.config_local_file = None
self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
self._system_config = None
self._global_config = None
self._repo_config = None
self._local_config = None
self.config = None
self.load(validate=validate)
self.load()
@staticmethod
def get_global_config_dir():
@@ -350,35 +347,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes
open(config_file, "w+").close()
return Config(dvc_dir)
def _load(self):
self._system_config = configobj.ConfigObj(self.system_config_file)
self._global_config = configobj.ConfigObj(self.global_config_file)
if self.config_file is not None:
self._repo_config = configobj.ConfigObj(self.config_file)
else:
self._repo_config = configobj.ConfigObj()
if self.config_local_file is not None:
self._local_config = configobj.ConfigObj(self.config_local_file)
else:
self._local_config = configobj.ConfigObj()
self.config = None
def _load_config(self, path):
config = configobj.ConfigObj(path)
config = self._lower(config)
self._resolve_paths(config, path)
return config
@staticmethod
def _resolve_path(path, config_file):
assert os.path.isabs(config_file)
config_dir = os.path.dirname(config_file)
return os.path.abspath(os.path.join(config_dir, path))
def _resolve_cache_path(self, config, fname):
def _resolve_cache_path(self, config):
cache = config.get(self.SECTION_CACHE)
if cache is None:
return
@@ -387,72 +356,86 @@ class Config(object): # pylint: disable=too-many-instance-attributes
if cache_dir is None:
return
cache[self.PRIVATE_CWD] = os.path.dirname(fname)
cache[self.PRIVATE_CWD] = os.path.dirname(config.filename)
def _resolve_paths(self, config, fname):
if fname is None:
return
def _resolve_paths(self, config):
if config.filename is None:
return config
self._resolve_cache_path(config, fname)
for section in config.values():
ret = copy.deepcopy(config)
self._resolve_cache_path(ret)
for section in ret.values():
if self.SECTION_REMOTE_URL not in section.keys():
continue
section[self.PRIVATE_CWD] = os.path.dirname(fname)
section[self.PRIVATE_CWD] = os.path.dirname(ret.filename)
def load(self, validate=True):
return ret
def _load_configs(self):
system_config_file = os.path.join(
self.get_system_config_dir(), self.CONFIG
)
global_config_file = os.path.join(
self.get_global_config_dir(), self.CONFIG
)
self._system_config = configobj.ConfigObj(system_config_file)
self._global_config = configobj.ConfigObj(global_config_file)
self._repo_config = configobj.ConfigObj()
self._local_config = configobj.ConfigObj()
if not self.dvc_dir:
return
config_file = os.path.join(self.dvc_dir, self.CONFIG)
config_local_file = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)
self._repo_config = configobj.ConfigObj(config_file)
self._local_config = configobj.ConfigObj(config_local_file)
@property
def config_local_file(self):
return self._local_config.filename
@property
def config_file(self):
return self._repo_config.filename
def load(self):
"""Loads config from all the config files.
Args:
validate (bool): optional flag to tell dvc if it should validate
the config or just load it as is. 'True' by default.
Raises:
dvc.config.ConfigError: thrown if config has invalid format.
"""
self._load()
self._load_configs()
self.config = configobj.ConfigObj()
for c in [
self._system_config,
self._global_config,
self._repo_config,
self._local_config,
]:
c = self._resolve_paths(c)
c = self._lower(c)
self.config.merge(c)
if not self.validate:
return
d = self.config.dict()
try:
self.config = self._load_config(self.system_config_file)
user = self._load_config(self.global_config_file)
config = self._load_config(self.config_file)
local = self._load_config(self.config_local_file)
# NOTE: schema doesn't support ConfigObj.Section validation, so we
# need to convert our config to dict before passing it to
for conf in [user, config, local]:
self.config = self._merge(self.config, conf)
if validate:
self.config = Schema(self.SCHEMA).validate(self.config)
# NOTE: now converting back to ConfigObj
self.config = configobj.ConfigObj(
self.config, write_empty_values=True
)
self.config.filename = self.config_file
self._resolve_paths(self.config, self.config_file)
except Exception as ex:
raise ConfigError(ex)
@staticmethod
def _get_key(conf, name, add=False):
for k in conf.keys():
if k.lower() == name.lower():
return k
if add:
conf[name] = {}
return name
return None
d = Schema(self.SCHEMA).validate(d)
except SchemaError as exc:
raise ConfigError("config format error", cause=exc)
self.config = configobj.ConfigObj(d, write_empty_values=True)
def save(self, config=None):
"""Saves config to config files.
Args:
config (configobj.ConfigObj): optional config object to save.
Raises:
dvc.config.ConfigError: thrown if failed to write config file.
"""
@@ -467,129 +450,103 @@ class Config(object): # pylint: disable=too-many-instance-attributes
]
for conf in clist:
if conf.filename is None:
continue
self._save(conf)
try:
logger.debug("Writing '{}'.".format(conf.filename))
dname = os.path.dirname(os.path.abspath(conf.filename))
try:
os.makedirs(dname)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
conf.write()
except Exception as exc:
msg = "failed to write config '{}'".format(conf.filename)
raise ConfigError(msg, exc)
def get_remote_settings(self, name):
import posixpath
"""
Args:
name (str): The name of the remote that we want to retrieve
Returns:
dict: The content beneath the given remote name.
Example:
>>> config = {'remote "server"': {'url': 'ssh://localhost/'}}
>>> get_remote_settings("server")
{'url': 'ssh://localhost/'}
"""
settings = self.config.get(
self.SECTION_REMOTE_FMT.format(name.lower())
)
if settings is None:
raise ConfigError(
"unable to find remote section '{}'".format(name)
)
parsed = urlparse(settings["url"])
# Support for cross referenced remotes.
# This will merge the settings, giving priority to the outer reference.
# For example, having:
#
# dvc remote add server ssh://localhost
# dvc remote modify server user root
# dvc remote modify server ask_password true
#
# dvc remote add images remote://server/tmp/pictures
# dvc remote modify images user alice
# dvc remote modify images ask_password false
# dvc remote modify images password asdf1234
#
# Results on a config dictionary like:
#
# {
# "url": "ssh://localhost/tmp/pictures",
# "user": "alice",
# "password": "asdf1234",
# "ask_password": False,
# }
#
if parsed.scheme == "remote":
reference = self.get_remote_settings(parsed.netloc)
url = posixpath.join(reference["url"], parsed.path.lstrip("/"))
merged = reference.copy()
merged.update(settings)
merged["url"] = url
return merged
return settings
self.load()
@staticmethod
def unset(config, section, opt=None):
def _save(config):
if config.filename is None:
return
logger.debug("Writing '{}'.".format(config.filename))
dname = os.path.dirname(os.path.abspath(config.filename))
try:
os.makedirs(dname)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
config.write()
def unset(self, section, opt=None, level=None, force=False):
"""Unsets specified option and/or section in the config.
Args:
config (configobj.ConfigObj): config to work on.
section (str): section name.
opt (str): optional option name.
level (int): config level to use.
force (bool): don't error-out even if section doesn't exist. False
by default.
Raises:
dvc.config.ConfigError: thrown if section doesn't exist and
`force != True`.
"""
config = self.get_configobj(level)
if section not in config.keys():
if force:
return
raise ConfigError("section '{}' doesn't exist".format(section))
if opt is None:
del config[section]
return
if opt:
if opt not in config[section].keys():
if force:
return
raise ConfigError(
"option '{}.{}' doesn't exist".format(section, opt)
)
del config[section][opt]
if opt not in config[section].keys():
raise ConfigError(
"option '{}.{}' doesn't exist".format(section, opt)
)
del config[section][opt]
if not config[section]:
if not config[section]:
del config[section]
else:
del config[section]
@staticmethod
def set(config, section, opt, value):
self.save(config)
def set(self, section, opt, value, level=None, force=True):
"""Sets specified option in the config.
Args:
config (configobj.ConfigObj): config to work on.
section (str): section name.
opt (str): option name.
value: value to set option to.
level (int): config level to use.
force (bool): set option even if section already exists. True by
default.
Raises:
dvc.config.ConfigError: thrown if section already exists and
`force != True`.
"""
config = self.get_configobj(level)
if section not in config.keys():
config[section] = {}
elif not force:
raise ConfigError(
"Section '{}' already exists. Use `-f|--force` to overwrite "
"section with new value.".format(section)
)
config[section][opt] = value
self.save(config)
@staticmethod
def show(config, section, opt):
"""Prints option value from the config.
def get(self, section, opt=None, level=None):
"""Return option value from the config.
Args:
config (configobj.ConfigObj): config to work on.
section (str): section name.
opt (str): option name.
level (int): config level to use.
Returns:
value (str, int): option value.
"""
config = self.get_configobj(level)
if section not in config.keys():
raise ConfigError("section '{}' doesn't exist".format(section))
@@ -598,25 +555,35 @@ class Config(object): # pylint: disable=too-many-instance-attributes
"option '{}.{}' doesn't exist".format(section, opt)
)
logger.info(config[section][opt])
@staticmethod
def _merge(first, second):
res = {}
sections = list(first.keys()) + list(second.keys())
for section in sections:
first_copy = first.get(section, {}).copy()
second_copy = second.get(section, {}).copy()
first_copy.update(second_copy)
res[section] = first_copy
return res
return config[section][opt]
@staticmethod
def _lower(config):
new_config = {}
new_config = configobj.ConfigObj()
for s_key, s_value in config.items():
new_s = {}
for key, value in s_value.items():
new_s[key.lower()] = str(value)
new_config[s_key.lower()] = new_s
return new_config
def get_configobj(self, level):
configs = {
self.LEVEL_LOCAL: self._local_config,
self.LEVEL_REPO: self._repo_config,
self.LEVEL_GLOBAL: self._global_config,
self.LEVEL_SYSTEM: self._system_config,
}
return configs.get(level, self._repo_config)
def list_options(self, section_regex, option, level=None):
ret = {}
config = self.get_configobj(level)
for section in config.keys():
r = re.match(section_regex, section)
if r:
name = r.group("name")
value = config[section].get(option, "")
ret[name] = value
return ret
dvc/data_cloud.py
+1 -3
View File
@@ -25,7 +25,6 @@ class DataCloud(object):
Args:
repo (dvc.repo.Repo): repo instance that belongs to the repo that
we are working on.
config (dict): config of the repo.
Raises:
config.ConfigError: thrown when config has invalid format.
@@ -75,8 +74,7 @@ class DataCloud(object):
)
def _init_remote(self, remote):
config = self.repo.config.get_remote_settings(remote)
return Remote(self.repo, config)
return Remote(self.repo, name=remote)
def _init_compat(self):
name = self._core.get(Config.SECTION_CORE_CLOUD, "").strip().lower()
dvc/dependency/__init__.py
+1 -2
View File
@@ -52,8 +52,7 @@ def _get(stage, p, info):
parsed = urlparse(p)
if parsed.scheme == "remote":
settings = stage.repo.config.get_remote_settings(parsed.netloc)
remote = Remote(stage.repo, settings)
remote = Remote(stage.repo, name=parsed.netloc)
return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
for d in DEPS:
dvc/main.py
+4
View File
@@ -5,6 +5,7 @@ from __future__ import unicode_literals
import logging
from dvc.cli import parse_args
from dvc.config import ConfigError
from dvc.analytics import Analytics
from dvc.exceptions import NotDvcRepoError, DvcParserError
@@ -36,6 +37,9 @@ def main(argv=None):
cmd = args.func(args)
ret = cmd.run_cmd()
except ConfigError:
logger.exception("configuration error")
ret = 251
except KeyboardInterrupt:
logger.exception("interrupted by the user")
ret = 252
dvc/output/__init__.py
+1 -2
View File
@@ -61,8 +61,7 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None):
parsed = urlparse(p)
if parsed.scheme == "remote":
settings = stage.repo.config.get_remote_settings(parsed.netloc)
remote = Remote(stage.repo, settings)
remote = Remote(stage.repo, name=parsed.netloc)
return OUTS_MAP[remote.scheme](
stage,
p,
dvc/remote/__init__.py
+10 -2
View File
@@ -10,6 +10,8 @@ from dvc.remote.http import RemoteHTTP
from dvc.remote.https import RemoteHTTPS
from dvc.remote.oss import RemoteOSS
from .config import RemoteConfig
REMOTES = [
RemoteAZURE,
@@ -31,5 +33,11 @@ def _get(config):
return RemoteLOCAL
def Remote(repo, config):
return _get(config)(repo, config)
def Remote(repo, **kwargs):
name = kwargs.get("name")
if name:
remote_config = RemoteConfig(repo.config)
settings = remote_config.get_settings(name)
else:
settings = kwargs
return _get(settings)(repo, settings)
dvc/remote/config.py
+151
View File
@@ -0,0 +1,151 @@
import os
import logging
import posixpath
from dvc.utils.compat import urlparse
from dvc.config import Config, ConfigError
logger = logging.getLogger(__name__)
class RemoteConfig(object):
def __init__(self, config):
self.config = config
def get_settings(self, name):
"""
Args:
name (str): The name of the remote that we want to retrieve
Returns:
dict: The content beneath the given remote name.
Example:
>>> config = {'remote "server"': {'url': 'ssh://localhost/'}}
>>> get_settings("server")
{'url': 'ssh://localhost/'}
"""
settings = self.config.config.get(
Config.SECTION_REMOTE_FMT.format(name.lower())
)
if settings is None:
raise ConfigError(
"unable to find remote section '{}'".format(name)
)
parsed = urlparse(settings["url"])
# Support for cross referenced remotes.
# This will merge the settings, giving priority to the outer reference.
# For example, having:
#
# dvc remote add server ssh://localhost
# dvc remote modify server user root
# dvc remote modify server ask_password true
#
# dvc remote add images remote://server/tmp/pictures
# dvc remote modify images user alice
# dvc remote modify images ask_password false
# dvc remote modify images password asdf1234
#
# Results on a config dictionary like:
#
# {
# "url": "ssh://localhost/tmp/pictures",
# "user": "alice",
# "password": "asdf1234",
# "ask_password": False,
# }
#
if parsed.scheme == "remote":
reference = self.get_settings(parsed.netloc)
url = posixpath.join(reference["url"], parsed.path.lstrip("/"))
merged = reference.copy()
merged.update(settings)
merged["url"] = url
return merged
return settings
@staticmethod
def resolve_path(path, config_file):
"""Resolve path relative to config file location.
Args:
path: Path to be resolved.
config_file: Path to config file, which `path` is specified
relative to.
Returns:
Path relative to the `config_file` location. If `path` is an
absolute path then it will be returned without change.
"""
if os.path.isabs(path):
return path
return os.path.relpath(path, os.path.dirname(config_file))
def add(self, name, url, default=False, force=False, level=None):
from dvc.remote import _get, RemoteLOCAL
configobj = self.config.get_configobj(level)
remote = _get({Config.SECTION_REMOTE_URL: url})
if remote == RemoteLOCAL and not url.startswith("remote://"):
url = self.resolve_path(url, configobj.filename)
self.config.set(
Config.SECTION_REMOTE_FMT.format(name),
Config.SECTION_REMOTE_URL,
url,
force=force,
level=level,
)
if default:
logger.info("Setting '{}' as a default remote.".format(name))
self.config.set(
Config.SECTION_CORE,
Config.SECTION_CORE_REMOTE,
name,
level=level,
)
def remove(self, name, level=None):
self.config.unset(Config.SECTION_REMOTE_FMT.format(name), level=level)
if level is None:
level = Config.LEVEL_REPO
for lev in [
Config.LEVEL_LOCAL,
Config.LEVEL_REPO,
Config.LEVEL_GLOBAL,
Config.LEVEL_SYSTEM,
]:
self.config.unset(
Config.SECTION_CORE,
Config.SECTION_CORE_REMOTE,
level=lev,
force=True,
)
if lev == level:
break
def modify(self, name, option, value, level=None):
self.config.set(
Config.SECTION_REMOTE_FMT.format(name), option, value, level=level
)
def list(self, level=None):
return self.config.list_options(
Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL, level=level
)
def set_default(self, name, unset=False, level=None):
if unset:
self.config.unset(Config.SECTION_CORE, Config.SECTION_CORE_REMOTE)
return
self.config.set(
Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, name, level=level
)
dvc/repo/__init__.py
+8 -8
View File
@@ -5,6 +5,7 @@ import logging
import dvc.prompt as prompt
from dvc.config import Config
from dvc.exceptions import (
DvcException,
NotDvcRepoError,
@@ -41,7 +42,6 @@ class Repo(object):
from dvc.repo.brancher import brancher
def __init__(self, root_dir=None):
from dvc.config import Config
from dvc.state import State
from dvc.lock import Lock
from dvc.scm import SCM
@@ -89,15 +89,15 @@ class Repo(object):
def __repr__(self):
return "Repo: '{root_dir}'".format(root_dir=self.root_dir)
@staticmethod
def find_root(root=None):
@classmethod
def find_root(cls, root=None):
if root is None:
root = os.getcwd()
else:
root = os.path.abspath(os.path.realpath(root))
while True:
dvc_dir = os.path.join(root, Repo.DVC_DIR)
dvc_dir = os.path.join(root, cls.DVC_DIR)
if os.path.isdir(dvc_dir):
return root
if os.path.ismount(root):
@@ -105,10 +105,10 @@ class Repo(object):
root = os.path.dirname(root)
raise NotDvcRepoError(root)
@staticmethod
def find_dvc_dir(root=None):
root_dir = Repo.find_root(root)
return os.path.join(root_dir, Repo.DVC_DIR)
@classmethod
def find_dvc_dir(cls, root=None):
root_dir = cls.find_root(root)
return os.path.join(root_dir, cls.DVC_DIR)
@staticmethod
def init(root_dir=os.curdir, no_scm=False, force=False):
tests/basic_env.py
+6 -3
View File
@@ -57,6 +57,10 @@ class TestDirFixture(object):
root_dir = self.mkdtemp()
self._root_dir = os.path.realpath(root_dir)
@property
def root_dir(self):
return self._root_dir
def _pushd(self, d):
if not hasattr(self, "_saved_dir"):
self._saved_dir = os.path.realpath(os.curdir)
@@ -193,9 +197,8 @@ class TestDvcDataFileFixture(TestDvcGitInitializedFixture):
shutil.copytree(self.dvc.cache.local.cache_dir, self.cache_dir)
class MockConfig:
system = None
glob = None
local = None
level = 1
force = False
default = True
name = self.REMOTE
url = self.cache_dir
tests/func/test_config.py
+5 -24
View File
@@ -1,7 +1,6 @@
import configobj
from dvc.main import main
from dvc.command.config import CmdConfig
from tests.basic_env import TestDvc
@@ -67,38 +66,20 @@ class TestConfigCLI(TestDvc):
self._do_test(True)
def test_non_existing(self):
# FIXME check stdout/err
ret = main(["config", "non_existing_section.field"])
self.assertEqual(ret, 1)
self.assertEqual(ret, 251)
ret = main(["config", "global.non_existing_field"])
self.assertEqual(ret, 1)
self.assertEqual(ret, 251)
ret = main(["config", "non_existing_section.field", "-u"])
self.assertEqual(ret, 1)
self.assertEqual(ret, 251)
ret = main(["config", "global.non_existing_field", "-u"])
self.assertEqual(ret, 1)
self.assertEqual(ret, 251)
ret = main(["config", "core.remote", "myremote"])
self.assertEqual(ret, 0)
ret = main(["config", "core.non_existing_field", "-u"])
self.assertEqual(ret, 1)
def test_failed_write(self):
class A(object):
system = False
glob = False
local = False
name = "core.remote"
value = "myremote"
unset = False
args = A()
cmd = CmdConfig(args)
cmd.configobj.write = None
ret = cmd.run()
self.assertNotEqual(ret, 0)
self.assertEqual(ret, 251)
tests/func/test_data_cloud.py
+1
View File
@@ -39,6 +39,7 @@ from tests.utils import spy
TEST_REMOTE = "upstream"
TEST_SECTION = 'remote "{}"'.format(TEST_REMOTE)
TEST_CONFIG = {
Config.SECTION_CACHE: {},
Config.SECTION_CORE: {Config.SECTION_CORE_REMOTE: TEST_REMOTE},
TEST_SECTION: {Config.SECTION_REMOTE_URL: ""},
}
tests/func/test_remote.py
+1 -18
View File
@@ -5,7 +5,6 @@ from dvc.remote.base import RemoteBASE
from mock import patch
from dvc.main import main
from dvc.command.remote import CmdRemoteAdd
from dvc.config import Config
from tests.basic_env import TestDvc
@@ -33,22 +32,6 @@ class TestRemote(TestDvc):
self.assertEqual(main(["remote", "list"]), 0)
def test_failed_write(self):
class A(object):
system = False
glob = False
local = False
name = "myremote"
url = "s3://remote"
unset = False
args = A()
cmd = CmdRemoteAdd(args)
cmd.configobj.write = None
ret = cmd.run()
self.assertNotEqual(ret, 0)
def test_relative_path(self):
dname = os.path.join("..", "path", "to", "dir")
ret = main(["remote", "add", "mylocal", dname])
@@ -64,7 +47,7 @@ class TestRemote(TestDvc):
remote_name = "a"
remote_url = "s3://bucket/name"
self.assertEqual(main(["remote", "add", remote_name, remote_url]), 0)
self.assertEqual(main(["remote", "add", remote_name, remote_url]), 1)
self.assertEqual(main(["remote", "add", remote_name, remote_url]), 251)
self.assertEqual(
main(["remote", "add", "-f", remote_name, remote_url]), 0
)