Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

config: support 'global' and 'system' configs

Browse Source 
Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2018-12-01 02:31:14 +02:00
parent a0ba0ef19a
commit c02fe1313f
9 changed files with 232 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/cli.py
+55
View File
@@ -638,6 +638,17 @@ def parse_args(argv=None):
nargs='?',
default=None,
help='Option value.')
config_parser.add_argument(
'--global',
dest='glob',
action='store_true',
default=False,
help='Use global config.')
config_parser.add_argument(
'--system',
action='store_true',
default=False,
help='Use system config.')
config_parser.add_argument(
'--local',
action='store_true',
@@ -672,6 +683,17 @@ def parse_args(argv=None):
remote_add_parser.add_argument(
'url',
help='URL.')
remote_add_parser.add_argument(
'--global',
dest='glob',
action='store_true',
default=False,
help='Use global config.')
remote_add_parser.add_argument(
'--system',
action='store_true',
default=False,
help='Use system config.')
remote_add_parser.add_argument(
'--local',
action='store_true',
@@ -694,6 +716,17 @@ def parse_args(argv=None):
remote_remove_parser.add_argument(
'name',
help='Name')
remote_remove_parser.add_argument(
'--global',
dest='glob',
action='store_true',
default=False,
help='Use global config.')
remote_remove_parser.add_argument(
'--system',
action='store_true',
default=False,
help='Use system config.')
remote_remove_parser.add_argument(
'--local',
action='store_true',
@@ -723,6 +756,17 @@ def parse_args(argv=None):
default=False,
action='store_true',
help='Unset option.')
remote_modify_parser.add_argument(
'--global',
dest='glob',
action='store_true',
default=False,
help='Use global config.')
remote_modify_parser.add_argument(
'--system',
action='store_true',
default=False,
help='Use system config.')
remote_modify_parser.add_argument(
'--local',
action='store_true',
@@ -736,6 +780,17 @@ def parse_args(argv=None):
parents=[parent_parser],
description=REMOTE_LIST_HELP,
help=REMOTE_LIST_HELP)
remote_list_parser.add_argument(
'--global',
dest='glob',
action='store_true',
default=False,
help='Use global config.')
remote_list_parser.add_argument(
'--system',
action='store_true',
default=False,
help='Use system config.')
remote_list_parser.add_argument(
'--local',
action='store_true',
dvc/command/config.py
+37 -71
View File
@@ -1,9 +1,9 @@
import os
import configobj
from dvc.command.base import CmdBase
from dvc.logger import Logger
from dvc.config import Config
from dvc.exceptions import DvcException
class CmdConfig(CmdBase):
@@ -11,92 +11,58 @@ class CmdConfig(CmdBase):
from dvc.project import Project
self.args = args
root_dir = self._find_root()
if args.local:
config = Config.CONFIG_LOCAL
dvc_dir = os.path.join(self._find_root(), Project.DVC_DIR)
self.config = Config(dvc_dir)
if self.args.system:
self.configobj = self.config._system_config
elif self.args.glob:
self.configobj = self.config._global_config
elif self.args.local:
self.configobj = self.config._local_config
else:
config = Config.CONFIG
self.config_file = os.path.join(root_dir, Project.DVC_DIR, config)
# Using configobj because it doesn't
# drop comments like configparser does.
self.configobj = configobj.ConfigObj(self.config_file)
self.configobj = self.config._project_config
def run_cmd(self):
return self.run()
def _get_key(self, d, name, add=False):
for k in d.keys():
if k.lower() == name.lower():
return k
def _unset(self, section, opt=None, configobj=None):
if configobj is None:
configobj = self.configobj
if add:
d[name] = {}
return name
return None
def save(self):
try:
self.configobj.write()
except Exception as exc:
msg = "Failed to write config '{}'".format(self.configobj.filename)
Logger.error(msg, exc)
self.config.unset(configobj, section, opt)
self.config.save(configobj)
except DvcException as exc:
Logger.error("Failed to unset '{}'".format(self.args.name), exc)
return 1
return 0
def unset(self, section, opt=None):
if section not in self.configobj.keys():
Logger.error("Section '{}' doesn't exist".format(section))
def _show(self, section, opt):
try:
self.config.show(self.configobj, section, opt)
except DvcException as exc:
Logger.error("Failed to show '{}'".format(self.args.name), exc)
return 1
if opt in self.configobj[section].keys():
del self.configobj[section][opt]
if len(self.configobj[section]) == 0 or opt is None:
del self.configobj[section]
return self.save()
def show(self):
Logger.info(self.configobj[self.section][self.opt])
return 0
def set(self, section, opt, value):
if section not in self.configobj.keys():
self.configobj[section] = {}
self.configobj[section][opt] = value
return self.save()
def check_opt(self):
_section, _opt = self.args.name.strip().split('.', 1)
add = (self.args.value is not None and self.args.unset is False)
section = self._get_key(self.configobj, _section, add)
if not section:
Logger.error('Invalid option name {}'.format(_section))
def _set(self, section, opt, value):
try:
self.config.set(self.configobj, section, opt, value)
self.config.save(self.configobj)
except DvcException as exc:
Logger.error("Failed to set '{}.{}' to '{}'".format(section,
opt,
value),
exc)
return 1
opt = self._get_key(self.configobj[section], _opt, add)
if not opt:
Logger.error('Invalid option value: {}'.format(_opt))
return 1
self.section = section
self.opt = opt
return 0
def run(self):
if self.check_opt() != 0:
return 1
section, opt = self.args.name.lower().strip().split('.', 1)
if self.args.unset:
return self.unset(self.section, self.opt)
if self.args.value is None:
return self.show()
return self.set(self.section, self.opt, self.args.value)
return self._unset(section, opt)
elif self.args.value is None:
return self._show(section, opt)
else:
return self._set(section, opt, self.args.value)
dvc/command/remote.py
+21 -22
View File
@@ -1,6 +1,4 @@
import os
import re
import configobj
from dvc.config import Config
from dvc.command.config import CmdConfig
@@ -10,49 +8,50 @@ from dvc.logger import Logger
class CmdRemoteAdd(CmdConfig):
def run(self):
section = Config.SECTION_REMOTE_FMT.format(self.args.name)
ret = self.set(section, Config.SECTION_REMOTE_URL, self.args.url)
ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url)
if ret != 0:
return ret
if self.args.default:
msg = 'Setting \'{}\' as a default remote.'.format(self.args.name)
Logger.info(msg)
ret = self.set(Config.SECTION_CORE,
Config.SECTION_CORE_REMOTE,
self.args.name)
ret = self._set(Config.SECTION_CORE,
Config.SECTION_CORE_REMOTE,
self.args.name)
return ret
class CmdRemoteRemove(CmdConfig):
def _remove_default(self, config_file, remote):
path = os.path.join(os.path.dirname(self.config_file),
config_file)
config = configobj.ConfigObj(path)
def _remove_default(self, config):
core = config.get(Config.SECTION_CORE, None)
if core is None:
return
return 0
default = core.get(Config.SECTION_CORE_REMOTE, None)
if default is None:
return
return 0
if default == remote:
del config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE]
if len(config[Config.SECTION_CORE]) == 0:
del config[Config.SECTION_CORE]
config.write()
if default == self.args.name:
return self._unset(Config.SECTION_CORE,
opt=Config.SECTION_CORE_REMOTE,
configobj=config)
def run(self):
section = Config.SECTION_REMOTE_FMT.format(self.args.name)
ret = self.unset(section)
ret = self._unset(section)
if ret != 0:
return ret
self._remove_default(Config.CONFIG, self.args.name)
self._remove_default(Config.CONFIG_LOCAL, self.args.name)
for configobj in [self.config._local_config,
self.config._project_config,
self.config._global_config,
self.config._system_config]:
self._remove_default(configobj)
self.config.save(configobj)
if configobj == self.configobj:
break
return 0
dvc/config.py
+110 -9
View File
@@ -2,9 +2,11 @@
DVC config objects.
"""
import os
import errno
import configobj
from schema import Schema, Optional, And, Use, Regex
from dvc.logger import Logger
from dvc.exceptions import DvcException
@@ -54,6 +56,9 @@ def is_percent(val):
class Config(object):
APPNAME = 'dvc'
APPAUTHOR = 'iterative'
CONFIG = 'config'
CONFIG_LOCAL = 'config.local'
@@ -187,18 +192,54 @@ class Config(object):
def __init__(self, dvc_dir):
self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
self.system_config_file = os.path.join(self.system_config_dir,
self.CONFIG)
self.global_config_file = os.path.join(self.global_config_dir,
self.CONFIG)
self.config_file = os.path.join(dvc_dir, self.CONFIG)
self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL)
self._load()
@property
def global_config_dir(self):
from appdirs import user_config_dir
return user_config_dir(appname=self.APPNAME,
appauthor=self.APPAUTHOR)
@property
def system_config_dir(self):
from appdirs import site_config_dir
return site_config_dir(appname=self.APPNAME,
appauthor=self.APPAUTHOR)
@staticmethod
def init(dvc_dir):
config_file = os.path.join(dvc_dir, Config.CONFIG)
open(config_file, 'w+').close()
return Config(dvc_dir)
def _load(self):
self._system_config = configobj.ConfigObj(self.system_config_file)
self._global_config = configobj.ConfigObj(self.global_config_file)
self._project_config = configobj.ConfigObj(self.config_file)
self._local_config = configobj.ConfigObj(self.config_local_file)
self._config = None
def load(self):
self._load()
try:
self._config = configobj.ConfigObj(self.config_file)
self._config = configobj.ConfigObj(self.system_config_file)
user = configobj.ConfigObj(self.global_config_file)
config = configobj.ConfigObj(self.config_file)
local = configobj.ConfigObj(self.config_local_file)
# NOTE: schema doesn't support ConfigObj.Section validation, so we
# need to convert our config to dict before passing it to
self._config = self._lower(self._config)
local = self._lower(local)
self._config = self._merge(self._config, local)
for c in [user, config, local]:
c = self._lower(c)
self._config = self._merge(self._config, c)
self._config = Schema(self.SCHEMA).validate(self._config)
@@ -209,6 +250,72 @@ class Config(object):
except Exception as ex:
raise ConfigError(ex)
def _get_key(self, d, name, add=False):
for k in d.keys():
if k.lower() == name.lower():
return k
if add:
d[name] = {}
return name
return None
def save(self, config=None):
if config is not None:
clist = [config]
else:
clist = [self._system_config,
self._global_config,
self._project_config,
self._local_config]
for conf in clist:
try:
Logger.debug("Writing '{}'.".format(conf.filename))
dname = os.path.dirname(os.path.abspath(conf.filename))
try:
os.makedirs(dname)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
conf.write()
except Exception as exc:
msg = "Failed to write config '{}'".format(conf.filename)
raise ConfigError(msg, exc)
def unset(self, config, section, opt=None):
if section not in config.keys():
raise ConfigError("Section '{}' doesn't exist".format(section))
if opt is None:
del config[section]
return
if opt not in config[section].keys():
raise ConfigError("Option '{}.{}' doesn't exist".format(section,
opt))
del config[section][opt]
if len(config[section]) == 0:
del config[section]
def set(self, config, section, opt, value):
if section not in config.keys():
config[section] = {}
config[section][opt] = value
def show(self, config, section, opt):
if section not in config.keys():
raise ConfigError("Section '{}' doesn't exist".format(section))
if opt not in config[section].keys():
raise ConfigError("Option '{}.{}' doesn't exist".format(section,
opt))
Logger.info(config[section][opt])
@staticmethod
def _merge(first, second):
res = {}
@@ -229,9 +336,3 @@ class Config(object):
new_s[key.lower()] = value
new_config[s_key.lower()] = new_s
return new_config
@staticmethod
def init(dvc_dir):
config_file = os.path.join(dvc_dir, Config.CONFIG)
open(config_file, 'w+').close()
return Config(dvc_dir)
dvc/project.py
+2
View File
@@ -35,6 +35,8 @@ class Project(object):
self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
self.config = Config(self.dvc_dir)
self.config.load()
self.scm = SCM(self.root_dir, project=self)
self.lock = Lock(self.dvc_dir)
# NOTE: storing state and link_state in the repository itself to avoid
requirements.txt
+1
View File
@@ -24,3 +24,4 @@ grandalf==0.6
pydot>=1.2.4
asciimatics>=1.10.0
distro>=1.3.0
appdirs>=1.4.3
setup.py
+1
View File
@@ -24,6 +24,7 @@ install_requires = [
"grandalf==0.6",
"asciimatics>=1.10.0",
"distro>=1.3.0",
"appdirs>=1.4.3",
]
# Extra dependencies for remote integrations
tests/test_config.py
+3 -1
View File
@@ -89,6 +89,8 @@ class TestConfigCLI(TestDvc):
def test_failed_write(self):
class A(object):
system = False
glob = False
local = False
name = 'core.remote'
value = 'myremote'
@@ -98,5 +100,5 @@ class TestConfigCLI(TestDvc):
cmd = CmdConfig(args)
cmd.configobj.write = None
ret = cmd.save()
ret = cmd.run()
self.assertNotEqual(ret, 0)
tests/test_remote.py
+2
View File
@@ -30,6 +30,8 @@ class TestRemote(TestDvc):
def test_failed_write(self):
class A(object):
system = False
glob = False
local = False
name = 'myremote'
url = 's3://remote'