Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-01 23:08:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

dvc: refactor config (#3298)

Browse Source 
* dvc: refactor config

Advantages:
- read like a dict
- modify as a dict for testing or manipulation purposes
- wrap modifications into context manager to save them:

    with config.edit("local") as conf:
        conf["core"]["remote"] = "new-default"

Design decisions:
- dropped string constants
- remote sections are parsed into a simple subdict upon load
- remote config schema is now defined by url scheme
- relative paths are handled transparently upon load/save
- user config and remote manipulations are moved to commands
- same dev manipulations are done by simply modifying a config dict

* test: do not set cache.type for remotes that doesn't support it

* dvc: fix windows absolute paths validation in config

* test: fix windows separate drive test

* dvc: stylistic fixes in config
This commit is contained in:
Alexander Schepanovski
2020-02-13 19:35:37 +02:00
committed by GitHub
parent ccca126749
commit 6ccae711fe
45 changed files with 635 additions and 1078 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/analytics.py
+2 -5
View File
@@ -54,11 +54,8 @@ def is_enabled():
return False
enabled = to_bool(
Config(validate=False)
.config.get(Config.SECTION_CORE, {})
.get(Config.SECTION_CORE_ANALYTICS, "true")
Config(validate=False).get("core", {}).get("analytics", "true")
)
logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))
return enabled
@@ -144,7 +141,7 @@ def _find_or_create_user_id():
IDs are generated randomly with UUID.
"""
config_dir = Config.get_global_config_dir()
config_dir = Config.get_dir("global")
fname = os.path.join(config_dir, "user_id")
lockfile = os.path.join(config_dir, "user_id.lock")
dvc/cache.py
+9 -41
View File
@@ -1,25 +1,8 @@
"""Manages cache of a DVC repo."""
import os
from collections import defaultdict
from funcy import cached_property
from dvc.config import Config
class CacheConfig(object):
def __init__(self, config):
self.config = config
def set_dir(self, dname, level=None):
from dvc.remote.config import RemoteConfig
configobj = self.config.get_configobj(level)
path = RemoteConfig.resolve_path(dname, configobj.filename)
self.config.set(
Config.SECTION_CACHE, Config.SECTION_CACHE_DIR, path, level=level
)
def _make_remote_property(name):
"""
@@ -70,37 +53,22 @@ class Cache(object):
from dvc.remote import Remote
self.repo = repo
self.config = config = repo.config["cache"]
self.config = config = repo.config.config[Config.SECTION_CACHE]
local = config.get(Config.SECTION_CACHE_LOCAL)
local = config.get("local")
if local:
name = Config.SECTION_REMOTE_FMT.format(local)
settings = repo.config.config[name]
settings = {"name": local}
else:
default_cache_dir = os.path.join(repo.dvc_dir, self.CACHE_DIR)
cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir)
cache_type = config.get(Config.SECTION_CACHE_TYPE)
protected = config.get(Config.SECTION_CACHE_PROTECTED)
shared = config.get(Config.SECTION_CACHE_SHARED)
settings = {
Config.PRIVATE_CWD: config.get(
Config.PRIVATE_CWD, repo.dvc_dir
),
Config.SECTION_REMOTE_URL: cache_dir,
Config.SECTION_CACHE_TYPE: cache_type,
Config.SECTION_CACHE_PROTECTED: protected,
Config.SECTION_CACHE_SHARED: shared,
}
settings = {**config, "url": config["dir"]}
self.local = Remote(repo, **settings)
s3 = _make_remote_property(Config.SECTION_CACHE_S3)
gs = _make_remote_property(Config.SECTION_CACHE_GS)
ssh = _make_remote_property(Config.SECTION_CACHE_SSH)
hdfs = _make_remote_property(Config.SECTION_CACHE_HDFS)
azure = _make_remote_property(Config.SECTION_CACHE_AZURE)
s3 = _make_remote_property("s3")
gs = _make_remote_property("gs")
ssh = _make_remote_property("ssh")
hdfs = _make_remote_property("hdfs")
azure = _make_remote_property("azure")
class NamedCache(object):
dvc/command/add.py
+2 -5
View File
@@ -1,11 +1,8 @@
import argparse
import logging
from dvc.command.base import append_doc_link
from dvc.command.base import CmdBase
from dvc.exceptions import DvcException
from dvc.exceptions import RecursiveAddingWhileUsingFilename
from dvc.command.base import append_doc_link, CmdBase
from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
logger = logging.getLogger(__name__)
dvc/command/base.py
+1 -1
View File
@@ -35,7 +35,7 @@ class CmdBase(object):
self.repo = Repo()
self.config = self.repo.config
self.args = args
hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
hardlink_lock = self.config["core"].get("hardlink_lock", False)
updater = Updater(self.repo.dvc_dir, hardlink_lock=hardlink_lock)
updater.check()
dvc/command/cache.py
+3 -8
View File
@@ -1,18 +1,13 @@
import argparse
from dvc.cache import CacheConfig
from dvc.command.base import append_doc_link
from dvc.command.base import fix_subparsers
from dvc.command.base import append_doc_link, fix_subparsers
from dvc.command.config import CmdConfig
class CmdCacheDir(CmdConfig):
def __init__(self, args):
super().__init__(args)
self.cache_config = CacheConfig(self.config)
def run(self):
self.cache_config.set_dir(self.args.value, level=self.args.level)
with self.config.edit(level=self.args.level) as edit:
edit["cache"]["dir"] = self.args.value
return 0
dvc/command/config.py
+29 -18
View File
@@ -1,10 +1,8 @@
import argparse
import logging
from dvc.command.base import append_doc_link
from dvc.command.base import CmdBaseNoRepo
from dvc.config import Config
from dvc.command.base import append_doc_link, CmdBaseNoRepo
from dvc.config import Config, ConfigError
logger = logging.getLogger(__name__)
@@ -18,17 +16,21 @@ class CmdConfig(CmdBaseNoRepo):
def run(self):
section, opt = self.args.name.lower().strip().split(".", 1)
if self.args.unset:
self.config.unset(section, opt, level=self.args.level)
elif self.args.value is None:
logger.info(self.config.get(section, opt, level=self.args.level))
else:
self.config.set(
section, opt, self.args.value, level=self.args.level
)
if self.args.value is None and not self.args.unset:
conf = self.config.load_one(self.args.level)
self._check(conf, section, opt)
logger.info(conf[section][opt])
return 0
is_write = self.args.unset or self.args.value is not None
if is_write and self.args.name == "cache.type":
with self.config.edit(self.args.level) as conf:
if self.args.unset:
self._check(conf, section, opt)
del conf[section][opt]
else:
self._check(conf, section)
conf[section][opt] = self.args.value
if self.args.name == "cache.type":
logger.warning(
"You have changed the 'cache.type' option. This doesn't update"
" any existing workspace file links, but it can be done with:"
@@ -37,30 +39,39 @@ class CmdConfig(CmdBaseNoRepo):
return 0
def _check(self, conf, section, opt=None):
if section not in conf:
msg = "section {} doesn't exist"
raise ConfigError(msg.format(self.args.name))
if opt and opt not in conf[section]:
msg = "option {} doesn't exist"
raise ConfigError(msg.format(self.args.name))
parent_config_parser = argparse.ArgumentParser(add_help=False)
parent_config_parser.add_argument(
"--global",
dest="level",
action="store_const",
const=Config.LEVEL_GLOBAL,
const="global",
help="Use global config.",
)
parent_config_parser.add_argument(
"--system",
dest="level",
action="store_const",
const=Config.LEVEL_SYSTEM,
const="system",
help="Use system config.",
)
parent_config_parser.add_argument(
"--local",
dest="level",
action="store_const",
const=Config.LEVEL_LOCAL,
const="local",
help="Use local config.",
)
parent_config_parser.set_defaults(level=Config.LEVEL_REPO)
parent_config_parser.set_defaults(level="repo")
def add_parser(subparsers, parent_parser):
dvc/command/remote.py
+60 -33
View File
@@ -1,72 +1,99 @@
import argparse
import logging
from dvc.command.base import append_doc_link
from dvc.command.base import fix_subparsers
from dvc.config import ConfigError
from dvc.command.base import append_doc_link, fix_subparsers
from dvc.command.config import CmdConfig
from dvc.remote.config import RemoteConfig
from dvc.utils import format_link
logger = logging.getLogger(__name__)
class CmdRemoteConfig(CmdConfig):
class CmdRemote(CmdConfig):
def __init__(self, args):
super().__init__(args)
self.remote_config = RemoteConfig(self.config)
if getattr(self.args, "name", None):
self.args.name = self.args.name.lower()
def _check_exists(self, conf):
if self.args.name not in conf["remote"]:
raise ConfigError(
"remote '{}' doesn't exists.".format(self.args.name)
)
class CmdRemoteAdd(CmdRemoteConfig):
class CmdRemoteAdd(CmdRemote):
def run(self):
if self.args.default:
logger.info(
"Setting '{}' as a default remote.".format(self.args.name)
)
self.remote_config.add(
self.args.name,
self.args.url,
force=self.args.force,
default=self.args.default,
level=self.args.level,
)
with self.config.edit(self.args.level) as conf:
if self.args.name in conf["remote"] and not self.args.force:
raise ConfigError(
"remote '{}' already exists. Use `-f|--force` to "
"overwrite it.".format(self.args.name)
)
conf["remote"][self.args.name] = {"url": self.args.url}
if self.args.default:
conf["core"]["remote"] = self.args.name
return 0
class CmdRemoteRemove(CmdRemoteConfig):
class CmdRemoteRemove(CmdRemote):
def run(self):
self.remote_config.remove(self.args.name, level=self.args.level)
with self.config.edit(self.args.level) as conf:
self._check_exists(conf)
del conf["remote"][self.args.name]
# Remove core.remote refs to this remote in any shadowing configs
for level in reversed(self.config.LEVELS):
with self.config.edit(level) as conf:
if conf["core"].get("remote") == self.args.name:
del conf["core"]["remote"]
if level == self.args.level:
break
return 0
class CmdRemoteModify(CmdRemoteConfig):
class CmdRemoteModify(CmdRemote):
def run(self):
self.remote_config.modify(
self.args.name,
self.args.option,
self.args.value,
level=self.args.level,
)
with self.config.edit(self.args.level) as conf:
self._check_exists(conf)
conf["remote"][self.args.name][self.args.option] = self.args.value
return 0
class CmdRemoteDefault(CmdRemoteConfig):
class CmdRemoteDefault(CmdRemote):
def run(self):
if self.args.name is None and not self.args.unset:
name = self.remote_config.get_default(level=self.args.level)
print(name)
conf = self.config.load_one(self.args.level)
try:
print(conf["core"]["remote"])
except KeyError:
logger.info("No default remote set")
return 1
else:
self.remote_config.set_default(
self.args.name, unset=self.args.unset, level=self.args.level
)
with self.config.edit(self.args.level) as conf:
if self.args.unset:
conf["core"].pop("remote", None)
else:
conf["core"]["remote"] = self.args.name
return 0
class CmdRemoteList(CmdRemoteConfig):
class CmdRemoteList(CmdRemote):
def run(self):
for name, url in self.remote_config.list(
level=self.args.level
).items():
logger.info("{}\t{}".format(name, url))
conf = self.config.load_one(self.args.level)
for name, conf in conf["remote"].items():
logger.info("{}\t{}".format(name, conf["url"]))
return 0
dvc/config.py
+249 -401
View File
@@ -1,17 +1,17 @@
"""DVC config objects."""
import copy
import errno
from contextlib import contextmanager
import logging
import os
import re
from urllib.parse import urlparse
from funcy import cached_property, re_find, walk_values, compact
import configobj
from voluptuous import Schema, Required, Optional, Invalid
from voluptuous import All, Any, Lower, Range, Coerce, Match
from voluptuous import Schema, Optional, Invalid, ALLOW_EXTRA
from voluptuous import All, Any, Lower, Range, Coerce
from dvc.exceptions import DvcException
from dvc.exceptions import NotDvcRepoError
from dvc.exceptions import DvcException, NotDvcRepoError
from dvc.utils import relpath
logger = logging.getLogger(__name__)
@@ -48,13 +48,12 @@ def supported_cache_type(types):
# Checks that value is either true or false and converts it to bool
Bool = All(
to_bool = Bool = All(
Lower,
Any("true", "false"),
lambda v: v == "true",
msg="expected true or false",
)
to_bool = Schema(Bool)
def Choices(*choices):
@@ -67,7 +66,125 @@ def Choices(*choices):
return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
class Config(object): # pylint: disable=too-many-instance-attributes
def ByUrl(mapping):
schemas = walk_values(Schema, mapping)
def validate(data):
if "url" not in data:
raise Invalid("expected 'url'")
parsed = urlparse(data["url"])
# Windows absolute paths should really have scheme == "" (local)
if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
return schemas[""](data)
if parsed.scheme not in schemas:
raise Invalid("Unsupported URL type {}://".format(parsed.scheme))
return schemas[parsed.scheme](data)
return validate
class RelPath(str):
pass
REMOTE_COMMON = {
"url": str,
"checksum_jobs": All(Coerce(int), Range(1)),
"no_traverse": Bool,
"verify": Bool,
}
LOCAL_COMMON = {
"type": supported_cache_type,
Optional("protected", default=False): Bool,
"shared": All(Lower, Choices("group")),
Optional("slow_link_warning", default=True): Bool,
}
SCHEMA = {
"core": {
"remote": Lower,
"checksum_jobs": All(Coerce(int), Range(1)),
"loglevel": All(Lower, Choices("info", "debug", "warning", "error")),
Optional("interactive", default=False): Bool,
Optional("analytics", default=True): Bool,
Optional("hardlink_lock", default=False): Bool,
},
"cache": {
"local": str,
"s3": str,
"gs": str,
"hdfs": str,
"ssh": str,
"azure": str,
# This is for default local cache
"dir": str,
**LOCAL_COMMON,
},
"remote": {
str: ByUrl(
{
"": {**LOCAL_COMMON, **REMOTE_COMMON},
"s3": {
"region": str,
"profile": str,
"credentialpath": str,
"endpointurl": str,
Optional("listobjects", default=False): Bool,
Optional("use_ssl", default=True): Bool,
"sse": str,
"acl": str,
"grant_read": str,
"grant_read_acp": str,
"grant_write_acp": str,
"grant_full_control": str,
**REMOTE_COMMON,
},
"gs": {
"projectname": str,
"credentialpath": str,
**REMOTE_COMMON,
},
"ssh": {
"type": supported_cache_type,
"port": Coerce(int),
"user": str,
"password": str,
"ask_password": Bool,
"keyfile": str,
"timeout": Coerce(int),
"gss_auth": Bool,
**REMOTE_COMMON,
},
"hdfs": {"user": str, **REMOTE_COMMON},
"azure": {"connection_string": str, **REMOTE_COMMON},
"oss": {
"oss_key_id": str,
"oss_key_secret": str,
"oss_endpoint": str,
**REMOTE_COMMON,
},
"gdrive": {
"gdrive_client_id": str,
"gdrive_client_secret": str,
"gdrive_user_credentials_file": str,
**REMOTE_COMMON,
},
"http": REMOTE_COMMON,
"https": REMOTE_COMMON,
"remote": {str: object}, # Any of the above options are valid
}
)
},
"state": {
"row_limit": All(Coerce(int), Range(1)),
"row_cleanup_quota": All(Coerce(int), Range(0, 100)),
},
}
COMPILED_SCHEMA = Schema(SCHEMA)
class Config(dict):
"""Class that manages configuration files for a DVC repo.
Args:
@@ -85,168 +202,14 @@ class Config(object): # pylint: disable=too-many-instance-attributes
APPNAME = "dvc"
APPAUTHOR = "iterative"
# NOTE: used internally in RemoteLOCAL to know config
# location, that url should resolved relative to.
PRIVATE_CWD = "_cwd"
# In the order they shadow each other
LEVELS = ("system", "global", "repo", "local")
CONFIG = "config"
CONFIG_LOCAL = "config.local"
CREDENTIALPATH = "credentialpath"
LEVEL_LOCAL = 0
LEVEL_REPO = 1
LEVEL_GLOBAL = 2
LEVEL_SYSTEM = 3
SECTION_CORE = "core"
SECTION_CORE_LOGLEVEL = "loglevel"
SECTION_CORE_LOGLEVEL_SCHEMA = All(
Lower, Choices("info", "debug", "warning", "error")
)
SECTION_CORE_REMOTE = "remote"
SECTION_CORE_INTERACTIVE = "interactive"
SECTION_CORE_ANALYTICS = "analytics"
SECTION_CORE_CHECKSUM_JOBS = "checksum_jobs"
SECTION_CORE_HARDLINK_LOCK = "hardlink_lock"
SECTION_CACHE = "cache"
SECTION_CACHE_DIR = "dir"
SECTION_CACHE_TYPE = "type"
SECTION_CACHE_PROTECTED = "protected"
SECTION_CACHE_SHARED = "shared"
SECTION_CACHE_SHARED_SCHEMA = All(Lower, Choices("group"))
SECTION_CACHE_LOCAL = "local"
SECTION_CACHE_S3 = "s3"
SECTION_CACHE_GS = "gs"
SECTION_CACHE_SSH = "ssh"
SECTION_CACHE_HDFS = "hdfs"
SECTION_CACHE_AZURE = "azure"
SECTION_CACHE_SLOW_LINK_WARNING = "slow_link_warning"
SECTION_CACHE_SCHEMA = {
SECTION_CACHE_LOCAL: str,
SECTION_CACHE_S3: str,
SECTION_CACHE_GS: str,
SECTION_CACHE_HDFS: str,
SECTION_CACHE_SSH: str,
SECTION_CACHE_AZURE: str,
SECTION_CACHE_DIR: str,
SECTION_CACHE_TYPE: supported_cache_type,
Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
SECTION_CACHE_SHARED: SECTION_CACHE_SHARED_SCHEMA,
PRIVATE_CWD: str,
Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): Bool,
}
SECTION_CORE_SCHEMA = {
SECTION_CORE_LOGLEVEL: SECTION_CORE_LOGLEVEL_SCHEMA,
SECTION_CORE_REMOTE: Lower,
Optional(SECTION_CORE_INTERACTIVE, default=False): Bool,
Optional(SECTION_CORE_ANALYTICS, default=True): Bool,
SECTION_CORE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
Optional(SECTION_CORE_HARDLINK_LOCK, default=False): Bool,
}
# aws specific options
SECTION_AWS_CREDENTIALPATH = CREDENTIALPATH
SECTION_AWS_ENDPOINT_URL = "endpointurl"
SECTION_AWS_LIST_OBJECTS = "listobjects"
SECTION_AWS_REGION = "region"
SECTION_AWS_PROFILE = "profile"
SECTION_AWS_USE_SSL = "use_ssl"
SECTION_AWS_SSE = "sse"
SECTION_AWS_ACL = "acl"
SECTION_AWS_GRANT_READ = "grant_read"
SECTION_AWS_GRANT_READ_ACP = "grant_read_acp"
SECTION_AWS_GRANT_WRITE_ACP = "grant_write_acp"
SECTION_AWS_GRANT_FULL_CONTROL = "grant_full_control"
# gcp specific options
SECTION_GCP_CREDENTIALPATH = CREDENTIALPATH
SECTION_GCP_PROJECTNAME = "projectname"
# azure specific option
SECTION_AZURE_CONNECTION_STRING = "connection_string"
# Alibabacloud oss options
SECTION_OSS_ACCESS_KEY_ID = "oss_key_id"
SECTION_OSS_ACCESS_KEY_SECRET = "oss_key_secret"
SECTION_OSS_ENDPOINT = "oss_endpoint"
# GDrive options
SECTION_GDRIVE_CLIENT_ID = "gdrive_client_id"
SECTION_GDRIVE_CLIENT_SECRET = "gdrive_client_secret"
SECTION_GDRIVE_USER_CREDENTIALS_FILE = "gdrive_user_credentials_file"
SECTION_REMOTE_CHECKSUM_JOBS = "checksum_jobs"
SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
SECTION_REMOTE_FMT = 'remote "{}"'
SECTION_REMOTE_URL = "url"
SECTION_REMOTE_USER = "user"
SECTION_REMOTE_PORT = "port"
SECTION_REMOTE_KEY_FILE = "keyfile"
SECTION_REMOTE_TIMEOUT = "timeout"
SECTION_REMOTE_PASSWORD = "password"
SECTION_REMOTE_ASK_PASSWORD = "ask_password"
SECTION_REMOTE_GSS_AUTH = "gss_auth"
SECTION_REMOTE_NO_TRAVERSE = "no_traverse"
SECTION_REMOTE_VERIFY = "verify"
SECTION_REMOTE_SCHEMA = {
Required(SECTION_REMOTE_URL): str,
SECTION_AWS_REGION: str,
SECTION_AWS_PROFILE: str,
SECTION_AWS_CREDENTIALPATH: str,
SECTION_AWS_ENDPOINT_URL: str,
Optional(SECTION_AWS_LIST_OBJECTS, default=False): Bool,
Optional(SECTION_AWS_USE_SSL, default=True): Bool,
SECTION_AWS_SSE: str,
SECTION_AWS_ACL: str,
SECTION_AWS_GRANT_READ: str,
SECTION_AWS_GRANT_READ_ACP: str,
SECTION_AWS_GRANT_WRITE_ACP: str,
SECTION_AWS_GRANT_FULL_CONTROL: str,
SECTION_GCP_PROJECTNAME: str,
SECTION_CACHE_TYPE: supported_cache_type,
Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
SECTION_REMOTE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
SECTION_REMOTE_USER: str,
SECTION_REMOTE_PORT: Coerce(int),
SECTION_REMOTE_KEY_FILE: str,
SECTION_REMOTE_TIMEOUT: Coerce(int),
SECTION_REMOTE_PASSWORD: str,
SECTION_REMOTE_ASK_PASSWORD: Bool,
SECTION_REMOTE_GSS_AUTH: Bool,
SECTION_AZURE_CONNECTION_STRING: str,
SECTION_OSS_ACCESS_KEY_ID: str,
SECTION_OSS_ACCESS_KEY_SECRET: str,
SECTION_OSS_ENDPOINT: str,
SECTION_GDRIVE_CLIENT_ID: str,
SECTION_GDRIVE_CLIENT_SECRET: str,
SECTION_GDRIVE_USER_CREDENTIALS_FILE: str,
PRIVATE_CWD: str,
SECTION_REMOTE_NO_TRAVERSE: Bool,
SECTION_REMOTE_VERIFY: Bool,
}
SECTION_STATE = "state"
SECTION_STATE_ROW_LIMIT = "row_limit"
SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota"
SECTION_STATE_SCHEMA = {
SECTION_STATE_ROW_LIMIT: All(Coerce(int), Range(1)),
SECTION_STATE_ROW_CLEANUP_QUOTA: All(Coerce(int), Range(0, 100)),
}
SCHEMA = {
Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
Match(SECTION_REMOTE_REGEX): SECTION_REMOTE_SCHEMA,
Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
}
COMPILED_SCHEMA = Schema(SCHEMA)
def __init__(self, dvc_dir=None, validate=True):
self.dvc_dir = dvc_dir
self.should_validate = validate
if not dvc_dir:
try:
@@ -258,33 +221,31 @@ class Config(object): # pylint: disable=too-many-instance-attributes
else:
self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
self.load()
self.load(validate=validate)
@staticmethod
def get_global_config_dir():
"""Returns global config location. E.g. ~/.config/dvc/config.
@classmethod
def get_dir(cls, level):
from appdirs import user_config_dir, site_config_dir
Returns:
str: path to the global config directory.
"""
from appdirs import user_config_dir
assert level in ("global", "system")
return user_config_dir(
appname=Config.APPNAME, appauthor=Config.APPAUTHOR
)
if level == "global":
return user_config_dir(cls.APPNAME, cls.APPAUTHOR)
if level == "system":
return site_config_dir(cls.APPNAME, cls.APPAUTHOR)
@staticmethod
def get_system_config_dir():
"""Returns system config location. E.g. /etc/dvc.conf.
@cached_property
def files(self):
files = {
level: os.path.join(self.get_dir(level), self.CONFIG)
for level in ("system", "global")
}
Returns:
str: path to the system config directory.
"""
from appdirs import site_config_dir
if self.dvc_dir is not None:
files["repo"] = os.path.join(self.dvc_dir, self.CONFIG)
files["local"] = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)
return site_config_dir(
appname=Config.APPNAME, appauthor=Config.APPAUTHOR
)
return files
@staticmethod
def init(dvc_dir):
@@ -300,250 +261,137 @@ class Config(object): # pylint: disable=too-many-instance-attributes
open(config_file, "w+").close()
return Config(dvc_dir)
def _resolve_cache_path(self, config):
cache = config.get(self.SECTION_CACHE)
if cache is None:
return
cache_dir = cache.get(self.SECTION_CACHE_DIR)
if cache_dir is None:
return
cache[self.PRIVATE_CWD] = os.path.dirname(config.filename)
def _resolve_paths(self, config):
if config.filename is None:
return config
ret = copy.deepcopy(config)
self._resolve_cache_path(ret)
for section in ret.values():
if self.SECTION_REMOTE_URL not in section.keys():
continue
section[self.PRIVATE_CWD] = os.path.dirname(ret.filename)
return ret
def _load_configs(self):
system_config_file = os.path.join(
self.get_system_config_dir(), self.CONFIG
)
global_config_file = os.path.join(
self.get_global_config_dir(), self.CONFIG
)
self._system_config = configobj.ConfigObj(system_config_file)
self._global_config = configobj.ConfigObj(global_config_file)
self._repo_config = configobj.ConfigObj()
self._local_config = configobj.ConfigObj()
if not self.dvc_dir:
return
config_file = os.path.join(self.dvc_dir, self.CONFIG)
config_local_file = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)
self._repo_config = configobj.ConfigObj(config_file)
self._local_config = configobj.ConfigObj(config_local_file)
@property
def config_local_file(self):
return self._local_config.filename
@property
def config_file(self):
return self._repo_config.filename
def load(self):
def load(self, validate=True):
"""Loads config from all the config files.
Raises:
dvc.config.ConfigError: thrown if config has invalid format.
"""
self._load_configs()
conf = {}
for level in self.LEVELS:
if level in self.files:
_merge(conf, self.load_one(level))
self.config = configobj.ConfigObj()
for c in [
self._system_config,
self._global_config,
self._repo_config,
self._local_config,
]:
c = self._resolve_paths(c)
c = self._lower(c)
self.config.merge(c)
if validate:
conf = self.validate(conf)
if not self.should_validate:
return
self.clear()
self.update(conf)
d = self.validate(self.config)
self.config = configobj.ConfigObj(d, write_empty_values=True)
# Add resolved default cache.dir
if not self["cache"].get("dir"):
self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
def save(self, config=None):
"""Saves config to config files.
def load_one(self, level):
conf = _load_config(self.files[level])
conf = self._load_paths(conf, self.files[level])
Raises:
dvc.config.ConfigError: thrown if failed to write config file.
"""
if config is not None:
clist = [config]
else:
clist = [
self._system_config,
self._global_config,
self._repo_config,
self._local_config,
]
# Autovivify sections
for key in COMPILED_SCHEMA.schema:
conf.setdefault(key, {})
for conf in clist:
self._save(conf)
return conf
@staticmethod
def _load_paths(conf, filename):
abs_conf_dir = os.path.abspath(os.path.dirname(filename))
def resolve(path):
if os.path.isabs(path) or re.match(r"\w+://", path):
return path
return RelPath(os.path.join(abs_conf_dir, path))
return Config._map_dirs(conf, resolve)
@staticmethod
def _save_paths(conf, filename):
conf_dir = os.path.dirname(filename)
def rel(path):
if re.match(r"\w+://", path):
return path
if isinstance(path, RelPath) or not os.path.isabs(path):
return relpath(path, conf_dir)
return path
return Config._map_dirs(conf, rel)
@staticmethod
def _map_dirs(conf, func):
dirs_schema = {"cache": {"dir": func}, "remote": {str: {"url": func}}}
return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
@contextmanager
def edit(self, level="repo"):
if level in {"repo", "local"} and self.dvc_dir is None:
raise ConfigError("Not inside a dvc repo")
conf = self.load_one(level)
yield conf
conf = self._save_paths(conf, self.files[level])
_save_config(self.files[level], conf)
self.load()
@staticmethod
def _save(config):
if config.filename is None:
return
logger.debug("Writing '{}'.".format(config.filename))
dname = os.path.dirname(os.path.abspath(config.filename))
def validate(data):
try:
os.makedirs(dname)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
config.write()
def validate(self, config):
try:
return self.COMPILED_SCHEMA(config.dict())
return COMPILED_SCHEMA(data)
except Invalid as exc:
raise ConfigError(str(exc)) from exc
raise ConfigError(str(exc)) from None
def unset(self, section, opt=None, level=None, force=False):
"""Unsets specified option and/or section in the config.
Args:
section (str): section name.
opt (str): optional option name.
level (int): config level to use.
force (bool): don't error-out even if section doesn't exist. False
by default.
def _load_config(filename):
conf_obj = configobj.ConfigObj(filename)
return _parse_remotes(_lower_keys(conf_obj.dict()))
Raises:
dvc.config.ConfigError: thrown if section doesn't exist and
`force != True`.
"""
config = self.get_configobj(level)
if section not in config.keys():
if force:
return
raise ConfigError("section '{}' doesn't exist".format(section))
def _save_config(filename, conf_dict):
logger.debug("Writing '{}'.".format(filename))
os.makedirs(os.path.dirname(filename), exist_ok=True)
if opt:
if opt not in config[section].keys():
if force:
return
raise ConfigError(
"option '{}.{}' doesn't exist".format(section, opt)
)
del config[section][opt]
config = configobj.ConfigObj(_pack_remotes(conf_dict))
config.filename = filename
config.write()
if not config[section]:
del config[section]
def _parse_remotes(conf):
result = {"remote": {}}
for section, val in conf.items():
name = re_find(r'^\s*remote\s*"(.*)"\s*$', section)
if name:
result["remote"][name] = val
else:
del config[section]
result[section] = val
self.save(config)
return result
def set(self, section, opt, value, level=None, force=True):
"""Sets specified option in the config.
Args:
section (str): section name.
opt (str): option name.
value: value to set option to.
level (int): config level to use.
force (bool): set option even if section already exists. True by
default.
def _pack_remotes(conf):
# Drop empty sections
result = compact(conf)
Raises:
dvc.config.ConfigError: thrown if section already exists and
`force != True`.
# Transform remote.name -> 'remote "name"'
for name, val in conf["remote"].items():
result['remote "{}"'.format(name)] = val
result.pop("remote", None)
"""
config = self.get_configobj(level)
return result
if section not in config.keys():
config[section] = {}
elif not force:
raise ConfigError(
"Section '{}' already exists. Use `-f|--force` to overwrite "
"section with new value.".format(section)
)
config[section][opt] = value
def _merge(into, update):
"""Merges second dict into first recursively"""
for key, val in update.items():
if isinstance(into.get(key), dict) and isinstance(val, dict):
_merge(into[key], val)
else:
into[key] = val
result = copy.deepcopy(self.config)
result.merge(config)
self.validate(result)
self.save(config)
def get(self, section, opt=None, level=None):
"""Return option value from the config.
Args:
section (str): section name.
opt (str): option name.
level (int): config level to use.
Returns:
value (str, int): option value.
"""
config = self.get_configobj(level)
if section not in config.keys():
raise ConfigError("section '{}' doesn't exist".format(section))
if opt not in config[section].keys():
raise ConfigError(
"option '{}.{}' doesn't exist".format(section, opt)
)
return config[section][opt]
@staticmethod
def _lower(config):
new_config = configobj.ConfigObj()
for s_key, s_value in config.items():
new_s = {}
for key, value in s_value.items():
new_s[key.lower()] = str(value)
new_config[s_key.lower()] = new_s
return new_config
def get_configobj(self, level):
configs = {
self.LEVEL_LOCAL: self._local_config,
self.LEVEL_REPO: self._repo_config,
self.LEVEL_GLOBAL: self._global_config,
self.LEVEL_SYSTEM: self._system_config,
}
return configs.get(level, self._repo_config)
def list_options(self, section_regex, option, level=None):
ret = {}
config = self.get_configobj(level)
for section in config.keys():
r = re.match(section_regex, section)
if r:
name = r.group("name")
value = config[section].get(option, "")
ret[name] = value
return ret
def _lower_keys(data):
return {
k.lower(): _lower_keys(v) if isinstance(v, dict) else v
for k, v in data.items()
}
dvc/data_cloud.py
+2 -17
View File
@@ -2,7 +2,6 @@
import logging
from dvc.config import Config
from dvc.config import NoRemoteError
from dvc.remote import Remote
@@ -24,28 +23,14 @@ class DataCloud(object):
def __init__(self, repo):
self.repo = repo
@property
def _config(self):
return self.repo.config.config
@property
def _core(self):
return self._config.get(Config.SECTION_CORE, {})
def get_remote(self, remote=None, command="<command>"):
if not remote:
remote = self._core.get(Config.SECTION_CORE_REMOTE)
remote = self.repo.config["core"].get("remote")
if remote:
return self._init_remote(remote)
has_non_default_remote = bool(
self.repo.config.list_options(
Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL
)
)
if has_non_default_remote:
if bool(self.repo.config["remote"]):
error_msg = (
"no remote specified. Setup default remote with\n"
" dvc remote default <remote name>\n"
dvc/external_repo.py
+8 -10
View File
@@ -9,11 +9,10 @@ from funcy import retry, suppress, wrap_with, cached_property
from dvc.compat import fspath
from dvc.repo import Repo
from dvc.config import Config, NoRemoteError, NotDvcRepoError
from dvc.config import NoRemoteError, NotDvcRepoError
from dvc.exceptions import NoRemoteInExternalRepoError
from dvc.exceptions import OutputNotFoundError, NoOutputInExternalRepoError
from dvc.exceptions import FileMissingError, PathMissingError
from dvc.remote import RemoteConfig
from dvc.utils.fs import remove, fs_copy
from dvc.scm.git import Git
@@ -110,19 +109,18 @@ class ExternalRepo(Repo):
# check if the URL is local and no default remote is present
# add default remote pointing to the original repo's cache location
if os.path.isdir(self.url):
rconfig = RemoteConfig(self.config)
if not rconfig.has_default():
if not self.config["core"].get("remote"):
src_repo = Repo(self.url)
try:
rconfig.add(
"auto-generated-upstream",
src_repo.cache.local.cache_dir,
default=True,
level=Config.LEVEL_LOCAL,
)
cache_dir = src_repo.cache.local.cache_dir
finally:
src_repo.close()
self.config["remote"]["auto-generated-upstream"] = {
"url": cache_dir
}
self.config["core"]["remote"] = "auto-generated-upstream"
class ExternalGitRepo:
def __init__(self, root_dir, url):
dvc/remote/__init__.py
+40 -7
View File
@@ -1,4 +1,6 @@
from .config import RemoteConfig
import posixpath
from urllib.parse import urlparse
from dvc.remote.azure import RemoteAZURE
from dvc.remote.gdrive import RemoteGDrive
from dvc.remote.gs import RemoteGS
@@ -25,9 +27,9 @@ REMOTES = [
]
def _get(config):
def _get(remote_conf):
for remote in REMOTES:
if remote.supported(config):
if remote.supported(remote_conf):
return remote
return RemoteLOCAL
@@ -35,8 +37,39 @@ def _get(config):
def Remote(repo, **kwargs):
name = kwargs.get("name")
if name:
remote_config = RemoteConfig(repo.config)
settings = remote_config.get_settings(name)
remote_conf = repo.config["remote"][name.lower()]
else:
settings = kwargs
return _get(settings)(repo, settings)
remote_conf = kwargs
remote_conf = _resolve_remote_refs(repo.config, remote_conf)
return _get(remote_conf)(repo, remote_conf)
def _resolve_remote_refs(config, remote_conf):
# Support for cross referenced remotes.
# This will merge the settings, shadowing base ref with remote_conf.
# For example, having:
#
# dvc remote add server ssh://localhost
# dvc remote modify server user root
# dvc remote modify server ask_password true
#
# dvc remote add images remote://server/tmp/pictures
# dvc remote modify images user alice
# dvc remote modify images ask_password false
# dvc remote modify images password asdf1234
#
# Results on a config dictionary like:
#
# {
# "url": "ssh://localhost/tmp/pictures",
# "user": "alice",
# "password": "asdf1234",
# "ask_password": False,
# }
parsed = urlparse(remote_conf["url"])
if parsed.scheme != "remote":
return remote_conf
base = config["remote"][parsed.netloc]
url = posixpath.join(base["url"], parsed.path.lstrip("/"))
return {**base, **remote_conf, "url": url}
dvc/remote/azure.py
+2 -3
View File
@@ -7,7 +7,6 @@ import threading
from funcy import cached_property, wrap_prop
from dvc.config import Config
from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.remote.base import RemoteBASE
@@ -33,7 +32,7 @@ class RemoteAZURE(RemoteBASE):
def __init__(self, repo, config):
super().__init__(repo, config)
url = config.get(Config.SECTION_REMOTE_URL, "azure://")
url = config.get("url", "azure://")
match = re.match(self.REGEX, url) # backward compatibility
path = match.group("path")
@@ -44,7 +43,7 @@ class RemoteAZURE(RemoteBASE):
)
self.connection_string = (
config.get(Config.SECTION_AZURE_CONNECTION_STRING)
config.get("connection_string")
or match.group("connection_string") # backward compatibility
or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
)
dvc/remote/base.py
+10 -33
View File
@@ -13,7 +13,6 @@ from operator import itemgetter
from shortuuid import uuid
import dvc.prompt as prompt
from dvc.config import Config
from dvc.exceptions import (
DvcException,
ConfirmRemoveError,
@@ -89,24 +88,18 @@ class RemoteBASE(object):
self.repo = repo
self._check_requires(config)
self.checksum_jobs = self._get_checksum_jobs(config)
self.checksum_jobs = (
config.get("checksum_jobs")
or (self.repo and self.repo.config["core"].get("checksum_jobs"))
or self.CHECKSUM_JOBS
)
self.protected = False
self.no_traverse = config.get(
Config.SECTION_REMOTE_NO_TRAVERSE, self.DEFAULT_NO_TRAVERSE
)
self.verify = config.get(
Config.SECTION_REMOTE_VERIFY, self.DEFAULT_VERIFY
)
self.no_traverse = config.get("no_traverse", self.DEFAULT_NO_TRAVERSE)
self.verify = config.get("verify", self.DEFAULT_VERIFY)
self._dir_info = {}
types = config.get(Config.SECTION_CACHE_TYPE, None)
if types:
if isinstance(types, str):
types = [t.strip() for t in types.split(",")]
self.cache_types = types
else:
self.cache_types = copy(self.DEFAULT_CACHE_TYPES)
self.cache_types = config.get("type") or copy(self.DEFAULT_CACHE_TYPES)
self.cache_type_confirmed = False
def _check_requires(self, config):
@@ -123,9 +116,7 @@ class RemoteBASE(object):
if not missing:
return
url = config.get(
Config.SECTION_REMOTE_URL, "{}://".format(self.scheme)
)
url = config.get("url", "{}://".format(self.scheme))
msg = (
"URL '{}' is supported but requires these missing "
"dependencies: {}. If you have installed dvc using pip, "
@@ -146,19 +137,6 @@ class RemoteBASE(object):
).format(url, missing, " ".join(missing), self.scheme)
raise RemoteMissingDepsError(msg)
def _get_checksum_jobs(self, config):
checksum_jobs = config.get(Config.SECTION_REMOTE_CHECKSUM_JOBS)
if checksum_jobs:
return checksum_jobs
if self.repo:
core = self.repo.config.config.get(Config.SECTION_CORE, {})
return core.get(
Config.SECTION_CORE_CHECKSUM_JOBS, self.CHECKSUM_JOBS
)
return self.CHECKSUM_JOBS
def __repr__(self):
return "{class_name}: '{path_info}'".format(
class_name=type(self).__name__,
@@ -170,7 +148,7 @@ class RemoteBASE(object):
if isinstance(config, (str, bytes)):
url = config
else:
url = config[Config.SECTION_REMOTE_URL]
url = config["url"]
# NOTE: silently skipping remote, calling code should handle that
parsed = urlparse(url)
@@ -893,7 +871,6 @@ class RemoteBASE(object):
"""Optional: Implement only if the remote needs to create
directories before copying/linking/moving data
"""
pass
def _checkout_dir(
self,
dvc/remote/config.py
-163
View File
@@ -1,163 +0,0 @@
import logging
import os
import posixpath
from urllib.parse import urlparse
from dvc.config import Config, ConfigError
from dvc.utils import relpath
logger = logging.getLogger(__name__)
class RemoteConfig(object):
def __init__(self, config):
self.config = config
def get_settings(self, name):
"""
Args:
name (str): The name of the remote that we want to retrieve
Returns:
dict: The content beneath the given remote name.
Example:
>>> config = {'remote "server"': {'url': 'ssh://localhost/'}}
>>> get_settings("server")
{'url': 'ssh://localhost/'}
"""
settings = self.config.config.get(
Config.SECTION_REMOTE_FMT.format(name.lower())
)
if settings is None:
raise ConfigError(
"Unable to find remote section '{}'".format(name)
)
parsed = urlparse(settings["url"])
# Support for cross referenced remotes.
# This will merge the settings, giving priority to the outer reference.
# For example, having:
#
# dvc remote add server ssh://localhost
# dvc remote modify server user root
# dvc remote modify server ask_password true
#
# dvc remote add images remote://server/tmp/pictures
# dvc remote modify images user alice
# dvc remote modify images ask_password false
# dvc remote modify images password asdf1234
#
# Results on a config dictionary like:
#
# {
# "url": "ssh://localhost/tmp/pictures",
# "user": "alice",
# "password": "asdf1234",
# "ask_password": False,
# }
#
if parsed.scheme == "remote":
reference = self.get_settings(parsed.netloc)
url = posixpath.join(reference["url"], parsed.path.lstrip("/"))
merged = reference.copy()
merged.update(settings)
merged["url"] = url
return merged
return settings
@staticmethod
def resolve_path(path, config_file):
"""Resolve path relative to config file location.
Args:
path: Path to be resolved.
config_file: Path to config file, which `path` is specified
relative to.
Returns:
Path relative to the `config_file` location. If `path` is an
absolute path then it will be returned without change.
"""
if os.path.isabs(path):
return path
return relpath(path, os.path.dirname(config_file))
def add(self, name, url, default=False, force=False, level=None):
from dvc.remote import _get, RemoteLOCAL
configobj = self.config.get_configobj(level)
remote = _get({Config.SECTION_REMOTE_URL: url})
if remote == RemoteLOCAL and not url.startswith("remote://"):
url = self.resolve_path(url, configobj.filename)
self.config.set(
Config.SECTION_REMOTE_FMT.format(name),
Config.SECTION_REMOTE_URL,
url,
force=force,
level=level,
)
if default:
self.config.set(
Config.SECTION_CORE,
Config.SECTION_CORE_REMOTE,
name,
level=level,
)
def remove(self, name, level=None):
self.config.unset(Config.SECTION_REMOTE_FMT.format(name), level=level)
if level is None:
level = Config.LEVEL_REPO
for lev in [
Config.LEVEL_LOCAL,
Config.LEVEL_REPO,
Config.LEVEL_GLOBAL,
Config.LEVEL_SYSTEM,
]:
self.config.unset(
Config.SECTION_CORE,
Config.SECTION_CORE_REMOTE,
level=lev,
force=True,
)
if lev == level:
break
def modify(self, name, option, value, level=None):
# try to get remote settings to verify that section exists
self.get_settings(name)
self.config.set(
Config.SECTION_REMOTE_FMT.format(name), option, value, level=level
)
def list(self, level=None):
return self.config.list_options(
Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL, level=level
)
def set_default(self, name, unset=False, level=None):
if unset:
self.config.unset(Config.SECTION_CORE, Config.SECTION_CORE_REMOTE)
return
self.config.set(
Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, name, level=level
)
def get_default(self, level=None):
return self.config.get(
Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, level=level
)
def has_default(self):
core = self.config.config[Config.SECTION_CORE]
return bool(core.get(Config.SECTION_CORE_REMOTE))
dvc/remote/gdrive.py
+7 -13
View File
@@ -12,7 +12,6 @@ from dvc.progress import Tqdm
from dvc.scheme import Schemes
from dvc.path_info import CloudURLInfo
from dvc.remote.base import RemoteBASE
from dvc.config import Config
from dvc.exceptions import DvcException
from dvc.utils import tmp_fname, format_link
@@ -95,25 +94,20 @@ class RemoteGDrive(RemoteBASE):
def __init__(self, repo, config):
super().__init__(repo, config)
url = config[Config.SECTION_REMOTE_URL]
self.path_info = self.path_cls(url)
self.config = config
self.path_info = self.path_cls(config["url"])
if not self.path_info.bucket:
raise DvcException(
"Empty Google Drive URL '{}'. Learn more at "
"{}.".format(
url, format_link("https://man.dvc.org/remote/add")
config["url"],
format_link("https://man.dvc.org/remote/add"),
)
)
self._bucket = self.path_info.bucket
self._client_id = self.config.get(
Config.SECTION_GDRIVE_CLIENT_ID, None
)
self._client_secret = self.config.get(
Config.SECTION_GDRIVE_CLIENT_SECRET, None
)
self._client_id = config.get("gdrive_client_id")
self._client_secret = config.get("gdrive_client_secret")
if not self._client_id or not self._client_secret:
raise DvcException(
"Please specify Google Drive's client id and "
@@ -123,8 +117,8 @@ class RemoteGDrive(RemoteBASE):
self._gdrive_user_credentials_path = (
tmp_fname(os.path.join(self.repo.tmp_dir, ""))
if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
else self.config.get(
Config.SECTION_GDRIVE_USER_CREDENTIALS_FILE,
else config.get(
"gdrive_user_credentials_file",
os.path.join(
self.repo.tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE
),
dvc/remote/gs.py
+3 -4
View File
@@ -7,7 +7,6 @@ import threading
from funcy import cached_property, wrap_prop
from dvc.config import Config
from dvc.exceptions import DvcException
from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
@@ -75,11 +74,11 @@ class RemoteGS(RemoteBASE):
def __init__(self, repo, config):
super().__init__(repo, config)
url = config.get(Config.SECTION_REMOTE_URL, "gs:///")
url = config.get("url", "gs:///")
self.path_info = self.path_cls(url)
self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
self.credentialpath = config.get(Config.SECTION_GCP_CREDENTIALPATH)
self.projectname = config.get("projectname", None)
self.credentialpath = config.get("credentialpath")
@wrap_prop(threading.Lock())
@cached_property
dvc/remote/hdfs.py
+2 -4
View File
@@ -10,7 +10,6 @@ from urllib.parse import urlparse
from .base import RemoteBASE, RemoteCmdError
from .pool import get_connection
from dvc.config import Config
from dvc.scheme import Schemes
from dvc.utils import fix_env, tmp_fname
@@ -26,13 +25,12 @@ class RemoteHDFS(RemoteBASE):
def __init__(self, repo, config):
super().__init__(repo, config)
self.path_info = None
url = config.get(Config.SECTION_REMOTE_URL)
url = config.get("url")
if not url:
return
parsed = urlparse(url)
user = parsed.username or config.get(Config.SECTION_REMOTE_USER)
user = parsed.username or config.get("user")
self.path_info = self.path_cls.from_parts(
scheme=self.scheme,
dvc/remote/http.py
+1 -2
View File
@@ -3,7 +3,6 @@ import threading
from funcy import cached_property, wrap_prop
from dvc.config import Config
from dvc.config import ConfigError
from dvc.exceptions import DvcException, HTTPError
from dvc.progress import Tqdm
@@ -24,7 +23,7 @@ class RemoteHTTP(RemoteBASE):
def __init__(self, repo, config):
super().__init__(repo, config)
url = config.get(Config.SECTION_REMOTE_URL)
url = config.get("url")
self.path_info = self.path_cls(url) if url else None
if not self.no_traverse:
dvc/remote/local.py
+9 -25
View File
@@ -7,27 +7,17 @@ from functools import partial
from shortuuid import uuid
from dvc.config import Config
from dvc.exceptions import DownloadError
from dvc.exceptions import DvcException
from dvc.exceptions import UploadError
from dvc.compat import fspath_py35
from dvc.exceptions import DvcException, DownloadError, UploadError
from dvc.path_info import PathInfo
from dvc.progress import Tqdm
from dvc.remote.base import RemoteBASE
from dvc.remote.base import STATUS_DELETED
from dvc.remote.base import STATUS_MAP
from dvc.remote.base import STATUS_MISSING
from dvc.remote.base import STATUS_NEW
from dvc.remote.base import RemoteBASE, STATUS_MAP
from dvc.remote.base import STATUS_DELETED, STATUS_MISSING, STATUS_NEW
from dvc.scheme import Schemes
from dvc.scm.tree import is_working_tree
from dvc.system import System
from dvc.utils.fs import copyfile
from dvc.utils import file_md5
from dvc.utils import relpath
from dvc.utils import tmp_fname
from dvc.compat import fspath_py35
from dvc.utils.fs import move, makedirs, remove
from dvc.utils.fs import walk_files
from dvc.utils import file_md5, relpath, tmp_fname
from dvc.utils.fs import copyfile, move, makedirs, remove, walk_files
logger = logging.getLogger(__name__)
@@ -46,22 +36,16 @@ class RemoteLOCAL(RemoteBASE):
def __init__(self, repo, config):
super().__init__(repo, config)
self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
self.protected = config.get("protected", False)
shared = config.get(Config.SECTION_CACHE_SHARED)
shared = config.get("shared")
self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared]
if self.protected:
# cache files are set to be read-only for everyone
self._file_mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH
cache_dir = config.get(Config.SECTION_REMOTE_URL)
if cache_dir is not None and not os.path.isabs(cache_dir):
cwd = config[Config.PRIVATE_CWD]
cache_dir = os.path.abspath(os.path.join(cwd, cache_dir))
self.cache_dir = cache_dir
self.cache_dir = config.get("url")
self._dir_info = {}
@property
dvc/remote/oss.py
+4 -7
View File
@@ -4,7 +4,6 @@ import threading
from funcy import cached_property, wrap_prop
from dvc.config import Config
from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.remote.base import RemoteBASE
@@ -42,21 +41,19 @@ class RemoteOSS(RemoteBASE):
def __init__(self, repo, config):
super().__init__(repo, config)
url = config.get(Config.SECTION_REMOTE_URL)
url = config.get("url")
self.path_info = self.path_cls(url) if url else None
self.endpoint = config.get(Config.SECTION_OSS_ENDPOINT) or os.getenv(
"OSS_ENDPOINT"
)
self.endpoint = config.get("oss_endpoint") or os.getenv("OSS_ENDPOINT")
self.key_id = (
config.get(Config.SECTION_OSS_ACCESS_KEY_ID)
config.get("oss_key_id")
or os.getenv("OSS_ACCESS_KEY_ID")
or "defaultId"
)
self.key_secret = (
config.get(Config.SECTION_OSS_ACCESS_KEY_SECRET)
config.get("oss_key_secret")
or os.getenv("OSS_ACCESS_KEY_SECRET")
or "defaultSecret"
)
dvc/remote/s3.py
+15 -17
View File
@@ -6,7 +6,7 @@ import threading
from funcy import cached_property, wrap_prop
from dvc.config import Config, ConfigError
from dvc.config import ConfigError
from dvc.exceptions import DvcException
from dvc.exceptions import ETagMismatchError
from dvc.path_info import CloudURLInfo
@@ -26,35 +26,33 @@ class RemoteS3(RemoteBASE):
def __init__(self, repo, config):
super().__init__(repo, config)
url = config.get(Config.SECTION_REMOTE_URL, "s3://")
url = config.get("url", "s3://")
self.path_info = self.path_cls(url)
self.region = config.get(Config.SECTION_AWS_REGION)
self.region = config.get("region")
self.profile = config.get("profile")
self.endpoint_url = config.get("endpointurl")
self.profile = config.get(Config.SECTION_AWS_PROFILE)
self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
if config.get(Config.SECTION_AWS_LIST_OBJECTS):
if config.get("listobjects"):
self.list_objects_api = "list_objects"
else:
self.list_objects_api = "list_objects_v2"
self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True)
self.use_ssl = config.get("use_ssl", True)
self.extra_args = {}
self.sse = config.get(Config.SECTION_AWS_SSE, "")
self.sse = config.get("sse")
if self.sse:
self.extra_args["ServerSideEncryption"] = self.sse
self.acl = config.get(Config.SECTION_AWS_ACL, "")
self.acl = config.get("acl")
if self.acl:
self.extra_args["ACL"] = self.acl
self._append_aws_grants_to_extra_args(config)
shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
shared_creds = config.get("credentialpath")
if shared_creds:
os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
@@ -322,14 +320,14 @@ class RemoteS3(RemoteBASE):
"""
grants = {
Config.SECTION_AWS_GRANT_FULL_CONTROL: "GrantFullControl",
Config.SECTION_AWS_GRANT_READ: "GrantRead",
Config.SECTION_AWS_GRANT_READ_ACP: "GrantReadACP",
Config.SECTION_AWS_GRANT_WRITE_ACP: "GrantWriteACP",
"grant_full_control": "GrantFullControl",
"grant_read": "GrantRead",
"grant_read_acp": "GrantReadACP",
"grant_write_acp": "GrantWriteACP",
}
for grant_option, extra_args_key in grants.items():
if config.get(grant_option, ""):
if config.get(grant_option):
if self.acl:
raise ConfigError(
"`acl` and `grant_*` AWS S3 config options "
dvc/remote/slow_link_detection.py
+4 -8
View File
@@ -1,12 +1,10 @@
import logging
import sys
import time
from functools import wraps
import colorama
from dvc.config import Config
logger = logging.getLogger(__name__)
this = sys.modules[__name__]
@@ -31,11 +29,9 @@ def slow_link_guard(f):
if this.already_displayed:
return f(remote, *args, **kwargs)
config = remote.repo.config.config.get(Config.SECTION_CACHE, {})
cache_type = config.get(Config.SECTION_CACHE_TYPE)
should_warn = config.get(Config.SECTION_CACHE_SLOW_LINK_WARNING, True)
if not should_warn or cache_type:
cache_conf = remote.repo.config["cache"]
slow_link_warning = cache_conf.get("slow_link_warning", True)
if not slow_link_warning or cache_conf.get("type"):
return f(remote, *args, **kwargs)
start = time.time()
dvc/remote/ssh/__init__.py
+11 -20
View File
@@ -9,10 +9,9 @@ from concurrent.futures import ThreadPoolExecutor
from contextlib import closing, contextmanager
from urllib.parse import urlparse
from funcy import memoize, wrap_with
from funcy import memoize, wrap_with, silent, first
import dvc.prompt as prompt
from dvc.config import Config
from dvc.progress import Tqdm
from dvc.remote.base import RemoteBASE
from dvc.remote.pool import get_connection
@@ -50,20 +49,20 @@ class RemoteSSH(RemoteBASE):
def __init__(self, repo, config):
super().__init__(repo, config)
url = config.get(Config.SECTION_REMOTE_URL)
url = config.get("url")
if url:
parsed = urlparse(url)
user_ssh_config = self._load_user_ssh_config(parsed.hostname)
host = user_ssh_config.get("hostname", parsed.hostname)
user = (
config.get(Config.SECTION_REMOTE_USER)
config.get("user")
or parsed.username
or user_ssh_config.get("user")
or getpass.getuser()
)
port = (
config.get(Config.SECTION_REMOTE_PORT)
config.get("port")
or parsed.port
or self._try_get_ssh_config_port(user_ssh_config)
or self.DEFAULT_PORT
@@ -80,14 +79,12 @@ class RemoteSSH(RemoteBASE):
user_ssh_config = {}
self.keyfile = config.get(
Config.SECTION_REMOTE_KEY_FILE
"keyfile"
) or self._try_get_ssh_config_keyfile(user_ssh_config)
self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
self.ask_password = config.get(
Config.SECTION_REMOTE_ASK_PASSWORD, False
)
self.gss_auth = config.get(Config.SECTION_REMOTE_GSS_AUTH, False)
self.timeout = config.get("timeout", self.TIMEOUT)
self.password = config.get("password", None)
self.ask_password = config.get("ask_password", False)
self.gss_auth = config.get("gss_auth", False)
@staticmethod
def ssh_config_filename():
@@ -110,17 +107,11 @@ class RemoteSSH(RemoteBASE):
@staticmethod
def _try_get_ssh_config_port(user_ssh_config):
try:
return int(user_ssh_config.get("port"))
except (ValueError, TypeError):
return None
return silent(int)(user_ssh_config.get("port"))
@staticmethod
def _try_get_ssh_config_keyfile(user_ssh_config):
identity_file = user_ssh_config.get("identityfile")
if identity_file and len(identity_file) > 0:
return identity_file[0]
return None
return first(user_ssh_config.get("identityfile") or ())
def ensure_credentials(self, path_info=None):
if path_info is None:
dvc/repo/__init__.py
+4 -6
View File
@@ -88,7 +88,7 @@ class Repo(object):
self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
makedirs(self.tmp_dir, exist_ok=True)
hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
hardlink_lock = self.config["core"].get("hardlink_lock", False)
self.lock = make_lock(
os.path.join(self.dvc_dir, "lock"),
tmp_dir=os.path.join(self.dvc_dir, "tmp"),
@@ -98,11 +98,9 @@ class Repo(object):
# NOTE: storing state and link_state in the repository itself to avoid
# any possible state corruption in 'shared cache dir' scenario.
self.state = State(self, self.config.config)
self.state = State(self)
core = self.config.config[Config.SECTION_CORE]
level = core.get(Config.SECTION_CORE_LOGLEVEL)
level = self.config.get("core", {}).get("loglevel")
if level:
logger.setLevel(level.upper())
@@ -170,7 +168,7 @@ class Repo(object):
updater = Updater(self.dvc_dir)
flist = (
[self.config.config_local_file, updater.updater_file]
[self.config.files["local"], updater.updater_file]
+ [self.lock.lockfile, updater.lock.lockfile, self.tmp_dir]
+ self.state.files
)
dvc/repo/init.py
+1 -1
View File
@@ -89,7 +89,7 @@ def init(root_dir=os.curdir, no_scm=False, force=False):
config = Config.init(dvc_dir)
proj = Repo(root_dir)
scm.add([config.config_file])
scm.add([config.files["repo"]])
if scm.ignore_file:
scm.add([os.path.join(dvc_dir, scm.ignore_file)])
dvc/repo/reproduce.py
+1 -5
View File
@@ -65,11 +65,7 @@ def reproduce(
interactive = kwargs.get("interactive", False)
if not interactive:
config = self.config
core = config.config[config.SECTION_CORE]
kwargs["interactive"] = core.get(
config.SECTION_CORE_INTERACTIVE, False
)
kwargs["interactive"] = self.config["core"].get("interactive", False)
active_graph = _get_active_graph(self.graph)
active_pipelines = get_pipelines(active_graph)
dvc/state.py
+4 -8
View File
@@ -6,7 +6,6 @@ import re
import sqlite3
from urllib.parse import urlunparse, urlencode
from dvc.config import Config
from dvc.exceptions import DvcException
from dvc.utils import current_timestamp
from dvc.utils import relpath
@@ -95,18 +94,15 @@ class State(object): # pylint: disable=too-many-instance-attributes
MAX_INT = 2 ** 63 - 1
MAX_UINT = 2 ** 64 - 2
def __init__(self, repo, config):
def __init__(self, repo):
self.repo = repo
self.dvc_dir = repo.dvc_dir
self.root_dir = repo.root_dir
state_config = config.get(Config.SECTION_STATE, {})
self.row_limit = state_config.get(
Config.SECTION_STATE_ROW_LIMIT, self.STATE_ROW_LIMIT
)
state_config = repo.config.get("state", {})
self.row_limit = state_config.get("row_limit", self.STATE_ROW_LIMIT)
self.row_cleanup_quota = state_config.get(
Config.SECTION_STATE_ROW_CLEANUP_QUOTA,
self.STATE_ROW_CLEANUP_QUOTA,
"row_cleanup_quota", self.STATE_ROW_CLEANUP_QUOTA
)
if not self.dvc_dir:
tests/dir_helpers.py
+5 -5
View File
@@ -264,15 +264,15 @@ def run_copy(tmp_dir, dvc):
@pytest.fixture
def erepo_dir(make_tmp_dir):
from dvc.remote.config import RemoteConfig
path = make_tmp_dir("erepo", scm=True, dvc=True)
# Chdir for git and dvc to work locally
with path.chdir():
rconfig = RemoteConfig(path.dvc.config)
rconfig.add("upstream", path.dvc.cache.local.cache_dir, default=True)
path.scm_add([path.dvc.config.config_file], commit="add remote")
with path.dvc.config.edit() as conf:
cache_dir = path.dvc.cache.local.cache_dir
conf["remote"]["upstream"] = {"url": cache_dir}
conf["core"]["remote"] = "upstream"
path.scm_add([path.dvc.config.files["repo"]], commit="add remote")
return path
tests/func/test_add.py
+2 -2
View File
@@ -546,7 +546,7 @@ def temporary_windows_drive(tmp_path_factory):
def test_windows_should_add_when_cache_on_different_drive(
tmp_dir, dvc, temporary_windows_drive
):
dvc.config.set("cache", "dir", temporary_windows_drive)
dvc.config["cache"]["dir"] = temporary_windows_drive
dvc.cache = Cache(dvc)
(stage,) = tmp_dir.dvc_gen({"file": "file"})
@@ -601,7 +601,7 @@ def test_should_relink_on_repeated_add(
):
from dvc.path_info import PathInfo
dvc.config.set("cache", "type", link)
dvc.config["cache"]["type"] = link
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
tests/func/test_api.py
+3 -4
View File
@@ -9,7 +9,6 @@ from dvc.compat import fspath
from dvc.exceptions import FileMissingError
from dvc.main import main
from dvc.path_info import URLInfo
from dvc.remote.config import RemoteConfig
from tests.remotes import Azure, GCP, HDFS, Local, OSS, S3, SSH
@@ -109,9 +108,9 @@ def test_missing(remote_url, tmp_dir, dvc):
def _set_remote_url_and_commit(repo, remote_url):
rconfig = RemoteConfig(repo.config)
rconfig.modify("upstream", "url", remote_url)
repo.scm.add([repo.config.config_file])
with repo.config.edit() as conf:
conf["remote"]["upstream"]["url"] = remote_url
repo.scm.add([repo.config.files["repo"]])
repo.scm.commit("modify remote")
tests/func/test_cache.py
+6 -7
View File
@@ -154,7 +154,7 @@ class TestCmdCacheDir(TestDvc):
ret = main(["cache", "dir", dname])
self.assertEqual(ret, 0)
config = configobj.ConfigObj(self.dvc.config.config_file)
config = configobj.ConfigObj(self.dvc.config.files["repo"])
self.assertEqual(config["cache"]["dir"], dname)
def test_relative_path(self):
@@ -166,7 +166,7 @@ class TestCmdCacheDir(TestDvc):
# NOTE: we are in the repo's root and config is in .dvc/, so
# dir path written to config should be just one level above.
rel = os.path.join("..", dname)
config = configobj.ConfigObj(self.dvc.config.config_file)
config = configobj.ConfigObj(self.dvc.config.files["repo"])
self.assertEqual(config["cache"]["dir"], rel)
ret = main(["add", self.FOO])
@@ -178,9 +178,8 @@ class TestCmdCacheDir(TestDvc):
self.assertEqual(len(files), 1)
class TestShouldCacheBeReflinkOrCopyByDefault(TestDvc):
def test(self):
self.assertEqual(self.dvc.cache.local.cache_types, ["reflink", "copy"])
def test_default_cache_type(dvc):
assert dvc.cache.local.cache_types == ["reflink", "copy"]
@pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.")
@@ -189,8 +188,8 @@ class TestShouldCacheBeReflinkOrCopyByDefault(TestDvc):
[(False, 0o775, 0o664), (True, 0o775, 0o444)],
)
def test_shared_cache(tmp_dir, dvc, protected, dir_mode, file_mode):
dvc.config.set("cache", "shared", "group")
dvc.config.set("cache", "protected", str(protected))
with dvc.config.edit() as conf:
conf["cache"].update({"shared": "group", "protected": str(protected)})
dvc.cache = Cache(dvc)
tmp_dir.dvc_gen(
tests/func/test_config.py
+14 -8
View File
@@ -2,16 +2,13 @@ import pytest
import configobj
from dvc.main import main
from dvc.config import Config, ConfigError
from dvc.config import ConfigError
from tests.basic_env import TestDvc
class TestConfigCLI(TestDvc):
def _contains(self, section, field, value, local=False):
if local:
fname = self.dvc.config.config_local_file
else:
fname = self.dvc.config.config_file
fname = self.dvc.config.files["local" if local else "repo"]
config = configobj.ConfigObj(fname)
if section not in config.keys():
@@ -89,9 +86,18 @@ class TestConfigCLI(TestDvc):
def test_set_invalid_key(dvc):
with pytest.raises(ConfigError, match=r"extra keys not allowed"):
dvc.config.set("core", "invalid.key", "value")
with dvc.config.edit() as conf:
conf["core"]["invalid_key"] = "value"
def test_merging_two_levels(dvc):
dvc.config.set('remote "test"', "url", "https://example.com")
dvc.config.set('remote "test"', "password", "1", level=Config.LEVEL_LOCAL)
with dvc.config.edit() as conf:
conf["remote"]["test"] = {"url": "ssh://example.com"}
with dvc.config.edit("local") as conf:
conf["remote"]["test"] = {"password": "1"}
assert dvc.config["remote"]["test"] == {
"url": "ssh://example.com",
"password": "1",
}
tests/func/test_data_cloud.py
+32 -43
View File
@@ -7,12 +7,11 @@ from unittest import SkipTest
import pytest
from dvc.cache import NamedCache
from dvc.compat import fspath
from dvc.config import Config
from dvc.cache import NamedCache
from dvc.data_cloud import DataCloud
from dvc.main import main
from dvc.remote import RemoteAZURE, RemoteConfig
from dvc.remote import RemoteAZURE
from dvc.remote import RemoteGDrive
from dvc.remote import RemoteGS
from dvc.remote import RemoteHDFS
@@ -21,12 +20,9 @@ from dvc.remote import RemoteLOCAL
from dvc.remote import RemoteOSS
from dvc.remote import RemoteS3
from dvc.remote import RemoteSSH
from dvc.remote.base import STATUS_DELETED
from dvc.remote.base import STATUS_NEW
from dvc.remote.base import STATUS_OK
from dvc.remote.base import STATUS_DELETED, STATUS_NEW, STATUS_OK
from dvc.utils import file_md5
from dvc.utils.stage import dump_stage_file
from dvc.utils.stage import load_stage_file
from dvc.utils.stage import dump_stage_file, load_stage_file
from tests.basic_env import TestDvc
from tests.remotes import (
@@ -39,7 +35,6 @@ from tests.remotes import (
SSHMocked,
OSS,
TEST_CONFIG,
TEST_SECTION,
TEST_GCP_CREDS_FILE,
TEST_GDRIVE_CLIENT_ID,
TEST_GDRIVE_CLIENT_SECRET,
@@ -49,7 +44,7 @@ from tests.remotes import (
class TestDataCloud(TestDvc):
def _test_cloud(self, config, cl):
self.dvc.config.config = config
self.dvc.config = config
cloud = DataCloud(self.dvc)
self.assertIsInstance(cloud.get_remote(), cl)
@@ -68,7 +63,7 @@ class TestDataCloud(TestDvc):
for scheme, cl in clist:
remote_url = scheme + str(uuid.uuid4())
config[TEST_SECTION][Config.SECTION_REMOTE_URL] = remote_url
config["remote"][TEST_REMOTE]["url"] = remote_url
self._test_cloud(config, cl)
@@ -100,9 +95,8 @@ class TestDataCloudBase(TestDvc):
keyfile = self._get_keyfile()
config = copy.deepcopy(TEST_CONFIG)
config[TEST_SECTION][Config.SECTION_REMOTE_URL] = repo
config[TEST_SECTION][Config.SECTION_REMOTE_KEY_FILE] = keyfile
self.dvc.config.config = config
config["remote"][TEST_REMOTE] = {"url": repo, "keyfile": keyfile}
self.dvc.config = config
self.cloud = DataCloud(self.dvc)
self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
@@ -196,15 +190,13 @@ class TestRemoteS3(S3, TestDataCloudBase):
def setup_gdrive_cloud(remote_url, dvc):
config = copy.deepcopy(TEST_CONFIG)
config[TEST_SECTION][Config.SECTION_REMOTE_URL] = remote_url
config[TEST_SECTION][
Config.SECTION_GDRIVE_CLIENT_ID
] = TEST_GDRIVE_CLIENT_ID
config[TEST_SECTION][
Config.SECTION_GDRIVE_CLIENT_SECRET
] = TEST_GDRIVE_CLIENT_SECRET
config["remote"][TEST_REMOTE] = {
"url": remote_url,
"gdrive_client_id": TEST_GDRIVE_CLIENT_ID,
"grdive_client_secret": TEST_GDRIVE_CLIENT_SECRET,
}
dvc.config.config = config
dvc.config = config
remote = DataCloud(dvc).get_remote()
remote._create_remote_dir("root", remote.path_info.path)
@@ -230,11 +222,11 @@ class TestRemoteGS(GCP, TestDataCloudBase):
repo = self.get_url()
config = copy.deepcopy(TEST_CONFIG)
config[TEST_SECTION][Config.SECTION_REMOTE_URL] = repo
config[TEST_SECTION][
Config.SECTION_GCP_CREDENTIALPATH
] = TEST_GCP_CREDS_FILE
self.dvc.config.config = config
config["remote"][TEST_REMOTE] = {
"url": repo,
"credentialpath": TEST_GCP_CREDS_FILE,
}
self.dvc.config = config
self.cloud = DataCloud(self.dvc)
self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
@@ -272,10 +264,12 @@ class TestRemoteSSHMocked(SSHMocked, TestDataCloudBase):
keyfile = self._get_keyfile()
config = copy.deepcopy(TEST_CONFIG)
config[TEST_SECTION][Config.SECTION_REMOTE_URL] = repo
config[TEST_SECTION][Config.SECTION_REMOTE_KEY_FILE] = keyfile
config[TEST_SECTION][Config.SECTION_REMOTE_NO_TRAVERSE] = False
self.dvc.config.config = config
config["remote"][TEST_REMOTE] = {
"url": repo,
"keyfile": keyfile,
"no_traverse": False,
}
self.dvc.config = config
self.cloud = DataCloud(self.dvc)
self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
@@ -425,7 +419,7 @@ class TestRemoteGDriveCLI(GDrive, TestDataCloudCLIBase):
"remote",
"modify",
TEST_REMOTE,
Config.SECTION_GDRIVE_CLIENT_ID,
"gdrive_client_id",
TEST_GDRIVE_CLIENT_ID,
]
)
@@ -434,7 +428,7 @@ class TestRemoteGDriveCLI(GDrive, TestDataCloudCLIBase):
"remote",
"modify",
TEST_REMOTE,
Config.SECTION_GDRIVE_CLIENT_SECRET,
"grdive_client_secret",
TEST_GDRIVE_CLIENT_SECRET,
]
)
@@ -669,11 +663,10 @@ def test_verify_checksums(tmp_dir, scm, dvc, mocker, tmp_path_factory):
tmp_dir.dvc_gen({"file": "file1 content"}, commit="add file")
tmp_dir.dvc_gen({"dir": {"subfile": "file2 content"}}, commit="add dir")
RemoteConfig(dvc.config).add(
"local_remote",
fspath(tmp_path_factory.mktemp("local_remote")),
default=True,
)
dvc.config["remote"]["local_remote"] = {
"url": fspath(tmp_path_factory.mktemp("local_remote"))
}
dvc.config["core"]["remote"] = "local_remote"
dvc.push()
# remove artifacts and cache to trigger fetching
@@ -689,11 +682,7 @@ def test_verify_checksums(tmp_dir, scm, dvc, mocker, tmp_path_factory):
# Removing cache will invalidate existing state entries
shutil.rmtree(dvc.cache.local.cache_dir)
dvc.config.set(
Config.SECTION_REMOTE_FMT.format("local_remote"),
Config.SECTION_REMOTE_VERIFY,
"True",
)
dvc.config["remote"]["local_remote"]["verify"] = True
dvc.pull()
assert checksum_spy.call_count == 3
tests/func/test_get.py
+3 -5
View File
@@ -4,7 +4,6 @@ import os
import pytest
from dvc.cache import Cache
from dvc.config import Config
from dvc.main import main
from dvc.exceptions import PathMissingError
from dvc.repo.get import GetDVCFileError
@@ -61,12 +60,11 @@ def test_get_git_dir(tmp_dir, erepo_dir):
def test_cache_type_is_properly_overridden(tmp_dir, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc.config.set(
Config.SECTION_CACHE, Config.SECTION_CACHE_TYPE, "symlink"
)
with erepo_dir.dvc.config.edit() as conf:
conf["cache"]["type"] = "symlink"
erepo_dir.dvc.cache = Cache(erepo_dir.dvc)
erepo_dir.scm_add(
[erepo_dir.dvc.config.config_file], "set cache type to symlinks"
[erepo_dir.dvc.config.files["repo"]], "set cache type to symlinks"
)
erepo_dir.dvc_gen("file", "contents", "create file")
assert System.is_symlink(erepo_dir / "file")
tests/func/test_import.py
+3 -5
View File
@@ -7,7 +7,6 @@ import pytest
from mock import patch
from dvc.cache import Cache
from dvc.config import Config
from dvc.exceptions import DownloadError, PathMissingError
from dvc.config import NoRemoteError
from dvc.stage import Stage
@@ -162,12 +161,11 @@ def test_pull_imported_stage(tmp_dir, dvc, erepo_dir):
def test_cache_type_is_properly_overridden(tmp_dir, scm, dvc, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc.config.set(
Config.SECTION_CACHE, Config.SECTION_CACHE_TYPE, "symlink"
)
with erepo_dir.dvc.config.edit() as conf:
conf["cache"]["type"] = "symlink"
erepo_dir.dvc.cache = Cache(erepo_dir.dvc)
erepo_dir.scm_add(
[erepo_dir.dvc.config.config_file],
[erepo_dir.dvc.config.files["repo"]],
"set source repo cache type to symlink",
)
erepo_dir.dvc_gen("foo", "foo content", "create foo")
tests/func/test_install.py
+3 -4
View File
@@ -5,7 +5,6 @@ import sys
import pytest
from dvc.exceptions import GitHookAlreadyExistsError
from dvc.remote import RemoteConfig
from dvc.utils import file_md5, fspath
@@ -52,9 +51,9 @@ class TestInstall(object):
git_remote = temp / "project.git"
storage_path = temp / "dvc_storage"
RemoteConfig(dvc.config).add(
"store", fspath(storage_path), default=True
)
with dvc.config.edit() as conf:
conf["remote"]["store"] = {"url": fspath(storage_path)}
conf["core"]["remote"] = "store"
tmp_dir.dvc_gen("file", "file_content", "commit message")
file_checksum = file_md5("file")[0]
tests/func/test_remote.py
+30 -62
View File
@@ -6,11 +6,11 @@ import configobj
import pytest
from mock import patch
from dvc.config import Config, ConfigError
from dvc.config import Config
from dvc.exceptions import DownloadError, UploadError
from dvc.main import main
from dvc.path_info import PathInfo
from dvc.remote import RemoteLOCAL, RemoteConfig
from dvc.remote import RemoteLOCAL
from dvc.remote.base import RemoteBASE, RemoteCacheRequiredError
from dvc.compat import fspath
from tests.basic_env import TestDvc
@@ -46,7 +46,7 @@ class TestRemote(TestDvc):
# NOTE: we are in the repo's root and config is in .dvc/, so
# dir path written to config should be just one level above.
rel = os.path.join("..", dname)
config = configobj.ConfigObj(self.dvc.config.config_file)
config = configobj.ConfigObj(self.dvc.config.files["repo"])
self.assertEqual(config['remote "mylocal"']["url"], rel)
def test_overwrite(self):
@@ -62,53 +62,29 @@ class TestRemote(TestDvc):
assert main(["remote", "add", "foo", "ssh://localhost/"]) == 0
assert main(["remote", "add", "bar", "remote://foo/dvc-storage"]) == 0
config = configobj.ConfigObj(self.dvc.config.config_file)
config = configobj.ConfigObj(self.dvc.config.files["repo"])
assert config['remote "bar"']["url"] == "remote://foo/dvc-storage"
class TestRemoteRemoveDefault(TestDvc):
def test(self):
remote = "mys3"
self.assertEqual(
main(["remote", "add", "--default", remote, "s3://bucket/name"]), 0
)
self.assertEqual(
main(["remote", "modify", remote, "profile", "default"]), 0
)
self.assertEqual(main(["config", "--local", "core.remote", remote]), 0)
def test_remove_default(tmp_dir, dvc):
remote = "mys3"
assert (
main(["remote", "add", "--default", remote, "s3://bucket/name"]) == 0
)
assert main(["remote", "modify", remote, "profile", "default"]) == 0
assert main(["config", "--local", "core.remote", remote]) == 0
config = configobj.ConfigObj(
os.path.join(self.dvc.dvc_dir, Config.CONFIG)
)
local_config = configobj.ConfigObj(
os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL)
)
self.assertEqual(
config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], remote
)
self.assertEqual(
local_config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE],
remote,
)
config = configobj.ConfigObj(dvc.config.files["repo"])
local_config = configobj.ConfigObj(dvc.config.files["local"])
assert config["core"]["remote"] == remote
assert local_config["core"]["remote"] == remote
self.assertEqual(main(["remote", "remove", remote]), 0)
config = configobj.ConfigObj(
os.path.join(self.dvc.dvc_dir, Config.CONFIG)
)
local_config = configobj.ConfigObj(
os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL)
)
section = Config.SECTION_REMOTE_FMT.format(remote)
self.assertTrue(section not in config.keys())
assert main(["remote", "remove", remote]) == 0
core = config.get(Config.SECTION_CORE, None)
if core is not None:
self.assertTrue(Config.SECTION_CORE_REMOTE not in core.keys())
core = local_config.get(Config.SECTION_CORE, None)
if core is not None:
self.assertTrue(Config.SECTION_CORE_REMOTE not in core.keys())
config = configobj.ConfigObj(dvc.config.files["repo"])
local_config = configobj.ConfigObj(dvc.config.files["local"])
assert config.get("core", {}).get("remote") is None
assert local_config.get("core", {}).get("remote") is None
class TestRemoteRemove(TestDvc):
@@ -134,17 +110,13 @@ class TestRemoteDefault(TestDvc):
self.assertEqual(ret, 0)
config_file = os.path.join(self.dvc.dvc_dir, Config.CONFIG)
config = configobj.ConfigObj(config_file)
default = config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE]
default = config["core"]["remote"]
self.assertEqual(default, remote)
ret = main(["remote", "default", "--unset"])
self.assertEqual(ret, 0)
config = configobj.ConfigObj(config_file)
core = config.get(Config.SECTION_CORE)
if core is not None:
default = core.get(Config.SECTION_CORE_REMOTE)
else:
default = None
default = config.get("core", {}).get("remote")
self.assertEqual(default, None)
@@ -199,10 +171,9 @@ def test_dir_checksum_should_be_key_order_agnostic(tmp_dir, dvc):
def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory):
remote_config = RemoteConfig(dvc.config)
remote_config.add(
"upstream", fspath(tmp_path_factory.mktemp("upstream")), default=True
)
url = fspath(tmp_path_factory.mktemp("upstream"))
dvc.config["remote"]["upstream"] = {"url": url}
dvc.config["core"]["remote"] = "upstream"
foo = tmp_dir.dvc_gen({"foo": "foo content"})[0].outs[0]
bar = tmp_dir.dvc_gen({"bar": "bar content"})[0].outs[0]
@@ -235,9 +206,9 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory):
def test_raise_on_too_many_open_files(tmp_dir, dvc, tmp_path_factory, mocker):
storage = tmp_path_factory.mktemp("test_remote_base")
remote_config = RemoteConfig(dvc.config)
remote_config.add("local_remote", fspath(storage), default=True)
storage = fspath(tmp_path_factory.mktemp("test_remote_base"))
dvc.config["remote"]["local_remote"] = {"url": storage}
dvc.config["core"]["remote"] = "local_remote"
tmp_dir.dvc_gen({"file": "file content"})
@@ -252,11 +223,8 @@ def test_raise_on_too_many_open_files(tmp_dir, dvc, tmp_path_factory, mocker):
assert e.errno == errno.EMFILE
def test_modify_missing_remote(dvc):
remote_config = RemoteConfig(dvc.config)
with pytest.raises(ConfigError, match=r"Unable to find remote section"):
remote_config.modify("myremote", "gdrive_client_id", "xxx")
def test_modify_missing_remote(tmp_dir, dvc):
assert main(["remote", "modify", "myremote", "user", "xxx"]) == 251
def test_external_dir_resource_on_no_cache(tmp_dir, dvc, tmp_path_factory):
tests/func/test_repo.py
+1 -1
View File
@@ -8,7 +8,7 @@ from dvc.compat import fspath
def test_destroy(tmp_dir, dvc):
dvc.config.set("cache", "type", "symlink")
dvc.config["cache"]["type"] = ["symlink"]
dvc.cache = Cache(dvc)
tmp_dir.dvc_gen("file", "text")
tests/func/test_repro.py
+13 -12
View File
@@ -808,6 +808,8 @@ class TestCmdReproChdir(TestDvc):
class TestReproExternalBase(TestDvc):
cache_type = None
@staticmethod
def should_test():
return False
@@ -816,10 +818,6 @@ class TestReproExternalBase(TestDvc):
def cache_scheme(self):
return self.scheme
@property
def cache_type(self):
return "copy"
@property
def scheme(self):
return None
@@ -876,8 +874,9 @@ class TestReproExternalBase(TestDvc):
self.assertEqual(ret, 0)
ret = main(["remote", "add", "myrepo", cache])
self.assertEqual(ret, 0)
ret = main(["remote", "modify", "myrepo", "type", self.cache_type])
self.assertEqual(ret, 0)
if self.cache_type:
ret = main(["remote", "modify", "myrepo", "type", self.cache_type])
self.assertEqual(ret, 0)
remote_name = "myremote"
remote_key = str(uuid.uuid4())
@@ -887,8 +886,11 @@ class TestReproExternalBase(TestDvc):
ret = main(["remote", "add", remote_name, remote])
self.assertEqual(ret, 0)
ret = main(["remote", "modify", remote_name, "type", self.cache_type])
self.assertEqual(ret, 0)
if self.cache_type:
ret = main(
["remote", "modify", remote_name, "type", self.cache_type]
)
self.assertEqual(ret, 0)
self.dvc = DvcRepo(".")
@@ -1052,6 +1054,7 @@ class TestReproExternalHDFS(HDFS, TestReproExternalBase):
@flaky(max_runs=3, min_passes=1)
class TestReproExternalSSH(SSH, TestReproExternalBase):
_dir = None
cache_type = "copy"
@property
def scheme(self):
@@ -1095,6 +1098,8 @@ class TestReproExternalSSH(SSH, TestReproExternalBase):
class TestReproExternalLOCAL(Local, TestReproExternalBase):
cache_type = "hardlink"
def setUp(self):
super().setUp()
self.tmpdir = self.mkdtemp()
@@ -1102,10 +1107,6 @@ class TestReproExternalLOCAL(Local, TestReproExternalBase):
self.assertEqual(ret, 0)
self.dvc = DvcRepo(".")
@property
def cache_type(self):
return "hardlink"
@property
def cache_scheme(self):
return "local"
tests/func/test_state.py
+3 -3
View File
@@ -11,7 +11,7 @@ def test_state(tmp_dir, dvc):
path_info = PathInfo(path)
md5 = file_md5(path)[0]
state = State(dvc, dvc.config.config)
state = State(dvc)
with state:
state.save(path_info, md5)
@@ -34,7 +34,7 @@ def test_state(tmp_dir, dvc):
def test_state_overflow(tmp_dir, dvc):
# NOTE: trying to add more entries than state can handle,
# to see if it will clean up and vacuum successfully
dvc.config.set("state", "row_limit", 10)
dvc.config["state"]["row_limit"] = 10
path = tmp_dir / "dir"
path.mkdir()
@@ -55,7 +55,7 @@ def mock_get_inode(inode):
def test_get_state_record_for_inode(get_inode_mock, tmp_dir, dvc):
tmp_dir.gen("foo", "foo content")
state = State(dvc, dvc.config.config)
state = State(dvc)
inode = state.MAX_INT + 2
assert inode != state._to_sqlite(inode)
tests/remotes.py
+3 -5
View File
@@ -7,7 +7,6 @@ from contextlib import contextmanager
from subprocess import CalledProcessError, check_output, Popen
from dvc.utils import env2bool
from dvc.config import Config
from dvc.remote import RemoteGDrive
from dvc.remote.gs import RemoteGS
from dvc.remote.s3 import RemoteS3
@@ -17,11 +16,10 @@ from moto.s3 import mock_s3
TEST_REMOTE = "upstream"
TEST_SECTION = 'remote "{}"'.format(TEST_REMOTE)
TEST_CONFIG = {
Config.SECTION_CACHE: {},
Config.SECTION_CORE: {Config.SECTION_CORE_REMOTE: TEST_REMOTE},
TEST_SECTION: {Config.SECTION_REMOTE_URL: ""},
"cache": {},
"core": {"remote": TEST_REMOTE},
"remote": {TEST_REMOTE: {"url": ""}},
}
TEST_AWS_REPO_BUCKET = os.environ.get("DVC_TEST_AWS_REPO_BUCKET", "dvc-test")
tests/unit/remote/test_remote.py
+8 -22
View File
@@ -1,41 +1,27 @@
from dvc.remote import Remote
def set_config_opts(dvc, commands):
list(map(lambda args: dvc.config.set(*args), commands))
def test_remote_with_checksum_jobs(dvc):
set_config_opts(
dvc,
[
('remote "with_checksum_jobs"', "url", "s3://bucket/name"),
('remote "with_checksum_jobs"', "checksum_jobs", 100),
("core", "checksum_jobs", 200),
],
)
dvc.config["remote"]["with_checksum_jobs"] = {
"url": "s3://bucket/name",
"checksum_jobs": 100,
}
dvc.config["core"]["checksum_jobs"] = 200
remote = Remote(dvc, name="with_checksum_jobs")
assert remote.checksum_jobs == 100
def test_remote_without_checksum_jobs(dvc):
set_config_opts(
dvc,
[
('remote "without_checksum_jobs"', "url", "s3://bucket/name"),
("core", "checksum_jobs", "200"),
],
)
dvc.config["remote"]["without_checksum_jobs"] = {"url": "s3://bucket/name"}
dvc.config["core"]["checksum_jobs"] = 200
remote = Remote(dvc, name="without_checksum_jobs")
assert remote.checksum_jobs == 200
def test_remote_without_checksum_jobs_default(dvc):
set_config_opts(
dvc, [('remote "without_checksum_jobs"', "url", "s3://bucket/name")]
)
dvc.config["remote"]["without_checksum_jobs"] = {"url": "s3://bucket/name"}
remote = Remote(dvc, name="without_checksum_jobs")
assert remote.checksum_jobs == remote.CHECKSUM_JOBS
tests/unit/remote/test_slow_link_detection.py
+3 -5
View File
@@ -2,7 +2,6 @@ import mock
import pytest
import dvc.remote.slow_link_detection
from dvc.config import Config
from dvc.remote.slow_link_detection import slow_link_guard
@@ -15,9 +14,8 @@ def timeout_immediately(monkeypatch):
def make_remote():
def _make_remote(cache_type=None, should_warn=True):
remote = mock.Mock()
remote.repo.config.config.get.return_value = {
Config.SECTION_CACHE_TYPE: cache_type,
Config.SECTION_CACHE_SLOW_LINK_WARNING: should_warn,
remote.repo.config = {
"cache": {"type": cache_type, "slow_link_warning": should_warn}
}
return remote
@@ -29,8 +27,8 @@ def test_show_warning_once(caplog, make_remote):
slow_link_guard(lambda x: None)(remote)
slow_link_guard(lambda x: None)(remote)
assert caplog.records[0].message == dvc.remote.slow_link_detection.message
assert len(caplog.records) == 1
assert caplog.records[0].message == dvc.remote.slow_link_detection.message
def test_dont_warn_when_cache_type_is_set(caplog, make_remote):
tests/unit/test_analytics.py
+11 -11
View File
@@ -10,19 +10,17 @@ from dvc.cli import parse_args
@pytest.fixture
def tmp_global_config(tmp_path):
def tmp_global_dir(tmp_path):
"""
Fixture to prevent modifying the actual global config
"""
with mock.patch(
"dvc.config.Config.get_global_config_dir", return_value=str(tmp_path)
):
with mock.patch("dvc.config.Config.get_dir", return_value=str(tmp_path)):
yield
@mock.patch("dvc.daemon._spawn")
@mock.patch("json.dump")
def test_collect_and_send_report(mock_json, mock_daemon, tmp_global_config):
def test_collect_and_send_report(mock_json, mock_daemon, tmp_global_dir):
analytics.collect_and_send_report()
report = mock_json.call_args[0][0]
@@ -44,7 +42,7 @@ def test_collect_and_send_report(mock_json, mock_daemon, tmp_global_config):
assert mock_daemon.call_count == 2
def test_runtime_info(tmp_global_config):
def test_runtime_info(tmp_global_dir):
schema = Schema(
{
"dvc_version": str,
@@ -86,10 +84,12 @@ def test_send(mock_post, tmp_path):
({"analytics": "false", "unknown": "broken"}, False),
],
)
def test_is_enabled(dvc, config, result, monkeypatch, tmp_global_config):
configobj = dvc.config._repo_config
configobj["core"] = config
configobj.write()
def test_is_enabled(dvc, config, result, monkeypatch, tmp_global_dir):
import configobj
conf = configobj.ConfigObj({"core": config})
conf.filename = dvc.config.files["repo"]
conf.write()
# reset DVC_TEST env var, which affects `is_enabled()`
monkeypatch.delenv("DVC_TEST")
@@ -127,7 +127,7 @@ def test_system_info():
assert schema(analytics._system_info())
def test_find_or_create_user_id(tmp_global_config):
def test_find_or_create_user_id(tmp_global_dir):
created = analytics._find_or_create_user_id()
found = analytics._find_or_create_user_id()
tests/unit/test_config.py
+9 -9
View File
@@ -1,16 +1,16 @@
from dvc.config import Config
from dvc.config import COMPILED_SCHEMA
def test_remote_config_no_traverse():
d = Config.COMPILED_SCHEMA({'remote "myremote"': {"url": "url"}})
assert "no_traverse" not in d['remote "myremote"']
d = COMPILED_SCHEMA({"remote": {"myremote": {"url": "url"}}})
assert "no_traverse" not in d["remote"]["myremote"]
d = Config.COMPILED_SCHEMA(
{'remote "myremote"': {"url": "url", "no_traverse": "fAlSe"}}
d = COMPILED_SCHEMA(
{"remote": {"myremote": {"url": "url", "no_traverse": "fAlSe"}}}
)
assert not d['remote "myremote"']["no_traverse"]
assert not d["remote"]["myremote"]["no_traverse"]
d = Config.COMPILED_SCHEMA(
{'remote "myremote"': {"url": "url", "no_traverse": "tRuE"}}
d = COMPILED_SCHEMA(
{"remote": {"myremote": {"url": "url", "no_traverse": "tRuE"}}}
)
assert d['remote "myremote"']["no_traverse"]
assert d["remote"]["myremote"]["no_traverse"]