Split config and schema (#5229)

so as not to compile a schema unless needed

dvc/cache/__init__.py

@@ -56,7 +56,7 @@ class Cache:
         elif "dir" not in config:
             settings = None
         else:
-            from ..config import LOCAL_COMMON
+            from ..config_schema import LOCAL_COMMON
 
             settings = {"url": config["dir"]}
             for opt in LOCAL_COMMON.keys():

dvc/config.py

@@ -4,25 +4,11 @@ import os
 import re
 from contextlib import contextmanager
 from functools import partial
-from urllib.parse import urlparse
 
-import configobj
-from funcy import cached_property, compact, re_find, walk_values
-from voluptuous import (
-    ALLOW_EXTRA,
-    All,
-    Any,
-    Coerce,
-    Invalid,
-    Lower,
-    Optional,
-    Range,
-    Schema,
-)
+from funcy import cached_property, compact, memoize, re_find
 
 from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.path_info import PathInfo
-from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
 
@@ -38,208 +24,19 @@ class NoRemoteError(ConfigError):
     pass
 
 
-def supported_cache_type(types):
-    """Checks if link type config option consists only of valid values.
+@memoize
+def get_compiled_schema():
+    from voluptuous import Schema
 
-    Args:
-        types (list/string): type(s) of links that dvc should try out.
-    """
-    if types is None:
-        return None
-    if isinstance(types, str):
-        types = [typ.strip() for typ in types.split(",")]
+    from .config_schema import SCHEMA
 
-    unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"}
-    if unsupported:
-        raise Invalid(
-            "Unsupported cache type(s): {}".format(", ".join(unsupported))
-        )
-
-    return types
+    return Schema(SCHEMA)
 
 
-# Checks that value is either true or false and converts it to bool
-to_bool = Bool = All(
-    Lower,
-    Any("true", "false"),
-    lambda v: v == "true",
-    msg="expected true or false",
-)
+def to_bool(value):
+    from .config_schema import Bool
 
-
-def Choices(*choices):
-    """Checks that value belongs to the specified set of values
-
-    Args:
-        *choices: pass allowed values as arguments, or pass a list or
-            tuple as a single argument
-    """
-    return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
-
-
-def ByUrl(mapping):
-    schemas = walk_values(Schema, mapping)
-
-    def validate(data):
-        if "url" not in data:
-            raise Invalid("expected 'url'")
-
-        parsed = urlparse(data["url"])
-        # Windows absolute paths should really have scheme == "" (local)
-        if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
-            return schemas[""](data)
-        if parsed.scheme not in schemas:
-            raise Invalid(f"Unsupported URL type {parsed.scheme}://")
-
-        return schemas[parsed.scheme](data)
-
-    return validate
-
-
-class RelPath(str):
-    pass
-
-
-REMOTE_COMMON = {
-    "url": str,
-    "checksum_jobs": All(Coerce(int), Range(1)),
-    "jobs": All(Coerce(int), Range(1)),
-    Optional("no_traverse"): Bool,  # obsoleted
-    "verify": Bool,
-}
-LOCAL_COMMON = {
-    "type": supported_cache_type,
-    Optional("protected", default=False): Bool,  # obsoleted
-    "shared": All(Lower, Choices("group")),
-    Optional("slow_link_warning", default=True): Bool,
-}
-HTTP_COMMON = {
-    "auth": All(Lower, Choices("basic", "digest", "custom")),
-    "custom_auth_header": str,
-    "user": str,
-    "password": str,
-    "ask_password": Bool,
-    "ssl_verify": Bool,
-    "method": str,
-}
-WEBDAV_COMMON = {
-    "user": str,
-    "password": str,
-    "ask_password": Bool,
-    "token": str,
-    "cert_path": str,
-    "key_path": str,
-    "timeout": Coerce(int),
-}
-
-SCHEMA = {
-    "core": {
-        "remote": Lower,
-        "checksum_jobs": All(Coerce(int), Range(1)),
-        Optional("interactive", default=False): Bool,
-        Optional("analytics", default=True): Bool,
-        Optional("hardlink_lock", default=False): Bool,
-        Optional("no_scm", default=False): Bool,
-        Optional("autostage", default=False): Bool,
-        Optional("experiments"): Bool,  # obsoleted
-        Optional("check_update", default=True): Bool,
-    },
-    "cache": {
-        "local": str,
-        "s3": str,
-        "gs": str,
-        "hdfs": str,
-        "webhdfs": str,
-        "ssh": str,
-        "azure": str,
-        # This is for default local cache
-        "dir": str,
-        **LOCAL_COMMON,
-    },
-    "remote": {
-        str: ByUrl(
-            {
-                "": {**LOCAL_COMMON, **REMOTE_COMMON},
-                "s3": {
-                    "region": str,
-                    "profile": str,
-                    "credentialpath": str,
-                    "endpointurl": str,
-                    "access_key_id": str,
-                    "secret_access_key": str,
-                    Optional("listobjects", default=False): Bool,  # obsoleted
-                    Optional("use_ssl", default=True): Bool,
-                    "sse": str,
-                    "sse_kms_key_id": str,
-                    "acl": str,
-                    "grant_read": str,
-                    "grant_read_acp": str,
-                    "grant_write_acp": str,
-                    "grant_full_control": str,
-                    **REMOTE_COMMON,
-                },
-                "gs": {
-                    "projectname": str,
-                    "credentialpath": str,
-                    **REMOTE_COMMON,
-                },
-                "ssh": {
-                    "type": supported_cache_type,
-                    "port": Coerce(int),
-                    "user": str,
-                    "password": str,
-                    "ask_password": Bool,
-                    "keyfile": str,
-                    "timeout": Coerce(int),
-                    "gss_auth": Bool,
-                    "allow_agent": Bool,
-                    **REMOTE_COMMON,
-                },
-                "hdfs": {"user": str, **REMOTE_COMMON},
-                "webhdfs": {
-                    "hdfscli_config": str,
-                    "webhdfs_token": str,
-                    "user": str,
-                    "webhdfs_alias": str,
-                    **REMOTE_COMMON,
-                },
-                "azure": {"connection_string": str, **REMOTE_COMMON},
-                "oss": {
-                    "oss_key_id": str,
-                    "oss_key_secret": str,
-                    "oss_endpoint": str,
-                    **REMOTE_COMMON,
-                },
-                "gdrive": {
-                    "gdrive_use_service_account": Bool,
-                    "gdrive_client_id": str,
-                    "gdrive_client_secret": str,
-                    "gdrive_user_credentials_file": str,
-                    "gdrive_service_account_email": str,
-                    "gdrive_service_account_user_email": str,
-                    "gdrive_service_account_p12_file_path": str,
-                    Optional("gdrive_trash_only", default=False): Bool,
-                    **REMOTE_COMMON,
-                },
-                "http": {**HTTP_COMMON, **REMOTE_COMMON},
-                "https": {**HTTP_COMMON, **REMOTE_COMMON},
-                "webdav": {**WEBDAV_COMMON, **REMOTE_COMMON},
-                "webdavs": {**WEBDAV_COMMON, **REMOTE_COMMON},
-                "remote": {str: object},  # Any of the above options are valid
-            }
-        )
-    },
-    "state": {
-        "row_limit": All(Coerce(int), Range(1)),
-        "row_cleanup_quota": All(Coerce(int), Range(0, 100)),
-    },
-    # section for experimental features
-    "feature": {
-        # enabled by default. It's of no use, kept for backward compatibility.
-        Optional("parametrization", default=True): Bool
-    },
-}
-COMPILED_SCHEMA = Schema(SCHEMA)
+    return Bool(value)
 
 
 class Config(dict):
@@ -352,17 +149,21 @@ class Config(dict):
         return self.tree if level == "repo" else self.wtree
 
     def _load_config(self, level):
+        from configobj import ConfigObj
+
         filename = self.files[level]
         tree = self._get_tree(level)
 
         if tree.exists(filename, use_dvcignore=False):
             with tree.open(filename) as fobj:
-                conf_obj = configobj.ConfigObj(fobj)
+                conf_obj = ConfigObj(fobj)
         else:
-            conf_obj = configobj.ConfigObj()
+            conf_obj = ConfigObj()
         return _parse_remotes(_lower_keys(conf_obj.dict()))
 
     def _save_config(self, level, conf_dict):
+        from configobj import ConfigObj
+
         filename = self.files[level]
         tree = self._get_tree(level)
 
@@ -370,7 +171,7 @@ class Config(dict):
 
         tree.makedirs(os.path.dirname(filename))
 
-        config = configobj.ConfigObj(_pack_remotes(conf_dict))
+        config = ConfigObj(_pack_remotes(conf_dict))
         with tree.open(filename, "wb") as fobj:
             config.write(fobj)
         config.filename = filename
@@ -380,7 +181,7 @@ class Config(dict):
         conf = self._load_paths(conf, self.files[level])
 
         # Auto-verify sections
-        for key in COMPILED_SCHEMA.schema:
+        for key in get_compiled_schema().schema:
             conf.setdefault(key, {})
 
         return conf
@@ -390,6 +191,8 @@ class Config(dict):
         abs_conf_dir = os.path.abspath(os.path.dirname(filename))
 
         def resolve(path):
+            from .config_schema import RelPath
+
             if os.path.isabs(path) or re.match(r"\w+://", path):
                 return path
 
@@ -404,6 +207,10 @@ class Config(dict):
 
     @staticmethod
     def _to_relpath(conf_dir, path):
+        from dvc.utils import relpath
+
+        from .config_schema import RelPath
+
         if re.match(r"\w+://", path):
             return path
 
@@ -421,6 +228,8 @@ class Config(dict):
 
     @staticmethod
     def _map_dirs(conf, func):
+        from voluptuous import ALLOW_EXTRA, Schema
+
         dirs_schema = {
             "cache": {"dir": func},
             "remote": {
@@ -473,8 +282,10 @@ class Config(dict):
 
     @staticmethod
     def validate(data):
+        from voluptuous import Invalid
+
         try:
-            return COMPILED_SCHEMA(data)
+            return get_compiled_schema()(data)
         except Invalid as exc:
             raise ConfigError(str(exc)) from None
 

dvc/config_schema.py

@@ -0,0 +1,215 @@
+import os
+from urllib.parse import urlparse
+
+from funcy import walk_values
+from voluptuous import (
+    All,
+    Any,
+    Coerce,
+    Invalid,
+    Lower,
+    Optional,
+    Range,
+    Schema,
+)
+
+Bool = All(
+    Lower,
+    Any("true", "false"),
+    lambda v: v == "true",
+    msg="expected true or false",
+)
+
+
+def supported_cache_type(types):
+    """Checks if link type config option consists only of valid values.
+
+    Args:
+        types (list/string): type(s) of links that dvc should try out.
+    """
+    if types is None:
+        return None
+    if isinstance(types, str):
+        types = [typ.strip() for typ in types.split(",")]
+
+    unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"}
+    if unsupported:
+        raise Invalid(
+            "Unsupported cache type(s): {}".format(", ".join(unsupported))
+        )
+
+    return types
+
+
+def Choices(*choices):
+    """Checks that value belongs to the specified set of values
+
+    Args:
+        *choices: pass allowed values as arguments, or pass a list or
+            tuple as a single argument
+    """
+    return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
+
+
+def ByUrl(mapping):
+    schemas = walk_values(Schema, mapping)
+
+    def validate(data):
+        if "url" not in data:
+            raise Invalid("expected 'url'")
+
+        parsed = urlparse(data["url"])
+        # Windows absolute paths should really have scheme == "" (local)
+        if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
+            return schemas[""](data)
+        if parsed.scheme not in schemas:
+            raise Invalid(f"Unsupported URL type {parsed.scheme}://")
+
+        return schemas[parsed.scheme](data)
+
+    return validate
+
+
+class RelPath(str):
+    pass
+
+
+REMOTE_COMMON = {
+    "url": str,
+    "checksum_jobs": All(Coerce(int), Range(1)),
+    "jobs": All(Coerce(int), Range(1)),
+    Optional("no_traverse"): Bool,  # obsoleted
+    "verify": Bool,
+}
+LOCAL_COMMON = {
+    "type": supported_cache_type,
+    Optional("protected", default=False): Bool,  # obsoleted
+    "shared": All(Lower, Choices("group")),
+    Optional("slow_link_warning", default=True): Bool,
+}
+HTTP_COMMON = {
+    "auth": All(Lower, Choices("basic", "digest", "custom")),
+    "custom_auth_header": str,
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+    "ssl_verify": Bool,
+    "method": str,
+}
+WEBDAV_COMMON = {
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+    "token": str,
+    "cert_path": str,
+    "key_path": str,
+    "timeout": Coerce(int),
+}
+
+SCHEMA = {
+    "core": {
+        "remote": Lower,
+        "checksum_jobs": All(Coerce(int), Range(1)),
+        Optional("interactive", default=False): Bool,
+        Optional("analytics", default=True): Bool,
+        Optional("hardlink_lock", default=False): Bool,
+        Optional("no_scm", default=False): Bool,
+        Optional("autostage", default=False): Bool,
+        Optional("experiments"): Bool,  # obsoleted
+        Optional("check_update", default=True): Bool,
+    },
+    "cache": {
+        "local": str,
+        "s3": str,
+        "gs": str,
+        "hdfs": str,
+        "webhdfs": str,
+        "ssh": str,
+        "azure": str,
+        # This is for default local cache
+        "dir": str,
+        **LOCAL_COMMON,
+    },
+    "remote": {
+        str: ByUrl(
+            {
+                "": {**LOCAL_COMMON, **REMOTE_COMMON},
+                "s3": {
+                    "region": str,
+                    "profile": str,
+                    "credentialpath": str,
+                    "endpointurl": str,
+                    "access_key_id": str,
+                    "secret_access_key": str,
+                    Optional("listobjects", default=False): Bool,  # obsoleted
+                    Optional("use_ssl", default=True): Bool,
+                    "sse": str,
+                    "sse_kms_key_id": str,
+                    "acl": str,
+                    "grant_read": str,
+                    "grant_read_acp": str,
+                    "grant_write_acp": str,
+                    "grant_full_control": str,
+                    **REMOTE_COMMON,
+                },
+                "gs": {
+                    "projectname": str,
+                    "credentialpath": str,
+                    **REMOTE_COMMON,
+                },
+                "ssh": {
+                    "type": supported_cache_type,
+                    "port": Coerce(int),
+                    "user": str,
+                    "password": str,
+                    "ask_password": Bool,
+                    "keyfile": str,
+                    "timeout": Coerce(int),
+                    "gss_auth": Bool,
+                    "allow_agent": Bool,
+                    **REMOTE_COMMON,
+                },
+                "hdfs": {"user": str, **REMOTE_COMMON},
+                "webhdfs": {
+                    "hdfscli_config": str,
+                    "webhdfs_token": str,
+                    "user": str,
+                    "webhdfs_alias": str,
+                    **REMOTE_COMMON,
+                },
+                "azure": {"connection_string": str, **REMOTE_COMMON},
+                "oss": {
+                    "oss_key_id": str,
+                    "oss_key_secret": str,
+                    "oss_endpoint": str,
+                    **REMOTE_COMMON,
+                },
+                "gdrive": {
+                    "gdrive_use_service_account": Bool,
+                    "gdrive_client_id": str,
+                    "gdrive_client_secret": str,
+                    "gdrive_user_credentials_file": str,
+                    "gdrive_service_account_email": str,
+                    "gdrive_service_account_user_email": str,
+                    "gdrive_service_account_p12_file_path": str,
+                    Optional("gdrive_trash_only", default=False): Bool,
+                    **REMOTE_COMMON,
+                },
+                "http": {**HTTP_COMMON, **REMOTE_COMMON},
+                "https": {**HTTP_COMMON, **REMOTE_COMMON},
+                "webdav": {**WEBDAV_COMMON, **REMOTE_COMMON},
+                "webdavs": {**WEBDAV_COMMON, **REMOTE_COMMON},
+                "remote": {str: object},  # Any of the above options are valid
+            }
+        )
+    },
+    "state": {
+        "row_limit": All(Coerce(int), Range(1)),
+        "row_cleanup_quota": All(Coerce(int), Range(0, 100)),
+    },
+    # section for experimental features
+    "feature": {
+        # enabled by default. It's of no use, kept for backward compatibility.
+        Optional("parametrization", default=True): Bool
+    },
+}

dvc/tree/__init__.py

@@ -80,7 +80,8 @@ def _resolve_remote_refs(config, remote_conf):
 
 
 def get_cloud_tree(repo, **kwargs):
-    from dvc.config import SCHEMA, ConfigError, Invalid
+    from dvc.config import ConfigError
+    from dvc.config_schema import SCHEMA, Invalid
 
     remote_conf = get_tree_config(repo.config, **kwargs)
     try: