From c04cdaf70fd3402a7d3ff14d7f1eefbb6e0e74eb Mon Sep 17 00:00:00 2001 From: Saugat Pachhai Date: Thu, 7 Jan 2021 17:32:14 +0545 Subject: [PATCH] Split config and schema (#5229) so as not to compile a schema unless needed --- dvc/cache/__init__.py | 2 +- dvc/config.py | 245 +++++------------------------------------- dvc/config_schema.py | 215 ++++++++++++++++++++++++++++++++++++ dvc/tree/__init__.py | 3 +- 4 files changed, 246 insertions(+), 219 deletions(-) create mode 100644 dvc/config_schema.py diff --git a/dvc/cache/__init__.py b/dvc/cache/__init__.py index dd6471983..73b13dedc 100644 --- a/dvc/cache/__init__.py +++ b/dvc/cache/__init__.py @@ -56,7 +56,7 @@ class Cache: elif "dir" not in config: settings = None else: - from ..config import LOCAL_COMMON + from ..config_schema import LOCAL_COMMON settings = {"url": config["dir"]} for opt in LOCAL_COMMON.keys(): diff --git a/dvc/config.py b/dvc/config.py index a6b992e72..27b28918f 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -4,25 +4,11 @@ import os import re from contextlib import contextmanager from functools import partial -from urllib.parse import urlparse -import configobj -from funcy import cached_property, compact, re_find, walk_values -from voluptuous import ( - ALLOW_EXTRA, - All, - Any, - Coerce, - Invalid, - Lower, - Optional, - Range, - Schema, -) +from funcy import cached_property, compact, memoize, re_find from dvc.exceptions import DvcException, NotDvcRepoError from dvc.path_info import PathInfo -from dvc.utils import relpath logger = logging.getLogger(__name__) @@ -38,208 +24,19 @@ class NoRemoteError(ConfigError): pass -def supported_cache_type(types): - """Checks if link type config option consists only of valid values. +@memoize +def get_compiled_schema(): + from voluptuous import Schema - Args: - types (list/string): type(s) of links that dvc should try out. - """ - if types is None: - return None - if isinstance(types, str): - types = [typ.strip() for typ in types.split(",")] + from .config_schema import SCHEMA - unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"} - if unsupported: - raise Invalid( - "Unsupported cache type(s): {}".format(", ".join(unsupported)) - ) - - return types + return Schema(SCHEMA) -# Checks that value is either true or false and converts it to bool -to_bool = Bool = All( - Lower, - Any("true", "false"), - lambda v: v == "true", - msg="expected true or false", -) +def to_bool(value): + from .config_schema import Bool - -def Choices(*choices): - """Checks that value belongs to the specified set of values - - Args: - *choices: pass allowed values as arguments, or pass a list or - tuple as a single argument - """ - return Any(*choices, msg="expected one of {}".format(", ".join(choices))) - - -def ByUrl(mapping): - schemas = walk_values(Schema, mapping) - - def validate(data): - if "url" not in data: - raise Invalid("expected 'url'") - - parsed = urlparse(data["url"]) - # Windows absolute paths should really have scheme == "" (local) - if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "": - return schemas[""](data) - if parsed.scheme not in schemas: - raise Invalid(f"Unsupported URL type {parsed.scheme}://") - - return schemas[parsed.scheme](data) - - return validate - - -class RelPath(str): - pass - - -REMOTE_COMMON = { - "url": str, - "checksum_jobs": All(Coerce(int), Range(1)), - "jobs": All(Coerce(int), Range(1)), - Optional("no_traverse"): Bool, # obsoleted - "verify": Bool, -} -LOCAL_COMMON = { - "type": supported_cache_type, - Optional("protected", default=False): Bool, # obsoleted - "shared": All(Lower, Choices("group")), - Optional("slow_link_warning", default=True): Bool, -} -HTTP_COMMON = { - "auth": All(Lower, Choices("basic", "digest", "custom")), - "custom_auth_header": str, - "user": str, - "password": str, - "ask_password": Bool, - "ssl_verify": Bool, - "method": str, -} -WEBDAV_COMMON = { - "user": str, - "password": str, - "ask_password": Bool, - "token": str, - "cert_path": str, - "key_path": str, - "timeout": Coerce(int), -} - -SCHEMA = { - "core": { - "remote": Lower, - "checksum_jobs": All(Coerce(int), Range(1)), - Optional("interactive", default=False): Bool, - Optional("analytics", default=True): Bool, - Optional("hardlink_lock", default=False): Bool, - Optional("no_scm", default=False): Bool, - Optional("autostage", default=False): Bool, - Optional("experiments"): Bool, # obsoleted - Optional("check_update", default=True): Bool, - }, - "cache": { - "local": str, - "s3": str, - "gs": str, - "hdfs": str, - "webhdfs": str, - "ssh": str, - "azure": str, - # This is for default local cache - "dir": str, - **LOCAL_COMMON, - }, - "remote": { - str: ByUrl( - { - "": {**LOCAL_COMMON, **REMOTE_COMMON}, - "s3": { - "region": str, - "profile": str, - "credentialpath": str, - "endpointurl": str, - "access_key_id": str, - "secret_access_key": str, - Optional("listobjects", default=False): Bool, # obsoleted - Optional("use_ssl", default=True): Bool, - "sse": str, - "sse_kms_key_id": str, - "acl": str, - "grant_read": str, - "grant_read_acp": str, - "grant_write_acp": str, - "grant_full_control": str, - **REMOTE_COMMON, - }, - "gs": { - "projectname": str, - "credentialpath": str, - **REMOTE_COMMON, - }, - "ssh": { - "type": supported_cache_type, - "port": Coerce(int), - "user": str, - "password": str, - "ask_password": Bool, - "keyfile": str, - "timeout": Coerce(int), - "gss_auth": Bool, - "allow_agent": Bool, - **REMOTE_COMMON, - }, - "hdfs": {"user": str, **REMOTE_COMMON}, - "webhdfs": { - "hdfscli_config": str, - "webhdfs_token": str, - "user": str, - "webhdfs_alias": str, - **REMOTE_COMMON, - }, - "azure": {"connection_string": str, **REMOTE_COMMON}, - "oss": { - "oss_key_id": str, - "oss_key_secret": str, - "oss_endpoint": str, - **REMOTE_COMMON, - }, - "gdrive": { - "gdrive_use_service_account": Bool, - "gdrive_client_id": str, - "gdrive_client_secret": str, - "gdrive_user_credentials_file": str, - "gdrive_service_account_email": str, - "gdrive_service_account_user_email": str, - "gdrive_service_account_p12_file_path": str, - Optional("gdrive_trash_only", default=False): Bool, - **REMOTE_COMMON, - }, - "http": {**HTTP_COMMON, **REMOTE_COMMON}, - "https": {**HTTP_COMMON, **REMOTE_COMMON}, - "webdav": {**WEBDAV_COMMON, **REMOTE_COMMON}, - "webdavs": {**WEBDAV_COMMON, **REMOTE_COMMON}, - "remote": {str: object}, # Any of the above options are valid - } - ) - }, - "state": { - "row_limit": All(Coerce(int), Range(1)), - "row_cleanup_quota": All(Coerce(int), Range(0, 100)), - }, - # section for experimental features - "feature": { - # enabled by default. It's of no use, kept for backward compatibility. - Optional("parametrization", default=True): Bool - }, -} -COMPILED_SCHEMA = Schema(SCHEMA) + return Bool(value) class Config(dict): @@ -352,17 +149,21 @@ class Config(dict): return self.tree if level == "repo" else self.wtree def _load_config(self, level): + from configobj import ConfigObj + filename = self.files[level] tree = self._get_tree(level) if tree.exists(filename, use_dvcignore=False): with tree.open(filename) as fobj: - conf_obj = configobj.ConfigObj(fobj) + conf_obj = ConfigObj(fobj) else: - conf_obj = configobj.ConfigObj() + conf_obj = ConfigObj() return _parse_remotes(_lower_keys(conf_obj.dict())) def _save_config(self, level, conf_dict): + from configobj import ConfigObj + filename = self.files[level] tree = self._get_tree(level) @@ -370,7 +171,7 @@ class Config(dict): tree.makedirs(os.path.dirname(filename)) - config = configobj.ConfigObj(_pack_remotes(conf_dict)) + config = ConfigObj(_pack_remotes(conf_dict)) with tree.open(filename, "wb") as fobj: config.write(fobj) config.filename = filename @@ -380,7 +181,7 @@ class Config(dict): conf = self._load_paths(conf, self.files[level]) # Auto-verify sections - for key in COMPILED_SCHEMA.schema: + for key in get_compiled_schema().schema: conf.setdefault(key, {}) return conf @@ -390,6 +191,8 @@ class Config(dict): abs_conf_dir = os.path.abspath(os.path.dirname(filename)) def resolve(path): + from .config_schema import RelPath + if os.path.isabs(path) or re.match(r"\w+://", path): return path @@ -404,6 +207,10 @@ class Config(dict): @staticmethod def _to_relpath(conf_dir, path): + from dvc.utils import relpath + + from .config_schema import RelPath + if re.match(r"\w+://", path): return path @@ -421,6 +228,8 @@ class Config(dict): @staticmethod def _map_dirs(conf, func): + from voluptuous import ALLOW_EXTRA, Schema + dirs_schema = { "cache": {"dir": func}, "remote": { @@ -473,8 +282,10 @@ class Config(dict): @staticmethod def validate(data): + from voluptuous import Invalid + try: - return COMPILED_SCHEMA(data) + return get_compiled_schema()(data) except Invalid as exc: raise ConfigError(str(exc)) from None diff --git a/dvc/config_schema.py b/dvc/config_schema.py new file mode 100644 index 000000000..095ebc733 --- /dev/null +++ b/dvc/config_schema.py @@ -0,0 +1,215 @@ +import os +from urllib.parse import urlparse + +from funcy import walk_values +from voluptuous import ( + All, + Any, + Coerce, + Invalid, + Lower, + Optional, + Range, + Schema, +) + +Bool = All( + Lower, + Any("true", "false"), + lambda v: v == "true", + msg="expected true or false", +) + + +def supported_cache_type(types): + """Checks if link type config option consists only of valid values. + + Args: + types (list/string): type(s) of links that dvc should try out. + """ + if types is None: + return None + if isinstance(types, str): + types = [typ.strip() for typ in types.split(",")] + + unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"} + if unsupported: + raise Invalid( + "Unsupported cache type(s): {}".format(", ".join(unsupported)) + ) + + return types + + +def Choices(*choices): + """Checks that value belongs to the specified set of values + + Args: + *choices: pass allowed values as arguments, or pass a list or + tuple as a single argument + """ + return Any(*choices, msg="expected one of {}".format(", ".join(choices))) + + +def ByUrl(mapping): + schemas = walk_values(Schema, mapping) + + def validate(data): + if "url" not in data: + raise Invalid("expected 'url'") + + parsed = urlparse(data["url"]) + # Windows absolute paths should really have scheme == "" (local) + if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "": + return schemas[""](data) + if parsed.scheme not in schemas: + raise Invalid(f"Unsupported URL type {parsed.scheme}://") + + return schemas[parsed.scheme](data) + + return validate + + +class RelPath(str): + pass + + +REMOTE_COMMON = { + "url": str, + "checksum_jobs": All(Coerce(int), Range(1)), + "jobs": All(Coerce(int), Range(1)), + Optional("no_traverse"): Bool, # obsoleted + "verify": Bool, +} +LOCAL_COMMON = { + "type": supported_cache_type, + Optional("protected", default=False): Bool, # obsoleted + "shared": All(Lower, Choices("group")), + Optional("slow_link_warning", default=True): Bool, +} +HTTP_COMMON = { + "auth": All(Lower, Choices("basic", "digest", "custom")), + "custom_auth_header": str, + "user": str, + "password": str, + "ask_password": Bool, + "ssl_verify": Bool, + "method": str, +} +WEBDAV_COMMON = { + "user": str, + "password": str, + "ask_password": Bool, + "token": str, + "cert_path": str, + "key_path": str, + "timeout": Coerce(int), +} + +SCHEMA = { + "core": { + "remote": Lower, + "checksum_jobs": All(Coerce(int), Range(1)), + Optional("interactive", default=False): Bool, + Optional("analytics", default=True): Bool, + Optional("hardlink_lock", default=False): Bool, + Optional("no_scm", default=False): Bool, + Optional("autostage", default=False): Bool, + Optional("experiments"): Bool, # obsoleted + Optional("check_update", default=True): Bool, + }, + "cache": { + "local": str, + "s3": str, + "gs": str, + "hdfs": str, + "webhdfs": str, + "ssh": str, + "azure": str, + # This is for default local cache + "dir": str, + **LOCAL_COMMON, + }, + "remote": { + str: ByUrl( + { + "": {**LOCAL_COMMON, **REMOTE_COMMON}, + "s3": { + "region": str, + "profile": str, + "credentialpath": str, + "endpointurl": str, + "access_key_id": str, + "secret_access_key": str, + Optional("listobjects", default=False): Bool, # obsoleted + Optional("use_ssl", default=True): Bool, + "sse": str, + "sse_kms_key_id": str, + "acl": str, + "grant_read": str, + "grant_read_acp": str, + "grant_write_acp": str, + "grant_full_control": str, + **REMOTE_COMMON, + }, + "gs": { + "projectname": str, + "credentialpath": str, + **REMOTE_COMMON, + }, + "ssh": { + "type": supported_cache_type, + "port": Coerce(int), + "user": str, + "password": str, + "ask_password": Bool, + "keyfile": str, + "timeout": Coerce(int), + "gss_auth": Bool, + "allow_agent": Bool, + **REMOTE_COMMON, + }, + "hdfs": {"user": str, **REMOTE_COMMON}, + "webhdfs": { + "hdfscli_config": str, + "webhdfs_token": str, + "user": str, + "webhdfs_alias": str, + **REMOTE_COMMON, + }, + "azure": {"connection_string": str, **REMOTE_COMMON}, + "oss": { + "oss_key_id": str, + "oss_key_secret": str, + "oss_endpoint": str, + **REMOTE_COMMON, + }, + "gdrive": { + "gdrive_use_service_account": Bool, + "gdrive_client_id": str, + "gdrive_client_secret": str, + "gdrive_user_credentials_file": str, + "gdrive_service_account_email": str, + "gdrive_service_account_user_email": str, + "gdrive_service_account_p12_file_path": str, + Optional("gdrive_trash_only", default=False): Bool, + **REMOTE_COMMON, + }, + "http": {**HTTP_COMMON, **REMOTE_COMMON}, + "https": {**HTTP_COMMON, **REMOTE_COMMON}, + "webdav": {**WEBDAV_COMMON, **REMOTE_COMMON}, + "webdavs": {**WEBDAV_COMMON, **REMOTE_COMMON}, + "remote": {str: object}, # Any of the above options are valid + } + ) + }, + "state": { + "row_limit": All(Coerce(int), Range(1)), + "row_cleanup_quota": All(Coerce(int), Range(0, 100)), + }, + # section for experimental features + "feature": { + # enabled by default. It's of no use, kept for backward compatibility. + Optional("parametrization", default=True): Bool + }, +} diff --git a/dvc/tree/__init__.py b/dvc/tree/__init__.py index c36f5216a..cc6d08e8c 100644 --- a/dvc/tree/__init__.py +++ b/dvc/tree/__init__.py @@ -80,7 +80,8 @@ def _resolve_remote_refs(config, remote_conf): def get_cloud_tree(repo, **kwargs): - from dvc.config import SCHEMA, ConfigError, Invalid + from dvc.config import ConfigError + from dvc.config_schema import SCHEMA, Invalid remote_conf = get_tree_config(repo.config, **kwargs) try: