From 8c19492ba622a1cd3ebfb72f861ed380949a184c Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 14 Sep 2020 12:15:38 +0300 Subject: [PATCH] tree: validate resolved remote:// sections (#4569) * tree: validate resolved remote:// sections * dvc: use checksum_jobs instead of hash_jobs This is a regression introduced during one of the refactorings. --- dvc/cache/__init__.py | 7 ++++++- dvc/tree/__init__.py | 6 ++++++ dvc/tree/base.py | 4 ++-- tests/unit/remote/test_remote.py | 6 +++--- tests/unit/tree/test_tree.py | 30 ++++++++++++++++++++++++++++++ 5 files changed, 47 insertions(+), 6 deletions(-) diff --git a/dvc/cache/__init__.py b/dvc/cache/__init__.py index 1cb0fc994..c4a742a5a 100644 --- a/dvc/cache/__init__.py +++ b/dvc/cache/__init__.py @@ -63,7 +63,12 @@ class Cache: if local: settings = {"name": local} else: - settings = {**config, "url": config["dir"]} + from ..config import LOCAL_COMMON + + settings = {"url": config["dir"]} + for opt in LOCAL_COMMON.keys(): + if opt in config: + settings[str(opt)] = config.get(opt) tree = get_cloud_tree(repo, **settings) self.local = LocalCache(tree) diff --git a/dvc/tree/__init__.py b/dvc/tree/__init__.py index aff54088a..cbd1595a6 100644 --- a/dvc/tree/__init__.py +++ b/dvc/tree/__init__.py @@ -78,5 +78,11 @@ def _resolve_remote_refs(repo, remote_conf): def get_cloud_tree(repo, **kwargs): + from dvc.config import SCHEMA, ConfigError, Invalid + remote_conf = _get_conf(repo, **kwargs) + try: + remote_conf = SCHEMA["remote"][str](remote_conf) + except Invalid as exc: + raise ConfigError(str(exc)) from None return _get_tree(remote_conf)(repo, remote_conf) diff --git a/dvc/tree/base.py b/dvc/tree/base.py index ef4ad89fd..466bb0ecf 100644 --- a/dvc/tree/base.py +++ b/dvc/tree/base.py @@ -75,8 +75,8 @@ class BaseTree: @cached_property def hash_jobs(self): return ( - self.config.get("hash_jobs") - or (self.repo and self.repo.config["core"].get("hash_jobs")) + self.config.get("checksum_jobs") + or (self.repo and self.repo.config["core"].get("checksum_jobs")) or self.HASH_JOBS ) diff --git a/tests/unit/remote/test_remote.py b/tests/unit/remote/test_remote.py index f7ea0c3ce..3769fb2c3 100644 --- a/tests/unit/remote/test_remote.py +++ b/tests/unit/remote/test_remote.py @@ -8,9 +8,9 @@ from dvc.tree.s3 import S3Tree def test_remote_with_hash_jobs(dvc): dvc.config["remote"]["with_hash_jobs"] = { "url": "s3://bucket/name", - "hash_jobs": 100, + "checksum_jobs": 100, } - dvc.config["core"]["hash_jobs"] = 200 + dvc.config["core"]["checksum_jobs"] = 200 tree = get_cloud_tree(dvc, name="with_hash_jobs") assert tree.hash_jobs == 100 @@ -18,7 +18,7 @@ def test_remote_with_hash_jobs(dvc): def test_remote_without_hash_jobs(dvc): dvc.config["remote"]["without_hash_jobs"] = {"url": "s3://bucket/name"} - dvc.config["core"]["hash_jobs"] = 200 + dvc.config["core"]["checksum_jobs"] = 200 tree = get_cloud_tree(dvc, name="without_hash_jobs") assert tree.hash_jobs == 200 diff --git a/tests/unit/tree/test_tree.py b/tests/unit/tree/test_tree.py index 645617610..2c4fc3af7 100644 --- a/tests/unit/tree/test_tree.py +++ b/tests/unit/tree/test_tree.py @@ -1,3 +1,6 @@ +import pytest + +from dvc.config import ConfigError from dvc.path_info import CloudURLInfo from dvc.tree import get_cloud_tree @@ -16,3 +19,30 @@ def test_get_cloud_tree(tmp_dir, dvc): assert get_cloud_tree(dvc, name="base").path_info == base assert get_cloud_tree(dvc, name="first").path_info == first assert get_cloud_tree(dvc, name="second").path_info == second + + +def test_get_cloud_tree_validate(tmp_dir, dvc): + tmp_dir.add_remote( + name="base", url="ssh://example.com/path", default=False + ) + tmp_dir.add_remote( + name="first", + config={"url": "remote://base/first", "type": "symlink"}, + default=False, + ) + tmp_dir.add_remote( + name="second", + config={"url": "remote://first/second", "oss_key_id": "mykey"}, + default=False, + ) + + assert get_cloud_tree(dvc, name="base").config == { + "url": "ssh://example.com/path" + } + assert get_cloud_tree(dvc, name="first").config == { + "url": "ssh://example.com/path/first", + "type": ["symlink"], + } + + with pytest.raises(ConfigError): + get_cloud_tree(dvc, name="second")