mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
rgvanwesepandRobert Van Wesep
59cfc2c4a4
* config, remote: Made S3 CA bundle customizable botocore allows a path to a custom CA bundle either by passing a path to the CA bundle file into the verify argument of boto3.session.Session.client or passing None (the default) which will fall back to the AWS config. Previously, the DVC config only accepted a boolean into the ssl_verify option in the remote S3 config. This changes the DVC config to accept both string and None in addition to boolean and defaults to None. I also changed the default for ssl_verfiy to None in BaseS3FileSystem. Thus, if ssl_verify is not provided, botocore will fall back to the AWS config. Testing Unit tests to cover the changes to the config schema and addition ssl_verify types that will be passed into S3FileSystem. Also, ran dvc push -r object-store data/cifar-10-python.tar.gz in my work environment that has a private S3 endpoint that requires a custom CA bundle, both with and without ssl_verify specified in the config. This was successful, showing that communication could be established. And I ran dvc remote modify object-store ssl_verify "$HOME/.aws/cabundle.pem" and confirmed that the custom CA bundle path was added to the config. Fixes #6012 * Removed default None on ssl_verify Responding to PR comment, removed the Optional, default None on ssl_verify since the config keys are optional by default. Rather than a missing ssl_verify producing a None that eventually gets filtered, it doesn't appear in the parsed config in the first place. Co-authored-by: Robert Van Wesep <[email protected]>
73 lines
1.9 KiB
Python
73 lines
1.9 KiB
Python
import os
|
|
import textwrap
|
|
|
|
import pytest
|
|
|
|
from dvc.config import Config
|
|
from dvc.fs.local import LocalFileSystem
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"path, expected",
|
|
[
|
|
("cache", "../cache"),
|
|
(os.path.join("..", "cache"), "../../cache"),
|
|
(os.getcwd(), os.getcwd().replace("\\", "/")),
|
|
("ssh://some/path", "ssh://some/path"),
|
|
],
|
|
)
|
|
def test_to_relpath(path, expected):
|
|
assert Config._to_relpath(os.path.join(".", "config"), path) == expected
|
|
|
|
|
|
def test_get_fs(tmp_dir, scm):
|
|
tmp_dir.scm_gen("foo", "foo", commit="add foo")
|
|
|
|
fs = scm.get_fs("master")
|
|
config = Config(fs=fs)
|
|
|
|
assert config.fs == fs
|
|
assert config.wfs != fs
|
|
assert isinstance(config.wfs, LocalFileSystem)
|
|
|
|
assert config._get_fs("repo") == fs
|
|
assert config._get_fs("local") == config.wfs
|
|
assert config._get_fs("global") == config.wfs
|
|
assert config._get_fs("system") == config.wfs
|
|
|
|
|
|
def test_s3_ssl_verify(tmp_dir, dvc):
|
|
config = Config(validate=False)
|
|
with config.edit() as conf:
|
|
conf["remote"]["remote-name"] = {"url": "s3://bucket/dvc"}
|
|
|
|
assert "ssl_verify" not in config["remote"]["remote-name"]
|
|
|
|
with config.edit() as conf:
|
|
section = conf["remote"]["remote-name"]
|
|
section["ssl_verify"] = False
|
|
|
|
assert (tmp_dir / ".dvc" / "config").read_text() == textwrap.dedent(
|
|
"""\
|
|
[core]
|
|
no_scm = True
|
|
['remote "remote-name"']
|
|
url = s3://bucket/dvc
|
|
ssl_verify = False
|
|
"""
|
|
)
|
|
|
|
with config.edit() as conf:
|
|
section = conf["remote"]["remote-name"]
|
|
section["ssl_verify"] = "/path/to/custom/cabundle.pem"
|
|
|
|
assert (tmp_dir / ".dvc" / "config").read_text() == textwrap.dedent(
|
|
"""\
|
|
[core]
|
|
no_scm = True
|
|
['remote "remote-name"']
|
|
url = s3://bucket/dvc
|
|
ssl_verify = /path/to/custom/cabundle.pem
|
|
"""
|
|
)
|