diff --git a/dvc/commands/get_url.py b/dvc/commands/get_url.py index 9270f57a3..8a454b8fe 100644 --- a/dvc/commands/get_url.py +++ b/dvc/commands/get_url.py @@ -11,6 +11,7 @@ logger = logging.getLogger(__name__) class CmdGetUrl(CmdBaseNoRepo): def run(self): + from dvc.config import Config from dvc.repo import Repo try: @@ -20,6 +21,7 @@ class CmdGetUrl(CmdBaseNoRepo): jobs=self.args.jobs, force=self.args.force, fs_config=self.args.fs_config, + config=Config.from_cwd(), ) return 0 except DvcException: diff --git a/dvc/commands/ls_url.py b/dvc/commands/ls_url.py index fbbdd86d0..021accbab 100644 --- a/dvc/commands/ls_url.py +++ b/dvc/commands/ls_url.py @@ -11,12 +11,14 @@ logger = logging.getLogger(__name__) class CmdListUrl(CmdBaseNoRepo): def run(self): + from dvc.config import Config from dvc.repo import Repo entries = Repo.ls_url( self.args.url, recursive=self.args.recursive, fs_config=self.args.fs_config, + config=Config.from_cwd(), ) if entries: show_entries(entries, with_color=True, with_size=self.args.size) diff --git a/dvc/fs/__init__.py b/dvc/fs/__init__.py index 3c948ad94..76605801e 100644 --- a/dvc/fs/__init__.py +++ b/dvc/fs/__init__.py @@ -88,11 +88,11 @@ def download( return len(to_infos) -def parse_external_url(url, config=None): - remote_config = dict(config) if config else {} +def parse_external_url(url, fs_config=None, config=None): + remote_config = dict(fs_config) if fs_config else {} remote_config["url"] = url - fs_cls, fs_config, fs_path = get_cloud_fs({}, **remote_config) - fs = fs_cls(**fs_config) + fs_cls, resolved_fs_config, fs_path = get_cloud_fs(config, **remote_config) + fs = fs_cls(**resolved_fs_config) return fs, fs_path diff --git a/dvc/repo/get_url.py b/dvc/repo/get_url.py index ff0e57cf4..9cbd69235 100644 --- a/dvc/repo/get_url.py +++ b/dvc/repo/get_url.py @@ -6,12 +6,12 @@ from dvc.fs import download, parse_external_url from dvc.utils import resolve_output -def get_url(url, out=None, *, fs_config=None, jobs=None, force=False): +def get_url(url, out=None, *, fs_config=None, jobs=None, force=False, config=None): out = resolve_output(url, out, force=force) out = os.path.abspath(out) (out,) = output.loads_from(None, [out], use_cache=False) - src_fs, src_path = parse_external_url(url, fs_config) + src_fs, src_path = parse_external_url(url, fs_config, config=config) if not src_fs.exists(src_path): raise URLMissingError(url) download(src_fs, src_path, out.fs_path, jobs=jobs) diff --git a/dvc/repo/ls_url.py b/dvc/repo/ls_url.py index bae2e9ff3..32253a9ef 100644 --- a/dvc/repo/ls_url.py +++ b/dvc/repo/ls_url.py @@ -2,8 +2,8 @@ from dvc.exceptions import URLMissingError from dvc.fs import parse_external_url -def ls_url(url, *, fs_config=None, recursive=False): - fs, fs_path = parse_external_url(url, config=fs_config) +def ls_url(url, *, fs_config=None, recursive=False, config=None): + fs, fs_path = parse_external_url(url, fs_config=fs_config, config=config) try: info = fs.info(fs_path) except FileNotFoundError as exc: diff --git a/tests/func/test_get_url.py b/tests/func/test_get_url.py index 1733895ce..e19f3c8e1 100644 --- a/tests/func/test_get_url.py +++ b/tests/func/test_get_url.py @@ -3,6 +3,7 @@ import os import pytest +from dvc.config import Config from dvc.exceptions import FileExistsLocallyError, URLMissingError from dvc.repo import Repo from dvc.testing.workspace_tests import TestGetUrl as _TestGetUrl @@ -66,3 +67,12 @@ def test_get_url_nonexistent(tmp_dir): class TestGetUrl(_TestGetUrl): pass + + +def test_get_url_config(tmp_dir, dvc, make_remote): + remote_path = make_remote("myremote", default=False, typ="local") + (remote_path / "foo").write_text("foo") + (remote_path / "bar").write_text("bar") + + Repo.get_url("remote://myremote/foo", config=Config.from_cwd()) + assert (tmp_dir / "foo").read_text() == "foo" diff --git a/tests/func/test_ls_url.py b/tests/func/test_ls_url.py index 4478cdd66..367c5fdaa 100644 --- a/tests/func/test_ls_url.py +++ b/tests/func/test_ls_url.py @@ -1,5 +1,23 @@ +from dvc.config import Config +from dvc.repo import Repo from dvc.testing.workspace_tests import TestLsUrl as _TestLsUrl class TestLsUrl(_TestLsUrl): pass + + +def test_ls_url_config(dvc, make_remote): + remote_path = make_remote("myremote", default=False, typ="local") + (remote_path / "foo").write_text("foo") + (remote_path / "bar").write_text("bar") + + actual = sorted( + Repo.ls_url("remote://myremote", config=Config.from_cwd()), + key=lambda entry: entry["path"], + ) + expected = [ + {"isdir": False, "path": "bar", "size": 3}, + {"isdir": False, "path": "foo", "size": 3}, + ] + assert actual == expected diff --git a/tests/unit/command/test_get_url.py b/tests/unit/command/test_get_url.py index 617940f15..5ffe7957f 100644 --- a/tests/unit/command/test_get_url.py +++ b/tests/unit/command/test_get_url.py @@ -1,8 +1,9 @@ from dvc.cli import parse_args from dvc.commands.get_url import CmdGetUrl +from dvc.config import Config -def test_get_url(mocker): +def test_get_url(mocker, M): cli_args = parse_args(["get-url", "src", "out", "-j", "5"]) assert cli_args.func == CmdGetUrl @@ -11,4 +12,11 @@ def test_get_url(mocker): assert cmd.run() == 0 - m.assert_called_once_with("src", out="out", jobs=5, force=False, fs_config=None) + m.assert_called_once_with( + "src", + out="out", + jobs=5, + force=False, + fs_config=None, + config=M.instance_of(Config), + ) diff --git a/tests/unit/command/test_ls_url.py b/tests/unit/command/test_ls_url.py index 20e2676bf..252c0061e 100644 --- a/tests/unit/command/test_ls_url.py +++ b/tests/unit/command/test_ls_url.py @@ -1,8 +1,9 @@ from dvc.cli import parse_args from dvc.commands.ls_url import CmdListUrl +from dvc.config import Config -def test_ls_url(mocker): +def test_ls_url(mocker, M): cli_args = parse_args(["ls-url", "src"]) assert cli_args.func == CmdListUrl cmd = cli_args.func(cli_args) @@ -10,10 +11,12 @@ def test_ls_url(mocker): assert cmd.run() == 0 - m.assert_called_once_with("src", recursive=False, fs_config=None) + m.assert_called_once_with( + "src", recursive=False, fs_config=None, config=M.instance_of(Config) + ) -def test_recursive(mocker): +def test_recursive(mocker, M): cli_args = parse_args(["ls-url", "-R", "src"]) assert cli_args.func == CmdListUrl cmd = cli_args.func(cli_args) @@ -21,4 +24,6 @@ def test_recursive(mocker): assert cmd.run() == 0 - m.assert_called_once_with("src", recursive=True, fs_config=None) + m.assert_called_once_with( + "src", recursive=True, fs_config=None, config=M.instance_of(Config) + )