get-url/ls-url: support remote:// with in-repo config

dvc/commands/get_url.py

@@ -11,6 +11,7 @@ logger = logging.getLogger(__name__)
 
 class CmdGetUrl(CmdBaseNoRepo):
     def run(self):
+        from dvc.config import Config
         from dvc.repo import Repo
 
         try:
@@ -20,6 +21,7 @@ class CmdGetUrl(CmdBaseNoRepo):
                 jobs=self.args.jobs,
                 force=self.args.force,
                 fs_config=self.args.fs_config,
+                config=Config.from_cwd(),
             )
             return 0
         except DvcException:

dvc/commands/ls_url.py

@@ -11,12 +11,14 @@ logger = logging.getLogger(__name__)
 
 class CmdListUrl(CmdBaseNoRepo):
     def run(self):
+        from dvc.config import Config
         from dvc.repo import Repo
 
         entries = Repo.ls_url(
             self.args.url,
             recursive=self.args.recursive,
             fs_config=self.args.fs_config,
+            config=Config.from_cwd(),
         )
         if entries:
             show_entries(entries, with_color=True, with_size=self.args.size)

dvc/fs/__init__.py

@@ -88,11 +88,11 @@ def download(
         return len(to_infos)
 
 
-def parse_external_url(url, config=None):
-    remote_config = dict(config) if config else {}
+def parse_external_url(url, fs_config=None, config=None):
+    remote_config = dict(fs_config) if fs_config else {}
     remote_config["url"] = url
-    fs_cls, fs_config, fs_path = get_cloud_fs({}, **remote_config)
-    fs = fs_cls(**fs_config)
+    fs_cls, resolved_fs_config, fs_path = get_cloud_fs(config, **remote_config)
+    fs = fs_cls(**resolved_fs_config)
     return fs, fs_path
 
 

dvc/repo/get_url.py

@@ -6,12 +6,12 @@ from dvc.fs import download, parse_external_url
 from dvc.utils import resolve_output
 
 
-def get_url(url, out=None, *, fs_config=None, jobs=None, force=False):
+def get_url(url, out=None, *, fs_config=None, jobs=None, force=False, config=None):
     out = resolve_output(url, out, force=force)
     out = os.path.abspath(out)
     (out,) = output.loads_from(None, [out], use_cache=False)
 
-    src_fs, src_path = parse_external_url(url, fs_config)
+    src_fs, src_path = parse_external_url(url, fs_config, config=config)
     if not src_fs.exists(src_path):
         raise URLMissingError(url)
     download(src_fs, src_path, out.fs_path, jobs=jobs)

dvc/repo/ls_url.py

@@ -2,8 +2,8 @@ from dvc.exceptions import URLMissingError
 from dvc.fs import parse_external_url
 
 
-def ls_url(url, *, fs_config=None, recursive=False):
-    fs, fs_path = parse_external_url(url, config=fs_config)
+def ls_url(url, *, fs_config=None, recursive=False, config=None):
+    fs, fs_path = parse_external_url(url, fs_config=fs_config, config=config)
     try:
         info = fs.info(fs_path)
     except FileNotFoundError as exc:

tests/func/test_get_url.py

@@ -3,6 +3,7 @@ import os
 
 import pytest
 
+from dvc.config import Config
 from dvc.exceptions import FileExistsLocallyError, URLMissingError
 from dvc.repo import Repo
 from dvc.testing.workspace_tests import TestGetUrl as _TestGetUrl
@@ -66,3 +67,12 @@ def test_get_url_nonexistent(tmp_dir):
 
 class TestGetUrl(_TestGetUrl):
     pass
+
+
+def test_get_url_config(tmp_dir, dvc, make_remote):
+    remote_path = make_remote("myremote", default=False, typ="local")
+    (remote_path / "foo").write_text("foo")
+    (remote_path / "bar").write_text("bar")
+
+    Repo.get_url("remote://myremote/foo", config=Config.from_cwd())
+    assert (tmp_dir / "foo").read_text() == "foo"

tests/func/test_ls_url.py

@@ -1,5 +1,23 @@
+from dvc.config import Config
+from dvc.repo import Repo
 from dvc.testing.workspace_tests import TestLsUrl as _TestLsUrl
 
 
 class TestLsUrl(_TestLsUrl):
     pass
+
+
+def test_ls_url_config(dvc, make_remote):
+    remote_path = make_remote("myremote", default=False, typ="local")
+    (remote_path / "foo").write_text("foo")
+    (remote_path / "bar").write_text("bar")
+
+    actual = sorted(
+        Repo.ls_url("remote://myremote", config=Config.from_cwd()),
+        key=lambda entry: entry["path"],
+    )
+    expected = [
+        {"isdir": False, "path": "bar", "size": 3},
+        {"isdir": False, "path": "foo", "size": 3},
+    ]
+    assert actual == expected

tests/unit/command/test_get_url.py

@@ -1,8 +1,9 @@
 from dvc.cli import parse_args
 from dvc.commands.get_url import CmdGetUrl
+from dvc.config import Config
 
 
-def test_get_url(mocker):
+def test_get_url(mocker, M):
     cli_args = parse_args(["get-url", "src", "out", "-j", "5"])
     assert cli_args.func == CmdGetUrl
 
@@ -11,4 +12,11 @@ def test_get_url(mocker):
 
     assert cmd.run() == 0
 
-    m.assert_called_once_with("src", out="out", jobs=5, force=False, fs_config=None)
+    m.assert_called_once_with(
+        "src",
+        out="out",
+        jobs=5,
+        force=False,
+        fs_config=None,
+        config=M.instance_of(Config),
+    )

tests/unit/command/test_ls_url.py

@@ -1,8 +1,9 @@
 from dvc.cli import parse_args
 from dvc.commands.ls_url import CmdListUrl
+from dvc.config import Config
 
 
-def test_ls_url(mocker):
+def test_ls_url(mocker, M):
     cli_args = parse_args(["ls-url", "src"])
     assert cli_args.func == CmdListUrl
     cmd = cli_args.func(cli_args)
@@ -10,10 +11,12 @@ def test_ls_url(mocker):
 
     assert cmd.run() == 0
 
-    m.assert_called_once_with("src", recursive=False, fs_config=None)
+    m.assert_called_once_with(
+        "src", recursive=False, fs_config=None, config=M.instance_of(Config)
+    )
 
 
-def test_recursive(mocker):
+def test_recursive(mocker, M):
     cli_args = parse_args(["ls-url", "-R", "src"])
     assert cli_args.func == CmdListUrl
     cmd = cli_args.func(cli_args)
@@ -21,4 +24,6 @@ def test_recursive(mocker):
 
     assert cmd.run() == 0
 
-    m.assert_called_once_with("src", recursive=True, fs_config=None)
+    m.assert_called_once_with(
+        "src", recursive=True, fs_config=None, config=M.instance_of(Config)
+    )