ls-url: create basic version of dvc ls-url command

For now, this only takes a URL (or local path) and lists its contents
if the protocol supports it. The command fails if the target doesn't
exist and returns the (absolute) path to the target if it's not a
directory.

dvc/cli/parser.py

@@ -29,6 +29,7 @@ from dvc.commands import (
     init,
     install,
     ls,
+    ls_url,
     machine,
     metrics,
     move,
@@ -75,6 +76,7 @@ COMMANDS = [
     install,
     root,
     ls,
+    ls_url,
     freeze,
     dag,
     daemon,

dvc/commands/ls_url.py

@@ -0,0 +1,42 @@
+import argparse
+import logging
+
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import append_doc_link
+from dvc.exceptions import DvcException
+from dvc.ui import ui
+
+from .ls import _prettify
+
+logger = logging.getLogger(__name__)
+
+
+class CmdListUrl(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+
+        try:
+            entries = Repo.ls_url(self.args.url)
+            if entries:
+                entries = _prettify(entries, with_color=True)
+                ui.write("\n".join(entries))
+            return 0
+        except DvcException:
+            logger.exception("failed to list '%s'", self.args.url)
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    LS_HELP = "List directory contents from URL."
+    get_parser = subparsers.add_parser(
+        "list-url",
+        aliases=["ls-url"],
+        parents=[parent_parser],
+        description=append_doc_link(LS_HELP, "ls-url"),
+        help=LS_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    get_parser.add_argument(
+        "url", help="See `dvc import-url -h` for full list of supported URLs."
+    )
+    get_parser.set_defaults(func=CmdListUrl)

dvc/exceptions.py

@@ -278,6 +278,11 @@ class PathMissingError(DvcException):
         self.dvc_only = dvc_only
 
 
+class URLMissingError(DvcException):
+    def __init__(self, url):
+        super().__init__(f"The path '{url}' does not exist.")
+
+
 class RemoteCacheRequiredError(DvcException):
     def __init__(self, scheme, fs_path):
 

dvc/repo/__init__.py

@@ -68,6 +68,7 @@ class Repo:
     from dvc.repo.imp_url import imp_url  # type: ignore[misc]
     from dvc.repo.install import install  # type: ignore[misc]
     from dvc.repo.ls import ls as _ls  # type: ignore[misc]
+    from dvc.repo.ls_url import ls_url as _ls_url  # type: ignore[misc]
     from dvc.repo.move import move  # type: ignore[misc]
     from dvc.repo.pull import pull  # type: ignore[misc]
     from dvc.repo.push import push  # type: ignore[misc]
@@ -80,6 +81,7 @@ class Repo:
     from .data import status as data_status  # type: ignore[misc]
 
     ls = staticmethod(_ls)
+    ls_url = staticmethod(_ls_url)
     get = staticmethod(_get)
     get_url = staticmethod(_get_url)
 

dvc/repo/ls_url.py

@@ -0,0 +1,24 @@
+from dvc.exceptions import URLMissingError
+from dvc.output import Output
+
+
+def ls_url(url):
+    out = Output(None, url)
+    fs, fs_path = out.fs, out.fs_path
+
+    try:
+        info = fs.info(fs_path)
+    except FileNotFoundError:
+        raise URLMissingError(url)
+    if info["type"] != "directory":
+        return [{"path": info["name"], "isdir": False}]
+
+    infos = fs.ls(fs_path, detail=True)
+    ret = []
+    for info in infos:
+        ls_info = {
+            "path": fs.path.relpath(info["name"], fs_path),
+            "isdir": info["type"] == "directory",
+        }
+        ret.append(ls_info)
+    return ret

tests/func/test_ls_url.py

@@ -0,0 +1,34 @@
+import os
+
+import pytest
+
+from dvc.exceptions import URLMissingError
+from dvc.repo import Repo
+
+
+def match_files(entries, expected):
+    entries_content = {(d["path"], d["isdir"]) for d in entries}
+    expected_content = {(d["path"], d["isdir"]) for d in expected}
+    assert entries_content == expected_content
+
+
+def test_file(tmp_dir):
+    tmp_dir.gen({"foo": "foo contents"})
+
+    result = Repo.ls_url("foo")
+    match_files(result, [{"path": os.path.abspath("foo"), "isdir": False}])
+
+
+def test_dir(tmp_dir):
+    tmp_dir.gen({"dir/foo": "foo contents", "dir/subdir/bar": "bar contents"})
+
+    result = Repo.ls_url("dir")
+    match_files(
+        result,
+        [{"path": "foo", "isdir": False}, {"path": "subdir", "isdir": True}],
+    )
+
+
+def test_nonexistent(tmp_dir):
+    with pytest.raises(URLMissingError):
+        Repo.ls_url("nonexistent")

tests/unit/command/test_ls_url.py

@@ -0,0 +1,14 @@
+from dvc.cli import parse_args
+from dvc.commands.ls_url import CmdListUrl
+
+
+def test_ls_url(mocker):
+    cli_args = parse_args(["ls-url", "src"])
+    assert cli_args.func == CmdListUrl
+
+    cmd = cli_args.func(cli_args)
+    m = mocker.patch("dvc.repo.Repo.ls_url", autospec=True)
+
+    assert cmd.run() == 0
+
+    m.assert_called_once_with("src")