diff --git a/dvc/cli/parser.py b/dvc/cli/parser.py index c626daf9d..c1ce67183 100644 --- a/dvc/cli/parser.py +++ b/dvc/cli/parser.py @@ -29,6 +29,7 @@ from dvc.commands import ( init, install, ls, + ls_url, machine, metrics, move, @@ -75,6 +76,7 @@ COMMANDS = [ install, root, ls, + ls_url, freeze, dag, daemon, diff --git a/dvc/commands/ls_url.py b/dvc/commands/ls_url.py new file mode 100644 index 000000000..49405d043 --- /dev/null +++ b/dvc/commands/ls_url.py @@ -0,0 +1,42 @@ +import argparse +import logging + +from dvc.cli.command import CmdBaseNoRepo +from dvc.cli.utils import append_doc_link +from dvc.exceptions import DvcException +from dvc.ui import ui + +from .ls import _prettify + +logger = logging.getLogger(__name__) + + +class CmdListUrl(CmdBaseNoRepo): + def run(self): + from dvc.repo import Repo + + try: + entries = Repo.ls_url(self.args.url) + if entries: + entries = _prettify(entries, with_color=True) + ui.write("\n".join(entries)) + return 0 + except DvcException: + logger.exception("failed to list '%s'", self.args.url) + return 1 + + +def add_parser(subparsers, parent_parser): + LS_HELP = "List directory contents from URL." + get_parser = subparsers.add_parser( + "list-url", + aliases=["ls-url"], + parents=[parent_parser], + description=append_doc_link(LS_HELP, "ls-url"), + help=LS_HELP, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + get_parser.add_argument( + "url", help="See `dvc import-url -h` for full list of supported URLs." + ) + get_parser.set_defaults(func=CmdListUrl) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index b85ef8e0a..2168e2697 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -278,6 +278,11 @@ class PathMissingError(DvcException): self.dvc_only = dvc_only +class URLMissingError(DvcException): + def __init__(self, url): + super().__init__(f"The path '{url}' does not exist.") + + class RemoteCacheRequiredError(DvcException): def __init__(self, scheme, fs_path): diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 520a575c0..86e8a4a46 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -68,6 +68,7 @@ class Repo: from dvc.repo.imp_url import imp_url # type: ignore[misc] from dvc.repo.install import install # type: ignore[misc] from dvc.repo.ls import ls as _ls # type: ignore[misc] + from dvc.repo.ls_url import ls_url as _ls_url # type: ignore[misc] from dvc.repo.move import move # type: ignore[misc] from dvc.repo.pull import pull # type: ignore[misc] from dvc.repo.push import push # type: ignore[misc] @@ -80,6 +81,7 @@ class Repo: from .data import status as data_status # type: ignore[misc] ls = staticmethod(_ls) + ls_url = staticmethod(_ls_url) get = staticmethod(_get) get_url = staticmethod(_get_url) diff --git a/dvc/repo/ls_url.py b/dvc/repo/ls_url.py new file mode 100644 index 000000000..affc16266 --- /dev/null +++ b/dvc/repo/ls_url.py @@ -0,0 +1,24 @@ +from dvc.exceptions import URLMissingError +from dvc.output import Output + + +def ls_url(url): + out = Output(None, url) + fs, fs_path = out.fs, out.fs_path + + try: + info = fs.info(fs_path) + except FileNotFoundError: + raise URLMissingError(url) + if info["type"] != "directory": + return [{"path": info["name"], "isdir": False}] + + infos = fs.ls(fs_path, detail=True) + ret = [] + for info in infos: + ls_info = { + "path": fs.path.relpath(info["name"], fs_path), + "isdir": info["type"] == "directory", + } + ret.append(ls_info) + return ret diff --git a/tests/func/test_ls_url.py b/tests/func/test_ls_url.py new file mode 100644 index 000000000..57a665d0b --- /dev/null +++ b/tests/func/test_ls_url.py @@ -0,0 +1,34 @@ +import os + +import pytest + +from dvc.exceptions import URLMissingError +from dvc.repo import Repo + + +def match_files(entries, expected): + entries_content = {(d["path"], d["isdir"]) for d in entries} + expected_content = {(d["path"], d["isdir"]) for d in expected} + assert entries_content == expected_content + + +def test_file(tmp_dir): + tmp_dir.gen({"foo": "foo contents"}) + + result = Repo.ls_url("foo") + match_files(result, [{"path": os.path.abspath("foo"), "isdir": False}]) + + +def test_dir(tmp_dir): + tmp_dir.gen({"dir/foo": "foo contents", "dir/subdir/bar": "bar contents"}) + + result = Repo.ls_url("dir") + match_files( + result, + [{"path": "foo", "isdir": False}, {"path": "subdir", "isdir": True}], + ) + + +def test_nonexistent(tmp_dir): + with pytest.raises(URLMissingError): + Repo.ls_url("nonexistent") diff --git a/tests/unit/command/test_ls_url.py b/tests/unit/command/test_ls_url.py new file mode 100644 index 000000000..ff1aba241 --- /dev/null +++ b/tests/unit/command/test_ls_url.py @@ -0,0 +1,14 @@ +from dvc.cli import parse_args +from dvc.commands.ls_url import CmdListUrl + + +def test_ls_url(mocker): + cli_args = parse_args(["ls-url", "src"]) + assert cli_args.func == CmdListUrl + + cmd = cli_args.func(cli_args) + m = mocker.patch("dvc.repo.Repo.ls_url", autospec=True) + + assert cmd.run() == 0 + + m.assert_called_once_with("src")