From fa7619dfeeff66c1f96b0adacfad5b8ca3a49e49 Mon Sep 17 00:00:00 2001 From: Ronan Lamy Date: Thu, 8 Sep 2022 17:53:20 +0100 Subject: [PATCH] ls-url: create basic version of dvc ls-url command For now, this only takes a URL (or local path) and lists its contents if the protocol supports it. The command fails if the target doesn't exist and returns the (absolute) path to the target if it's not a directory. --- dvc/cli/parser.py | 2 ++ dvc/commands/ls_url.py | 42 +++++++++++++++++++++++++++++++ dvc/exceptions.py | 5 ++++ dvc/repo/__init__.py | 2 ++ dvc/repo/ls_url.py | 24 ++++++++++++++++++ tests/func/test_ls_url.py | 34 +++++++++++++++++++++++++ tests/unit/command/test_ls_url.py | 14 +++++++++++ 7 files changed, 123 insertions(+) create mode 100644 dvc/commands/ls_url.py create mode 100644 dvc/repo/ls_url.py create mode 100644 tests/func/test_ls_url.py create mode 100644 tests/unit/command/test_ls_url.py diff --git a/dvc/cli/parser.py b/dvc/cli/parser.py index c626daf9d..c1ce67183 100644 --- a/dvc/cli/parser.py +++ b/dvc/cli/parser.py @@ -29,6 +29,7 @@ from dvc.commands import ( init, install, ls, + ls_url, machine, metrics, move, @@ -75,6 +76,7 @@ COMMANDS = [ install, root, ls, + ls_url, freeze, dag, daemon, diff --git a/dvc/commands/ls_url.py b/dvc/commands/ls_url.py new file mode 100644 index 000000000..49405d043 --- /dev/null +++ b/dvc/commands/ls_url.py @@ -0,0 +1,42 @@ +import argparse +import logging + +from dvc.cli.command import CmdBaseNoRepo +from dvc.cli.utils import append_doc_link +from dvc.exceptions import DvcException +from dvc.ui import ui + +from .ls import _prettify + +logger = logging.getLogger(__name__) + + +class CmdListUrl(CmdBaseNoRepo): + def run(self): + from dvc.repo import Repo + + try: + entries = Repo.ls_url(self.args.url) + if entries: + entries = _prettify(entries, with_color=True) + ui.write("\n".join(entries)) + return 0 + except DvcException: + logger.exception("failed to list '%s'", self.args.url) + return 1 + + +def add_parser(subparsers, parent_parser): + LS_HELP = "List directory contents from URL." + get_parser = subparsers.add_parser( + "list-url", + aliases=["ls-url"], + parents=[parent_parser], + description=append_doc_link(LS_HELP, "ls-url"), + help=LS_HELP, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + get_parser.add_argument( + "url", help="See `dvc import-url -h` for full list of supported URLs." + ) + get_parser.set_defaults(func=CmdListUrl) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index b85ef8e0a..2168e2697 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -278,6 +278,11 @@ class PathMissingError(DvcException): self.dvc_only = dvc_only +class URLMissingError(DvcException): + def __init__(self, url): + super().__init__(f"The path '{url}' does not exist.") + + class RemoteCacheRequiredError(DvcException): def __init__(self, scheme, fs_path): diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 520a575c0..86e8a4a46 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -68,6 +68,7 @@ class Repo: from dvc.repo.imp_url import imp_url # type: ignore[misc] from dvc.repo.install import install # type: ignore[misc] from dvc.repo.ls import ls as _ls # type: ignore[misc] + from dvc.repo.ls_url import ls_url as _ls_url # type: ignore[misc] from dvc.repo.move import move # type: ignore[misc] from dvc.repo.pull import pull # type: ignore[misc] from dvc.repo.push import push # type: ignore[misc] @@ -80,6 +81,7 @@ class Repo: from .data import status as data_status # type: ignore[misc] ls = staticmethod(_ls) + ls_url = staticmethod(_ls_url) get = staticmethod(_get) get_url = staticmethod(_get_url) diff --git a/dvc/repo/ls_url.py b/dvc/repo/ls_url.py new file mode 100644 index 000000000..affc16266 --- /dev/null +++ b/dvc/repo/ls_url.py @@ -0,0 +1,24 @@ +from dvc.exceptions import URLMissingError +from dvc.output import Output + + +def ls_url(url): + out = Output(None, url) + fs, fs_path = out.fs, out.fs_path + + try: + info = fs.info(fs_path) + except FileNotFoundError: + raise URLMissingError(url) + if info["type"] != "directory": + return [{"path": info["name"], "isdir": False}] + + infos = fs.ls(fs_path, detail=True) + ret = [] + for info in infos: + ls_info = { + "path": fs.path.relpath(info["name"], fs_path), + "isdir": info["type"] == "directory", + } + ret.append(ls_info) + return ret diff --git a/tests/func/test_ls_url.py b/tests/func/test_ls_url.py new file mode 100644 index 000000000..57a665d0b --- /dev/null +++ b/tests/func/test_ls_url.py @@ -0,0 +1,34 @@ +import os + +import pytest + +from dvc.exceptions import URLMissingError +from dvc.repo import Repo + + +def match_files(entries, expected): + entries_content = {(d["path"], d["isdir"]) for d in entries} + expected_content = {(d["path"], d["isdir"]) for d in expected} + assert entries_content == expected_content + + +def test_file(tmp_dir): + tmp_dir.gen({"foo": "foo contents"}) + + result = Repo.ls_url("foo") + match_files(result, [{"path": os.path.abspath("foo"), "isdir": False}]) + + +def test_dir(tmp_dir): + tmp_dir.gen({"dir/foo": "foo contents", "dir/subdir/bar": "bar contents"}) + + result = Repo.ls_url("dir") + match_files( + result, + [{"path": "foo", "isdir": False}, {"path": "subdir", "isdir": True}], + ) + + +def test_nonexistent(tmp_dir): + with pytest.raises(URLMissingError): + Repo.ls_url("nonexistent") diff --git a/tests/unit/command/test_ls_url.py b/tests/unit/command/test_ls_url.py new file mode 100644 index 000000000..ff1aba241 --- /dev/null +++ b/tests/unit/command/test_ls_url.py @@ -0,0 +1,14 @@ +from dvc.cli import parse_args +from dvc.commands.ls_url import CmdListUrl + + +def test_ls_url(mocker): + cli_args = parse_args(["ls-url", "src"]) + assert cli_args.func == CmdListUrl + + cmd = cli_args.func(cli_args) + m = mocker.patch("dvc.repo.Repo.ls_url", autospec=True) + + assert cmd.run() == 0 + + m.assert_called_once_with("src")