diff --git a/dvc/commands/ls_url.py b/dvc/commands/ls_url.py index 49405d043..05cd036b0 100644 --- a/dvc/commands/ls_url.py +++ b/dvc/commands/ls_url.py @@ -16,7 +16,7 @@ class CmdListUrl(CmdBaseNoRepo): from dvc.repo import Repo try: - entries = Repo.ls_url(self.args.url) + entries = Repo.ls_url(self.args.url, recursive=self.args.recursive) if entries: entries = _prettify(entries, with_color=True) ui.write("\n".join(entries)) @@ -28,7 +28,7 @@ class CmdListUrl(CmdBaseNoRepo): def add_parser(subparsers, parent_parser): LS_HELP = "List directory contents from URL." - get_parser = subparsers.add_parser( + lsurl_parser = subparsers.add_parser( "list-url", aliases=["ls-url"], parents=[parent_parser], @@ -36,7 +36,13 @@ def add_parser(subparsers, parent_parser): help=LS_HELP, formatter_class=argparse.RawDescriptionHelpFormatter, ) - get_parser.add_argument( + lsurl_parser.add_argument( "url", help="See `dvc import-url -h` for full list of supported URLs." ) - get_parser.set_defaults(func=CmdListUrl) + lsurl_parser.add_argument( + "-R", + "--recursive", + action="store_true", + help="Recursively list files.", + ) + lsurl_parser.set_defaults(func=CmdListUrl) diff --git a/dvc/repo/ls_url.py b/dvc/repo/ls_url.py index cf8263465..f0dba9c82 100644 --- a/dvc/repo/ls_url.py +++ b/dvc/repo/ls_url.py @@ -8,7 +8,7 @@ def parse_external_url(url): return fs, fs.path.abspath(fs.path.normpath(fs_path)) -def ls_url(url): +def ls_url(url, recursive=False): fs, fs_path = parse_external_url(url) try: info = fs.info(fs_path) @@ -17,12 +17,19 @@ def ls_url(url): if info["type"] != "directory": return [{"path": info["name"], "isdir": False}] - infos = fs.ls(fs_path, detail=True) ret = [] - for info in infos: - ls_info = { - "path": fs.path.relpath(info["name"], fs_path), - "isdir": info["type"] == "directory", - } - ret.append(ls_info) + for _, dirs, files in fs.walk(fs_path, detail=True): + if not recursive: + files.update(dirs) + + for info in files.values(): + ls_info = { + "path": fs.path.relpath(info["name"], fs_path), + "isdir": info["type"] == "directory", + } + ret.append(ls_info) + + if not recursive: + break + return ret diff --git a/tests/func/test_ls_url.py b/tests/func/test_ls_url.py index 94fb777fe..339b8710f 100644 --- a/tests/func/test_ls_url.py +++ b/tests/func/test_ls_url.py @@ -31,6 +31,19 @@ def test_dir(tmp_dir): ) +def test_recursive(tmp_dir): + tmp_dir.gen({"dir/foo": "foo contents", "dir/subdir/bar": "bar contents"}) + + result = Repo.ls_url("dir", recursive=True) + match_files( + result, + [ + {"path": "foo", "isdir": False}, + {"path": "subdir/bar", "isdir": False}, + ], + ) + + def test_nonexistent(tmp_dir): with pytest.raises(URLMissingError): Repo.ls_url("nonexistent")