ls-url: add --recursive option

dvc/commands/ls_url.py

@@ -16,7 +16,7 @@ class CmdListUrl(CmdBaseNoRepo):
         from dvc.repo import Repo
 
         try:
-            entries = Repo.ls_url(self.args.url)
+            entries = Repo.ls_url(self.args.url, recursive=self.args.recursive)
             if entries:
                 entries = _prettify(entries, with_color=True)
                 ui.write("\n".join(entries))
@@ -28,7 +28,7 @@ class CmdListUrl(CmdBaseNoRepo):
 
 def add_parser(subparsers, parent_parser):
     LS_HELP = "List directory contents from URL."
-    get_parser = subparsers.add_parser(
+    lsurl_parser = subparsers.add_parser(
         "list-url",
         aliases=["ls-url"],
         parents=[parent_parser],
@@ -36,7 +36,13 @@ def add_parser(subparsers, parent_parser):
         help=LS_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    get_parser.add_argument(
+    lsurl_parser.add_argument(
         "url", help="See `dvc import-url -h` for full list of supported URLs."
     )
-    get_parser.set_defaults(func=CmdListUrl)
+    lsurl_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        help="Recursively list files.",
+    )
+    lsurl_parser.set_defaults(func=CmdListUrl)

dvc/repo/ls_url.py

@@ -8,7 +8,7 @@ def parse_external_url(url):
     return fs, fs.path.abspath(fs.path.normpath(fs_path))
 
 
-def ls_url(url):
+def ls_url(url, recursive=False):
     fs, fs_path = parse_external_url(url)
     try:
         info = fs.info(fs_path)
@@ -17,12 +17,19 @@ def ls_url(url):
     if info["type"] != "directory":
         return [{"path": info["name"], "isdir": False}]
 
-    infos = fs.ls(fs_path, detail=True)
     ret = []
-    for info in infos:
-        ls_info = {
-            "path": fs.path.relpath(info["name"], fs_path),
-            "isdir": info["type"] == "directory",
-        }
-        ret.append(ls_info)
+    for _, dirs, files in fs.walk(fs_path, detail=True):
+        if not recursive:
+            files.update(dirs)
+
+        for info in files.values():
+            ls_info = {
+                "path": fs.path.relpath(info["name"], fs_path),
+                "isdir": info["type"] == "directory",
+            }
+            ret.append(ls_info)
+
+        if not recursive:
+            break
+
     return ret

tests/func/test_ls_url.py

@@ -31,6 +31,19 @@ def test_dir(tmp_dir):
     )
 
 
+def test_recursive(tmp_dir):
+    tmp_dir.gen({"dir/foo": "foo contents", "dir/subdir/bar": "bar contents"})
+
+    result = Repo.ls_url("dir", recursive=True)
+    match_files(
+        result,
+        [
+            {"path": "foo", "isdir": False},
+            {"path": "subdir/bar", "isdir": False},
+        ],
+    )
+
+
 def test_nonexistent(tmp_dir):
     with pytest.raises(URLMissingError):
         Repo.ls_url("nonexistent")