get, get-url: add --jobs option (#5362)

dvc/command/get.py

@@ -39,6 +39,7 @@ class CmdGet(CmdBaseNoRepo):
                 path=self.args.path,
                 out=self.args.out,
                 rev=self.args.rev,
+                jobs=self.args.jobs,
             )
             return 0
         except DvcException:
@@ -84,4 +85,15 @@ def add_parser(subparsers, parent_parser):
         help="Print the storage location (URL) the target data would be "
         "downloaded from, and exit.",
     )
+    get_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
+        metavar="<number>",
+    )
     get_parser.set_defaults(func=CmdGet)

dvc/command/get_url.py

@@ -14,7 +14,7 @@ class CmdGetUrl(CmdBaseNoRepo):
         from dvc.repo import Repo
 
         try:
-            Repo.get_url(self.args.url, out=self.args.out)
+            Repo.get_url(self.args.url, out=self.args.out, jobs=self.args.jobs)
             return 0
         except DvcException:
             logger.exception(f"failed to get '{self.args.url}'")
@@ -36,4 +36,15 @@ def add_parser(subparsers, parent_parser):
     get_parser.add_argument(
         "out", nargs="?", help="Destination path to put data to.",
     ).complete = completion.DIR
+    get_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
+        metavar="<number>",
+    )
     get_parser.set_defaults(func=CmdGetUrl)

dvc/repo/get.py

@@ -17,7 +17,7 @@ class GetDVCFileError(DvcException):
         )
 
 
-def get(url, path, out=None, rev=None):
+def get(url, path, out=None, rev=None, jobs=None):
     import shortuuid
 
     from dvc.dvcfile import is_valid_filename
@@ -52,6 +52,8 @@ def get(url, path, out=None, rev=None):
         ) as repo:
             from_info = PathInfo(repo.root_dir) / path
             to_info = PathInfo(out)
-            repo.repo_tree.download(from_info, to_info, follow_subrepos=False)
+            repo.repo_tree.download(
+                from_info, to_info, jobs=jobs, follow_subrepos=False
+            )
     finally:
         remove(tmp_dir)

dvc/repo/get_url.py

@@ -1,7 +1,7 @@
 import os
 
 
-def get_url(url, out=None):
+def get_url(url, out=None, jobs=None):
     import dvc.dependency as dependency
     import dvc.output as output
     from dvc.utils import resolve_output
@@ -17,4 +17,4 @@ def get_url(url, out=None):
     (out,) = output.loads_from(None, [out], use_cache=False)
     if not dep.exists:
         raise dep.DoesNotExistError(dep)
-    dep.download(out)
+    dep.download(out, jobs=jobs)

tests/unit/command/test_get.py

@@ -4,7 +4,17 @@ from dvc.command.get import CmdGet
 
 def test_get(mocker):
     cli_args = parse_args(
-        ["get", "repo_url", "src", "--out", "out", "--rev", "version"]
+        [
+            "get",
+            "repo_url",
+            "src",
+            "--out",
+            "out",
+            "--rev",
+            "version",
+            "--jobs",
+            "4",
+        ]
     )
     assert cli_args.func == CmdGet
 
@@ -13,7 +23,9 @@ def test_get(mocker):
 
     assert cmd.run() == 0
 
-    m.assert_called_once_with("repo_url", path="src", out="out", rev="version")
+    m.assert_called_once_with(
+        "repo_url", path="src", out="out", rev="version", jobs=4
+    )
 
 
 def test_get_url(mocker, caplog):

tests/unit/command/test_get_url.py

@@ -3,7 +3,7 @@ from dvc.command.get_url import CmdGetUrl
 
 
 def test_get_url(mocker):
-    cli_args = parse_args(["get-url", "src", "out"])
+    cli_args = parse_args(["get-url", "src", "out", "-j", "5"])
     assert cli_args.func == CmdGetUrl
 
     cmd = cli_args.func(cli_args)
@@ -11,4 +11,4 @@ def test_get_url(mocker):
 
     assert cmd.run() == 0
 
-    m.assert_called_once_with("src", out="out")
+    m.assert_called_once_with("src", out="out", jobs=5)