ls/import/get: introduce --remote and --remote-config

Fixes #9656

dvc/cli/utils.py

@@ -1,3 +1,27 @@
+import argparse
+
+
+class DictAction(argparse.Action):
+    def __call__(self, parser, args, values, option_string=None):  # noqa: ARG002
+        d = getattr(args, self.dest) or {}
+
+        if isinstance(values, list):
+            kvs = values
+        else:
+            kvs = [values]
+
+        for kv in kvs:
+            key, value = kv.split("=")
+            if not value:
+                raise argparse.ArgumentError(
+                    self,
+                    f'Could not parse argument "{values}" as k1=v1 k2=v2 ... format',
+                )
+            d[key] = value
+
+        setattr(args, self.dest, d)
+
+
 def fix_subparsers(subparsers):
     """Workaround for bug in Python 3. See more info at:
     https://bugs.python.org/issue16308

dvc/commands/get.py

@@ -3,7 +3,7 @@ import logging
 
 from dvc.cli import completion
 from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import append_doc_link
+from dvc.cli.utils import DictAction, append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -38,6 +38,8 @@ class CmdGet(CmdBaseNoRepo):
                 jobs=self.args.jobs,
                 force=self.args.force,
                 config=self.args.config,
+                remote=self.args.remote,
+                remote_config=self.args.remote_config,
             )
             return 0
         except CloneError:
@@ -111,4 +113,19 @@ def add_parser(subparsers, parent_parser):
             "in the target repository."
         ),
     )
+    get_parser.add_argument(
+        "--remote",
+        type=str,
+        help="Remote name to set as a default in the target repository.",
+    )
+    get_parser.add_argument(
+        "--remote-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help=(
+            "Remote config options to merge with a remote's config (default or one "
+            "specified by '--remote') in the target repository."
+        ),
+    )
     get_parser.set_defaults(func=CmdGet)

dvc/commands/imp.py

@@ -3,7 +3,7 @@ import logging
 
 from dvc.cli import completion
 from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+from dvc.cli.utils import DictAction, append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -23,6 +23,8 @@ class CmdImport(CmdBase):
                 no_download=self.args.no_download,
                 jobs=self.args.jobs,
                 config=self.args.config,
+                remote=self.args.remote,
+                remote_config=self.args.remote_config,
             )
         except CloneError:
             logger.exception("failed to import '%s'", self.args.path)
@@ -103,4 +105,19 @@ def add_parser(subparsers, parent_parser):
             "in the target repository."
         ),
     )
+    import_parser.add_argument(
+        "--remote",
+        type=str,
+        help="Remote name to set as a default in the target repository.",
+    )
+    import_parser.add_argument(
+        "--remote-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help=(
+            "Remote config options to merge with a remote's config (default or one "
+            "specified by '--remote') in the target repository."
+        ),
+    )
     import_parser.set_defaults(func=CmdImport)

dvc/commands/ls/__init__.py

@@ -3,7 +3,7 @@ import logging
 
 from dvc.cli import completion
 from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import append_doc_link
+from dvc.cli.utils import DictAction, append_doc_link
 from dvc.commands.ls.ls_colors import LsColors
 from dvc.exceptions import DvcException
 from dvc.ui import ui
@@ -35,6 +35,8 @@ class CmdList(CmdBaseNoRepo):
                 recursive=self.args.recursive,
                 dvc_only=self.args.dvc_only,
                 config=self.args.config,
+                remote=self.args.remote,
+                remote_config=self.args.remote_config,
             )
             if self.args.json:
                 ui.write_json(entries)
@@ -89,6 +91,21 @@ def add_parser(subparsers, parent_parser):
             "in the target repository."
         ),
     )
+    list_parser.add_argument(
+        "--remote",
+        type=str,
+        help="Remote name to set as a default in the target repository.",
+    )
+    list_parser.add_argument(
+        "--remote-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help=(
+            "Remote config options to merge with a remote's config (default or one "
+            "specified by '--remote') in the target repository."
+        ),
+    )
     list_parser.add_argument(
         "path",
         nargs="?",

dvc/dependency/repo.py

@@ -1,6 +1,6 @@
 from typing import TYPE_CHECKING, Dict, Optional, Union
 
-from voluptuous import Required
+from voluptuous import Any, Required
 
 from dvc.utils import as_posix
 
@@ -17,6 +17,7 @@ class RepoDependency(Dependency):
     PARAM_REV = "rev"
     PARAM_REV_LOCK = "rev_lock"
     PARAM_CONFIG = "config"
+    PARAM_REMOTE = "remote"
 
     REPO_SCHEMA = {
         PARAM_REPO: {
@@ -24,6 +25,7 @@ class RepoDependency(Dependency):
             PARAM_REV: str,
             PARAM_REV_LOCK: str,
             PARAM_CONFIG: str,
+            PARAM_REMOTE: Any(str, dict),
         }
     }
 
@@ -76,6 +78,10 @@ class RepoDependency(Dependency):
         if config:
             repo[self.PARAM_CONFIG] = config
 
+        remote = self.def_repo.get(self.PARAM_REMOTE)
+        if remote:
+            repo[self.PARAM_REMOTE] = remote
+
         return {
             self.PARAM_PATH: self.def_path,
             self.PARAM_REPO: repo,
@@ -99,6 +105,14 @@ class RepoDependency(Dependency):
         from dvc.config import Config
         from dvc.fs import DVCFileSystem
 
+        rem = self.def_repo.get("remote")
+        if isinstance(rem, dict):
+            remote = None  # type: ignore[unreachable]
+            remote_config = rem
+        else:
+            remote = rem
+            remote_config = None
+
         conf = self.def_repo.get("config")
         if conf:
             config = Config.load_file(conf)
@@ -113,6 +127,8 @@ class RepoDependency(Dependency):
             rev=rev or self._get_rev(locked=locked),
             subrepos=True,
             config=config,
+            remote=remote,
+            remote_config=remote_config,
         )
 
     def _get_rev(self, locked: bool = True):

dvc/repo/get.py

@@ -19,7 +19,17 @@ class GetDVCFileError(DvcException):
         )
 
 
-def get(url, path, out=None, rev=None, jobs=None, force=False, config=None):
+def get(
+    url,
+    path,
+    out=None,
+    rev=None,
+    jobs=None,
+    force=False,
+    config=None,
+    remote=None,
+    remote_config=None,
+):
     from dvc.config import Config
     from dvc.dvcfile import is_valid_filename
     from dvc.repo import Repo
@@ -38,6 +48,8 @@ def get(url, path, out=None, rev=None, jobs=None, force=False, config=None):
         subrepos=True,
         uninitialized=True,
         config=config,
+        remote=remote,
+        remote_config=remote_config,
     ) as repo:
         from dvc.fs import download
         from dvc.fs.data import DataFileSystem

dvc/repo/imp.py

@@ -1,4 +1,14 @@
-def imp(self, url, path, out=None, rev=None, config=None, **kwargs):
+def imp(
+    self,
+    url,
+    path,
+    out=None,
+    rev=None,
+    config=None,
+    remote=None,
+    remote_config=None,
+    **kwargs,
+):
     erepo = {"url": url}
     if rev is not None:
         erepo["rev"] = rev
@@ -6,4 +16,17 @@ def imp(self, url, path, out=None, rev=None, config=None, **kwargs):
     if config is not None:
         erepo["config"] = config
 
+    if remote is not None and remote_config is not None:
+        conf = erepo.get("config") or {}
+        remotes = conf.get("remote") or {}
+        remote_conf = remotes.get(remote) or {}
+        remote_conf.update(remote_config)
+        remotes[remote] = remote_conf
+        conf["remote"] = remotes
+        erepo["config"] = conf
+    elif remote is not None:
+        erepo["remote"] = remote
+    elif remote_config is not None:
+        erepo["remote"] = remote_config
+
     return self.imp_url(path, out=out, erepo=erepo, frozen=True, **kwargs)

dvc/repo/ls.py

@@ -14,6 +14,8 @@ def ls(
     recursive: Optional[bool] = None,
     dvc_only: bool = False,
     config: Optional[str] = None,
+    remote: Optional[str] = None,
+    remote_config: Optional[dict] = None,
 ):
     """Methods for getting files and outputs for the repo.
 
@@ -23,7 +25,9 @@ def ls(
         rev (str, optional): SHA commit, branch or tag name
         recursive (bool, optional): recursively walk the repo
         dvc_only (bool, optional): show only DVC-artifacts
-        config (bool, optional): path to config file
+        config (str, optional): path to config file
+        remote (str, optional): remote name to set as a default remote in the repo
+        remote_config (str, dict): remote config to merge with a remote in the repo
 
     Returns:
         list of `entry`
@@ -47,7 +51,13 @@ def ls(
         config_dict = None
 
     with Repo.open(
-        url, rev=rev, subrepos=True, uninitialized=True, config=config_dict
+        url,
+        rev=rev,
+        subrepos=True,
+        uninitialized=True,
+        config=config_dict,
+        remote=remote,
+        remote_config=remote_config,
     ) as repo:
         path = path or ""
 

tests/unit/command/ls/test_ls.py

@@ -19,7 +19,14 @@ def test_list(mocker):
     url = "local_dir"
     m = _test_cli(mocker, url)
     m.assert_called_once_with(
-        url, None, recursive=False, rev=None, dvc_only=False, config=None
+        url,
+        None,
+        recursive=False,
+        rev=None,
+        dvc_only=False,
+        config=None,
+        remote=None,
+        remote_config=None,
     )
 
 
@@ -27,7 +34,14 @@ def test_list_recursive(mocker):
     url = "local_dir"
     m = _test_cli(mocker, url, "-R")
     m.assert_called_once_with(
-        url, None, recursive=True, rev=None, dvc_only=False, config=None
+        url,
+        None,
+        recursive=True,
+        rev=None,
+        dvc_only=False,
+        config=None,
+        remote=None,
+        remote_config=None,
     )
 
 
@@ -35,7 +49,14 @@ def test_list_git_ssh_rev(mocker):
     url = "[email protected]:repo"
     m = _test_cli(mocker, url, "--rev", "123")
     m.assert_called_once_with(
-        url, None, recursive=False, rev="123", dvc_only=False, config=None
+        url,
+        None,
+        recursive=False,
+        rev="123",
+        dvc_only=False,
+        config=None,
+        remote=None,
+        remote_config=None,
     )
 
 
@@ -44,7 +65,14 @@ def test_list_targets(mocker):
     target = "subdir"
     m = _test_cli(mocker, url, target)
     m.assert_called_once_with(
-        url, target, recursive=False, rev=None, dvc_only=False, config=None
+        url,
+        target,
+        recursive=False,
+        rev=None,
+        dvc_only=False,
+        config=None,
+        remote=None,
+        remote_config=None,
     )
 
 
@@ -52,15 +80,40 @@ def test_list_outputs_only(mocker):
     url = "local_dir"
     m = _test_cli(mocker, url, None, "--dvc-only")
     m.assert_called_once_with(
-        url, None, recursive=False, rev=None, dvc_only=True, config=None
+        url,
+        None,
+        recursive=False,
+        rev=None,
+        dvc_only=True,
+        config=None,
+        remote=None,
+        remote_config=None,
     )
 
 
 def test_list_config(mocker):
     url = "local_dir"
-    m = _test_cli(mocker, url, None, "--config", "myconfig")
+    m = _test_cli(
+        mocker,
+        url,
+        None,
+        "--config",
+        "myconfig",
+        "--remote",
+        "myremote",
+        "--remote-config",
+        "k1=v1",
+        "k2=v2",
+    )
     m.assert_called_once_with(
-        url, None, recursive=False, rev=None, dvc_only=False, config="myconfig"
+        url,
+        None,
+        recursive=False,
+        rev=None,
+        dvc_only=False,
+        config="myconfig",
+        remote="myremote",
+        remote_config={"k1": "v1", "k2": "v2"},
     )
 
 

tests/unit/command/test_get.py

@@ -16,6 +16,11 @@ def test_get(mocker):
             "4",
             "--config",
             "myconfig",
+            "--remote",
+            "myremote",
+            "--remote-config",
+            "k1=v1",
+            "k2=v2",
         ]
     )
     assert cli_args.func == CmdGet
@@ -33,6 +38,8 @@ def test_get(mocker):
         jobs=4,
         config="myconfig",
         force=False,
+        remote="myremote",
+        remote_config={"k1": "v1", "k2": "v2"},
     )
 
 

tests/unit/command/test_imp.py

@@ -16,6 +16,11 @@ def test_import(mocker, dvc):
             "3",
             "--config",
             "myconfig",
+            "--remote",
+            "myremote",
+            "--remote-config",
+            "k1=v1",
+            "k2=v2",
         ]
     )
     assert cli_args.func == CmdImport
@@ -34,6 +39,8 @@ def test_import(mocker, dvc):
         no_download=False,
         jobs=3,
         config="myconfig",
+        remote="myremote",
+        remote_config={"k1": "v1", "k2": "v2"},
     )
 
 
@@ -65,6 +72,8 @@ def test_import_no_exec(mocker, dvc):
         no_download=False,
         jobs=None,
         config=None,
+        remote=None,
+        remote_config=None,
     )
 
 
@@ -96,4 +105,6 @@ def test_import_no_download(mocker, dvc):
         no_download=True,
         jobs=None,
         config=None,
+        remote=None,
+        remote_config=None,
     )