checkout: Expose --allow-missing arg. (#9919)

Implements fsspec's fsid https://github.com/fsspec/filesystem_spec/pull/1122

Required for #9904

.github/workflows/build.yaml

@@ -15,7 +15,7 @@ jobs:
   pip:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
 

.github/workflows/codeql.yml

@@ -28,7 +28,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2

.github/workflows/plugin_tests.yaml

@@ -23,11 +23,11 @@ jobs:
         plugin: ["dvc-s3"]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         path: dvc
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         repository: iterative/${{ matrix.plugin }}
         ref: main

.github/workflows/tests.yaml

@@ -26,7 +26,7 @@ jobs:
         pyv: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Set up Python
@@ -66,7 +66,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Slack Notification
-      uses: rtCamp/[email protected].0
+      uses: rtCamp/[email protected].1
       env:
         SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         SLACK_COLOR: ${{ job.status }}

.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
     hooks:
       - id: isort
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.0.282'
+    rev: 'v0.0.287'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

dvc/__pyinstaller/hook-dvc.py

@@ -22,6 +22,7 @@ hiddenimports = [
     "dvc_hdfs",
     "dvc_oss",
     "dvc_s3",
+    "dvc_ssh",
     "dvc_webdav",
     "dvc_webhdfs",
     # https://github.com/pypa/setuptools/issues/1963

dvc/cachemgr.py

@@ -42,11 +42,8 @@ class CacheManager:
         self.config = config = repo.config["cache"]
         self._odb = {}
 
-        default = None
-        if repo and repo.local_dvc_dir:
-            default = os.path.join(repo.local_dvc_dir, self.CACHE_DIR)
-
         local = config.get("local")
+        default = self.default_local_cache_dir
 
         if local:
             settings = {"name": local}
@@ -103,6 +100,13 @@ class CacheManager:
         """
         return self.legacy.path
 
+    @property
+    def default_local_cache_dir(self) -> Optional[str]:
+        repo = self._repo
+        if repo and repo.local_dvc_dir:
+            return os.path.join(repo.local_dvc_dir, self.CACHE_DIR)
+        return None
+
 
 def migrate_2_to_3(repo: "Repo", dry: bool = False):
     """Migrate legacy 2.x objects to 3.x cache.

dvc/cli/parser.py

@@ -31,7 +31,6 @@ from dvc.commands import (
     install,
     ls,
     ls_url,
-    machine,
     metrics,
     move,
     params,
@@ -89,7 +88,6 @@ COMMANDS = [
     stage,
     experiments,
     check_ignore,
-    machine,
     data,
 ]
 

dvc/cli/utils.py

@@ -11,7 +11,7 @@ class DictAction(argparse.Action):
             kvs = [values]
 
         for kv in kvs:
-            key, value = kv.split("=")
+            key, value = kv.split("=", 1)
             if not value:
                 raise argparse.ArgumentError(
                     self,

dvc/commands/checkout.py

@@ -37,6 +37,7 @@ class CmdCheckout(CmdBase):
                 force=self.args.force,
                 relink=self.args.relink,
                 recursive=self.args.recursive,
+                allow_missing=self.args.allow_missing,
             )
         except CheckoutError as _exc:
             exc = _exc
@@ -101,6 +102,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Recreate links or copies from cache to workspace.",
     )
+    checkout_parser.add_argument(
+        "--allow-missing",
+        action="store_true",
+        default=False,
+        help="Ignore errors if some of the files or directories are missing.",
+    )
     checkout_parser.add_argument(
         "targets",
         nargs="*",

dvc/commands/experiments/ls.py

@@ -23,15 +23,22 @@ class CmdExperimentsList(CmdBase):
             git_remote=git_remote,
         )
 
+        from dvc.repo.experiments.utils import describe
+        from dvc.scm import Git
+
+        if name_only or sha_only:
+            names = {}
+        else:
+            assert isinstance(self.repo.scm, Git)
+            names = describe(
+                self.repo.scm,
+                (baseline for baseline in exps),
+                logger=logger,
+            )
+
         for baseline in exps:
             if not (name_only or sha_only):
-                tag_base = "refs/tags/"
-                branch_base = "refs/heads/"
-                name = baseline[:7]
-                if baseline.startswith(tag_base):
-                    name = baseline[len(tag_base) :]
-                elif baseline.startswith(branch_base):
-                    name = baseline[len(branch_base) :]
+                name = names.get(baseline) or baseline[:7]
                 ui.write(f"{name}:")
             for exp_name, rev in exps[baseline]:
                 if name_only:
@@ -52,6 +59,7 @@ def add_parser(experiments_subparsers, parent_parser):
     EXPERIMENTS_LIST_HELP = "List local and remote experiments."
     experiments_list_parser = experiments_subparsers.add_parser(
         "list",
+        aliases=["ls"],
         parents=[parent_parser],
         description=append_doc_link(EXPERIMENTS_LIST_HELP, "exp/list"),
         help=EXPERIMENTS_LIST_HELP,

dvc/commands/experiments/remove.py

@@ -50,6 +50,7 @@ def add_parser(experiments_subparsers, parent_parser):
     EXPERIMENTS_REMOVE_HELP = "Remove experiments."
     experiments_remove_parser = experiments_subparsers.add_parser(
         "remove",
+        aliases=["rm"],
         parents=[parent_parser],
         description=append_doc_link(EXPERIMENTS_REMOVE_HELP, "exp/remove"),
         help=EXPERIMENTS_REMOVE_HELP,

dvc/commands/experiments/run.py

@@ -17,7 +17,6 @@ class CmdExperimentsRun(CmdRepro):
             jobs=self.args.jobs,
             params=self.args.set_param,
             tmp_dir=self.args.tmp_dir,
-            machine=self.args.machine,
             copy_paths=self.args.copy_paths,
             message=self.args.message,
             **self._common_kwargs,
@@ -90,15 +89,6 @@ def _add_run_common(parser):
             "your workspace."
         ),
     )
-    parser.add_argument(
-        "--machine",
-        default=None,
-        help=argparse.SUPPRESS,
-        # help=(
-        #     "Run this experiment on the specified 'dvc machine' instance."
-        # )
-        # metavar="<name>",
-    )
     parser.add_argument(
         "-C",
         "--copy-paths",

dvc/commands/experiments/save.py

@@ -27,10 +27,6 @@ class CmdExperimentsSave(CmdBase):
         else:
             name = self.repo.experiments.get_exact_name([ref])[ref]
             ui.write(f"Experiment has been saved as: {name}")
-            ui.write(
-                "\nTo promote an experiment to a Git branch run:\n\n"
-                "\tdvc exp branch <exp> <branch>\n"
-            )
 
         return 0
 

dvc/commands/get_url.py

@@ -3,7 +3,7 @@ import logging
 
 from dvc.cli import completion
 from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import append_doc_link
+from dvc.cli.utils import DictAction, append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -19,6 +19,7 @@ class CmdGetUrl(CmdBaseNoRepo):
                 out=self.args.out,
                 jobs=self.args.jobs,
                 force=self.args.force,
+                fs_config=self.args.fs_config,
             )
             return 0
         except DvcException:
@@ -58,4 +59,11 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Override local file or folder if exists.",
     )
+    get_parser.add_argument(
+        "--fs-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help="Config options for the target url.",
+    )
     get_parser.set_defaults(func=CmdGetUrl)

dvc/commands/imp_url.py

@@ -3,7 +3,7 @@ import logging
 
 from dvc.cli import completion
 from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+from dvc.cli.utils import DictAction, append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -22,6 +22,7 @@ class CmdImportUrl(CmdBase):
                 jobs=self.args.jobs,
                 force=self.args.force,
                 version_aware=self.args.version_aware,
+                fs_config=self.args.fs_config,
             )
         except DvcException:
             logger.exception(
@@ -114,4 +115,11 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Import using cloud versioning. Implied if the URL contains a version ID.",
     )
+    import_parser.add_argument(
+        "--fs-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help="Config options for the target url.",
+    )
     import_parser.set_defaults(func=CmdImportUrl)

dvc/commands/ls/__init__.py

@@ -11,7 +11,18 @@ from dvc.ui import ui
 logger = logging.getLogger(__name__)
 
 
-def _prettify(entries, with_color=False):
+def _format_entry(entry, fmt):
+    from dvc.utils.humanize import naturalsize
+
+    size = entry.get("size")
+    if size is None:
+        size = ""
+    else:
+        size = naturalsize(size)
+    return size, fmt(entry)
+
+
+def show_entries(entries, with_color=False, with_size=False):
     if with_color:
         ls_colors = LsColors()
         fmt = ls_colors.format
@@ -20,7 +31,12 @@ def _prettify(entries, with_color=False):
         def fmt(entry):
             return entry["path"]
 
-    return [fmt(entry) for entry in entries]
+    if with_size:
+        ui.table([_format_entry(entry, fmt) for entry in entries])
+        return
+
+    # NOTE: this is faster than ui.table for very large number of entries
+    ui.write("\n".join(fmt(entry) for entry in entries))
 
 
 class CmdList(CmdBaseNoRepo):
@@ -41,8 +57,7 @@ class CmdList(CmdBaseNoRepo):
             if self.args.json:
                 ui.write_json(entries)
             elif entries:
-                entries = _prettify(entries, with_color=True)
-                ui.write("\n".join(entries))
+                show_entries(entries, with_color=True, with_size=self.args.size)
             return 0
         except DvcException:
             logger.exception("failed to list '%s'", self.args.url)
@@ -106,6 +121,11 @@ def add_parser(subparsers, parent_parser):
             "specified by '--remote') in the target repository."
         ),
     )
+    list_parser.add_argument(
+        "--size",
+        action="store_true",
+        help="Show sizes.",
+    )
     list_parser.add_argument(
         "path",
         nargs="?",

dvc/commands/ls_url.py

@@ -2,10 +2,9 @@ import argparse
 import logging
 
 from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import append_doc_link
-from dvc.ui import ui
+from dvc.cli.utils import DictAction, append_doc_link
 
-from .ls import _prettify
+from .ls import show_entries
 
 logger = logging.getLogger(__name__)
 
@@ -14,10 +13,13 @@ class CmdListUrl(CmdBaseNoRepo):
     def run(self):
         from dvc.repo import Repo
 
-        entries = Repo.ls_url(self.args.url, recursive=self.args.recursive)
+        entries = Repo.ls_url(
+            self.args.url,
+            recursive=self.args.recursive,
+            fs_config=self.args.fs_config,
+        )
         if entries:
-            entries = _prettify(entries, with_color=True)
-            ui.write("\n".join(entries))
+            show_entries(entries, with_color=True, with_size=self.args.size)
         return 0
 
 
@@ -40,4 +42,16 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         help="Recursively list files.",
     )
+    lsurl_parser.add_argument(
+        "--size",
+        action="store_true",
+        help="Show sizes.",
+    )
+    lsurl_parser.add_argument(
+        "--fs-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help="Config options for the target url.",
+    )
     lsurl_parser.set_defaults(func=CmdListUrl)

dvc/commands/machine.py

@@ -1,490 +0,0 @@
-import argparse
-from typing import Dict, List
-
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link, fix_subparsers
-from dvc.commands.config import CmdConfig
-from dvc.compare import TabularData
-from dvc.config import ConfigError
-from dvc.exceptions import DvcException
-from dvc.ui import ui
-from dvc.utils import format_link
-
-
-class MachineDisabledError(ConfigError):
-    def __init__(self):
-        super().__init__("Machine feature is disabled")
-
-
-class CmdMachineConfig(CmdConfig):
-    def __init__(self, args):
-        super().__init__(args)
-        if not self.config["feature"].get("machine", False):
-            raise MachineDisabledError
-
-        if getattr(self.args, "name", None):
-            self.args.name = self.args.name.lower()
-
-    def _check_exists(self, conf):
-        if self.args.name not in conf["machine"]:
-            raise ConfigError(f"machine '{self.args.name}' doesn't exist.")
-
-
-class CmdMachineAdd(CmdMachineConfig):
-    def run(self):
-        from dvc.machine import validate_name
-
-        validate_name(self.args.name)
-
-        if self.args.default:
-            ui.write(f"Setting '{self.args.name}' as a default machine.")
-
-        with self.config.edit(self.args.level) as conf:
-            if self.args.name in conf["machine"] and not self.args.force:
-                raise ConfigError(
-                    "machine '{}' already exists. Use `-f|--force` to "
-                    "overwrite it.".format(self.args.name)
-                )
-
-            conf["machine"][self.args.name] = {"cloud": self.args.cloud}
-            if self.args.default:
-                conf["core"]["machine"] = self.args.name
-
-        return 0
-
-
-class CmdMachineRemove(CmdMachineConfig):
-    def run(self):
-        with self.config.edit(self.args.level) as conf:
-            self._check_exists(conf)
-            del conf["machine"][self.args.name]
-
-        up_to_level = self.args.level or "repo"
-        # Remove core.machine refs to this machine in any shadowing configs
-        for level in reversed(self.config.LEVELS):
-            with self.config.edit(level) as conf:
-                if conf["core"].get("machine") == self.args.name:
-                    del conf["core"]["machine"]
-
-            if level == up_to_level:
-                break
-
-        return 0
-
-
-class CmdMachineList(CmdMachineConfig):
-    TABLE_COLUMNS = [
-        "name",
-        "cloud",
-        "region",
-        "image",
-        "spot",
-        "spot_price",
-        "instance_hdd_size",
-        "instance_type",
-        "ssh_private",
-        "startup_script",
-    ]
-
-    PRIVATE_COLUMNS = ["ssh_private", "startup_script"]
-
-    def _hide_private(self, conf):
-        for machine in conf:
-            for column in self.PRIVATE_COLUMNS:
-                if column in conf[machine]:
-                    conf[machine][column] = "***"
-
-    def _show_origin(self):
-        levels = [self.args.level] if self.args.level else self.config.LEVELS
-        for level in levels:
-            conf = self.config.read(level)["machine"]
-            if self.args.name:
-                conf = conf.get(self.args.name, {})
-            self._hide_private(conf)
-            prefix = self._config_file_prefix(True, self.config, level)
-            configs = list(self._format_config(conf, prefix))
-            if configs:
-                ui.write("\n".join(configs))
-
-    def _show_table(self):
-        td = TabularData(self.TABLE_COLUMNS, fill_value="-")
-        conf = self.config.read()["machine"]
-        if self.args.name:
-            conf = {self.args.name: conf.get(self.args.name, {})}
-        self._hide_private(conf)
-        for machine, machine_config in conf.items():
-            machine_config["name"] = machine
-            td.row_from_dict(machine_config)
-        td.dropna("cols", "all")
-        td.render()
-
-    def run(self):
-        if self.args.show_origin:
-            self._show_origin()
-        else:
-            self._show_table()
-        return 0
-
-
-class CmdMachineModify(CmdMachineConfig):
-    def run(self):
-        from dvc.config import merge
-
-        with self.config.edit(self.args.level) as conf:
-            merged = self.config.load_config_to_level(self.args.level)
-            merge(merged, conf)
-            self._check_exists(merged)
-
-            if self.args.name not in conf["machine"]:
-                conf["machine"][self.args.name] = {}
-            section = conf["machine"][self.args.name]
-            if self.args.unset:
-                section.pop(self.args.option, None)
-            else:
-                section[self.args.option] = self.args.value
-        return 0
-
-
-class CmdMachineRename(CmdBase):
-    def _check_exists(self, conf):
-        if self.args.name not in conf["machine"]:
-            raise ConfigError(f"machine '{self.args.name}' doesn't exist.")
-
-    def _rename_default(self, conf):
-        if conf["core"].get("machine") == self.args.name:
-            conf["core"]["machine"] = self.args.new
-
-    def _check_before_rename(self):
-        from dvc.machine import validate_name
-
-        validate_name(self.args.new)
-
-        all_config = self.config.load_config_to_level(None)
-        if self.args.new in all_config.get("machine", {}):
-            raise ConfigError(
-                f"Rename failed. Machine '{self.args.new}' already exists."
-            )
-        ui.write(f"Rename machine '{self.args.name}' to '{self.args.new}'.")
-
-    def run(self):
-        self._check_before_rename()
-
-        with self.config.edit(self.args.level) as conf:
-            self._check_exists(conf)
-            conf["machine"][self.args.new] = conf["machine"][self.args.name]
-            try:
-                assert self.repo.machine
-                self.repo.machine.rename(self.args.name, self.args.new)
-            except DvcException as error:
-                del conf["machine"][self.args.new]
-                raise ConfigError("terraform rename failed") from error
-            del conf["machine"][self.args.name]
-            self._rename_default(conf)
-
-        up_to_level = self.args.level or "repo"
-        for level in reversed(self.config.LEVELS):
-            if level == up_to_level:
-                break
-            with self.config.edit(level) as level_conf:
-                self._rename_default(level_conf)
-
-        return 0
-
-
-class CmdMachineDefault(CmdMachineConfig):
-    def run(self):
-        if self.args.name is None and not self.args.unset:
-            conf = self.config.read(self.args.level)
-            try:
-                ui.write(conf["core"]["machine"])
-            except KeyError:
-                ui.write("No default machine set")
-                return 1
-        else:
-            with self.config.edit(self.args.level) as conf:
-                if self.args.unset:
-                    conf["core"].pop("machine", None)
-                else:
-                    merged_conf = self.config.load_config_to_level(self.args.level)
-                    if (
-                        self.args.name in conf["machine"]
-                        or self.args.name in merged_conf["machine"]
-                    ):
-                        conf["core"]["machine"] = self.args.name
-                    else:
-                        raise ConfigError(
-                            "default machine must be present in machine list."
-                        )
-        return 0
-
-
-class CmdMachineCreate(CmdBase):
-    def run(self):
-        if self.repo.machine is None:
-            raise MachineDisabledError
-
-        self.repo.machine.create(self.args.name)
-        return 0
-
-
-class CmdMachineStatus(CmdBase):
-    INSTANCE_FIELD = ["name", "instance", "status"]
-    SHOWN_FIELD = [
-        "cloud",
-        "instance_ip",
-        "instance_type",
-        "instance_hdd_size",
-        "instance_gpu",
-    ]
-    FILL_VALUE = "-"
-
-    def _add_row(
-        self,
-        name: str,
-        all_status: List[Dict],
-        td: TabularData,
-    ):
-        if not all_status:
-            row = [
-                name,
-                self.FILL_VALUE,
-                "offline",
-            ]  # back to `None` after #7167
-            td.append(row)
-        for i, status in enumerate(all_status, start=1):
-            row = [name, f"num_{i}", "running" if status else "offline"]
-            for field in self.SHOWN_FIELD:
-                value = str(status.get(field, ""))
-                row.append(value)
-            td.append(row)
-
-    def run(self):
-        if self.repo.machine is None:
-            raise MachineDisabledError
-
-        td = TabularData(
-            self.INSTANCE_FIELD + self.SHOWN_FIELD, fill_value=self.FILL_VALUE
-        )
-
-        if self.args.name:
-            all_status = list(self.repo.machine.status(self.args.name))
-            self._add_row(self.args.name, all_status, td)
-        else:
-            name_set = set()
-            for level in self.repo.config.LEVELS:
-                conf = self.repo.config.read(level)["machine"]
-                name_set.update(conf.keys())
-            name_list = list(name_set)
-            for name in sorted(name_list):
-                all_status = list(self.repo.machine.status(name))
-                self._add_row(name, all_status, td)
-
-        td.dropna("cols", "all")
-        td.render()
-        return 0
-
-
-class CmdMachineDestroy(CmdBase):
-    def run(self):
-        if self.repo.machine is None:
-            raise MachineDisabledError
-
-        self.repo.machine.destroy(self.args.name)
-        return 0
-
-
-class CmdMachineSsh(CmdBase):
-    def run(self):
-        if self.repo.machine is None:
-            raise MachineDisabledError
-
-        self.repo.machine.run_shell(self.args.name)
-        return 0
-
-
-def add_parser(subparsers, parent_parser):  # noqa: PLR0915
-    from dvc.commands.config import parent_config_parser
-
-    machine_HELP = "Set up and manage cloud machines."
-    machine_parser = subparsers.add_parser(
-        "machine",
-        parents=[parent_parser],
-        description=append_doc_link(machine_HELP, "machine"),
-        # NOTE: suppress help during development to hide command
-        # help=machine_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-
-    machine_subparsers = machine_parser.add_subparsers(
-        dest="cmd",
-        help="Use `dvc machine CMD --help` for command-specific help.",
-    )
-
-    fix_subparsers(machine_subparsers)
-
-    machine_ADD_HELP = "Add a new data machine."
-    machine_add_parser = machine_subparsers.add_parser(
-        "add",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(machine_ADD_HELP, "machine/add"),
-        help=machine_ADD_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_add_parser.add_argument("name", help="Name of the machine")
-    machine_add_parser.add_argument(
-        "cloud",
-        help="Machine cloud. See full list of supported clouds at {}".format(
-            format_link(
-                "https://github.com/iterative/terraform-provider-iterative#machine"
-            )
-        ),
-    )
-    machine_add_parser.add_argument(
-        "-d",
-        "--default",
-        action="store_true",
-        default=False,
-        help="Set as default machine.",
-    )
-    machine_add_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Force overwriting existing configs",
-    )
-    machine_add_parser.set_defaults(func=CmdMachineAdd)
-
-    machine_DEFAULT_HELP = "Set/unset the default machine."
-    machine_default_parser = machine_subparsers.add_parser(
-        "default",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(machine_DEFAULT_HELP, "machine/default"),
-        help=machine_DEFAULT_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_default_parser.add_argument("name", nargs="?", help="Name of the machine")
-    machine_default_parser.add_argument(
-        "-u",
-        "--unset",
-        action="store_true",
-        default=False,
-        help="Unset default machine.",
-    )
-    machine_default_parser.set_defaults(func=CmdMachineDefault)
-
-    machine_LIST_HELP = "List the configuration of one/all machines."
-    machine_list_parser = machine_subparsers.add_parser(
-        "list",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(machine_LIST_HELP, "machine/list"),
-        help=machine_LIST_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_list_parser.add_argument(
-        "--show-origin",
-        default=False,
-        action="store_true",
-        help="Show the source file containing each config value.",
-    )
-    machine_list_parser.add_argument(
-        "name",
-        nargs="?",
-        type=str,
-        help="name of machine to specify",
-    )
-    machine_list_parser.set_defaults(func=CmdMachineList)
-    machine_MODIFY_HELP = "Modify the configuration of an machine."
-    machine_modify_parser = machine_subparsers.add_parser(
-        "modify",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(machine_MODIFY_HELP, "machine/modify"),
-        help=machine_MODIFY_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_modify_parser.add_argument("name", help="Name of the machine")
-    machine_modify_parser.add_argument("option", help="Name of the option to modify.")
-    machine_modify_parser.add_argument(
-        "value", nargs="?", help="(optional) Value of the option."
-    )
-    machine_modify_parser.add_argument(
-        "-u",
-        "--unset",
-        default=False,
-        action="store_true",
-        help="Unset option.",
-    )
-    machine_modify_parser.set_defaults(func=CmdMachineModify)
-
-    machine_RENAME_HELP = "Rename a machine "
-    machine_rename_parser = machine_subparsers.add_parser(
-        "rename",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(machine_RENAME_HELP, "remote/rename"),
-        help=machine_RENAME_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_rename_parser.add_argument("name", help="Machine to be renamed")
-    machine_rename_parser.add_argument("new", help="New name of the machine")
-    machine_rename_parser.set_defaults(func=CmdMachineRename)
-
-    machine_REMOVE_HELP = "Remove an machine."
-    machine_remove_parser = machine_subparsers.add_parser(
-        "remove",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(machine_REMOVE_HELP, "machine/remove"),
-        help=machine_REMOVE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_remove_parser.add_argument("name", help="Name of the machine to remove.")
-    machine_remove_parser.set_defaults(func=CmdMachineRemove)
-
-    machine_CREATE_HELP = "Create and start a machine instance."
-    machine_create_parser = machine_subparsers.add_parser(
-        "create",
-        parents=[parent_parser],
-        description=append_doc_link(machine_CREATE_HELP, "machine/create"),
-        help=machine_CREATE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_create_parser.add_argument("name", help="Name of the machine to create.")
-    machine_create_parser.set_defaults(func=CmdMachineCreate)
-
-    machine_STATUS_HELP = "List the status of running instances for one/all machines."
-    machine_status_parser = machine_subparsers.add_parser(
-        "status",
-        parents=[parent_parser],
-        description=append_doc_link(machine_STATUS_HELP, "machine/status"),
-        help=machine_STATUS_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_status_parser.add_argument(
-        "name", nargs="?", help="(optional) Name of the machine."
-    )
-    machine_status_parser.set_defaults(func=CmdMachineStatus)
-
-    machine_DESTROY_HELP = "Destroy an machine instance."
-    machine_destroy_parser = machine_subparsers.add_parser(
-        "destroy",
-        parents=[parent_parser],
-        description=append_doc_link(machine_DESTROY_HELP, "machine/destroy"),
-        help=machine_DESTROY_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_destroy_parser.add_argument(
-        "name", help="Name of the machine instance to destroy."
-    )
-    machine_destroy_parser.set_defaults(func=CmdMachineDestroy)
-
-    machine_SSH_HELP = "Connect to a machine via SSH."
-    machine_ssh_parser = machine_subparsers.add_parser(
-        "ssh",
-        parents=[parent_parser],
-        description=append_doc_link(machine_SSH_HELP, "machine/ssh"),
-        help=machine_SSH_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_ssh_parser.add_argument(
-        "name", help="Name of the machine instance to connect to."
-    )
-    machine_ssh_parser.set_defaults(func=CmdMachineSsh)

dvc/commands/move.py

@@ -30,6 +30,7 @@ def add_parser(subparsers, parent_parser):
 
     move_parser = subparsers.add_parser(
         "move",
+        aliases=["mv"],
         parents=[parent_parser],
         description=append_doc_link(MOVE_DESCRIPTION, "move"),
         help=MOVE_HELP,

dvc/commands/remote.py

@@ -30,8 +30,8 @@ class CmdRemoteAdd(CmdRemote):
         with self.config.edit(self.args.level) as conf:
             if self.args.name in conf["remote"] and not self.args.force:
                 raise ConfigError(
-                    "remote '{}' already exists. Use `-f|--force` to "
-                    "overwrite it.".format(self.args.name)
+                    f"remote '{self.args.name}' already exists. Use `-f|--force` to "
+                    "overwrite it."
                 )
 
             conf["remote"][self.args.name] = {"url": self.args.url}

dvc/commands/remove.py

@@ -26,6 +26,7 @@ def add_parser(subparsers, parent_parser):
     )
     remove_parser = subparsers.add_parser(
         "remove",
+        aliases=["rm"],
         parents=[parent_parser],
         description=append_doc_link(REMOVE_HELP, "remove"),
         help=REMOVE_HELP,

dvc/config.py

@@ -90,6 +90,7 @@ class Config(dict):
     def __init__(
         self,
         dvc_dir: Optional["StrPath"] = None,
+        local_dvc_dir: Optional["StrPath"] = None,
         validate: bool = True,
         fs: Optional["FileSystem"] = None,
         config: Optional["DictStrAny"] = None,
@@ -98,6 +99,7 @@ class Config(dict):
     ):  # pylint: disable=super-init-not-called
         from dvc.fs import LocalFileSystem
 
+        dvc_dir = os.fspath(dvc_dir) if dvc_dir else None
         self.dvc_dir = dvc_dir
         self.wfs = LocalFileSystem()
         self.fs = fs or self.wfs
@@ -105,6 +107,10 @@ class Config(dict):
         if dvc_dir:
             self.dvc_dir = self.fs.path.abspath(dvc_dir)
 
+        self.local_dvc_dir = local_dvc_dir
+        if not fs and not local_dvc_dir:
+            self.local_dvc_dir = dvc_dir
+
         self.load(
             validate=validate, config=config, remote=remote, remote_config=remote_config
         )
@@ -140,7 +146,9 @@ class Config(dict):
 
         if self.dvc_dir is not None:
             files["repo"] = self.fs.path.join(self.dvc_dir, self.CONFIG)
-            files["local"] = self.fs.path.join(self.dvc_dir, self.CONFIG_LOCAL)
+
+        if self.local_dvc_dir is not None:
+            files["local"] = self.wfs.path.join(self.local_dvc_dir, self.CONFIG_LOCAL)
 
         return files
 
@@ -158,6 +166,9 @@ class Config(dict):
         with open(config_file, "w+", encoding="utf-8"):
             return Config(dvc_dir)
 
+    def merge(self, config):
+        merge(self, config)
+
     def load(
         self,
         validate: bool = True,
@@ -250,21 +261,30 @@ class Config(dict):
         return conf
 
     @staticmethod
-    def _load_paths(conf, filename):
-        abs_conf_dir = os.path.abspath(os.path.dirname(filename))
+    def _resolve(conf_dir, path):
+        from .config_schema import ExpPath, RelPath
 
-        def resolve(path):
-            from .config_schema import RelPath
+        if re.match(r"\w+://", path):
+            return path
 
-            if os.path.isabs(path) or re.match(r"\w+://", path):
-                return path
+        if os.path.isabs(path):
+            return path
 
-            # on windows convert slashes to backslashes
-            # to have path compatible with abs_conf_dir
-            if os.path.sep == "\\" and "/" in path:
-                path = path.replace("/", "\\")
+        # on windows convert slashes to backslashes
+        # to have path compatible with abs_conf_dir
+        if os.path.sep == "\\" and "/" in path:
+            path = path.replace("/", "\\")
 
-            return RelPath(os.path.join(abs_conf_dir, path))
+        expanded = os.path.expanduser(path)
+        if os.path.isabs(expanded):
+            return ExpPath(expanded, path)
+
+        return RelPath(os.path.abspath(os.path.join(conf_dir, path)))
+
+    @classmethod
+    def _load_paths(cls, conf, filename):
+        conf_dir = os.path.abspath(os.path.dirname(filename))
+        resolve = partial(cls._resolve, conf_dir)
 
         return Config._map_dirs(conf, resolve)
 
@@ -273,11 +293,17 @@ class Config(dict):
         from dvc.fs import localfs
         from dvc.utils import relpath
 
-        from .config_schema import RelPath
+        from .config_schema import ExpPath, RelPath
 
         if re.match(r"\w+://", path):
             return path
 
+        if isinstance(path, ExpPath):
+            return path.def_path
+
+        if os.path.expanduser(path) != path:
+            return localfs.path.as_posix(path)
+
         if isinstance(path, RelPath) or not os.path.isabs(path):
             path = relpath(path, conf_dir)
             return localfs.path.as_posix(path)

dvc/config_schema.py

@@ -75,6 +75,17 @@ def ByUrl(mapping):  # noqa: N802
     return validate
 
 
+class ExpPath(str):
+    __slots__ = ("def_path",)
+
+    def_path: str
+
+    def __new__(cls, string, def_path):
+        ret = super().__new__(cls, string)
+        ret.def_path = def_path
+        return ret
+
+
 class RelPath(str):
     __slots__ = ()
 

dvc/dependency/__init__.py

@@ -15,6 +15,7 @@ SCHEMA: Mapping[str, Any] = {
     **ARTIFACT_SCHEMA,
     **RepoDependency.REPO_SCHEMA,
     Output.PARAM_FILES: [DIR_FILES_SCHEMA],
+    Output.PARAM_FS_CONFIG: dict,
 }
 
 
@@ -36,7 +37,10 @@ def loadd_from(stage, d_list):
         p = d.pop(Output.PARAM_PATH, None)
         files = d.pop(Output.PARAM_FILES, None)
         hash_name = d.pop(Output.PARAM_HASH, None)
-        ret.append(_get(stage, p, d, files=files, hash_name=hash_name))
+        fs_config = d.pop(Output.PARAM_FS_CONFIG, None)
+        ret.append(
+            _get(stage, p, d, files=files, hash_name=hash_name, fs_config=fs_config)
+        )
     return ret
 
 

dvc/dependency/base.py

@@ -44,9 +44,6 @@ class Dependency(Output):
         if self.fs.version_aware:
             self.fs_path = self.fs.path.version_path(self.fs_path, rev)
             self.meta = self.get_meta()
-            self.def_path = self.fs.path.version_path(
-                self.def_path, self.meta.version_id
-            )
             self.fs_path = self.fs.path.version_path(self.fs_path, self.meta.version_id)
 
     def download(self, to, jobs=None):

dvc/dependency/repo.py

@@ -1,3 +1,4 @@
+from copy import deepcopy
 from typing import TYPE_CHECKING, Dict, Optional, Union
 
 from voluptuous import Any, Required
@@ -24,12 +25,12 @@ class RepoDependency(Dependency):
             Required(PARAM_URL): str,
             PARAM_REV: str,
             PARAM_REV_LOCK: str,
-            PARAM_CONFIG: str,
+            PARAM_CONFIG: Any(str, dict),
             PARAM_REMOTE: Any(str, dict),
         }
     }
 
-    def __init__(self, def_repo: Dict[str, str], stage: "Stage", *args, **kwargs):
+    def __init__(self, def_repo: Dict[str, Any], stage: "Stage", *args, **kwargs):
         self.def_repo = def_repo
         super().__init__(stage, *args, **kwargs)
 
@@ -107,17 +108,17 @@ class RepoDependency(Dependency):
 
         rem = self.def_repo.get("remote")
         if isinstance(rem, dict):
-            remote = None  # type: ignore[unreachable]
+            remote = None
             remote_config = rem
         else:
             remote = rem
             remote_config = None
 
-        conf = self.def_repo.get("config")
-        if conf:
-            config = Config.load_file(conf)
+        conf = self.def_repo.get("config", {})
+        if isinstance(conf, dict):
+            config = deepcopy(conf)
         else:
-            config = {}
+            config = Config.load_file(conf)
 
         config["cache"] = self.repo.config["cache"]
         config["cache"]["dir"] = self.repo.cache.local_cache_dir

dvc/exceptions.py

@@ -1,9 +1,12 @@
 """Exceptions raised by the dvc."""
 import errno
-from typing import Dict, List
+from typing import TYPE_CHECKING, Dict, List, Set
 
 from dvc.utils import format_link
 
+if TYPE_CHECKING:
+    from dvc.stage import Stage
+
 
 class DvcException(Exception):
     """Base class for all dvc exceptions."""
@@ -31,24 +34,25 @@ class OutputDuplicationError(DvcException):
         stages (list): list of paths to stages.
     """
 
-    def __init__(self, output, stages):
+    def __init__(self, output: str, stages: Set["Stage"]):
         from funcy import first
 
         assert isinstance(output, str)
         assert all(hasattr(stage, "relpath") for stage in stages)
-        msg = ""
-        stage_names = [s.addressing for s in stages]
-        stages_str = " ".join(stage_names)
         if len(stages) == 1:
-            stage_name = first(stages)
-            msg = f"output '{output}' is already specified in {stage_name}."
+            stage = first(stages)
+            msg = (
+                f"output '{output}' is already specified in {stage}."
+                f"\nUse `dvc remove {stage.addressing}` to stop tracking the "
+                "overlapping output."
+            )
         else:
-            msg = "output '{}' is already specified in stages:\n{}".format(
-                output, "\n".join(f"\t- {s}" for s in stage_names)
+            stage_names = "\n".join(["\t- " + s.addressing for s in stages])
+            msg = (
+                f"output '{output}' is specified in:\n{stage_names}"
+                "\nUse `dvc remove` with any of the above targets to stop tracking the "
+                "overlapping output."
             )
-        msg += (
-            f"\nUse `dvc remove {stages_str}` to stop tracking the overlapping output."
-        )
         super().__init__(msg)
         self.stages = stages
         self.output = output
@@ -196,7 +200,7 @@ class ETagMismatchError(DvcException):
     def __init__(self, etag, cached_etag):
         super().__init__(
             "ETag mismatch detected when copying file to cache! "
-            "(expected: '{}', actual: '{}')".format(etag, cached_etag)
+            f"(expected: '{etag}', actual: '{cached_etag}')"
         )
 
 

dvc/fs/dvc.py

@@ -139,7 +139,7 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
             self.repo_factory = repo_factory
 
         def _getcwd():
-            relparts = ()
+            relparts: Tuple[str, ...] = ()
             assert repo is not None
             if repo.fs.path.isin(repo.fs.path.getcwd(), repo.root_dir):
                 relparts = repo.fs.path.relparts(repo.fs.path.getcwd(), repo.root_dir)
@@ -162,7 +162,17 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         if hasattr(repo, "dvc_dir"):
             self._datafss[key] = DataFileSystem(index=repo.index.data["repo"])
 
+    @functools.cached_property
+    def fsid(self) -> str:
+        from fsspec.utils import tokenize
+
+        return "dvcfs_" + tokenize(
+            self.repo.url or self.repo.root_dir,
+            self.repo.get_rev() if self.repo.scm else None,
+        )
+
     def _get_key(self, path: "StrPath") -> Key:
+        path = os.fspath(path)
         parts = self.repo.fs.path.relparts(path, self.repo.root_dir)
         if parts == (os.curdir,):
             return ()
@@ -261,15 +271,12 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         try:
             return self.repo.fs.open(fs_path, mode=mode)
         except FileNotFoundError:
-            repo, dvc_fs, subkey = self._get_subrepo_info(key)
+            _, dvc_fs, subkey = self._get_subrepo_info(key)
             if not dvc_fs:
                 raise
 
         dvc_path = _get_dvc_path(dvc_fs, subkey)
-        kw = {}
-        if kwargs.get("cache_remote_stream", False):
-            kw["cache_odb"] = repo.cache.local
-        return dvc_fs.open(dvc_path, mode=mode, **kw)
+        return dvc_fs.open(dvc_path, mode=mode, cache=kwargs.get("cache", False))
 
     def isdvc(self, path, **kwargs) -> bool:
         """Is this entry dvc-tracked?"""
@@ -415,6 +422,10 @@ class DVCFileSystem(FileSystem):
     def fs(self) -> "_DVCFileSystem":
         return _DVCFileSystem(**self.fs_args)
 
+    @property
+    def fsid(self) -> str:
+        return self.fs.fsid
+
     def isdvc(self, path, **kwargs) -> bool:
         return self.fs.isdvc(path, **kwargs)
 

dvc/ignore.py

@@ -148,7 +148,9 @@ class DvcIgnorePatterns(DvcIgnore):
         return bool(self.pattern_list)
 
 
-CheckIgnoreResult = namedtuple("CheckIgnoreResult", ["file", "match", "patterns"])
+CheckIgnoreResult = namedtuple(  # noqa: PYI024
+    "CheckIgnoreResult", ["file", "match", "patterns"]
+)
 
 
 def _no_match(path):

dvc/machine/__init__.py

@@ -1,227 +0,0 @@
-import logging
-import os
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Dict,
-    Iterable,
-    Iterator,
-    Mapping,
-    Optional,
-    Tuple,
-    Type,
-)
-
-from dvc.exceptions import DvcException
-
-if TYPE_CHECKING:
-    from dvc.repo import Repo
-    from dvc.types import StrPath
-
-    from .backend.base import BaseMachineBackend
-
-    BackendCls = Type[BaseMachineBackend]
-
-logger = logging.getLogger(__name__)
-
-
-RESERVED_NAMES = {"local", "localhost"}
-
-DEFAULT_STARTUP_SCRIPT = """#!/bin/bash
-sudo apt-get update
-sudo apt-get install --yes python3 python3-pip
-python3 -m pip install --upgrade pip
-
-pushd /etc/apt/sources.list.d
-sudo wget https://dvc.org/deb/dvc.list
-sudo apt-get update
-sudo apt-get install --yes dvc
-popd
-
-sudo echo "OK" > /var/log/dvc-machine-init.log
-"""
-
-
-def validate_name(name: str):
-    from dvc.exceptions import InvalidArgumentError
-
-    name = name.lower()
-    if name in RESERVED_NAMES:
-        raise InvalidArgumentError(
-            f"Machine name '{name}' is reserved for internal DVC use."
-        )
-
-
-class MachineBackends(Mapping):
-    try:
-        from .backend.terraform import TerraformBackend
-    except ImportError:
-        TerraformBackend = None  # type: ignore[assignment, misc]
-
-    DEFAULT: Dict[str, Optional["BackendCls"]] = {
-        "terraform": TerraformBackend,
-    }
-
-    def __getitem__(self, key: str) -> "BaseMachineBackend":
-        """Lazily initialize backends and cache it afterwards"""
-        initialized = self.initialized.get(key)
-        if not initialized:
-            backend = self.backends[key]
-            initialized = backend(os.path.join(self.tmp_dir, key), **self.kwargs)
-            self.initialized[key] = initialized
-        return initialized
-
-    def __init__(
-        self,
-        selected: Optional[Iterable[str]],
-        tmp_dir: "StrPath",
-        **kwargs,
-    ) -> None:
-        selected = selected or list(self.DEFAULT)
-        self.backends: Dict[str, "BackendCls"] = {}
-        for key in selected:
-            cls = self.DEFAULT.get(key)
-            if cls is None:
-                raise DvcException(
-                    f"'dvc machine' backend '{key}' is missing required "
-                    "dependencies. Install them with:\n"
-                    f"\tpip install dvc[{key}]"
-                )
-            self.backends[key] = cls
-
-        self.initialized: Dict[str, "BaseMachineBackend"] = {}
-
-        self.tmp_dir = tmp_dir
-        self.kwargs = kwargs
-
-    def __iter__(self):
-        return iter(self.backends)
-
-    def __len__(self) -> int:
-        return len(self.backends)
-
-    def close_initialized(self) -> None:
-        for backend in self.initialized.values():
-            backend.close()
-
-
-class MachineManager:
-    """Class that manages dvc cloud machines.
-
-    Args:
-        repo (dvc.repo.Repo): repo instance that belongs to the repo that
-            we are working on.
-
-    Raises:
-        config.ConfigError: thrown when config has invalid format.
-    """
-
-    CLOUD_BACKENDS = {
-        "aws": "terraform",
-        "azure": "terraform",
-    }
-
-    def __init__(
-        self, repo: "Repo", backends: Optional[Iterable[str]] = None, **kwargs
-    ):
-        self.repo = repo
-        assert self.repo.tmp_dir
-        tmp_dir = os.path.join(self.repo.tmp_dir, "machine")
-        self.backends = MachineBackends(backends, tmp_dir=tmp_dir, **kwargs)
-
-    def get_config_and_backend(
-        self,
-        name: Optional[str] = None,
-    ) -> Tuple[dict, "BaseMachineBackend"]:
-        from dvc.config import NoMachineError
-
-        if not name:
-            name = self.repo.config["core"].get("machine")
-
-        if name:
-            config = self._get_config(name=name)
-            backend = self._get_backend(config["cloud"])
-            return config, backend
-
-        if bool(self.repo.config["machine"]):
-            error_msg = (
-                "no machine specified. Setup default machine with\n"
-                "    dvc machine default <name>\n"
-            )
-        else:
-            error_msg = (
-                "no machine specified. Create a default machine with\n"
-                "    dvc machine add -d <name> <cloud>"
-            )
-
-        raise NoMachineError(error_msg)
-
-    def _get_config(self, **kwargs):
-        config = self.repo.config
-        name = kwargs.get("name")
-        if name:
-            try:
-                conf = config["machine"][name.lower()]
-                conf["name"] = name
-            except KeyError:
-                from dvc.config import MachineNotFoundError
-
-                raise MachineNotFoundError(  # noqa: B904
-                    f"machine '{name}' doesn't exist"
-                )
-        else:
-            conf = kwargs
-        return conf
-
-    def _get_backend(self, cloud: str) -> "BaseMachineBackend":
-        from dvc.config import NoMachineError
-
-        try:
-            backend = self.CLOUD_BACKENDS[cloud]
-            return self.backends[backend]
-        except KeyError:
-            raise NoMachineError(  # noqa: B904
-                f"Machine platform '{cloud}' unsupported"
-            )
-
-    def create(self, name: Optional[str]):
-        """Create and start the specified machine instance."""
-        config, backend = self.get_config_and_backend(name)
-        if "startup_script" in config:
-            with open(config["startup_script"], encoding="utf-8") as fobj:
-                startup_script = fobj.read()
-        else:
-            startup_script = DEFAULT_STARTUP_SCRIPT
-        config["startup_script"] = startup_script
-        config.pop("setup_script", None)
-        return backend.create(**config)
-
-    def destroy(self, name: Optional[str]):
-        """Destroy the specified machine instance."""
-        config, backend = self.get_config_and_backend(name)
-        return backend.destroy(**config)
-
-    def get_sshfs(self, name: Optional[str]):
-        config, backend = self.get_config_and_backend(name)
-        return backend.get_sshfs(**config)
-
-    def run_shell(self, name: Optional[str]):
-        config, backend = self.get_config_and_backend(name)
-        return backend.run_shell(**config)
-
-    def status(self, name: str) -> Iterator[Dict[Any, Any]]:
-        config, backend = self.get_config_and_backend(name)
-        return backend.instances(**config)
-
-    def rename(self, name: str, new: str):
-        """move configuration to a new location if the machine is running."""
-        config, backend = self.get_config_and_backend(name)
-        return backend.rename(new=new, **config)
-
-    def get_executor_kwargs(self, name: Optional[str]):
-        config, backend = self.get_config_and_backend(name)
-        return backend.get_executor_kwargs(**config)
-
-    def get_setup_script(self, name: Optional[str]) -> Optional[str]:
-        config, _backend = self.get_config_and_backend(name)
-        return config.get("setup_script")

dvc/machine/backend/base.py

@@ -1,50 +0,0 @@
-from abc import ABC, abstractmethod
-from contextlib import contextmanager
-from typing import TYPE_CHECKING, Iterator, Optional
-
-if TYPE_CHECKING:
-    from dvc_ssh import SSHFileSystem
-
-    from dvc.types import StrPath
-
-
-class BaseMachineBackend(ABC):
-    def __init__(self, tmp_dir: "StrPath", **kwargs):
-        self.tmp_dir = tmp_dir
-
-    @abstractmethod
-    def create(self, name: Optional[str] = None, **config):
-        """Create and start an instance of the specified machine."""
-
-    @abstractmethod
-    def destroy(self, name: Optional[str] = None, **config):
-        """Stop and destroy all instances of the specified machine."""
-
-    @abstractmethod
-    def instances(self, name: Optional[str] = None, **config) -> Iterator[dict]:
-        """Iterate over status of all instances of the specified machine."""
-
-    def close(self):  # noqa: B027
-        pass
-
-    @abstractmethod
-    def run_shell(self, name: Optional[str] = None, **config):
-        """Spawn an interactive SSH shell for the specified machine."""
-
-    @abstractmethod
-    def get_executor_kwargs(self, name: str, **config) -> dict:
-        """Return SSHExecutor kwargs which can be used for DVC
-        experiment/pipeline execution on the specified machine.
-        """
-
-    @abstractmethod
-    @contextmanager
-    def get_sshfs(
-        self, name: Optional[str] = None, **config
-    ) -> Iterator["SSHFileSystem"]:
-        """Return an sshfs instance for the default directory on the
-        specified machine."""
-
-    @abstractmethod
-    def rename(self, name: str, new: str, **config):
-        """Rename a machine instance."""

dvc/machine/backend/terraform.py

@@ -1,99 +0,0 @@
-import os
-from contextlib import contextmanager
-from functools import partial, partialmethod
-from typing import TYPE_CHECKING, Iterator, Optional
-
-from dvc_ssh import DEFAULT_PORT, SSHFileSystem
-
-from dvc.exceptions import DvcException
-
-from .base import BaseMachineBackend
-
-if TYPE_CHECKING:
-    from dvc.types import StrPath
-
-
-@contextmanager
-def _sshfs(resource: dict):
-    from tpi import TerraformProviderIterative
-
-    with TerraformProviderIterative.pemfile(resource) as pem:
-        fs = SSHFileSystem(
-            host=resource["instance_ip"],
-            user="ubuntu",
-            keyfile=pem,
-        )
-        yield fs
-
-
-class TerraformBackend(BaseMachineBackend):
-    def __init__(self, tmp_dir: "StrPath", **kwargs):
-        super().__init__(tmp_dir, **kwargs)
-        os.makedirs(tmp_dir, exist_ok=True)
-
-    @contextmanager
-    def make_tpi(self, name: str):
-        from tpi import TPIError
-        from tpi.terraform import TerraformBackend as TPIBackend
-
-        try:
-            working_dir = os.path.join(self.tmp_dir, name)
-            os.makedirs(working_dir, exist_ok=True)
-            yield TPIBackend(working_dir=working_dir)
-        except TPIError as exc:
-            raise DvcException("TPI operation failed") from exc
-
-    def _tpi_func(self, fname, name: Optional[str] = None, **config):
-        from tpi import TPIError
-
-        assert name
-        with self.make_tpi(name) as tpi:
-            func = getattr(tpi, fname)
-            try:
-                return func(name=name, **config)
-            except TPIError as exc:
-                raise DvcException(f"TPI {fname} failed") from exc
-
-    create = partialmethod(_tpi_func, "create")  # type: ignore[assignment]
-    destroy = partialmethod(_tpi_func, "destroy")  # type: ignore[assignment]
-    instances = partialmethod(_tpi_func, "instances")  # type: ignore[assignment]
-    run_shell = partialmethod(_tpi_func, "run_shell")  # type: ignore[assignment]
-
-    def get_executor_kwargs(self, name: str, **config) -> dict:
-        with self.make_tpi(name) as tpi:
-            resource = tpi.default_resource(name)
-        return {
-            "host": resource["instance_ip"],
-            "port": DEFAULT_PORT,
-            "username": "ubuntu",
-            "fs_factory": partial(_sshfs, dict(resource)),
-        }
-
-    @contextmanager
-    def get_sshfs(  # pylint: disable=unused-argument
-        self, name: Optional[str] = None, **config
-    ) -> Iterator["SSHFileSystem"]:
-        assert name
-        with self.make_tpi(name) as tpi:
-            resource = tpi.default_resource(name)
-        with _sshfs(resource) as fs:
-            yield fs
-
-    def rename(self, name: str, new: str, **config):
-        """rename a dvc machine instance to a new name"""
-        import shutil
-
-        mtype = "iterative_machine"
-
-        new_dir = os.path.join(self.tmp_dir, new)
-        old_dir = os.path.join(self.tmp_dir, name)
-        if os.path.exists(new_dir):
-            raise DvcException(f"rename failed: path {new_dir} already exists")
-
-        if not os.path.exists(old_dir):
-            return
-
-        with self.make_tpi(name) as tpi:
-            tpi.state_mv(f"{mtype}.{name}", f"{mtype}.{new}", **config)
-
-        shutil.move(old_dir, new_dir)

dvc/output.py

@@ -97,6 +97,7 @@ def loadd_from(stage, d_list):
         files = d.pop(Output.PARAM_FILES, None)
         push = d.pop(Output.PARAM_PUSH, True)
         hash_name = d.pop(Output.PARAM_HASH, None)
+        fs_config = d.pop(Output.PARAM_FS_CONFIG, None)
         ret.append(
             _get(
                 stage,
@@ -111,6 +112,7 @@ def loadd_from(stage, d_list):
                 files=files,
                 push=push,
                 hash_name=hash_name,
+                fs_config=fs_config,
             )
         )
     return ret
@@ -313,6 +315,7 @@ class Output:
     PARAM_PUSH = "push"
     PARAM_CLOUD = "cloud"
     PARAM_HASH = "hash"
+    PARAM_FS_CONFIG = "fs_config"
 
     DoesNotExistError: Type[DvcException] = OutputDoesNotExistError
     IsNotFileOrDirError: Type[DvcException] = OutputIsNotFileOrDirError
@@ -351,6 +354,7 @@ class Output:
         if meta.version_id or files:
             fs_kwargs["version_aware"] = True
 
+        self.def_fs_config = fs_config
         if fs_config is not None:
             fs_kwargs.update(**fs_config)
 
@@ -872,6 +876,9 @@ class Output:
 
         ret[self.PARAM_PATH] = path
 
+        if self.def_fs_config:
+            ret[self.PARAM_FS_CONFIG] = self.def_fs_config
+
         if not self.IS_DEPENDENCY:
             ret.update(self.annot.to_dict())
             if not self.use_cache:
@@ -1090,7 +1097,7 @@ class Output:
         return len(obj) if obj else 0
 
     def unprotect(self):
-        if self.exists:
+        if self.exists and self.use_cache:
             self.cache.unprotect(self.fs_path)
 
     def get_dir_cache(self, **kwargs) -> Optional["Tree"]:
@@ -1142,7 +1149,7 @@ class Output:
             if not force and not prompt.confirm(msg.format(self.fs_path)):
                 raise CollectCacheError(  # noqa: B904
                     "unable to fully collect used cache"
-                    " without cache for directory '{}'".format(self)
+                    f" without cache for directory '{self}'"
                 )
             return None
 
@@ -1171,19 +1178,15 @@ class Output:
 
         if not self.hash_info:
             msg = (
-                "Output '{}'({}) is missing version info. "
+                f"Output '{self}'({self.stage}) is missing version info. "
                 "Cache for it will not be collected. "
-                "Use `dvc repro` to get your pipeline up to date.".format(
-                    self, self.stage
-                )
+                "Use `dvc repro` to get your pipeline up to date."
             )
             if self.exists:
                 msg += (
                     "\n"
-                    "You can also use `dvc commit {stage.addressing}` "
-                    "to associate existing '{out}' with {stage}.".format(
-                        out=self, stage=self.stage
-                    )
+                    f"You can also use `dvc commit {self.stage.addressing}` "
+                    f"to associate existing '{self}' with {self.stage}."
                 )
             logger.warning(msg)
             return {}
@@ -1541,4 +1544,5 @@ SCHEMA = {
     Output.PARAM_REMOTE: str,
     Output.PARAM_PUSH: bool,
     Output.PARAM_FILES: [DIR_FILES_SCHEMA],
+    Output.PARAM_FS_CONFIG: dict,
 }

dvc/pathspec_math.py

@@ -8,7 +8,7 @@ from pathspec.util import normalize_file
 
 from dvc.utils import relpath
 
-PatternInfo = namedtuple("PatternInfo", ["patterns", "file_info"])
+PatternInfo = namedtuple("PatternInfo", ["patterns", "file_info"])  # noqa: PYI024
 
 
 def _not_ignore(rule):

dvc/repo/__init__.py

@@ -15,8 +15,6 @@ from typing import (
 
 from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
 from dvc.ignore import DvcIgnoreFilter
-from dvc.utils import env2bool
-from dvc.utils.fs import path_isin
 from dvc.utils.objects import cached_property
 
 if TYPE_CHECKING:
@@ -24,7 +22,6 @@ if TYPE_CHECKING:
     from dvc.fs.data import DataFileSystem
     from dvc.fs.dvc import DVCFileSystem
     from dvc.lock import LockBase
-    from dvc.machine import MachineManager
     from dvc.scm import Git, NoSCM
     from dvc.stage import Stage
     from dvc.types import DictStrAny
@@ -244,6 +241,7 @@ class Repo:
 
         return Config(
             self.dvc_dir,
+            local_dvc_dir=self.local_dvc_dir,
             fs=self.fs,
             config=self._config,
             remote=self._remote,
@@ -351,16 +349,6 @@ class Repo:
 
         return Experiments(self)
 
-    @cached_property
-    def machine(self) -> Optional["MachineManager"]:
-        from dvc.machine import MachineManager
-
-        if self.tmp_dir and (
-            self.config["feature"].get("machine", False) or env2bool("DVC_TEST")
-        ):
-            return MachineManager(self)
-        return None
-
     @property
     def fs(self) -> "FileSystem":
         return self._fs
@@ -466,8 +454,9 @@ class Repo:
         flist = [self.config.files["local"]]
         if tmp_dir := self.tmp_dir:
             flist.append(tmp_dir)
-        if path_isin(self.cache.legacy.path, self.root_dir):
-            flist.append(self.cache.legacy.path)
+
+        if cache_dir := self.cache.default_local_cache_dir:
+            flist.append(cache_dir)
 
         for file in flist:
             self.scm_context.ignore(file)
@@ -616,12 +605,19 @@ class Repo:
         finally:
             os.umask(umask)
 
+        # NOTE: Some number to change the generated token if none of the
+        # components were changed (useful to prevent newer dvc versions from
+        # using older broken cache). Please reset this back to 0 if other parts
+        # of the token components are changed.
+        salt = 0
+
         md5 = hashlib.md5(  # noqa: S324  # nosec B324, B303
             str(
                 (
                     root_dir,
                     getpass.getuser(),
                     version_tuple[0],
+                    salt,
                 )
             ).encode()
         )

dvc/repo/add.py

@@ -120,7 +120,7 @@ def translate_graph_error(stages: List["Stage"]) -> Iterator[None]:
         )
     except OutputDuplicationError as exc:
         raise OutputDuplicationError(  # noqa: B904
-            exc.output, list(set(exc.stages) - set(stages))
+            exc.output, set(exc.stages) - set(stages)
         )
 
 

dvc/repo/brancher.py

@@ -57,11 +57,11 @@ def brancher(  # noqa: E302
 
     from dvc.fs import LocalFileSystem
 
-    repo_root_parts = ()
+    repo_root_parts: Tuple[str, ...] = ()
     if self.fs.path.isin(self.root_dir, self.scm.root_dir):
         repo_root_parts = self.fs.path.relparts(self.root_dir, self.scm.root_dir)
 
-    cwd_parts = ()
+    cwd_parts: Tuple[str, ...] = ()
     if self.fs.path.isin(self.fs.path.getcwd(), self.scm.root_dir):
         cwd_parts = self.fs.path.relparts(self.fs.path.getcwd(), self.scm.root_dir)
 
@@ -144,11 +144,11 @@ def switch(repo: "Repo", rev: str) -> Iterator[str]:
     if rev != "workspace":
         rev = resolve_rev(repo.scm, rev)
 
-    repo_root_parts = ()
+    repo_root_parts: Tuple[str, ...] = ()
     if repo.fs.path.isin(repo.root_dir, repo.scm.root_dir):
         repo_root_parts = repo.fs.path.relparts(repo.root_dir, repo.scm.root_dir)
 
-    cwd_parts = ()
+    cwd_parts: Tuple[str, ...] = ()
     if repo.fs.path.isin(repo.fs.path.getcwd(), repo.scm.root_dir):
         cwd_parts = repo.fs.path.relparts(repo.fs.path.getcwd(), repo.scm.root_dir)
 

dvc/repo/cache.py

@@ -26,6 +26,14 @@ def check_missing(repo, rev=None, max_size=None, types=None):
             continue
 
         if not fs.exists(path):
-            ret.append(os.path.join(*entry.key))
+            typ = "directory" if (entry.meta and entry.meta.isdir) else "file"
+            ret.append(
+                (
+                    typ,
+                    entry.hash_info.name,
+                    entry.hash_info.value,
+                    os.path.join(*entry.key),
+                )
+            )
 
     return ret

dvc/repo/checkout.py

@@ -51,20 +51,16 @@ def _build_out_changes(index, changes):
         out_keys.append(key)
 
     out_changes = {}
-    for objects in changes.values():
-        for change in objects:
-            for out_key in out_keys:
-                if (
-                    len(out_key) > len(change.key)
-                    or change.key[: len(out_key)] != out_key
-                ):
-                    continue
+    for key, change in changes.items():
+        for out_key in out_keys:
+            if len(out_key) > len(key) or key[: len(out_key)] != out_key:
+                continue
 
-                if change.key == out_key:
-                    out_changes[out_key] = change.typ
-                elif not out_changes.get(out_key):
-                    out_changes[out_key] = MODIFY
-                break
+            if key == out_key:
+                out_changes[out_key] = change.typ
+            elif not out_changes.get(out_key):
+                out_changes[out_key] = MODIFY
+            break
 
     return out_changes
 
@@ -85,7 +81,7 @@ def _check_can_delete(
         if cache_fs.exists(cache_path):
             continue
 
-        entry_paths.append(fs.path.join(path, *entry.key))
+        entry_paths.append(fs.path.join(path, *(entry.key or ())))
 
     if not entry_paths:
         return

dvc/repo/experiments/__init__.py

@@ -136,10 +136,6 @@ class Experiments:
         **kwargs,
     ) -> "QueueEntry":
         """Queue a single experiment."""
-        if kwargs.pop("machine", None) is not None:
-            # TODO: decide how to handle queued remote execution
-            raise NotImplementedError
-
         return self.new(
             queue,
             **kwargs,
@@ -214,10 +210,6 @@ class Experiments:
             )
         else:
             ui.write("Experiment results have been applied to your workspace.")
-        ui.write(
-            "\nTo promote an experiment to a Git branch run:\n\n"
-            "\tdvc exp branch <exp> <branch>\n"
-        )
 
     def new(
         self,

dvc/repo/experiments/collect.py

@@ -23,6 +23,7 @@ from dvc.scm import Git, SCMError, iter_revs
 from .exceptions import InvalidExpRefError
 from .refs import EXEC_BRANCH, ExpRefInfo
 from .serialize import ExpRange, ExpState, SerializableError, SerializableExp
+from .utils import describe
 
 if TYPE_CHECKING:
     from dvc.repo import Repo
@@ -308,7 +309,9 @@ def collect(
     if sha_only:
         baseline_names: Dict[str, Optional[str]] = {}
     else:
-        baseline_names = _describe(repo.scm, baseline_revs, refs=cached_refs)
+        baseline_names = describe(
+            repo.scm, baseline_revs, refs=cached_refs, logger=logger
+        )
 
     workspace_data = collect_rev(repo, "workspace", **kwargs)
     result: List["ExpState"] = [workspace_data]
@@ -337,55 +340,6 @@ def collect(
     return result
 
 
-def _describe(
-    scm: "Git",
-    revs: Iterable[str],
-    refs: Optional[Iterable[str]] = None,
-) -> Dict[str, Optional[str]]:
-    """Describe revisions using a tag, branch.
-
-    The first matching name will be returned for each rev. Names are preferred in this
-    order:
-        - current branch (if rev matches HEAD and HEAD is a branch)
-        - tags
-        - branches
-
-    Returns:
-        Dict mapping revisions from revs to a name.
-    """
-
-    head_rev = scm.get_rev()
-    head_ref = scm.get_ref("HEAD", follow=False)
-    if head_ref and head_ref.startswith("refs/heads/"):
-        head_branch = head_ref[len("refs/heads/") :]
-    else:
-        head_branch = None
-
-    tags = {}
-    branches = {}
-    ref_it = iter(refs) if refs else scm.iter_refs()
-    for ref in ref_it:
-        is_tag = ref.startswith("refs/tags/")
-        is_branch = ref.startswith("refs/heads/")
-        if not (is_tag or is_branch):
-            continue
-        rev = scm.get_ref(ref)
-        if not rev:
-            logger.debug("unresolved ref %s", ref)
-            continue
-        if is_tag and rev not in tags:
-            tags[rev] = ref[len("refs/tags/") :]
-        if is_branch and rev not in branches:
-            branches[rev] = ref[len("refs/heads/") :]
-    names: Dict[str, Optional[str]] = {}
-    for rev in revs:
-        if rev == head_rev and head_branch:
-            names[rev] = head_branch
-        else:
-            names[rev] = tags.get(rev) or branches.get(rev)
-    return names
-
-
 def _sorted_ranges(exp_ranges: Iterable["ExpRange"]) -> List["ExpRange"]:
     """Return list of ExpRange sorted by (timestamp, rev)."""
 

dvc/repo/experiments/executor/base.py

@@ -129,7 +129,6 @@ class BaseExecutor(ABC):
 
     PACKED_ARGS_FILE = "repro.dat"
     WARN_UNTRACKED = False
-    QUIET = False
     INFOFILE_EXT = ".run"
     DEFAULT_LOCATION: str = "workspace"
 
@@ -255,15 +254,6 @@ class BaseExecutor(ABC):
             **kwargs,
         )
 
-    @classmethod
-    def _get_stage_files(cls, stages: List["Stage"]) -> List[str]:
-        from dvc.stage.utils import _get_stage_files
-
-        ret: List[str] = []
-        for stage in stages:
-            ret.extend(_get_stage_files(stage))
-        return ret
-
     @classmethod
     def _get_top_level_paths(cls, repo: "Repo") -> List["str"]:
         return list(
@@ -519,10 +509,8 @@ class BaseExecutor(ABC):
                     recursive=kwargs.get("recursive", False),
                 )
 
+            kwargs["repro_fn"] = cls._repro_and_track
             stages = dvc.reproduce(*args, **kwargs)
-            if paths := cls._get_stage_files(stages):
-                logger.debug("Staging stage-related files: %s", paths)
-                dvc.scm_context.add(paths)
             if paths := cls._get_top_level_paths(dvc):
                 logger.debug("Staging top-level files: %s", paths)
                 dvc.scm_context.add(paths)
@@ -547,6 +535,17 @@ class BaseExecutor(ABC):
         # multiprocessing calls
         return ExecutorResult(exp_hash, exp_ref, repro_force)
 
+    @staticmethod
+    def _repro_and_track(stage: "Stage", **kwargs) -> Optional["Stage"]:
+        from dvc.repo.reproduce import _reproduce_stage
+        from dvc.stage.utils import _get_stage_files
+
+        ret = _reproduce_stage(stage, **kwargs)
+        if not kwargs.get("dry") and (paths := _get_stage_files(stage)):
+            logger.debug("Staging stage-related files: %s", paths)
+            stage.repo.scm_context.add(paths)
+        return ret
+
     @classmethod
     def _repro_commit(
         cls,
@@ -602,8 +601,7 @@ class BaseExecutor(ABC):
             info.status = TaskStatus.RUNNING
             if infofile is not None:
                 info.dump_json(infofile)
-            if cls.QUIET:
-                dvc.scm_context.quiet = cls.QUIET
+            dvc.scm_context.quiet = True
             old_cwd = os.getcwd()
 
             for path in copy_paths or []:

dvc/repo/experiments/executor/local.py

@@ -65,7 +65,6 @@ class TempDirExecutor(BaseLocalExecutor):
     # debugging user code), and suppress other DVC hints (like `git add`
     # suggestions) that are not applicable outside of workspace runs
     WARN_UNTRACKED = True
-    QUIET = True
     DEFAULT_LOCATION = "tempdir"
 
     @retry(180, errors=LockError, timeout=1)

dvc/repo/experiments/executor/ssh.py

@@ -1,292 +0,0 @@
-import logging
-import os
-import posixpath
-import sys
-from contextlib import contextmanager
-from typing import TYPE_CHECKING, Callable, Iterable, List, Optional
-
-from dvc_ssh import SSHFileSystem
-from funcy import first
-
-from dvc.repo.experiments.refs import (
-    EXEC_BASELINE,
-    EXEC_BRANCH,
-    EXEC_HEAD,
-    EXEC_MERGE,
-    EXEC_NAMESPACE,
-)
-
-from .base import BaseExecutor, ExecutorInfo, ExecutorResult, TaskStatus
-
-if TYPE_CHECKING:
-    from queue import Queue
-
-    from dvc.repo import Repo
-    from dvc.repo.experiments.refs import ExpRefInfo
-    from dvc.repo.experiments.stash import ExpStashEntry
-    from dvc.scm import Git
-
-logger = logging.getLogger(__name__)
-
-
-@contextmanager
-def _sshfs(fs_factory, **kwargs):
-    if fs_factory:
-        with fs_factory() as fs:
-            yield fs
-        return
-    yield SSHFileSystem(**kwargs)
-
-
-class SSHExecutor(BaseExecutor):
-    """SSH experiment executor."""
-
-    WARN_UNTRACKED = True
-    QUIET = True
-    SETUP_SCRIPT_FILENAME = "exec-setup.sh"
-
-    def __init__(
-        self,
-        *args,
-        host: Optional[str] = None,
-        port: Optional[int] = None,
-        username: Optional[str] = None,
-        fs_factory: Optional[Callable] = None,
-        setup_script: Optional[str] = None,
-        **kwargs,
-    ):
-        assert host
-
-        super().__init__(*args, **kwargs)
-        self.host: str = host
-        self.port = port
-        self.username = username
-        self._fs_factory = fs_factory
-        self._repo_abspath = ""
-        self._setup_script = setup_script
-
-    @classmethod
-    def gen_dirname(cls, name: Optional[str] = None):
-        from shortuuid import uuid
-
-        return "-".join([name or "dvc-exp", "executor", uuid()])
-
-    @classmethod
-    def from_stash_entry(
-        cls,
-        repo: "Repo",
-        entry: "ExpStashEntry",
-        **kwargs,
-    ):
-        machine_name: Optional[str] = kwargs.pop("machine_name", None)
-        assert repo.machine
-        executor = cls._from_stash_entry(
-            repo,
-            entry,
-            cls.gen_dirname(entry.name),
-            location=machine_name,
-            **repo.machine.get_executor_kwargs(machine_name),
-            setup_script=repo.machine.get_setup_script(machine_name),
-        )
-        logger.debug("Init SSH executor for host '%s'", executor.host)
-        return executor
-
-    def sshfs(self):
-        return _sshfs(self._fs_factory, host=self.host, port=self.port)
-
-    @property
-    def git_url(self) -> str:
-        user = f"{self.username}@" if self.username else ""
-        port = f":{self.port}" if self.port is not None else ""
-        path = f"{self.root_dir}" if self.root_dir else ""
-        if path and not posixpath.isabs(path):
-            path = f"/~/{path}"
-        return f"ssh://{user}{self.host}{port}{path}"
-
-    @property
-    def abs_url(self) -> str:
-        assert self._repo_abspath
-        user = f"{self.username}@" if self.username else ""
-        port = f":{self.port}" if self.port is not None else ""
-        return f"ssh://{user}{self.host}{port}{self._repo_abspath}"
-
-    @staticmethod
-    def _git_client_args(fs):
-        return {
-            "password": fs.fs_args.get("password"),
-            "key_filename": first(fs.fs_args.get("client_keys", [])),
-        }
-
-    def init_git(
-        self,
-        repo: "Repo",  # noqa: ARG002
-        scm: "Git",
-        stash_rev: str,
-        entry: "ExpStashEntry",
-        infofile: Optional[str],
-        branch: Optional[str] = None,
-    ):
-        from dvc.repo.experiments.utils import push_refspec
-
-        self.status = TaskStatus.PREPARING
-        if infofile:
-            self.info.dump_json(infofile)
-
-        with self.sshfs() as fs:
-            fs.makedirs(self.root_dir)
-            self._ssh_cmd(fs, "git init .")
-            self._ssh_cmd(fs, "git config user.name dvc-exp")
-            self._ssh_cmd(fs, "git config user.email [email protected]")
-
-            result = self._ssh_cmd(fs, "pwd")
-            path = result.stdout.strip()
-            self._repo_abspath = path
-
-            # TODO: support multiple client key retries in git backends
-            # (see https://github.com/iterative/dvc/issues/6508)
-            kwargs = self._git_client_args(fs)
-
-            ref_dict = {
-                EXEC_HEAD: entry.head_rev,
-                EXEC_MERGE: stash_rev,
-                EXEC_BASELINE: entry.baseline_rev,
-            }
-            with self.set_temp_refs(scm, ref_dict):
-                exec_namespace = f"{EXEC_NAMESPACE}/"
-                refspec = [(exec_namespace, exec_namespace)]
-                push_refspec(scm, self.git_url, refspec, **kwargs)
-
-            if branch:
-                push_refspec(scm, self.git_url, [(branch, branch)], **kwargs)
-                self._ssh_cmd(fs, f"git symbolic-ref {EXEC_BRANCH} {branch}")
-            else:
-                self._ssh_cmd(fs, f"git symbolic-ref -d {EXEC_BRANCH}", check=False)
-
-            # checkout EXEC_HEAD and apply EXEC_MERGE on top of it without
-            # committing
-            head = EXEC_BRANCH if branch else EXEC_HEAD
-            self._ssh_cmd(fs, f"git checkout {head}")
-            merge_rev = scm.get_ref(EXEC_MERGE)
-            self._ssh_cmd(fs, f"git stash apply {merge_rev}")
-
-            if self._setup_script:
-                self._init_setup_script(fs)
-
-    @classmethod
-    def _setup_script_path(cls, dvc_dir: str):
-        return posixpath.join(
-            dvc_dir,
-            "tmp",
-            cls.SETUP_SCRIPT_FILENAME,
-        )
-
-    def _init_setup_script(self, fs: "SSHFileSystem"):
-        assert self._repo_abspath
-        script_path = self._setup_script_path(
-            posixpath.join(self._repo_abspath, self.dvc_dir)
-        )
-        assert self._setup_script
-        fs.put_file(self._setup_script, script_path)
-
-    def _ssh_cmd(self, sshfs, cmd, chdir=None, **kwargs):
-        working_dir = chdir or self.root_dir
-        return sshfs.fs.execute(f"cd {working_dir};{cmd}", **kwargs)
-
-    def init_cache(self, repo: "Repo", rev: str, run_cache: bool = True):
-        from dvc.repo.push import push
-
-        with self.get_odb() as odb:
-            push(
-                repo,
-                revs=[rev],
-                run_cache=run_cache,
-                odb=odb,
-                include_imports=True,
-            )
-
-    def collect_cache(
-        self, repo: "Repo", exp_ref: "ExpRefInfo", run_cache: bool = True
-    ):
-        """Collect DVC cache."""
-        from dvc.repo.experiments.pull import _pull_cache
-
-        _pull_cache(repo, exp_ref, run_cache=run_cache)
-
-    @contextmanager
-    def get_odb(self):
-        from dvc.cachemgr import CacheManager, get_odb
-
-        cache_path = posixpath.join(
-            self._repo_abspath,
-            self.dvc_dir,
-            CacheManager.CACHE_DIR,
-        )
-
-        with self.sshfs() as fs:
-            yield get_odb(fs, cache_path, **fs.config)
-
-    def fetch_exps(self, *args, **kwargs) -> Iterable[str]:
-        with self.sshfs() as fs:
-            kwargs.update(self._git_client_args(fs))
-            return super().fetch_exps(*args, **kwargs)
-
-    @classmethod
-    def reproduce(
-        cls,
-        info: "ExecutorInfo",
-        rev: str,  # noqa: ARG003
-        queue: Optional["Queue"] = None,  # noqa: ARG003
-        infofile: Optional[str] = None,
-        log_errors: bool = True,
-        log_level: Optional[int] = None,
-        copy_paths: Optional[List[str]] = None,  # noqa: ARG003
-        message: Optional[str] = None,  # noqa: ARG003
-        **kwargs,
-    ) -> "ExecutorResult":
-        """Reproduce an experiment on a remote machine over SSH.
-
-        Internally uses 'dvc exp exec-run' over SSH.
-        """
-        import json
-        import time
-        from tempfile import TemporaryFile
-
-        from asyncssh import ProcessError
-
-        fs_factory: Optional[Callable] = kwargs.pop("fs_factory", None)
-        if log_errors and log_level is not None:
-            cls._set_log_level(log_level)
-
-        with _sshfs(fs_factory) as fs:
-            while not fs.exists("/var/log/dvc-machine-init.log"):
-                logger.info("Waiting for dvc-machine startup script to complete...")
-                time.sleep(5)
-            logger.info("Reproducing experiment on '%s'", fs.fs_args.get("host"))
-            with TemporaryFile(mode="w+", encoding="utf-8") as fobj:
-                json.dump(info.asdict(), fobj)
-                fobj.seek(0)
-                fs.put_file(fobj, infofile)
-            cmd = ["source ~/.profile"]
-            script_path = cls._setup_script_path(info.dvc_dir)
-            if fs.exists(posixpath.join(info.root_dir, script_path)):
-                cmd.extend([f"pushd {info.root_dir}", f"source {script_path}", "popd"])
-            exec_cmd = f"dvc exp exec-run --infofile {infofile}"
-            if log_level is not None:
-                if log_level <= logging.TRACE:  # type: ignore[attr-defined]
-                    exec_cmd += " -vv"
-                elif log_level <= logging.DEBUG:
-                    exec_cmd += " -v"
-            cmd.append(exec_cmd)
-            try:
-                sys.stdout.flush()
-                sys.stderr.flush()
-                stdout = os.dup(sys.stdout.fileno())
-                stderr = os.dup(sys.stderr.fileno())
-                fs.fs.execute("; ".join(cmd), stdout=stdout, stderr=stderr)
-                with fs.open(infofile) as fobj:
-                    result_info = ExecutorInfo.from_dict(json.load(fobj))
-                if result_info.result_hash:
-                    return result_info.result
-            except ProcessError:
-                pass
-            return ExecutorResult(None, None, False)

dvc/repo/experiments/ls.py

@@ -31,23 +31,15 @@ def ls(
             rev = [rev]
         revs = iter_revs(repo.scm, rev, num)
         rev_set = set(revs.keys())
+
     ref_info_dict = exp_refs_by_baseline(repo.scm, rev_set, git_remote)
-
-    tags = repo.scm.describe(ref_info_dict.keys())
-    remained = {baseline for baseline, tag in tags.items() if tag is None}
-    base = "refs/heads"
-    ref_heads = repo.scm.describe(remained, base=base)
-
     results = defaultdict(list)
     for baseline in ref_info_dict:
-        name = baseline
-        if tags[baseline] or ref_heads[baseline]:
-            name = tags[baseline] or ref_heads[baseline]
         for info in ref_info_dict[baseline]:
             if git_remote:
                 exp_rev = None
             else:
                 exp_rev = repo.scm.get_ref(str(info))
-            results[name].append((info.name, exp_rev))
+            results[baseline].append((info.name, exp_rev))
 
     return results

dvc/repo/experiments/queue/base.py

@@ -21,7 +21,7 @@ from dvc_studio_client.post_live_metrics import get_studio_config
 from funcy import retry
 
 from dvc.dependency import ParamsDependency
-from dvc.env import DVC_EXP_BASELINE_REV, DVC_EXP_NAME
+from dvc.env import DVC_EXP_BASELINE_REV, DVC_EXP_NAME, DVC_ROOT
 from dvc.lock import LockError
 from dvc.repo.experiments.exceptions import ExperimentExistsError
 from dvc.repo.experiments.executor.base import BaseExecutor
@@ -348,6 +348,9 @@ class BaseStashQueue(ABC):
                     if not name:
                         name = get_random_exp_name(self.scm, baseline_rev)
                     run_env[DVC_EXP_NAME] = name
+                    # Override DVC_ROOT env var to point to the parent DVC repo
+                    # root (and not an executor tempdir root)
+                    run_env[DVC_ROOT] = self.repo.root_dir
 
                     # save studio config to read later by dvc and dvclive
                     studio_config = get_studio_config(

dvc/repo/experiments/show.py

@@ -177,26 +177,31 @@ def _build_rows(
                 )
 
 
-def _sort_column(
+def _sort_column(  # noqa: C901
     sort_by: str,
     metric_names: Mapping[str, Iterable[str]],
     param_names: Mapping[str, Iterable[str]],
 ) -> Tuple[str, str, str]:
-    path, _, sort_name = sort_by.rpartition(":")
+    sep = ":"
+    parts = sort_by.split(sep)
     matches: Set[Tuple[str, str, str]] = set()
 
-    if path:
+    for split_num in range(len(parts)):
+        path = sep.join(parts[:split_num])
+        sort_name = sep.join(parts[split_num:])
+        if not path:  # handles ':metric_name' case
+            sort_by = sort_name
         if path in metric_names and sort_name in metric_names[path]:
             matches.add((path, sort_name, "metrics"))
         if path in param_names and sort_name in param_names[path]:
             matches.add((path, sort_name, "params"))
-    else:
+    if not matches:
         for path in metric_names:
-            if sort_name in metric_names[path]:
-                matches.add((path, sort_name, "metrics"))
+            if sort_by in metric_names[path]:
+                matches.add((path, sort_by, "metrics"))
         for path in param_names:
-            if sort_name in param_names[path]:
-                matches.add((path, sort_name, "params"))
+            if sort_by in param_names[path]:
+                matches.add((path, sort_by, "params"))
 
     if len(matches) == 1:
         return matches.pop()
@@ -238,19 +243,16 @@ def _exp_range_rows(
     is_base: bool = False,
     **kwargs,
 ) -> Iterator[Tuple["CellT", ...]]:
-    for i, exp in enumerate(exp_range.revs):
+    from funcy import first
+
+    if len(exp_range.revs) > 1:
+        logger.debug("Returning tip commit for legacy checkpoint exp")
+    exp = first(exp_range.revs)
+    if exp:
         row: Dict[str, "CellT"] = {k: fill_value for k in all_headers}
         row["Experiment"] = exp.name or ""
         row["rev"] = exp.rev[:7] if Git.is_sha(exp.rev) else exp.rev
-        if len(exp_range.revs) > 1:
-            if i == 0:
-                row["typ"] = "checkpoint_tip"
-            elif i == len(exp_range.revs) - 1:
-                row["typ"] = "checkpoint_base"
-            else:
-                row["typ"] = "checkpoint_commit"
-        else:
-            row["typ"] = "branch_base" if is_base else "branch_commit"
+        row["typ"] = "branch_base" if is_base else "branch_commit"
         row["parent"] = ""
         if exp_range.executor:
             row["State"] = exp_range.executor.state.capitalize()

dvc/repo/experiments/utils.py

@@ -346,3 +346,55 @@ def to_studio_params(dvc_params):
             result[file_name] = file_data.get("data", {})
 
     return result
+
+
+def describe(
+    scm: "Git",
+    revs: Iterable[str],
+    logger,
+    refs: Optional[Iterable[str]] = None,
+) -> Dict[str, Optional[str]]:
+    """Describe revisions using a tag, branch.
+
+    The first matching name will be returned for each rev. Names are preferred in this
+    order:
+        - current branch (if rev matches HEAD and HEAD is a branch)
+        - tags
+        - branches
+
+    Returns:
+        Dict mapping revisions from revs to a name.
+    """
+
+    head_rev = scm.get_rev()
+    head_ref = scm.get_ref("HEAD", follow=False)
+    if head_ref and head_ref.startswith("refs/heads/"):
+        head_branch = head_ref[len("refs/heads/") :]
+    else:
+        head_branch = None
+
+    tags = {}
+    branches = {}
+    ref_it = iter(refs) if refs else scm.iter_refs()
+    for ref in ref_it:
+        is_tag = ref.startswith("refs/tags/")
+        is_branch = ref.startswith("refs/heads/")
+        if not (is_tag or is_branch):
+            continue
+        rev = scm.get_ref(ref)
+        if not rev:
+            logger.debug("unresolved ref %s", ref)
+            continue
+        if is_tag and rev not in tags:
+            tags[rev] = ref[len("refs/tags/") :]
+        if is_branch and rev not in branches:
+            branches[rev] = ref[len("refs/heads/") :]
+
+    names: Dict[str, Optional[str]] = {}
+    for rev in revs:
+        if rev == head_rev and head_branch:
+            names[rev] = head_branch
+        else:
+            names[rev] = tags.get(rev) or branches.get(rev)
+
+    return names

dvc/repo/fetch.py

@@ -48,7 +48,7 @@ def _collect_indexes(  # noqa: PLR0913
         all_commits=all_commits,
     ):
         try:
-            repo.config.update(config)
+            repo.config.merge(config)
 
             idx = repo.index.targets_view(
                 targets,

dvc/repo/get_url.py

@@ -6,12 +6,12 @@ from dvc.fs import download, parse_external_url
 from dvc.utils import resolve_output
 
 
-def get_url(url, out=None, *, config=None, jobs=None, force=False):
+def get_url(url, out=None, *, fs_config=None, jobs=None, force=False):
     out = resolve_output(url, out, force=force)
     out = os.path.abspath(out)
     (out,) = output.loads_from(None, [out], use_cache=False)
 
-    src_fs, src_path = parse_external_url(url, config)
+    src_fs, src_path = parse_external_url(url, fs_config)
     if not src_fs.exists(src_path):
         raise URLMissingError(url)
     download(src_fs, src_path, out.fs_path, jobs=jobs)

dvc/repo/imp.py

@@ -13,16 +13,28 @@ def imp(
     if rev is not None:
         erepo["rev"] = rev
 
+    if remote and remote_config and isinstance(config, str):
+        raise ValueError(
+            "Can't specify config path together with both remote and remote_config"
+        )
+
     if config is not None:
         erepo["config"] = config
 
     if remote is not None and remote_config is not None:
         conf = erepo.get("config") or {}
+
+        core = conf.get("core") or {}
+        core["remote"] = remote
+
         remotes = conf.get("remote") or {}
         remote_conf = remotes.get(remote) or {}
         remote_conf.update(remote_config)
         remotes[remote] = remote_conf
+
+        conf["core"] = core
         conf["remote"] = remotes
+
         erepo["config"] = conf
     elif remote is not None:
         erepo["remote"] = remote

dvc/repo/index.py

@@ -413,7 +413,7 @@ class Index:
             by_workspace[workspace].add(key)
 
         for path in self._plot_sources:
-            key = self.repo.fs.path.relparts(path, self.repo.root_dir)
+            key = self.repo.fs.path.parts(path)
             by_workspace["repo"].add(key)
 
         return dict(by_workspace)

dvc/repo/ls.py

@@ -109,6 +109,7 @@ def _ls(
             "isout": dvc_info.get("isout", False),
             "isdir": info["type"] == "directory",
             "isexec": info.get("isexec", False),
+            "size": info.get("size"),
         }
 
     return ret

dvc/repo/ls_url.py

@@ -2,8 +2,8 @@ from dvc.exceptions import URLMissingError
 from dvc.fs import parse_external_url
 
 
-def ls_url(url, *, config=None, recursive=False):
-    fs, fs_path = parse_external_url(url, config=config)
+def ls_url(url, *, fs_config=None, recursive=False):
+    fs, fs_path = parse_external_url(url, config=fs_config)
     try:
         info = fs.info(fs_path)
     except FileNotFoundError as exc:
@@ -20,6 +20,7 @@ def ls_url(url, *, config=None, recursive=False):
             ls_info = {
                 "path": fs.path.relpath(info["name"], fs_path),
                 "isdir": info["type"] == "directory",
+                "size": info.get("size"),
             }
             ret.append(ls_info)
 

dvc/repo/plots/__init__.py

@@ -46,8 +46,8 @@ SUPPORTED_IMAGE_EXTENSIONS = ImageRenderer.EXTENSIONS
 class PlotMetricTypeError(DvcException):
     def __init__(self, file):
         super().__init__(
-            "'{}' - file type error\n"
-            "Only JSON, YAML, CSV and TSV formats are supported.".format(file)
+            f"'{file}' - file type error\n"
+            "Only JSON, YAML, CSV and TSV formats are supported."
         )
 
 
@@ -212,7 +212,7 @@ class Plots:
             short_rev = "workspace"
             if rev := getattr(self.repo.fs, "rev", None):
                 short_rev = rev[:7]
-            _resolve_data_sources(data, short_rev, cache_remote_stream=True)
+            _resolve_data_sources(data, short_rev, cache=True)
             result.update(data)
 
         errored = errored_revisions(result)
@@ -281,9 +281,7 @@ def _is_plot(out: "Output") -> bool:
     return bool(out.plot)
 
 
-def _resolve_data_sources(
-    plots_data: Dict, rev: str, cache_remote_stream: bool = False
-):
+def _resolve_data_sources(plots_data: Dict, rev: str, cache: bool = False):
     from dvc.progress import Tqdm
 
     values = list(plots_data.values())
@@ -298,7 +296,7 @@ def _resolve_data_sources(
     def resolve(value):
         data_source = value.pop("data_source")
         assert callable(data_source)
-        value.update(data_source(cache_remote_stream=cache_remote_stream))
+        value.update(data_source(cache=cache))
 
     if not to_resolve:
         return
@@ -448,6 +446,7 @@ def _resolve_definitions(
     definitions: "DictStrAny",
     onerror: Optional[Callable[[Any], Any]] = None,
 ):
+    config_path = os.fspath(config_path)
     config_dir = fs.path.dirname(config_path)
     result: Dict[str, Dict] = {}
     for plot_id, plot_props in definitions.items():

dvc/repo/trie.py

@@ -14,7 +14,7 @@ def build_outs_trie(stages):
             # Check for dup outs
             if out_key in outs:
                 dup_stages = [stage, outs[out_key].stage]
-                raise OutputDuplicationError(str(out), dup_stages)
+                raise OutputDuplicationError(str(out), set(dup_stages))
 
             # Check for overlapping outs
             if outs.has_subtrie(out_key):

dvc/repo/worktree.py

@@ -166,7 +166,7 @@ def push_worktree(
                     latest_only=remote_obj.worktree,
                     jobs=jobs,
                 )
-                pushed += sum(len(changes) for changes in diff.changes.values())
+                pushed += len(diff.files_create)
             except VersioningNotSupported:
                 logger.exception("")
                 raise DvcException(

dvc/rwlock.py

@@ -104,7 +104,7 @@ def _check_blockers(tmp_dir, lock, info, *, mode, waiters):  # noqa: C901, PLR09
                 logger.warning(
                     (
                         "Process '%s' with (Pid %s), in RWLock-file '%s'"
-                        " had been killed. Auto remove it from the lock file."
+                        " had been killed. Auto removed it from the lock file."
                     ),
                     cmd,
                     pid,

dvc/stage/__init__.py

@@ -285,15 +285,6 @@ class Stage(params.StageParams):
                 return line.strip()
         return desc
 
-    def env(self) -> Env:
-        from dvc.env import DVC_ROOT
-
-        env: Env = {}
-        if self.repo:
-            env.update({DVC_ROOT: self.repo.root_dir})
-
-        return env
-
     def changed_deps(
         self, allow_missing: bool = False, upstream: Optional[List] = None
     ) -> bool:

dvc/stage/exceptions.py

@@ -98,7 +98,7 @@ class StageNameUnspecified(DvcException):
     def __init__(self, file):
         super().__init__(
             "Stage name not provided."
-            "Please specify the name as: `{}:stage_name`".format(file.relpath)
+            f"Please specify the name as: `{file.relpath}:stage_name`"
         )
 
 

dvc/stage/run.py

@@ -39,10 +39,14 @@ def _enforce_cmd_list(cmd):
 
 
 def prepare_kwargs(stage, run_env=None):
+    from dvc.env import DVC_ROOT
+
     kwargs = {"cwd": stage.wdir, "env": fix_env(None), "close_fds": True}
 
     if run_env:
         kwargs["env"].update(run_env)
+    if DVC_ROOT not in kwargs["env"]:
+        kwargs["env"][DVC_ROOT] = stage.repo.root_dir
 
     # NOTE: when you specify `shell=True`, `Popen` [1] will default to
     # `/bin/sh` on *nix and will add ["/bin/sh", "-c"] to your command.

dvc/testing/api_tests.py

@@ -117,10 +117,10 @@ class TestAPI:
             assert fobj.read() == b"script1"
 
         tmp = make_tmp_dir("temp-download")
-        fs.get_file("data/foo", tmp / "foo")
+        fs.get_file("data/foo", (tmp / "foo").fs_path)
         assert (tmp / "foo").read_text() == "foo"
 
-        fs.get_file("scripts/script1", tmp / "script1")
+        fs.get_file("scripts/script1", (tmp / "script1").fs_path)
         assert (tmp / "script1").read_text() == "script1"
 
         fs.get("/", (tmp / "all").fs_path, recursive=True)

dvc/testing/benchmarks/cli/stories/test_modify_data.py

@@ -29,8 +29,8 @@ def test_partial_add(bench_dvc, tmp_dir, dvc, dataset, remote):
     shutil.copytree("partial-copy", dataset, dirs_exist_ok=True)
 
     # Benchmark operations for adding files to a dataset
-    bench_dvc("add", dataset, name="partial-add")
-    bench_dvc("push", name="partial-add")
+    bench_dvc("add", dataset, name="partial")
+    bench_dvc("push", name="partial")
     bench_dvc("gc", "-f", "-w", name="noop")
     bench_dvc("gc", "-f", "-w", "-c", name="cloud-noop")
 
@@ -51,7 +51,7 @@ def test_partial_remove(bench_dvc, tmp_dir, dvc, dataset, remote):
                 shutil.rmtree(dataset / f)
 
     # Benchmark operations for removing files from dataset
-    bench_dvc("add", dataset)
-    bench_dvc("push")
+    bench_dvc("add", dataset, name="update")
+    bench_dvc("push", name="update")
     bench_dvc("gc", "-f", "-w")
     bench_dvc("gc", "-f", "-w", "-c", name="cloud")

dvc/testing/tmp_dir.py

@@ -115,8 +115,8 @@ class TmpDir(pathlib.Path):
     def _require(self, name):
         if not hasattr(self, name):
             raise TypeError(
-                "Can't use {name} for this temporary dir. "
-                'Did you forget to use "{name}" fixture?'.format(name=name)
+                f"Can't use {name} for this temporary dir. "
+                f'Did you forget to use "{name}" fixture?'
             )
 
     # Bootstrapping methods

dvc/testing/workspace_tests.py

@@ -95,6 +95,9 @@ class TestImportURLVersionAware:
         assert (tmp_dir / "file").read_text() == "file"
         assert dvc.status() == {}
 
+        orig_version_id = stage.deps[0].meta.version_id
+        orig_def_path = stage.deps[0].def_path
+
         dvc.cache.local.clear()
         remove(tmp_dir / "file")
         dvc.pull()
@@ -108,6 +111,10 @@ class TestImportURLVersionAware:
         assert (tmp_dir / "file").read_text() == "modified"
         assert dvc.status() == {}
 
+        stage = first(dvc.index.stages)
+        assert orig_version_id != stage.deps[0].meta.version_id
+        assert orig_def_path == stage.deps[0].def_path
+
         dvc.cache.local.clear()
         remove(tmp_dir / "file")
         dvc.pull()
@@ -166,7 +173,7 @@ class TestLsUrl:
     def test_file(self, cloud, fname):
         cloud.gen({fname: "foo contents"})
         fs, fs_path = parse_external_url(cloud.url, cloud.config)
-        result = ls_url(str(cloud / fname), config=cloud.config)
+        result = ls_url(str(cloud / fname), fs_config=cloud.config)
         match_files(
             fs,
             result,
@@ -178,7 +185,7 @@ class TestLsUrl:
         if not (cloud / "dir").is_dir():
             pytest.skip("Cannot create directories on this cloud")
         fs, _ = parse_external_url(cloud.url, cloud.config)
-        result = ls_url(str(cloud / "dir"), config=cloud.config)
+        result = ls_url(str(cloud / "dir"), fs_config=cloud.config)
         match_files(
             fs,
             result,
@@ -193,7 +200,7 @@ class TestLsUrl:
         if not (cloud / "dir").is_dir():
             pytest.skip("Cannot create directories on this cloud")
         fs, _ = parse_external_url(cloud.url, cloud.config)
-        result = ls_url(str(cloud / "dir"), config=cloud.config, recursive=True)
+        result = ls_url(str(cloud / "dir"), fs_config=cloud.config, recursive=True)
         match_files(
             fs,
             result,
@@ -205,14 +212,14 @@ class TestLsUrl:
 
     def test_nonexistent(self, cloud):
         with pytest.raises(URLMissingError):
-            ls_url(str(cloud / "dir"), config=cloud.config)
+            ls_url(str(cloud / "dir"), fs_config=cloud.config)
 
 
 class TestGetUrl:
     def test_get_file(self, cloud, tmp_dir):
         cloud.gen({"foo": "foo contents"})
 
-        Repo.get_url(str(cloud / "foo"), "foo_imported", config=cloud.config)
+        Repo.get_url(str(cloud / "foo"), "foo_imported", fs_config=cloud.config)
 
         assert (tmp_dir / "foo_imported").is_file()
         assert (tmp_dir / "foo_imported").read_text() == "foo contents"
@@ -222,7 +229,7 @@ class TestGetUrl:
         if not (cloud / "foo").is_dir():
             pytest.skip("Cannot create directories on this cloud")
 
-        Repo.get_url(str(cloud / "foo"), "foo_imported", config=cloud.config)
+        Repo.get_url(str(cloud / "foo"), "foo_imported", fs_config=cloud.config)
 
         assert (tmp_dir / "foo_imported").is_dir()
         assert (tmp_dir / "foo_imported" / "foo").is_file()
@@ -235,14 +242,14 @@ class TestGetUrl:
             pytest.skip("Cannot create directories on this cloud")
         tmp_dir.gen({"dir": {"subdir": {}}})
 
-        Repo.get_url(str(cloud / "src" / "foo"), dname, config=cloud.config)
+        Repo.get_url(str(cloud / "src" / "foo"), dname, fs_config=cloud.config)
 
         assert (tmp_dir / dname).is_dir()
         assert (tmp_dir / dname / "foo").read_text() == "foo contents"
 
     def test_get_url_nonexistent(self, cloud):
         with pytest.raises(URLMissingError):
-            Repo.get_url(str(cloud / "nonexistent"), config=cloud.config)
+            Repo.get_url(str(cloud / "nonexistent"), fs_config=cloud.config)
 
 
 class TestToRemote:

dvc/utils/humanize.py

@@ -31,3 +31,11 @@ def truncate_text(text: str, max_length: int, with_ellipsis: bool = True) -> str
         return text[: max_length - 1] + ELLIPSIS
 
     return text[:max_length]
+
+
+def naturalsize(value: float, base: int = 1024) -> str:
+    from tqdm import tqdm
+
+    if value < base:
+        return f"{value:.0f}"
+    return tqdm.format_sizeof(value, divisor=base)

pyproject.toml

@@ -34,7 +34,7 @@ dependencies = [
     "configobj>=5.0.6",
     "distro>=1.3",
     "dpath<3,>=2.1.0",
-    "dvc-data>=2.12.0,<2.13.0",
+    "dvc-data>=2.16.0,<2.17.0",
     "dvc-http>=2.29.0",
     "dvc-render>=0.3.1,<1",
     "dvc-studio-client>=0.9.2,<1",
@@ -57,7 +57,7 @@ dependencies = [
     "requests>=2.22",
     "rich>=12",
     "ruamel.yaml>=0.17.11",
-    "scmrepo>=1.0.4,<2",
+    "scmrepo>=1.2.1,<2",
     "shortuuid>=0.5",
     "shtab<2,>=1.3.4",
     "tabulate>=0.8.7",
@@ -70,12 +70,12 @@ dependencies = [
 [project.optional-dependencies]
 all = ["dvc[azure,gdrive,gs,hdfs,oss,s3,ssh,webdav,webhdfs]"]
 azure = ["dvc-azure>=2.21.2"]
-dev = ["dvc[azure,gdrive,gs,hdfs,lint,oss,s3,ssh,terraform,tests,webdav,webhdfs]"]
+dev = ["dvc[azure,gdrive,gs,hdfs,lint,oss,s3,ssh,tests,webdav,webhdfs]"]
 gdrive = ["dvc-gdrive==2.20"]
 gs = ["dvc-gs==2.22.1"]
 hdfs = ["dvc-hdfs==2.19"]
 lint = [
-    "mypy==1.4.1",
+    "mypy==1.5.1",
     "pylint==2.17.5",
     "types-colorama",
     "types-psutil",
@@ -85,11 +85,10 @@ lint = [
     "types-toml",
     "types-tqdm",
 ]
-oss = ["dvc-oss==2.19"]
+oss = ["dvc-oss>=2.20.0"]
 s3 = ["dvc-s3==2.23.0"]
 ssh = ["dvc-ssh>=2.22.1,<3"]
 ssh_gssapi = ["dvc-ssh[gssapi]>=2.22.1,<3"]
-terraform = ["tpi[ssh]>=2.1"]
 testing = [
   "pytest-test-utils",
   "pytest-benchmark[histogram]",
@@ -103,7 +102,7 @@ tests = [
     "flaky",
     "pytest<8,>=7",
     "pytest-cov>=4.1.0",
-    "pytest-docker>=1,<2",
+    "pytest-docker>=1,<3",
     "pytest-lazy-fixture",
     "pytest-mock",
     "pytest-test-utils",

tests/func/experiments/executor/test_ssh.py

@@ -1,162 +0,0 @@
-import posixpath
-from contextlib import contextmanager
-from functools import partial
-from urllib.parse import urlparse
-
-import pytest
-from dvc_ssh import SSHFileSystem
-from dvc_ssh.tests.cloud import TEST_SSH_KEY_PATH, TEST_SSH_USER
-
-from dvc.repo.experiments.executor.base import ExecutorInfo, ExecutorResult
-from dvc.repo.experiments.executor.ssh import SSHExecutor
-from dvc.repo.experiments.refs import EXEC_HEAD, EXEC_MERGE
-
-# pylint: disable-next=wildcard-import
-from tests.func.machine.conftest import *  # noqa: F403
-
-
-@contextmanager
-def _ssh_factory(cloud):
-    yield SSHFileSystem(
-        host=cloud.host,
-        port=cloud.port,
-        user=TEST_SSH_USER,
-        keyfile=TEST_SSH_KEY_PATH,
-    )
-
-
-def test_init_from_stash(tmp_dir, scm, dvc, machine_instance, mocker):
-    mock = mocker.patch.object(SSHExecutor, "_from_stash_entry")
-    mock_entry = mocker.Mock()
-    mock_entry.name = ""
-    SSHExecutor.from_stash_entry(
-        dvc,
-        mock_entry,
-        machine_name="foo",
-    )
-    _args, kwargs = mock.call_args
-    assert kwargs["host"] == machine_instance["instance_ip"]
-
-
-@pytest.mark.needs_internet
-@pytest.mark.parametrize("cloud", [pytest.lazy_fixture("git_ssh")])
-def test_init_git(tmp_dir, dvc, scm, cloud, mocker):
-    tmp_dir.scm_gen({"foo": "foo", "dir": {"bar": "bar"}}, commit="init")
-    baseline_rev = scm.get_rev()
-    tmp_dir.gen("foo", "stashed")
-    scm.gitpython.git.stash()
-    rev = scm.resolve_rev("stash@{0}")
-
-    mock = mocker.Mock(baseline_rev=baseline_rev, head_rev=baseline_rev)
-
-    root_url = cloud / SSHExecutor.gen_dirname()
-
-    executor = SSHExecutor(
-        root_dir=root_url.path,
-        dvc_dir=".dvc",
-        baseline_rev=baseline_rev,
-        host=root_url.host,
-        port=root_url.port,
-        username=TEST_SSH_USER,
-        fs_factory=partial(_ssh_factory, cloud),
-    )
-    infofile = str((root_url / "foo.run").path)
-    executor.init_git(dvc, scm, rev, mock, infofile=infofile)
-    assert root_url.path == executor._repo_abspath
-
-    fs = cloud._ssh
-    assert fs.exists(posixpath.join(executor._repo_abspath, "foo"))
-    assert fs.exists(posixpath.join(executor._repo_abspath, "dir"))
-    assert fs.exists(posixpath.join(executor._repo_abspath, "dir", "bar"))
-
-
-@pytest.mark.needs_internet
-@pytest.mark.parametrize("cloud", [pytest.lazy_fixture("git_ssh")])
-def test_init_cache(tmp_dir, dvc, scm, cloud):
-    foo = tmp_dir.dvc_gen("foo", "foo", commit="init")[0].outs[0]
-    rev = scm.get_rev()
-    scm.set_ref(EXEC_HEAD, rev)
-    scm.set_ref(EXEC_MERGE, rev)
-    root_url = cloud / SSHExecutor.gen_dirname()
-
-    executor = SSHExecutor(
-        root_dir=root_url.path,
-        dvc_dir=".dvc",
-        baseline_rev=rev,
-        host=root_url.host,
-        port=root_url.port,
-        username=TEST_SSH_USER,
-        fs_factory=partial(_ssh_factory, cloud),
-    )
-    executor.init_cache(dvc, rev)
-
-    fs = cloud._ssh
-    foo_hash = foo.hash_info.value
-    assert fs.exists(
-        posixpath.join(
-            executor._repo_abspath, ".dvc", "cache", foo_hash[:2], foo_hash[2:]
-        )
-    )
-
-
-@pytest.mark.needs_internet
-@pytest.mark.parametrize("cloud", [pytest.lazy_fixture("git_ssh")])
-def test_reproduce(tmp_dir, scm, dvc, cloud, exp_stage, mocker):
-    from sshfs import SSHFileSystem as _sshfs  # noqa: N813
-
-    rev = scm.get_rev()
-    root_url = cloud / SSHExecutor.gen_dirname()
-    mocker.patch.object(SSHFileSystem, "exists", return_value=True)
-    mock_execute = mocker.patch.object(_sshfs, "execute")
-    info = ExecutorInfo(
-        str(root_url),
-        rev,
-        "machine-foo",
-        str(root_url.path),
-        ".dvc",
-    )
-    infofile = str((root_url / "foo.run").path)
-    SSHExecutor.reproduce(
-        info,
-        rev,
-        fs_factory=partial(_ssh_factory, cloud),
-    )
-    mock_execute.assert_called_once()
-    _name, args, _kwargs = mock_execute.mock_calls[0]
-    assert f"dvc exp exec-run --infofile {infofile}" in args[0]
-
-
-@pytest.mark.needs_internet
-@pytest.mark.parametrize("cloud", [pytest.lazy_fixture("git_ssh")])
-def test_run_machine(tmp_dir, scm, dvc, cloud, exp_stage, mocker):
-    baseline = scm.get_rev()
-    factory = partial(_ssh_factory, cloud)
-    mocker.patch.object(
-        dvc.machine,
-        "get_executor_kwargs",
-        return_value={
-            "host": cloud.host,
-            "port": cloud.port,
-            "username": TEST_SSH_USER,
-            "fs_factory": factory,
-        },
-    )
-    mocker.patch.object(dvc.machine, "get_setup_script", return_value=None)
-    mock_repro = mocker.patch.object(
-        SSHExecutor,
-        "reproduce",
-        return_value=ExecutorResult("abc123", None, False),
-    )
-
-    tmp_dir.gen("params.yaml", "foo: 2")
-    dvc.experiments.run(exp_stage.addressing, machine="foo")
-    mock_repro.assert_called_once()
-    _name, _args, kwargs = mock_repro.mock_calls[0]
-    info = kwargs["info"]
-    url = urlparse(info.git_url)
-    assert url.scheme == "ssh"
-    assert url.hostname == cloud.host
-    assert url.port == cloud.port
-    assert info.baseline_rev == baseline
-    assert kwargs["infofile"] is not None
-    assert kwargs["fs_factory"] is not None

tests/func/experiments/test_experiments.py

@@ -12,6 +12,7 @@ from dvc.dvcfile import PROJECT_FILE
 from dvc.env import (
     DVC_EXP_BASELINE_REV,
     DVC_EXP_NAME,
+    DVC_ROOT,
     DVC_STUDIO_OFFLINE,
     DVC_STUDIO_REPO_URL,
     DVC_STUDIO_TOKEN,
@@ -348,7 +349,8 @@ def test_packed_args_exists(tmp_dir, scm, dvc, exp_stage, caplog):
 
 
 def test_list(tmp_dir, scm, dvc, exp_stage):
-    baseline_a = scm.get_rev()
+    baseline_old = scm.get_rev()
+
     results = dvc.experiments.run(exp_stage.addressing, params=["foo=2"])
     exp_a = first(results)
     ref_info_a = first(exp_refs_by_rev(scm, exp_a))
@@ -358,34 +360,36 @@ def test_list(tmp_dir, scm, dvc, exp_stage):
     ref_info_b = first(exp_refs_by_rev(scm, exp_b))
 
     tmp_dir.scm_gen("new", "new", commit="new")
+    baseline_new = scm.get_rev()
+
     results = dvc.experiments.run(exp_stage.addressing, params=["foo=4"])
     exp_c = first(results)
     ref_info_c = first(exp_refs_by_rev(scm, exp_c))
 
-    assert dvc.experiments.ls() == {"refs/heads/master": [(ref_info_c.name, exp_c)]}
+    assert dvc.experiments.ls() == {baseline_new: [(ref_info_c.name, exp_c)]}
 
     exp_list = dvc.experiments.ls(rev=ref_info_a.baseline_sha)
     assert {key: set(val) for key, val in exp_list.items()} == {
-        baseline_a: {(ref_info_a.name, exp_a), (ref_info_b.name, exp_b)}
+        baseline_old: {(ref_info_a.name, exp_a), (ref_info_b.name, exp_b)}
     }
 
-    exp_list = dvc.experiments.ls(rev=[baseline_a, scm.get_rev()])
+    exp_list = dvc.experiments.ls(rev=[baseline_old, baseline_new])
     assert {key: set(val) for key, val in exp_list.items()} == {
-        baseline_a: {(ref_info_a.name, exp_a), (ref_info_b.name, exp_b)},
-        "refs/heads/master": {(ref_info_c.name, exp_c)},
+        baseline_old: {(ref_info_a.name, exp_a), (ref_info_b.name, exp_b)},
+        baseline_new: {(ref_info_c.name, exp_c)},
     }
 
     exp_list = dvc.experiments.ls(all_commits=True)
     assert {key: set(val) for key, val in exp_list.items()} == {
-        baseline_a: {(ref_info_a.name, exp_a), (ref_info_b.name, exp_b)},
-        "refs/heads/master": {(ref_info_c.name, exp_c)},
+        baseline_old: {(ref_info_a.name, exp_a), (ref_info_b.name, exp_b)},
+        baseline_new: {(ref_info_c.name, exp_c)},
     }
 
     scm.checkout("branch", True)
     exp_list = dvc.experiments.ls(all_commits=True)
     assert {key: set(val) for key, val in exp_list.items()} == {
-        baseline_a: {(ref_info_a.name, exp_a), (ref_info_b.name, exp_b)},
-        "refs/heads/branch": {(ref_info_c.name, exp_c)},
+        baseline_old: {(ref_info_a.name, exp_a), (ref_info_b.name, exp_b)},
+        baseline_new: {(ref_info_c.name, exp_c)},
     }
 
 
@@ -617,6 +621,7 @@ def test_run_env(tmp_dir, dvc, scm, mocker):
         from dvc.env import (
             DVC_EXP_BASELINE_REV,
             DVC_EXP_NAME,
+            DVC_ROOT,
             DVC_STUDIO_OFFLINE,
             DVC_STUDIO_REPO_URL,
             DVC_STUDIO_TOKEN,
@@ -625,6 +630,7 @@ def test_run_env(tmp_dir, dvc, scm, mocker):
         for v in (
             DVC_EXP_BASELINE_REV,
             DVC_EXP_NAME,
+            DVC_ROOT,
             DVC_STUDIO_OFFLINE,
             DVC_STUDIO_REPO_URL,
             DVC_STUDIO_TOKEN,
@@ -652,6 +658,7 @@ def test_run_env(tmp_dir, dvc, scm, mocker):
     dvc.experiments.run()
     assert (tmp_dir / DVC_EXP_BASELINE_REV).read_text().strip() == baseline
     assert (tmp_dir / DVC_EXP_NAME).read_text().strip()
+    assert (tmp_dir / DVC_ROOT).read_text().strip() == dvc.root_dir
     assert (tmp_dir / DVC_STUDIO_TOKEN).read_text().strip() == "TOKEN"
     assert (tmp_dir / DVC_STUDIO_REPO_URL).read_text().strip() == "REPO_URL"
     assert (tmp_dir / DVC_STUDIO_URL).read_text().strip() == "BASE_URL"
@@ -666,7 +673,7 @@ def test_experiment_unchanged(tmp_dir, scm, dvc, exp_stage):
     dvc.experiments.run(exp_stage.addressing)
     dvc.experiments.run(exp_stage.addressing)
 
-    assert len(dvc.experiments.ls()["refs/heads/master"]) == 2
+    assert len(dvc.experiments.ls()[scm.get_rev()]) == 2
 
 
 def test_experiment_run_dry(tmp_dir, scm, dvc, exp_stage):

tests/func/experiments/test_remote.py

@@ -139,7 +139,7 @@ def test_push_ambiguous_name(tmp_dir, scm, dvc, git_upstream, exp_stage):
 
 @pytest.mark.parametrize("use_url", [True, False])
 def test_list_remote(tmp_dir, scm, dvc, git_downstream, exp_stage, use_url):
-    baseline_a = scm.get_rev()
+    baseline_old = scm.get_rev()
     results = dvc.experiments.run(exp_stage.addressing, params=["foo=2"])
     exp_a = first(results)
     ref_info_a = first(exp_refs_by_rev(scm, exp_a))
@@ -149,6 +149,8 @@ def test_list_remote(tmp_dir, scm, dvc, git_downstream, exp_stage, use_url):
     ref_info_b = first(exp_refs_by_rev(scm, exp_b))
 
     tmp_dir.scm_gen("new", "new", commit="new")
+    baseline_new = scm.get_rev()
+
     results = dvc.experiments.run(exp_stage.addressing, params=["foo=4"])
     exp_c = first(results)
     ref_info_c = first(exp_refs_by_rev(scm, exp_c))
@@ -160,15 +162,15 @@ def test_list_remote(tmp_dir, scm, dvc, git_downstream, exp_stage, use_url):
     assert downstream_exp.ls(git_remote=remote) == {}
 
     git_downstream.tmp_dir.scm.fetch_refspecs(remote, ["master:master"])
-    exp_list = downstream_exp.ls(rev=baseline_a, git_remote=remote)
+    exp_list = downstream_exp.ls(rev=baseline_old, git_remote=remote)
     assert {key: set(val) for key, val in exp_list.items()} == {
-        baseline_a: {(ref_info_a.name, None), (ref_info_b.name, None)}
+        baseline_old: {(ref_info_a.name, None), (ref_info_b.name, None)}
     }
 
     exp_list = downstream_exp.ls(all_commits=True, git_remote=remote)
     assert {key: set(val) for key, val in exp_list.items()} == {
-        baseline_a: {(ref_info_a.name, None), (ref_info_b.name, None)},
-        "refs/heads/master": {(ref_info_c.name, None)},
+        baseline_old: {(ref_info_a.name, None), (ref_info_b.name, None)},
+        baseline_new: {(ref_info_c.name, None)},
     }
 
 

tests/func/experiments/test_save.py

@@ -67,11 +67,12 @@ def test_exp_save_after_commit(tmp_dir, dvc, scm):
     dvc.experiments.save(name="exp-1", force=True)
 
     tmp_dir.scm_gen({"new_file": "new_file"}, commit="new baseline")
+    baseline_new = scm.get_rev()
     dvc.experiments.save(name="exp-2", force=True)
 
     all_exps = dvc.experiments.ls(all_commits=True)
     assert all_exps[baseline][0][0] == "exp-1"
-    assert all_exps["refs/heads/master"][0][0] == "exp-2"
+    assert all_exps[baseline_new][0][0] == "exp-2"
 
 
 def test_exp_save_with_staged_changes(tmp_dir, dvc, scm):

tests/func/experiments/test_show.py

@@ -352,6 +352,20 @@ def test_show_sort(tmp_dir, scm, dvc, exp_stage, caplog):
     assert main(["exp", "show", "--no-pager", "--sort-by=metrics.yaml:foo"]) == 0
 
 
+def test_show_sort_metric_sep(tmp_dir, scm, dvc, caplog):
+    metrics_path = tmp_dir / "metrics:1.json"
+    metrics_path.write_text('{"my::metric": 1, "other_metric": 0.5}')
+    metrics_path = tmp_dir / "metrics:2.json"
+    metrics_path.write_text('{"my::metric": 2}')
+    dvcyaml_path = tmp_dir / "dvc.yaml"
+    dvcyaml_path.write_text("metrics: ['metrics:1.json', 'metrics:2.json']")
+    dvc.experiments.save()
+    assert (
+        main(["exp", "show", "--no-pager", "--sort-by=metrics:1.json:my::metric"]) == 0
+    )
+    assert main(["exp", "show", "--no-pager", "--sort-by=:other_metric"]) == 0
+
+
 @pytest.mark.vscode
 @pytest.mark.parametrize(
     "status, pid_exists",

tests/func/machine/conftest.py

@@ -1,55 +0,0 @@
-import textwrap
-
-import pytest
-
-BASIC_CONFIG = textwrap.dedent(
-    """\
-        [feature]
-            machine = true
-        ['machine "foo"']
-            cloud = aws
-    """
-)
-
-TEST_INSTANCE = {
-    "aws_security_group": None,
-    "cloud": "aws",
-    "id": "iterative-2jyhw8j9ieov6",
-    "image": "ubuntu-bionic-18.04-amd64-server-20210818",
-    "instance_gpu": None,
-    "instance_hdd_size": 35,
-    "instance_ip": "123.123.123.123",
-    "instance_launch_time": "2021-08-25T07:13:03Z",
-    "instance_type": "m",
-    "name": "test-resource",
-    "region": "us-west",
-    "spot": False,
-    "spot_price": -1,
-    "ssh_name": None,
-    "ssh_private": "-----BEGIN RSA PRIVATE KEY-----\\n",
-    "startup_script": "IyEvYmluL2Jhc2g=",
-    "timeouts": None,
-}
-
-
-@pytest.fixture
-def machine_config(tmp_dir):
-    (tmp_dir / ".dvc" / "config").write_text(BASIC_CONFIG)
-    return BASIC_CONFIG
-
-
-@pytest.fixture
-def machine_instance(tmp_dir, dvc, mocker):
-    with dvc.config.edit() as conf:
-        conf["machine"]["foo"] = {"cloud": "aws"}
-
-    def mock_instances(name=None, **kwargs):
-        if name == "foo":
-            return iter([TEST_INSTANCE])
-        return iter([])
-
-    mocker.patch(
-        "tpi.terraform.TerraformBackend.instances",
-        mocker.MagicMock(side_effect=mock_instances),
-    )
-    return TEST_INSTANCE

tests/func/machine/test_machine_config.py

@@ -1,163 +0,0 @@
-import os
-import textwrap
-
-import pytest
-
-from dvc.cli import main
-from dvc.ui import ui
-from tests.utils import console_width
-
-from .conftest import BASIC_CONFIG
-
-
-@pytest.mark.parametrize(
-    "slot,value",
-    [
-        ("region", "us-west"),
-        ("image", "iterative-cml"),
-        ("spot", "True"),
-        ("spot_price", "1.2345"),
-        ("spot_price", "12345"),
-        ("instance_hdd_size", "10"),
-        ("instance_type", "l"),
-        ("instance_gpu", "tesla"),
-        ("ssh_private", "secret"),
-    ],
-)
-def test_machine_modify_susccess(tmp_dir, dvc, machine_config, slot, value):
-    assert main(["machine", "modify", "foo", slot, value]) == 0
-    assert (
-        tmp_dir / ".dvc" / "config"
-    ).read_text() == machine_config + f"    {slot} = {value}\n"
-    assert main(["machine", "modify", "--unset", "foo", slot]) == 0
-    assert (tmp_dir / ".dvc" / "config").read_text() == machine_config
-
-
-def test_machine_modify_startup_script(tmp_dir, dvc, machine_config):
-    slot, value = "startup_script", "start.sh"
-    assert main(["machine", "modify", "foo", slot, value]) == 0
-    assert (
-        tmp_dir / ".dvc" / "config"
-    ).read_text() == machine_config + f"    {slot} = ../{value}\n"
-    assert main(["machine", "modify", "--unset", "foo", slot]) == 0
-    assert (tmp_dir / ".dvc" / "config").read_text() == machine_config
-
-
-@pytest.mark.parametrize(
-    "slot,value,msg",
-    [
-        (
-            "region",
-            "other-west",
-            "expected one of us-west, us-east, eu-west, eu-north",
-        ),
-        ("spot_price", "NUM", "expected float"),
-        ("instance_hdd_size", "BIG", "expected int"),
-    ],
-)
-def test_machine_modify_fail(tmp_dir, dvc, machine_config, caplog, slot, value, msg):
-    assert main(["machine", "modify", "foo", slot, value]) == 251
-    assert (tmp_dir / ".dvc" / "config").read_text() == machine_config
-    assert msg in caplog.text
-
-
-FULL_CONFIG_TEXT = textwrap.dedent(
-    """\
-        [feature]
-            machine = true
-        ['machine \"bar\"']
-            cloud = azure
-        ['machine \"foo\"']
-            cloud = aws
-            region = us-west
-            image = iterative-cml
-            spot = True
-            spot_price = 1.2345
-            instance_hdd_size = 10
-            instance_type = l
-            instance_gpu = tesla
-            ssh_private = secret
-            startup_script = {}
-    """.format(
-        os.path.join("..", "start.sh")
-    )
-)
-
-
-def test_machine_list(tmp_dir, dvc, capsys):
-    from dvc.commands.machine import CmdMachineList
-
-    (tmp_dir / ".dvc" / "config").write_text(FULL_CONFIG_TEXT)
-
-    with console_width(ui.rich_console, 255):
-        assert main(["machine", "list"]) == 0
-    out, _ = capsys.readouterr()
-    for key in CmdMachineList.TABLE_COLUMNS:
-        assert f"{key}" in out
-    assert "bar     azure    -         -" in out
-    assert "foo     aws      us-west   iterative-cml  True    1.2345" in out
-    assert "10                   l                ***            ***" in out
-    assert "tesla" in out
-
-    with console_width(ui.rich_console, 255):
-        assert main(["machine", "list", "bar"]) == 0
-    out, _ = capsys.readouterr()
-    assert "foo" not in out
-    assert "name    cloud" in out
-    assert "bar     azure" in out
-
-
-def test_machine_rename_success(tmp_dir, scm, dvc, machine_config, capsys, mocker):
-    import tpi
-
-    config_file = tmp_dir / ".dvc" / "config"
-
-    mocker.patch.object(
-        tpi.terraform.TerraformBackend,
-        "state_mv",
-        autospec=True,
-        return_value=True,
-    )
-
-    os.makedirs(tmp_dir / ".dvc" / "tmp" / "machine" / "terraform" / "foo")
-
-    assert main(["machine", "rename", "foo", "bar"]) == 0
-    cap = capsys.readouterr()
-    assert "Rename machine 'foo' to 'bar'." in cap.out
-    assert config_file.read_text() == machine_config.replace("foo", "bar")
-    assert not (tmp_dir / ".dvc" / "tmp" / "machine" / "terraform" / "foo").exists()
-    assert (tmp_dir / ".dvc" / "tmp" / "machine" / "terraform" / "bar").exists()
-
-
-def test_machine_rename_none_exist(tmp_dir, scm, dvc, caplog):
-    config_alice = BASIC_CONFIG.replace("foo", "alice")
-    config_file = tmp_dir / ".dvc" / "config"
-    config_file.write_text(config_alice)
-    assert main(["machine", "rename", "foo", "bar"]) == 251
-    assert config_file.read_text() == config_alice
-    assert "machine 'foo' doesn't exist." in caplog.text
-
-
-def test_machine_rename_exist(tmp_dir, scm, dvc, caplog):
-    config_bar = BASIC_CONFIG + "['machine \"bar\"']\n    cloud = aws"
-    config_file = tmp_dir / ".dvc" / "config"
-    config_file.write_text(config_bar)
-    assert main(["machine", "rename", "foo", "bar"]) == 251
-    assert config_file.read_text() == config_bar
-    assert "Machine 'bar' already exists." in caplog.text
-
-
-def test_machine_rename_error(tmp_dir, scm, dvc, machine_config, caplog, mocker):
-    import tpi
-
-    config_file = tmp_dir / ".dvc" / "config"
-    os.makedirs(tmp_dir / ".dvc" / "tmp" / "machine" / "terraform" / "foo")
-
-    def cmd_error(self, source, destination, **kwargs):
-        raise tpi.TPIError("test error")
-
-    mocker.patch.object(tpi.terraform.TerraformBackend, "state_mv", cmd_error)
-
-    assert main(["machine", "rename", "foo", "bar"]) == 251
-    assert config_file.read_text() == machine_config
-    assert "rename failed" in caplog.text

tests/func/machine/test_machine_status.py

@@ -1,22 +0,0 @@
-from dvc.cli import main
-from dvc.ui import ui
-from tests.utils import console_width
-
-
-def test_status(tmp_dir, scm, dvc, machine_config, machine_instance, capsys):
-    assert main(["machine", "add", "bar", "aws"]) == 0
-    with console_width(ui.rich_console, 255):
-        assert main(["machine", "status"]) == 0
-    cap = capsys.readouterr()
-    assert (
-        "name    instance    status    cloud    instance_ip      "
-        "instance_type    instance_hdd_size    instance_gpu" in cap.out
-    )
-    assert (
-        "bar     -           offline   -        -                "
-        "-                -                    -" in cap.out
-    )
-    assert (
-        "foo     num_1       running   aws      123.123.123.123  "
-        "m                35                   None" in cap.out
-    )

tests/func/repro/test_repro.py

@@ -1366,3 +1366,27 @@ def test_repro_ignore_errors(mocker, tmp_dir, dvc, copy_script):
         [bar_call, stage1_call, foo_call, stage2_call],
         [foo_call, bar_call, stage1_call, stage2_call],
     )
+
+
+@pytest.mark.parametrize("persist", [True, False])
+def test_repro_external_outputs(tmp_dir, dvc, local_workspace, persist):
+    local_workspace.gen("foo", "foo")
+    foo_path = str(local_workspace / "foo")
+    bar_path = str(local_workspace / "bar")
+    outs = {"outs_no_cache": [bar_path]}
+    if persist:
+        outs = {"outs_persist_no_cache": [bar_path]}
+    dvc.run(
+        name="mystage",
+        cmd=f"cp {foo_path} {bar_path}",
+        deps=[foo_path],
+        no_exec=True,
+        **outs,
+    )
+
+    dvc.reproduce()
+    dvc.reproduce(force=True)
+
+    assert (local_workspace / "foo").read_text() == "foo"
+    assert (local_workspace / "bar").read_text() == "foo"
+    assert not (local_workspace / "cache").exists()

tests/func/test_add.py

@@ -12,7 +12,11 @@ from dvc.cachemgr import CacheManager
 from dvc.cli import main
 from dvc.config import ConfigError
 from dvc.dvcfile import DVC_FILE_SUFFIX
-from dvc.exceptions import DvcException, OverlappingOutputPathsError
+from dvc.exceptions import (
+    DvcException,
+    OutputDuplicationError,
+    OverlappingOutputPathsError,
+)
 from dvc.fs import LocalFileSystem, system
 from dvc.output import (
     OutputAlreadyTrackedError,
@@ -22,7 +26,7 @@ from dvc.output import (
 from dvc.stage import Stage
 from dvc.stage.exceptions import StageExternalOutputsError, StagePathNotFoundError
 from dvc.utils.fs import path_isin
-from dvc.utils.serialize import YAMLFileCorruptedError
+from dvc.utils.serialize import YAMLFileCorruptedError, dump_yaml
 from dvc_data.hashfile.hash import file_md5
 from dvc_data.hashfile.hash_info import HashInfo
 from tests.utils import get_gitignore_content
@@ -656,6 +660,25 @@ def test_try_adding_pipeline_tracked_output(tmp_dir, dvc, run_copy):
         dvc.add("bar")
 
 
+def test_try_adding_multiple_overlaps(tmp_dir, dvc):
+    tmp_dir.dvc_gen("foo", "foo")
+    dvcyaml_content = {
+        "stages": {
+            "echo-foo": {
+                "cmd": "echo foo > foo",
+                "outs": ["foo"],
+            }
+        }
+    }
+    dump_yaml("dvc.yaml", dvcyaml_content)
+    msg = (
+        "\nUse `dvc remove` with any of the above targets to stop tracking the "
+        "overlapping output."
+    )
+    with pytest.raises(OutputDuplicationError, match=msg):
+        dvc.add("foo")
+
+
 def test_add_pipeline_file(tmp_dir, dvc, run_copy):
     from dvc.dvcfile import PROJECT_FILE
 

tests/func/test_config.py

@@ -265,14 +265,16 @@ def test_load_relative_paths(dvc, field, remote_url):
 
     # check if written paths are correct
     dvc_dir = dvc.config.dvc_dir
-    assert dvc.config["remote"]["test"][field] == os.path.join(
-        dvc_dir, "..", "file.txt"
+    assert dvc.config["remote"]["test"][field] == os.path.abspath(
+        os.path.join(dvc_dir, "..", "file.txt")
     )
 
     # load config and check that it contains what we expect
     # (relative paths are evaluated correctly)
     cfg = Config(dvc_dir)
-    assert cfg["remote"]["test"][field] == os.path.join(dvc_dir, "..", "file.txt")
+    assert cfg["remote"]["test"][field] == os.path.abspath(
+        os.path.join(dvc_dir, "..", "file.txt")
+    )
 
 
 def test_config_gdrive_fields(tmp_dir, dvc):

tests/func/test_import.py

@@ -658,3 +658,75 @@ def test_parameterized_repo(tmp_dir, dvc, scm, erepo_dir, paths):
         "url": os.fspath(erepo_dir),
         "rev_lock": erepo_dir.scm.get_rev(),
     }
+
+
+@pytest.mark.parametrize(
+    "options, def_repo",
+    [
+        ({"config": "myconfig"}, {"config": "myconfig"}),
+        ({"remote": "myremote"}, {"remote": "myremote"}),
+        ({"remote_config": {"key": "value"}}, {"remote": {"key": "value"}}),
+        (
+            {
+                "remote": "myremote",
+                "remote_config": {"key": "value"},
+            },
+            {
+                "config": {
+                    "core": {"remote": "myremote"},
+                    "remote": {
+                        "myremote": {"key": "value"},
+                    },
+                },
+            },
+        ),
+        (
+            {
+                "remote": "myremote",
+                "remote_config": {"key": "value"},
+                "config": {"otherkey": "othervalue"},
+            },
+            {
+                "config": {
+                    "core": {"remote": "myremote"},
+                    "remote": {
+                        "myremote": {"key": "value"},
+                    },
+                    "otherkey": "othervalue",
+                },
+            },
+        ),
+    ],
+)
+def test_import_configs(tmp_dir, scm, dvc, erepo_dir, options, def_repo):
+    with erepo_dir.chdir():
+        erepo_dir.dvc_gen("foo", "foo content", commit="create foo")
+
+    (tmp_dir / "myconfig").touch()
+
+    stage = dvc.imp(
+        os.fspath(erepo_dir), "foo", "foo_imported", no_exec=True, **options
+    )
+    assert stage.deps[0].def_repo == {
+        "url": os.fspath(erepo_dir),
+        **def_repo,
+    }
+
+
+def test_import_invalid_configs(tmp_dir, scm, dvc, erepo_dir):
+    with erepo_dir.chdir():
+        erepo_dir.dvc_gen("foo", "foo content", commit="create foo")
+
+    with pytest.raises(
+        ValueError,
+        match="Can't specify config path together with both remote and remote_config",
+    ):
+        dvc.imp(
+            os.fspath(erepo_dir),
+            "foo",
+            "foo_imported",
+            no_exec=True,
+            config="myconfig",
+            remote="myremote",
+            remote_config={"key": "value"},
+        )

tests/func/test_import_url.py

@@ -249,9 +249,19 @@ def test_import_url_fs_config(tmp_dir, dvc, workspace, mocker):
     dep_init = mocker.spy(Dependency, "__init__")
     dvc.imp_url(url, fs_config={"jobs": 42})
 
+    stage = load_file(dvc, "foo.dvc").stage
+    assert stage.deps[0].def_fs_config == {"jobs": 42}
+
     dep_init_kwargs = dep_init.call_args[1]
     assert dep_init_kwargs.get("fs_config") == {"jobs": 42}
 
     assert get_fs_config.call_args_list[0][1] == {"url": "foo"}
     assert get_fs_config.call_args_list[1][1] == {"url": url, "jobs": 42}
     assert get_fs_config.call_args_list[2][1] == {"name": "workspace"}
+
+    dep_init.reset_mock()
+
+    dvc.pull("foo.dvc")
+
+    dep_init_kwargs = dep_init.call_args[1]
+    assert dep_init_kwargs.get("fs_config") == {"jobs": 42}

tests/func/test_init.py

@@ -113,10 +113,7 @@ def test_init_when_ignored_by_git(tmp_dir, scm, caplog):
     with caplog.at_level(logging.ERROR, logger="dvc"):
         assert main(["init"]) == 1
     assert (
-        "{dvc_dir} is ignored by your SCM tool. \n"
+        f"{tmp_dir / DvcRepo.DVC_DIR} is ignored by your SCM tool. \n"
         "Make sure that it's tracked, "
-        "for example, by adding '!.dvc' to .gitignore.".format(
-            dvc_dir=tmp_dir / DvcRepo.DVC_DIR
-        )
-        in caplog.text
+        "for example, by adding '!.dvc' to .gitignore." in caplog.text
     )

tests/func/test_ls.py

@@ -460,7 +460,7 @@ def test_ls_shows_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
     match_files(files, ((("bar",), True),))
 
 
-def test_ls_granular(erepo_dir):
+def test_ls_granular(erepo_dir, M):
     with erepo_dir.chdir():
         erepo_dir.dvc_gen(
             {
@@ -475,15 +475,21 @@ def test_ls_granular(erepo_dir):
 
     entries = Repo.ls(os.fspath(erepo_dir), os.path.join("dir", "subdir"))
     assert entries == [
-        {"isout": True, "isdir": False, "isexec": False, "path": "bar"},
-        {"isout": True, "isdir": False, "isexec": False, "path": "foo"},
+        {"isout": True, "isdir": False, "isexec": False, "path": "bar", "size": 3},
+        {"isout": True, "isdir": False, "isexec": False, "path": "foo", "size": 3},
     ]
 
     entries = Repo.ls(os.fspath(erepo_dir), "dir")
     assert entries == [
-        {"isout": True, "isdir": False, "isexec": False, "path": "1"},
-        {"isout": True, "isdir": False, "isexec": False, "path": "2"},
-        {"isout": True, "isdir": True, "isexec": False, "path": "subdir"},
+        {"isout": True, "isdir": False, "isexec": False, "path": "1", "size": 1},
+        {"isout": True, "isdir": False, "isexec": False, "path": "2", "size": 1},
+        {
+            "isout": True,
+            "isdir": True,
+            "isexec": False,
+            "path": "subdir",
+            "size": M.instance_of(int),
+        },
     ]
 
 
@@ -508,14 +514,14 @@ def test_ls_target(erepo_dir, use_scm):
         return Repo.ls(os.fspath(erepo_dir), path)
 
     assert _ls(os.path.join("dir", "1")) == [
-        {"isout": isout, "isdir": False, "isexec": False, "path": "1"}
+        {"isout": isout, "isdir": False, "isexec": False, "path": "1", "size": 1}
     ]
     assert _ls(os.path.join("dir", "subdir", "foo")) == [
-        {"isout": isout, "isdir": False, "isexec": False, "path": "foo"}
+        {"isout": isout, "isdir": False, "isexec": False, "path": "foo", "size": 3}
     ]
     assert _ls(os.path.join("dir", "subdir")) == [
-        {"isdir": False, "isexec": 0, "isout": isout, "path": "bar"},
-        {"isdir": False, "isexec": 0, "isout": isout, "path": "foo"},
+        {"isdir": False, "isexec": 0, "isout": isout, "path": "bar", "size": 3},
+        {"isdir": False, "isexec": 0, "isout": isout, "path": "foo", "size": 3},
     ]
 
 
@@ -559,7 +565,7 @@ def test_subrepo(dvc_top_level, erepo):
     assert _list_files(subrepo, "dvc_dir") == {"lorem"}
 
 
-def test_broken_symlink(tmp_dir, dvc):
+def test_broken_symlink(tmp_dir, dvc, M):
     from dvc.fs import system
 
     tmp_dir.gen("file", "content")
@@ -575,17 +581,19 @@ def test_broken_symlink(tmp_dir, dvc):
             "isdir": False,
             "isexec": False,
             "path": ".dvcignore",
+            "size": M.instance_of(int),
         },
         {
             "isout": False,
             "isdir": False,
             "isexec": False,
             "path": "link",
+            "size": 0,
         },
     ]
 
 
-def test_ls_broken_dir(tmp_dir, dvc):
+def test_ls_broken_dir(tmp_dir, dvc, M):
     from dvc_data.index import DataIndexDirError
 
     tmp_dir.dvc_gen(
@@ -606,13 +614,43 @@ def test_ls_broken_dir(tmp_dir, dvc):
 
     entries = Repo.ls(os.fspath(tmp_dir))
     assert entries == [
-        {"isdir": False, "isexec": False, "isout": False, "path": ".dvcignore"},
-        {"isdir": True, "isexec": False, "isout": True, "path": "broken"},
-        {"isdir": False, "isexec": False, "isout": False, "path": "broken.dvc"},
-        {"isdir": True, "isexec": False, "isout": True, "path": "dir"},
-        {"isdir": False, "isexec": False, "isout": False, "path": "dir.dvc"},
-        {"isdir": False, "isexec": False, "isout": True, "path": "foo"},
-        {"isdir": False, "isexec": False, "isout": False, "path": "foo.dvc"},
+        {
+            "isdir": False,
+            "isexec": False,
+            "isout": False,
+            "path": ".dvcignore",
+            "size": M.instance_of(int),
+        },
+        {"isdir": True, "isexec": False, "isout": True, "path": "broken", "size": 3},
+        {
+            "isdir": False,
+            "isexec": False,
+            "isout": False,
+            "path": "broken.dvc",
+            "size": M.instance_of(int),
+        },
+        {
+            "isdir": True,
+            "isexec": False,
+            "isout": True,
+            "path": "dir",
+            "size": M.instance_of(int),
+        },
+        {
+            "isdir": False,
+            "isexec": False,
+            "isout": False,
+            "path": "dir.dvc",
+            "size": M.instance_of(int),
+        },
+        {"isdir": False, "isexec": False, "isout": True, "path": "foo", "size": 3},
+        {
+            "isdir": False,
+            "isexec": False,
+            "isout": False,
+            "path": "foo.dvc",
+            "size": M.instance_of(int),
+        },
     ]
 
     with pytest.raises(DataIndexDirError):

tests/unit/command/test_checkout.py

@@ -16,6 +16,7 @@ def test_checkout(tmp_dir, dvc, mocker):
         recursive=False,
         relink=True,
         with_deps=True,
+        allow_missing=False,
     )
 
 

tests/unit/command/test_diff.py

@@ -45,10 +45,10 @@ def test_default(mocker, capsys, dvc):
         "    file\n"
         "\n"
         "Renamed:\n"
-        "    data{sep}file_old -> data{sep}file_new\n"
+        f"    data{os.path.sep}file_old -> data{os.path.sep}file_new\n"
         "\n"
         "files summary: 1 added, 1 renamed"
-    ).format(sep=os.path.sep) in capsys.readouterr()[0]
+    ) in capsys.readouterr()[0]
 
 
 def test_show_hash(mocker, capsys, dvc):
@@ -252,14 +252,14 @@ def test_show_markdown(capsys):
         "|--------------|----------------------|\n"
         "| added        | file                 |\n"
         "| deleted      | zoo                  |\n"
-        "| deleted      | data{sep}                |\n"
-        "| deleted      | data{sep}foo             |\n"
-        "| deleted      | data{sep}bar             |\n"
+        f"| deleted      | data{os.path.sep}                |\n"
+        f"| deleted      | data{os.path.sep}foo             |\n"
+        f"| deleted      | data{os.path.sep}bar             |\n"
         "| renamed      | file_old -> file_new |\n"
         "| modified     | file                 |\n"
         "| not in cache | file2                |\n"
         "\n"
-    ).format(sep=os.path.sep)
+    )
 
 
 def test_show_markdown_with_hash(capsys):
@@ -289,14 +289,14 @@ def test_show_markdown_with_hash(capsys):
         "|--------------|--------------------|----------------------|\n"
         "| added        | 00000000           | file                 |\n"
         "| deleted      | 22222              | zoo                  |\n"
-        "| deleted      | XXXXXXXX           | data{sep}                |\n"
-        "| deleted      | 11111111           | data{sep}foo             |\n"
-        "| deleted      | 00000000           | data{sep}bar             |\n"
+        f"| deleted      | XXXXXXXX           | data{os.path.sep}                |\n"
+        f"| deleted      | 11111111           | data{os.path.sep}foo             |\n"
+        f"| deleted      | 00000000           | data{os.path.sep}bar             |\n"
         "| renamed      | 11111111           | file_old -> file_new |\n"
         "| modified     | AAAAAAAA..BBBBBBBB | file                 |\n"
         "| not in cache | 12345678           | file2                |\n"
         "\n"
-    ).format(sep=os.path.sep)
+    )
 
 
 def test_show_markdown_hide_missing(capsys):
@@ -321,13 +321,13 @@ def test_show_markdown_hide_missing(capsys):
         "|----------|----------------------|\n"
         "| added    | file                 |\n"
         "| deleted  | zoo                  |\n"
-        "| deleted  | data{sep}                |\n"
-        "| deleted  | data{sep}foo             |\n"
-        "| deleted  | data{sep}bar             |\n"
+        f"| deleted  | data{os.path.sep}                |\n"
+        f"| deleted  | data{os.path.sep}foo             |\n"
+        f"| deleted  | data{os.path.sep}bar             |\n"
         "| renamed  | file_old -> file_new |\n"
         "| modified | file                 |\n"
         "\n"
-    ).format(sep=os.path.sep)
+    )
 
 
 def test_hide_missing(mocker, capsys, dvc):

tests/unit/command/test_experiments.py

@@ -125,7 +125,6 @@ def test_experiments_run(dvc, scm, mocker):
         "run_all": False,
         "jobs": 1,
         "tmp_dir": False,
-        "machine": None,
         "copy_paths": [],
         "message": None,
     }
@@ -189,7 +188,7 @@ def test_experiments_list(dvc, scm, mocker):
 @pytest.mark.parametrize(
     "args,expected",
     [
-        ([], "main:\n\tsha-a [exp-a]\n"),
+        ([], "master:\n\tsha-a [exp-a]\n"),
         (["--name-only"], "exp-a\n"),
         (["--sha-only"], "sha-a\n"),
     ],
@@ -198,7 +197,7 @@ def test_experiments_list_format(mocker, capsys, args, expected, dvc, scm):
     mocker.patch(
         "dvc.repo.experiments.ls.ls",
         return_value={
-            "refs/heads/main": [
+            scm.get_rev(): [
                 ("exp-a", "sha-a"),
             ]
         },

tests/unit/command/test_get_url.py

@@ -11,4 +11,4 @@ def test_get_url(mocker):
 
     assert cmd.run() == 0
 
-    m.assert_called_once_with("src", out="out", jobs=5, force=False)
+    m.assert_called_once_with("src", out="out", jobs=5, force=False, fs_config=None)

tests/unit/command/test_help.py

@@ -25,18 +25,18 @@ def command_tuples():
 
     main_parser = get_main_parser()
     recurse_parser(main_parser)
+
+    # the no. of commands will usually go up,
+    # but if we ever remove commands and drop below, adjust the magic number accordingly
+    assert len(commands) >= 96
     return sorted(commands)
 
 
-def pytest_generate_tests(metafunc):
-    def ids(values):
-        if values:
-            return "-".join(values)
-        return "dvc"
-
-    metafunc.parametrize("command_tuples", command_tuples(), ids=ids)
+def ids(values):
+    return "-".join(values) or "dvc"
 
 
+@pytest.mark.parametrize("command_tuples", command_tuples(), ids=ids)
 def test_help(caplog, capsys, command_tuples):
     with caplog.at_level(logging.INFO), pytest.raises(SystemExit) as e:
         main([*command_tuples, "--help"])

tests/unit/command/test_imp_url.py

@@ -34,6 +34,7 @@ def test_import_url(mocker, dvc):
         jobs=4,
         force=False,
         version_aware=False,
+        fs_config=None,
     )
 
 
@@ -83,6 +84,7 @@ def test_import_url_no_exec_download_flags(mocker, flag, expected, dvc):
         jobs=None,
         force=False,
         version_aware=False,
+        fs_config=None,
         **expected,
     )
 
@@ -115,6 +117,7 @@ def test_import_url_to_remote(mocker, dvc):
         jobs=None,
         force=False,
         version_aware=False,
+        fs_config=None,
     )
 
 

tests/unit/command/test_ls_url.py

@@ -10,7 +10,7 @@ def test_ls_url(mocker):
 
     assert cmd.run() == 0
 
-    m.assert_called_once_with("src", recursive=False)
+    m.assert_called_once_with("src", recursive=False, fs_config=None)
 
 
 def test_recursive(mocker):
@@ -21,4 +21,4 @@ def test_recursive(mocker):
 
     assert cmd.run() == 0
 
-    m.assert_called_once_with("src", recursive=True)
+    m.assert_called_once_with("src", recursive=True, fs_config=None)

tests/unit/command/test_machine.py

@@ -1,167 +0,0 @@
-import os
-
-import configobj
-import pytest
-
-from dvc.cli import parse_args
-from dvc.commands.machine import (
-    CmdMachineAdd,
-    CmdMachineCreate,
-    CmdMachineDestroy,
-    CmdMachineList,
-    CmdMachineModify,
-    CmdMachineRemove,
-    CmdMachineRename,
-    CmdMachineSsh,
-    CmdMachineStatus,
-)
-
-DATA = {
-    ".dvc": {
-        "config": (
-            "[feature]\n"
-            "  machine = true\n"
-            "['machine \"foo\"']\n"
-            "  cloud = aws\n"
-            "['machine \"myaws\"']\n"
-            "  cloud = aws"
-        )
-    }
-}
-
-
-def test_add(tmp_dir):
-    tmp_dir.gen({".dvc": {"config": "[feature]\n  machine = true"}})
-    cli_args = parse_args(["machine", "add", "foo", "aws"])
-    assert cli_args.func == CmdMachineAdd
-    cmd = cli_args.func(cli_args)
-    assert cmd.run() == 0
-    config = configobj.ConfigObj(str(tmp_dir / ".dvc" / "config"))
-    assert config['machine "foo"']["cloud"] == "aws"
-
-
-def test_remove(tmp_dir):
-    tmp_dir.gen(DATA)
-    cli_args = parse_args(["machine", "remove", "foo"])
-    assert cli_args.func == CmdMachineRemove
-    cmd = cli_args.func(cli_args)
-    assert cmd.run() == 0
-    config = configobj.ConfigObj(str(tmp_dir / ".dvc" / "config"))
-    assert list(config.keys()) == ["feature", 'machine "myaws"']
-
-
-def test_create(tmp_dir, dvc, mocker):
-    cli_args = parse_args(["machine", "create", "foo"])
-    assert cli_args.func == CmdMachineCreate
-
-    cmd = cli_args.func(cli_args)
-    m = mocker.patch.object(cmd.repo.machine, "create", autospec=True, return_value=0)
-
-    assert cmd.run() == 0
-    m.assert_called_once_with("foo")
-
-
-def test_status(tmp_dir, scm, dvc, mocker):
-    tmp_dir.gen(DATA)
-    cli_args = parse_args(["machine", "status", "foo"])
-    assert cli_args.func == CmdMachineStatus
-
-    cmd = cli_args.func(cli_args)
-    m = mocker.patch.object(cmd.repo.machine, "status", autospec=True, return_value=[])
-    assert cmd.run() == 0
-    m.assert_called_once_with("foo")
-
-    cli_args = parse_args(["machine", "status"])
-    cmd = cli_args.func(cli_args)
-    m = mocker.patch.object(cmd.repo.machine, "status", autospec=True, return_value=[])
-    assert cmd.run() == 0
-    assert m.call_count == 2
-    m.assert_has_calls([mocker.call("foo"), mocker.call("myaws")])
-
-
-def test_destroy(tmp_dir, dvc, mocker):
-    cli_args = parse_args(["machine", "destroy", "foo"])
-    assert cli_args.func == CmdMachineDestroy
-
-    cmd = cli_args.func(cli_args)
-    m = mocker.patch.object(cmd.repo.machine, "destroy", autospec=True, return_value=0)
-
-    assert cmd.run() == 0
-    m.assert_called_once_with("foo")
-
-
-def test_ssh(tmp_dir, dvc, mocker):
-    cli_args = parse_args(["machine", "ssh", "foo"])
-    assert cli_args.func == CmdMachineSsh
-
-    cmd = cli_args.func(cli_args)
-    m = mocker.patch.object(
-        cmd.repo.machine, "run_shell", autospec=True, return_value=0
-    )
-
-    assert cmd.run() == 0
-    m.assert_called_once_with("foo")
-
-
-@pytest.mark.parametrize("show_origin", [["--show-origin"], []])
-def test_list(tmp_dir, mocker, show_origin):
-    from dvc.compare import TabularData
-    from dvc.ui import ui
-
-    tmp_dir.gen(DATA)
-    cli_args = parse_args(["machine", "list", *show_origin, "foo"])
-    assert cli_args.func == CmdMachineList
-    cmd = cli_args.func(cli_args)
-    if show_origin:
-        m = mocker.patch.object(ui, "write", autospec=True)
-    else:
-        m = mocker.patch.object(TabularData, "render", autospec=True)
-    assert cmd.run() == 0
-    if show_origin:
-        m.assert_called_once_with(f".dvc{os.sep}config	cloud=aws")
-    else:
-        m.assert_called_once()
-
-
-def test_modified(tmp_dir):
-    tmp_dir.gen(DATA)
-    cli_args = parse_args(["machine", "modify", "foo", "cloud", "azure"])
-    assert cli_args.func == CmdMachineModify
-    cmd = cli_args.func(cli_args)
-    assert cmd.run() == 0
-    config = configobj.ConfigObj(str(tmp_dir / ".dvc" / "config"))
-    assert config['machine "foo"']["cloud"] == "azure"
-
-
-def test_rename(tmp_dir, scm, dvc):
-    tmp_dir.gen(DATA)
-    cli_args = parse_args(["machine", "rename", "foo", "bar"])
-    assert cli_args.func == CmdMachineRename
-    cmd = cli_args.func(cli_args)
-    assert cmd.run() == 0
-    config = configobj.ConfigObj(str(tmp_dir / ".dvc" / "config"))
-    assert config['machine "bar"']["cloud"] == "aws"
-
-
-@pytest.mark.parametrize(
-    "cmd,use_config",
-    [
-        ("add", True),
-        ("default", True),
-        ("remove", True),
-        ("list", True),
-        ("modify", True),
-        ("create", False),
-        ("destroy", False),
-        ("rename", True),
-        ("status", False),
-        ("ssh", False),
-    ],
-)
-def test_help_message(tmp_dir, scm, dvc, cmd, use_config, capsys):
-    try:
-        parse_args(["machine", cmd, "--help"])
-    except SystemExit:
-        pass
-    cap = capsys.readouterr()
-    assert ("--global" in cap.out) is use_config

tests/unit/fs/test_azure.py

@@ -1,84 +0,0 @@
-import pytest
-from dvc_azure import AzureAuthError, AzureFileSystem
-
-container_name = "container-name"
-connection_string = (
-    "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;"
-    "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsu"
-    "Fq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;"
-    "BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;"
-)
-
-
-def test_strip_protocol_env_var(monkeypatch, dvc):
-    monkeypatch.setenv("AZURE_STORAGE_CONTAINER_NAME", container_name)
-    monkeypatch.setenv("AZURE_STORAGE_CONNECTION_STRING", connection_string)
-
-    assert AzureFileSystem._strip_protocol("azure://") == container_name
-
-
-def test_strip_protocol(dvc):
-    assert (
-        AzureFileSystem._strip_protocol(f"azure://{container_name}") == container_name
-    )
-
-
-def test_init(dvc):
-    config = {"connection_string": connection_string}
-    fs = AzureFileSystem(**config)
-    assert fs.fs_args["connection_string"] == connection_string
-
-
-def test_azure_login_methods():
-    def get_login_method(config):
-        fs = AzureFileSystem(**config)
-        return fs.login_method
-
-    with pytest.raises(AzureAuthError):
-        get_login_method({})
-
-    assert get_login_method({"connection_string": "test"}) == "connection string"
-    assert get_login_method({"account_name": "test"}).startswith("default credentials")
-    assert (
-        get_login_method({"account_name": "test", "allow_anonymous_login": True})
-        == "anonymous login"
-    )
-
-    with pytest.raises(AzureAuthError):
-        get_login_method(
-            {"tenant_id": "test", "client_id": "test", "client_secret": "test"}
-        )
-
-    assert (
-        get_login_method(
-            {
-                "account_name": "test",
-                "tenant_id": "test",
-                "client_id": "test",
-                "client_secret": "test",
-            }
-        )
-        == "AD service principal"
-    )
-
-    assert (
-        get_login_method({"account_name": "test", "account_key": "test"})
-        == "account key"
-    )
-    assert (
-        get_login_method({"account_name": "test", "sas_token": "test"}) == "SAS token"
-    )
-    assert (
-        get_login_method(
-            {
-                "connection_string": "test",
-                "account_name": "test",
-                "sas_token": "test",
-            }
-        )
-        == "connection string"
-    )
-    assert (
-        get_login_method({"connection_string": "test", "sas_token": "test"})
-        == "connection string"
-    )

tests/unit/fs/test_s3.py

@@ -1,137 +0,0 @@
-import os
-
-import pytest
-from dvc_s3 import S3FileSystem
-
-from dvc.fs import ConfigError
-
-bucket_name = "bucket-name"
-prefix = "some/prefix"
-url = f"s3://{bucket_name}/{prefix}"
-key_id = "key-id"
-key_secret = "key-secret"
-session_token = "session-token"
-
-
-@pytest.fixture(autouse=True)
-def grants():
-    return {
-        "grant_read": "id=read-permission-id,id=other-read-permission-id",
-        "grant_read_acp": "id=read-acp-permission-id",
-        "grant_write_acp": "id=write-acp-permission-id",
-        "grant_full_control": "id=full-control-permission-id",
-    }
-
-
-def test_verify_ssl_default_param(dvc):
-    config = {"url": url}
-    fs = S3FileSystem(**config)
-
-    assert "client_kwargs" not in fs.fs_args
-
-    config = {
-        "url": url,
-        "endpointurl": "https://my.custom.s3:1234",
-    }
-    fs = S3FileSystem(**config)
-
-    assert "verify" not in fs.fs_args["client_kwargs"]
-
-
-def test_s3_config_credentialpath(dvc, monkeypatch):
-    environment = {}
-    monkeypatch.setattr(os, "environ", environment)
-
-    config = {"url": url, "credentialpath": "somewhere"}
-    S3FileSystem(**config)
-    assert environment["AWS_SHARED_CREDENTIALS_FILE"] == "somewhere"
-    environment.clear()
-
-    config = {"url": url, "configpath": "somewhere"}
-    S3FileSystem(**config)
-    assert environment["AWS_CONFIG_FILE"] == "somewhere"
-    environment.clear()
-
-    config = {
-        "url": url,
-        "credentialpath": "somewhere",
-        "configpath": "elsewhere",
-    }
-    S3FileSystem(**config)
-    assert environment["AWS_SHARED_CREDENTIALS_FILE"] == "somewhere"
-    assert environment["AWS_CONFIG_FILE"] == "elsewhere"
-    environment.clear()
-
-
-def test_ssl_verify_bool_param(dvc):
-    config = {"url": url, "ssl_verify": False}
-    fs = S3FileSystem(**config)
-
-    assert fs.fs_args["client_kwargs"]["verify"] == config["ssl_verify"]
-
-
-def test_ssl_verify_path_param(dvc):
-    config = {"url": url, "ssl_verify": "/path/to/custom/cabundle.pem"}
-    fs = S3FileSystem(**config)
-
-    assert fs.fs_args["client_kwargs"]["verify"] == config["ssl_verify"]
-
-
-def test_ssl_verify_none_param(dvc):
-    config = {"url": url, "ssl_verify": None}
-    fs = S3FileSystem(**config)
-
-    assert "client_kwargs" not in fs.fs_args
-
-    config = {
-        "url": url,
-        "endpointurl": "https://my.custom.s3:1234",
-        "ssl_verify": None,
-    }
-    fs = S3FileSystem(**config)
-
-    assert "verify" not in fs.fs_args["client_kwargs"]
-
-
-def test_grants(dvc):
-    config = {
-        "url": url,
-        "grant_read": "id=read-permission-id,id=other-read-permission-id",
-        "grant_read_acp": "id=read-acp-permission-id",
-        "grant_write_acp": "id=write-acp-permission-id",
-        "grant_full_control": "id=full-control-permission-id",
-    }
-    fs = S3FileSystem(**config)
-
-    extra_args = fs.fs_args["s3_additional_kwargs"]
-    assert (
-        extra_args["GrantRead"] == "id=read-permission-id,id=other-read-permission-id"
-    )
-    assert extra_args["GrantReadACP"] == "id=read-acp-permission-id"
-    assert extra_args["GrantWriteACP"] == "id=write-acp-permission-id"
-    assert extra_args["GrantFullControl"] == "id=full-control-permission-id"
-
-
-def test_grants_mutually_exclusive_acl_error(dvc, grants):
-    for grant_option, grant_value in grants.items():
-        config = {"url": url, "acl": "public-read", grant_option: grant_value}
-
-        with pytest.raises(ConfigError):
-            S3FileSystem(**config)
-
-
-def test_sse_kms_key_id(dvc):
-    fs = S3FileSystem(url=url, sse_kms_key_id="key")
-    assert fs.fs_args["s3_additional_kwargs"]["SSEKMSKeyId"] == "key"
-
-
-def test_key_id_and_secret(dvc):
-    fs = S3FileSystem(
-        url=url,
-        access_key_id=key_id,
-        secret_access_key=key_secret,
-        session_token=session_token,
-    )
-    assert fs.fs_args["key"] == key_id
-    assert fs.fs_args["secret"] == key_secret
-    assert fs.fs_args["token"] == session_token

tests/unit/machine/test_machine.py

@@ -1,36 +0,0 @@
-import pytest
-
-
-def test_validate_name():
-    from dvc.exceptions import InvalidArgumentError
-    from dvc.machine import RESERVED_NAMES, validate_name
-
-    for name in RESERVED_NAMES:
-        with pytest.raises(InvalidArgumentError):
-            validate_name(name)
-
-
-@pytest.mark.parametrize("cloud", ["aws", "azure"])
-def test_get_config_and_backend(tmp_dir, dvc, cloud):
-    from dvc.machine.backend.terraform import TerraformBackend
-
-    name = "foo"
-    with dvc.config.edit() as conf:
-        conf["machine"][name] = {"cloud": cloud}
-    config, backend = dvc.machine.get_config_and_backend(name)
-    assert config == {"cloud": cloud, "name": name}
-    assert isinstance(backend, TerraformBackend)
-
-
-def test_get_config_and_backend_nonexistent(tmp_dir, dvc):
-    from dvc.config import MachineNotFoundError
-
-    with pytest.raises(MachineNotFoundError):
-        dvc.machine.get_config_and_backend("foo")
-
-
-def test_get_config_and_backend_default(tmp_dir, dvc):
-    from dvc.config import NoMachineError
-
-    with pytest.raises(NoMachineError):
-        dvc.machine.get_config_and_backend()

tests/unit/repo/test_scm_context.py

@@ -100,13 +100,14 @@ def test_scm_context_on_no_files_to_track(caplog, scm_context, autostage, quiet)
     assert not caplog.text
 
 
-def test_scm_context_remind_to_track(caplog, scm_context):
-    with scm_context() as context:
+@pytest.mark.parametrize("quiet", [True, False])
+def test_scm_context_remind_to_track(caplog, scm_context, quiet):
+    with scm_context(quiet=quiet) as context:
         context.track_file("foo")
         context.track_file("lorem ipsum")
         assert context.files_to_track == {"foo", "lorem ipsum"}
 
-    if isinstance(scm_context.scm, NoSCM):
+    if quiet or isinstance(scm_context.scm, NoSCM):
         assert not caplog.text
     else:
         assert "To track the changes with git, run:" in caplog.text

tests/unit/scm/test_scm.py

@@ -76,6 +76,12 @@ def test_iter_revs(
                 commit_time_offset=0,
                 message="dummy",
                 parents=["dummy"],
+                committer_name="dummy",
+                committer_email="dummy",
+                author_name="dummy",
+                author_email="dummy",
+                author_time=datetime(2022, 6, 28).timestamp(),
+                author_time_offset=0,
             )
         if rev == rev_old:
             raise SCMError
@@ -85,6 +91,12 @@ def test_iter_revs(
             commit_time_offset=0,
             message="dummy",
             parents=["dummy"],
+            committer_name="dummy",
+            committer_email="dummy",
+            author_name="dummy",
+            author_email="dummy",
+            author_time=datetime(2022, 6, 28).timestamp(),
+            author_time_offset=0,
         )
 
     mocker.patch(

tests/unit/test_config.py

@@ -21,6 +21,29 @@ def test_to_relpath(path, expected):
     assert Config._to_relpath(os.path.join(".", "config"), path) == expected
 
 
+@pytest.mark.parametrize(
+    "path, expected",
+    [
+        ("cache", os.path.abspath(os.path.join("conf_dir", "cache"))),
+        ("dir/cache", os.path.abspath(os.path.join("conf_dir", "dir", "cache"))),
+        ("../cache", os.path.abspath("cache")),
+        (os.getcwd(), os.getcwd()),
+        ("ssh://some/path", "ssh://some/path"),
+    ],
+)
+def test_resolve(path, expected):
+    conf_dir = os.path.abspath("conf_dir")
+    assert Config._resolve(conf_dir, path) == expected
+
+
+def test_resolve_homedir():
+    # NOTE: our test suit patches $HOME, but that only works within the
+    # test itself, so we can't use expanduser in @parametrize here.
+    conf_dir = os.path.abspath("conf_dir")
+    expected = os.path.expanduser(os.path.join("~", "cache"))
+    assert Config._resolve(conf_dir, "~/cache") == expected
+
+
 def test_get_fs(tmp_dir, scm):
     tmp_dir.scm_gen("foo", "foo", commit="add foo")