analytics: fix mac not sending reports (#10026)

Bumps [pylint](https://github.com/pylint-dev/pylint) from 2.17.6 to 3.0.1.
- [Release notes](https://github.com/pylint-dev/pylint/releases)
- [Commits](https://github.com/pylint-dev/pylint/compare/v2.17.6...v3.0.1)

---
updated-dependencies:
- dependency-name: pylint
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <[email protected]>

.flake8

@@ -1,19 +1,20 @@
 [flake8]
-ignore=
+max_line_length = 88
+extend-ignore=
     # Whitespace before ':'
     E203
-    # Too many leading '#' for block comment
-    E266
     # Line break occurred before a binary operator
     W503
     # unindexed parameters in the str.format, see:
     # https://pypi.org/project/flake8-string-format/
     P1
-    # use `!r` instead of surrounding by quotes
-    B028
-max_line_length = 88
 max-complexity = 10
-select = B,C,E,F,W,T4,B902,T,P,TYP
 show_source = true
 count = true
 min_python_version = 3.8.0
+per-file-ignores =
+    dvc/testing/benchmarks/conftest.py:F401
+    dvc/testing/conftest.py:F401
+    dvc/testing/plugin.py:F401
+    tests/conftest.py:F401
+    tests/func/experiments/executor/test_ssh.py:F401

.github/workflows/build.yaml

@@ -1,21 +1,25 @@
-name: Build package
+name: Build and upload package
 on:
   push:
     branches:
     - main
   release:
-    types: [released, prereleased]
+    types:
+    - published
   workflow_dispatch:
 
+env:
+  FORCE_COLOR: 1
+
 permissions:
   contents: read
-  id-token: write
 
 jobs:
-  pip:
+  build:
+    name: Build & Verify package
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
 
@@ -24,24 +28,65 @@ jobs:
         python-version: 3.8
 
     - run: python -m pip install --upgrade pip setuptools_scm build twine
+
     - name: Force version for Test PyPI upload
       if: ${{ !startsWith(github.ref, 'refs/tags') }}
       run: echo version=$(python -m setuptools_scm | awk -F+ '{print $1}' | tail -1) >> $GITHUB_ENV
 
-    - run: |
-        echo 'PKG = "pip"'>dvc/_build.py
-        python -m build
+    - run: echo 'PKG = "pip"'>dvc/_build.py
+    - run: python -m build
       env:
-        SETUPTOOLS_SCM_PRETEND_VERSION: ${{ env.version }}
+        SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DVC: ${{ env.version }}
     - run: twine check --strict dist/*
 
-    - name: Publish to PyPI
-      if: github.event_name == 'release'
-      uses: pypa/gh-action-pypi-publish@release/v1
+    - uses: actions/upload-artifact@v3
+      with:
+        name: Packages
+        path: dist/
 
-    - name: Publish to Test PyPI
-      if: ${{ github.event_name == 'release' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }}
+  test-pypi-publish:
+    name: Publish dev package to test.pypi.org
+    runs-on: ubuntu-latest
+    needs: build
+    if: ${{ github.event.action == 'published' || (github.repository == 'iterative/dvc' && github.event_name == 'push' && github.ref == 'refs/heads/main') }}
+
+    environment:
+      name: test-pypi
+      url: https://test.pypi.org/p/dvc
+
+    permissions:
+      id-token: write
+
+    steps:
+    - uses: actions/download-artifact@v3
+      with:
+        name: Packages
+        path: dist
+
+    - name: Upload package to Test PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         repository-url: https://test.pypi.org/legacy/
         skip-existing: true
+
+  pypi-publish:
+    name: Publish released package to pypi.org
+    runs-on: ubuntu-latest
+    needs: build
+    if: github.event.action == 'published'
+
+    environment:
+      name: pypi
+      url: https://pypi.org/p/dvc
+
+    permissions:
+      id-token: write
+
+    steps:
+    - uses: actions/download-artifact@v3
+      with:
+        name: Packages
+        path: dist
+
+    - name: Upload package to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/codeql.yml

@@ -28,7 +28,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2

.github/workflows/plugin_tests.yaml

@@ -23,11 +23,11 @@ jobs:
         plugin: ["dvc-s3"]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         path: dvc
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         repository: iterative/${{ matrix.plugin }}
         ref: main

.github/workflows/tests.yaml

@@ -26,7 +26,7 @@ jobs:
         pyv: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Set up Python
@@ -66,7 +66,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Slack Notification
-      uses: rtCamp/[email protected].0
+      uses: rtCamp/[email protected].1
       env:
         SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         SLACK_COLOR: ${{ job.status }}

.pre-commit-config.yaml

@@ -23,19 +23,8 @@ repos:
       - id: mixed-line-ending
       - id: sort-simple-yaml
       - id: trailing-whitespace
-  - repo: https://github.com/lovesegfault/beautysh
-    rev: v6.2.1
-    hooks:
-      - id: beautysh
-        args: ["-i", "2"]
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.9.0
-    hooks:
-      - id: pyupgrade
-        args:
-          - --py38-plus
   - repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 23.9.1
     hooks:
       - id: black
   - repo: https://github.com/PyCQA/isort
@@ -43,22 +32,20 @@ repos:
     hooks:
       - id: isort
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.0.278'
+    rev: 'v0.0.291'
     hooks:
       - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
   - repo: https://github.com/codespell-project/codespell
     rev: v2.2.5
     hooks:
       - id: codespell
         additional_dependencies: ["tomli"]
   - repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
       - id: flake8
         additional_dependencies:
-          - flake8-bugbear
-          - flake8-comprehensions
-          - flake8-debugger
           - flake8-string-format
           - flake8-typing-imports
   - repo: https://github.com/PyCQA/bandit

dvc/__pyinstaller/hook-dvc.py

@@ -22,6 +22,7 @@ hiddenimports = [
     "dvc_hdfs",
     "dvc_oss",
     "dvc_s3",
+    "dvc_ssh",
     "dvc_webdav",
     "dvc_webhdfs",
     # https://github.com/pypa/setuptools/issues/1963

dvc/api/__init__.py

@@ -1,5 +1,6 @@
 from dvc.fs.dvc import _DVCFileSystem as DVCFileSystem
 
+from .artifacts import artifacts_show
 from .data import open  # pylint: disable=redefined-builtin
 from .data import get_url, read
 from .experiments import exp_save, exp_show
@@ -10,6 +11,7 @@ __all__ = [
     "all_branches",
     "all_commits",
     "all_tags",
+    "artifacts_show",
     "exp_save",
     "exp_show",
     "get_url",

dvc/api/artifacts.py

@@ -0,0 +1,47 @@
+from typing import Any, Dict, Optional
+
+from dvc.repo import Repo
+
+
+def artifacts_show(
+    name: str,
+    version: Optional[str] = None,
+    stage: Optional[str] = None,
+    repo: Optional[str] = None,
+) -> Dict[str, str]:
+    """
+    Return path and Git revision for an artifact in a DVC project.
+
+    The resulting path and revision can be used in conjunction with other dvc.api
+    calls to open and read the artifact.
+
+    Args:
+        name (str): name of the artifact to open.
+        version (str, optional): version of the artifact to open. Defaults to
+            the latest version.
+        stage (str, optional): name of the model registry stage.
+        repo: (str, optional): path or URL for the DVC repo.
+
+    Returns:
+        Dictionary of the form:
+            {
+                "rev": ...,
+                "path": ...,
+            }
+
+    Raises:
+        dvc.exceptions.ArtifactNotFoundError: The specified artifact was not found in
+            the repo.
+    """
+    if version and stage:
+        raise ValueError("Artifact version and stage are mutually exclusive.")
+
+    repo_kwargs: Dict[str, Any] = {
+        "subrepos": True,
+        "uninitialized": True,
+    }
+    with Repo.open(repo, **repo_kwargs) as _repo:
+        rev = _repo.artifacts.get_rev(name, version=version, stage=stage)
+        with _repo.switch(rev):
+            path = _repo.artifacts.get_path(name)
+        return {"rev": rev, "path": path}

dvc/api/data.py

@@ -67,7 +67,7 @@ class _OpenContextManager(GCM):
         raise AttributeError("dvc.api.open() should be used in a with statement.")
 
 
-def open(  # noqa, pylint: disable=redefined-builtin
+def open(  # noqa: A001, pylint: disable=redefined-builtin
     path: str,
     repo: Optional[str] = None,
     rev: Optional[str] = None,
@@ -75,6 +75,7 @@ def open(  # noqa, pylint: disable=redefined-builtin
     mode: str = "r",
     encoding: Optional[str] = None,
     config: Optional[Dict[str, Any]] = None,
+    remote_config: Optional[Dict[str, Any]] = None,
 ):
     """
     Opens a file tracked in a DVC project.
@@ -118,6 +119,9 @@ def open(  # noqa, pylint: disable=redefined-builtin
             Mirrors the namesake parameter in builtin `open()`_.
         config(dict, optional): config to be passed to the DVC repository.
             Defaults to None.
+        remote_config(dict, optional): remote config to be passed to the DVC
+            repository.
+            Defaults to None.
 
     Returns:
         _OpenContextManager: A context manager that generatse a corresponding
@@ -214,22 +218,27 @@ def open(  # noqa, pylint: disable=redefined-builtin
         "mode": mode,
         "encoding": encoding,
         "config": config,
+        "remote_config": remote_config,
     }
     return _OpenContextManager(_open, args, kwargs)
 
 
-def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None, config=None):
-    if remote:
-        if config is not None:
-            raise ValueError(
-                "can't specify both `remote` and `config` at the same time"
-            )
-        config = {"core": {"remote": remote}}
-
+def _open(
+    path,
+    repo=None,
+    rev=None,
+    remote=None,
+    mode="r",
+    encoding=None,
+    config=None,
+    remote_config=None,
+):
     repo_kwargs: Dict[str, Any] = {
         "subrepos": True,
         "uninitialized": True,
+        "remote": remote,
         "config": config,
+        "remote_config": remote_config,
     }
 
     with Repo.open(repo, rev=rev, **repo_kwargs) as _repo:
@@ -265,7 +274,16 @@ def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None, confi
                 raise DvcIsADirectoryError(f"'{path}' is a directory") from exc
 
 
-def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None, config=None):
+def read(
+    path,
+    repo=None,
+    rev=None,
+    remote=None,
+    mode="r",
+    encoding=None,
+    config=None,
+    remote_config=None,
+):
     """
     Returns the contents of a tracked file (by DVC or Git). For Git repos, HEAD
     is used unless a rev argument is supplied. The default remote is tried
@@ -279,5 +297,6 @@ def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None, config
         mode=mode,
         encoding=encoding,
         config=config,
+        remote_config=remote_config,
     ) as fd:
         return fd.read()

dvc/api/show.py

@@ -7,10 +7,6 @@ from funcy import first
 from dvc.repo import Repo
 
 
-def _onerror_raise(exception: Exception, *args, **kwargs):
-    raise exception
-
-
 def _postprocess(results):
     processed: Dict[str, Dict] = {}
     for rev, rev_data in results.items():
@@ -23,7 +19,6 @@ def _postprocess(results):
         for file_data in rev_data["data"].values():
             for k in file_data["data"]:
                 counts[k] += 1
-
         for file_name, file_data in rev_data["data"].items():
             to_merge = {
                 (k if counts[k] == 1 else f"{file_name}:{k}"): v
@@ -138,13 +133,17 @@ def metrics_show(
     .. _Git revision:
         https://git-scm.com/docs/revisions
     """
+    from dvc.repo.metrics.show import to_relpath
 
     with Repo.open(repo, config=config) as _repo:
         metrics = _repo.metrics.show(
             targets=targets,
             revs=rev if rev is None else [rev],
-            onerror=_onerror_raise,
+            on_error="raise",
         )
+        metrics = {
+            k: to_relpath(_repo.fs, _repo.root_dir, v) for k, v in metrics.items()
+        }
 
     metrics = _postprocess(metrics)
 
@@ -382,6 +381,8 @@ def params_show(
         https://git-scm.com/docs/revisions
 
     """
+    from dvc.repo.metrics.show import to_relpath
+
     if isinstance(stages, str):
         stages = [stages]
 
@@ -389,10 +390,11 @@ def params_show(
         params = _repo.params.show(
             revs=rev if rev is None else [rev],
             targets=targets,
-            deps=deps,
-            onerror=_onerror_raise,
+            deps_only=deps,
+            on_error="raise",
             stages=stages,
         )
+        params = {k: to_relpath(_repo.fs, _repo.root_dir, v) for k, v in params.items()}
 
     params = _postprocess(params)
 

dvc/cachemgr.py

@@ -36,17 +36,15 @@ def _get_odb(
 class CacheManager:
     CACHE_DIR = "cache"
     FILES_DIR = "files"
+    FS_DIR = "fs"
 
     def __init__(self, repo):
         self._repo = repo
         self.config = config = repo.config["cache"]
         self._odb = {}
 
-        default = None
-        if repo and repo.local_dvc_dir:
-            default = os.path.join(repo.local_dvc_dir, self.CACHE_DIR)
-
         local = config.get("local")
+        default = self.default_local_cache_dir
 
         if local:
             settings = {"name": local}
@@ -76,6 +74,21 @@ class CacheManager:
         legacy_odb = _get_odb(repo, settings, hash_name="md5-dos2unix", **kwargs)
         self._odb["legacy"] = legacy_odb
 
+    @property
+    def fs_cache(self):
+        """Filesystem-based cache.
+
+        Currently used as a temporary location to download files that we don't
+        yet have a regular oid (e.g. md5) for.
+        """
+        from dvc_data.index import FileStorage
+
+        return FileStorage(
+            key=(),
+            fs=self.local.fs,
+            path=self.local.fs.path.join(self.default_local_cache_dir, self.FS_DIR),
+        )
+
     def _init_odb(self, schemes):
         for scheme in schemes:
             remote = self.config.get(scheme)
@@ -103,6 +116,13 @@ class CacheManager:
         """
         return self.legacy.path
 
+    @property
+    def default_local_cache_dir(self) -> Optional[str]:
+        repo = self._repo
+        if repo and repo.local_dvc_dir:
+            return os.path.join(repo.local_dvc_dir, self.CACHE_DIR)
+        return None
+
 
 def migrate_2_to_3(repo: "Repo", dry: bool = False):
     """Migrate legacy 2.x objects to 3.x cache.

dvc/cli/__init__.py

@@ -230,7 +230,7 @@ def main(argv=None):  # noqa: C901, PLR0912, PLR0915
         logger.exception("")
     except DvcParserError:
         ret = 254
-    except Exception as exc:  # noqa, pylint: disable=broad-except
+    except Exception as exc:  # noqa: BLE001, pylint: disable=broad-except
         # pylint: disable=no-member
         ret = _log_exceptions(exc) or 255
 

dvc/cli/parser.py

@@ -7,6 +7,7 @@ from functools import lru_cache
 from dvc import __version__
 from dvc.commands import (
     add,
+    artifacts,
     cache,
     check_ignore,
     checkout,
@@ -31,7 +32,6 @@ from dvc.commands import (
     install,
     ls,
     ls_url,
-    machine,
     metrics,
     move,
     params,
@@ -89,8 +89,8 @@ COMMANDS = [
     stage,
     experiments,
     check_ignore,
-    machine,
     data,
+    artifacts,
 ]
 
 

dvc/cli/utils.py

@@ -1,3 +1,27 @@
+import argparse
+
+
+class DictAction(argparse.Action):
+    def __call__(self, parser, args, values, option_string=None):  # noqa: ARG002
+        d = getattr(args, self.dest) or {}
+
+        if isinstance(values, list):
+            kvs = values
+        else:
+            kvs = [values]
+
+        for kv in kvs:
+            key, value = kv.split("=", 1)
+            if not value:
+                raise argparse.ArgumentError(
+                    self,
+                    f'Could not parse argument "{values}" as k1=v1 k2=v2 ... format',
+                )
+            d[key] = value
+
+        setattr(args, self.dest, d)
+
+
 def fix_subparsers(subparsers):
     """Workaround for bug in Python 3. See more info at:
     https://bugs.python.org/issue16308

dvc/commands/artifacts.py

@@ -0,0 +1,133 @@
+import argparse
+import logging
+
+from dvc.cli import completion
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import DictAction, append_doc_link, fix_subparsers
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+class CmdArtifactsGet(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo.artifacts import Artifacts
+        from dvc.scm import CloneError
+        from dvc.ui import ui
+
+        try:
+            count, out = Artifacts.get(
+                self.args.url,
+                name=self.args.name,
+                version=self.args.rev,
+                stage=self.args.stage,
+                force=self.args.force,
+                config=self.args.config,
+                remote=self.args.remote,
+                remote_config=self.args.remote_config,
+                out=self.args.out,
+            )
+            ui.write(f"Downloaded {count} file(s) to '{out}'")
+            return 0
+        except CloneError:
+            logger.exception("failed to get '%s'", self.args.name)
+            return 1
+        except DvcException:
+            logger.exception(
+                "failed to get '%s' from '%s'", self.args.name, self.args.url
+            )
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    ARTIFACTS_HELP = "DVC model registry artifact commands."
+
+    artifacts_parser = subparsers.add_parser(
+        "artifacts",
+        parents=[parent_parser],
+        description=append_doc_link(ARTIFACTS_HELP, "artifacts"),
+        help=ARTIFACTS_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    artifacts_subparsers = artifacts_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc artifacts CMD --help` to display command-specific help.",
+    )
+    fix_subparsers(artifacts_subparsers)
+
+    ARTIFACTS_GET_HELP = "Download an artifact from a DVC project."
+    get_parser = artifacts_subparsers.add_parser(
+        "get",
+        parents=[parent_parser],
+        description=append_doc_link(ARTIFACTS_GET_HELP, "artifacts/get"),
+        help=ARTIFACTS_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    get_parser.add_argument("url", help="Location of DVC repository to download from")
+    get_parser.add_argument(
+        "name", help="Name of artifact in the repository"
+    ).complete = completion.FILE
+    get_parser.add_argument(
+        "--rev",
+        nargs="?",
+        help="Artifact version",
+        metavar="<version>",
+    )
+    get_parser.add_argument(
+        "--stage",
+        nargs="?",
+        help="Artifact stage",
+        metavar="<stage>",
+    )
+    get_parser.add_argument(
+        "-o",
+        "--out",
+        nargs="?",
+        help="Destination path to download artifact to",
+        metavar="<path>",
+    ).complete = completion.DIR
+    get_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+        ),
+        metavar="<number>",
+    )
+    get_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Override local file or folder if exists.",
+    )
+    get_parser.add_argument(
+        "--config",
+        type=str,
+        help=(
+            "Path to a config file that will be merged with the config "
+            "in the target repository."
+        ),
+    )
+    get_parser.add_argument(
+        "--remote",
+        type=str,
+        help=(
+            "Remote name to set as a default in the target repository "
+            "(only applicable when downloading from DVC remote)."
+        ),
+    )
+    get_parser.add_argument(
+        "--remote-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help=(
+            "Remote config options to merge with a remote's config (default or one "
+            "specified by '--remote') in the target repository (only applicable "
+            "when downloading from DVC remote)."
+        ),
+    )
+    get_parser.set_defaults(func=CmdArtifactsGet)

dvc/commands/cache.py

@@ -40,8 +40,11 @@ class CmdCacheDir(CmdConfig):
 class CmdCacheMigrate(CmdBase):
     def run(self):
         from dvc.cachemgr import migrate_2_to_3
+        from dvc.repo.commit import commit_2_to_3
 
         migrate_2_to_3(self.repo, dry=self.args.dry)
+        if self.args.dvc_files:
+            commit_2_to_3(self.repo, dry=self.args.dry)
         return 0
 
 
@@ -102,6 +105,14 @@ def add_parser(subparsers, parent_parser):
         help=CACHE_MIGRATE_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    cache_migrate_parser.add_argument(
+        "--dvc-files",
+        help=(
+            "Migrate entries in all existing DVC files in the repository "
+            "to the DVC 3.0 format."
+        ),
+        action="store_true",
+    )
     cache_migrate_parser.add_argument(
         "--dry",
         help=(

dvc/commands/checkout.py

@@ -37,6 +37,7 @@ class CmdCheckout(CmdBase):
                 force=self.args.force,
                 relink=self.args.relink,
                 recursive=self.args.recursive,
+                allow_missing=self.args.allow_missing,
             )
         except CheckoutError as _exc:
             exc = _exc
@@ -101,6 +102,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Recreate links or copies from cache to workspace.",
     )
+    checkout_parser.add_argument(
+        "--allow-missing",
+        action="store_true",
+        default=False,
+        help="Ignore errors if some of the files or directories are missing.",
+    )
     checkout_parser.add_argument(
         "targets",
         nargs="*",

dvc/commands/commit.py

@@ -47,7 +47,10 @@ def add_parser(subparsers, parent_parser):
         "--force",
         action="store_true",
         default=False,
-        help="Commit even if hash value for dependencies/outputs changed.",
+        help=(
+            "Commit data even if hash values for dependencies or "
+            "outputs did not change."
+        ),
     )
     commit_parser.add_argument(
         "-d",

dvc/commands/completion.py

@@ -9,6 +9,9 @@ from dvc.ui import ui
 logger = logging.getLogger(__name__)
 
 
+SUPPORTED_SHELLS = ["bash", "zsh"]
+
+
 class CmdCompletion(CmdBaseNoRepo):
     def run(self):
         import shtab
@@ -35,6 +38,6 @@ def add_parser(subparsers, parent_parser):
         "--shell",
         help="Shell syntax for completions.",
         default="bash",
-        choices=["bash", "zsh"],
+        choices=SUPPORTED_SHELLS,
     )
     completion_parser.set_defaults(func=CmdCompletion)

dvc/commands/data_sync.py

@@ -14,6 +14,10 @@ class CmdDataBase(CmdBase):
         from dvc.utils.humanize import get_summary
 
         default_msg = "Everything is up to date."
+
+        if not self.args.remote and not self.repo.config["core"].get("remote"):
+            ui.warn("No remote provided and no default remote set.")
+
         ui.write(get_summary(stats.items()) or default_msg)
 
 

dvc/commands/experiments/__init__.py

@@ -12,6 +12,7 @@ from dvc.commands.experiments import (
     push,
     queue_worker,
     remove,
+    rename,
     run,
     save,
     show,
@@ -28,6 +29,7 @@ SUB_COMMANDS = [
     push,
     queue_worker,
     remove,
+    rename,
     run,
     save,
     show,

dvc/commands/experiments/apply.py

@@ -4,21 +4,13 @@ import logging
 from dvc.cli import completion
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
-from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsApply(CmdBase):
     def run(self):
-        if not self.args.force:
-            ui.write(
-                "The --no-force option is deprecated and will be removed in a future"
-                " DVC release. To revert the result of 'exp apply', run:\n"
-                "\n\tgit reset --hard\n"
-                "\tgit stash apply refs/exps/apply/stash\n"
-            )
-        self.repo.experiments.apply(self.args.experiment)
+        self.repo.experiments.apply(self.args.experiment, force=self.args.force)
 
         return 0
 
@@ -33,10 +25,11 @@ def add_parser(experiments_subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     experiments_apply_parser.add_argument(
-        "--no-force",
-        action="store_false",
-        dest="force",
-        help="Fail if this command would overwrite conflicting changes.",
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Do not prompt when removing working directory files.",
     )
     experiments_apply_parser.add_argument(
         "experiment", help="Experiment to be applied."

dvc/commands/experiments/exec_run.py

@@ -49,7 +49,7 @@ def add_parser(experiments_subparsers, parent_parser):
         ),
     )
     exec_run_parser.add_argument(
-        "-M",
+        "-m",
         "--message",
         type=str,
         default=None,

dvc/commands/experiments/ls.py

@@ -23,15 +23,22 @@ class CmdExperimentsList(CmdBase):
             git_remote=git_remote,
         )
 
+        from dvc.repo.experiments.utils import describe
+        from dvc.scm import Git
+
+        if name_only or sha_only:
+            names = {}
+        else:
+            assert isinstance(self.repo.scm, Git)
+            names = describe(
+                self.repo.scm,
+                (baseline for baseline in exps),
+                logger=logger,
+            )
+
         for baseline in exps:
             if not (name_only or sha_only):
-                tag_base = "refs/tags/"
-                branch_base = "refs/heads/"
-                name = baseline[:7]
-                if baseline.startswith(tag_base):
-                    name = baseline[len(tag_base) :]
-                elif baseline.startswith(branch_base):
-                    name = baseline[len(branch_base) :]
+                name = names.get(baseline) or baseline[:7]
                 ui.write(f"{name}:")
             for exp_name, rev in exps[baseline]:
                 if name_only:
@@ -52,6 +59,7 @@ def add_parser(experiments_subparsers, parent_parser):
     EXPERIMENTS_LIST_HELP = "List local and remote experiments."
     experiments_list_parser = experiments_subparsers.add_parser(
         "list",
+        aliases=["ls"],
         parents=[parent_parser],
         description=append_doc_link(EXPERIMENTS_LIST_HELP, "exp/list"),
         help=EXPERIMENTS_LIST_HELP,

dvc/commands/experiments/remove.py

@@ -50,6 +50,7 @@ def add_parser(experiments_subparsers, parent_parser):
     EXPERIMENTS_REMOVE_HELP = "Remove experiments."
     experiments_remove_parser = experiments_subparsers.add_parser(
         "remove",
+        aliases=["rm"],
         parents=[parent_parser],
         description=append_doc_link(EXPERIMENTS_REMOVE_HELP, "exp/remove"),
         help=EXPERIMENTS_REMOVE_HELP,

dvc/commands/experiments/rename.py

@@ -0,0 +1,68 @@
+import argparse
+import logging
+
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.exceptions import InvalidArgumentError
+from dvc.ui import ui
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsRename(CmdBase):
+    def run(self):
+        from dvc.utils import humanize
+
+        if not (self.args.experiment and self.args.name):
+            raise InvalidArgumentError(
+                "An experiment to rename and a new experiment name are required."
+            )
+        renamed = self.repo.experiments.rename(
+            exp_name=self.args.experiment,
+            new_name=self.args.name,
+            git_remote=self.args.git_remote,
+            force=self.args.force,
+        )
+        if renamed:
+            ui.write(f"Renamed experiments: {humanize.join(map(repr, renamed))}")
+        else:
+            ui.write("No experiments to rename.")
+
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+    EXPERIMENTS_RENAME_HELP = "Rename experiments."
+    experiments_rename_parser = experiments_subparsers.add_parser(
+        "rename",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_RENAME_HELP, "exp/rename"),
+        help=EXPERIMENTS_RENAME_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    rename_group = experiments_rename_parser.add_mutually_exclusive_group()
+    rename_group.add_argument(
+        "-g",
+        "--git-remote",
+        metavar="<git_remote>",
+        help="Name or URL of the Git remote to rename the experiment from",
+    )
+    experiments_rename_parser.add_argument(
+        "experiment",
+        help="Experiment to rename.",
+        nargs="?",
+        metavar="<experiment>",
+    )
+    experiments_rename_parser.add_argument(
+        "name",
+        help="New name for the experiment.",
+        metavar="<name>",
+    )
+    experiments_rename_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Replace experiment if it already exists.",
+    )
+    experiments_rename_parser.set_defaults(func=CmdExperimentsRename)

dvc/commands/experiments/run.py

@@ -17,7 +17,6 @@ class CmdExperimentsRun(CmdRepro):
             jobs=self.args.jobs,
             params=self.args.set_param,
             tmp_dir=self.args.tmp_dir,
-            machine=self.args.machine,
             copy_paths=self.args.copy_paths,
             message=self.args.message,
             **self._common_kwargs,
@@ -27,7 +26,7 @@ class CmdExperimentsRun(CmdRepro):
 
 
 def add_parser(experiments_subparsers, parent_parser):
-    EXPERIMENTS_RUN_HELP = "Run or resume an experiment."
+    EXPERIMENTS_RUN_HELP = "Run an experiment."
     experiments_run_parser = experiments_subparsers.add_parser(
         "run",
         parents=[parent_parser],
@@ -90,15 +89,6 @@ def _add_run_common(parser):
             "your workspace."
         ),
     )
-    parser.add_argument(
-        "--machine",
-        default=None,
-        help=argparse.SUPPRESS,
-        # help=(
-        #     "Run this experiment on the specified 'dvc machine' instance."
-        # )
-        # metavar="<name>",
-    )
     parser.add_argument(
         "-C",
         "--copy-paths",
@@ -116,3 +106,8 @@ def _add_run_common(parser):
         default=None,
         help="Custom commit message to use when committing the experiment.",
     )
+    parser.add_argument(
+        "-M",  # obsolete
+        dest="message",
+        help=argparse.SUPPRESS,
+    )

dvc/commands/experiments/save.py

@@ -27,10 +27,6 @@ class CmdExperimentsSave(CmdBase):
         else:
             name = self.repo.experiments.get_exact_name([ref])[ref]
             ui.write(f"Experiment has been saved as: {name}")
-            ui.write(
-                "\nTo promote an experiment to a Git branch run:\n\n"
-                "\tdvc exp branch <exp> <branch>\n"
-            )
 
         return 0
 
@@ -76,10 +72,15 @@ def add_parser(experiments_subparsers, parent_parser):
         metavar="<path>",
     )
     save_parser.add_argument(
-        "-M",
+        "-m",
         "--message",
         type=str,
         default=None,
         help="Custom commit message to use when committing the experiment.",
     )
+    save_parser.add_argument(
+        "-M",  # obsolete
+        dest="message",
+        help=argparse.SUPPRESS,
+    )
     save_parser.set_defaults(func=CmdExperimentsSave)

dvc/commands/gc.py

@@ -81,6 +81,7 @@ class CmdGC(CmdBase):
             rev=self.args.rev,
             num=self.args.num,
             not_in_remote=self.args.not_in_remote,
+            dry=self.args.dry,
         )
         return 0
 
@@ -213,4 +214,10 @@ def add_parser(subparsers, parent_parser):
         ),
         metavar="<paths>",
     )
+    gc_parser.add_argument(
+        "--dry",
+        action="store_true",
+        default=False,
+        help=("Only print what would get removed without actually removing."),
+    )
     gc_parser.set_defaults(func=CmdGC)

dvc/commands/get.py

@@ -3,7 +3,7 @@ import logging
 
 from dvc.cli import completion
 from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import append_doc_link
+from dvc.cli.utils import DictAction, append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -38,6 +38,8 @@ class CmdGet(CmdBaseNoRepo):
                 jobs=self.args.jobs,
                 force=self.args.force,
                 config=self.args.config,
+                remote=self.args.remote,
+                remote_config=self.args.remote_config,
             )
             return 0
         except CloneError:
@@ -111,4 +113,19 @@ def add_parser(subparsers, parent_parser):
             "in the target repository."
         ),
     )
+    get_parser.add_argument(
+        "--remote",
+        type=str,
+        help="Remote name to set as a default in the target repository.",
+    )
+    get_parser.add_argument(
+        "--remote-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help=(
+            "Remote config options to merge with a remote's config (default or one "
+            "specified by '--remote') in the target repository."
+        ),
+    )
     get_parser.set_defaults(func=CmdGet)

dvc/commands/get_url.py

@@ -3,7 +3,7 @@ import logging
 
 from dvc.cli import completion
 from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import append_doc_link
+from dvc.cli.utils import DictAction, append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -11,6 +11,7 @@ logger = logging.getLogger(__name__)
 
 class CmdGetUrl(CmdBaseNoRepo):
     def run(self):
+        from dvc.config import Config
         from dvc.repo import Repo
 
         try:
@@ -19,6 +20,8 @@ class CmdGetUrl(CmdBaseNoRepo):
                 out=self.args.out,
                 jobs=self.args.jobs,
                 force=self.args.force,
+                fs_config=self.args.fs_config,
+                config=Config.from_cwd(),
             )
             return 0
         except DvcException:
@@ -58,4 +61,11 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Override local file or folder if exists.",
     )
+    get_parser.add_argument(
+        "--fs-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help="Config options for the target url.",
+    )
     get_parser.set_defaults(func=CmdGetUrl)

dvc/commands/imp.py

@@ -3,7 +3,7 @@ import logging
 
 from dvc.cli import completion
 from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+from dvc.cli.utils import DictAction, append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -23,6 +23,9 @@ class CmdImport(CmdBase):
                 no_download=self.args.no_download,
                 jobs=self.args.jobs,
                 config=self.args.config,
+                remote=self.args.remote,
+                remote_config=self.args.remote_config,
+                force=self.args.force,
             )
         except CloneError:
             logger.exception("failed to import '%s'", self.args.path)
@@ -63,6 +66,13 @@ def add_parser(subparsers, parent_parser):
         help="Destination path to download files to",
         metavar="<path>",
     ).complete = completion.DIR
+    import_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Override destination file or folder if exists.",
+    )
     import_parser.add_argument(
         "--rev",
         nargs="?",
@@ -103,4 +113,19 @@ def add_parser(subparsers, parent_parser):
             "in the target repository."
         ),
     )
+    import_parser.add_argument(
+        "--remote",
+        type=str,
+        help="Remote name to set as a default in the target repository.",
+    )
+    import_parser.add_argument(
+        "--remote-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help=(
+            "Remote config options to merge with a remote's config (default or one "
+            "specified by '--remote') in the target repository."
+        ),
+    )
     import_parser.set_defaults(func=CmdImport)

dvc/commands/imp_url.py

@@ -3,7 +3,7 @@ import logging
 
 from dvc.cli import completion
 from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+from dvc.cli.utils import DictAction, append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -22,6 +22,7 @@ class CmdImportUrl(CmdBase):
                 jobs=self.args.jobs,
                 force=self.args.force,
                 version_aware=self.args.version_aware,
+                fs_config=self.args.fs_config,
             )
         except DvcException:
             logger.exception(
@@ -114,4 +115,11 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Import using cloud versioning. Implied if the URL contains a version ID.",
     )
+    import_parser.add_argument(
+        "--fs-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help="Config options for the target url.",
+    )
     import_parser.set_defaults(func=CmdImportUrl)

dvc/commands/ls/__init__.py

@@ -3,7 +3,7 @@ import logging
 
 from dvc.cli import completion
 from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import append_doc_link
+from dvc.cli.utils import DictAction, append_doc_link
 from dvc.commands.ls.ls_colors import LsColors
 from dvc.exceptions import DvcException
 from dvc.ui import ui
@@ -11,7 +11,18 @@ from dvc.ui import ui
 logger = logging.getLogger(__name__)
 
 
-def _prettify(entries, with_color=False):
+def _format_entry(entry, fmt):
+    from dvc.utils.humanize import naturalsize
+
+    size = entry.get("size")
+    if size is None:
+        size = ""
+    else:
+        size = naturalsize(size)
+    return size, fmt(entry)
+
+
+def show_entries(entries, with_color=False, with_size=False):
     if with_color:
         ls_colors = LsColors()
         fmt = ls_colors.format
@@ -20,7 +31,12 @@ def _prettify(entries, with_color=False):
         def fmt(entry):
             return entry["path"]
 
-    return [fmt(entry) for entry in entries]
+    if with_size:
+        ui.table([_format_entry(entry, fmt) for entry in entries])
+        return
+
+    # NOTE: this is faster than ui.table for very large number of entries
+    ui.write("\n".join(fmt(entry) for entry in entries))
 
 
 class CmdList(CmdBaseNoRepo):
@@ -35,12 +51,13 @@ class CmdList(CmdBaseNoRepo):
                 recursive=self.args.recursive,
                 dvc_only=self.args.dvc_only,
                 config=self.args.config,
+                remote=self.args.remote,
+                remote_config=self.args.remote_config,
             )
             if self.args.json:
                 ui.write_json(entries)
             elif entries:
-                entries = _prettify(entries, with_color=True)
-                ui.write("\n".join(entries))
+                show_entries(entries, with_color=True, with_size=self.args.size)
             return 0
         except DvcException:
             logger.exception("failed to list '%s'", self.args.url)
@@ -89,6 +106,26 @@ def add_parser(subparsers, parent_parser):
             "in the target repository."
         ),
     )
+    list_parser.add_argument(
+        "--remote",
+        type=str,
+        help="Remote name to set as a default in the target repository.",
+    )
+    list_parser.add_argument(
+        "--remote-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help=(
+            "Remote config options to merge with a remote's config (default or one "
+            "specified by '--remote') in the target repository."
+        ),
+    )
+    list_parser.add_argument(
+        "--size",
+        action="store_true",
+        help="Show sizes.",
+    )
     list_parser.add_argument(
         "path",
         nargs="?",

dvc/commands/ls_url.py

@@ -2,22 +2,26 @@ import argparse
 import logging
 
 from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import append_doc_link
-from dvc.ui import ui
+from dvc.cli.utils import DictAction, append_doc_link
 
-from .ls import _prettify
+from .ls import show_entries
 
 logger = logging.getLogger(__name__)
 
 
 class CmdListUrl(CmdBaseNoRepo):
     def run(self):
+        from dvc.config import Config
         from dvc.repo import Repo
 
-        entries = Repo.ls_url(self.args.url, recursive=self.args.recursive)
+        entries = Repo.ls_url(
+            self.args.url,
+            recursive=self.args.recursive,
+            fs_config=self.args.fs_config,
+            config=Config.from_cwd(),
+        )
         if entries:
-            entries = _prettify(entries, with_color=True)
-            ui.write("\n".join(entries))
+            show_entries(entries, with_color=True, with_size=self.args.size)
         return 0
 
 
@@ -40,4 +44,16 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         help="Recursively list files.",
     )
+    lsurl_parser.add_argument(
+        "--size",
+        action="store_true",
+        help="Show sizes.",
+    )
+    lsurl_parser.add_argument(
+        "--fs-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help="Config options for the target url.",
+    )
     lsurl_parser.set_defaults(func=CmdListUrl)

dvc/commands/machine.py

@@ -1,490 +0,0 @@
-import argparse
-from typing import Dict, List
-
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link, fix_subparsers
-from dvc.commands.config import CmdConfig
-from dvc.compare import TabularData
-from dvc.config import ConfigError
-from dvc.exceptions import DvcException
-from dvc.ui import ui
-from dvc.utils import format_link
-
-
-class MachineDisabledError(ConfigError):
-    def __init__(self):
-        super().__init__("Machine feature is disabled")
-
-
-class CmdMachineConfig(CmdConfig):
-    def __init__(self, args):
-        super().__init__(args)
-        if not self.config["feature"].get("machine", False):
-            raise MachineDisabledError
-
-        if getattr(self.args, "name", None):
-            self.args.name = self.args.name.lower()
-
-    def _check_exists(self, conf):
-        if self.args.name not in conf["machine"]:
-            raise ConfigError(f"machine '{self.args.name}' doesn't exist.")
-
-
-class CmdMachineAdd(CmdMachineConfig):
-    def run(self):
-        from dvc.machine import validate_name
-
-        validate_name(self.args.name)
-
-        if self.args.default:
-            ui.write(f"Setting '{self.args.name}' as a default machine.")
-
-        with self.config.edit(self.args.level) as conf:
-            if self.args.name in conf["machine"] and not self.args.force:
-                raise ConfigError(
-                    "machine '{}' already exists. Use `-f|--force` to "
-                    "overwrite it.".format(self.args.name)
-                )
-
-            conf["machine"][self.args.name] = {"cloud": self.args.cloud}
-            if self.args.default:
-                conf["core"]["machine"] = self.args.name
-
-        return 0
-
-
-class CmdMachineRemove(CmdMachineConfig):
-    def run(self):
-        with self.config.edit(self.args.level) as conf:
-            self._check_exists(conf)
-            del conf["machine"][self.args.name]
-
-        up_to_level = self.args.level or "repo"
-        # Remove core.machine refs to this machine in any shadowing configs
-        for level in reversed(self.config.LEVELS):
-            with self.config.edit(level) as conf:
-                if conf["core"].get("machine") == self.args.name:
-                    del conf["core"]["machine"]
-
-            if level == up_to_level:
-                break
-
-        return 0
-
-
-class CmdMachineList(CmdMachineConfig):
-    TABLE_COLUMNS = [
-        "name",
-        "cloud",
-        "region",
-        "image",
-        "spot",
-        "spot_price",
-        "instance_hdd_size",
-        "instance_type",
-        "ssh_private",
-        "startup_script",
-    ]
-
-    PRIVATE_COLUMNS = ["ssh_private", "startup_script"]
-
-    def _hide_private(self, conf):
-        for machine in conf:
-            for column in self.PRIVATE_COLUMNS:
-                if column in conf[machine]:
-                    conf[machine][column] = "***"
-
-    def _show_origin(self):
-        levels = [self.args.level] if self.args.level else self.config.LEVELS
-        for level in levels:
-            conf = self.config.read(level)["machine"]
-            if self.args.name:
-                conf = conf.get(self.args.name, {})
-            self._hide_private(conf)
-            prefix = self._config_file_prefix(True, self.config, level)
-            configs = list(self._format_config(conf, prefix))
-            if configs:
-                ui.write("\n".join(configs))
-
-    def _show_table(self):
-        td = TabularData(self.TABLE_COLUMNS, fill_value="-")
-        conf = self.config.read()["machine"]
-        if self.args.name:
-            conf = {self.args.name: conf.get(self.args.name, {})}
-        self._hide_private(conf)
-        for machine, machine_config in conf.items():
-            machine_config["name"] = machine
-            td.row_from_dict(machine_config)
-        td.dropna("cols", "all")
-        td.render()
-
-    def run(self):
-        if self.args.show_origin:
-            self._show_origin()
-        else:
-            self._show_table()
-        return 0
-
-
-class CmdMachineModify(CmdMachineConfig):
-    def run(self):
-        from dvc.config import merge
-
-        with self.config.edit(self.args.level) as conf:
-            merged = self.config.load_config_to_level(self.args.level)
-            merge(merged, conf)
-            self._check_exists(merged)
-
-            if self.args.name not in conf["machine"]:
-                conf["machine"][self.args.name] = {}
-            section = conf["machine"][self.args.name]
-            if self.args.unset:
-                section.pop(self.args.option, None)
-            else:
-                section[self.args.option] = self.args.value
-        return 0
-
-
-class CmdMachineRename(CmdBase):
-    def _check_exists(self, conf):
-        if self.args.name not in conf["machine"]:
-            raise ConfigError(f"machine '{self.args.name}' doesn't exist.")
-
-    def _rename_default(self, conf):
-        if conf["core"].get("machine") == self.args.name:
-            conf["core"]["machine"] = self.args.new
-
-    def _check_before_rename(self):
-        from dvc.machine import validate_name
-
-        validate_name(self.args.new)
-
-        all_config = self.config.load_config_to_level(None)
-        if self.args.new in all_config.get("machine", {}):
-            raise ConfigError(
-                f"Rename failed. Machine '{self.args.new}' already exists."
-            )
-        ui.write(f"Rename machine '{self.args.name}' to '{self.args.new}'.")
-
-    def run(self):
-        self._check_before_rename()
-
-        with self.config.edit(self.args.level) as conf:
-            self._check_exists(conf)
-            conf["machine"][self.args.new] = conf["machine"][self.args.name]
-            try:
-                assert self.repo.machine
-                self.repo.machine.rename(self.args.name, self.args.new)
-            except DvcException as error:
-                del conf["machine"][self.args.new]
-                raise ConfigError("terraform rename failed") from error
-            del conf["machine"][self.args.name]
-            self._rename_default(conf)
-
-        up_to_level = self.args.level or "repo"
-        for level in reversed(self.config.LEVELS):
-            if level == up_to_level:
-                break
-            with self.config.edit(level) as level_conf:
-                self._rename_default(level_conf)
-
-        return 0
-
-
-class CmdMachineDefault(CmdMachineConfig):
-    def run(self):
-        if self.args.name is None and not self.args.unset:
-            conf = self.config.read(self.args.level)
-            try:
-                ui.write(conf["core"]["machine"])
-            except KeyError:
-                ui.write("No default machine set")
-                return 1
-        else:
-            with self.config.edit(self.args.level) as conf:
-                if self.args.unset:
-                    conf["core"].pop("machine", None)
-                else:
-                    merged_conf = self.config.load_config_to_level(self.args.level)
-                    if (
-                        self.args.name in conf["machine"]
-                        or self.args.name in merged_conf["machine"]
-                    ):
-                        conf["core"]["machine"] = self.args.name
-                    else:
-                        raise ConfigError(
-                            "default machine must be present in machine list."
-                        )
-        return 0
-
-
-class CmdMachineCreate(CmdBase):
-    def run(self):
-        if self.repo.machine is None:
-            raise MachineDisabledError
-
-        self.repo.machine.create(self.args.name)
-        return 0
-
-
-class CmdMachineStatus(CmdBase):
-    INSTANCE_FIELD = ["name", "instance", "status"]
-    SHOWN_FIELD = [
-        "cloud",
-        "instance_ip",
-        "instance_type",
-        "instance_hdd_size",
-        "instance_gpu",
-    ]
-    FILL_VALUE = "-"
-
-    def _add_row(
-        self,
-        name: str,
-        all_status: List[Dict],
-        td: TabularData,
-    ):
-        if not all_status:
-            row = [
-                name,
-                self.FILL_VALUE,
-                "offline",
-            ]  # back to `None` after #7167
-            td.append(row)
-        for i, status in enumerate(all_status, start=1):
-            row = [name, f"num_{i}", "running" if status else "offline"]
-            for field in self.SHOWN_FIELD:
-                value = str(status.get(field, ""))
-                row.append(value)
-            td.append(row)
-
-    def run(self):
-        if self.repo.machine is None:
-            raise MachineDisabledError
-
-        td = TabularData(
-            self.INSTANCE_FIELD + self.SHOWN_FIELD, fill_value=self.FILL_VALUE
-        )
-
-        if self.args.name:
-            all_status = list(self.repo.machine.status(self.args.name))
-            self._add_row(self.args.name, all_status, td)
-        else:
-            name_set = set()
-            for level in self.repo.config.LEVELS:
-                conf = self.repo.config.read(level)["machine"]
-                name_set.update(conf.keys())
-            name_list = list(name_set)
-            for name in sorted(name_list):
-                all_status = list(self.repo.machine.status(name))
-                self._add_row(name, all_status, td)
-
-        td.dropna("cols", "all")
-        td.render()
-        return 0
-
-
-class CmdMachineDestroy(CmdBase):
-    def run(self):
-        if self.repo.machine is None:
-            raise MachineDisabledError
-
-        self.repo.machine.destroy(self.args.name)
-        return 0
-
-
-class CmdMachineSsh(CmdBase):
-    def run(self):
-        if self.repo.machine is None:
-            raise MachineDisabledError
-
-        self.repo.machine.run_shell(self.args.name)
-        return 0
-
-
-def add_parser(subparsers, parent_parser):  # noqa: PLR0915
-    from dvc.commands.config import parent_config_parser
-
-    machine_HELP = "Set up and manage cloud machines."
-    machine_parser = subparsers.add_parser(
-        "machine",
-        parents=[parent_parser],
-        description=append_doc_link(machine_HELP, "machine"),
-        # NOTE: suppress help during development to hide command
-        # help=machine_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-
-    machine_subparsers = machine_parser.add_subparsers(
-        dest="cmd",
-        help="Use `dvc machine CMD --help` for command-specific help.",
-    )
-
-    fix_subparsers(machine_subparsers)
-
-    machine_ADD_HELP = "Add a new data machine."
-    machine_add_parser = machine_subparsers.add_parser(
-        "add",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(machine_ADD_HELP, "machine/add"),
-        help=machine_ADD_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_add_parser.add_argument("name", help="Name of the machine")
-    machine_add_parser.add_argument(
-        "cloud",
-        help="Machine cloud. See full list of supported clouds at {}".format(
-            format_link(
-                "https://github.com/iterative/terraform-provider-iterative#machine"
-            )
-        ),
-    )
-    machine_add_parser.add_argument(
-        "-d",
-        "--default",
-        action="store_true",
-        default=False,
-        help="Set as default machine.",
-    )
-    machine_add_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Force overwriting existing configs",
-    )
-    machine_add_parser.set_defaults(func=CmdMachineAdd)
-
-    machine_DEFAULT_HELP = "Set/unset the default machine."
-    machine_default_parser = machine_subparsers.add_parser(
-        "default",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(machine_DEFAULT_HELP, "machine/default"),
-        help=machine_DEFAULT_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_default_parser.add_argument("name", nargs="?", help="Name of the machine")
-    machine_default_parser.add_argument(
-        "-u",
-        "--unset",
-        action="store_true",
-        default=False,
-        help="Unset default machine.",
-    )
-    machine_default_parser.set_defaults(func=CmdMachineDefault)
-
-    machine_LIST_HELP = "List the configuration of one/all machines."
-    machine_list_parser = machine_subparsers.add_parser(
-        "list",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(machine_LIST_HELP, "machine/list"),
-        help=machine_LIST_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_list_parser.add_argument(
-        "--show-origin",
-        default=False,
-        action="store_true",
-        help="Show the source file containing each config value.",
-    )
-    machine_list_parser.add_argument(
-        "name",
-        nargs="?",
-        type=str,
-        help="name of machine to specify",
-    )
-    machine_list_parser.set_defaults(func=CmdMachineList)
-    machine_MODIFY_HELP = "Modify the configuration of an machine."
-    machine_modify_parser = machine_subparsers.add_parser(
-        "modify",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(machine_MODIFY_HELP, "machine/modify"),
-        help=machine_MODIFY_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_modify_parser.add_argument("name", help="Name of the machine")
-    machine_modify_parser.add_argument("option", help="Name of the option to modify.")
-    machine_modify_parser.add_argument(
-        "value", nargs="?", help="(optional) Value of the option."
-    )
-    machine_modify_parser.add_argument(
-        "-u",
-        "--unset",
-        default=False,
-        action="store_true",
-        help="Unset option.",
-    )
-    machine_modify_parser.set_defaults(func=CmdMachineModify)
-
-    machine_RENAME_HELP = "Rename a machine "
-    machine_rename_parser = machine_subparsers.add_parser(
-        "rename",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(machine_RENAME_HELP, "remote/rename"),
-        help=machine_RENAME_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_rename_parser.add_argument("name", help="Machine to be renamed")
-    machine_rename_parser.add_argument("new", help="New name of the machine")
-    machine_rename_parser.set_defaults(func=CmdMachineRename)
-
-    machine_REMOVE_HELP = "Remove an machine."
-    machine_remove_parser = machine_subparsers.add_parser(
-        "remove",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(machine_REMOVE_HELP, "machine/remove"),
-        help=machine_REMOVE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_remove_parser.add_argument("name", help="Name of the machine to remove.")
-    machine_remove_parser.set_defaults(func=CmdMachineRemove)
-
-    machine_CREATE_HELP = "Create and start a machine instance."
-    machine_create_parser = machine_subparsers.add_parser(
-        "create",
-        parents=[parent_parser],
-        description=append_doc_link(machine_CREATE_HELP, "machine/create"),
-        help=machine_CREATE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_create_parser.add_argument("name", help="Name of the machine to create.")
-    machine_create_parser.set_defaults(func=CmdMachineCreate)
-
-    machine_STATUS_HELP = "List the status of running instances for one/all machines."
-    machine_status_parser = machine_subparsers.add_parser(
-        "status",
-        parents=[parent_parser],
-        description=append_doc_link(machine_STATUS_HELP, "machine/status"),
-        help=machine_STATUS_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_status_parser.add_argument(
-        "name", nargs="?", help="(optional) Name of the machine."
-    )
-    machine_status_parser.set_defaults(func=CmdMachineStatus)
-
-    machine_DESTROY_HELP = "Destroy an machine instance."
-    machine_destroy_parser = machine_subparsers.add_parser(
-        "destroy",
-        parents=[parent_parser],
-        description=append_doc_link(machine_DESTROY_HELP, "machine/destroy"),
-        help=machine_DESTROY_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_destroy_parser.add_argument(
-        "name", help="Name of the machine instance to destroy."
-    )
-    machine_destroy_parser.set_defaults(func=CmdMachineDestroy)
-
-    machine_SSH_HELP = "Connect to a machine via SSH."
-    machine_ssh_parser = machine_subparsers.add_parser(
-        "ssh",
-        parents=[parent_parser],
-        description=append_doc_link(machine_SSH_HELP, "machine/ssh"),
-        help=machine_SSH_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_ssh_parser.add_argument(
-        "name", help="Name of the machine instance to connect to."
-    )
-    machine_ssh_parser.set_defaults(func=CmdMachineSsh)

dvc/commands/metrics.py

@@ -4,7 +4,6 @@ import logging
 from dvc.cli import completion
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link, fix_subparsers
-from dvc.exceptions import DvcException
 from dvc.ui import ui
 from dvc.utils.serialize import encode_exception
 
@@ -20,17 +19,25 @@ class CmdMetricsBase(CmdBase):
 
 class CmdMetricsShow(CmdMetricsBase):
     def run(self):
-        try:
-            metrics = self.repo.metrics.show(
-                self.args.targets,
-                all_branches=self.args.all_branches,
-                all_tags=self.args.all_tags,
-                all_commits=self.args.all_commits,
-                recursive=self.args.recursive,
+        from dvc.repo.metrics.show import to_relpath
+        from dvc.utils import errored_revisions
+
+        metrics = self.repo.metrics.show(
+            self.args.targets,
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
+        )
+        metrics = {
+            k: to_relpath(self.repo.fs, self.repo.root_dir, v)
+            for k, v in metrics.items()
+        }
+
+        if errored := errored_revisions(metrics):
+            ui.error_write(
+                "DVC failed to load some metrics for following revisions:"
+                f" '{', '.join(errored)}'."
             )
-        except DvcException:
-            logger.exception("")
-            return 1
 
         if self.args.json:
             ui.write_json(metrics, default=encode_exception)
@@ -52,17 +59,26 @@ class CmdMetricsShow(CmdMetricsBase):
 
 class CmdMetricsDiff(CmdMetricsBase):
     def run(self):
-        try:
-            diff = self.repo.metrics.diff(
-                a_rev=self.args.a_rev,
-                b_rev=self.args.b_rev,
-                targets=self.args.targets,
-                recursive=self.args.recursive,
-                all=self.args.all,
+        import os
+        from os.path import relpath
+
+        diff_result = self.repo.metrics.diff(
+            a_rev=self.args.a_rev,
+            b_rev=self.args.b_rev,
+            targets=self.args.targets,
+            all=self.args.all,
+        )
+
+        errored = [rev for rev, err in diff_result.get("errors", {}).items() if err]
+        if errored:
+            ui.error_write(
+                "DVC failed to load some metrics for following revisions:"
+                f" '{', '.join(errored)}'."
             )
-        except DvcException:
-            logger.exception("failed to show metrics diff")
-            return 1
+
+        start = relpath(os.getcwd(), self.repo.root_dir)
+        diff = diff_result.get("diff", {})
+        diff = {relpath(path, start): result for path, result in diff.items()}
 
         if self.args.json:
             ui.write_json(diff)
@@ -184,10 +200,14 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     metrics_diff_parser.add_argument(
-        "a_rev", nargs="?", help="Old Git commit to compare (defaults to HEAD)"
+        "a_rev",
+        nargs="?",
+        help="Old Git commit to compare (defaults to HEAD)",
+        default="HEAD",
     )
     metrics_diff_parser.add_argument(
         "b_rev",
+        default="workspace",
         nargs="?",
         help="New Git commit to compare (defaults to the current workspace)",
     )

dvc/commands/move.py

@@ -30,6 +30,7 @@ def add_parser(subparsers, parent_parser):
 
     move_parser = subparsers.add_parser(
         "move",
+        aliases=["mv"],
         parents=[parent_parser],
         description=append_doc_link(MOVE_DESCRIPTION, "move"),
         help=MOVE_HELP,

dvc/commands/params.py

@@ -4,7 +4,6 @@ import logging
 from dvc.cli import completion
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link, fix_subparsers
-from dvc.exceptions import DvcException
 from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
@@ -14,17 +13,27 @@ class CmdParamsDiff(CmdBase):
     UNINITIALIZED = True
 
     def run(self):
-        try:
-            diff = self.repo.params.diff(
-                a_rev=self.args.a_rev,
-                b_rev=self.args.b_rev,
-                targets=self.args.targets,
-                all=self.args.all,
-                deps=self.args.deps,
+        import os
+        from os.path import relpath
+
+        diff_result = self.repo.params.diff(
+            a_rev=self.args.a_rev,
+            b_rev=self.args.b_rev,
+            targets=self.args.targets,
+            all=self.args.all,
+            deps_only=self.args.deps,
+        )
+
+        errored = [rev for rev, err in diff_result.get("errors", {}).items() if err]
+        if errored:
+            ui.error_write(
+                "DVC failed to load some metrics for following revisions:"
+                f" '{', '.join(errored)}'."
             )
-        except DvcException:
-            logger.exception("failed to show params diff")
-            return 1
+
+        start = relpath(os.getcwd(), self.repo.root_dir)
+        diff = diff_result.get("diff", {})
+        diff = {relpath(path, start): result for path, result in diff.items()}
 
         if self.args.json:
             ui.write_json(diff)
@@ -74,10 +83,14 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     params_diff_parser.add_argument(
-        "a_rev", nargs="?", help="Old Git commit to compare (defaults to HEAD)"
+        "a_rev",
+        nargs="?",
+        default="HEAD",
+        help="Old Git commit to compare (defaults to HEAD)",
     )
     params_diff_parser.add_argument(
         "b_rev",
+        default="workspace",
         nargs="?",
         help="New Git commit to compare (defaults to the current workspace)",
     )

dvc/commands/remote.py

@@ -30,8 +30,8 @@ class CmdRemoteAdd(CmdRemote):
         with self.config.edit(self.args.level) as conf:
             if self.args.name in conf["remote"] and not self.args.force:
                 raise ConfigError(
-                    "remote '{}' already exists. Use `-f|--force` to "
-                    "overwrite it.".format(self.args.name)
+                    f"remote '{self.args.name}' already exists. Use `-f|--force` to "
+                    "overwrite it."
                 )
 
             conf["remote"][self.args.name] = {"url": self.args.url}

dvc/commands/remove.py

@@ -26,6 +26,7 @@ def add_parser(subparsers, parent_parser):
     )
     remove_parser = subparsers.add_parser(
         "remove",
+        aliases=["rm"],
         parents=[parent_parser],
         description=append_doc_link(REMOVE_HELP, "remove"),
         help=REMOVE_HELP,

dvc/compare.py

@@ -57,7 +57,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
         return self._columns[name]
 
     def items(self) -> ItemsView[str, Column]:
-        projection = {k: self.column(k) for k in self.keys()}  # noqa: SIM118
+        projection = {k: self.column(k) for k in self.keys()}
         return projection.items()
 
     def keys(self) -> List[str]:
@@ -171,8 +171,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
                 self.add_column(key)
 
         row: List["CellT"] = [
-            with_value(d.get(key), self._fill_value)
-            for key in self.keys()  # noqa: SIM118
+            with_value(d.get(key), self._fill_value) for key in self.keys()
         ]
         self.append(row)
 
@@ -227,7 +226,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
             to_drop -= match_line
 
         if axis == "rows":
-            for name in self.keys():  # noqa: SIM118
+            for name in self.keys():
                 self._columns[name] = Column(
                     [x for n, x in enumerate(self._columns[name]) if n not in to_drop]
                 )
@@ -275,7 +274,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
                 else:
                     unique_rows.append(tuple_row)
 
-            for name in self.keys():  # noqa: SIM118
+            for name in self.keys():
                 self._columns[name] = Column(
                     [
                         x

dvc/config.py

@@ -90,20 +90,30 @@ class Config(dict):
     def __init__(
         self,
         dvc_dir: Optional["StrPath"] = None,
+        local_dvc_dir: Optional["StrPath"] = None,
         validate: bool = True,
         fs: Optional["FileSystem"] = None,
         config: Optional["DictStrAny"] = None,
+        remote: Optional[str] = None,
+        remote_config: Optional["DictStrAny"] = None,
     ):  # pylint: disable=super-init-not-called
         from dvc.fs import LocalFileSystem
 
+        dvc_dir = os.fspath(dvc_dir) if dvc_dir else None
         self.dvc_dir = dvc_dir
         self.wfs = LocalFileSystem()
         self.fs = fs or self.wfs
 
         if dvc_dir:
-            self.dvc_dir = self.fs.path.abspath(self.fs.path.realpath(dvc_dir))
+            self.dvc_dir = self.fs.path.abspath(dvc_dir)
 
-        self.load(validate=validate, config=config)
+        self.local_dvc_dir = local_dvc_dir
+        if not fs and not local_dvc_dir:
+            self.local_dvc_dir = dvc_dir
+
+        self.load(
+            validate=validate, config=config, remote=remote, remote_config=remote_config
+        )
 
     @classmethod
     def from_cwd(cls, fs: Optional["FileSystem"] = None, **kwargs):
@@ -136,7 +146,9 @@ class Config(dict):
 
         if self.dvc_dir is not None:
             files["repo"] = self.fs.path.join(self.dvc_dir, self.CONFIG)
-            files["local"] = self.fs.path.join(self.dvc_dir, self.CONFIG_LOCAL)
+
+        if self.local_dvc_dir is not None:
+            files["local"] = self.wfs.path.join(self.local_dvc_dir, self.CONFIG_LOCAL)
 
         return files
 
@@ -154,7 +166,16 @@ class Config(dict):
         with open(config_file, "w+", encoding="utf-8"):
             return Config(dvc_dir)
 
-    def load(self, validate: bool = True, config: Optional["DictStrAny"] = None):
+    def merge(self, config):
+        merge(self, config)
+
+    def load(
+        self,
+        validate: bool = True,
+        config: Optional["DictStrAny"] = None,
+        remote: Optional[str] = None,
+        remote_config: Optional["DictStrAny"] = None,
+    ):
         """Loads config from all the config files.
 
         Raises:
@@ -169,6 +190,17 @@ class Config(dict):
             conf = self.validate(conf)
 
         self.clear()
+
+        if remote:
+            conf["core"]["remote"] = remote
+
+        if remote_config:
+            remote = remote or conf["core"].get("remote")
+            if not remote:
+                raise ValueError("Missing remote name")
+
+            merge(conf, {"remote": {remote: remote_config}})
+
         self.update(conf)
 
     def _get_fs(self, level):
@@ -229,21 +261,30 @@ class Config(dict):
         return conf
 
     @staticmethod
-    def _load_paths(conf, filename):
-        abs_conf_dir = os.path.abspath(os.path.dirname(filename))
+    def _resolve(conf_dir, path):
+        from .config_schema import ExpPath, RelPath
 
-        def resolve(path):
-            from .config_schema import RelPath
+        if re.match(r"\w+://", path):
+            return path
 
-            if os.path.isabs(path) or re.match(r"\w+://", path):
-                return path
+        if os.path.isabs(path):
+            return path
 
-            # on windows convert slashes to backslashes
-            # to have path compatible with abs_conf_dir
-            if os.path.sep == "\\" and "/" in path:
-                path = path.replace("/", "\\")
+        # on windows convert slashes to backslashes
+        # to have path compatible with abs_conf_dir
+        if os.path.sep == "\\" and "/" in path:
+            path = path.replace("/", "\\")
 
-            return RelPath(os.path.join(abs_conf_dir, path))
+        expanded = os.path.expanduser(path)
+        if os.path.isabs(expanded):
+            return ExpPath(expanded, path)
+
+        return RelPath(os.path.abspath(os.path.join(conf_dir, path)))
+
+    @classmethod
+    def _load_paths(cls, conf, filename):
+        conf_dir = os.path.abspath(os.path.dirname(filename))
+        resolve = partial(cls._resolve, conf_dir)
 
         return Config._map_dirs(conf, resolve)
 
@@ -252,11 +293,17 @@ class Config(dict):
         from dvc.fs import localfs
         from dvc.utils import relpath
 
-        from .config_schema import RelPath
+        from .config_schema import ExpPath, RelPath
 
         if re.match(r"\w+://", path):
             return path
 
+        if isinstance(path, ExpPath):
+            return path.def_path
+
+        if os.path.expanduser(path) != path:
+            return localfs.path.as_posix(path)
+
         if isinstance(path, RelPath) or not os.path.isabs(path):
             path = relpath(path, conf_dir)
             return localfs.path.as_posix(path)

dvc/config_schema.py

@@ -8,6 +8,7 @@ from voluptuous import (
     All,
     Any,
     Coerce,
+    Exclusive,
     Invalid,
     Lower,
     Optional,
@@ -74,6 +75,17 @@ def ByUrl(mapping):  # noqa: N802
     return validate
 
 
+class ExpPath(str):
+    __slots__ = ("def_path",)
+
+    def_path: str
+
+    def __new__(cls, string, def_path):
+        ret = super().__new__(cls, string)
+        ret.def_path = def_path
+        return ret
+
+
 class RelPath(str):
     __slots__ = ()
 
@@ -324,7 +336,8 @@ SCHEMA = {
     },
     "hydra": {
         Optional("enabled", default=False): Bool,
-        "config_dir": str,
+        Exclusive("config_dir", "config_source"): str,
+        Exclusive("config_module", "config_source"): str,
         "config_name": str,
     },
     "studio": {

dvc/daemon.py

@@ -3,7 +3,6 @@
 import inspect
 import logging
 import os
-import platform
 import sys
 from subprocess import Popen  # nosec B404
 from typing import List
@@ -93,13 +92,8 @@ def _spawn_posix(cmd, env):
     sys.stderr.close()
     os.closerange(0, 3)
 
-    if platform.system() == "Darwin":
-        # workaround for MacOS bug
-        # https://github.com/iterative/dvc/issues/4294
-        _popen(cmd, env=env).communicate()
-    else:
-        os.environ.update(env)
-        main(cmd)
+    os.environ.update(env)
+    main(cmd)
 
     os._exit(0)  # pylint: disable=protected-access
 

dvc/data_cloud.py

@@ -4,6 +4,7 @@ import logging
 from typing import TYPE_CHECKING, Iterable, Optional, Set, Tuple
 
 from dvc.config import NoRemoteError, RemoteConfigError
+from dvc.fs.callbacks import Callback
 from dvc.utils.objects import cached_property
 from dvc_data.hashfile.db import get_index
 from dvc_data.hashfile.transfer import TransferResult
@@ -134,7 +135,7 @@ class DataCloud:
 
         remote = self.get_remote(name=name, command=command)
         if remote.fs.version_aware or remote.worktree:
-            raise NoRemoteError(
+            raise RemoteConfigError(
                 f"'{command}' is unsupported for cloud versioned remotes"
             )
         if hash_name in LEGACY_HASH_NAMES:
@@ -209,15 +210,20 @@ class DataCloud:
             cache = self.repo.cache.legacy
         else:
             cache = self.repo.cache.local
-        return self.transfer(
-            cache,
-            odb,
-            objs,
-            jobs=jobs,
-            dest_index=get_index(odb),
-            cache_odb=cache,
-            validate_status=self._log_missing,
-        )
+        with Callback.as_tqdm_callback(
+            desc=f"Pushing to {odb.fs.unstrip_protocol(odb.path)}",
+            unit="file",
+        ) as cb:
+            return self.transfer(
+                cache,
+                odb,
+                objs,
+                jobs=jobs,
+                dest_index=get_index(odb),
+                cache_odb=cache,
+                validate_status=self._log_missing,
+                callback=cb,
+            )
 
     def pull(
         self,
@@ -263,16 +269,21 @@ class DataCloud:
             cache = self.repo.cache.legacy
         else:
             cache = self.repo.cache.local
-        return self.transfer(
-            odb,
-            cache,
-            objs,
-            jobs=jobs,
-            src_index=get_index(odb),
-            cache_odb=cache,
-            verify=odb.verify,
-            validate_status=self._log_missing,
-        )
+        with Callback.as_tqdm_callback(
+            desc=f"Fetching from {odb.fs.unstrip_protocol(odb.path)}",
+            unit="file",
+        ) as cb:
+            return self.transfer(
+                odb,
+                cache,
+                objs,
+                jobs=jobs,
+                src_index=get_index(odb),
+                cache_odb=cache,
+                verify=odb.verify,
+                validate_status=self._log_missing,
+                callback=cb,
+            )
 
     def status(
         self,

dvc/dependency/__init__.py

@@ -1,5 +1,5 @@
 from collections import defaultdict
-from typing import Any, Mapping, Set
+from typing import Any, Dict, List, Mapping, Set
 
 from dvc.output import ARTIFACT_SCHEMA, DIR_FILES_SCHEMA, Output
 
@@ -15,6 +15,7 @@ SCHEMA: Mapping[str, Any] = {
     **ARTIFACT_SCHEMA,
     **RepoDependency.REPO_SCHEMA,
     Output.PARAM_FILES: [DIR_FILES_SCHEMA],
+    Output.PARAM_FS_CONFIG: dict,
 }
 
 
@@ -36,7 +37,10 @@ def loadd_from(stage, d_list):
         p = d.pop(Output.PARAM_PATH, None)
         files = d.pop(Output.PARAM_FILES, None)
         hash_name = d.pop(Output.PARAM_HASH, None)
-        ret.append(_get(stage, p, d, files=files, hash_name=hash_name))
+        fs_config = d.pop(Output.PARAM_FS_CONFIG, None)
+        ret.append(
+            _get(stage, p, d, files=files, hash_name=hash_name, fs_config=fs_config)
+        )
     return ret
 
 
@@ -54,7 +58,7 @@ def loads_from(stage, s_list, erepo=None, fs_config=None):
     ]
 
 
-def _merge_params(s_list):
+def _merge_params(s_list) -> Dict[str, List[str]]:
     d = defaultdict(list)
     default_file = ParamsDependency.DEFAULT_PARAMS_FILE
 

dvc/dependency/base.py

@@ -44,9 +44,6 @@ class Dependency(Output):
         if self.fs.version_aware:
             self.fs_path = self.fs.path.version_path(self.fs_path, rev)
             self.meta = self.get_meta()
-            self.def_path = self.fs.path.version_path(
-                self.def_path, self.meta.version_id
-            )
             self.fs_path = self.fs.path.version_path(self.fs_path, self.meta.version_id)
 
     def download(self, to, jobs=None):

dvc/dependency/param.py

@@ -2,7 +2,7 @@ import logging
 import os
 import typing
 from collections import defaultdict
-from typing import Any, Dict
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
 import dpath
 
@@ -12,6 +12,9 @@ from dvc_data.hashfile.hash_info import HashInfo
 
 from .base import Dependency
 
+if TYPE_CHECKING:
+    from dvc.fs import FileSystem
+
 logger = logging.getLogger(__name__)
 
 
@@ -31,6 +34,37 @@ class BadParamFileError(DvcException):
     pass
 
 
+def read_param_file(
+    fs: "FileSystem",
+    path: str,
+    key_paths: Optional[List[str]] = None,
+    flatten: bool = False,
+    **load_kwargs,
+) -> Any:
+    config = load_path(path, fs, **load_kwargs)
+    if not key_paths:
+        return config
+
+    ret = {}
+    if flatten:
+        for key_path in key_paths:
+            try:
+                ret[key_path] = dpath.get(config, key_path, separator=".")
+            except KeyError:
+                continue
+        return ret
+
+    from dpath import merge
+
+    for key_path in key_paths:
+        merge(
+            ret,
+            dpath.search(config, key_path, separator="."),
+            separator=".",
+        )
+    return ret
+
+
 class ParamsDependency(Dependency):
     PARAM_PARAMS = "params"
     DEFAULT_PARAMS_FILE = "params.yaml"
@@ -75,31 +109,19 @@ class ParamsDependency(Dependency):
         self, flatten: bool = True, **kwargs: typing.Any
     ) -> Dict[str, typing.Any]:
         try:
-            config = self.read_file()
+            self.validate_filepath()
         except MissingParamsFile:
-            config = {}
+            return {}
 
-        if not self.params:
-            return config
-
-        ret = {}
-        if flatten:
-            for param in self.params:
-                try:
-                    ret[param] = dpath.get(config, param, separator=".")
-                except KeyError:
-                    continue
-            return ret
-
-        from dpath import merge
-
-        for param in self.params:
-            merge(
-                ret,
-                dpath.search(config, param, separator="."),
-                separator=".",
+        try:
+            return read_param_file(
+                self.repo.fs,
+                self.fs_path,
+                list(self.params) if self.params else None,
+                flatten=flatten,
             )
-        return ret
+        except ParseError as exc:
+            raise BadParamFileError(f"Unable to read parameters from '{self}'") from exc
 
     def workspace_status(self):
         if not self.exists:
@@ -149,13 +171,6 @@ class ParamsDependency(Dependency):
                 f"'{self}' is a directory, expected a parameters file"
             )
 
-    def read_file(self):
-        self.validate_filepath()
-        try:
-            return load_path(self.fs_path, self.repo.fs)
-        except ParseError as exc:
-            raise BadParamFileError(f"Unable to read parameters from '{self}'") from exc
-
     def get_hash(self):
         info = self.read_params()
 

dvc/dependency/repo.py

@@ -1,6 +1,7 @@
+from copy import deepcopy
 from typing import TYPE_CHECKING, Dict, Optional, Union
 
-from voluptuous import Required
+from voluptuous import Any, Required
 
 from dvc.utils import as_posix
 
@@ -17,17 +18,19 @@ class RepoDependency(Dependency):
     PARAM_REV = "rev"
     PARAM_REV_LOCK = "rev_lock"
     PARAM_CONFIG = "config"
+    PARAM_REMOTE = "remote"
 
     REPO_SCHEMA = {
         PARAM_REPO: {
             Required(PARAM_URL): str,
             PARAM_REV: str,
             PARAM_REV_LOCK: str,
-            PARAM_CONFIG: str,
+            PARAM_CONFIG: Any(str, dict),
+            PARAM_REMOTE: Any(str, dict),
         }
     }
 
-    def __init__(self, def_repo: Dict[str, str], stage: "Stage", *args, **kwargs):
+    def __init__(self, def_repo: Dict[str, Any], stage: "Stage", *args, **kwargs):
         self.def_repo = def_repo
         super().__init__(stage, *args, **kwargs)
 
@@ -76,6 +79,10 @@ class RepoDependency(Dependency):
         if config:
             repo[self.PARAM_CONFIG] = config
 
+        remote = self.def_repo.get(self.PARAM_REMOTE)
+        if remote:
+            repo[self.PARAM_REMOTE] = remote
+
         return {
             self.PARAM_PATH: self.def_path,
             self.PARAM_REPO: repo,
@@ -99,11 +106,19 @@ class RepoDependency(Dependency):
         from dvc.config import Config
         from dvc.fs import DVCFileSystem
 
-        conf = self.def_repo.get("config")
-        if conf:
-            config = Config.load_file(conf)
+        rem = self.def_repo.get("remote")
+        if isinstance(rem, dict):
+            remote = None
+            remote_config = rem
         else:
-            config = {}
+            remote = rem
+            remote_config = None
+
+        conf = self.def_repo.get("config", {})
+        if isinstance(conf, dict):
+            config = deepcopy(conf)
+        else:
+            config = Config.load_file(conf)
 
         config["cache"] = self.repo.config["cache"]
         config["cache"]["dir"] = self.repo.cache.local_cache_dir
@@ -113,6 +128,8 @@ class RepoDependency(Dependency):
             rev=rev or self._get_rev(locked=locked),
             subrepos=True,
             config=config,
+            remote=remote,
+            remote_config=remote_config,
         )
 
     def _get_rev(self, locked: bool = True):

dvc/exceptions.py

@@ -1,9 +1,12 @@
 """Exceptions raised by the dvc."""
 import errno
-from typing import Dict, List
+from typing import TYPE_CHECKING, Dict, List, Optional, Set
 
 from dvc.utils import format_link
 
+if TYPE_CHECKING:
+    from dvc.stage import Stage
+
 
 class DvcException(Exception):
     """Base class for all dvc exceptions."""
@@ -31,24 +34,25 @@ class OutputDuplicationError(DvcException):
         stages (list): list of paths to stages.
     """
 
-    def __init__(self, output, stages):
+    def __init__(self, output: str, stages: Set["Stage"]):
         from funcy import first
 
         assert isinstance(output, str)
         assert all(hasattr(stage, "relpath") for stage in stages)
-        msg = ""
-        stage_names = [s.addressing for s in stages]
-        stages_str = " ".join(stage_names)
         if len(stages) == 1:
-            stage_name = first(stages)
-            msg = f"output '{output}' is already specified in {stage_name}."
+            stage = first(stages)
+            msg = (
+                f"output '{output}' is already specified in {stage}."
+                f"\nUse `dvc remove {stage.addressing}` to stop tracking the "
+                "overlapping output."
+            )
         else:
-            msg = "output '{}' is already specified in stages:\n{}".format(
-                output, "\n".join(f"\t- {s}" for s in stage_names)
+            stage_names = "\n".join(["\t- " + s.addressing for s in stages])
+            msg = (
+                f"output '{output}' is specified in:\n{stage_names}"
+                "\nUse `dvc remove` with any of the above targets to stop tracking the "
+                "overlapping output."
             )
-        msg += (
-            f"\nUse `dvc remove {stages_str}` to stop tracking the overlapping output."
-        )
         super().__init__(msg)
         self.stages = stages
         self.output = output
@@ -71,6 +75,10 @@ class OutputNotFoundError(DvcException):
         )
 
 
+class StageNotFoundError(DvcException):
+    pass
+
+
 class StagePathAsOutputError(DvcException):
     """Thrown if directory that stage is going to be saved in is specified as
     an output of another stage.
@@ -192,7 +200,7 @@ class ETagMismatchError(DvcException):
     def __init__(self, etag, cached_etag):
         super().__init__(
             "ETag mismatch detected when copying file to cache! "
-            "(expected: '{}', actual: '{}')".format(etag, cached_etag)
+            f"(expected: '{etag}', actual: '{cached_etag}')"
         )
 
 
@@ -293,7 +301,7 @@ class URLMissingError(DvcException):
         super().__init__(f"The path '{url}' does not exist")
 
 
-class IsADirectoryError(DvcException):  # noqa,pylint:disable=redefined-builtin
+class IsADirectoryError(DvcException):  # noqa: A001,pylint:disable=redefined-builtin
     """Raised when a file operation is requested on a directory."""
 
 
@@ -328,3 +336,24 @@ class CacheLinkError(DvcException):
 class PrettyDvcException(DvcException):
     def __pretty_exc__(self, **kwargs):
         """Print prettier exception message."""
+
+
+class ArtifactNotFoundError(DvcException):
+    """Thrown if an artifact is not found in the DVC repo.
+
+    Args:
+        name (str): artifact name.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        version: Optional[str] = None,
+        stage: Optional[str] = None,
+    ):
+        self.name = name
+        self.version = version
+        self.stage = stage
+
+        desc = f" @ {stage or version}" if (stage or version) else ""
+        super().__init__(f"Unable to find artifact '{name}{desc}'")

dvc/fs/__init__.py

@@ -49,7 +49,9 @@ known_implementations.update(
 # pylint: enable=unused-import
 
 
-def download(fs: "FileSystem", fs_path: str, to: str, jobs: Optional[int] = None):
+def download(
+    fs: "FileSystem", fs_path: str, to: str, jobs: Optional[int] = None
+) -> int:
     with Callback.as_tqdm_callback(
         desc=f"Downloading {fs.path.name(fs_path)}",
         unit="files",
@@ -63,7 +65,8 @@ def download(fs: "FileSystem", fs_path: str, to: str, jobs: Optional[int] = None
                 if not path.endswith(fs.path.flavour.sep)
             ]
             if not from_infos:
-                return localfs.makedirs(to, exist_ok=True)
+                localfs.makedirs(to, exist_ok=True)
+                return 0
             to_infos = [
                 localfs.path.join(to, *fs.path.relparts(info, fs_path))
                 for info in from_infos
@@ -82,13 +85,14 @@ def download(fs: "FileSystem", fs_path: str, to: str, jobs: Optional[int] = None
             callback=cb,
             batch_size=jobs,
         )
+        return len(to_infos)
 
 
-def parse_external_url(url, config=None):
-    remote_config = dict(config) if config else {}
+def parse_external_url(url, fs_config=None, config=None):
+    remote_config = dict(fs_config) if fs_config else {}
     remote_config["url"] = url
-    fs_cls, fs_config, fs_path = get_cloud_fs({}, **remote_config)
-    fs = fs_cls(**fs_config)
+    fs_cls, resolved_fs_config, fs_path = get_cloud_fs(config, **remote_config)
+    fs = fs_cls(**resolved_fs_config)
     return fs, fs_path
 
 

dvc/fs/dvc.py

@@ -18,7 +18,7 @@ from .data import DataFileSystem
 
 if TYPE_CHECKING:
     from dvc.repo import Repo
-    from dvc.types import StrPath
+    from dvc.types import DictStrAny, StrPath
 
 logger = logging.getLogger(__name__)
 
@@ -76,14 +76,22 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
     cachable = False
     root_marker = "/"
 
-    def __init__(
+    def __init__(  # noqa: PLR0913
         self,
         url: Optional[str] = None,
         rev: Optional[str] = None,
         repo: Optional["Repo"] = None,
         subrepos: bool = False,
         repo_factory: Optional[RepoFactory] = None,
-        **repo_kwargs: Any,
+        fo: Optional[str] = None,
+        # pylint:disable-next=unused-argument
+        target_options: Optional[Dict[str, Any]] = None,  # noqa: ARG002
+        # pylint:disable-next=unused-argument
+        target_protocol: Optional[str] = None,  # noqa: ARG002
+        config: Optional["DictStrAny"] = None,
+        remote: Optional[str] = None,
+        remote_config: Optional["DictStrAny"] = None,
+        **kwargs,
     ) -> None:
         """DVC + git-tracked files fs.
 
@@ -103,6 +111,9 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
                 By default, it ignores subrepos.
             repo_factory (callable): A function to initialize subrepo with.
                 The default is `Repo`.
+            config (dict): Repo config to be passed into `repo_factory`.
+            remote (str): Remote name to be passed into `repo_factory`.
+            remote_config(dict): Remote config to be passed into `repo_factory`.
 
         Examples:
             - Opening a filesystem from repo in current working directory
@@ -125,7 +136,15 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         super().__init__()
         self._repo_stack = ExitStack()
         if repo is None:
-            repo = self._make_repo(url=url, rev=rev, subrepos=subrepos, **repo_kwargs)
+            url = url if url is not None else fo
+            repo = self._make_repo(
+                url=url,
+                rev=rev,
+                subrepos=subrepos,
+                config=config,
+                remote=remote,
+                remote_config=remote_config,
+            )
             assert repo is not None
             # pylint: disable=protected-access
             repo_factory = repo._fs_conf["repo_factory"]
@@ -139,7 +158,7 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
             self.repo_factory = repo_factory
 
         def _getcwd():
-            relparts = ()
+            relparts: Tuple[str, ...] = ()
             assert repo is not None
             if repo.fs.path.isin(repo.fs.path.getcwd(), repo.root_dir):
                 relparts = repo.fs.path.relparts(repo.fs.path.getcwd(), repo.root_dir)
@@ -162,13 +181,26 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         if hasattr(repo, "dvc_dir"):
             self._datafss[key] = DataFileSystem(index=repo.index.data["repo"])
 
+    @functools.cached_property
+    def fsid(self) -> str:
+        from fsspec.utils import tokenize
+
+        from dvc.scm import NoSCM
+
+        return "dvcfs_" + tokenize(
+            self.repo.url or self.repo.root_dir,
+            self.repo.get_rev() if not isinstance(self.repo.scm, NoSCM) else None,
+        )
+
     def _get_key(self, path: "StrPath") -> Key:
+        path = os.fspath(path)
         parts = self.repo.fs.path.relparts(path, self.repo.root_dir)
         if parts == (os.curdir,):
             return ()
         return parts
 
     def _get_key_from_relative(self, path) -> Key:
+        path = self._strip_protocol(path)
         parts = self.path.relparts(path, self.root_marker)
         if parts and parts[0] == os.curdir:
             return parts[1:]
@@ -261,15 +293,12 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         try:
             return self.repo.fs.open(fs_path, mode=mode)
         except FileNotFoundError:
-            repo, dvc_fs, subkey = self._get_subrepo_info(key)
+            _, dvc_fs, subkey = self._get_subrepo_info(key)
             if not dvc_fs:
                 raise
 
         dvc_path = _get_dvc_path(dvc_fs, subkey)
-        kw = {}
-        if kwargs.get("cache_remote_stream", False):
-            kw["cache_odb"] = repo.cache.local
-        return dvc_fs.open(dvc_path, mode=mode, **kw)
+        return dvc_fs.open(dvc_path, mode=mode, cache=kwargs.get("cache", False))
 
     def isdvc(self, path, **kwargs) -> bool:
         """Is this entry dvc-tracked?"""
@@ -352,7 +381,7 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
                 dvc_info = dvc_fs.fs.index.info(subkey)
                 dvc_path = _get_dvc_path(dvc_fs, subkey)
                 dvc_info["name"] = dvc_path
-            except FileNotFoundError:
+            except KeyError:
                 pass
 
         fs_info = None
@@ -415,6 +444,10 @@ class DVCFileSystem(FileSystem):
     def fs(self) -> "_DVCFileSystem":
         return _DVCFileSystem(**self.fs_args)
 
+    @property
+    def fsid(self) -> str:
+        return self.fs.fsid
+
     def isdvc(self, path, **kwargs) -> bool:
         return self.fs.isdvc(path, **kwargs)
 

dvc/ignore.py

@@ -148,7 +148,9 @@ class DvcIgnorePatterns(DvcIgnore):
         return bool(self.pattern_list)
 
 
-CheckIgnoreResult = namedtuple("CheckIgnoreResult", ["file", "match", "patterns"])
+CheckIgnoreResult = namedtuple(  # noqa: PYI024
+    "CheckIgnoreResult", ["file", "match", "patterns"]
+)
 
 
 def _no_match(path):

dvc/logger.py

@@ -160,7 +160,7 @@ class LoggerHandler(logging.StreamHandler):
             self.flush()
         except (BrokenPipeError, RecursionError):
             raise
-        except Exception:  # noqa, pylint: disable=broad-except
+        except Exception:  # noqa: BLE001, pylint: disable=broad-except
             self.handleError(record)
 
 

dvc/machine/__init__.py

@@ -1,227 +0,0 @@
-import logging
-import os
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Dict,
-    Iterable,
-    Iterator,
-    Mapping,
-    Optional,
-    Tuple,
-    Type,
-)
-
-from dvc.exceptions import DvcException
-
-if TYPE_CHECKING:
-    from dvc.repo import Repo
-    from dvc.types import StrPath
-
-    from .backend.base import BaseMachineBackend
-
-    BackendCls = Type[BaseMachineBackend]
-
-logger = logging.getLogger(__name__)
-
-
-RESERVED_NAMES = {"local", "localhost"}
-
-DEFAULT_STARTUP_SCRIPT = """#!/bin/bash
-sudo apt-get update
-sudo apt-get install --yes python3 python3-pip
-python3 -m pip install --upgrade pip
-
-pushd /etc/apt/sources.list.d
-sudo wget https://dvc.org/deb/dvc.list
-sudo apt-get update
-sudo apt-get install --yes dvc
-popd
-
-sudo echo "OK" > /var/log/dvc-machine-init.log
-"""
-
-
-def validate_name(name: str):
-    from dvc.exceptions import InvalidArgumentError
-
-    name = name.lower()
-    if name in RESERVED_NAMES:
-        raise InvalidArgumentError(
-            f"Machine name '{name}' is reserved for internal DVC use."
-        )
-
-
-class MachineBackends(Mapping):
-    try:
-        from .backend.terraform import TerraformBackend
-    except ImportError:
-        TerraformBackend = None  # type: ignore[assignment, misc]
-
-    DEFAULT: Dict[str, Optional["BackendCls"]] = {
-        "terraform": TerraformBackend,
-    }
-
-    def __getitem__(self, key: str) -> "BaseMachineBackend":
-        """Lazily initialize backends and cache it afterwards"""
-        initialized = self.initialized.get(key)
-        if not initialized:
-            backend = self.backends[key]
-            initialized = backend(os.path.join(self.tmp_dir, key), **self.kwargs)
-            self.initialized[key] = initialized
-        return initialized
-
-    def __init__(
-        self,
-        selected: Optional[Iterable[str]],
-        tmp_dir: "StrPath",
-        **kwargs,
-    ) -> None:
-        selected = selected or list(self.DEFAULT)
-        self.backends: Dict[str, "BackendCls"] = {}
-        for key in selected:
-            cls = self.DEFAULT.get(key)
-            if cls is None:
-                raise DvcException(
-                    f"'dvc machine' backend '{key}' is missing required "
-                    "dependencies. Install them with:\n"
-                    f"\tpip install dvc[{key}]"
-                )
-            self.backends[key] = cls
-
-        self.initialized: Dict[str, "BaseMachineBackend"] = {}
-
-        self.tmp_dir = tmp_dir
-        self.kwargs = kwargs
-
-    def __iter__(self):
-        return iter(self.backends)
-
-    def __len__(self) -> int:
-        return len(self.backends)
-
-    def close_initialized(self) -> None:
-        for backend in self.initialized.values():
-            backend.close()
-
-
-class MachineManager:
-    """Class that manages dvc cloud machines.
-
-    Args:
-        repo (dvc.repo.Repo): repo instance that belongs to the repo that
-            we are working on.
-
-    Raises:
-        config.ConfigError: thrown when config has invalid format.
-    """
-
-    CLOUD_BACKENDS = {
-        "aws": "terraform",
-        "azure": "terraform",
-    }
-
-    def __init__(
-        self, repo: "Repo", backends: Optional[Iterable[str]] = None, **kwargs
-    ):
-        self.repo = repo
-        assert self.repo.tmp_dir
-        tmp_dir = os.path.join(self.repo.tmp_dir, "machine")
-        self.backends = MachineBackends(backends, tmp_dir=tmp_dir, **kwargs)
-
-    def get_config_and_backend(
-        self,
-        name: Optional[str] = None,
-    ) -> Tuple[dict, "BaseMachineBackend"]:
-        from dvc.config import NoMachineError
-
-        if not name:
-            name = self.repo.config["core"].get("machine")
-
-        if name:
-            config = self._get_config(name=name)
-            backend = self._get_backend(config["cloud"])
-            return config, backend
-
-        if bool(self.repo.config["machine"]):
-            error_msg = (
-                "no machine specified. Setup default machine with\n"
-                "    dvc machine default <name>\n"
-            )
-        else:
-            error_msg = (
-                "no machine specified. Create a default machine with\n"
-                "    dvc machine add -d <name> <cloud>"
-            )
-
-        raise NoMachineError(error_msg)
-
-    def _get_config(self, **kwargs):
-        config = self.repo.config
-        name = kwargs.get("name")
-        if name:
-            try:
-                conf = config["machine"][name.lower()]
-                conf["name"] = name
-            except KeyError:
-                from dvc.config import MachineNotFoundError
-
-                raise MachineNotFoundError(  # noqa: B904
-                    f"machine '{name}' doesn't exist"
-                )
-        else:
-            conf = kwargs
-        return conf
-
-    def _get_backend(self, cloud: str) -> "BaseMachineBackend":
-        from dvc.config import NoMachineError
-
-        try:
-            backend = self.CLOUD_BACKENDS[cloud]
-            return self.backends[backend]
-        except KeyError:
-            raise NoMachineError(  # noqa: B904
-                f"Machine platform '{cloud}' unsupported"
-            )
-
-    def create(self, name: Optional[str]):
-        """Create and start the specified machine instance."""
-        config, backend = self.get_config_and_backend(name)
-        if "startup_script" in config:
-            with open(config["startup_script"], encoding="utf-8") as fobj:
-                startup_script = fobj.read()
-        else:
-            startup_script = DEFAULT_STARTUP_SCRIPT
-        config["startup_script"] = startup_script
-        config.pop("setup_script", None)
-        return backend.create(**config)
-
-    def destroy(self, name: Optional[str]):
-        """Destroy the specified machine instance."""
-        config, backend = self.get_config_and_backend(name)
-        return backend.destroy(**config)
-
-    def get_sshfs(self, name: Optional[str]):
-        config, backend = self.get_config_and_backend(name)
-        return backend.get_sshfs(**config)
-
-    def run_shell(self, name: Optional[str]):
-        config, backend = self.get_config_and_backend(name)
-        return backend.run_shell(**config)
-
-    def status(self, name: str) -> Iterator[Dict[Any, Any]]:
-        config, backend = self.get_config_and_backend(name)
-        return backend.instances(**config)
-
-    def rename(self, name: str, new: str):
-        """move configuration to a new location if the machine is running."""
-        config, backend = self.get_config_and_backend(name)
-        return backend.rename(new=new, **config)
-
-    def get_executor_kwargs(self, name: Optional[str]):
-        config, backend = self.get_config_and_backend(name)
-        return backend.get_executor_kwargs(**config)
-
-    def get_setup_script(self, name: Optional[str]) -> Optional[str]:
-        config, _backend = self.get_config_and_backend(name)
-        return config.get("setup_script")

dvc/machine/backend/base.py

@@ -1,50 +0,0 @@
-from abc import ABC, abstractmethod
-from contextlib import contextmanager
-from typing import TYPE_CHECKING, Iterator, Optional
-
-if TYPE_CHECKING:
-    from dvc_ssh import SSHFileSystem
-
-    from dvc.types import StrPath
-
-
-class BaseMachineBackend(ABC):
-    def __init__(self, tmp_dir: "StrPath", **kwargs):
-        self.tmp_dir = tmp_dir
-
-    @abstractmethod
-    def create(self, name: Optional[str] = None, **config):
-        """Create and start an instance of the specified machine."""
-
-    @abstractmethod
-    def destroy(self, name: Optional[str] = None, **config):
-        """Stop and destroy all instances of the specified machine."""
-
-    @abstractmethod
-    def instances(self, name: Optional[str] = None, **config) -> Iterator[dict]:
-        """Iterate over status of all instances of the specified machine."""
-
-    def close(self):  # noqa: B027
-        pass
-
-    @abstractmethod
-    def run_shell(self, name: Optional[str] = None, **config):
-        """Spawn an interactive SSH shell for the specified machine."""
-
-    @abstractmethod
-    def get_executor_kwargs(self, name: str, **config) -> dict:
-        """Return SSHExecutor kwargs which can be used for DVC
-        experiment/pipeline execution on the specified machine.
-        """
-
-    @abstractmethod
-    @contextmanager
-    def get_sshfs(
-        self, name: Optional[str] = None, **config
-    ) -> Iterator["SSHFileSystem"]:
-        """Return an sshfs instance for the default directory on the
-        specified machine."""
-
-    @abstractmethod
-    def rename(self, name: str, new: str, **config):
-        """Rename a machine instance."""

dvc/machine/backend/terraform.py

@@ -1,99 +0,0 @@
-import os
-from contextlib import contextmanager
-from functools import partial, partialmethod
-from typing import TYPE_CHECKING, Iterator, Optional
-
-from dvc_ssh import DEFAULT_PORT, SSHFileSystem
-
-from dvc.exceptions import DvcException
-
-from .base import BaseMachineBackend
-
-if TYPE_CHECKING:
-    from dvc.types import StrPath
-
-
-@contextmanager
-def _sshfs(resource: dict):
-    from tpi import TerraformProviderIterative
-
-    with TerraformProviderIterative.pemfile(resource) as pem:
-        fs = SSHFileSystem(
-            host=resource["instance_ip"],
-            user="ubuntu",
-            keyfile=pem,
-        )
-        yield fs
-
-
-class TerraformBackend(BaseMachineBackend):
-    def __init__(self, tmp_dir: "StrPath", **kwargs):
-        super().__init__(tmp_dir, **kwargs)
-        os.makedirs(tmp_dir, exist_ok=True)
-
-    @contextmanager
-    def make_tpi(self, name: str):
-        from tpi import TPIError
-        from tpi.terraform import TerraformBackend as TPIBackend
-
-        try:
-            working_dir = os.path.join(self.tmp_dir, name)
-            os.makedirs(working_dir, exist_ok=True)
-            yield TPIBackend(working_dir=working_dir)
-        except TPIError as exc:
-            raise DvcException("TPI operation failed") from exc
-
-    def _tpi_func(self, fname, name: Optional[str] = None, **config):
-        from tpi import TPIError
-
-        assert name
-        with self.make_tpi(name) as tpi:
-            func = getattr(tpi, fname)
-            try:
-                return func(name=name, **config)
-            except TPIError as exc:
-                raise DvcException(f"TPI {fname} failed") from exc
-
-    create = partialmethod(_tpi_func, "create")  # type: ignore[assignment]
-    destroy = partialmethod(_tpi_func, "destroy")  # type: ignore[assignment]
-    instances = partialmethod(_tpi_func, "instances")  # type: ignore[assignment]
-    run_shell = partialmethod(_tpi_func, "run_shell")  # type: ignore[assignment]
-
-    def get_executor_kwargs(self, name: str, **config) -> dict:
-        with self.make_tpi(name) as tpi:
-            resource = tpi.default_resource(name)
-        return {
-            "host": resource["instance_ip"],
-            "port": DEFAULT_PORT,
-            "username": "ubuntu",
-            "fs_factory": partial(_sshfs, dict(resource)),
-        }
-
-    @contextmanager
-    def get_sshfs(  # pylint: disable=unused-argument
-        self, name: Optional[str] = None, **config
-    ) -> Iterator["SSHFileSystem"]:
-        assert name
-        with self.make_tpi(name) as tpi:
-            resource = tpi.default_resource(name)
-        with _sshfs(resource) as fs:
-            yield fs
-
-    def rename(self, name: str, new: str, **config):
-        """rename a dvc machine instance to a new name"""
-        import shutil
-
-        mtype = "iterative_machine"
-
-        new_dir = os.path.join(self.tmp_dir, new)
-        old_dir = os.path.join(self.tmp_dir, name)
-        if os.path.exists(new_dir):
-            raise DvcException(f"rename failed: path {new_dir} already exists")
-
-        if not os.path.exists(old_dir):
-            return
-
-        with self.make_tpi(name) as tpi:
-            tpi.state_mv(f"{mtype}.{name}", f"{mtype}.{new}", **config)
-
-        shutil.move(old_dir, new_dir)

dvc/output.py

@@ -44,7 +44,6 @@ from .utils.fs import path_isin
 if TYPE_CHECKING:
     from dvc_data.hashfile.obj import HashFile
     from dvc_data.index import DataIndexKey
-    from dvc_objects.db import ObjectDB
 
     from .ignore import DvcIgnoreFilter
 
@@ -97,6 +96,7 @@ def loadd_from(stage, d_list):
         files = d.pop(Output.PARAM_FILES, None)
         push = d.pop(Output.PARAM_PUSH, True)
         hash_name = d.pop(Output.PARAM_HASH, None)
+        fs_config = d.pop(Output.PARAM_FS_CONFIG, None)
         ret.append(
             _get(
                 stage,
@@ -111,6 +111,7 @@ def loadd_from(stage, d_list):
                 files=files,
                 push=push,
                 hash_name=hash_name,
+                fs_config=fs_config,
             )
         )
     return ret
@@ -313,6 +314,7 @@ class Output:
     PARAM_PUSH = "push"
     PARAM_CLOUD = "cloud"
     PARAM_HASH = "hash"
+    PARAM_FS_CONFIG = "fs_config"
 
     DoesNotExistError: Type[DvcException] = OutputDoesNotExistError
     IsNotFileOrDirError: Type[DvcException] = OutputIsNotFileOrDirError
@@ -351,6 +353,7 @@ class Output:
         if meta.version_id or files:
             fs_kwargs["version_aware"] = True
 
+        self.def_fs_config = fs_config
         if fs_config is not None:
             fs_kwargs.update(**fs_config)
 
@@ -718,7 +721,7 @@ class Output:
         if self.metric:
             self.verify_metric()
 
-        self._update_legacy_hash_name()
+        self.update_legacy_hash_name()
         if self.use_cache:
             _, self.meta, self.obj = self._build(
                 self.cache,
@@ -742,8 +745,8 @@ class Output:
         self.hash_info = self.obj.hash_info
         self.files = None
 
-    def _update_legacy_hash_name(self):
-        if self.hash_name == "md5-dos2unix" and self.changed_checksum():
+    def update_legacy_hash_name(self, force: bool = False):
+        if self.hash_name == "md5-dos2unix" and (force or self.changed_checksum()):
             self.hash_name = "md5"
 
     def set_exec(self) -> None:
@@ -787,13 +790,18 @@ class Output:
                     self.hash_name,
                     ignore=self.dvcignore,
                 )
-                otransfer(
-                    staging,
-                    self.cache,
-                    {obj.hash_info},
-                    shallow=False,
-                    hardlink=hardlink,
-                )
+                with Callback.as_tqdm_callback(
+                    desc=f"Committing {self} to cache",
+                    unit="file",
+                ) as cb:
+                    otransfer(
+                        staging,
+                        self.cache,
+                        {obj.hash_info},
+                        shallow=False,
+                        hardlink=hardlink,
+                        callback=cb,
+                    )
             if relink:
                 rel = self.fs.path.relpath(filter_info or self.fs_path)
                 with CheckoutCallback(desc=f"Checking out {rel}", unit="files") as cb:
@@ -822,13 +830,18 @@ class Output:
         save_obj = obj.filter(prefix)
         assert isinstance(save_obj, Tree)
         checkout_obj = save_obj.get_obj(self.cache, prefix)
-        otransfer(
-            staging,
-            self.cache,
-            {save_obj.hash_info} | {oid for _, _, oid in save_obj},
-            shallow=True,
-            hardlink=hardlink,
-        )
+        with Callback.as_tqdm_callback(
+            desc=f"Committing {self} to cache",
+            unit="file",
+        ) as cb:
+            otransfer(
+                staging,
+                self.cache,
+                {save_obj.hash_info} | {oid for _, _, oid in save_obj},
+                shallow=True,
+                hardlink=hardlink,
+                callback=cb,
+            )
         return checkout_obj
 
     def dumpd(self, **kwargs):  # noqa: C901, PLR0912
@@ -862,6 +875,9 @@ class Output:
 
         ret[self.PARAM_PATH] = path
 
+        if self.def_fs_config:
+            ret[self.PARAM_FS_CONFIG] = self.def_fs_config
+
         if not self.IS_DEPENDENCY:
             ret.update(self.annot.to_dict())
             if not self.use_cache:
@@ -1048,14 +1064,19 @@ class Output:
             upload=upload,
             no_progress_bar=no_progress_bar,
         )
-        otransfer(
-            staging,
-            odb,
-            {obj.hash_info},
-            jobs=jobs,
-            hardlink=False,
-            shallow=False,
-        )
+        with Callback.as_tqdm_callback(
+            desc=f"Transferring to {odb.fs.unstrip_protocol(odb.path)}",
+            unit="file",
+        ) as cb:
+            otransfer(
+                staging,
+                odb,
+                {obj.hash_info},
+                jobs=jobs,
+                hardlink=False,
+                shallow=False,
+                callback=cb,
+            )
 
         self.hash_info = obj.hash_info
         self.files = None
@@ -1075,7 +1096,7 @@ class Output:
         return len(obj) if obj else 0
 
     def unprotect(self):
-        if self.exists:
+        if self.exists and self.use_cache:
             self.cache.unprotect(self.fs_path)
 
     def get_dir_cache(self, **kwargs) -> Optional["Tree"]:
@@ -1127,7 +1148,7 @@ class Output:
             if not force and not prompt.confirm(msg.format(self.fs_path)):
                 raise CollectCacheError(  # noqa: B904
                     "unable to fully collect used cache"
-                    " without cache for directory '{}'".format(self)
+                    f" without cache for directory '{self}'"
                 )
             return None
 
@@ -1139,10 +1160,11 @@ class Output:
             return obj.filter(prefix)
         return obj
 
-    def get_used_objs(  # noqa: C901
+    def get_used_objs(  # noqa: C901, PLR0911
         self, **kwargs
-    ) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
+    ) -> Dict[Optional["HashFileDB"], Set["HashInfo"]]:
         """Return filtered set of used object IDs for this out."""
+        from dvc.cachemgr import LEGACY_HASH_NAMES
 
         if not self.use_cache:
             return {}
@@ -1156,19 +1178,15 @@ class Output:
 
         if not self.hash_info:
             msg = (
-                "Output '{}'({}) is missing version info. "
+                f"Output '{self}'({self.stage}) is missing version info. "
                 "Cache for it will not be collected. "
-                "Use `dvc repro` to get your pipeline up to date.".format(
-                    self, self.stage
-                )
+                "Use `dvc repro` to get your pipeline up to date."
             )
             if self.exists:
                 msg += (
                     "\n"
-                    "You can also use `dvc commit {stage.addressing}` "
-                    "to associate existing '{out}' with {stage}.".format(
-                        out=self, stage=self.stage
-                    )
+                    f"You can also use `dvc commit {self.stage.addressing}` "
+                    f"to associate existing '{self}' with {self.stage}."
                 )
             logger.warning(msg)
             return {}
@@ -1185,13 +1203,17 @@ class Output:
             return {}
 
         if self.remote:
-            remote = self.repo.cloud.get_remote_odb(
+            remote_odb = self.repo.cloud.get_remote_odb(
                 name=self.remote, hash_name=self.hash_name
             )
-        else:
-            remote = None
-
-        return {remote: self._named_obj_ids(obj)}
+            other_odb = self.repo.cloud.get_remote_odb(
+                name=self.remote,
+                hash_name=(
+                    "md5" if self.hash_name in LEGACY_HASH_NAMES else "md5-dos2unix"
+                ),
+            )
+            return {remote_odb: self._named_obj_ids(obj), other_odb: set()}
+        return {None: self._named_obj_ids(obj)}
 
     def _named_obj_ids(self, obj):
         name = str(self)
@@ -1369,7 +1391,7 @@ class Output:
             )
 
         assert self.repo
-        self._update_legacy_hash_name()
+        self.update_legacy_hash_name()
         cache = self.cache if self.use_cache else self.local_cache
         assert isinstance(cache, HashFileDB)
 
@@ -1418,7 +1440,18 @@ class Output:
 
         assert staging
         assert obj.hash_info
-        otransfer(staging, self.cache, {obj.hash_info}, hardlink=relink, shallow=False)
+        with Callback.as_tqdm_callback(
+            desc=f"Adding {self} to cache",
+            unit="file",
+        ) as cb:
+            otransfer(
+                staging,
+                self.cache,
+                {obj.hash_info},
+                hardlink=relink,
+                shallow=False,
+                callback=cb,
+            )
 
         if relink:
             with CheckoutCallback(
@@ -1515,4 +1548,5 @@ SCHEMA = {
     Output.PARAM_REMOTE: str,
     Output.PARAM_PUSH: bool,
     Output.PARAM_FILES: [DIR_FILES_SCHEMA],
+    Output.PARAM_FS_CONFIG: dict,
 }

dvc/parsing/__init__.py

@@ -2,6 +2,7 @@ import logging
 import os
 from collections.abc import Mapping, Sequence
 from copy import deepcopy
+from itertools import product
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -51,6 +52,7 @@ VARS_KWD = "vars"
 WDIR_KWD = "wdir"
 PARAMS_KWD = "params"
 FOREACH_KWD = "foreach"
+MATRIX_KWD = "matrix"
 DO_KWD = "do"
 
 DEFAULT_PARAMS_FILE = "params.yaml"
@@ -105,7 +107,7 @@ def is_map_or_seq(data: Any) -> bool:
     return not isinstance(data, str) and _is_map_or_seq(data)
 
 
-def split_foreach_name(name: str) -> Tuple[str, Optional[str]]:
+def split_group_name(name: str) -> Tuple[str, Optional[str]]:
     group, *keys = name.rsplit(JOIN, maxsplit=1)
     return group, first(keys)
 
@@ -120,13 +122,15 @@ def check_interpolations(data: "DictStrAny", where: str, path: str):
     return recurse(func)(data)
 
 
-Definition = Union["ForeachDefinition", "EntryDefinition"]
+Definition = Union["ForeachDefinition", "EntryDefinition", "MatrixDefinition"]
 
 
 def make_definition(
     resolver: "DataResolver", name: str, definition: "DictStrAny", **kwargs
 ) -> Definition:
     args = resolver, resolver.context, name, definition
+    if MATRIX_KWD in definition:
+        return MatrixDefinition(*args, **kwargs)
     if FOREACH_KWD in definition:
         return ForeachDefinition(*args, **kwargs)
     return EntryDefinition(*args, **kwargs)
@@ -159,7 +163,8 @@ class DataResolver:
         self.tracked_vars: Dict[str, Mapping] = {}
 
         stages_data = d.get(STAGES_KWD, {})
-        # we wrap the definitions into ForeachDefinition and EntryDefinition,
+        # we wrap the definitions into:
+        # ForeachDefinition, MatrixDefinition, and EntryDefinition
         # that helps us to optimize, cache and selectively load each one of
         # them as we need, and simplify all of this DSL/parsing logic.
         self.definitions: Dict[str, Definition] = {
@@ -168,12 +173,13 @@ class DataResolver:
         }
 
     def resolve_one(self, name: str):
-        group, key = split_foreach_name(name)
+        group, key = split_group_name(name)
 
         if not self._has_group_and_key(group, key):
             raise EntryNotFound(f"Could not find '{name}'")
 
-        # all of the checks for `key` not being None for `ForeachDefinition`
+        # all of the checks for `key` not being None for
+        # `ForeachDefinition`/`MatrixDefinition`
         # and/or `group` not existing in the `interim`, etc. should be
         # handled by the `self.has_key()` above.
         definition = self.definitions[group]
@@ -190,7 +196,7 @@ class DataResolver:
         return {STAGES_KWD: data}
 
     def has_key(self, key: str):
-        return self._has_group_and_key(*split_foreach_name(key))
+        return self._has_group_and_key(*split_group_name(key))
 
     def _has_group_and_key(self, group: str, key: Optional[str] = None):
         try:
@@ -198,16 +204,14 @@ class DataResolver:
         except KeyError:
             return False
 
-        if key:
-            return isinstance(definition, ForeachDefinition) and definition.has_member(
-                key
-            )
-        return not isinstance(definition, ForeachDefinition)
+        if not isinstance(definition, (ForeachDefinition, MatrixDefinition)):
+            return key is None
+        return key is not None and definition.has_member(key)
 
     @collecting
     def get_keys(self):
         for name, definition in self.definitions.items():
-            if isinstance(definition, ForeachDefinition):
+            if isinstance(definition, (ForeachDefinition, MatrixDefinition)):
                 yield from definition.get_generated_names()
                 continue
             yield name
@@ -256,8 +260,9 @@ class EntryDefinition:
         name = self.name
         if not skip_checks:
             # we can check for syntax errors as we go for interpolated entries,
-            # but for foreach-generated ones, once is enough, which it does
-            # that itself. See `ForeachDefinition.do_definition`.
+            # but for foreach and matrix generated ones, once is enough, which it does
+            # that itself. See `ForeachDefinition.template`
+            # and `MatrixDefinition.template`.
             check_syntax_errors(self.definition, name, self.relpath)
 
         # we need to pop vars from generated/evaluated data
@@ -331,17 +336,18 @@ class ForeachDefinition:
         self.name = name
 
         assert DO_KWD in definition
+        assert MATRIX_KWD not in definition
         self.foreach_data = definition[FOREACH_KWD]
-        self._do_definition = definition[DO_KWD]
+        self._template = definition[DO_KWD]
 
         self.pair = IterationPair()
         self.where = where
 
     @cached_property
-    def do_definition(self):
+    def template(self):
         # optimization: check for syntax errors only once for `foreach` stages
-        check_syntax_errors(self._do_definition, self.name, self.relpath)
-        return self._do_definition
+        check_syntax_errors(self._template, self.name, self.relpath)
+        return self._template
 
     @cached_property
     def resolved_iterable(self):
@@ -444,16 +450,111 @@ class ForeachDefinition:
             # i.e. quadratic complexity).
             generated = self._generate_name(key)
             entry = EntryDefinition(
-                self.resolver, self.context, generated, self.do_definition
+                self.resolver, self.context, generated, self.template
             )
             try:
                 # optimization: skip checking for syntax errors on each foreach
-                # generated stages. We do it once when accessing do_definition.
+                # generated stages. We do it once when accessing template.
                 return entry.resolve_stage(skip_checks=True)
             except ContextError as exc:
                 format_and_raise(exc, f"stage '{generated}'", self.relpath)
 
-            # let mypy know that this state is unreachable as format_and_raise
-            # does not return at all (it's not able to understand it for some
-            # reason)
-            raise AssertionError("unreachable")
+
+class MatrixDefinition:
+    def __init__(
+        self,
+        resolver: DataResolver,
+        context: Context,
+        name: str,
+        definition: "DictStrAny",
+        where: str = STAGES_KWD,
+    ):
+        self.resolver = resolver
+        self.relpath = self.resolver.relpath
+        self.context = context
+        self.name = name
+
+        assert MATRIX_KWD in definition
+        assert DO_KWD not in definition
+        assert FOREACH_KWD not in definition
+
+        self._template = definition.copy()
+        self.matrix_data = self._template.pop(MATRIX_KWD)
+
+        self.pair = IterationPair()
+        self.where = where
+
+    @cached_property
+    def template(self) -> "DictStrAny":
+        # optimization: check for syntax errors only once for `matrix` stages
+        check_syntax_errors(self._template, self.name, self.relpath)
+        return self._template
+
+    @cached_property
+    def resolved_iterable(self) -> Dict[str, List]:
+        return self._resolve_matrix_data()
+
+    def _resolve_matrix_data(self) -> Dict[str, List]:
+        try:
+            iterable = self.context.resolve(self.matrix_data, unwrap=False)
+        except (ContextError, ParseError) as exc:
+            format_and_raise(exc, f"'{self.where}.{self.name}.matrix'", self.relpath)
+        return iterable
+
+    @cached_property
+    def normalized_iterable(self) -> Dict[str, "DictStrAny"]:
+        """Convert sequence to Mapping with keys normalized."""
+        iterable = self.resolved_iterable
+        assert isinstance(iterable, Mapping)
+
+        ret: Dict[str, "DictStrAny"] = {}
+        matrix = {key: enumerate(v) for key, v in iterable.items()}
+        for combination in product(*matrix.values()):
+            d: "DictStrAny" = {}
+            fragments: List[str] = []
+            for k, (i, v) in zip(matrix.keys(), combination):
+                d[k] = v
+                fragments.append(f"{k}{i}" if is_map_or_seq(v) else to_str(v))
+
+            key = "-".join(fragments)
+            ret[key] = d
+        return ret
+
+    def has_member(self, key: str) -> bool:
+        return key in self.normalized_iterable
+
+    def get_generated_names(self) -> List[str]:
+        return list(map(self._generate_name, self.normalized_iterable))
+
+    def _generate_name(self, key: str) -> str:
+        return f"{self.name}{JOIN}{key}"
+
+    def resolve_all(self) -> "DictStrAny":
+        return join(map(self.resolve_one, self.normalized_iterable))
+
+    def resolve_one(self, key: str) -> "DictStrAny":
+        return self._each_iter(key)
+
+    def _each_iter(self, key: str) -> "DictStrAny":
+        err_message = f"Could not find '{key}' in matrix group '{self.name}'"
+        with reraise(KeyError, EntryNotFound(err_message)):
+            value = self.normalized_iterable[key]
+
+        temp_dict = {self.pair.value: value}
+        with self.context.set_temporarily(temp_dict, reserve=True):
+            # optimization: item and key can be removed on __exit__() as they
+            # are top-level values, and are not merged recursively.
+            # This helps us avoid cloning context, which is slower
+            # (increasing the size of the context might increase
+            # the no. of items to be generated which means more cloning,
+            # i.e. quadratic complexity).
+            generated = self._generate_name(key)
+            entry = EntryDefinition(
+                self.resolver, self.context, generated, self.template
+            )
+            try:
+                # optimization: skip checking for syntax errors on each matrix
+                # generated stages. We do it once when accessing template.
+                return entry.resolve_stage(skip_checks=True)
+            except ContextError as exc:
+                format_and_raise(exc, f"stage '{generated}'", self.relpath)

dvc/pathspec_math.py

@@ -8,7 +8,7 @@ from pathspec.util import normalize_file
 
 from dvc.utils import relpath
 
-PatternInfo = namedtuple("PatternInfo", ["patterns", "file_info"])
+PatternInfo = namedtuple("PatternInfo", ["patterns", "file_info"])  # noqa: PYI024
 
 
 def _not_ignore(rule):

dvc/render/match.py

@@ -8,7 +8,7 @@ import dpath.options
 from funcy import get_in, last
 
 from dvc.repo.plots import _normpath, infer_data_sources
-from dvc.utils.plots import get_plot_id
+from dvc.utils.plots import group_definitions_by_id
 
 from .convert import _get_converter
 
@@ -36,14 +36,9 @@ class PlotsData:
     def group_definitions(self):
         groups = defaultdict(list)
         for rev, rev_content in self.data.items():
-            for config_file, config_file_content in (
-                rev_content.get("definitions", {}).get("data", {}).items()
-            ):
-                for plot_id, plot_definition in config_file_content.get(
-                    "data", {}
-                ).items():
-                    full_id = get_plot_id(plot_id, config_file)
-                    groups[full_id].append((rev, plot_id, plot_definition))
+            definitions = rev_content.get("definitions", {}).get("data", {})
+            for plot_id, definition in group_definitions_by_id(definitions).items():
+                groups[plot_id].append((rev, *definition))
         return dict(groups)
 
     def get_definition_data(self, target_files, rev):

dvc/repo/__init__.py

@@ -15,8 +15,6 @@ from typing import (
 
 from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
 from dvc.ignore import DvcIgnoreFilter
-from dvc.utils import env2bool
-from dvc.utils.fs import path_isin
 from dvc.utils.objects import cached_property
 
 if TYPE_CHECKING:
@@ -24,7 +22,6 @@ if TYPE_CHECKING:
     from dvc.fs.data import DataFileSystem
     from dvc.fs.dvc import DVCFileSystem
     from dvc.lock import LockBase
-    from dvc.machine import MachineManager
     from dvc.scm import Git, NoSCM
     from dvc.stage import Stage
     from dvc.types import DictStrAny
@@ -93,6 +90,7 @@ class Repo:
     from dvc.repo.status import status  # type: ignore[misc]
     from dvc.repo.update import update  # type: ignore[misc]
 
+    from .cache import check_missing as cache_check_missing  # type: ignore[misc]
     from .data import status as data_status  # type: ignore[misc]
 
     ls = staticmethod(_ls)
@@ -133,7 +131,7 @@ class Repo:
         assert root_dir
         return root_dir, dvc_dir
 
-    def __init__(  # noqa: PLR0915
+    def __init__(  # noqa: PLR0915, PLR0913
         self,
         root_dir: Optional[str] = None,
         fs: Optional["FileSystem"] = None,
@@ -144,6 +142,8 @@ class Repo:
         url: Optional[str] = None,
         repo_factory: Optional[Callable] = None,
         scm: Optional[Union["Git", "NoSCM"]] = None,
+        remote: Optional[str] = None,
+        remote_config: Optional["DictStrAny"] = None,
     ):
         from dvc.cachemgr import CacheManager
         from dvc.data_cloud import DataCloud
@@ -163,6 +163,8 @@ class Repo:
         self._fs = fs or localfs
         self._scm = scm
         self._config = config
+        self._remote = remote
+        self._remote_config = remote_config
         self._data_index = None
 
         if rev and not fs:
@@ -237,7 +239,14 @@ class Repo:
     def config(self):
         from dvc.config import Config
 
-        return Config(self.dvc_dir, fs=self.fs, config=self._config)
+        return Config(
+            self.dvc_dir,
+            local_dvc_dir=self.local_dvc_dir,
+            fs=self.fs,
+            config=self._config,
+            remote=self._remote,
+            remote_config=self._remote_config,
+        )
 
     @cached_property
     def local_dvc_dir(self):
@@ -340,16 +349,6 @@ class Repo:
 
         return Experiments(self)
 
-    @cached_property
-    def machine(self) -> Optional["MachineManager"]:
-        from dvc.machine import MachineManager
-
-        if self.tmp_dir and (
-            self.config["feature"].get("machine", False) or env2bool("DVC_TEST")
-        ):
-            return MachineManager(self)
-        return None
-
     @property
     def fs(self) -> "FileSystem":
         return self._fs
@@ -411,7 +410,7 @@ class Repo:
 
         fs = fs or localfs
         root = root or os.curdir
-        root_dir = fs.path.realpath(root)
+        root_dir = fs.path.abspath(root)
 
         if not fs.isdir(root_dir):
             raise NotDvcRepoError(f"directory '{root}' does not exist")
@@ -455,8 +454,9 @@ class Repo:
         flist = [self.config.files["local"]]
         if tmp_dir := self.tmp_dir:
             flist.append(tmp_dir)
-        if path_isin(self.cache.legacy.path, self.root_dir):
-            flist.append(self.cache.legacy.path)
+
+        if cache_dir := self.cache.default_local_cache_dir:
+            flist.append(cache_dir)
 
         for file in flist:
             self.scm_context.ignore(file)
@@ -487,7 +487,7 @@ class Repo:
         used_run_cache=None,
         revs=None,
         num=1,
-        push: bool = False,  # noqa: F811
+        push: bool = False,
     ):
         """Get the stages related to the given target and collect
         the `info` of its outputs.
@@ -577,6 +577,26 @@ class Repo:
 
         return DVCFileSystem(repo=self, subrepos=self.subrepos, **self._fs_conf)
 
+    @cached_property
+    def _btime(self):
+        if not self.tmp_dir:
+            return None
+
+        # Not all python versions/filesystems/platforms provide creation
+        # time (st_birthtime, stx_btime, etc), so we use our own dummy
+        # file and its mtime instead.
+        path = os.path.join(self.tmp_dir, "btime")
+
+        try:
+            with open(path, "x"):  # pylint: disable=unspecified-encoding
+                pass
+        except FileNotFoundError:
+            return None
+        except FileExistsError:
+            pass
+
+        return os.path.getmtime(path)
+
     @cached_property
     def site_cache_dir(self) -> str:
         import getpass
@@ -605,12 +625,24 @@ class Repo:
         finally:
             os.umask(umask)
 
+        # NOTE: Some number to change the generated token if none of the
+        # components were changed (useful to prevent newer dvc versions from
+        # using older broken cache). Please reset this back to 0 if other parts
+        # of the token components are changed.
+        salt = 1
+
+        # NOTE: This helps us avoid accidentally reusing cache for repositories
+        # that just happened to be at the same path as old deleted ones.
+        btime = self._btime or getattr(os.stat(root_dir), "st_birthtime", None)
+
         md5 = hashlib.md5(  # noqa: S324  # nosec B324, B303
             str(
                 (
                     root_dir,
+                    btime,
                     getpass.getuser(),
                     version_tuple[0],
+                    salt,
                 )
             ).encode()
         )

dvc/repo/add.py

@@ -120,7 +120,7 @@ def translate_graph_error(stages: List["Stage"]) -> Iterator[None]:
         )
     except OutputDuplicationError as exc:
         raise OutputDuplicationError(  # noqa: B904
-            exc.output, list(set(exc.stages) - set(stages))
+            exc.output, set(exc.stages) - set(stages)
         )
 
 

dvc/repo/artifacts.py

@@ -1,42 +1,61 @@
 import logging
-import re
+import os
+import posixpath
 from pathlib import Path
-from typing import Dict, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
 
 from dvc.annotations import Artifact
 from dvc.dvcfile import PROJECT_FILE
-from dvc.exceptions import InvalidArgumentError
-from dvc.repo import Repo
-from dvc.utils import relpath
+from dvc.exceptions import (
+    ArtifactNotFoundError,
+    DvcException,
+    FileExistsLocallyError,
+    InvalidArgumentError,
+)
+from dvc.utils import relpath, resolve_output
+from dvc.utils.objects import cached_property
 from dvc.utils.serialize import modify_yaml
 
+if TYPE_CHECKING:
+    from gto.tag import Tag as GTOTag
+    from scmrepo.git import GitTag
+
+    from dvc.repo import Repo
+    from dvc.scm import Git
+
 logger = logging.getLogger(__name__)
 
 
-# Constants are taken from GTO.
-# When we make it a dependency, we can import them instead
-SEPARATOR_IN_NAME = ":"
-DIRNAME = r"[a-z0-9-_./]+"
-NAME = r"[a-z0-9]([a-z0-9-/]*[a-z0-9])?"
-NAME_RE = re.compile(f"^{NAME}$")
-FULLNAME = f"((?P<dirname>{DIRNAME}){SEPARATOR_IN_NAME})?(?P<name>{NAME})"
-FULLNAME_RE = re.compile(f"^{FULLNAME}$")
+def check_name_format(name: str) -> None:
+    from gto.constants import assert_name_is_valid
+    from gto.exceptions import ValidationError
+
+    try:
+        assert_name_is_valid(name)
+    except ValidationError as exc:
+        raise InvalidArgumentError(
+            f"Can't use '{name}' as artifact name (ID)."
+        ) from exc
 
 
 def name_is_compatible(name: str) -> bool:
-    return bool(NAME_RE.search(name))
+    """
+    Only needed by DVCLive per iterative/dvclive#715
+    Will be removed in future release.
+    """
+    from gto.constants import assert_name_is_valid
+    from gto.exceptions import ValidationError
 
-
-def check_name_format(name: str) -> None:
-    if not name_is_compatible(name):
-        raise InvalidArgumentError(
-            f"Can't use '{name}' as artifact name (ID)."
-            " You can use letters and numbers, and use '-' as separator"
-            " (but not at the start or end)."
-        )
+    try:
+        assert_name_is_valid(name)
+        return True
+    except ValidationError:
+        return False
 
 
 def check_for_nested_dvc_repo(dvcfile: Path):
+    from dvc.repo import Repo
+
     if dvcfile.is_absolute():
         raise InvalidArgumentError("Use relative path to dvc.yaml.")
     path = dvcfile.parent
@@ -48,11 +67,36 @@ def check_for_nested_dvc_repo(dvcfile: Path):
         path = path.parent
 
 
+def _reformat_name(name: str) -> str:
+    from gto.constants import SEPARATOR_IN_NAME, fullname_re
+
+    # NOTE: DVC accepts names like
+    #   path/to/dvc.yaml:artifact_name
+    # but Studio/GTO tags are generated with
+    #   path/to:artifact_name
+    m = fullname_re.match(name)
+    if m and m.group("dirname"):
+        group = m.group("dirname").rstrip(SEPARATOR_IN_NAME)
+        dirname, basename = posixpath.split(group)
+        if basename == PROJECT_FILE:
+            name = f"{dirname}{SEPARATOR_IN_NAME}{m.group('name')}"
+    return name
+
+
 class Artifacts:
     def __init__(self, repo: "Repo") -> None:
         self.repo = repo
 
+    @cached_property
+    def scm(self) -> Optional["Git"]:
+        from dvc.scm import Git
+
+        if isinstance(self.repo.scm, Git):
+            return self.repo.scm
+        return None
+
     def read(self) -> Dict[str, Dict[str, Artifact]]:
+        """Read artifacts from dvc.yaml."""
         artifacts: Dict[str, Dict[str, Artifact]] = {}
         for (
             dvcfile,
@@ -69,6 +113,7 @@ class Artifacts:
         return artifacts
 
     def add(self, name: str, artifact: Artifact, dvcfile: Optional[str] = None):
+        """Add artifact to dvc.yaml."""
         with self.repo.scm_context(quiet=True):
             check_name_format(name)
             dvcyaml = Path(dvcfile or PROJECT_FILE)
@@ -85,3 +130,219 @@ class Artifacts:
             self.repo.scm_context.track_file(dvcfile)
 
         return artifacts.get(name)
+
+    def get_rev(
+        self, name: str, version: Optional[str] = None, stage: Optional[str] = None
+    ):
+        """Return revision containing the given artifact."""
+        from gto.base import sort_versions
+        from gto.tag import find, parse_tag
+
+        assert not (version and stage)
+        name = _reformat_name(name)
+        tags: List["GitTag"] = find(
+            name=name, version=version, stage=stage, scm=self.scm
+        )
+        if not tags:
+            raise ArtifactNotFoundError(name, version=version, stage=stage)
+        if version or stage:
+            return tags[-1].target
+        gto_tags: List["GTOTag"] = sort_versions(parse_tag(tag) for tag in tags)
+        return gto_tags[0].tag.target
+
+    def get_path(self, name: str):
+        """Return repo fspath for the given artifact."""
+        from gto.constants import SEPARATOR_IN_NAME, fullname_re
+
+        name = _reformat_name(name)
+        m = fullname_re.match(name)
+        if not m:
+            raise ArtifactNotFoundError(name)
+        dirname = m.group("dirname")
+        if dirname:
+            dirname = dirname.rstrip(SEPARATOR_IN_NAME)
+        dvcyaml = os.path.join(dirname, PROJECT_FILE) if dirname else PROJECT_FILE
+        artifact_name = m.group("name")
+        try:
+            artifact = self.read()[dvcyaml][artifact_name]
+        except KeyError as exc:
+            raise ArtifactNotFoundError(name) from exc
+        return os.path.join(dirname, artifact.path) if dirname else artifact.path
+
+    def download(
+        self,
+        name: str,
+        version: Optional[str] = None,
+        stage: Optional[str] = None,
+        out: Optional[str] = None,
+        force: bool = False,
+        jobs: Optional[int] = None,
+    ) -> Tuple[int, str]:
+        """Download the specified artifact."""
+        from dvc.fs import download as fs_download
+
+        logger.debug("Trying to download artifact '%s' via DVC", name)
+        rev = self.get_rev(name, version=version, stage=stage)
+        with self.repo.switch(rev):
+            path = self.get_path(name)
+            out = resolve_output(path, out, force=force)
+            fs = self.repo.dvcfs
+            fs_path = fs.from_os_path(path)
+            count = fs_download(
+                fs,
+                fs_path,
+                os.path.abspath(out),
+                jobs=jobs,
+            )
+        return count, out
+
+    @staticmethod
+    def _download_studio(
+        repo_url: str,
+        name: str,
+        version: Optional[str] = None,
+        stage: Optional[str] = None,
+        out: Optional[str] = None,
+        force: bool = False,
+        jobs: Optional[int] = None,
+        studio_config: Optional[Dict[str, Any]] = None,
+        **kwargs,
+    ) -> Tuple[int, str]:
+        from dvc_studio_client.model_registry import get_download_uris
+
+        from dvc.fs import Callback, HTTPFileSystem, generic, localfs
+
+        logger.debug("Trying to download artifact '%s' via studio", name)
+        out = out or os.getcwd()
+        to_infos: List[str] = []
+        from_infos: List[str] = []
+        if studio_config is None:
+            studio_config = {}
+        studio_config["repo_url"] = repo_url
+        try:
+            kwargs["dvc_studio_config"] = studio_config
+            for path, url in get_download_uris(
+                repo_url,
+                name,
+                version=version,
+                stage=stage,
+                **kwargs,
+            ).items():
+                to_info = localfs.path.join(out, path)
+                if localfs.exists(to_info) and not force:
+                    hint = "\nTo override it, re-run with '--force'."
+                    raise FileExistsLocallyError(  # noqa: TRY301
+                        relpath(to_info), hint=hint
+                    )
+                to_infos.append(to_info)
+                from_infos.append(url)
+        except DvcException:
+            raise
+        except Exception as exc:  # noqa: BLE001, pylint: disable=broad-except
+            raise DvcException(
+                f"Failed to download artifact '{name}' via Studio"
+            ) from exc
+        fs = HTTPFileSystem()
+        jobs = jobs or fs.jobs
+        with Callback.as_tqdm_callback(
+            desc=f"Downloading '{name}' from '{repo_url}'",
+            unit="files",
+        ) as cb:
+            cb.set_size(len(from_infos))
+            generic.copy(
+                fs, from_infos, localfs, to_infos, callback=cb, batch_size=jobs
+            )
+
+        return len(to_infos), relpath(localfs.path.commonpath(to_infos))
+
+    @classmethod
+    def get(  # noqa: C901, PLR0913
+        cls,
+        url: str,
+        name: str,
+        version: Optional[str] = None,
+        stage: Optional[str] = None,
+        config: Optional[Union[str, Dict[str, Any]]] = None,
+        remote: Optional[str] = None,
+        remote_config: Optional[Union[str, Dict[str, Any]]] = None,
+        out: Optional[str] = None,
+        force: bool = False,
+        jobs: Optional[int] = None,
+    ):
+        from dvc.config import Config
+        from dvc.repo import Repo
+
+        if version and stage:
+            raise InvalidArgumentError(
+                "Artifact version and stage are mutually exclusive."
+            )
+
+        # NOTE: We try to download the artifact up to three times
+        # 1. via studio with studio config loaded from environment
+        # 2. via studio with studio config loaded from DVC repo 'studio'
+        #    section + environment
+        # 3. via DVC remote
+
+        name = _reformat_name(name)
+        saved_exc: Optional[Exception] = None
+        try:
+            logger.trace("Trying studio-only config")  # type: ignore[attr-defined]
+            return cls._download_studio(
+                url,
+                name,
+                version=version,
+                stage=stage,
+                out=out,
+                force=force,
+                jobs=jobs,
+            )
+        except FileExistsLocallyError:
+            raise
+        except Exception as exc:  # noqa: BLE001, pylint: disable=broad-except
+            saved_exc = exc
+
+        if config and not isinstance(config, dict):
+            config = Config.load_file(config)
+        with Repo.open(
+            url=url,
+            subrepos=True,
+            uninitialized=True,
+            config=config,
+            remote=remote,
+            remote_config=remote_config,
+        ) as repo:
+            logger.trace("Trying repo [studio] config")  # type: ignore[attr-defined]
+            studio_config = dict(repo.config.get("studio"))
+            try:
+                return cls._download_studio(
+                    url,
+                    name,
+                    version=version,
+                    stage=stage,
+                    out=out,
+                    force=force,
+                    jobs=jobs,
+                    dvc_studio_config=studio_config,
+                )
+            except FileExistsLocallyError:
+                raise
+            except Exception as exc:  # noqa: BLE001, pylint: disable=broad-except
+                saved_exc = exc
+
+            try:
+                return repo.artifacts.download(
+                    name,
+                    version=version,
+                    stage=stage,
+                    out=out,
+                    force=force,
+                    jobs=jobs,
+                )
+            except FileExistsLocallyError:
+                raise
+            except Exception as exc:  # noqa: BLE001, pylint: disable=broad-except
+                if saved_exc:
+                    logger.exception(str(saved_exc), exc_info=saved_exc.__cause__)
+                raise DvcException(
+                    f"Failed to download artifact '{name}' via DVC remote"
+                ) from exc

dvc/repo/brancher.py

@@ -57,11 +57,11 @@ def brancher(  # noqa: E302
 
     from dvc.fs import LocalFileSystem
 
-    repo_root_parts = ()
+    repo_root_parts: Tuple[str, ...] = ()
     if self.fs.path.isin(self.root_dir, self.scm.root_dir):
         repo_root_parts = self.fs.path.relparts(self.root_dir, self.scm.root_dir)
 
-    cwd_parts = ()
+    cwd_parts: Tuple[str, ...] = ()
     if self.fs.path.isin(self.fs.path.getcwd(), self.scm.root_dir):
         cwd_parts = self.fs.path.relparts(self.fs.path.getcwd(), self.scm.root_dir)
 
@@ -144,11 +144,11 @@ def switch(repo: "Repo", rev: str) -> Iterator[str]:
     if rev != "workspace":
         rev = resolve_rev(repo.scm, rev)
 
-    repo_root_parts = ()
+    repo_root_parts: Tuple[str, ...] = ()
     if repo.fs.path.isin(repo.root_dir, repo.scm.root_dir):
         repo_root_parts = repo.fs.path.relparts(repo.root_dir, repo.scm.root_dir)
 
-    cwd_parts = ()
+    cwd_parts: Tuple[str, ...] = ()
     if repo.fs.path.isin(repo.fs.path.getcwd(), repo.scm.root_dir):
         cwd_parts = repo.fs.path.relparts(repo.fs.path.getcwd(), repo.scm.root_dir)
 

dvc/repo/cache.py

@@ -0,0 +1,39 @@
+import os
+
+
+def check_missing(repo, rev=None, max_size=None, types=None):
+    from dvc_data.index import StorageKeyError
+
+    with repo.switch(rev or "workspace"):
+        idx = repo.index.targets_view(
+            None,
+            max_size=max_size,
+            types=types,
+        )
+
+        index = idx.data["repo"]
+
+    def onerror(_entry, _exc):
+        pass
+
+    index.onerror = onerror
+
+    ret = []
+    for _, entry in index.iteritems():
+        try:
+            fs, path = index.storage_map.get_cache(entry)
+        except (StorageKeyError, ValueError):
+            continue
+
+        if not fs.exists(path):
+            typ = "directory" if (entry.meta and entry.meta.isdir) else "file"
+            ret.append(
+                (
+                    typ,
+                    entry.hash_info.name,
+                    entry.hash_info.value,
+                    os.path.join(*entry.key),
+                )
+            )
+
+    return ret

dvc/repo/checkout.py

@@ -51,20 +51,16 @@ def _build_out_changes(index, changes):
         out_keys.append(key)
 
     out_changes = {}
-    for objects in changes.values():
-        for change in objects:
-            for out_key in out_keys:
-                if (
-                    len(out_key) > len(change.key)
-                    or change.key[: len(out_key)] != out_key
-                ):
-                    continue
+    for key, change in changes.items():
+        for out_key in out_keys:
+            if len(out_key) > len(key) or key[: len(out_key)] != out_key:
+                continue
 
-                if change.key == out_key:
-                    out_changes[out_key] = change.typ
-                elif not out_changes.get(out_key):
-                    out_changes[out_key] = MODIFY
-                break
+            if key == out_key:
+                out_changes[out_key] = change.typ
+            elif not out_changes.get(out_key):
+                out_changes[out_key] = MODIFY
+            break
 
     return out_changes
 
@@ -85,7 +81,7 @@ def _check_can_delete(
         if cache_fs.exists(cache_path):
             continue
 
-        entry_paths.append(fs.path.join(path, *entry.key))
+        entry_paths.append(fs.path.join(path, *(entry.key or ())))
 
     if not entry_paths:
         return
@@ -144,7 +140,8 @@ def checkout(  # noqa: C901
 
     with ui.progress(
         unit="entry",
-        desc="Building data index",
+        desc="Building workspace index",
+        leave=True,
     ) as pb:
         old = build_data_index(
             view,
@@ -159,6 +156,7 @@ def checkout(  # noqa: C901
     with ui.progress(
         desc="Comparing indexes",
         unit="entry",
+        leave=True,
     ) as pb:
         diff = compare(old, new, relink=relink, delete=True, callback=pb.as_callback())
 
@@ -179,7 +177,8 @@ def checkout(  # noqa: C901
 
     with ui.progress(
         unit="file",
-        desc="Checkout",
+        desc="Applying changes",
+        leave=True,
     ) as pb:
         apply(
             diff,

dvc/repo/commit.py

@@ -1,6 +1,12 @@
+from typing import TYPE_CHECKING
+
 from dvc import prompt
 
 from . import locked
+from .scm_context import scm_context
+
+if TYPE_CHECKING:
+    from . import Repo
 
 
 def _prepare_message(stage, changes):
@@ -68,3 +74,48 @@ def commit(
         )
         stage.dump(update_pipeline=False)
     return [s.stage for s in stages_info]
+
+
+@locked
+@scm_context
+def commit_2_to_3(repo: "Repo", dry: bool = False):
+    """Force-commit all legacy outputs to use DVC 3.0 hashes."""
+    from dvc.dvcfile import ProjectFile
+    from dvc.ui import ui
+
+    view = repo.index.targets_view(
+        targets=None,
+        outs_filter=lambda o: o.hash_name == "md5-dos2unix",
+        recursive=True,
+    )
+    migrated = set()
+    for out in view.outs:
+        dvcfile = out.stage.dvcfile.relpath
+        if isinstance(out.stage.dvcfile, ProjectFile):
+            # pylint: disable-next=protected-access
+            lockfile = out.stage.dvcfile._lockfile.relpath
+            migrated.add(f"{dvcfile} ({lockfile})")
+        else:
+            migrated.add(dvcfile)
+    if not migrated:
+        ui.write("No DVC files in the repo to migrate to the 3.0 format.")
+        return
+    if dry:
+        ui.write("Entries in following DVC files will be migrated to the 3.0 format:")
+        ui.write("\n".join(sorted(f"\t{file}" for file in migrated)))
+        return
+    for stage, filter_info in view._stage_infos:  # pylint: disable=protected-access
+        outs_filter = view._outs_filter  # pylint: disable=protected-access
+        outs = {
+            out
+            for out in stage.filter_outs(filter_info)
+            if outs_filter is not None and outs_filter(out)
+        }
+        if outs:
+            for out in outs:
+                out.update_legacy_hash_name(force=True)
+            stage.save(allow_missing=True)
+            stage.commit(allow_missing=True, relink=True)
+            if not isinstance(stage.dvcfile, ProjectFile):
+                ui.write(f"Updating DVC file '{stage.dvcfile.relpath}'")
+            stage.dump(update_pipeline=False)

dvc/repo/diff.py

@@ -22,7 +22,7 @@ def _hash(entry):
     return None
 
 
-def _diff(old, new, with_missing=False):
+def _diff(old, new, data_keys, with_missing=False):
     from dvc_data.index.diff import ADD, DELETE, MODIFY, RENAME
     from dvc_data.index.diff import diff as idiff
 
@@ -34,12 +34,23 @@ def _diff(old, new, with_missing=False):
         "not in cache": [],
     }
 
+    def meta_cmp_key(meta):
+        if not meta:
+            return meta
+        return meta.isdir
+
     for change in idiff(
         old,
         new,
         with_renames=True,
-        hash_only=True,
+        meta_cmp_key=meta_cmp_key,
+        roots=data_keys,
     ):
+        if (change.old and change.old.isdir and not change.old.hash_info) or (
+            change.new and change.new.isdir and not change.new.hash_info
+        ):
+            continue
+
         if change.typ == ADD:
             ret["added"].append(
                 {
@@ -116,6 +127,7 @@ def diff(
         b_rev = "workspace"
         with_missing = True
 
+    data_keys = set()
     for rev in self.brancher(revs=[a_rev, b_rev]):
         if rev == "workspace" and b_rev != "workspace":
             # brancher always returns workspace, but we only need to compute
@@ -132,6 +144,8 @@ def diff(
             recursive=recursive,
         )
 
+        data_keys.update(view.data_keys.get("repo", set()))
+
         if rev == "workspace":
             from .index import build_data_index
 
@@ -165,4 +179,4 @@ def diff(
         new = indexes[b_rev]
 
     with ui.status("Calculating diff"):
-        return _diff(old, new, with_missing=with_missing)
+        return _diff(old, new, data_keys, with_missing=with_missing)

dvc/repo/experiments/__init__.py

@@ -136,10 +136,6 @@ class Experiments:
         **kwargs,
     ) -> "QueueEntry":
         """Queue a single experiment."""
-        if kwargs.pop("machine", None) is not None:
-            # TODO: decide how to handle queued remote execution
-            raise NotImplementedError
-
         return self.new(
             queue,
             **kwargs,
@@ -214,10 +210,6 @@ class Experiments:
             )
         else:
             ui.write("Experiment results have been applied to your workspace.")
-        ui.write(
-            "\nTo promote an experiment to a Git branch run:\n\n"
-            "\tdvc exp branch <exp> <branch>\n"
-        )
 
     def new(
         self,
@@ -249,7 +241,7 @@ class Experiments:
                 self.check_baseline(last_applied)
             return last_applied
         except BaselineMismatchError:
-            # If HEAD has moved since the the last applied experiment,
+            # If HEAD has moved since the last applied experiment,
             # the applied experiment is no longer relevant
             self.scm.remove_ref(EXEC_APPLY)
         return None
@@ -403,6 +395,11 @@ class Experiments:
 
         return remove(self.repo, *args, **kwargs)
 
+    def rename(self, *args, **kwargs):
+        from dvc.repo.experiments.rename import rename
+
+        return rename(self.repo, *args, **kwargs)
+
     def clean(self, *args, **kwargs):
         from dvc.repo.experiments.clean import clean
 

dvc/repo/experiments/collect.py

@@ -23,6 +23,7 @@ from dvc.scm import Git, SCMError, iter_revs
 from .exceptions import InvalidExpRefError
 from .refs import EXEC_BRANCH, ExpRefInfo
 from .serialize import ExpRange, ExpState, SerializableError, SerializableExp
+from .utils import describe
 
 if TYPE_CHECKING:
     from dvc.repo import Repo
@@ -308,7 +309,9 @@ def collect(
     if sha_only:
         baseline_names: Dict[str, Optional[str]] = {}
     else:
-        baseline_names = _describe(repo.scm, baseline_revs, refs=cached_refs)
+        baseline_names = describe(
+            repo.scm, baseline_revs, refs=cached_refs, logger=logger
+        )
 
     workspace_data = collect_rev(repo, "workspace", **kwargs)
     result: List["ExpState"] = [workspace_data]
@@ -337,55 +340,6 @@ def collect(
     return result
 
 
-def _describe(
-    scm: "Git",
-    revs: Iterable[str],
-    refs: Optional[Iterable[str]] = None,
-) -> Dict[str, Optional[str]]:
-    """Describe revisions using a tag, branch.
-
-    The first matching name will be returned for each rev. Names are preferred in this
-    order:
-        - current branch (if rev matches HEAD and HEAD is a branch)
-        - tags
-        - branches
-
-    Returns:
-        Dict mapping revisions from revs to a name.
-    """
-
-    head_rev = scm.get_rev()
-    head_ref = scm.get_ref("HEAD", follow=False)
-    if head_ref and head_ref.startswith("refs/heads/"):
-        head_branch = head_ref[len("refs/heads/") :]
-    else:
-        head_branch = None
-
-    tags = {}
-    branches = {}
-    ref_it = iter(refs) if refs else scm.iter_refs()
-    for ref in ref_it:
-        is_tag = ref.startswith("refs/tags/")
-        is_branch = ref.startswith("refs/heads/")
-        if not (is_tag or is_branch):
-            continue
-        rev = scm.get_ref(ref)
-        if not rev:
-            logger.debug("unresolved ref %s", ref)
-            continue
-        if is_tag and rev not in tags:
-            tags[rev] = ref[len("refs/tags/") :]
-        if is_branch and rev not in branches:
-            branches[rev] = ref[len("refs/heads/") :]
-    names: Dict[str, Optional[str]] = {}
-    for rev in revs:
-        if rev == head_rev and head_branch:
-            names[rev] = head_branch
-        else:
-            names[rev] = tags.get(rev) or branches.get(rev)
-    return names
-
-
 def _sorted_ranges(exp_ranges: Iterable["ExpRange"]) -> List["ExpRange"]:
     """Return list of ExpRange sorted by (timestamp, rev)."""
 

dvc/repo/experiments/exceptions.py

@@ -24,8 +24,6 @@ class ExperimentExistsError(DvcException):
             "Experiment conflicts with existing experiment "
             f"'{name}'. To overwrite the existing experiment run:\n\n"
             f"\tdvc exp {command} -f ...\n\n"
-            f"To {command} this experiment with a different name run:\n\n"
-            f"\tdvc exp {command} -n <new_name> ...\n"
         )
         super().__init__(msg)
         self.name = name

dvc/repo/experiments/executor/base.py

@@ -23,7 +23,6 @@ from typing import (
     Union,
 )
 
-from funcy import get_in
 from scmrepo.exceptions import SCMError
 
 from dvc.env import DVC_EXP_AUTO_PUSH, DVC_EXP_GIT_REMOTE
@@ -129,7 +128,6 @@ class BaseExecutor(ABC):
 
     PACKED_ARGS_FILE = "repro.dat"
     WARN_UNTRACKED = False
-    QUIET = False
     INFOFILE_EXT = ".run"
     DEFAULT_LOCATION: str = "workspace"
 
@@ -255,15 +253,6 @@ class BaseExecutor(ABC):
             **kwargs,
         )
 
-    @classmethod
-    def _get_stage_files(cls, stages: List["Stage"]) -> List[str]:
-        from dvc.stage.utils import _get_stage_files
-
-        ret: List[str] = []
-        for stage in stages:
-            ret.extend(_get_stage_files(stage))
-        return ret
-
     @classmethod
     def _get_top_level_paths(cls, repo: "Repo") -> List["str"]:
         return list(
@@ -306,7 +295,7 @@ class BaseExecutor(ABC):
             stages = dvc.commit([], force=True, relink=False)
             exp_hash = cls.hash_exp(stages)
             if include_untracked:
-                dvc.scm.add(include_untracked)
+                dvc.scm.add(include_untracked, force=True)  # type: ignore[call-arg]
             cls.commit(
                 dvc.scm,  # type: ignore[arg-type]
                 exp_hash,
@@ -519,10 +508,8 @@ class BaseExecutor(ABC):
                     recursive=kwargs.get("recursive", False),
                 )
 
+            kwargs["repro_fn"] = cls._repro_and_track
             stages = dvc.reproduce(*args, **kwargs)
-            if paths := cls._get_stage_files(stages):
-                logger.debug("Staging stage-related files: %s", paths)
-                dvc.scm_context.add(paths)
             if paths := cls._get_top_level_paths(dvc):
                 logger.debug("Staging top-level files: %s", paths)
                 dvc.scm_context.add(paths)
@@ -547,6 +534,17 @@ class BaseExecutor(ABC):
         # multiprocessing calls
         return ExecutorResult(exp_hash, exp_ref, repro_force)
 
+    @staticmethod
+    def _repro_and_track(stage: "Stage", **kwargs) -> Optional["Stage"]:
+        from dvc.repo.reproduce import _reproduce_stage
+        from dvc.stage.utils import _get_stage_files
+
+        ret = _reproduce_stage(stage, **kwargs)
+        if not kwargs.get("dry") and (paths := _get_stage_files(stage)):
+            logger.debug("Staging stage-related files: %s", paths)
+            stage.repo.scm_context.add(paths)
+        return ret
+
     @classmethod
     def _repro_commit(
         cls,
@@ -602,12 +600,11 @@ class BaseExecutor(ABC):
             info.status = TaskStatus.RUNNING
             if infofile is not None:
                 info.dump_json(infofile)
-            if cls.QUIET:
-                dvc.scm_context.quiet = cls.QUIET
+            dvc.scm_context.quiet = True
             old_cwd = os.getcwd()
 
             for path in copy_paths or []:
-                cls._copy_path(os.path.realpath(path), os.path.join(dvc.root_dir, path))
+                cls._copy_path(os.path.abspath(path), os.path.join(dvc.root_dir, path))
 
             if info.wdir:
                 os.chdir(os.path.join(dvc.scm.root_dir, info.wdir))
@@ -646,13 +643,15 @@ class BaseExecutor(ABC):
                 info.status = TaskStatus.FAILED
                 raise
             finally:
+                from dvc.repo.metrics.show import _gather_metrics
+
                 post_live_metrics(
                     "done",
                     info.baseline_rev,
                     info.name,  # type: ignore[arg-type]
                     "dvc",
                     experiment_rev=dvc.experiments.scm.get_ref(EXEC_BRANCH),
-                    metrics=get_in(dvc.metrics.show(), ["", "data"]),
+                    metrics=_gather_metrics(dvc, on_error="return"),
                     dvc_studio_config=dvc_studio_config,
                 )
 

dvc/repo/experiments/executor/local.py

@@ -65,7 +65,6 @@ class TempDirExecutor(BaseLocalExecutor):
     # debugging user code), and suppress other DVC hints (like `git add`
     # suggestions) that are not applicable outside of workspace runs
     WARN_UNTRACKED = True
-    QUIET = True
     DEFAULT_LOCATION = "tempdir"
 
     @retry(180, errors=LockError, timeout=1)

dvc/repo/experiments/executor/ssh.py

@@ -1,292 +0,0 @@
-import logging
-import os
-import posixpath
-import sys
-from contextlib import contextmanager
-from typing import TYPE_CHECKING, Callable, Iterable, List, Optional
-
-from dvc_ssh import SSHFileSystem
-from funcy import first
-
-from dvc.repo.experiments.refs import (
-    EXEC_BASELINE,
-    EXEC_BRANCH,
-    EXEC_HEAD,
-    EXEC_MERGE,
-    EXEC_NAMESPACE,
-)
-
-from .base import BaseExecutor, ExecutorInfo, ExecutorResult, TaskStatus
-
-if TYPE_CHECKING:
-    from queue import Queue
-
-    from dvc.repo import Repo
-    from dvc.repo.experiments.refs import ExpRefInfo
-    from dvc.repo.experiments.stash import ExpStashEntry
-    from dvc.scm import Git
-
-logger = logging.getLogger(__name__)
-
-
-@contextmanager
-def _sshfs(fs_factory, **kwargs):
-    if fs_factory:
-        with fs_factory() as fs:
-            yield fs
-        return
-    yield SSHFileSystem(**kwargs)
-
-
-class SSHExecutor(BaseExecutor):
-    """SSH experiment executor."""
-
-    WARN_UNTRACKED = True
-    QUIET = True
-    SETUP_SCRIPT_FILENAME = "exec-setup.sh"
-
-    def __init__(
-        self,
-        *args,
-        host: Optional[str] = None,
-        port: Optional[int] = None,
-        username: Optional[str] = None,
-        fs_factory: Optional[Callable] = None,
-        setup_script: Optional[str] = None,
-        **kwargs,
-    ):
-        assert host
-
-        super().__init__(*args, **kwargs)
-        self.host: str = host
-        self.port = port
-        self.username = username
-        self._fs_factory = fs_factory
-        self._repo_abspath = ""
-        self._setup_script = setup_script
-
-    @classmethod
-    def gen_dirname(cls, name: Optional[str] = None):
-        from shortuuid import uuid
-
-        return "-".join([name or "dvc-exp", "executor", uuid()])
-
-    @classmethod
-    def from_stash_entry(
-        cls,
-        repo: "Repo",
-        entry: "ExpStashEntry",
-        **kwargs,
-    ):
-        machine_name: Optional[str] = kwargs.pop("machine_name", None)
-        assert repo.machine
-        executor = cls._from_stash_entry(
-            repo,
-            entry,
-            cls.gen_dirname(entry.name),
-            location=machine_name,
-            **repo.machine.get_executor_kwargs(machine_name),
-            setup_script=repo.machine.get_setup_script(machine_name),
-        )
-        logger.debug("Init SSH executor for host '%s'", executor.host)
-        return executor
-
-    def sshfs(self):
-        return _sshfs(self._fs_factory, host=self.host, port=self.port)
-
-    @property
-    def git_url(self) -> str:
-        user = f"{self.username}@" if self.username else ""
-        port = f":{self.port}" if self.port is not None else ""
-        path = f"{self.root_dir}" if self.root_dir else ""
-        if path and not posixpath.isabs(path):
-            path = f"/~/{path}"
-        return f"ssh://{user}{self.host}{port}{path}"
-
-    @property
-    def abs_url(self) -> str:
-        assert self._repo_abspath
-        user = f"{self.username}@" if self.username else ""
-        port = f":{self.port}" if self.port is not None else ""
-        return f"ssh://{user}{self.host}{port}{self._repo_abspath}"
-
-    @staticmethod
-    def _git_client_args(fs):
-        return {
-            "password": fs.fs_args.get("password"),
-            "key_filename": first(fs.fs_args.get("client_keys", [])),
-        }
-
-    def init_git(
-        self,
-        repo: "Repo",  # noqa: ARG002
-        scm: "Git",
-        stash_rev: str,
-        entry: "ExpStashEntry",
-        infofile: Optional[str],
-        branch: Optional[str] = None,
-    ):
-        from dvc.repo.experiments.utils import push_refspec
-
-        self.status = TaskStatus.PREPARING
-        if infofile:
-            self.info.dump_json(infofile)
-
-        with self.sshfs() as fs:
-            fs.makedirs(self.root_dir)
-            self._ssh_cmd(fs, "git init .")
-            self._ssh_cmd(fs, "git config user.name dvc-exp")
-            self._ssh_cmd(fs, "git config user.email [email protected]")
-
-            result = self._ssh_cmd(fs, "pwd")
-            path = result.stdout.strip()
-            self._repo_abspath = path
-
-            # TODO: support multiple client key retries in git backends
-            # (see https://github.com/iterative/dvc/issues/6508)
-            kwargs = self._git_client_args(fs)
-
-            ref_dict = {
-                EXEC_HEAD: entry.head_rev,
-                EXEC_MERGE: stash_rev,
-                EXEC_BASELINE: entry.baseline_rev,
-            }
-            with self.set_temp_refs(scm, ref_dict):
-                exec_namespace = f"{EXEC_NAMESPACE}/"
-                refspec = [(exec_namespace, exec_namespace)]
-                push_refspec(scm, self.git_url, refspec, **kwargs)
-
-            if branch:
-                push_refspec(scm, self.git_url, [(branch, branch)], **kwargs)
-                self._ssh_cmd(fs, f"git symbolic-ref {EXEC_BRANCH} {branch}")
-            else:
-                self._ssh_cmd(fs, f"git symbolic-ref -d {EXEC_BRANCH}", check=False)
-
-            # checkout EXEC_HEAD and apply EXEC_MERGE on top of it without
-            # committing
-            head = EXEC_BRANCH if branch else EXEC_HEAD
-            self._ssh_cmd(fs, f"git checkout {head}")
-            merge_rev = scm.get_ref(EXEC_MERGE)
-            self._ssh_cmd(fs, f"git stash apply {merge_rev}")
-
-            if self._setup_script:
-                self._init_setup_script(fs)
-
-    @classmethod
-    def _setup_script_path(cls, dvc_dir: str):
-        return posixpath.join(
-            dvc_dir,
-            "tmp",
-            cls.SETUP_SCRIPT_FILENAME,
-        )
-
-    def _init_setup_script(self, fs: "SSHFileSystem"):
-        assert self._repo_abspath
-        script_path = self._setup_script_path(
-            posixpath.join(self._repo_abspath, self.dvc_dir)
-        )
-        assert self._setup_script
-        fs.put_file(self._setup_script, script_path)
-
-    def _ssh_cmd(self, sshfs, cmd, chdir=None, **kwargs):
-        working_dir = chdir or self.root_dir
-        return sshfs.fs.execute(f"cd {working_dir};{cmd}", **kwargs)
-
-    def init_cache(self, repo: "Repo", rev: str, run_cache: bool = True):
-        from dvc.repo.push import push
-
-        with self.get_odb() as odb:
-            push(
-                repo,
-                revs=[rev],
-                run_cache=run_cache,
-                odb=odb,
-                include_imports=True,
-            )
-
-    def collect_cache(
-        self, repo: "Repo", exp_ref: "ExpRefInfo", run_cache: bool = True
-    ):
-        """Collect DVC cache."""
-        from dvc.repo.experiments.pull import _pull_cache
-
-        _pull_cache(repo, exp_ref, run_cache=run_cache)
-
-    @contextmanager
-    def get_odb(self):
-        from dvc.cachemgr import CacheManager, get_odb
-
-        cache_path = posixpath.join(
-            self._repo_abspath,
-            self.dvc_dir,
-            CacheManager.CACHE_DIR,
-        )
-
-        with self.sshfs() as fs:
-            yield get_odb(fs, cache_path, **fs.config)
-
-    def fetch_exps(self, *args, **kwargs) -> Iterable[str]:
-        with self.sshfs() as fs:
-            kwargs.update(self._git_client_args(fs))
-            return super().fetch_exps(*args, **kwargs)
-
-    @classmethod
-    def reproduce(
-        cls,
-        info: "ExecutorInfo",
-        rev: str,  # noqa: ARG003
-        queue: Optional["Queue"] = None,  # noqa: ARG003
-        infofile: Optional[str] = None,
-        log_errors: bool = True,
-        log_level: Optional[int] = None,
-        copy_paths: Optional[List[str]] = None,  # noqa: ARG003
-        message: Optional[str] = None,  # noqa: ARG003
-        **kwargs,
-    ) -> "ExecutorResult":
-        """Reproduce an experiment on a remote machine over SSH.
-
-        Internally uses 'dvc exp exec-run' over SSH.
-        """
-        import json
-        import time
-        from tempfile import TemporaryFile
-
-        from asyncssh import ProcessError
-
-        fs_factory: Optional[Callable] = kwargs.pop("fs_factory", None)
-        if log_errors and log_level is not None:
-            cls._set_log_level(log_level)
-
-        with _sshfs(fs_factory) as fs:
-            while not fs.exists("/var/log/dvc-machine-init.log"):
-                logger.info("Waiting for dvc-machine startup script to complete...")
-                time.sleep(5)
-            logger.info("Reproducing experiment on '%s'", fs.fs_args.get("host"))
-            with TemporaryFile(mode="w+", encoding="utf-8") as fobj:
-                json.dump(info.asdict(), fobj)
-                fobj.seek(0)
-                fs.put_file(fobj, infofile)
-            cmd = ["source ~/.profile"]
-            script_path = cls._setup_script_path(info.dvc_dir)
-            if fs.exists(posixpath.join(info.root_dir, script_path)):
-                cmd.extend([f"pushd {info.root_dir}", f"source {script_path}", "popd"])
-            exec_cmd = f"dvc exp exec-run --infofile {infofile}"
-            if log_level is not None:
-                if log_level <= logging.TRACE:  # type: ignore[attr-defined]
-                    exec_cmd += " -vv"
-                elif log_level <= logging.DEBUG:
-                    exec_cmd += " -v"
-            cmd.append(exec_cmd)
-            try:
-                sys.stdout.flush()
-                sys.stderr.flush()
-                stdout = os.dup(sys.stdout.fileno())
-                stderr = os.dup(sys.stderr.fileno())
-                fs.fs.execute("; ".join(cmd), stdout=stdout, stderr=stderr)
-                with fs.open(infofile) as fobj:
-                    result_info = ExecutorInfo.from_dict(json.load(fobj))
-                if result_info.result_hash:
-                    return result_info.result
-            except ProcessError:
-                pass
-            return ExecutorResult(None, None, False)

dvc/repo/experiments/ls.py

@@ -31,23 +31,15 @@ def ls(
             rev = [rev]
         revs = iter_revs(repo.scm, rev, num)
         rev_set = set(revs.keys())
+
     ref_info_dict = exp_refs_by_baseline(repo.scm, rev_set, git_remote)
-
-    tags = repo.scm.describe(ref_info_dict.keys())
-    remained = {baseline for baseline, tag in tags.items() if tag is None}
-    base = "refs/heads"
-    ref_heads = repo.scm.describe(remained, base=base)
-
     results = defaultdict(list)
     for baseline in ref_info_dict:
-        name = baseline
-        if tags[baseline] or ref_heads[baseline]:
-            name = tags[baseline] or ref_heads[baseline]
         for info in ref_info_dict[baseline]:
             if git_remote:
                 exp_rev = None
             else:
                 exp_rev = repo.scm.get_ref(str(info))
-            results[name].append((info.name, exp_rev))
+            results[baseline].append((info.name, exp_rev))
 
     return results

dvc/repo/experiments/queue/base.py

@@ -21,7 +21,7 @@ from dvc_studio_client.post_live_metrics import get_studio_config
 from funcy import retry
 
 from dvc.dependency import ParamsDependency
-from dvc.env import DVC_EXP_BASELINE_REV, DVC_EXP_NAME
+from dvc.env import DVC_EXP_BASELINE_REV, DVC_EXP_NAME, DVC_ROOT
 from dvc.lock import LockError
 from dvc.repo.experiments.exceptions import ExperimentExistsError
 from dvc.repo.experiments.executor.base import BaseExecutor
@@ -348,6 +348,9 @@ class BaseStashQueue(ABC):
                     if not name:
                         name = get_random_exp_name(self.scm, baseline_rev)
                     run_env[DVC_EXP_NAME] = name
+                    # Override DVC_ROOT env var to point to the parent DVC repo
+                    # root (and not an executor tempdir root)
+                    run_env[DVC_ROOT] = self.repo.root_dir
 
                     # save studio config to read later by dvc and dvclive
                     studio_config = get_studio_config(
@@ -476,13 +479,17 @@ class BaseStashQueue(ABC):
         hydra_output_file = ParamsDependency.DEFAULT_PARAMS_FILE
         for path, overrides in params.items():
             if hydra_enabled and path == hydra_output_file:
-                config_dir = os.path.join(
-                    self.repo.root_dir, hydra_config.get("config_dir", "conf")
-                )
+                if (config_module := hydra_config.get("config_module")) is None:
+                    config_dir = os.path.join(
+                        self.repo.root_dir, hydra_config.get("config_dir", "conf")
+                    )
+                else:
+                    config_dir = None
                 config_name = hydra_config.get("config_name", "config")
                 compose_and_dump(
                     path,
                     config_dir,
+                    config_module,
                     config_name,
                     overrides,
                 )

dvc/repo/experiments/rename.py

@@ -0,0 +1,61 @@
+import logging
+from typing import TYPE_CHECKING, Dict, List, Optional, Union
+
+from dvc.repo.experiments.exceptions import (
+    ExperimentExistsError,
+    UnresolvedExpNamesError,
+)
+from dvc.repo.experiments.utils import check_ref_format, resolve_name
+from dvc.scm import Git
+
+from .refs import ExpRefInfo
+
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+
+logger = logging.getLogger(__name__)
+
+
+def rename(
+    repo: "Repo",
+    new_name: str,
+    exp_name: Union[str, None] = None,
+    git_remote: Optional[str] = None,
+    force: bool = False,
+) -> Union[List[str], None]:
+    renamed: List[str] = []
+    remained: List[str] = []
+    assert isinstance(repo.scm, Git)
+
+    if exp_name == new_name:
+        return None
+
+    if exp_name:
+        results: Dict[str, Union[ExpRefInfo, None]] = resolve_name(
+            scm=repo.scm, exp_names=exp_name, git_remote=git_remote
+        )
+        for name, result in results.items():
+            if result is None:
+                remained.append(name)
+                continue
+
+            new_ref = ExpRefInfo(baseline_sha=result.baseline_sha, name=new_name)
+            if repo.scm.get_ref(str(new_ref)) and not force:
+                raise ExperimentExistsError(new_name)
+
+            check_ref_format(repo.scm, new_ref)
+            _rename_exp(scm=repo.scm, ref_info=result, new_name=new_name)
+            renamed.append(name)
+
+    if remained:
+        raise UnresolvedExpNamesError(remained, git_remote=git_remote)
+
+    return renamed
+
+
+def _rename_exp(scm: "Git", ref_info: "ExpRefInfo", new_name: str):
+    rev = scm.get_ref(str(ref_info))
+    scm.remove_ref(str(ref_info))
+    ref_info.name = new_name
+    scm.set_ref(str(ref_info), rev)
+    return new_name

dvc/repo/experiments/run.py

@@ -30,6 +30,9 @@ def run(  # noqa: C901, PLR0912
     Returns a dict mapping new experiment SHAs to the results
     of `repro` for that experiment.
     """
+    if kwargs.get("dry"):
+        tmp_dir = True
+
     if run_all:
         return repo.experiments.reproduce_celery(jobs=jobs)
 

dvc/repo/experiments/serialize.py

@@ -1,15 +1,16 @@
 import json
 from dataclasses import asdict, dataclass, field
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Literal, Optional
+from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Literal, Optional
 
 from dvc.exceptions import DvcException
 from dvc.repo.metrics.show import _gather_metrics
 from dvc.repo.params.show import _gather_params
-from dvc.utils import onerror_collect, relpath
+from dvc.utils import relpath
 
 if TYPE_CHECKING:
     from dvc.repo import Repo
+    from dvc.repo.metrics.show import FileResult
 
 
 class DeserializeError(DvcException):
@@ -29,8 +30,8 @@ class SerializableExp:
 
     rev: str
     timestamp: Optional[datetime] = None
-    params: Dict[str, Any] = field(default_factory=dict)
-    metrics: Dict[str, Any] = field(default_factory=dict)
+    params: Dict[str, "FileResult"] = field(default_factory=dict)
+    metrics: Dict[str, "FileResult"] = field(default_factory=dict)
     deps: Dict[str, "ExpDep"] = field(default_factory=dict)
     outs: Dict[str, "ExpOut"] = field(default_factory=dict)
     meta: Dict[str, Any] = field(default_factory=dict)
@@ -40,7 +41,6 @@ class SerializableExp:
         cls,
         repo: "Repo",
         rev: Optional[str] = None,
-        onerror: Optional[Callable] = None,
         param_deps: bool = False,
         **kwargs,
     ) -> "SerializableExp":
@@ -51,25 +51,11 @@ class SerializableExp:
         """
         from dvc.dependency import ParamsDependency, RepoDependency
 
-        if not onerror:
-            onerror = onerror_collect
-
         rev = rev or repo.get_rev()
         assert rev
-        # NOTE: _gather_params/_gather_metrics return defaultdict which is not
-        # supported in dataclasses.asdict() on all python releases
-        # see https://bugs.python.org/issue35540
-        params = dict(_gather_params(repo, deps=param_deps, onerror=onerror))
-        params = {k: dict(v) for k, v in params.items()}
-        metrics = dict(
-            _gather_metrics(
-                repo,
-                targets=None,
-                rev=rev[:7],
-                recursive=False,
-                onerror=onerror_collect,
-            )
-        )
+
+        params = _gather_params(repo, deps_only=param_deps, on_error="return")
+        metrics = _gather_metrics(repo, on_error="return")
         return cls(
             rev=rev,
             params=params,
@@ -119,11 +105,8 @@ class SerializableExp:
 
     @property
     def contains_error(self) -> bool:
-        return (
-            self.params.get("error")
-            or any(value.get("error") for value in self.params.values())
-            or self.metrics.get("error")
-            or any(value.get("error") for value in self.metrics.values())
+        return any(value.get("error") for value in self.params.values()) or any(
+            value.get("error") for value in self.metrics.values()
         )
 
 

dvc/repo/experiments/show.py

@@ -177,26 +177,31 @@ def _build_rows(
                 )
 
 
-def _sort_column(
+def _sort_column(  # noqa: C901
     sort_by: str,
     metric_names: Mapping[str, Iterable[str]],
     param_names: Mapping[str, Iterable[str]],
 ) -> Tuple[str, str, str]:
-    path, _, sort_name = sort_by.rpartition(":")
+    sep = ":"
+    parts = sort_by.split(sep)
     matches: Set[Tuple[str, str, str]] = set()
 
-    if path:
+    for split_num in range(len(parts)):
+        path = sep.join(parts[:split_num])
+        sort_name = sep.join(parts[split_num:])
+        if not path:  # handles ':metric_name' case
+            sort_by = sort_name
         if path in metric_names and sort_name in metric_names[path]:
             matches.add((path, sort_name, "metrics"))
         if path in param_names and sort_name in param_names[path]:
             matches.add((path, sort_name, "params"))
-    else:
+    if not matches:
         for path in metric_names:
-            if sort_name in metric_names[path]:
-                matches.add((path, sort_name, "metrics"))
+            if sort_by in metric_names[path]:
+                matches.add((path, sort_by, "metrics"))
         for path in param_names:
-            if sort_name in param_names[path]:
-                matches.add((path, sort_name, "params"))
+            if sort_by in param_names[path]:
+                matches.add((path, sort_by, "params"))
 
     if len(matches) == 1:
         return matches.pop()
@@ -238,19 +243,16 @@ def _exp_range_rows(
     is_base: bool = False,
     **kwargs,
 ) -> Iterator[Tuple["CellT", ...]]:
-    for i, exp in enumerate(exp_range.revs):
+    from funcy import first
+
+    if len(exp_range.revs) > 1:
+        logger.debug("Returning tip commit for legacy checkpoint exp")
+    exp = first(exp_range.revs)
+    if exp:
         row: Dict[str, "CellT"] = {k: fill_value for k in all_headers}
         row["Experiment"] = exp.name or ""
         row["rev"] = exp.rev[:7] if Git.is_sha(exp.rev) else exp.rev
-        if len(exp_range.revs) > 1:
-            if i == 0:
-                row["typ"] = "checkpoint_tip"
-            elif i == len(exp_range.revs) - 1:
-                row["typ"] = "checkpoint_base"
-            else:
-                row["typ"] = "checkpoint_commit"
-        else:
-            row["typ"] = "branch_base" if is_base else "branch_commit"
+        row["typ"] = "branch_base" if is_base else "branch_commit"
         row["parent"] = ""
         if exp_range.executor:
             row["State"] = exp_range.executor.state.capitalize()

dvc/repo/fetch.py

@@ -2,6 +2,7 @@ import logging
 from typing import List, Tuple
 
 from dvc.exceptions import DownloadError
+from dvc.ui import ui
 from dvc_data.index import DataIndex, FileStorage
 
 from . import locked
@@ -9,6 +10,14 @@ from . import locked
 logger = logging.getLogger(__name__)
 
 
+def _make_index_onerror(onerror, rev):
+    def _onerror(entry, exc):
+        if onerror:
+            return onerror(rev, entry, exc)
+
+    return _onerror
+
+
 def _collect_indexes(  # noqa: PLR0913
     repo,
     targets=None,
@@ -22,6 +31,7 @@ def _collect_indexes(  # noqa: PLR0913
     max_size=None,
     types=None,
     config=None,
+    onerror=None,
 ):
     indexes = {}
     collection_exc = None
@@ -39,7 +49,7 @@ def _collect_indexes(  # noqa: PLR0913
         all_commits=all_commits,
     ):
         try:
-            repo.config.update(config)
+            repo.config.merge(config)
 
             idx = repo.index.targets_view(
                 targets,
@@ -48,8 +58,14 @@ def _collect_indexes(  # noqa: PLR0913
                 max_size=max_size,
                 types=types,
             )
-            indexes[idx.data_tree.hash_info.value] = idx.data["repo"]
+
+            data = idx.data["repo"]
+            data.onerror = _make_index_onerror(onerror, rev)
+
+            indexes[idx.data_tree.hash_info.value] = data
         except Exception as exc:  # pylint: disable=broad-except
+            if onerror:
+                onerror(rev, None, exc)
             collection_exc = exc
             logger.exception("failed to collect '%s'", rev or "workspace")
 
@@ -75,6 +91,7 @@ def fetch(  # noqa: C901, PLR0913
     max_size=None,
     types=None,
     config=None,
+    onerror=None,
 ) -> int:
     """Download data items from a cloud and imported repositories
 
@@ -90,7 +107,6 @@ def fetch(  # noqa: C901, PLR0913
     """
     from fsspec.utils import tokenize
 
-    from dvc.fs.callbacks import Callback
     from dvc_data.index.fetch import collect
     from dvc_data.index.fetch import fetch as ifetch
 
@@ -119,33 +135,36 @@ def fetch(  # noqa: C901, PLR0913
         max_size=max_size,
         types=types,
         config=config,
+        onerror=onerror,
     )
 
     cache_key = ("fetch", tokenize(sorted(indexes.keys())))
 
-    with Callback.as_tqdm_callback(
+    with ui.progress(
         desc="Collecting",
         unit="entry",
-    ) as cb:
+        leave=True,
+    ) as pb:
         data = collect(
             indexes.values(),
             "remote",
             cache_index=self.data_index,
             cache_key=cache_key,
-            callback=cb,
+            callback=pb.as_callback(),
         )
     data, unversioned_count = _log_unversioned(data)
     failed_count += unversioned_count
 
-    with Callback.as_tqdm_callback(
+    with ui.progress(
         desc="Fetching",
-        unit="file",
-    ) as cb:
+        bar_format="{desc}",
+        leave=True,
+    ) as pb:
         try:
             fetch_transferred, fetch_failed = ifetch(
                 data,
                 jobs=jobs,
-                callback=cb,
+                callback=pb.as_callback(),
             )  # pylint: disable=assignment-from-no-return
         finally:
             for fs_index in data:

dvc/repo/gc.py

@@ -1,5 +1,5 @@
 import logging
-from typing import TYPE_CHECKING, List, Optional, Set, Tuple
+from typing import TYPE_CHECKING, List, Optional
 
 from dvc.exceptions import InvalidArgumentError
 
@@ -7,8 +7,7 @@ from . import locked
 
 if TYPE_CHECKING:
     from dvc.repo import Repo
-    from dvc_data.hashfile.hash_info import HashInfo
-    from dvc_objects.db import ObjectDB
+    from dvc.repo.index import ObjectContainer
 
 logger = logging.getLogger(__name__)
 
@@ -33,22 +32,25 @@ def _validate_args(**kwargs):
         raise InvalidArgumentError("`--num` can only be used alongside `--rev`")
 
 
-def _used_obj_ids_not_in_remote(repo, default_remote, jobs, remote_odb_to_obj_ids):
+def _used_obj_ids_not_in_remote(
+    remote_odb_to_obj_ids: "ObjectContainer", jobs: Optional[int] = None
+):
     used_obj_ids = set()
     remote_oids = set()
-    for remote_odb, obj_ids in remote_odb_to_obj_ids:
-        if remote_odb is None:
-            remote_odb = repo.cloud.get_remote_odb(default_remote, "gc --not-in-remote")
-
+    for remote_odb, obj_ids in remote_odb_to_obj_ids.items():
+        assert remote_odb
         remote_oids.update(
-            remote_odb.list_oids_exists({x.value for x in obj_ids}, jobs=jobs)
+            remote_odb.list_oids_exists(
+                {x.value for x in obj_ids if x.value},
+                jobs=jobs,
+            )
         )
         used_obj_ids.update(obj_ids)
     return {obj for obj in used_obj_ids if obj.value not in remote_oids}
 
 
 @locked
-def gc(  # noqa: PLR0912, PLR0913, C901
+def gc(  # noqa: PLR0913, C901
     self: "Repo",
     all_branches: bool = False,
     cloud: bool = False,
@@ -65,6 +67,7 @@ def gc(  # noqa: PLR0912, PLR0913, C901
     rev: Optional[str] = None,
     num: Optional[int] = None,
     not_in_remote: bool = False,
+    dry: bool = False,
 ):
     # require `workspace` to be true to come into effect.
     # assume `workspace` to be enabled if any of `all_tags`, `all_commits`,
@@ -92,8 +95,7 @@ def gc(  # noqa: PLR0912, PLR0913, C901
         repos = []
     all_repos = [Repo(path) for path in repos]
 
-    odb_to_obj_ids: List[Tuple[Optional["ObjectDB"], Set["HashInfo"]]] = []
-
+    odb_to_obj_ids: "ObjectContainer" = {}
     with ExitStack() as stack:
         for repo in all_repos:
             stack.enter_context(repo.lock)
@@ -112,33 +114,50 @@ def gc(  # noqa: PLR0912, PLR0913, C901
                 revs=[rev] if rev else None,
                 num=num or 1,
             ).items():
-                odb_to_obj_ids.append((odb, obj_ids))
-
-    if not odb_to_obj_ids:
-        odb_to_obj_ids = [(None, set())]
+                if odb not in odb_to_obj_ids:
+                    odb_to_obj_ids[odb] = set()
+                odb_to_obj_ids[odb].update(obj_ids)
 
+    if cloud or not_in_remote:
+        _merge_remote_obj_ids(self, remote, odb_to_obj_ids)
     if not_in_remote:
-        used_obj_ids = _used_obj_ids_not_in_remote(self, remote, jobs, odb_to_obj_ids)
+        used_obj_ids = _used_obj_ids_not_in_remote(odb_to_obj_ids, jobs=jobs)
     else:
         used_obj_ids = set()
-        for _, obj_ids in odb_to_obj_ids:
-            used_obj_ids.update(obj_ids)
+        used_obj_ids.update(*odb_to_obj_ids.values())
 
     for scheme, odb in self.cache.by_scheme():
         if not odb:
             continue
-        removed = ogc(odb, used_obj_ids, jobs=jobs)
-        if not removed:
+        num_removed = ogc(odb, used_obj_ids, jobs=jobs, dry=dry)
+        if num_removed:
+            logger.info("Removed %d objects from %s cache.", num_removed, scheme)
+        else:
             logger.info("No unused '%s' cache to remove.", scheme)
 
     if not cloud:
         return
 
-    for remote_odb, obj_ids in odb_to_obj_ids:
-        if remote_odb is None:
-            remote_odb = repo.cloud.get_remote_odb(remote, "gc -c")
-        removed = ogc(remote_odb, obj_ids, jobs=jobs)
-        if removed:
+    for remote_odb, obj_ids in odb_to_obj_ids.items():
+        assert remote_odb is not None
+        num_removed = ogc(remote_odb, obj_ids, jobs=jobs, dry=dry)
+        if num_removed:
             get_index(remote_odb).clear()
+            logger.info("Removed %d objects from remote.", num_removed)
         else:
             logger.info("No unused cache to remove from remote.")
+
+
+def _merge_remote_obj_ids(
+    repo: "Repo", remote: Optional[str], used_objs: "ObjectContainer"
+):
+    # Merge default remote used objects with remote-per-output used objects
+    default_obj_ids = used_objs.pop(None, set())
+    remote_odb = repo.cloud.get_remote_odb(remote, "gc -c", hash_name="md5")
+    if remote_odb not in used_objs:
+        used_objs[remote_odb] = set()
+    used_objs[remote_odb].update(default_obj_ids)
+    legacy_odb = repo.cloud.get_remote_odb(remote, "gc -c", hash_name="md5-dos2unix")
+    if legacy_odb not in used_objs:
+        used_objs[legacy_odb] = set()
+    used_objs[legacy_odb].update(default_obj_ids)

dvc/repo/get.py

@@ -19,7 +19,17 @@ class GetDVCFileError(DvcException):
         )
 
 
-def get(url, path, out=None, rev=None, jobs=None, force=False, config=None):
+def get(
+    url,
+    path,
+    out=None,
+    rev=None,
+    jobs=None,
+    force=False,
+    config=None,
+    remote=None,
+    remote_config=None,
+):
     from dvc.config import Config
     from dvc.dvcfile import is_valid_filename
     from dvc.repo import Repo
@@ -38,6 +48,8 @@ def get(url, path, out=None, rev=None, jobs=None, force=False, config=None):
         subrepos=True,
         uninitialized=True,
         config=config,
+        remote=remote,
+        remote_config=remote_config,
     ) as repo:
         from dvc.fs import download
         from dvc.fs.data import DataFileSystem

dvc/repo/get_url.py

@@ -6,12 +6,12 @@ from dvc.fs import download, parse_external_url
 from dvc.utils import resolve_output
 
 
-def get_url(url, out=None, *, config=None, jobs=None, force=False):
+def get_url(url, out=None, *, fs_config=None, jobs=None, force=False, config=None):
     out = resolve_output(url, out, force=force)
     out = os.path.abspath(out)
     (out,) = output.loads_from(None, [out], use_cache=False)
 
-    src_fs, src_path = parse_external_url(url, config)
+    src_fs, src_path = parse_external_url(url, fs_config, config=config)
     if not src_fs.exists(src_path):
         raise URLMissingError(url)
     download(src_fs, src_path, out.fs_path, jobs=jobs)

dvc/repo/graph.py

@@ -31,6 +31,9 @@ def check_acyclic(graph: "DiGraph") -> None:
 
 def get_pipeline(pipelines, node):
     found = [i for i in pipelines if i.has_node(node)]
+    if not found:
+        return None
+
     assert len(found) == 1
     return found[0]
 
@@ -60,6 +63,9 @@ def collect_pipeline(stage: "Stage", graph: "DiGraph") -> Iterator["Stage"]:
     import networkx as nx
 
     pipeline = get_pipeline(get_pipelines(graph), stage)
+    if not pipeline:
+        return iter([])
+
     return nx.dfs_postorder_nodes(pipeline, stage)
 
 
@@ -152,10 +158,9 @@ def build_outs_graph(graph, outs_trie):
 
     outs_graph.add_nodes_from(outs_trie.values())
     for stage in graph.nodes():
+        if stage.is_repo_import:
+            continue
         for dep in stage.deps:
-            if dep.fs_path is None:
-                # RepoDependency don't have a path
-                continue
             dep_key = dep.fs.path.parts(dep.fs_path)
             overlapping = [n.value for n in outs_trie.prefixes(dep_key)]
             if outs_trie.has_subtrie(dep_key):

dvc/repo/imp.py

@@ -1,9 +1,44 @@
-def imp(self, url, path, out=None, rev=None, config=None, **kwargs):
+def imp(
+    self,
+    url,
+    path,
+    out=None,
+    rev=None,
+    config=None,
+    remote=None,
+    remote_config=None,
+    **kwargs,
+):
     erepo = {"url": url}
     if rev is not None:
         erepo["rev"] = rev
 
+    if remote and remote_config and isinstance(config, str):
+        raise ValueError(
+            "Can't specify config path together with both remote and remote_config"
+        )
+
     if config is not None:
         erepo["config"] = config
 
+    if remote is not None and remote_config is not None:
+        conf = erepo.get("config") or {}
+
+        core = conf.get("core") or {}
+        core["remote"] = remote
+
+        remotes = conf.get("remote") or {}
+        remote_conf = remotes.get(remote) or {}
+        remote_conf.update(remote_config)
+        remotes[remote] = remote_conf
+
+        conf["core"] = core
+        conf["remote"] = remotes
+
+        erepo["config"] = conf
+    elif remote is not None:
+        erepo["remote"] = remote
+    elif remote_config is not None:
+        erepo["remote"] = remote_config
+
     return self.imp_url(path, out=out, erepo=erepo, frozen=True, **kwargs)

dvc/repo/index.py

@@ -1,5 +1,6 @@
 import logging
 import time
+from collections import defaultdict
 from functools import partial
 from typing import (
     TYPE_CHECKING,
@@ -33,14 +34,14 @@ if TYPE_CHECKING:
     from dvc.repo.stage import StageInfo
     from dvc.stage import Stage
     from dvc.types import TargetType
+    from dvc_data.hashfile.db import HashFileDB
     from dvc_data.hashfile.hash_info import HashInfo
     from dvc_data.index import DataIndex, DataIndexKey, DataIndexView
-    from dvc_objects.db import ObjectDB
     from dvc_objects.fs.base import FileSystem
 
 
 logger = logging.getLogger(__name__)
-ObjectContainer = Dict[Optional["ObjectDB"], Set["HashInfo"]]
+ObjectContainer = Dict[Optional["HashFileDB"], Set["HashInfo"]]
 
 
 def log_walk(seq):
@@ -186,12 +187,13 @@ def _load_storage_from_out(storage_map, key, out):
             from fsspec.utils import tokenize
 
             # partial import
+            fs_cache = out.repo.cache.fs_cache
             storage_map.add_cache(
                 FileStorage(
                     key,
-                    out.cache.fs,
-                    out.cache.fs.path.join(
-                        out.cache.path, "fs", dep.fs.protocol, tokenize(dep.fs_path)
+                    fs_cache.fs,
+                    fs_cache.fs.path.join(
+                        fs_cache.path, dep.fs.protocol, tokenize(dep.fs_path)
                     ),
                 )
             )
@@ -318,6 +320,22 @@ class Index:
         for stage in self.stages:
             yield from stage.outs
 
+    @cached_property
+    def out_data_keys(self) -> Dict[str, Set["DataIndexKey"]]:
+        by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
+
+        by_workspace["repo"] = set()
+        by_workspace["local"] = set()
+
+        for out in self.outs:
+            if not out.use_cache:
+                continue
+
+            ws, key = out.index_key
+            by_workspace[ws].add(key)
+
+        return dict(by_workspace)
+
     @property
     def decorated_outs(self) -> Iterator["Output"]:
         for output in self.outs:
@@ -358,8 +376,6 @@ class Index:
 
     @cached_property
     def data_keys(self) -> Dict[str, Set["DataIndexKey"]]:
-        from collections import defaultdict
-
         by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
 
         by_workspace["repo"] = set()
@@ -376,8 +392,6 @@ class Index:
 
     @cached_property
     def metric_keys(self) -> Dict[str, Set["DataIndexKey"]]:
-        from collections import defaultdict
-
         from .metrics.show import _collect_top_level_metrics
 
         by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
@@ -399,8 +413,6 @@ class Index:
 
     @cached_property
     def plot_keys(self) -> Dict[str, Set["DataIndexKey"]]:
-        from collections import defaultdict
-
         by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
 
         by_workspace["repo"] = set()
@@ -413,7 +425,7 @@ class Index:
             by_workspace[workspace].add(key)
 
         for path in self._plot_sources:
-            key = self.repo.fs.path.relparts(path, self.repo.root_dir)
+            key = self.repo.fs.path.parts(path)
             by_workspace["repo"].add(key)
 
         return dict(by_workspace)
@@ -457,6 +469,9 @@ class Index:
             if not out.use_cache:
                 continue
 
+            if not out.is_in_repo:
+                continue
+
             ws, key = out.index_key
             if ws not in by_workspace:
                 by_workspace[ws] = index.view((*prefix, ws))
@@ -516,8 +531,6 @@ class Index:
         jobs: Optional[int] = None,
         push: bool = False,
     ) -> "ObjectContainer":
-        from collections import defaultdict
-
         used: "ObjectContainer" = defaultdict(set)
         pairs = self.collect_targets(targets, recursive=recursive, with_deps=with_deps)
         for stage, filter_info in pairs:
@@ -636,9 +649,23 @@ class IndexView:
         yield from {out for (out, _) in self._filtered_outs}
 
     @cached_property
-    def _data_prefixes(self) -> Dict[str, "_DataPrefixes"]:
-        from collections import defaultdict
+    def out_data_keys(self) -> Dict[str, Set["DataIndexKey"]]:
+        by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
 
+        by_workspace["repo"] = set()
+        by_workspace["local"] = set()
+
+        for out in self.outs:
+            if not out.use_cache:
+                continue
+
+            ws, key = out.index_key
+            by_workspace[ws].add(key)
+
+        return dict(by_workspace)
+
+    @cached_property
+    def _data_prefixes(self) -> Dict[str, "_DataPrefixes"]:
         prefixes: Dict[str, "_DataPrefixes"] = defaultdict(
             lambda: _DataPrefixes(set(), set())
         )
@@ -656,8 +683,6 @@ class IndexView:
 
     @cached_property
     def data_keys(self) -> Dict[str, Set["DataIndexKey"]]:
-        from collections import defaultdict
-
         ret: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
 
         for out, filter_info in self._filtered_outs:
@@ -710,7 +735,7 @@ class IndexView:
         return data
 
 
-def build_data_index(  # noqa: C901
+def build_data_index(  # noqa: C901, PLR0912
     index: Union["Index", "IndexView"],
     path: str,
     fs: "FileSystem",
@@ -773,14 +798,6 @@ def build_data_index(  # noqa: C901
             data.add(entry)
             callback.relative_update(1)
 
-        if compute_hash:
-            tree_meta, tree = build_tree(data, key, name=hash_name)
-            out_entry.meta = tree_meta
-            out_entry.hash_info = tree.hash_info
-            out_entry.loaded = True
-            data.add(out_entry)
-            callback.relative_update(1)
-
     for key in parents:
         parent_path = fs.path.join(path, *key)
         if not fs.exists(parent_path):
@@ -789,6 +806,23 @@ def build_data_index(  # noqa: C901
         data.add(direntry)
         callback.relative_update(1)
 
+    if compute_hash:
+        out_keys = index.out_data_keys.get(workspace, set())
+        data_keys = index.data_keys.get(workspace, set())
+        for key in data_keys.intersection(out_keys):
+            hash_name = _get_entry_hash_name(index, workspace, key)
+
+            out_entry = data.get(key)
+            if not out_entry or not out_entry.isdir:
+                continue
+
+            tree_meta, tree = build_tree(data, key, name=hash_name)
+            out_entry.meta = tree_meta
+            out_entry.hash_info = tree.hash_info
+            out_entry.loaded = True
+            data.add(out_entry)
+            callback.relative_update(1)
+
     return data
 
 
@@ -797,10 +831,14 @@ def _get_entry_hash_name(
 ) -> str:
     from dvc_data.hashfile.hash import DEFAULT_ALGORITHM
 
-    try:
-        src_entry = index.data[workspace][key]
+    for idx in reversed(range(len(key) + 1)):
+        prefix = key[:idx]
+        try:
+            src_entry = index.data[workspace][prefix]
+        except KeyError:
+            continue
+
         if src_entry.hash_info and src_entry.hash_info.name:
             return src_entry.hash_info.name
-    except KeyError:
-        pass
+
     return DEFAULT_ALGORITHM

dvc/repo/init.py

@@ -37,7 +37,7 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False):  # noqa:
             "Cannot initialize repo with `--no-scm` and `--subdir`"
         )
 
-    root_dir = os.path.realpath(root_dir)
+    root_dir = os.path.abspath(root_dir)
     dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
 
     try:

dvc/repo/ls.py

@@ -14,6 +14,8 @@ def ls(
     recursive: Optional[bool] = None,
     dvc_only: bool = False,
     config: Optional[str] = None,
+    remote: Optional[str] = None,
+    remote_config: Optional[dict] = None,
 ):
     """Methods for getting files and outputs for the repo.
 
@@ -23,7 +25,9 @@ def ls(
         rev (str, optional): SHA commit, branch or tag name
         recursive (bool, optional): recursively walk the repo
         dvc_only (bool, optional): show only DVC-artifacts
-        config (bool, optional): path to config file
+        config (str, optional): path to config file
+        remote (str, optional): remote name to set as a default remote in the repo
+        remote_config (str, dict): remote config to merge with a remote in the repo
 
     Returns:
         list of `entry`
@@ -47,7 +51,13 @@ def ls(
         config_dict = None
 
     with Repo.open(
-        url, rev=rev, subrepos=True, uninitialized=True, config=config_dict
+        url,
+        rev=rev,
+        subrepos=True,
+        uninitialized=True,
+        config=config_dict,
+        remote=remote,
+        remote_config=remote_config,
     ) as repo:
         path = path or ""
 
@@ -99,6 +109,7 @@ def _ls(
             "isout": dvc_info.get("isout", False),
             "isdir": info["type"] == "directory",
             "isexec": info.get("isexec", False),
+            "size": info.get("size"),
         }
 
     return ret

dvc/repo/ls_url.py

@@ -2,8 +2,8 @@ from dvc.exceptions import URLMissingError
 from dvc.fs import parse_external_url
 
 
-def ls_url(url, *, config=None, recursive=False):
-    fs, fs_path = parse_external_url(url, config=config)
+def ls_url(url, *, fs_config=None, recursive=False, config=None):
+    fs, fs_path = parse_external_url(url, fs_config=fs_config, config=config)
     try:
         info = fs.info(fs_path)
     except FileNotFoundError as exc:
@@ -20,6 +20,7 @@ def ls_url(url, *, config=None, recursive=False):
             ls_info = {
                 "path": fs.path.relpath(info["name"], fs_path),
                 "isdir": info["type"] == "directory",
+                "size": info.get("size"),
             }
             ret.append(ls_info)
 

dvc/repo/metrics/diff.py

@@ -1,21 +1,61 @@
-from dvc.utils.diff import diff as _diff
+from typing import TYPE_CHECKING, Dict, TypedDict, Union
+
+from funcy import compact
+
+from dvc.utils.diff import diff as _diff_dict
 from dvc.utils.diff import format_dict
 
+if TYPE_CHECKING:
+    from dvc.repo import Repo
 
-def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
+    from .show import Result
+
+
+class DiffResult(TypedDict, total=False):
+    errors: Dict[str, Union[Exception, Dict[str, Exception]]]
+    diff: Dict[str, Dict[str, Dict]]
+
+
+def _diff(
+    result: Dict[str, "Result"],
+    old_rev: str,
+    new_rev: str,
+    **kwargs,
+) -> DiffResult:
+    old = result.get(old_rev, {})
+    new = result.get(new_rev, {})
+
+    old_data = old.get("data", {})
+    new_data = new.get("data", {})
+
+    res = DiffResult()
+    errors = res.setdefault("errors", {})
+
+    if old_error := old.get("error"):
+        errors[old_rev] = old_error
+    else:
+        errors[old_rev] = {f: d["error"] for f, d in old_data.items() if "error" in d}
+
+    if new_error := new.get("error"):
+        errors[new_rev] = new_error
+    else:
+        errors[new_rev] = {f: d["error"] for f, d in new_data.items() if "error" in d}
+
+    diff_data = _diff_dict(format_dict(old_data), format_dict(new_data), **kwargs)
+    res = DiffResult(errors=errors, diff=diff_data)
+    res["errors"] = compact(res.get("errors", {}))  # type: ignore[assignment]
+    return compact(res)  # type: ignore[no-any-return]
+
+
+def diff(
+    repo: "Repo",
+    a_rev: str = "HEAD",
+    b_rev: str = "workspace",
+    all: bool = False,  # noqa: A002 # pylint: disable=redefined-builtin
+    **kwargs,
+) -> DiffResult:
     if repo.scm.no_commits:
         return {}
 
-    with_unchanged = kwargs.pop("all", False)
-
-    a_rev = a_rev or "HEAD"
-    b_rev = b_rev or "workspace"
-
-    metrics = repo.metrics.show(
-        *args, **kwargs, revs=[a_rev, b_rev], hide_workspace=False
-    )
-
-    old = metrics.get(a_rev, {}).get("data", {})
-    new = metrics.get(b_rev, {}).get("data", {})
-
-    return _diff(format_dict(old), format_dict(new), with_unchanged=with_unchanged)
+    metrics = repo.metrics.show(revs=[a_rev, b_rev], hide_workspace=False, **kwargs)
+    return _diff(metrics, a_rev, b_rev, with_unchanged=all)

dvc/repo/metrics/show.py

@@ -1,36 +1,38 @@
 import logging
 import os
-from typing import TYPE_CHECKING, List
+from itertools import chain
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Tuple,
+    TypedDict,
+    Union,
+)
 
+from funcy import ldistinct
 from scmrepo.exceptions import SCMError
 
-from dvc.fs.dvc import DVCFileSystem
 from dvc.repo import locked
-from dvc.repo.collect import collect
 from dvc.scm import NoSCMError
-from dvc.utils import as_posix, error_handler, errored_revisions, onerror_collect
+from dvc.utils import as_posix, expand_paths
 from dvc.utils.collections import ensure_list
 from dvc.utils.serialize import load_path
 
 if TYPE_CHECKING:
+    from dvc.fs import FileSystem
     from dvc.output import Output
+    from dvc.repo import Repo
+    from dvc.scm import Git, NoSCM
 
 logger = logging.getLogger(__name__)
 
 
-def _is_metric(out: "Output") -> bool:
-    return bool(out.metric)
-
-
-def _to_fs_paths(metrics: List["Output"]) -> List["str"]:
-    result = []
-    for out in metrics:
-        if out.metric:
-            result.append(out.repo.dvcfs.from_os_path(out.fs_path))
-    return result
-
-
-def _collect_top_level_metrics(repo):
+def _collect_top_level_metrics(repo: "Repo") -> Iterator[str]:
     top_metrics = repo.index._metrics  # pylint: disable=protected-access
     for dvcfile, metrics in top_metrics.items():
         wdir = repo.fs.path.relpath(repo.fs.path.parent(dvcfile), repo.root_dir)
@@ -39,14 +41,7 @@ def _collect_top_level_metrics(repo):
             yield repo.fs.path.normpath(path)
 
 
-def _collect_metrics(repo, targets, recursive):
-    metrics, fs_paths = collect(
-        repo, targets=targets, output_filter=_is_metric, recursive=recursive
-    )
-    return _to_fs_paths(metrics) + list(fs_paths)
-
-
-def _extract_metrics(metrics, path, rev):
+def _extract_metrics(metrics, path: str):
     if isinstance(metrics, (int, float, str)):
         return metrics
 
@@ -55,74 +50,157 @@ def _extract_metrics(metrics, path, rev):
 
     ret = {}
     for key, val in metrics.items():
-        m = _extract_metrics(val, path, rev)
+        m = _extract_metrics(val, path)
         if m not in (None, {}):
             ret[key] = m
         else:
             logger.debug(
-                (
-                    "Could not parse '%s' metric from '%s' at '%s' "
-                    "due to its unsupported type: '%s'"
-                ),
+                "Could not parse %r metric from %r due to its unsupported type: %r",
                 key,
                 path,
-                rev,
                 type(val).__name__,
             )
 
     return ret
 
 
-@error_handler
-def _read_metric(path, fs, rev, **kwargs):
-    val = load_path(path, fs)
-    val = _extract_metrics(val, path, rev)
+def _read_metric(fs: "FileSystem", path: str, **load_kwargs) -> Any:
+    val = load_path(path, fs, **load_kwargs)
+    val = _extract_metrics(val, path)
     return val or {}
 
 
-def _read_metrics(repo, metrics, rev, onerror=None):
-    fs = DVCFileSystem(repo=repo)
-
-    relpath = ""
-    if repo.root_dir != repo.fs.path.getcwd():
-        relpath = repo.fs.path.relpath(repo.root_dir, repo.fs.path.getcwd())
-
-    res = {}
+def _read_metrics(
+    fs: "FileSystem", metrics: Iterable[str], **load_kwargs
+) -> Iterator[Tuple[str, Union[Exception, Any]]]:
     for metric in metrics:
-        rel_metric_path = os.path.join(relpath, *fs.path.parts(metric))
-        if not fs.isfile(metric):
-            if fs.isfile(rel_metric_path):
-                metric = rel_metric_path
-            else:
-                continue
+        try:
+            yield metric, _read_metric(fs, metric, **load_kwargs)
+        except Exception as exc:  # noqa: BLE001 # pylint:disable=broad-exception-caught
+            logger.debug(exc)
+            yield metric, exc
 
-        res[rel_metric_path] = _read_metric(metric, fs, rev, onerror=onerror)
+
+def metrics_from_target(repo: "Repo", targets: List[str]) -> Iterator["Output"]:
+    stages = chain.from_iterable(repo.stage.collect(target) for target in targets)
+    for stage in stages:
+        yield from stage.metrics
+
+
+def _collect_metrics(
+    repo: "Repo",
+    targets: Optional[List[str]] = None,
+    stages: Optional[List[str]] = None,
+    outs_only: bool = False,
+) -> List[str]:
+    metrics: List[str] = []
+
+    if targets:
+        # target is a repo-relative path
+        metrics.extend(targets)
+
+    if not targets or outs_only:
+        outs = metrics_from_target(repo, stages) if stages else repo.index.metrics
+        relpath = repo.fs.path.relpath
+        metrics.extend(relpath(out.fs_path, repo.root_dir) for out in outs)
+
+    if not targets and not outs_only and not stages:
+        # _collect_top_level_metrics returns repo-relative paths
+        metrics.extend(_collect_top_level_metrics(repo))
+
+    fs = repo.dvcfs
+
+    # convert to posixpath for DVCFileSystem
+    paths = (fs.from_os_path(metric) for metric in metrics)
+    # make paths absolute for DVCFileSystem
+    repo_paths = (f"{fs.root_marker}{path}" for path in paths)
+    return ldistinct(expand_paths(fs, repo_paths))
+
+
+class FileResult(TypedDict, total=False):
+    data: Any
+    error: Exception
+
+
+class Result(TypedDict, total=False):
+    data: Dict[str, FileResult]
+    error: Exception
+
+
+def to_relpath(fs: "FileSystem", root_dir: str, d: Result) -> Result:
+    relpath = fs.path.relpath
+    cwd = fs.path.getcwd()
+
+    start = relpath(cwd, root_dir)
+    data = d.get("data")
+    if data is not None:
+        d["data"] = {relpath(path, start): result for path, result in data.items()}
+    return d
+
+
+def _gather_metrics(
+    repo: "Repo",
+    targets: Optional[List[str]] = None,
+    outs_only: bool = False,
+    stages: Optional[List[str]] = None,
+    on_error: str = "return",
+) -> Dict[str, FileResult]:
+    assert on_error in ("raise", "return", "ignore")
+
+    # `files` is a repo-relative posixpath that can be passed to DVCFileSystem
+    # It is absolute, i.e. has a root_marker `/` in front which we strip when returning
+    # the result and convert to appropriate repo-relative os.path.
+    files = _collect_metrics(repo, targets=targets, stages=stages, outs_only=outs_only)
+    data = {}
+
+    fs = repo.dvcfs
+    for fs_path, result in _read_metrics(fs, files, cache=True):
+        repo_path = fs_path.lstrip(fs.root_marker)
+        repo_os_path = os.sep.join(fs.path.parts(repo_path))
+        if not isinstance(result, Exception):
+            data.update({repo_os_path: FileResult(data=result)})
+            continue
+
+        if on_error == "raise":
+            raise result
+        if on_error == "return":
+            data.update({repo_os_path: FileResult(error=result)})
+    return data
+
+
+def _hide_workspace(
+    scm: Union["Git", "NoSCM"], res: Dict[str, Result]
+) -> Dict[str, Result]:
+    # Hide workspace params if they are the same as in the active branch
+    try:
+        active_branch = scm.active_branch()
+    except (SCMError, NoSCMError):
+        # SCMError - detached head
+        # NoSCMError - no repo case
+        pass
+    else:
+        if res.get("workspace") == res.get(active_branch):
+            res.pop("workspace", None)
 
     return res
 
 
-def _gather_metrics(repo, targets, rev, recursive, onerror=None):
-    metrics = _collect_metrics(repo, targets, recursive)
-    metrics.extend(_collect_top_level_metrics(repo))
-    return _read_metrics(repo, metrics, rev, onerror=onerror)
-
-
 @locked
 def show(
-    repo,
-    targets=None,
-    all_branches=False,
-    all_tags=False,
-    recursive=False,
-    revs=None,
-    all_commits=False,
-    onerror=None,
-    hide_workspace=True,
-):
-    if onerror is None:
-        onerror = onerror_collect
+    repo: "Repo",
+    targets: Optional[List[str]] = None,
+    stages: Optional[List[str]] = None,
+    outs_only: bool = False,
+    all_branches: bool = False,
+    all_tags: bool = False,
+    revs: Optional[List[str]] = None,
+    all_commits: bool = False,
+    hide_workspace: bool = True,
+    on_error: str = "return",
+) -> Dict[str, Result]:
+    assert on_error in ("raise", "return", "ignore")
 
-    targets = ensure_list(targets)
+    targets = [os.path.abspath(target) for target in ensure_list(targets)]
     targets = [repo.dvcfs.from_os_path(target) for target in targets]
 
     res = {}
@@ -132,29 +210,23 @@ def show(
         all_tags=all_tags,
         all_commits=all_commits,
     ):
-        res[rev] = error_handler(_gather_metrics)(
-            repo, targets, rev, recursive, onerror=onerror
-        )
+        try:
+            result = _gather_metrics(
+                repo,
+                targets=targets,
+                stages=stages,
+                outs_only=outs_only,
+                on_error=on_error,
+            )
+            res[rev] = Result(data=result)
+        except Exception as exc:  # noqa: BLE001 # pylint:disable=broad-exception-caught
+            if on_error == "raise":
+                raise
+
+            logger.warning("failed to load metrics in revision %r, %s", rev, str(exc))
+            if on_error == "return":
+                res[rev] = Result(error=exc)
 
     if hide_workspace:
-        # Hide workspace metrics if they are the same as in the active branch
-        try:
-            active_branch = repo.scm.active_branch()
-        except (SCMError, NoSCMError):
-            # SCMError - detached head
-            # NoSCMError - no repo case
-            pass
-        else:
-            if res.get("workspace") == res.get(active_branch):
-                res.pop("workspace", None)
-
-    errored = errored_revisions(res)
-    if errored:
-        from dvc.ui import ui
-
-        ui.error_write(
-            "DVC failed to load some metrics for following revisions:"
-            f" '{', '.join(errored)}'."
-        )
-
+        _hide_workspace(repo.scm, res)
     return res

dvc/repo/params/diff.py

@@ -1,21 +1,21 @@
-from dvc.utils.diff import diff as _diff
-from dvc.utils.diff import format_dict
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+    from dvc.repo.metrics.diff import DiffResult
 
 
-def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
+def diff(
+    repo: "Repo",
+    a_rev: str = "HEAD",
+    b_rev: str = "workspace",
+    all: bool = False,  # noqa: A002 # pylint: disable=redefined-builtin
+    **kwargs,
+) -> "DiffResult":
     if repo.scm.no_commits:
         return {}
 
-    with_unchanged = kwargs.pop("all", False)
+    from dvc.repo.metrics.diff import _diff
 
-    a_rev = a_rev or "HEAD"
-    b_rev = b_rev or "workspace"
-
-    params = repo.params.show(
-        *args, **kwargs, revs=[a_rev, b_rev], hide_workspace=False
-    )
-
-    old = params.get(a_rev, {}).get("data", {})
-    new = params.get(b_rev, {}).get("data", {})
-
-    return _diff(format_dict(old), format_dict(new), with_unchanged=with_unchanged)
+    params = repo.params.show(revs=[a_rev, b_rev], hide_workspace=False, **kwargs)
+    return _diff(params, a_rev, b_rev, with_unchanged=all)

dvc/repo/params/show.py

@@ -1,33 +1,24 @@
 import logging
 import os
 from collections import defaultdict
-from copy import copy
-from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Tuple
+from itertools import chain
+from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple, Union
 
-from scmrepo.exceptions import SCMError
-
-from dvc.dependency.param import ParamsDependency
+from dvc.dependency.param import ParamsDependency, read_param_file
 from dvc.repo import locked
-from dvc.repo.collect import collect
-from dvc.scm import NoSCMError
+from dvc.repo.metrics.show import FileResult, Result
 from dvc.stage import PipelineStage
-from dvc.ui import ui
-from dvc.utils import as_posix, error_handler, errored_revisions, onerror_collect
+from dvc.utils import as_posix, expand_paths
 from dvc.utils.collections import ensure_list
-from dvc.utils.serialize import load_path
 
 if TYPE_CHECKING:
-    from dvc.output import Output
+    from dvc.fs import FileSystem
     from dvc.repo import Repo
 
 logger = logging.getLogger(__name__)
 
 
-def _is_params(dep: "Output"):
-    return isinstance(dep, ParamsDependency)
-
-
-def _collect_top_level_params(repo):
+def _collect_top_level_params(repo: "Repo") -> Iterator[str]:
     top_params = repo.index._params  # pylint: disable=protected-access
     for dvcfile, params in top_params.items():
         wdir = repo.fs.path.relpath(repo.fs.path.parent(dvcfile), repo.root_dir)
@@ -36,67 +27,58 @@ def _collect_top_level_params(repo):
             yield repo.fs.path.normpath(path)
 
 
-def _collect_configs(
-    repo: "Repo", targets=None, deps=False, stages=None
-) -> Tuple[List["Output"], List[str]]:
-    params, fs_paths = collect(
-        repo,
-        targets=targets or [],
-        deps=True,
-        output_filter=_is_params,
-        duplicates=deps or stages is not None,
-    )
-    all_fs_paths = fs_paths + [p.fs_path for p in params]
-    if not any([deps, targets, stages]):
-        default_params = repo.fs.path.join(
-            repo.root_dir, ParamsDependency.DEFAULT_PARAMS_FILE
+def params_from_target(
+    repo: "Repo", targets: List[str]
+) -> Iterator["ParamsDependency"]:
+    stages = chain.from_iterable(repo.stage.collect(target) for target in targets)
+    for stage in stages:
+        yield from stage.params
+
+
+def _collect_params(
+    repo: "Repo",
+    targets: Union[List[str], Dict[str, List[str]], None] = None,
+    stages: Optional[List[str]] = None,
+    deps_only: bool = False,
+    default_file: Optional[str] = None,
+) -> Dict[str, List[str]]:
+    from dvc.dependency import _merge_params
+
+    if isinstance(targets, list):
+        targets = {target: [] for target in targets}
+
+    params: List[Dict[str, List[str]]] = []
+
+    if targets:
+        # target is a repo-relative path
+        params.extend({file: params} for file, params in targets.items())
+
+    if not targets or stages:
+        deps = params_from_target(repo, stages) if stages else repo.index.params
+        relpath = repo.fs.path.relpath
+        params.extend(
+            {relpath(dep.fs_path, repo.root_dir): list(dep.params)} for dep in deps
         )
-        if default_params not in all_fs_paths and repo.fs.exists(default_params):
-            fs_paths.append(default_params)
-    if targets and (deps or stages) and not params:
-        # A target has been provided but it is not used in the stages
-        fs_paths = []
-    return params, fs_paths
 
+    fs = repo.dvcfs
 
-@error_handler
-def _read_fs_path(fs, fs_path, **kwargs):
-    return load_path(fs_path, fs)
+    if not targets and not deps_only and not stages:
+        # _collect_top_level_params returns repo-relative paths
+        params.extend({param: []} for param in _collect_top_level_params(repo))
+        if default_file and fs.exists(f"{fs.root_marker}{default_file}"):
+            params.append({default_file: []})
 
+    # combine all the param files and the keypaths to track
+    all_params = _merge_params(params)
 
-def _read_params(
-    repo,
-    params,
-    params_fs_paths,
-    deps=False,
-    onerror: Optional[Callable] = None,
-    stages: Optional[Iterable[str]] = None,
-):
-    res: Dict[str, Dict] = defaultdict(lambda: defaultdict(dict))
-    fs_paths = copy(params_fs_paths)
-
-    if deps or stages:
-        for param in params:
-            if stages and param.stage.addressing not in stages:
-                continue
-            params_dict = error_handler(param.read_params)(
-                onerror=onerror, flatten=False
-            )
-            if params_dict:
-                name = os.sep.join(repo.fs.path.relparts(param.fs_path))
-                res[name]["data"].update(params_dict["data"])
-                if name in fs_paths:
-                    fs_paths.remove(name)
-    else:
-        fs_paths += [param.fs_path for param in params]
-
-    for fs_path in fs_paths:
-        from_path = _read_fs_path(repo.fs, fs_path, onerror=onerror)
-        if from_path:
-            name = os.sep.join(repo.fs.path.relparts(fs_path))
-            res[name] = from_path
-
-    return res
+    ret = {}
+    for param, _params in all_params.items():
+        # convert to posixpath for DVCFileSystem
+        path = fs.from_os_path(param)
+        # make paths absolute for DVCFileSystem
+        repo_path = f"{fs.root_marker}{path}"
+        ret.update({file: _params for file in expand_paths(fs, [repo_path])})
+    return ret
 
 
 def _collect_vars(repo, params, stages=None) -> Dict:
@@ -109,82 +91,116 @@ def _collect_vars(repo, params, stages=None) -> Dict:
             for file, vars_ in stage.tracked_vars.items():
                 # `params` file are shown regardless of `tracked` or not
                 # to reduce noise and duplication, they are skipped
-                if file in params:
+
+                # `file` is relative
+                abspath = repo.fs.path.abspath(file)
+                repo_path = repo.dvcfs.from_os_path(abspath)
+                if repo_path in params:
                     continue
 
-                name = os.sep.join(repo.fs.path.parts(file))
-                vars_params[name].update(vars_)
-    return vars_params
+                vars_params[repo_path].update(vars_)
+    return dict(vars_params)
+
+
+def _read_params(
+    fs: "FileSystem", params: Dict[str, List[str]], **load_kwargs
+) -> Iterator[Tuple[str, Union[Exception, Any]]]:
+    for file_path, key_paths in params.items():
+        try:
+            yield file_path, read_param_file(fs, file_path, key_paths, **load_kwargs)
+        except Exception as exc:  # noqa: BLE001 # pylint:disable=broad-exception-caught
+            logger.debug(exc)
+            yield file_path, exc
+
+
+def _gather_params(
+    repo: "Repo",
+    targets: Union[List[str], Dict[str, List[str]], None] = None,
+    deps_only: bool = False,
+    stages: Optional[List[str]] = None,
+    on_error: str = "return",
+):
+    assert on_error in ("raise", "return", "ignore")
+
+    # `files` is a repo-relative posixpath that can be passed to DVCFileSystem
+    # It is absolute, i.e. has a root_marker `/` in front which we strip when returning
+    # the result and convert to appropriate repo-relative os.path.
+    files_keypaths = _collect_params(
+        repo,
+        targets=targets,
+        stages=stages,
+        deps_only=deps_only,
+        default_file=ParamsDependency.DEFAULT_PARAMS_FILE,
+    )
+
+    data: Dict[str, FileResult] = {}
+
+    fs = repo.dvcfs
+    for fs_path, result in _read_params(fs, files_keypaths, cache=True):
+        repo_path = fs_path.lstrip(fs.root_marker)
+        repo_os_path = os.sep.join(fs.path.parts(repo_path))
+        if not isinstance(result, Exception):
+            data.update({repo_os_path: FileResult(data=result)})
+            continue
+
+        if on_error == "raise":
+            raise result
+        if on_error == "return":
+            data.update({repo_os_path: FileResult(error=result)})
+
+    if not (stages or targets):
+        data.update(
+            {
+                path: FileResult(data=result)
+                for path, result in _collect_vars(repo, data).items()
+            }
+        )
+    return data
 
 
 @locked
 def show(
-    repo,
-    revs=None,
-    targets=None,
-    deps=False,
-    onerror: Optional[Callable] = None,
-    stages=None,
-    hide_workspace=True,
-):
-    if onerror is None:
-        onerror = onerror_collect
+    repo: "Repo",
+    targets: Optional[List[str]] = None,
+    stages: Optional[List[str]] = None,
+    deps_only: bool = False,
+    all_branches: bool = False,
+    all_tags: bool = False,
+    revs: Optional[List[str]] = None,
+    all_commits: bool = False,
+    hide_workspace: bool = True,
+    on_error: str = "return",
+) -> Dict[str, Result]:
+    assert on_error in ("raise", "return", "ignore")
     res = {}
 
     targets = ensure_list(targets)
     targets = [repo.dvcfs.from_os_path(target) for target in targets]
 
-    for branch in repo.brancher(revs=revs):
-        params = error_handler(_gather_params)(
-            repo=repo,
-            targets=targets,
-            deps=deps,
-            onerror=onerror,
-            stages=stages,
-        )
-
-        if params:
-            res[branch] = params
+    for rev in repo.brancher(
+        revs=revs,
+        all_branches=all_branches,
+        all_tags=all_tags,
+        all_commits=all_commits,
+    ):
+        try:
+            params = _gather_params(
+                repo=repo,
+                targets=targets,
+                stages=stages,
+                deps_only=deps_only,
+                on_error=on_error,
+            )
+            res[rev] = Result(data=params)
+        except Exception as exc:  # noqa: BLE001 # pylint:disable=broad-exception-caught
+            if on_error == "raise":
+                raise
+            logger.warning("failed to load params in revision %r, %s", rev, str(exc))
+            if on_error == "return":
+                res[rev] = Result(error=exc)
 
     if hide_workspace:
-        # Hide workspace params if they are the same as in the active branch
-        try:
-            active_branch = repo.scm.active_branch()
-        except (SCMError, NoSCMError):
-            # SCMError - detached head
-            # NoSCMError - no repo case
-            pass
-        else:
-            if res.get("workspace") == res.get(active_branch):
-                res.pop("workspace", None)
-
-    errored = errored_revisions(res)
-    if errored:
-        ui.error_write(
-            "DVC failed to load some parameters for following revisions:"
-            f" '{', '.join(errored)}'."
-        )
+        from dvc.repo.metrics.show import _hide_workspace
 
+        _hide_workspace(repo.scm, res)
     return res
-
-
-def _gather_params(repo, targets=None, deps=False, onerror=None, stages=None):
-    param_outs, params_fs_paths = _collect_configs(
-        repo, targets=targets, deps=deps, stages=stages
-    )
-    params_fs_paths.extend(_collect_top_level_params(repo=repo))
-    params = _read_params(
-        repo,
-        params=param_outs,
-        params_fs_paths=params_fs_paths,
-        deps=deps,
-        onerror=onerror,
-        stages=stages,
-    )
-    vars_params = _collect_vars(repo, params, stages=stages)
-
-    # NOTE: only those that are not added as a ParamDependency are
-    # included so we don't need to recursively merge them yet.
-    for key, vals in vars_params.items():
-        params[key]["data"] = vals
-    return params

dvc/repo/plots/__init__.py

@@ -46,8 +46,8 @@ SUPPORTED_IMAGE_EXTENSIONS = ImageRenderer.EXTENSIONS
 class PlotMetricTypeError(DvcException):
     def __init__(self, file):
         super().__init__(
-            "'{}' - file type error\n"
-            "Only JSON, YAML, CSV and TSV formats are supported.".format(file)
+            f"'{file}' - file type error\n"
+            "Only JSON, YAML, CSV and TSV formats are supported."
         )
 
 
@@ -212,7 +212,7 @@ class Plots:
             short_rev = "workspace"
             if rev := getattr(self.repo.fs, "rev", None):
                 short_rev = rev[:7]
-            _resolve_data_sources(data, short_rev, cache_remote_stream=True)
+            _resolve_data_sources(data, short_rev, cache=True)
             result.update(data)
 
         errored = errored_revisions(result)
@@ -281,9 +281,7 @@ def _is_plot(out: "Output") -> bool:
     return bool(out.plot)
 
 
-def _resolve_data_sources(
-    plots_data: Dict, rev: str, cache_remote_stream: bool = False
-):
+def _resolve_data_sources(plots_data: Dict, rev: str, cache: bool = False):
     from dvc.progress import Tqdm
 
     values = list(plots_data.values())
@@ -298,7 +296,7 @@ def _resolve_data_sources(
     def resolve(value):
         data_source = value.pop("data_source")
         assert callable(data_source)
-        value.update(data_source(cache_remote_stream=cache_remote_stream))
+        value.update(data_source(cache=cache))
 
     if not to_resolve:
         return
@@ -448,6 +446,7 @@ def _resolve_definitions(
     definitions: "DictStrAny",
     onerror: Optional[Callable[[Any], Any]] = None,
 ):
+    config_path = os.fspath(config_path)
     config_dir = fs.path.dirname(config_path)
     result: Dict[str, Dict] = {}
     for plot_id, plot_props in definitions.items():
@@ -483,7 +482,7 @@ def _collect_pipeline_files(repo, targets: List[str], props, onerror=None):
             if isinstance(elem, str):
                 dvcfile_defs_dict[elem] = None
             else:
-                k, v = list(elem.items())[0]
+                k, v = next(iter(elem.items()))
                 dvcfile_defs_dict[k] = v
 
         resolved = _resolve_definitions(