logger: add DVC_SHOW_TRACEBACK envvar to show traceback on exception (#9310)

This will come useful in debugging, where you don't have to add --verbose just to
see where it failed or go through verbose logs (which might be useful when working
with UI issues as you want to see what the user sees).

Ideally this should be handled in main when we exit with error instead of using
logger.{warning,error,exception} to log exception as the intention is to only log
last exception that makes it exit. But since we don't have that capability and
we don't abuse logging.{warning,error,exception} to log and suppress exceptions, this is a good
enough solution for now without complicating `dvc.cli.main()`.

.github/ISSUE_TEMPLATE/epic_story.md

@@ -0,0 +1,60 @@
+---
+name: Epic/Story Issue Template
+about: Template for "top" level issues - Epics (>2 weeks) / Stories (<2 weeks)
+title: 'Epic: New Feature'
+labels: epic
+assignees:
+
+---
+
+## Summary / Background
+What do you want to achieve, why? business context
+...
+
+## Scope
+
+What will be impacted and what won't be?
+What needs implementation and what is invariant?
+e.g.
+- user should be able to run workflow X from UI
+- enable workflow Y, Z from CLI
+
+## Assumptions
+Product / UX assumptions as well as technical assumptions / limitations
+e.g.
+* Support only Python Runtime
+* Focus on DVC experiments only
+* Deployment environments don't change often and can be picked up from shared configuration
+
+## Open Questions
+e.g.
+- How should access control work for shared artifacts (workflow X)
+- Python runtime assumption - is it really valid? in light of <...>
+
+## Blockers / Dependencies
+List issues or other conditions / blockers
+
+## General Approach
+Invocation example:
+```shell
+$ mapper-run task.tar.gz --ray-cluster <ip>:<port>
+```
+
+## Steps
+
+### Must have (p1)
+- [ ] subissue2
+- [ ] step 2
+ - info
+ - info
+
+### Optional / followup (p2)
+- [ ] ⌛ step 3 wip
+- [ ] step 4
+
+## Timelines
+
+Put your estimations here. Update once certainty changes
+- end of week (Feb 3) for prototype with workflows X, Y
+- Feb 15 - MVP in production
+- Low priority followups can be done later

.github/workflows/packages.yaml

@@ -17,9 +17,9 @@ jobs:
     strategy:
       matrix:
         include:
-         - {pkg: "deb",  os: "ubuntu-18.04", asset: "dvc_${{ github.event.release.tag_name }}_amd64.deb"}
-         - {pkg: "rpm", os: "ubuntu-18.04", asset: "dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm"}
-         - {pkg: "osxpkg", os: "macos-10.15", asset: "dvc-${{ github.event.release.tag_name }}.pkg"}
+         - {pkg: "deb",  os: "ubuntu-20.04", asset: "dvc_${{ github.event.release.tag_name }}_amd64.deb"}
+         - {pkg: "rpm", os: "ubuntu-20.04", asset: "dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm"}
+         - {pkg: "osxpkg", os: "macos-11", asset: "dvc-${{ github.event.release.tag_name }}.pkg"}
          - {pkg: "exe",  os: "windows-2019", asset: "dvc-${{ github.event.release.tag_name }}.exe"}
 
     name: ${{ matrix.pkg }}

.github/workflows/tests.yaml

@@ -17,7 +17,7 @@ permissions:
 
 jobs:
   tests:
-    timeout-minutes: 45
+    timeout-minutes: 50
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -41,9 +41,9 @@ jobs:
         pip install -e ".[dev]"
     - uses: pre-commit/[email protected]
     - name: run tests
-      timeout-minutes: 30
+      timeout-minutes: 40
       run: >
-        pytest -nauto --timeout=300 --durations=100
+        pytest -n=logical --timeout=300 --durations=100
         --cov --cov-report=xml --cov-report=term
     - name: upload coverage report
       uses: codecov/codecov-action@v3

.pre-commit-config.yaml

@@ -35,7 +35,7 @@ repos:
         args:
           - --py38-plus
   - repo: https://github.com/psf/black
-    rev: 23.1.0
+    rev: 23.3.0
     hooks:
       - id: black
   - repo: https://github.com/PyCQA/isort
@@ -43,11 +43,11 @@ repos:
     hooks:
       - id: isort
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.0.238'
+    rev: 'v0.0.260'
     hooks:
       - id: ruff
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.2
+    rev: v2.2.4
     hooks:
       - id: codespell
         additional_dependencies: ["tomli"]
@@ -61,6 +61,13 @@ repos:
           - flake8-debugger
           - flake8-string-format
           - flake8-typing-imports
+  - repo: https://github.com/PyCQA/bandit
+    rev: '1.7.5'
+    hooks:
+    - id: bandit
+      args: ["-c", "pyproject.toml", "--quiet", "--format=custom"]
+      files: ^dvc/
+      additional_dependencies: [".[toml]"]
   - repo: local
     hooks:
     - id: mypy

dvc/analytics.py

@@ -47,7 +47,7 @@ def is_enabled():
     enabled = not os.getenv(DVC_NO_ANALYTICS)
     if enabled:
         enabled = to_bool(
-            Config(validate=False).get("core", {}).get("analytics", "true")
+            Config.from_cwd(validate=False).get("core", {}).get("analytics", "true")
         )
 
     logger.debug("Analytics is %sabled.", "en" if enabled else "dis")

dvc/annotations.py

@@ -2,6 +2,7 @@ from dataclasses import asdict, dataclass, field, fields
 from typing import Any, ClassVar, Dict, List, Optional
 
 from funcy import compact
+from voluptuous import Required
 
 
 @dataclass
@@ -26,6 +27,12 @@ class Annotation:
         return compact(asdict(self))
 
 
+@dataclass
+class Artifact(Annotation):
+    PARAM_PATH: ClassVar[str] = "path"
+    path: Optional[str] = None
+
+
 ANNOTATION_FIELDS = [field.name for field in fields(Annotation)]
 ANNOTATION_SCHEMA = {
     Annotation.PARAM_DESC: str,
@@ -33,3 +40,7 @@ ANNOTATION_SCHEMA = {
     Annotation.PARAM_LABELS: [str],
     Annotation.PARAM_META: object,
 }
+ARTIFACT_SCHEMA = {
+    Required(Artifact.PARAM_PATH): str,
+    **ANNOTATION_SCHEMA,  # type: ignore[arg-type]
+}

dvc/api/__init__.py

@@ -2,14 +2,16 @@ from dvc.fs.dvc import _DVCFileSystem as DVCFileSystem
 
 from .data import open  # pylint: disable=redefined-builtin
 from .data import get_url, read
-from .experiments import make_checkpoint
-from .params import params_show
+from .experiments import exp_save, make_checkpoint
+from .show import metrics_show, params_show
 
 __all__ = [
+    "exp_save",
     "get_url",
     "make_checkpoint",
     "open",
     "params_show",
+    "metrics_show",
     "read",
     "DVCFileSystem",
 ]

dvc/api/data.py

@@ -1,12 +1,30 @@
 from contextlib import _GeneratorContextManager as GCM
-from typing import Optional
+from contextlib import contextmanager
+from typing import Any, Dict, Optional
 
 from funcy import reraise
 
-from dvc.exceptions import OutputNotFoundError, PathMissingError
+from dvc.exceptions import FileMissingError, OutputNotFoundError, PathMissingError
 from dvc.repo import Repo
 
 
+@contextmanager
+def _wrap_exceptions(repo, url):
+    from dvc.config import NoRemoteError
+    from dvc.exceptions import NoOutputInExternalRepoError, NoRemoteInExternalRepoError
+
+    try:
+        yield
+    except NoRemoteError as exc:
+        raise NoRemoteInExternalRepoError(url) from exc
+    except OutputNotFoundError as exc:
+        if exc.repo is repo:
+            raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url) from exc
+        raise
+    except FileMissingError as exc:
+        raise PathMissingError(exc.path, url) from exc
+
+
 def get_url(path, repo=None, rev=None, remote=None):
     """
     Returns the URL to the storage location of a data file or directory tracked
@@ -19,18 +37,20 @@ def get_url(path, repo=None, rev=None, remote=None):
     directory in the remote storage.
     """
     with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
-        fs_path = _repo.dvcfs.from_os_path(path)
-        with reraise(FileNotFoundError, PathMissingError(path, repo)):
-            info = _repo.dvcfs.info(fs_path)
+        with _wrap_exceptions(_repo, path):
+            fs_path = _repo.dvcfs.from_os_path(path)
 
-        dvc_info = info.get("dvc_info")
-        if not dvc_info:
-            raise OutputNotFoundError(path, repo)
+            with reraise(FileNotFoundError, PathMissingError(path, repo)):
+                info = _repo.dvcfs.info(fs_path)
 
-        dvc_repo = info["repo"]  # pylint: disable=unsubscriptable-object
-        md5 = dvc_info["md5"]
+            dvc_info = info.get("dvc_info")
+            if not dvc_info:
+                raise OutputNotFoundError(path, repo)
 
-        return dvc_repo.cloud.get_url_for(remote, checksum=md5)
+            dvc_repo = info["repo"]  # pylint: disable=unsubscriptable-object
+            md5 = dvc_info["md5"]
+
+            return dvc_repo.cloud.get_url_for(remote, checksum=md5)
 
 
 class _OpenContextManager(GCM):
@@ -194,11 +214,41 @@ def open(  # noqa, pylint: disable=redefined-builtin
 
 
 def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
-        with _repo.open_by_relpath(
-            path, remote=remote, mode=mode, encoding=encoding
-        ) as fd:
-            yield fd
+    repo_kwargs: Dict[str, Any] = {"subrepos": True, "uninitialized": True}
+    if remote:
+        repo_kwargs["config"] = {"core": {"remote": remote}}
+
+    with Repo.open(repo, rev=rev, **repo_kwargs) as _repo:
+        with _wrap_exceptions(_repo, path):
+            import os
+            from typing import TYPE_CHECKING, Union
+
+            from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
+            from dvc.fs.data import DataFileSystem
+            from dvc.fs.dvc import DVCFileSystem
+
+            if TYPE_CHECKING:
+                from dvc.fs import FileSystem
+
+            fs: Union["FileSystem", DataFileSystem, DVCFileSystem]
+            if os.path.isabs(path):
+                fs = DataFileSystem(index=_repo.index.data["local"])
+                fs_path = path
+            else:
+                fs = DVCFileSystem(repo=_repo, subrepos=True)
+                fs_path = fs.from_os_path(path)
+
+            try:
+                with fs.open(
+                    fs_path,
+                    mode=mode,
+                    encoding=encoding,
+                ) as fobj:
+                    yield fobj
+            except FileNotFoundError as exc:
+                raise FileMissingError(path) from exc
+            except IsADirectoryError as exc:
+                raise DvcIsADirectoryError(f"'{path}' is a directory") from exc
 
 
 def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):

dvc/api/experiments.py

@@ -1,6 +1,7 @@
 import builtins
 import os
 from time import sleep
+from typing import List, Optional
 
 from dvc.env import DVC_CHECKPOINT, DVC_ROOT
 from dvc.repo import Repo
@@ -31,3 +32,40 @@ def make_checkpoint():
         os.fsync(fobj.fileno())
     while os.path.exists(signal_file):
         sleep(0.1)
+
+
+def exp_save(
+    name: Optional[str] = None,
+    force: bool = False,
+    include_untracked: Optional[List[str]] = None,
+):
+    """
+    Create a new DVC experiment using `exp save`.
+
+    See https://dvc.org/doc/command-reference/exp/save.
+
+    Args:
+        name (str, optional): specify a name for this experiment.
+            If `None`, a default one will be generated, such as `urban-sign`.
+            Defaults to `None`.
+        force (bool):  overwrite the experiment if an experiment with the same
+            name already exists.
+            Defaults to `False`.
+        include_untracked (List[str], optional): specify untracked file(s) to
+            be included in the saved experiment.
+            Defaults to `None`.
+
+    Returns:
+        str: The `Git revision`_ of the created experiment.
+
+    Raises:
+        ExperimentExistsError: If an experiment with `name` already exists and
+            `force=False`.
+
+    .. _Git revision:
+        https://git-scm.com/docs/revisions
+    """
+    with Repo() as repo:
+        return repo.experiments.save(
+            name=name, force=force, include_untracked=include_untracked
+        )

dvc/api/show.py

@@ -4,10 +4,153 @@ from typing import Dict, Iterable, Optional, Union
 
 from funcy import first
 
-from dvc.exceptions import DvcException
 from dvc.repo import Repo
 
 
+def _onerror_raise(exception: Exception, *args, **kwargs):
+    raise exception
+
+
+def _postprocess(results):
+    processed: Dict[str, Dict] = {}
+    for rev, rev_data in results.items():
+        if not rev_data:
+            continue
+
+        processed[rev] = {}
+
+        counts: typing.Counter[str] = Counter()
+        for file_data in rev_data["data"].values():
+            for k in file_data["data"]:
+                counts[k] += 1
+
+        for file_name, file_data in rev_data["data"].items():
+            to_merge = {
+                (k if counts[k] == 1 else f"{file_name}:{k}"): v
+                for k, v in file_data["data"].items()
+            }
+            processed[rev] = {**processed[rev], **to_merge}
+
+    if "workspace" in processed:
+        del processed["workspace"]
+
+    return processed
+
+
+def metrics_show(
+    *targets: str,
+    repo: Optional[str] = None,
+    rev: Optional[str] = None,
+) -> Dict:
+    """Get metrics tracked in `repo`.
+
+    Without arguments, this function will retrieve all metrics from all tracked
+    metric files, for the current working tree.
+
+    See the options below to restrict the metrics retrieved.
+
+    Args:
+        *targets (str, optional): Names of the metric files to retrieve
+        metrics from. For example, "classifier_eval.json,
+        clustering_eval.json".
+        If no `targets` are provided, all metric files tracked in `dvc.yaml`
+        will be used.
+        Note that targets don't necessarily have to be defined in `dvc.yaml`.
+        repo (str, optional): Location of the DVC repository.
+            Defaults to the current project (found by walking up from the
+            current working directory tree).
+            It can be a URL or a file system path.
+            Both HTTP and SSH protocols are supported for online Git repos
+            (e.g. [user@]server:project.git).
+        rev (str, optional): Name of the `Git revision`_ to retrieve metrics
+            from.
+            Defaults to `None`.
+            An example of git revision can be a branch or tag name, a commit
+            hash or a dvc experiment name.
+            If `repo` is not a Git repo, this option is ignored.
+            If `None`, the current working tree will be used.
+
+    Returns:
+        Dict: See Examples below.
+
+    Examples:
+
+        - No arguments.
+
+        Working on https://github.com/iterative/example-get-started
+
+        >>> import dvc.api
+        >>> metrics = dvc.api.metrics_show()
+        >>> print(json.dumps(metrics, indent=4))
+        {
+            "avg_prec": 0.9249974999612706,
+            "roc_auc": 0.9460213440787918
+        }
+
+        ---
+
+        - Using `rev`.
+
+        Working on https://github.com/iterative/example-get-started
+
+        >>> import json
+        >>> import dvc.api
+        >>> metrics = dvc.api.metrics_show(rev="tune-hyperparams")
+        >>> print(json.dumps(metrics, indent=4))
+        {
+            "avg_prec": 0.9268792615819422,
+            "roc_auc": 0.945093365854111
+        }
+
+        ---
+
+        - Using `targets`.
+
+        Working on https://github.com/iterative/example-get-started
+
+        >>> import json
+        >>> import dvc.api
+        >>> metrics = dvc.api.metrics_show("evaluation.json")
+        >>> print(json.dumps(metrics, indent=4))
+        {
+            "avg_prec": 0.9249974999612706,
+            "roc_auc": 0.9460213440787918
+        }
+
+        ---
+
+        - Git URL as `repo`.
+
+        >>> import json
+        >>> import dvc.api
+        >>> metrics = dvc.api.metrics_show(
+        ...     repo="https://github.com/iterative/demo-fashion-mnist")
+        >>> print(json.dumps(metrics, indent=4))
+        {
+            "loss": 0.25284987688064575,
+            "accuracy": 0.9071000218391418
+        }
+
+
+    .. _Git revision:
+        https://git-scm.com/docs/revisions
+    """
+
+    with Repo.open(repo) as _repo:
+        metrics = _repo.metrics.show(
+            targets=targets,
+            revs=rev if rev is None else [rev],
+            onerror=_onerror_raise,
+        )
+
+    metrics = _postprocess(metrics)
+
+    if not metrics:
+        return {}
+
+    return metrics[first(metrics)]
+
+
 def params_show(
     *targets: str,
     repo: Optional[str] = None,
@@ -56,9 +199,6 @@ def params_show(
     Returns:
         Dict: See Examples below.
 
-    Raises:
-        DvcException: If no params are found in `repo`.
-
     Examples:
 
         - No arguments.
@@ -239,34 +379,6 @@ def params_show(
     if isinstance(stages, str):
         stages = [stages]
 
-    def _onerror_raise(_result: Dict, exception: Exception, *args, **kwargs):
-        raise exception
-
-    def _postprocess(params):
-        processed: Dict[str, Dict] = {}
-        for rev, rev_data in params.items():
-            processed[rev] = {}
-
-            counts: typing.Counter[str] = Counter()
-            for file_data in rev_data["data"].values():
-                for k in file_data["data"]:
-                    counts[k] += 1
-
-            for file_name, file_data in rev_data["data"].items():
-                to_merge = {
-                    (k if counts[k] == 1 else f"{file_name}:{k}"): v
-                    for k, v in file_data["data"].items()
-                }
-                processed[rev] = {**processed[rev], **to_merge}
-
-        if "workspace" in processed:
-            del processed["workspace"]
-
-        if not processed:
-            raise DvcException("No params found")
-
-        return processed[first(processed)]
-
     with Repo.open(repo) as _repo:
         params = _repo.params.show(
             revs=rev if rev is None else [rev],
@@ -276,4 +388,9 @@ def params_show(
             stages=stages,
         )
 
-    return _postprocess(params)
+    params = _postprocess(params)
+
+    if not params:
+        return {}
+
+    return params[first(params)]

dvc/cachemgr.py

@@ -1,3 +1,5 @@
+import os
+
 from dvc.fs import GitFileSystem, Schemes
 from dvc_data.hashfile.db import get_odb
 
@@ -9,7 +11,6 @@ def _get_odb(repo, settings, fs=None):
         return None
 
     cls, config, fs_path = get_cloud_fs(repo, **settings)
-    config["tmp_dir"] = repo.tmp_dir
     fs = fs or cls(**config)
     return get_odb(fs, fs_path, state=repo.state, **config)
 
@@ -29,16 +30,21 @@ class CacheManager:
         self.config = config = repo.config["cache"]
         self._odb = {}
 
+        default = None
+        if repo and repo.local_dvc_dir:
+            default = os.path.join(repo.local_dvc_dir, self.CACHE_DIR)
+
         local = config.get("local")
 
         if local:
             settings = {"name": local}
-        elif "dir" not in config:
+        elif "dir" not in config and not default:
             settings = None
         else:
             from dvc.config_schema import LOCAL_COMMON
 
-            settings = {"url": config["dir"]}
+            url = config.get("dir") or default
+            settings = {"url": url}
             for opt in LOCAL_COMMON:
                 if opt in config:
                     settings[str(opt)] = config.get(opt)

dvc/cli/__init__.py

@@ -39,12 +39,17 @@ def parse_args(argv=None):
 
 def _log_unknown_exceptions() -> None:
     from dvc.info import get_dvc_info
-    from dvc.logger import FOOTER
+    from dvc.ui import ui
+    from dvc.utils import colorize
 
     logger.exception("unexpected error")
     if logger.isEnabledFor(logging.DEBUG):
         logger.debug("Version info for developers:\n%s", get_dvc_info())
-    logger.info(FOOTER)
+
+    q = colorize("Having any troubles?", "yellow")
+    link = colorize("https://dvc.org/support", "blue")
+    footer = f"\n{q} Hit us up at {link}, we are always happy to help!"
+    ui.error_write(footer)
 
 
 def _log_exceptions(exc: Exception) -> Optional[int]:
@@ -141,7 +146,7 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
     return None
 
 
-def main(argv=None):  # noqa: C901
+def main(argv=None):  # noqa: C901, PLR0912, PLR0915
     """Main entry point for dvc CLI.
 
     Args:
@@ -167,7 +172,7 @@ def main(argv=None):  # noqa: C901
 
     args = None
 
-    outerLogLevel = logger.level
+    outer_log_level = logger.level
     try:
         args = parse_args(argv)
 
@@ -238,7 +243,7 @@ def main(argv=None):  # noqa: C901
 
         return ret
     finally:
-        logger.setLevel(outerLogLevel)
+        logger.setLevel(outer_log_level)
 
         from dvc.external_repo import clean_repos
 

dvc/cli/parser.py

@@ -2,6 +2,7 @@
 import argparse
 import logging
 import os
+from functools import lru_cache
 
 from dvc import __version__
 from dvc.commands import (
@@ -150,6 +151,7 @@ def get_parent_parser():
     return parent_parser
 
 
+@lru_cache(maxsize=1)
 def get_main_parser():
     parent_parser = get_parent_parser()
 

dvc/commands/cache.py

@@ -1,4 +1,5 @@
 import argparse
+import os
 
 from dvc.cli import completion
 from dvc.cli.utils import append_doc_link, fix_subparsers
@@ -9,13 +10,21 @@ from dvc.ui import ui
 class CmdCacheDir(CmdConfig):
     def run(self):
         if self.args.value is None and not self.args.unset:
+            from dvc.config import ConfigError
+
             if self.args.level:
                 conf = self.config.read(level=self.args.level)
             else:
                 # Use merged config with default values
                 conf = self.config
-            self._check(conf, False, "cache", "dir")
-            ui.write(conf["cache"]["dir"])
+            try:
+                self._check(conf, False, "cache", "dir")
+                path = conf["cache"]["dir"]
+            except ConfigError:
+                if not self.config.dvc_dir or self.args.level:
+                    raise
+                path = os.path.join(self.config.dvc_dir, "cache")
+            ui.write(path)
             return 0
         with self.config.edit(level=self.args.level) as conf:
             if self.args.unset:

dvc/commands/check_ignore.py

@@ -39,7 +39,7 @@ class CmdCheckIgnore(CmdBase):
                 target = input()
             except (KeyboardInterrupt, EOFError):
                 break
-            if target == "":
+            if not target:
                 break
             if not self._check_one_file(target):
                 ret = 0

dvc/commands/completion.py

@@ -15,7 +15,7 @@ class CmdCompletion(CmdBaseNoRepo):
 
         shell = self.args.shell
         parser = self.args.parser
-        script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
+        script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)  # nosec B604
         ui.write(script, force=True)
         return 0
 

dvc/commands/config.py

@@ -32,7 +32,7 @@ class CmdConfig(CmdBaseNoRepo):
 
         super().__init__(args)
 
-        self.config = Config(validate=False)
+        self.config = Config.from_cwd(validate=False)
 
     def run(self):
         if self.args.show_origin and (self.args.value or self.args.unset):

dvc/commands/dag.py

@@ -9,11 +9,11 @@ if TYPE_CHECKING:
     from networkx import DiGraph
 
 
-def _show_ascii(G: "DiGraph"):
+def _show_ascii(graph: "DiGraph"):
     from dvc.dagascii import draw
     from dvc.repo.graph import get_pipelines
 
-    pipelines = get_pipelines(G)
+    pipelines = get_pipelines(graph)
 
     ret = []
     for pipeline in pipelines:
@@ -31,7 +31,7 @@ def _quote_label(node):
     return label
 
 
-def _show_dot(G: "DiGraph"):
+def _show_dot(graph: "DiGraph"):
     import io
 
     import networkx as nx
@@ -39,15 +39,15 @@ def _show_dot(G: "DiGraph"):
 
     dot_file = io.StringIO()
 
-    nx.relabel_nodes(G, _quote_label, copy=False)
-    write_dot(G.reverse(), dot_file)
+    nx.relabel_nodes(graph, _quote_label, copy=False)
+    write_dot(graph.reverse(), dot_file)
     return dot_file.getvalue()
 
 
-def _show_mermaid(G, markdown: bool = False):
+def _show_mermaid(graph, markdown: bool = False):
     from dvc.repo.graph import get_pipelines
 
-    pipelines = get_pipelines(G)
+    pipelines = get_pipelines(graph)
 
     graph = "flowchart TD"
 
@@ -100,43 +100,42 @@ def _transform(index, outs):
     def _relabel(node) -> str:
         return node.addressing if isinstance(node, Stage) else str(node)
 
-    G = index.outs_graph if outs else index.graph
-    return nx.relabel_nodes(G, _relabel, copy=True)
+    graph = index.outs_graph if outs else index.graph
+    return nx.relabel_nodes(graph, _relabel, copy=True)
 
 
-def _filter(G, targets, full):
+def _filter(graph, targets, full):
     import networkx as nx
 
     if not targets:
-        return G
+        return graph
 
-    H = G.copy()
+    new_graph = graph.copy()
     if not full:
         descendants = set()
         for target in targets:
-            descendants.update(nx.descendants(G, target))
+            descendants.update(nx.descendants(graph, target))
             descendants.add(target)
-        H.remove_nodes_from(set(G.nodes()) - descendants)
+        new_graph.remove_nodes_from(set(graph.nodes()) - descendants)
 
-    undirected = H.to_undirected()
+    undirected = new_graph.to_undirected()
     connected = set()
     for target in targets:
         connected.update(nx.node_connected_component(undirected, target))
 
-    H.remove_nodes_from(set(H.nodes()) - connected)
-
-    return H
+    new_graph.remove_nodes_from(set(new_graph.nodes()) - connected)
+    return new_graph
 
 
 def _build(repo, target=None, full=False, outs=False):
     targets = _collect_targets(repo, target, outs)
-    G = _transform(repo.index, outs)
-    return _filter(G, targets, full)
+    graph = _transform(repo.index, outs)
+    return _filter(graph, targets, full)
 
 
 class CmdDAG(CmdBase):
     def run(self):
-        G = _build(
+        graph = _build(
             self.repo,
             target=self.args.target,
             full=self.args.full,
@@ -144,12 +143,12 @@ class CmdDAG(CmdBase):
         )
 
         if self.args.dot:
-            ui.write(_show_dot(G))
+            ui.write(_show_dot(graph))
         elif self.args.mermaid or self.args.markdown:
-            ui.write(_show_mermaid(G, self.args.markdown))
+            ui.write(_show_mermaid(graph, self.args.markdown))
         else:
             with ui.pager():
-                ui.write(_show_ascii(G))
+                ui.write(_show_ascii(graph))
 
         return 0
 

dvc/commands/data.py

@@ -21,12 +21,14 @@ logger = logging.getLogger(__name__)
 
 class CmdDataStatus(CmdBase):
     COLORS = {
+        "not_in_remote": "red",
         "not_in_cache": "red",
         "committed": "green",
         "uncommitted": "yellow",
         "untracked": "cyan",
     }
     LABELS = {
+        "not_in_remote": "Not in remote",
         "not_in_cache": "Not in cache",
         "committed": "DVC committed changes",
         "uncommitted": "DVC uncommitted changes",
@@ -34,6 +36,7 @@ class CmdDataStatus(CmdBase):
         "unchanged": "DVC unchanged files",
     }
     HINTS = {
+        "not_in_remote": ('use "dvc push <file>..." to upload files',),
         "not_in_cache": ('use "dvc fetch <file>..." to download files',),
         "committed": ("git commit the corresponding dvc files to update the repo",),
         "uncommitted": (
@@ -114,6 +117,7 @@ class CmdDataStatus(CmdBase):
             status = self.repo.data_status(
                 granular=self.args.granular,
                 untracked_files=self.args.untracked_files,
+                remote_refresh=self.args.remote_refresh,
             )
 
         if not self.args.unchanged:
@@ -239,6 +243,12 @@ def add_parser(subparsers, parent_parser):
         nargs="?",
         help="Show untracked files.",
     )
+    data_status_parser.add_argument(
+        "--remote-refresh",
+        action="store_true",
+        default=False,
+        help="Refresh remote index.",
+    )
     data_status_parser.set_defaults(func=CmdDataStatus)
 
     DATA_LS_HELP = "List data tracked by DVC with its metadata."

dvc/commands/data_sync.py

@@ -40,6 +40,7 @@ class CmdDataPull(CmdDataBase):
                 recursive=self.args.recursive,
                 run_cache=self.args.run_cache,
                 glob=self.args.glob,
+                allow_missing=self.args.allow_missing,
             )
             self.log_summary(stats)
         except (CheckoutError, DvcException) as exc:
@@ -197,6 +198,12 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help=argparse.SUPPRESS,
     )
+    pull_parser.add_argument(
+        "--allow-missing",
+        action="store_true",
+        default=False,
+        help="Ignore errors if some of the files or directories are missing.",
+    )
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push

dvc/commands/experiments/__init__.py

@@ -4,6 +4,7 @@ from dvc.cli.utils import append_doc_link, fix_subparsers, hide_subparsers_from_
 from dvc.commands.experiments import (
     apply,
     branch,
+    clean,
     diff,
     exec_run,
     gc,
@@ -21,6 +22,7 @@ from dvc.commands.experiments import (
 SUB_COMMANDS = [
     apply,
     branch,
+    clean,
     diff,
     exec_run,
     gc,

dvc/commands/experiments/apply.py

@@ -4,13 +4,21 @@ import logging
 from dvc.cli import completion
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsApply(CmdBase):
     def run(self):
-        self.repo.experiments.apply(self.args.experiment, force=self.args.force)
+        if not self.args.force:
+            ui.write(
+                "The --no-force option is deprecated and will be removed in a future"
+                " DVC release. To revert the result of 'exp apply', run:\n"
+                "\n\tgit reset --hard\n"
+                "\tgit stash apply refs/exps/apply/stash\n"
+            )
+        self.repo.experiments.apply(self.args.experiment)
 
         return 0
 

dvc/commands/experiments/branch.py

@@ -26,5 +26,13 @@ def add_parser(experiments_subparsers, parent_parser):
     experiments_branch_parser.add_argument(
         "experiment", help="Experiment to be promoted."
     )
-    experiments_branch_parser.add_argument("branch", help="Name for the new Git branch")
+    experiments_branch_parser.add_argument(
+        "branch",
+        nargs="?",
+        default=None,
+        help=(
+            "Optional name for the new Git branch. "
+            "Defaults to '{experiment-name}-branch'."
+        ),
+    )
     experiments_branch_parser.set_defaults(func=CmdExperimentsBranch)

dvc/commands/experiments/clean.py

@@ -0,0 +1,25 @@
+import argparse
+import logging
+
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsClean(CmdBase):
+    def run(self):
+        self.repo.experiments.clean()
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+    EXPERIMENTS_CLEAN_HELP = "Cleanup experiments temporary internal files."
+    experiments_clean_parser = experiments_subparsers.add_parser(
+        "clean",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_CLEAN_HELP, "exp/clean"),
+        help=EXPERIMENTS_CLEAN_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_clean_parser.set_defaults(func=CmdExperimentsClean)

dvc/commands/experiments/diff.py

@@ -52,7 +52,7 @@ class CmdExperimentsDiff(CmdBase):
 
 
 def add_parser(experiments_subparsers, parent_parser):
-    EXPERIMENTS_DIFF_HELP = "Show changes between experiments in the DVC repository."
+    EXPERIMENTS_DIFF_HELP = "Show changes between experiments."
 
     experiments_diff_parser = experiments_subparsers.add_parser(
         "diff",

dvc/commands/experiments/exec_run.py

@@ -19,6 +19,7 @@ class CmdExecutorRun(CmdBaseNoRepo):
             queue=None,
             log_level=logger.getEffectiveLevel(),
             infofile=self.args.infofile,
+            copy_paths=self.args.copy_paths,
         )
         return 0
 
@@ -36,4 +37,14 @@ def add_parser(experiments_subparsers, parent_parser):
         help="Path to executor info file",
         default=None,
     )
+    exec_run_parser.add_argument(
+        "-C",
+        "--copy-paths",
+        action="append",
+        default=[],
+        help=(
+            "List of ignored or untracked paths to copy into the temp directory."
+            " Only used if `--temp` or `--queue` is specified."
+        ),
+    )
     exec_run_parser.set_defaults(func=CmdExecutorRun)

dvc/commands/experiments/gc.py

@@ -78,7 +78,8 @@ class CmdExperimentsGC(CmdBase):
 def add_parser(experiments_subparsers, parent_parser):
     EXPERIMENTS_GC_HELP = "Garbage collect unneeded experiments."
     EXPERIMENTS_GC_DESCRIPTION = (
-        "Removes all experiments which are not derived from the specifiedGit revisions."
+        "Removes all experiments which are not derived"
+        " from the specified Git revisions."
     )
     experiments_gc_parser = experiments_subparsers.add_parser(
         "gc",

dvc/commands/experiments/push.py

@@ -1,5 +1,6 @@
 import argparse
 import logging
+from typing import Any, Dict
 
 from dvc.cli import completion
 from dvc.cli.command import CmdBase
@@ -18,29 +19,64 @@ class CmdExperimentsPush(CmdBase):
                 "`--rev` or `--all-commits` flag."
             )
 
+    @staticmethod
+    def log_result(result: Dict[str, Any], remote: str):
+        from dvc.utils import humanize
+
+        def join_exps(exps):
+            return humanize.join([f"[bold]{e}[/]" for e in exps])
+
+        if diverged_exps := result.get("diverged"):
+            exps = join_exps(diverged_exps)
+            ui.error_write(
+                f"[yellow]Local experiment {exps} has diverged "
+                "from remote experiment with the same name.\n"
+                "To override the remote experiment re-run with '--force'.",
+                styled=True,
+            )
+        if uptodate_exps := result.get("up_to_date"):
+            exps = join_exps(uptodate_exps)
+            verb = "are" if len(uptodate_exps) > 1 else "is"
+            ui.write(
+                f"Experiment {exps} {verb} up to date on Git remote {remote!r}.",
+                styled=True,
+            )
+        if pushed_exps := result.get("success"):
+            exps = join_exps(pushed_exps)
+            ui.write(f"Pushed experiment {exps} to Git remote {remote!r}.", styled=True)
+        if not uptodate_exps and not pushed_exps:
+            ui.write("No experiments to push.")
+
+        if uploaded := result.get("uploaded"):
+            stats = {"uploaded": uploaded}
+            ui.write(humanize.get_summary(stats.items()))
+
+        if project_url := result.get("url"):
+            ui.write("[yellow]View your experiments at", project_url, styled=True)
+
     def run(self):
+        from dvc.repo.experiments.push import UploadError
+
         self.raise_error_if_all_disabled()
 
-        pushed_exps = self.repo.experiments.push(
-            self.args.git_remote,
-            self.args.experiment,
-            all_commits=self.args.all_commits,
-            rev=self.args.rev,
-            num=self.args.num,
-            force=self.args.force,
-            push_cache=self.args.push_cache,
-            dvc_remote=self.args.dvc_remote,
-            jobs=self.args.jobs,
-            run_cache=self.args.run_cache,
-        )
-
-        if pushed_exps:
-            ui.write(
-                f"Pushed experiment '{pushed_exps}'"
-                f"to Git remote '{self.args.git_remote}'."
+        try:
+            result = self.repo.experiments.push(
+                self.args.git_remote,
+                self.args.experiment,
+                all_commits=self.args.all_commits,
+                rev=self.args.rev,
+                num=self.args.num,
+                force=self.args.force,
+                push_cache=self.args.push_cache,
+                dvc_remote=self.args.dvc_remote,
+                jobs=self.args.jobs,
+                run_cache=self.args.run_cache,
             )
-        else:
-            ui.write("No experiments to push.")
+        except UploadError as e:
+            self.log_result(e.result, self.args.git_remote)
+            raise
+
+        self.log_result(result, self.args.git_remote)
         if not self.args.push_cache:
             ui.write(
                 "To push cached outputs",

dvc/commands/experiments/queue_worker.py

@@ -9,7 +9,9 @@ class CmdQueueWorker(CmdBase):
     """Run the exp queue worker."""
 
     def run(self):
-        self.repo.experiments.celery_queue.worker.start(self.args.name)
+        self.repo.experiments.celery_queue.worker.start(
+            self.args.name, fsapp_clean=self.args.clean
+        )
         return 0
 
 
@@ -22,4 +24,9 @@ def add_parser(experiments_subparsers, parent_parser):
         add_help=False,
     )
     parser.add_argument("name", help="Celery worker name.")
+    parser.add_argument(
+        "--clean",
+        action="store_true",
+        help="Automatically cleanup celery broker on shutdown.",
+    )
     parser.set_defaults(func=CmdQueueWorker)

dvc/commands/experiments/remove.py

@@ -24,9 +24,11 @@ class CmdExperimentsRemove(CmdBase):
             )
 
     def run(self):
+        from dvc.utils import humanize
+
         self.check_arguments()
 
-        removed_list = self.repo.experiments.remove(
+        removed = self.repo.experiments.remove(
             exp_names=self.args.experiment,
             all_commits=self.args.all_commits,
             rev=self.args.rev,
@@ -34,8 +36,10 @@ class CmdExperimentsRemove(CmdBase):
             queue=self.args.queue,
             git_remote=self.args.git_remote,
         )
-        removed = ",".join(removed_list)
-        ui.write(f"Removed experiments: {removed}")
+        if removed:
+            ui.write(f"Removed experiments: {humanize.join(map(repr, removed))}")
+        else:
+            ui.write("No experiments to remove.")
 
         return 0
 

dvc/commands/experiments/run.py

@@ -36,6 +36,7 @@ class CmdExperimentsRun(CmdRepro):
             reset=self.args.reset,
             tmp_dir=self.args.tmp_dir,
             machine=self.args.machine,
+            copy_paths=self.args.copy_paths,
             **self._common_kwargs,
         )
 
@@ -136,3 +137,13 @@ def _add_run_common(parser):
         # )
         # metavar="<name>",
     )
+    parser.add_argument(
+        "-C",
+        "--copy-paths",
+        action="append",
+        default=[],
+        help=(
+            "List of ignored or untracked paths to copy into the temp directory."
+            " Only used if `--temp` or `--queue` is specified."
+        ),
+    )

dvc/commands/experiments/save.py

@@ -35,7 +35,7 @@ class CmdExperimentsSave(CmdBase):
 
 
 def add_parser(experiments_subparsers, parent_parser):
-    EXPERIMENTS_SAVE_HELP = "Save current workspace as a dvc experiment."
+    EXPERIMENTS_SAVE_HELP = "Save current workspace as an experiment."
     save_parser = experiments_subparsers.add_parser(
         "save",
         parents=[parent_parser],

dvc/commands/experiments/show.py

@@ -62,7 +62,7 @@ experiment_types = {
 }
 
 
-def _collect_rows(  # noqa: C901
+def _collect_rows(  # noqa: C901, PLR0912, PLR0913, PLR0915
     base_rev,
     experiments,
     all_headers,
@@ -244,7 +244,7 @@ def _extend_row(row, names, headers, items, precision, fill_value=FILL_VALUE):
                 row[f"{fname}:{name}"] = value
 
 
-def experiments_table(
+def experiments_table(  # noqa: PLR0913
     all_experiments,
     headers,
     metric_headers,
@@ -308,7 +308,7 @@ def baseline_styler(typ):
     return {"style": "bold"} if typ == "baseline" else {}
 
 
-def show_experiments(  # noqa: C901
+def show_experiments(  # noqa: C901, PLR0912
     all_experiments,
     keep=None,
     drop=None,
@@ -453,8 +453,12 @@ def _normalize_headers(names, count):
 
 
 def _format_json(item):
+    from dvc.repo.experiments.show import _CachedError
+
     if isinstance(item, (date, datetime)):
         return item.isoformat()
+    if isinstance(item, _CachedError):
+        return {"type": getattr(item, "typ", "_CachedError"), "msg": str(item)}
     return encode_exception(item)
 
 
@@ -472,6 +476,7 @@ class CmdExperimentsShow(CmdBase):
                 sha_only=self.args.sha,
                 param_deps=self.args.param_deps,
                 fetch_running=self.args.fetch_running,
+                force=self.args.force,
             )
         except DvcException:
             logger.exception("failed to show experiments")
@@ -650,4 +655,10 @@ def add_parser(experiments_subparsers, parent_parser):
         action="store_false",
         help=argparse.SUPPRESS,
     )
+    experiments_show_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        help="Force re-collection of experiments instead of loading from exp cache.",
+    )
     experiments_show_parser.set_defaults(func=CmdExperimentsShow)

dvc/commands/get.py

@@ -14,12 +14,8 @@ class CmdGet(CmdBaseNoRepo):
         from dvc.api import get_url
         from dvc.ui import ui
 
-        try:
-            url = get_url(self.args.path, repo=self.args.url, rev=self.args.rev)
-            ui.write(url, force=True)
-        except DvcException:
-            logger.exception("failed to show URL")
-            return 1
+        url = get_url(self.args.path, repo=self.args.url, rev=self.args.rev)
+        ui.write(url, force=True)
 
         return 0
 

dvc/commands/machine.py

@@ -302,7 +302,7 @@ class CmdMachineSsh(CmdBase):
         return 0
 
 
-def add_parser(subparsers, parent_parser):
+def add_parser(subparsers, parent_parser):  # noqa: PLR0915
     from dvc.commands.config import parent_config_parser
 
     machine_HELP = "Set up and manage cloud machines."

dvc/commands/plots.py

@@ -1,9 +1,9 @@
 import argparse
-import json
 import logging
 import os
+from typing import TYPE_CHECKING, Dict, List, Optional
 
-from funcy import first
+from funcy import compact, first, get_in
 
 from dvc.cli import completion
 from dvc.cli.command import CmdBase
@@ -12,14 +12,51 @@ from dvc.exceptions import DvcException
 from dvc.ui import ui
 from dvc.utils import format_link
 
+if TYPE_CHECKING:
+    from dvc.render.match import RendererWithErrors
+
+
 logger = logging.getLogger(__name__)
 
 
-def _show_json(renderers, split=False):
+def _show_json(
+    renderers_with_errors: List["RendererWithErrors"],
+    split=False,
+    errors: Optional[Dict[str, Exception]] = None,
+):
     from dvc.render.convert import to_json
+    from dvc.utils.serialize import encode_exception
 
-    result = {renderer.name: to_json(renderer, split) for renderer in renderers}
-    ui.write_json(result)
+    all_errors: List[Dict] = []
+    data = {}
+
+    for renderer, src_errors, def_errors in renderers_with_errors:
+        name = renderer.name
+        data[name] = to_json(renderer, split)
+        all_errors.extend(
+            {
+                "name": name,
+                "rev": rev,
+                "source": source,
+                **encode_exception(e),
+            }
+            for rev, per_rev_src_errors in src_errors.items()
+            for source, e in per_rev_src_errors.items()
+        )
+        all_errors.extend(
+            {
+                "name": name,
+                "rev": rev,
+                **encode_exception(e),
+            }
+            for rev, e in def_errors.items()
+        )
+
+    # these errors are not tied to any renderers
+    errors = errors or {}
+    all_errors.extend({"rev": rev, **encode_exception(e)} for rev, e in errors.items())
+
+    ui.write_json(compact({"errors": all_errors, "data": data}), highlight=False)
 
 
 def _adjust_vega_renderers(renderers):
@@ -76,10 +113,6 @@ class CmdPlots(CmdBase):
         props = {p: getattr(self.args, p) for p in PLOT_PROPS}
         return {k: v for k, v in props.items() if v is not None}
 
-    def _config_files(self):
-        if self.args.from_config:
-            return {self.args.from_config}
-
     def _html_template_path(self):
         html_template_path = self.args.html_template
         if not html_template_path:
@@ -89,9 +122,9 @@ class CmdPlots(CmdBase):
             if html_template_path and not os.path.isabs(html_template_path):
                 assert self.repo.dvc_dir
                 html_template_path = os.path.join(self.repo.dvc_dir, html_template_path)
-        return html_template_path  # noqa: RET504
+        return html_template_path
 
-    def run(self) -> int:  # noqa: C901
+    def run(self) -> int:  # noqa: C901, PLR0911, PLR0912
         from pathlib import Path
 
         from dvc.render.match import match_defs_renderers
@@ -114,10 +147,9 @@ class CmdPlots(CmdBase):
             plots_data = self._func(
                 targets=self.args.targets,
                 props=self._props(),
-                config_files=self._config_files(),
             )
 
-            if not plots_data:
+            if not plots_data and not self.args.json:
                 ui.error_write(
                     "No plots were loaded, visualization file will not be created."
                 )
@@ -127,21 +159,27 @@ class CmdPlots(CmdBase):
             )
 
             renderers_out = out if self.args.json else os.path.join(out, "static")
-
-            renderers = match_defs_renderers(
+            renderers_with_errors = match_defs_renderers(
                 data=plots_data,
                 out=renderers_out,
                 templates_dir=self.repo.plots.templates_dir,
             )
             if self.args.json:
-                _show_json(renderers, self.args.split)
+                errors = compact(
+                    {
+                        rev: get_in(data, ["definitions", "error"])
+                        for rev, data in plots_data.items()
+                    }
+                )
+                _show_json(renderers_with_errors, self.args.split, errors=errors)
                 return 0
 
+            renderers = [r.renderer for r in renderers_with_errors]
             _adjust_vega_renderers(renderers)
             if self.args.show_vega:
                 renderer = first(filter(lambda r: r.TYPE == "vega", renderers))
                 if renderer:
-                    ui.write_json(json.loads(renderer.get_filled_template()))
+                    ui.write_json(renderer.get_filled_template(as_string=False))
                 return 0
 
             output_file: Path = (Path.cwd() / out).resolve() / "index.html"
@@ -150,7 +188,7 @@ class CmdPlots(CmdBase):
                 render_html(
                     renderers=renderers,
                     output_file=output_file,
-                    template_path=self._html_template_path(),
+                    html_template=self._html_template_path(),
                 )
 
                 ui.write(output_file.as_uri())
@@ -412,9 +450,3 @@ def _add_ui_arguments(parser):
         help="Custom HTML template for VEGA visualization.",
         metavar="<path>",
     )
-    parser.add_argument(
-        "--from-config",
-        default=None,
-        metavar="<path>",
-        help=argparse.SUPPRESS,
-    )

dvc/commands/update.py

@@ -31,8 +31,7 @@ class CmdUpdate(CmdBase):
 def add_parser(subparsers, parent_parser):
     UPDATE_HELP = (
         "Update data artifact imported (via dvc import or dvc import-url) "
-        "from an external DVC repository or URL, or update data from a "
-        "DVC worktree remote."
+        "from an external DVC repository or URL."
     )
     update_parser = subparsers.add_parser(
         "update",

dvc/compare.py

@@ -209,7 +209,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
         keys = self.keys() if cols is None else set(cols)
         return [{k: self._columns[k][i] for k in keys} for i in range(len(self))]
 
-    def dropna(  # noqa: C901
+    def dropna(  # noqa: C901, PLR0912
         self,
         axis: str = "rows",
         how="any",
@@ -371,7 +371,7 @@ def diff_table(
     return td
 
 
-def show_diff(
+def show_diff(  # noqa: PLR0913
     diff,
     title: str,
     old: bool = True,

dvc/config.py

@@ -103,21 +103,25 @@ class Config(dict):
         self.wfs = LocalFileSystem()
         self.fs = fs or self.wfs
 
-        if not dvc_dir:
-            try:
-                from dvc.repo import Repo
-
-                self.dvc_dir = Repo.find_dvc_dir()
-            except NotDvcRepoError:
-                self.dvc_dir = None
-        else:
+        if dvc_dir:
             self.dvc_dir = self.fs.path.abspath(self.fs.path.realpath(dvc_dir))
 
         self.load(validate=validate, config=config)
 
+    @classmethod
+    def from_cwd(cls, fs: Optional["FileSystem"] = None, **kwargs):
+        from dvc.repo import Repo
+
+        try:
+            dvc_dir = Repo.find_dvc_dir(fs=fs)
+        except NotDvcRepoError:
+            dvc_dir = None
+
+        return cls(dvc_dir=dvc_dir, fs=fs, **kwargs)
+
     @classmethod
     def get_dir(cls, level):
-        from appdirs import site_config_dir, user_config_dir
+        from platformdirs import site_config_dir, user_config_dir
 
         assert level in ("global", "system")
 
@@ -170,10 +174,6 @@ class Config(dict):
         self.clear()
         self.update(conf)
 
-        # Add resolved default cache.dir
-        if not self["cache"].get("dir") and self.dvc_dir:
-            self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
-
     def _get_fs(self, level):
         # NOTE: this might be a Gitfs, which doesn't see things outside of
         # the repo.

dvc/config_schema.py

@@ -1,8 +1,21 @@
+import logging
 import os
 from urllib.parse import urlparse
 
-from funcy import walk_values
-from voluptuous import All, Any, Coerce, Invalid, Lower, Optional, Range, Schema
+from funcy import once, walk_values
+from voluptuous import (
+    REMOVE_EXTRA,
+    All,
+    Any,
+    Coerce,
+    Invalid,
+    Lower,
+    Optional,
+    Range,
+    Schema,
+)
+
+logger = logging.getLogger(__name__)
 
 Bool = All(
     Lower,
@@ -30,7 +43,7 @@ def supported_cache_type(types):
     return types
 
 
-def Choices(*choices):
+def Choices(*choices):  # noqa: N802
     """Checks that value belongs to the specified set of values
 
     Args:
@@ -40,7 +53,7 @@ def Choices(*choices):
     return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
 
 
-def ByUrl(mapping):
+def ByUrl(mapping):  # noqa: N802
     schemas = walk_values(Schema, mapping)
 
     def validate(data):
@@ -49,7 +62,9 @@ def ByUrl(mapping):
 
         parsed = urlparse(data["url"])
         # Windows absolute paths should really have scheme == "" (local)
-        if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
+        if os.name == "nt" and len(parsed.scheme) == 1 and not parsed.netloc:
+            return schemas[""](data)
+        if not parsed.netloc:
             return schemas[""](data)
         if parsed.scheme not in schemas:
             raise Invalid(f"Unsupported URL type {parsed.scheme}://")
@@ -63,6 +78,27 @@ class RelPath(str):
     pass
 
 
+class FeatureSchema(Schema):
+    def __init__(self, schema, required=False):
+        super().__init__(schema, required=required, extra=REMOVE_EXTRA)
+
+    @staticmethod
+    @once
+    def _log_deprecated(keys):
+        # only run this once per session
+        message = "%s config option%s unsupported"
+        paths = ", ".join(f"'feature.{key}'" for key in keys)
+        pluralize = " is" if len(keys) == 1 else "s are"
+        logger.warning(message, paths, pluralize)
+
+    def __call__(self, data):
+        ret = super().__call__(data)
+        extra_keys = data.keys() - ret.keys()
+        if extra_keys:
+            self._log_deprecated(sorted(extra_keys))
+        return ret
+
+
 REMOTE_COMMON = {
     "url": str,
     "checksum_jobs": All(Coerce(int), Range(1)),
@@ -114,6 +150,7 @@ SCHEMA = {
         Optional("autostage", default=False): Bool,
         Optional("experiments"): Bool,  # obsoleted
         Optional("check_update", default=True): Bool,
+        "site_cache_dir": str,
         "machine": Lower,
     },
     "cache": {
@@ -237,12 +274,12 @@ SCHEMA = {
         )
     },
     "state": {
-        "dir": str,
+        "dir": str,  # obsoleted
         "row_limit": All(Coerce(int), Range(1)),  # obsoleted
         "row_cleanup_quota": All(Coerce(int), Range(0, 100)),  # obsoleted
     },
     "index": {
-        "dir": str,
+        "dir": str,  # obsoleted
     },
     "machine": {
         str: {
@@ -260,11 +297,15 @@ SCHEMA = {
         },
     },
     # section for experimental features
-    "feature": {
-        Optional("machine", default=False): Bool,
-        # enabled by default. It's of no use, kept for backward compatibility.
-        Optional("parametrization", default=True): Bool,
-    },
+    # only specified keys are validated, others get logged and then ignored/removed
+    "feature": FeatureSchema(
+        {
+            Optional("machine", default=False): Bool,
+            Optional("push_exp_to_studio", default=False): Bool,
+            "studio_token": str,
+            "studio_url": str,
+        },
+    ),
     "plots": {
         "html_template": str,
         Optional("auto_open", default=False): Bool,

dvc/daemon.py

@@ -5,7 +5,7 @@ import logging
 import os
 import platform
 import sys
-from subprocess import Popen
+from subprocess import Popen  # nosec B404
 from typing import List
 
 from dvc.env import DVC_DAEMON
@@ -14,19 +14,26 @@ from dvc.utils import fix_env, is_binary
 logger = logging.getLogger(__name__)
 
 
-def _popen(cmd, **kwargs):
+def _suppress_resource_warning(popen: Popen):
+    """Sets the returncode to avoid ResourceWarning when popen is garbage collected."""
+    # only use for daemon processes.
+    # See https://bugs.python.org/issue38890.
+    popen.returncode = 0
+
+
+def _popen(cmd, **kwargs) -> Popen:
     prefix = [sys.executable]
     if not is_binary():
         main_entrypoint = os.path.join(
             os.path.abspath(os.path.dirname(__file__)), "__main__.py"
         )
         prefix += [main_entrypoint]
-    return Popen(prefix + cmd, close_fds=True, shell=False, **kwargs)
+    return Popen(prefix + cmd, close_fds=True, shell=False, **kwargs)  # nosec B603
 
 
 def _spawn_windows(cmd, env):
     if sys.platform == "win32":
-        from subprocess import (
+        from subprocess import (  # nosec B404
             CREATE_NEW_PROCESS_GROUP,
             CREATE_NO_WINDOW,
             STARTF_USESHOWWINDOW,
@@ -40,12 +47,22 @@ def _spawn_windows(cmd, env):
         startupinfo = STARTUPINFO()
         startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 
-        _popen(cmd, env=env, creationflags=creationflags, startupinfo=startupinfo)
+        popen = _popen(
+            cmd, env=env, creationflags=creationflags, startupinfo=startupinfo
+        )
+        _suppress_resource_warning(popen)
 
 
 def _spawn_posix(cmd, env):
     from dvc.cli import main
 
+    # `fork` will copy buffers, so we need to flush them before forking.
+    # Otherwise, we will get duplicated outputs.
+    if sys.stdout and not sys.stdout.closed:
+        sys.stdout.flush()
+    if sys.stderr and not sys.stderr.closed:
+        sys.stderr.flush()
+
     # NOTE: using os._exit instead of sys.exit, because dvc built
     # with PyInstaller has trouble with SystemExit exception and throws
     # errors such as "[26338] Failed to execute script __main__"
@@ -72,6 +89,7 @@ def _spawn_posix(cmd, env):
     sys.stdin.close()
     sys.stdout.close()
     sys.stderr.close()
+    os.closerange(0, 3)
 
     if platform.system() == "Darwin":
         # workaround for MacOS bug

dvc/dagascii.py

@@ -80,7 +80,7 @@ class AsciiCanvas:
 
         self.canvas[y][x] = char
 
-    def line(self, x0, y0, x1, y1, char):  # noqa: C901
+    def line(self, x0, y0, x1, y1, char):  # noqa: C901, PLR0912
         """Create a line on ASCII canvas.
 
         Args:
@@ -213,8 +213,8 @@ def draw(vertices, edges):
     # pylint: disable=too-many-locals
     # NOTE: coordinates might me negative, so we need to shift
     # everything to the positive plane before we actually draw it.
-    Xs = []  # pylint: disable=invalid-name
-    Ys = []  # pylint: disable=invalid-name
+    Xs = []  # noqa: N806, pylint: disable=invalid-name
+    Ys = []  # noqa: N806, pylint: disable=invalid-name
 
     sug = _build_sugiyama_layout(vertices, edges)
 

dvc/data_cloud.py

@@ -18,10 +18,11 @@ logger = logging.getLogger(__name__)
 
 
 class Remote:
-    def __init__(self, name: str, path: str, fs: "FileSystem", **config):
+    def __init__(self, name: str, path: str, fs: "FileSystem", *, index=None, **config):
         self.path = path
         self.fs = fs
         self.name = name
+        self.index = index
 
         self.worktree: bool = config.pop("worktree", False)
         self.config = config
@@ -34,7 +35,7 @@ class Remote:
         if self.worktree:
             path = self.fs.path.join(path, ".dvc", "cache")
 
-        return get_odb(self.fs, path, **self.config)
+        return get_odb(self.fs, path, hash_name="md5", **self.config)
 
 
 class DataCloud:
@@ -74,19 +75,23 @@ class DataCloud:
                     config["version_aware"] = True
 
             fs = cls(**config)
-            config["tmp_dir"] = self.repo.index_db_dir
-            return Remote(name, fs_path, fs, **config)
+            config["tmp_dir"] = self.repo.site_cache_dir
+            if self.repo.data_index is not None:
+                index = self.repo.data_index.view(("remote", name))
+            else:
+                index = None
+            return Remote(name, fs_path, fs, index=index, **config)
 
         if bool(self.repo.config["remote"]):
             error_msg = (
-                "no remote specified. Setup default remote with\n"
+                f"no remote specified in {self.repo}. Setup default remote with\n"
                 "    dvc remote default <remote name>\n"
                 "or use:\n"
-                "    dvc {} -r <remote name>".format(command)
+                f"    dvc {command} -r <remote name>"
             )
         else:
             error_msg = (
-                "no remote specified. Create a default remote with\n"
+                f"no remote specified in {self.repo}. Create a default remote with\n"
                 "    dvc remote add -d <remote name> <remote url>"
             )
 

dvc/dependency/param.py

@@ -2,9 +2,9 @@ import logging
 import os
 import typing
 from collections import defaultdict
-from typing import Dict, cast
+from typing import Dict
 
-import dpath.util
+import dpath
 from voluptuous import Any
 
 from dvc.exceptions import DvcException
@@ -87,17 +87,17 @@ class ParamsDependency(Dependency):
         if flatten:
             for param in self.params:
                 try:
-                    ret[param] = dpath.util.get(config, param, separator=".")
+                    ret[param] = dpath.get(config, param, separator=".")
                 except KeyError:
                     continue
             return ret
 
-        from dpath.util import merge
+        from dpath import merge
 
         for param in self.params:
             merge(
                 ret,
-                dpath.util.search(config, param, separator="."),
+                dpath.search(config, param, separator="."),
                 separator=".",
             )
         return ret
@@ -111,7 +111,8 @@ class ParamsDependency(Dependency):
         from funcy import ldistinct
 
         status: Dict[str, Any] = defaultdict(dict)
-        info = cast(dict, self.hash_info.value) if self.hash_info else {}
+        assert isinstance(self.hash_info.value, dict)
+        info = self.hash_info.value if self.hash_info else {}
         actual = self.read_params()
 
         # NOTE: we want to preserve the order of params as specified in the

dvc/dependency/repo.py

@@ -1,24 +1,14 @@
-import os
-from collections import defaultdict
-from copy import copy
-from typing import TYPE_CHECKING, Dict, Optional, Set, Tuple, Union
+from typing import TYPE_CHECKING, Dict, Optional, Union
 
 from voluptuous import Required
 
-from dvc.prompt import confirm
+from dvc.utils import as_posix
 
 from .base import Dependency
 
 if TYPE_CHECKING:
-    from typing import ContextManager
-
-    from dvc.output import Output
-    from dvc.repo import Repo
+    from dvc.fs import DVCFileSystem
     from dvc.stage import Stage
-    from dvc_data.hashfile.hash_info import HashInfo
-    from dvc_data.hashfile.meta import Meta
-    from dvc_data.hashfile.obj import HashFile
-    from dvc_objects.db import ObjectDB
 
 
 class RepoDependency(Dependency):
@@ -37,10 +27,11 @@ class RepoDependency(Dependency):
 
     def __init__(self, def_repo: Dict[str, str], stage: "Stage", *args, **kwargs):
         self.def_repo = def_repo
-        self._objs: Dict[str, "HashFile"] = {}
-        self._meta: Dict[str, "Meta"] = {}
         super().__init__(stage, *args, **kwargs)
 
+        self.fs = self._make_fs()
+        self.fs_path = as_posix(self.def_path)
+
     def _parse_path(self, fs, fs_path):  # noqa: ARG002
         return None
 
@@ -52,8 +43,8 @@ class RepoDependency(Dependency):
         return f"{self.def_path} ({self.def_repo[self.PARAM_URL]})"
 
     def workspace_status(self):
-        current = self.get_obj(locked=True).hash_info
-        updated = self.get_obj(locked=False).hash_info
+        current = self._make_fs(locked=True).repo.get_rev()
+        updated = self._make_fs(locked=False).repo.get_rev()
 
         if current != updated:
             return {str(self): "update available"}
@@ -64,33 +55,18 @@ class RepoDependency(Dependency):
         return self.workspace_status()
 
     def save(self):
-        pass
+        rev = self.fs.repo.get_rev()
+        if self.def_repo.get(self.PARAM_REV_LOCK) is None:
+            self.def_repo[self.PARAM_REV_LOCK] = rev
 
     def dumpd(self, **kwargs) -> Dict[str, Union[str, Dict[str, str]]]:
         return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
 
-    def download(self, to: "Output", jobs: Optional[int] = None):
-        from dvc_data.hashfile.checkout import checkout
-
-        for odb, objs in self.get_used_objs().items():
-            self.repo.cloud.pull(objs, jobs=jobs, odb=odb)
-
-        obj = self.get_obj()
-        checkout(
-            to.fs_path,
-            to.fs,
-            obj,
-            self.repo.cache.local,
-            ignore=None,
-            state=self.repo.state,
-            prompt=confirm,
-        )
-
     def update(self, rev: Optional[str] = None):
         if rev:
             self.def_repo[self.PARAM_REV] = rev
-        with self._make_repo(locked=False) as repo:
-            self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
+        self.fs = self._make_fs(rev=rev, locked=False)
+        self.def_repo[self.PARAM_REV_LOCK] = self.fs.repo.get_rev()
 
     def changed_checksum(self) -> bool:
         # From current repo point of view what describes RepoDependency is its
@@ -98,124 +74,16 @@ class RepoDependency(Dependency):
         # immutable, hence its impossible for checksum to change.
         return False
 
-    def get_used_objs(self, **kwargs) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
-        used, _, _ = self._get_used_and_obj(**kwargs)
-        return used
+    def _make_fs(
+        self, rev: Optional[str] = None, locked: bool = True
+    ) -> "DVCFileSystem":
+        from dvc.fs import DVCFileSystem
 
-    def _get_used_and_obj(
-        self, obj_only: bool = False, **kwargs
-    ) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], "Meta", "HashFile"]:
-        from dvc.config import NoRemoteError
-        from dvc.exceptions import NoOutputOrStageError, PathMissingError
-        from dvc.utils import as_posix
-        from dvc_data.hashfile.build import build
-        from dvc_data.hashfile.tree import Tree, TreeError
-
-        local_odb = self.repo.cache.local
-        locked = kwargs.pop("locked", True)
-        with self._make_repo(locked=locked, cache_dir=local_odb.path) as repo:
-            used_obj_ids = defaultdict(set)
-            rev = repo.get_rev()
-            if locked and self.def_repo.get(self.PARAM_REV_LOCK) is None:
-                self.def_repo[self.PARAM_REV_LOCK] = rev
-
-            if not obj_only:
-                try:
-                    for odb, obj_ids in repo.used_objs(
-                        [os.path.join(repo.root_dir, self.def_path)],
-                        force=True,
-                        jobs=kwargs.get("jobs"),
-                        recursive=True,
-                    ).items():
-                        if odb is None:
-                            odb = repo.cloud.get_remote_odb()
-                            odb.read_only = True
-                        self._check_circular_import(odb, obj_ids)
-                        used_obj_ids[odb].update(obj_ids)
-                except (NoRemoteError, NoOutputOrStageError):
-                    pass  # noqa: S110
-
-            try:
-                object_store, meta, obj = build(
-                    local_odb,
-                    as_posix(self.def_path),
-                    repo.dvcfs,
-                    local_odb.fs.PARAM_CHECKSUM,
-                )
-            except (FileNotFoundError, TreeError) as exc:
-                raise PathMissingError(
-                    self.def_path, self.def_repo[self.PARAM_URL]
-                ) from exc
-            object_store = copy(object_store)
-            object_store.read_only = True
-
-            self._objs[rev] = obj
-            self._meta[rev] = meta
-
-            used_obj_ids[object_store].add(obj.hash_info)
-            if isinstance(obj, Tree):
-                used_obj_ids[object_store].update(oid for _, _, oid in obj)
-            return used_obj_ids, meta, obj
-
-    def _check_circular_import(self, odb: "ObjectDB", obj_ids: Set["HashInfo"]) -> None:
-        from dvc.exceptions import CircularImportError
-        from dvc.fs.dvc import DVCFileSystem
-        from dvc_data.hashfile.db.reference import ReferenceHashFileDB
-        from dvc_data.hashfile.tree import Tree
-
-        if not isinstance(odb, ReferenceHashFileDB):
-            return
-
-        def iter_objs():
-            for hash_info in obj_ids:
-                if hash_info.isdir:
-                    tree = Tree.load(odb, hash_info)
-                    yield from (odb.get(hi.value) for _, _, hi in tree)
-                else:
-                    assert hash_info.value
-                    yield odb.get(hash_info.value)
-
-        checked_urls = set()
-        for obj in iter_objs():
-            if not isinstance(obj.fs, DVCFileSystem):
-                continue
-            if obj.fs.repo_url in checked_urls or obj.fs.repo.root_dir in checked_urls:
-                continue
-            self_url = self.repo.url or self.repo.root_dir
-            if (
-                obj.fs.repo_url is not None
-                and obj.fs.repo_url == self_url
-                or obj.fs.repo.root_dir == self.repo.root_dir
-            ):
-                raise CircularImportError(self, obj.fs.repo_url, self_url)
-            checked_urls.update([obj.fs.repo_url, obj.fs.repo.root_dir])
-
-    def get_obj(self, filter_info=None, **kwargs):
-        locked = kwargs.get("locked", True)
-        rev = self._get_rev(locked=locked)
-        if rev in self._objs:
-            return self._objs[rev]
-        _, _, obj = self._get_used_and_obj(
-            obj_only=True, filter_info=filter_info, **kwargs
+        return DVCFileSystem(
+            url=self.def_repo[self.PARAM_URL],
+            rev=rev or self._get_rev(locked=locked),
+            subrepos=True,
         )
-        return obj
-
-    def get_meta(self, filter_info=None, **kwargs):
-        locked = kwargs.get("locked", True)
-        rev = self._get_rev(locked=locked)
-        if rev in self._meta:
-            return self._meta[rev]
-        _, meta, _ = self._get_used_and_obj(
-            obj_only=True, filter_info=filter_info, **kwargs
-        )
-        return meta
-
-    def _make_repo(self, locked: bool = True, **kwargs) -> "ContextManager[Repo]":
-        from dvc.external_repo import external_repo
-
-        d = self.def_repo
-        rev = self._get_rev(locked=locked)
-        return external_repo(d[self.PARAM_URL], rev=rev, **kwargs)
 
     def _get_rev(self, locked: bool = True):
         d = self.def_repo

dvc/dvcfile.py

@@ -187,11 +187,12 @@ class FileMixin:
 
 class SingleStageFile(FileMixin):
     from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
-    from dvc.stage.loader import SingleStageLoader as LOADER
+    from dvc.stage.loader import SingleStageLoader as LOADER  # noqa: N814
 
     metrics: List[str] = []
     plots: Any = {}
     params: List[str] = []
+    artifacts: Dict[str, Optional[Dict[str, Any]]] = {}
 
     @property
     def stage(self) -> "Stage":
@@ -231,7 +232,7 @@ class ProjectFile(FileMixin):
     """Abstraction for pipelines file, .yaml + .lock combined."""
 
     from dvc.schema import COMPILED_MULTI_STAGE_SCHEMA as SCHEMA
-    from dvc.stage.loader import StageLoader as LOADER
+    from dvc.stage.loader import StageLoader as LOADER  # noqa: N814
 
     @property
     def _lockfile(self):
@@ -323,6 +324,10 @@ class ProjectFile(FileMixin):
     def params(self) -> List[str]:
         return self.contents.get("params", [])
 
+    @property
+    def artifacts(self) -> Dict[str, Optional[Dict[str, Any]]]:
+        return self.contents.get("artifacts", {})
+
     def remove(self, force=False):
         if not force:
             logger.warning("Cannot remove pipeline file.")
@@ -366,7 +371,7 @@ def get_lockfile_schema(d):
 
 
 def migrate_lock_v1_to_v2(d, version_info):
-    stages = {k: v for k, v in d.items()}
+    stages = dict(d)
 
     for key in stages:
         d.pop(key)
@@ -409,11 +414,10 @@ class Lockfile(FileMixin):
             if version == LOCKFILE_VERSION.V1:
                 logger.info("Migrating lock file '%s' from v1 to v2", self.relpath)
                 migrate_lock_v1_to_v2(data, self.latest_version_info)
-            else:
-                if not data:
-                    data.update(self.latest_version_info)
-                    # order is important, meta should always be at the top
-                    logger.info("Generating lock file '%s'", self.relpath)
+            elif not data:
+                data.update(self.latest_version_info)
+                # order is important, meta should always be at the top
+                logger.info("Generating lock file '%s'", self.relpath)
 
             data["stages"] = data.get("stages", {})
             modified = data["stages"].get(stage.name, {}) != stage_data.get(

dvc/env.py

@@ -9,3 +9,4 @@ DVC_EXP_NAME = "DVC_EXP_NAME"
 DVC_EXP_GIT_REMOTE = "DVC_EXP_GIT_REMOTE"
 DVC_EXP_AUTO_PUSH = "DVC_EXP_AUTO_PUSH"
 DVC_NO_ANALYTICS = "DVC_NO_ANALYTICS"
+DVC_SHOW_TRACEBACK = "DVC_SHOW_TRACEBACK"

dvc/exceptions.py

@@ -31,14 +31,19 @@ class OutputDuplicationError(DvcException):
 
         assert isinstance(output, str)
         assert all(hasattr(stage, "relpath") for stage in stages)
+        msg = ""
+        stage_names = [s.addressing for s in stages]
+        stages_str = " ".join(stage_names)
         if len(stages) == 1:
-            msg = "output '{}' is already specified in {}.".format(
-                output, first(stages)
-            )
+            stage_name = first(stages)
+            msg = f"output '{output}' is already specified in {stage_name}."
         else:
             msg = "output '{}' is already specified in stages:\n{}".format(
-                output, "\n".join(f"\t- {s.addressing}" for s in stages)
+                output, "\n".join(f"\t- {s}" for s in stage_names)
             )
+        msg += (
+            f"\nUse `dvc remove {stages_str}` to stop tracking the overlapping output."
+        )
         super().__init__(msg)
         self.stages = stages
         self.output = output
@@ -333,14 +338,6 @@ class CacheLinkError(DvcException):
         self.fs_paths = fs_paths
 
 
-class CircularImportError(DvcException):
-    def __init__(self, dep, a, b):
-        super().__init__(
-            f"'{dep}' contains invalid circular import. "
-            f"DVC repo '{a}' already imports from '{b}'."
-        )
-
-
 class PrettyDvcException(DvcException):
     def __pretty_exc__(self, **kwargs):
         """Print prettier exception message."""

dvc/external_repo.py

@@ -7,14 +7,7 @@ from typing import TYPE_CHECKING, Collection, Dict, Iterator, Optional, Tuple
 
 from funcy import retry, wrap_with
 
-from dvc.exceptions import (
-    FileMissingError,
-    NoOutputInExternalRepoError,
-    NoRemoteInExternalRepoError,
-    NotDvcRepoError,
-    OutputNotFoundError,
-    PathMissingError,
-)
+from dvc.exceptions import NotDvcRepoError
 from dvc.repo import Repo
 from dvc.scm import CloneError, map_scm_exception
 from dvc.utils import relpath
@@ -37,10 +30,6 @@ def external_repo(
     cache_types: Optional[Collection[str]] = None,
     **kwargs,
 ) -> Iterator["Repo"]:
-    from dvc.config import NoRemoteError
-    from dvc.fs import GitFileSystem
-    from dvc.scm import Git
-
     logger.debug("Creating external repo %s@%s", url, rev)
     path = _cached_clone(url, rev, for_write=for_write)
     # Local HEAD points to the tip of whatever branch we first cloned from
@@ -54,44 +43,27 @@ def external_repo(
 
     config = _get_remote_config(url) if os.path.isdir(url) else {}
     config.update(cache_config)
+    config.update(kwargs.pop("config", None) or {})
 
+    main_root = "/"
     if for_write:
-        scm = None
-        root_dir = path
-        fs = None
-    else:
-        scm = Git(path)
-        fs = GitFileSystem(scm=scm, rev=rev)
-        root_dir = "/"
+        # we already checked out needed revision
+        rev = None
+        main_root = path
 
     repo_kwargs = dict(
-        root_dir=root_dir,
+        root_dir=path,
         url=url,
-        fs=fs,
         config=config,
-        repo_factory=erepo_factory(url, root_dir, cache_config),
-        scm=scm,
+        repo_factory=erepo_factory(url, main_root, cache_config),
+        rev=rev,
         **kwargs,
     )
 
-    if "subrepos" not in repo_kwargs:
-        repo_kwargs["subrepos"] = True
-
-    if "uninitialized" not in repo_kwargs:
-        repo_kwargs["uninitialized"] = True
-
     repo = Repo(**repo_kwargs)
 
     try:
         yield repo
-    except NoRemoteError as exc:
-        raise NoRemoteInExternalRepoError(url) from exc
-    except OutputNotFoundError as exc:
-        if exc.repo is repo:
-            raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url) from exc
-        raise
-    except FileMissingError as exc:
-        raise PathMissingError(exc.path, url) from exc
     finally:
         repo.close()
         if for_write:
@@ -189,7 +161,7 @@ def _cached_clone(url, rev, for_write=False):
 
 
 @wrap_with(threading.Lock())
-def _clone_default_branch(url, rev, for_write=False):  # noqa: C901
+def _clone_default_branch(url, rev, for_write=False):  # noqa: C901, PLR0912
     """Get or create a clean clone of the url.
 
     The cloned is reactualized with git pull unless rev is a known sha.

dvc/fs/__init__.py

@@ -6,7 +6,6 @@ from dvc.config import ConfigError as RepoConfigError
 from dvc.config_schema import SCHEMA, Invalid
 
 # pylint: disable=unused-import
-from dvc_objects.fs import utils  # noqa: F401
 from dvc_objects.fs import (  # noqa: F401
     LocalFileSystem,
     MemoryFileSystem,
@@ -17,6 +16,7 @@ from dvc_objects.fs import (  # noqa: F401
     localfs,
     registry,
     system,
+    utils,
 )
 from dvc_objects.fs.base import AnyFSPath, FileSystem  # noqa: F401
 from dvc_objects.fs.errors import (  # noqa: F401
@@ -121,7 +121,7 @@ def _get_cloud_fs(repo_config, **kwargs):
 
     remote_conf = get_fs_config(repo_config, **kwargs)
     try:
-        remote_conf = SCHEMA["remote"][str](remote_conf)
+        remote_conf = SCHEMA["remote"][str](remote_conf)  # type: ignore[index]
     except Invalid as exc:
         raise RepoConfigError(str(exc)) from None
 

dvc/fs/callbacks.py

@@ -1,6 +1,6 @@
 # pylint: disable=unused-import
 from contextlib import ExitStack
-from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional, Union, cast
+from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional, Union
 
 from dvc.utils.objects import cached_property
 from dvc_objects.fs.callbacks import (  # noqa: F401
@@ -54,7 +54,8 @@ class RichCallback(Callback):
             disable=self.disable,
             console=ui.error_console,
         )
-        return cast(RichTransferProgress, self._stack.enter_context(progress))
+        self._stack.enter_context(progress)
+        return progress
 
     @cached_property
     def task(self) -> "TaskID":

dvc/fs/dvc.py

@@ -251,12 +251,15 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         try:
             return self.repo.fs.open(fs_path, mode=mode)
         except FileNotFoundError:
-            _, dvc_fs, subkey = self._get_subrepo_info(key)
+            repo, dvc_fs, subkey = self._get_subrepo_info(key)
             if not dvc_fs:
                 raise
 
         dvc_path = _get_dvc_path(dvc_fs, subkey)
-        return dvc_fs.open(dvc_path, mode=mode)
+        kw = {}
+        if kwargs.get("cache_remote_stream", False):
+            kw["cache_odb"] = repo.cache.local
+        return dvc_fs.open(dvc_path, mode=mode, **kw)
 
     def isdvc(self, path, **kwargs) -> bool:
         """Is this entry dvc-tracked?"""
@@ -265,19 +268,22 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         dvc_path = _get_dvc_path(dvc_fs, subkey)
         return dvc_fs is not None and dvc_fs.isdvc(dvc_path, **kwargs)
 
-    def ls(  # pylint: disable=arguments-differ
+    def ls(  # pylint: disable=arguments-differ # noqa: C901
         self, path, detail=True, dvc_only=False, **kwargs
     ):
         key = self._get_key_from_relative(path)
         repo, dvc_fs, subkey = self._get_subrepo_info(key)
 
+        dvc_exists = False
         dvc_infos = {}
         if dvc_fs:
+            dvc_path = _get_dvc_path(dvc_fs, subkey)
             with suppress(FileNotFoundError):
-                dvc_path = _get_dvc_path(dvc_fs, subkey)
                 for info in dvc_fs.ls(dvc_path, detail=True):
                     dvc_infos[dvc_fs.path.name(info["name"])] = info
+            dvc_exists = bool(dvc_infos) or dvc_fs.exists(dvc_path)
 
+        fs_exists = False
         fs_infos = {}
         ignore_subrepos = kwargs.get("ignore_subrepos", True)
         if not dvc_only:
@@ -289,13 +295,20 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
                 ):
                     fs_infos[fs.path.name(info["name"])] = info
             except (FileNotFoundError, NotADirectoryError):
-                pass  # noqa: S110
+                pass
+
+            fs_exists = bool(fs_infos) or fs.exists(fs_path)
 
         dvcfiles = kwargs.get("dvcfiles", False)
 
         infos = []
         paths = []
         names = set(dvc_infos.keys()) | set(fs_infos.keys())
+
+        if not names and (dvc_exists or fs_exists):
+            # broken symlink or TreeError
+            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path)
+
         for name in names:
             if not dvcfiles and _is_dvc_file(name):
                 continue
@@ -316,7 +329,9 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         ignore_subrepos = kwargs.get("ignore_subrepos", True)
         return self._info(key, path, ignore_subrepos=ignore_subrepos)
 
-    def _info(self, key, path, ignore_subrepos=True, check_ignored=True):  # noqa: C901
+    def _info(  # noqa: C901, PLR0912
+        self, key, path, ignore_subrepos=True, check_ignored=True
+    ):
         repo, dvc_fs, subkey = self._get_subrepo_info(key)
 
         dvc_info = None
@@ -359,6 +374,19 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         info["name"] = path
         return info
 
+    def get_file(self, rpath, lpath, **kwargs):  # pylint: disable=arguments-differ
+        key = self._get_key_from_relative(rpath)
+        fs_path = self._from_key(key)
+        try:
+            return self.repo.fs.get_file(fs_path, lpath, **kwargs)
+        except FileNotFoundError:
+            _, dvc_fs, subkey = self._get_subrepo_info(key)
+            if not dvc_fs:
+                raise
+
+        dvc_path = _get_dvc_path(dvc_fs, subkey)
+        return dvc_fs.get_file(dvc_path, lpath, **kwargs)
+
 
 class DVCFileSystem(FileSystem):
     protocol = "local"

dvc/info.py

@@ -19,7 +19,6 @@ SUBPROJECTS = (
     "dvc_objects",
     "dvc_render",
     "dvc_task",
-    "dvclive",
     "scmrepo",
 )
 package = "" if PKG is None else f" ({PKG})"
@@ -55,6 +54,7 @@ def get_dvc_info():
             fs_root = get_fs_type(os.path.abspath(root_directory))
             info.append(f"Workspace directory: {fs_root}")
             info.append(f"Repo: {_get_dvc_repo_info(repo)}")
+            info.append(f"Repo.site_cache_dir: {repo.site_cache_dir}")
     except NotDvcRepoError:
         pass
     except SCMError:
@@ -139,7 +139,7 @@ def _get_supported_remotes():
 def get_fs_type(path):
     partition = {}
     for part in psutil.disk_partitions(all=True):
-        if part.fstype != "":
+        if part.fstype:
             try:
                 mountpoint = pathlib.Path(part.mountpoint).resolve()
                 partition[mountpoint] = part.fstype + " on " + part.device

dvc/lock.py

@@ -187,7 +187,9 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
 
         if self._tmp_dir is not None:
             # Under Windows file path length is limited so we hash it
-            hasher = hashlib.md5(self._claimfile.encode())  # noqa: S324
+            hasher = hashlib.md5(  # nosec B324, B303  # noqa: S324
+                self._claimfile.encode()
+            )
             filename = hasher.hexdigest()
             self._claimfile = os.path.join(self._tmp_dir, filename + ".lock")
 

dvc/logger.py

@@ -3,63 +3,55 @@
 import logging
 import logging.config
 import logging.handlers
+import os
 import sys
 
 import colorama
 
+from dvc.env import DVC_SHOW_TRACEBACK
 from dvc.progress import Tqdm
 
-FOOTER = (
-    "\n{yellow}Having any troubles?{nc}"
-    " Hit us up at {blue}https://dvc.org/support{nc},"
-    " we are always happy to help!"
-).format(
-    blue=colorama.Fore.BLUE,
-    nc=colorama.Fore.RESET,
-    yellow=colorama.Fore.YELLOW,
-)
 
-
-def addLoggingLevel(levelName, levelNum, methodName=None):
+def add_logging_level(level_name, level_num, method_name=None):
     """
     Adds a new logging level to the `logging` module and the
     currently configured logging class.
 
-    Uses the existing numeric levelNum if already defined.
+    Uses the existing numeric level_num if already defined.
 
     Based on https://stackoverflow.com/questions/2183233
     """
-    if methodName is None:
-        methodName = levelName.lower()
+    if method_name is None:
+        method_name = level_name.lower()
 
     # If the level name is already defined as a top-level `logging`
     # constant, then adopt the existing numeric level.
-    if hasattr(logging, levelName):
-        existingLevelNum = getattr(logging, levelName)
-        assert isinstance(existingLevelNum, int)
-        levelNum = existingLevelNum
+    if hasattr(logging, level_name):
+        existing_level_num = getattr(logging, level_name)
+        assert isinstance(existing_level_num, int)
+        level_num = existing_level_num
 
-    def logForLevel(self, message, *args, **kwargs):
-        if self.isEnabledFor(levelNum):
+    def log_for_level(self, message, *args, **kwargs):
+        if self.isEnabledFor(level_num):
             # pylint: disable=protected-access
-            self._log(levelNum, message, args, **kwargs)
+            self._log(level_num, message, args, **kwargs)
 
-    def logToRoot(message, *args, **kwargs):
-        logging.log(levelNum, message, *args, **kwargs)
+    def log_to_root(message, *args, **kwargs):
+        logging.log(level_num, message, *args, **kwargs)
 
     # getLevelName resolves the numeric log level if already defined,
     # otherwise returns a string
-    if not isinstance(logging.getLevelName(levelName), int):
-        logging.addLevelName(levelNum, levelName)
+    if not isinstance(logging.getLevelName(level_name), int):
+        logging.addLevelName(level_num, level_name)
 
-    if not hasattr(logging, levelName):
-        setattr(logging, levelName, levelNum)
+    if not hasattr(logging, level_name):
+        setattr(logging, level_name, level_num)
 
-    if not hasattr(logging.getLoggerClass(), methodName):
-        setattr(logging.getLoggerClass(), methodName, logForLevel)
+    if not hasattr(logging.getLoggerClass(), method_name):
+        setattr(logging.getLoggerClass(), method_name, log_for_level)
 
-    if not hasattr(logging, methodName):
-        setattr(logging, methodName, logToRoot)
+    if not hasattr(logging, method_name):
+        setattr(logging, method_name, log_to_root)
 
 
 class LoggingException(Exception):
@@ -96,9 +88,10 @@ class ColorFormatter(logging.Formatter):
         "CRITICAL": colorama.Fore.RED,
     }
 
-    def __init__(self, log_colors: bool = True) -> None:
+    def __init__(self, log_colors: bool = True, show_traceback: bool = False) -> None:
         super().__init__()
         self.log_colors = log_colors
+        self.show_traceback = show_traceback
 
     def format(self, record) -> str:  # noqa: A003, C901
         record.message = record.getMessage()
@@ -116,8 +109,10 @@ class ColorFormatter(logging.Formatter):
             msg = msg + sep + cause
 
         asctime = ""
-        if _is_verbose():
+        verbose = _is_verbose()
+        if verbose:
             asctime = self.formatTime(record, self.datefmt)
+        if verbose or self.show_traceback:
             if ei and not record.exc_text:
                 record.exc_text = self.formatException(ei)
             if record.exc_text:
@@ -139,7 +134,7 @@ class ColorFormatter(logging.Formatter):
 
 
 class LoggerHandler(logging.StreamHandler):
-    def handleError(self, record):
+    def handleError(self, record):  # noqa: N802
         super().handleError(record)
         raise LoggingException(record)
 
@@ -156,8 +151,9 @@ class LoggerHandler(logging.StreamHandler):
                         self.emit_pretty_exception(exc, verbose=_is_verbose())
                         if not _is_verbose():
                             return
-                    except Exception:  # noqa, pylint: disable=broad-except
-                        pass  # noqa: S110
+                    # pylint: disable-next=broad-except
+                    except Exception:  # noqa: BLE001, S110  # nosec B110
+                        pass
 
             msg = self.format(record)
             Tqdm.write(msg, file=self.stream, end=getattr(self, "terminator", "\n"))
@@ -200,14 +196,17 @@ def setup(level: int = logging.INFO, log_colors: bool = True) -> None:
     console_debug.setFormatter(formatter)
     console_debug.addFilter(exclude_filter(logging.INFO))
 
-    addLoggingLevel("TRACE", logging.DEBUG - 5)
+    add_logging_level("TRACE", logging.DEBUG - 5)
 
     console_trace = LoggerHandler(sys.stdout)
     console_trace.setLevel(logging.TRACE)  # type: ignore[attr-defined]
     console_trace.setFormatter(formatter)
     console_trace.addFilter(exclude_filter(logging.DEBUG))
 
-    err_formatter = ColorFormatter(log_colors=log_colors and sys.stderr.isatty())
+    show_traceback = bool(os.environ.get(DVC_SHOW_TRACEBACK))
+    err_formatter = ColorFormatter(
+        log_colors=log_colors and sys.stderr.isatty(), show_traceback=show_traceback
+    )
     console_errors = LoggerHandler(sys.stderr)
     console_errors.setLevel(logging.WARNING)
     console_errors.setFormatter(err_formatter)

dvc/output.py

@@ -2,8 +2,9 @@ import logging
 import os
 import posixpath
 from collections import defaultdict
+from contextlib import suppress
 from operator import itemgetter
-from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Type, cast
+from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Type
 from urllib.parse import urlparse
 
 from funcy import collecting, first, project
@@ -20,7 +21,6 @@ from dvc.exceptions import (
     RemoteCacheRequiredError,
 )
 from dvc.utils.objects import cached_property
-from dvc_data.hashfile import Tree
 from dvc_data.hashfile import check as ocheck
 from dvc_data.hashfile import load as oload
 from dvc_data.hashfile.build import build
@@ -29,6 +29,7 @@ from dvc_data.hashfile.hash_info import HashInfo
 from dvc_data.hashfile.istextfile import istextfile
 from dvc_data.hashfile.meta import Meta
 from dvc_data.hashfile.transfer import transfer as otransfer
+from dvc_data.hashfile.tree import Tree
 from dvc_objects.errors import ObjectFormatError
 
 from .annotations import ANNOTATION_FIELDS, ANNOTATION_SCHEMA, Annotation
@@ -312,7 +313,7 @@ class Output:
     IsStageFileError: Type[DvcException] = OutputIsStageFileError
     IsIgnoredError: Type[DvcException] = OutputIsIgnoredError
 
-    def __init__(
+    def __init__(  # noqa: PLR0913
         self,
         stage,
         path,
@@ -426,7 +427,7 @@ class Output:
             self.hash_info = tree.hash_info
             self.meta.isdir = True
             self.meta.nfiles = len(self.files)
-            self.meta.size = sum(f.get("size") for f in self.files)
+            self.meta.size = sum(filter(None, (f.get("size") for f in self.files)))
             self.meta.remote = first(f.get("remote") for f in self.files)
         elif self.meta.nfiles or self.hash_info and self.hash_info.isdir:
             self.meta.isdir = True
@@ -709,7 +710,7 @@ class Output:
         except _CheckoutError as exc:
             raise CheckoutError(exc.paths, {})  # noqa: B904
 
-    def commit(self, filter_info=None) -> None:
+    def commit(self, filter_info=None, relink=True) -> None:
         if not self.exists:
             raise self.DoesNotExistError(self)
 
@@ -719,8 +720,11 @@ class Output:
             granular = (
                 self.is_dir_checksum and filter_info and filter_info != self.fs_path
             )
+            # NOTE: trying to use hardlink during transfer only if we will be
+            # relinking later
+            hardlink = relink
             if granular:
-                obj = self._commit_granular_dir(filter_info)
+                obj = self._commit_granular_dir(filter_info, hardlink)
             else:
                 staging, _, obj = build(
                     self.cache,
@@ -734,41 +738,43 @@ class Output:
                     self.cache,
                     {obj.hash_info},
                     shallow=False,
-                    hardlink=True,
+                    hardlink=hardlink,
                 )
-            self._checkout(
-                filter_info or self.fs_path,
-                self.fs,
-                obj,
-                self.cache,
-                relink=True,
-                state=self.repo.state,
-                prompt=prompt.confirm,
-            )
-            self.set_exec()
+            if relink:
+                self._checkout(
+                    filter_info or self.fs_path,
+                    self.fs,
+                    obj,
+                    self.cache,
+                    relink=True,
+                    state=self.repo.state,
+                    prompt=prompt.confirm,
+                )
+                self.set_exec()
 
-    def _commit_granular_dir(self, filter_info) -> Optional["HashFile"]:
+    def _commit_granular_dir(self, filter_info, hardlink) -> Optional["HashFile"]:
         prefix = self.fs.path.parts(self.fs.path.relpath(filter_info, self.fs_path))
-        staging, _, save_obj = build(
+        staging, _, obj = build(
             self.cache,
             self.fs_path,
             self.fs,
             self.hash_name,
             ignore=self.dvcignore,
         )
-        save_obj = cast(Tree, save_obj)
-        save_obj = cast(Tree, save_obj.filter(prefix))
+        assert isinstance(obj, Tree)
+        save_obj = obj.filter(prefix)
+        assert isinstance(save_obj, Tree)
         checkout_obj = save_obj.get_obj(self.cache, prefix)
         otransfer(
             staging,
             self.cache,
             {save_obj.hash_info} | {oid for _, _, oid in save_obj},
             shallow=True,
-            hardlink=True,
+            hardlink=hardlink,
         )
         return checkout_obj
 
-    def dumpd(self, **kwargs):  # noqa: C901
+    def dumpd(self, **kwargs):  # noqa: C901, PLR0912
         ret: Dict[str, Any] = {}
         with_files = (
             (not self.IS_DEPENDENCY or self.stage.is_import)
@@ -822,7 +828,7 @@ class Output:
         if with_files:
             obj = self.obj or self.get_obj()
             if obj:
-                obj = cast("Tree", obj)
+                assert isinstance(obj, Tree)
                 ret[self.PARAM_FILES] = [
                     split_file_meta_from_cloud(f)
                     for f in _serialize_tree_obj_to_files(obj)
@@ -870,7 +876,7 @@ class Output:
         fs_path = self.fs.path
         if filter_info and filter_info != self.fs_path:
             prefix = fs_path.relparts(filter_info, self.fs_path)
-            obj = cast(Tree, obj)
+            assert isinstance(obj, Tree)
             obj = obj.get_obj(self.cache, prefix)
 
         return obj
@@ -1012,7 +1018,8 @@ class Output:
         except FileNotFoundError:
             if self.remote:
                 kwargs["remote"] = self.remote
-            self.repo.cloud.pull([obj.hash_info], **kwargs)
+            with suppress(Exception):
+                self.repo.cloud.pull([obj.hash_info], **kwargs)
 
         if self.obj:
             return self.obj
@@ -1052,12 +1059,12 @@ class Output:
             return None
 
         obj = self.get_obj()
+        assert obj is None or isinstance(obj, Tree)
         if filter_info and filter_info != self.fs_path:
             assert obj
             prefix = self.fs.path.parts(self.fs.path.relpath(filter_info, self.fs_path))
-            obj = cast(Tree, obj)
             return obj.filter(prefix)
-        return cast(Tree, obj)
+        return obj
 
     def get_used_objs(  # noqa: C901
         self, **kwargs
@@ -1069,9 +1076,7 @@ class Output:
 
         push: bool = kwargs.pop("push", False)
         if self.stage.is_repo_import:
-            if push:
-                return {}
-            return self.get_used_external(**kwargs)
+            return {}
 
         if push and not self.can_push:
             return {}
@@ -1123,15 +1128,6 @@ class Output:
                 oids.add(oid)
         return oids
 
-    def get_used_external(
-        self, **kwargs
-    ) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
-        if not self.use_cache or not self.stage.is_repo_import:
-            return {}
-
-        (dep,) = self.stage.deps
-        return dep.get_used_objs(**kwargs)
-
     def _validate_output_path(self, path, stage=None):
         from dvc.dvcfile import is_valid_filename
 

dvc/parsing/__init__.py

@@ -392,7 +392,7 @@ class ForeachDefinition:
         """Convert sequence to Mapping with keys normalized."""
         iterable = self.resolved_iterable
         if isinstance(iterable, Mapping):
-            return iterable
+            return {to_str(k): v for k, v in iterable.items()}
 
         assert isinstance(iterable, Sequence)
         if any(map(is_map_or_seq, iterable)):

dvc/parsing/context.py

@@ -155,7 +155,7 @@ class Value(Node):
 PRIMITIVES = (int, float, str, bytes, bool)
 
 
-class Container(Node, ABC):  # noqa: B024
+class Container(Node, ABC):
     meta: Meta
     data: Union[list, dict]
     _key_transform = staticmethod(identity)

dvc/parsing/interpolate.py

@@ -73,7 +73,7 @@ def embrace(s: str):
 
 def escape_str(value):
     if os.name == "nt":
-        from subprocess import list2cmdline
+        from subprocess import list2cmdline  # nosec B404
 
         return list2cmdline([value])
     from shlex import quote
@@ -95,16 +95,15 @@ def _(obj: bool):
 def _(obj: dict):  # noqa: C901
     from dvc.config import Config
 
-    config = Config().get("parsing", {})
+    config = Config.from_cwd().get("parsing", {})
 
     result = ""
     for k, v in flatten(obj).items():
         if isinstance(v, bool):
             if v:
                 result += f"--{k} "
-            else:
-                if config.get("bool", "store_true") == "boolean_optional":
-                    result += f"--no-{k} "
+            elif config.get("bool", "store_true") == "boolean_optional":
+                result += f"--no-{k} "
 
         elif isinstance(v, str):
             result += f"--{k} {escape_str(v)} "

dvc/parsing/versions.py

@@ -19,7 +19,7 @@ class VersionEnum(str, enum.Enum):
         return [v.value for v in cls]
 
 
-class LOCKFILE_VERSION(VersionEnum):
+class LOCKFILE_VERSION(VersionEnum):  # noqa: N801
     V1 = "1.0"
     V2 = "2.0"
 

dvc/proc/exceptions.py

@@ -1,23 +0,0 @@
-from dvc.exceptions import DvcException
-
-
-class ProcessNotTerminatedError(DvcException):
-    def __init__(self, name):
-        super().__init__(f"Managed process '{name}' has not been terminated.")
-
-
-class ProcessNotFoundError(DvcException):
-    def __init__(self, name):
-        super().__init__(f"Managed process '{name}' does not exist.")
-
-
-class TimeoutExpired(DvcException):
-    def __init__(self, cmd, timeout):
-        super().__init__(f"'{cmd}' did not complete before timeout '{timeout}'")
-        self.cmd = cmd
-        self.timeout = timeout
-
-
-class UnsupportedSignalError(DvcException):
-    def __init__(self, sig):
-        super().__init__(f"Unsupported signal: {sig}")

dvc/proc/manager.py

@@ -1,149 +0,0 @@
-"""Serverless process manager."""
-
-import json
-import logging
-import os
-import signal
-import sys
-from typing import Generator, List, Optional, Tuple, Union
-
-from funcy.flow import reraise
-from shortuuid import uuid
-
-from .exceptions import ProcessNotTerminatedError, UnsupportedSignalError
-from .process import ManagedProcess, ProcessInfo
-
-logger = logging.getLogger(__name__)
-
-
-class ProcessManager:
-    """Manager for controlling background ManagedProcess(es).
-
-    Spawned process entries are kept in the manager directory until they
-    are explicitly removed (with remove() or cleanup()) so that return
-    value and log information can be accessed after a process has completed.
-    """
-
-    def __init__(self, wdir: Optional[str] = None):
-        self.wdir = wdir or "."
-
-    def __iter__(self) -> Generator[str, None, None]:
-        if not os.path.exists(self.wdir):
-            return
-        yield from os.listdir(self.wdir)
-
-    def __getitem__(self, key: str) -> "ProcessInfo":
-        info_path = os.path.join(self.wdir, key, f"{key}.json")
-        try:
-            with open(info_path, encoding="utf-8") as fobj:
-                return ProcessInfo.from_dict(json.load(fobj))
-        except FileNotFoundError:
-            raise KeyError  # noqa: B904
-
-    @reraise(FileNotFoundError, KeyError)
-    def __setitem__(self, key: str, value: "ProcessInfo"):
-        info_path = os.path.join(self.wdir, key, f"{key}.json")
-        with open(info_path, "w", encoding="utf-8") as fobj:
-            return json.dump(value.asdict(), fobj)
-
-    def __delitem__(self, key: str) -> None:
-        from dvc.utils.fs import remove
-
-        path = os.path.join(self.wdir, key)
-        if os.path.exists(path):
-            remove(path)
-
-    def get(self, key: str, default=None):
-        try:
-            return self[key]
-        except KeyError:
-            return default
-
-    def processes(self) -> Generator[Tuple[str, "ProcessInfo"], None, None]:
-        for name in self:
-            try:
-                yield name, self[name]
-            except KeyError:
-                continue
-
-    def spawn(self, args: Union[str, List[str]], name: Optional[str] = None):
-        """Run the given command in the background."""
-        name = name or uuid()
-        pid = ManagedProcess.spawn(
-            args,
-            wdir=os.path.join(self.wdir, name),
-            name=name,
-        )
-        logger.debug(
-            "Spawned managed process '%s' (PID: '%d')",
-            name,
-            pid,
-        )
-
-    def send_signal(self, name: str, sig: int):
-        """Send `signal` to the specified named process."""
-        process_info = self[name]
-        if sys.platform == "win32" and sig not in (
-            signal.SIGTERM,
-            signal.CTRL_C_EVENT,  # pylint: disable=no-member
-            signal.CTRL_BREAK_EVENT,  # pylint: disable=no-member
-        ):
-            raise UnsupportedSignalError(sig)
-
-        def handle_closed_process():
-            logging.warning("Process %s had already aborted unexpectedly.", name)
-            process_info.returncode = -1
-            self[name] = process_info
-
-        if process_info.returncode is None:
-            try:
-                os.kill(process_info.pid, sig)
-            except ProcessLookupError:
-                handle_closed_process()
-                raise
-            except OSError as exc:
-                if getattr(exc, "winerror", None) == 87:
-                    handle_closed_process()
-                    raise ProcessLookupError from exc
-                raise
-
-    def terminate(self, name: str):
-        """Terminate the specified named process."""
-        self.send_signal(name, signal.SIGTERM)
-
-    def kill(self, name: str):
-        """Kill the specified named process."""
-        if sys.platform == "win32":
-            self.send_signal(name, signal.SIGTERM)
-        else:
-            self.send_signal(name, signal.SIGKILL)  # pylint: disable=no-member
-
-    def remove(self, name: str, force: bool = False):
-        """Remove the specified named process from this manager.
-
-        If the specified process is still running, it will be forcefully killed
-        if `force` is True`, otherwise an exception will be raised.
-
-        Raises:
-            ProcessNotTerminatedError if the specified process is still
-            running and was not forcefully killed.
-        """
-        try:
-            process_info = self[name]
-        except KeyError:
-            return
-        if process_info.returncode is None and not force:
-            raise ProcessNotTerminatedError(name)
-        try:
-            self.kill(name)
-        except ProcessLookupError:
-            pass
-        del self[name]
-
-    def cleanup(self, force: bool = False):
-        """Remove stale (terminated) processes from this manager."""
-        for name in self:
-            try:
-                self.remove(name, force)
-            except ProcessNotTerminatedError:
-                continue

dvc/proc/process.py

@@ -1,179 +0,0 @@
-import json
-import logging
-import os
-import shlex
-import subprocess
-from contextlib import AbstractContextManager
-from dataclasses import asdict, dataclass
-from typing import BinaryIO, List, Optional, Union
-
-from shortuuid import uuid
-
-from dvc.utils.objects import cached_property
-
-from .exceptions import TimeoutExpired
-
-logger = logging.getLogger(__name__)
-
-
-@dataclass
-class ProcessInfo:
-    pid: int
-    stdin: Optional[str]
-    stdout: Optional[str]
-    stderr: Optional[str]
-    returncode: Optional[int]
-
-    @classmethod
-    def from_dict(cls, d):
-        return cls(**d)
-
-    def asdict(self):
-        return asdict(self)
-
-
-class ManagedProcess(AbstractContextManager):
-    """Run the specified command with redirected output.
-
-    stdout and stderr will both be redirected to <name>.out.
-    Interactive processes (requiring stdin input) are currently unsupported.
-
-    Parameters:
-        args: Command to be run.
-        wdir: If specified, redirected output files will be placed in `wdir`.
-        name: Name to use for this process, if not specified a UUID will be
-            generated instead.
-    """
-
-    def __init__(
-        self,
-        args: Union[str, List[str]],
-        wdir: Optional[str] = None,
-        name: Optional[str] = None,
-    ):
-        self.args: List[str] = (
-            shlex.split(args, posix=os.name == "posix")
-            if isinstance(args, str)
-            else list(args)
-        )
-        self.wdir = wdir
-        self.name = name or uuid()
-        self.returncode: Optional[int] = None
-        self._stdout: Optional[BinaryIO] = None
-        self._stderr: Optional[BinaryIO] = None
-
-        self._run()
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        self.wait()
-
-    def _close_fds(self):
-        if self._stdout:
-            self._stdout.close()
-            self._stdout = None
-        if self._stderr:
-            self._stderr.close()
-            self._stderr = None
-
-    def _make_path(self, path: str) -> str:
-        return os.path.join(self.wdir, path) if self.wdir else path
-
-    @cached_property
-    def stdout_path(self) -> str:
-        return self._make_path(f"{self.name}.out")
-
-    @cached_property
-    def info_path(self) -> str:
-        return self._make_path(f"{self.name}.json")
-
-    @cached_property
-    def pidfile_path(self) -> str:
-        return self._make_path(f"{self.name}.pid")
-
-    @property
-    def info(self) -> "ProcessInfo":
-        return ProcessInfo(
-            pid=self.pid,
-            stdin=None,
-            stdout=self.stdout_path,
-            stderr=None,
-            returncode=self.returncode,
-        )
-
-    def _make_wdir(self):
-        if self.wdir:
-            os.makedirs(self.wdir, exist_ok=True)
-
-    def _dump(self):
-        self._make_wdir()
-        with open(self.info_path, "w", encoding="utf-8") as fobj:
-            json.dump(self.info.asdict(), fobj)
-        with open(self.pidfile_path, "w", encoding="utf-8") as fobj:
-            fobj.write(str(self.pid))
-
-    def _run(self) -> None:
-        self._make_wdir()
-        logger.debug(
-            "Appending output to '%s'",
-            self.stdout_path,
-        )
-        self._stdout = open(self.stdout_path, "ab")  # noqa: SIM115
-        try:
-            self._proc = subprocess.Popen(
-                self.args,
-                stdin=subprocess.DEVNULL,
-                stdout=self._stdout,
-                stderr=subprocess.STDOUT,
-                close_fds=True,
-                shell=False,
-            )
-            self.pid: int = self._proc.pid
-            self._dump()
-        except Exception:
-            if self._proc is not None:
-                self._proc.kill()
-            self._close_fds()
-            raise
-
-    def wait(self, timeout: Optional[int] = None) -> Optional[int]:
-        """Block until a process started with `run` has completed.
-
-        Raises:
-            TimeoutExpired if `timeout` was set and the process
-            did not terminate after `timeout` seconds.
-        """
-        if self.returncode is not None:
-            return self.returncode
-        try:
-            self._proc.wait(timeout=timeout)
-        except subprocess.TimeoutExpired as exc:
-            raise TimeoutExpired(exc.cmd, exc.timeout) from exc
-        self.returncode = self._proc.returncode
-        self._close_fds()
-        self._dump()
-        return self.returncode
-
-    @classmethod
-    def spawn(cls, *args, **kwargs) -> Optional[int]:
-        """Spawn a ManagedProcess command in the background.
-
-        Returns: The spawned process PID.
-        """
-        import multiprocessing as mp
-
-        proc = mp.Process(
-            target=cls._spawn,
-            args=args,
-            kwargs=kwargs,
-            daemon=True,
-        )
-        proc.start()
-        # Do not terminate the child daemon when the main process exits
-        # pylint: disable=protected-access
-        mp.process._children.discard(proc)  # type: ignore[attr-defined]
-        return proc.pid
-
-    @classmethod
-    def _spawn(cls, *args, **kwargs):
-        with cls(*args, **kwargs):
-            pass

dvc/progress.py

@@ -36,7 +36,7 @@ class Tqdm(tqdm):
         "miniters": 1,
     }
 
-    def __init__(
+    def __init__(  # noqa: PLR0913
         self,
         iterable=None,
         disable=None,

dvc/render/convert.py

@@ -1,4 +1,3 @@
-import json
 from collections import defaultdict
 from typing import Dict, List, Union
 
@@ -29,20 +28,20 @@ def _group_by_rev(datapoints):
 
 
 def to_json(renderer, split: bool = False) -> List[Dict]:
-    from copy import deepcopy
-
     if renderer.TYPE == "vega":
-        grouped = _group_by_rev(deepcopy(renderer.datapoints))
+        grouped = _group_by_rev(renderer.datapoints)
         if split:
-            content = renderer.get_filled_template(skip_anchors=["data"])
+            content = renderer.get_filled_template(
+                skip_anchors=["data"], as_string=False
+            )
         else:
-            content = renderer.get_filled_template()
+            content = renderer.get_filled_template(as_string=False)
         if grouped:
             return [
                 {
                     TYPE_KEY: renderer.TYPE,
                     REVISIONS_KEY: sorted(grouped.keys()),
-                    "content": json.loads(content),
+                    "content": content,
                     "datapoints": grouped,
                 }
             ]

dvc/render/converter/vega.py

@@ -1,4 +1,4 @@
-from copy import deepcopy
+import os
 from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
 
 from funcy import first, last
@@ -56,7 +56,6 @@ def _verify_field(file2datapoints: Dict[str, List], filename: str, field: str):
         datapoint = first(file2datapoints[filename])
         if field not in datapoint:
             raise FieldNotFoundError(field, datapoint.keys())
-    return
 
 
 def _get_xs(properties: Dict, file2datapoints: Dict[str, List[Dict]]):
@@ -194,7 +193,7 @@ class VegaConverter(Converter):
             return first(fields)
         return "x"
 
-    def flat_datapoints(self, revision):  # noqa: C901
+    def flat_datapoints(self, revision):  # noqa: C901, PLR0912
         file2datapoints, properties = self.convert()
 
         props_update = {}
@@ -222,7 +221,13 @@ class VegaConverter(Converter):
             )
 
         all_datapoints = []
-        all_y_fields = {y_field for _, y_field in ys}
+        if ys:
+            all_y_files, all_y_fields = list(zip(*ys))
+            all_y_fields = set(all_y_fields)
+            all_y_files = set(all_y_files)
+        else:
+            all_y_files = set()
+            all_y_fields = set()
 
         # override to unified y field name if there are different y fields
         if len(all_y_fields) > 1:
@@ -230,10 +235,16 @@ class VegaConverter(Converter):
         else:
             props_update["y"] = first(all_y_fields)
 
+        # get common prefix to drop from file names
+        if len(all_y_files) > 1:
+            common_prefix_len = len(os.path.commonpath(all_y_files))
+        else:
+            common_prefix_len = 0
+
         for i, (y_file, y_field) in enumerate(ys):
             if num_xs > 1:
                 x_file, x_field = xs[i]
-            datapoints = deepcopy(file2datapoints.get(y_file, []))
+            datapoints = [{**d} for d in file2datapoints.get(y_file, [])]
 
             if props_update.get("y", None) == "dvc_inferred_y_value":
                 _update_from_field(
@@ -259,12 +270,13 @@ class VegaConverter(Converter):
                         "They have to have same length."
                     )
 
+            y_file_short = y_file[common_prefix_len:].strip("/\\")
             _update_all(
                 datapoints,
                 update_dict={
                     VERSION_FIELD: {
                         "revision": revision,
-                        FILENAME_FIELD: y_file,
+                        FILENAME_FIELD: y_file_short,
                         "field": y_field,
                     }
                 },

dvc/render/match.py

@@ -1,17 +1,20 @@
+import os
 from collections import defaultdict
-from typing import TYPE_CHECKING, Dict, List, Optional
+from typing import TYPE_CHECKING, DefaultDict, Dict, List, NamedTuple, Optional
 
+import dpath
 import dpath.options
-import dpath.util
-from funcy import last
+from funcy import get_in, last
 
-from dvc.repo.plots import infer_data_sources
+from dvc.repo.plots import _normpath, infer_data_sources
 from dvc.utils.plots import get_plot_id
 
 from .convert import _get_converter
 
 if TYPE_CHECKING:
     from dvc.types import StrPath
+    from dvc_render.base import Renderer
+
 
 dpath.options.ALLOW_EMPTY_STRING_KEYS = True
 
@@ -43,35 +46,48 @@ class PlotsData:
 
     def get_definition_data(self, target_files, rev):
         result = {}
-        for file in target_files:
+        for definition_file in target_files:
+            if os.name == "nt":
+                source_file = _normpath(definition_file).replace("\\", "/")
+            else:
+                source_file = definition_file
             file_content = (
                 self.data.get(rev, {})
                 .get("sources", {})
                 .get("data", {})
-                .get(file, {})
+                .get(source_file, {})
                 .get("data", {})
             )
             if file_content:
-                result[file] = file_content
-
+                result[definition_file] = file_content
         return result
 
 
-def match_defs_renderers(
+class RendererWithErrors(NamedTuple):
+    renderer: "Renderer"
+    source_errors: Dict[str, Dict[str, Exception]]
+    definition_errors: Dict[str, Exception]
+
+
+def match_defs_renderers(  # noqa: C901, PLR0912
     data,
     out=None,
     templates_dir: Optional["StrPath"] = None,
-):
+) -> List[RendererWithErrors]:
     from dvc_render import ImageRenderer, VegaRenderer
 
     plots_data = PlotsData(data)
     renderers = []
     renderer_cls = None
+
     for plot_id, group in plots_data.group_definitions().items():
         plot_datapoints: List[Dict] = []
         props = _squash_plots_properties(group)
         final_props: Dict = {}
 
+        def_errors: Dict[str, Exception] = {}
+        src_errors: DefaultDict[str, Dict[str, Exception]] = defaultdict(dict)
+
         if out is not None:
             props["out"] = out
         if templates_dir is not None:
@@ -90,13 +106,24 @@ def match_defs_renderers(
 
             converter = _get_converter(renderer_cls, inner_id, props, definitions_data)
 
-            dps, rev_props = converter.flat_datapoints(rev)
+            for src in plot_sources:
+                if error := get_in(data, [rev, "sources", "data", src, "error"]):
+                    src_errors[rev][src] = error
+
+            try:
+                dps, rev_props = converter.flat_datapoints(rev)
+            except Exception as e:  # noqa: BLE001, pylint: disable=broad-except
+                def_errors[rev] = e
+                continue
+
             if not final_props and rev_props:
                 final_props = rev_props
             plot_datapoints.extend(dps)
 
         if "title" not in final_props:
             final_props["title"] = renderer_id
+
         if renderer_cls is not None:
-            renderers.append(renderer_cls(plot_datapoints, renderer_id, **final_props))
+            renderer = renderer_cls(plot_datapoints, renderer_id, **final_props)
+            renderers.append(RendererWithErrors(renderer, dict(src_errors), def_errors))
     return renderers

dvc/repo/__init__.py

@@ -3,10 +3,17 @@ import os
 from collections import defaultdict
 from contextlib import contextmanager
 from functools import wraps
-from typing import TYPE_CHECKING, Callable, Iterable, List, Optional, Tuple, Union, cast
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    ContextManager,
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+    Union,
+)
 
-from dvc.exceptions import FileMissingError
-from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
 from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
 from dvc.ignore import DvcIgnoreFilter
 from dvc.utils import env2bool
@@ -14,7 +21,7 @@ from dvc.utils.fs import path_isin
 from dvc.utils.objects import cached_property
 
 if TYPE_CHECKING:
-    from dvc.fs import FileSystem, GitFileSystem
+    from dvc.fs import FileSystem
     from dvc.fs.data import DataFileSystem
     from dvc.fs.dvc import DVCFileSystem
     from dvc.lock import LockBase
@@ -23,6 +30,7 @@ if TYPE_CHECKING:
     from dvc.stage import Stage
     from dvc.types import DictStrAny
     from dvc_data.hashfile.state import StateBase
+    from dvc_data.index import DataIndex
 
     from .experiments import Experiments
     from .index import Index
@@ -99,17 +107,15 @@ class Repo:
         fs: Optional["FileSystem"] = None,
         uninitialized: bool = False,
         scm: Optional[Union["Git", "NoSCM"]] = None,
-    ) -> Tuple[str, Optional[str], Optional[str]]:
+    ) -> Tuple[str, Optional[str]]:
         from dvc.fs import localfs
         from dvc.scm import SCM, SCMError
 
         dvc_dir: Optional[str] = None
-        tmp_dir: Optional[str] = None
         try:
             root_dir = self.find_root(root_dir, fs)
             fs = fs or localfs
             dvc_dir = fs.path.join(root_dir, self.DVC_DIR)
-            tmp_dir = fs.path.join(dvc_dir, "tmp")
         except NotDvcRepoError:
             if not uninitialized:
                 raise
@@ -117,6 +123,8 @@ class Repo:
             if not scm:
                 try:
                     scm = SCM(root_dir or os.curdir)
+                    if scm.dulwich.repo.bare:
+                        raise NotDvcRepoError(f"{scm.root_dir} is a bare git repo")
                 except SCMError:
                     scm = SCM(os.curdir, no_scm=True)
 
@@ -124,33 +132,9 @@ class Repo:
                 root_dir = scm.root_dir
 
         assert root_dir
-        return root_dir, dvc_dir, tmp_dir
+        return root_dir, dvc_dir
 
-    def _get_database_dir(self, db_name: str) -> Optional[str]:
-        # NOTE: by default, store SQLite-based remote indexes and state's
-        # `links` and `md5s` caches in the repository itself to avoid any
-        # possible state corruption in 'shared cache dir' scenario, but allow
-        # user to override this through config when, say, the repository is
-        # located on a mounted volume — see
-        # https://github.com/iterative/dvc/issues/4420
-        base_db_dir = self.config.get(db_name, {}).get("dir", None)
-        if not base_db_dir:
-            return self.tmp_dir
-
-        import hashlib
-
-        root_dir_hash = hashlib.sha224(self.root_dir.encode("utf-8")).hexdigest()
-
-        db_dir = self.fs.path.join(
-            base_db_dir,
-            self.DVC_DIR,
-            f"{self.fs.path.name(self.root_dir)}-{root_dir_hash[0:7]}",
-        )
-
-        self.fs.makedirs(db_dir, exist_ok=True)
-        return db_dir
-
-    def __init__(
+    def __init__(  # noqa: PLR0915
         self,
         root_dir: Optional[str] = None,
         fs: Optional["FileSystem"] = None,
@@ -167,6 +151,7 @@ class Repo:
         from dvc.data_cloud import DataCloud
         from dvc.fs import GitFileSystem, LocalFileSystem, localfs
         from dvc.lock import LockNoop, make_lock
+        from dvc.repo.artifacts import Artifacts
         from dvc.repo.metrics import Metrics
         from dvc.repo.params import Params
         from dvc.repo.plots import Plots
@@ -179,6 +164,7 @@ class Repo:
         self._fs_conf = {"repo_factory": repo_factory}
         self._fs = fs or localfs
         self._scm = scm
+        self._data_index = None
 
         if rev and not fs:
             self._scm = scm = SCM(root_dir or os.curdir)
@@ -187,8 +173,10 @@ class Repo:
 
         self.root_dir: str
         self.dvc_dir: Optional[str]
-        self.tmp_dir: Optional[str]
-        self.root_dir, self.dvc_dir, self.tmp_dir = self._get_repo_dirs(
+        (
+            self.root_dir,
+            self.dvc_dir,
+        ) = self._get_repo_dirs(
             root_dir=root_dir,
             fs=self.fs,
             uninitialized=uninitialized,
@@ -211,19 +199,19 @@ class Repo:
             self.lock = LockNoop()
             self.state = StateNoop()
             self.cache = CacheManager(self)
-            self.tmp_dir = None
         else:
-            self.fs.makedirs(cast(str, self.tmp_dir), exist_ok=True)
-
             if isinstance(self.fs, LocalFileSystem):
+                assert self.tmp_dir
+                self.fs.makedirs(self.tmp_dir, exist_ok=True)
+
                 self.lock = make_lock(
                     self.fs.path.join(self.tmp_dir, "lock"),
                     tmp_dir=self.tmp_dir,
                     hardlink_lock=self.config["core"].get("hardlink_lock", False),
                     friendly=True,
                 )
-                state_db_dir = self._get_database_dir("state")
-                self.state = State(self.root_dir, state_db_dir, self.dvcignore)
+                os.makedirs(self.site_cache_dir, exist_ok=True)
+                self.state = State(self.root_dir, self.site_cache_dir, self.dvcignore)
             else:
                 self.lock = LockNoop()
                 self.state = StateNoop()
@@ -237,6 +225,7 @@ class Repo:
         self.metrics: Metrics = Metrics(self)
         self.plots: Plots = Plots(self)
         self.params: Params = Params(self)
+        self.artifacts: Artifacts = Artifacts(self)
 
         self.stage_collection_error_handler: Optional[
             Callable[[str, Exception], None]
@@ -246,6 +235,41 @@ class Repo:
     def __str__(self):
         return self.url or self.root_dir
 
+    @cached_property
+    def local_dvc_dir(self):
+        from dvc.fs import GitFileSystem, LocalFileSystem
+
+        if not self.dvc_dir:
+            return None
+
+        if isinstance(self.fs, LocalFileSystem):
+            return self.dvc_dir
+
+        if not isinstance(self.fs, GitFileSystem):
+            return None
+
+        relparts = ()
+        if self.root_dir != "/":
+            # subrepo
+            relparts = self.fs.path.relparts(self.root_dir, "/")
+
+        dvc_dir = os.path.join(
+            self.scm.root_dir,
+            *relparts,
+            self.DVC_DIR,
+        )
+        if os.path.exists(dvc_dir):
+            return dvc_dir
+
+        return None
+
+    @cached_property
+    def tmp_dir(self):
+        if self.local_dvc_dir is None:
+            return None
+
+        return os.path.join(self.local_dvc_dir, "tmp")
+
     @cached_property
     def index(self) -> "Index":
         from dvc.repo.index import Index
@@ -304,7 +328,7 @@ class Repo:
         return DvcIgnoreFilter(self.fs, self.root_dir)
 
     def get_rev(self):
-        from dvc.fs import LocalFileSystem
+        from dvc.fs import GitFileSystem, LocalFileSystem
 
         assert self.scm
         if isinstance(self.fs, LocalFileSystem):
@@ -312,7 +336,8 @@ class Repo:
 
             with map_scm_exception():
                 return self.scm.get_rev()
-        return cast("GitFileSystem", self.fs).rev
+        assert isinstance(self.fs, GitFileSystem)
+        return self.fs.rev
 
     @cached_property
     def experiments(self) -> "Experiments":
@@ -341,6 +366,17 @@ class Repo:
         # fs.
         self._reset()
 
+    @property
+    def data_index(self) -> "DataIndex":
+        from dvc_data.index import DataIndex
+
+        if self._data_index is None:
+            index_dir = os.path.join(self.site_cache_dir, "index", "data")
+            os.makedirs(index_dir, exist_ok=True)
+            self._data_index = DataIndex.open(os.path.join(index_dir, "db.db"))
+
+        return self._data_index
+
     def __repr__(self):
         return f"{self.__class__.__name__}: '{self.root_dir}'"
 
@@ -374,9 +410,12 @@ class Repo:
         raise NotDvcRepoError(msg)
 
     @classmethod
-    def find_dvc_dir(cls, root=None) -> str:
-        root_dir = cls.find_root(root)
-        return os.path.join(root_dir, cls.DVC_DIR)
+    def find_dvc_dir(cls, root=None, fs=None) -> str:
+        from dvc.fs import localfs
+
+        fs = fs or localfs
+        root_dir = cls.find_root(root, fs=fs)
+        return fs.path.join(root_dir, cls.DVC_DIR)
 
     @staticmethod
     def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False) -> "Repo":
@@ -402,7 +441,12 @@ class Repo:
 
         return brancher(self, *args, **kwargs)
 
-    def used_objs(
+    def switch(self, rev: str) -> ContextManager[str]:
+        from dvc.repo.brancher import switch
+
+        return switch(self, rev)
+
+    def used_objs(  # noqa: PLR0913
         self,
         targets=None,
         all_branches=False,
@@ -515,59 +559,56 @@ class Repo:
         return DVCFileSystem(repo=self, subrepos=self.subrepos, **self._fs_conf)
 
     @cached_property
-    def index_db_dir(self):
-        return self._get_database_dir("index")
+    def site_cache_dir(self) -> str:
+        import getpass
+        import hashlib
 
-    @contextmanager
-    def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
-        """Opens a specified resource as a file descriptor"""
-        from dvc.fs.data import DataFileSystem
-        from dvc.fs.dvc import DVCFileSystem
+        import platformdirs
 
-        fs: Union["FileSystem", DataFileSystem, DVCFileSystem]
-        if os.path.isabs(path):
-            fs = DataFileSystem(index=self.index.data["local"])
-            fs_path = path
+        from dvc.fs import GitFileSystem
+
+        default = platformdirs.site_cache_dir("dvc", "iterative", opinion=True)
+        cache_dir = self.config["core"].get("site_cache_dir") or default
+
+        if isinstance(self.fs, GitFileSystem):
+            relparts = ()
+            if self.root_dir != "/":
+                # subrepo
+                relparts = self.fs.path.relparts(self.root_dir, "/")
+            root_dir = os.path.join(self.scm.root_dir, *relparts)
         else:
-            fs = DVCFileSystem(repo=self, subrepos=True)
-            fs_path = fs.from_os_path(path)
+            root_dir = self.root_dir
 
+        repos_dir = os.path.join(cache_dir, "repo")
+
+        umask = os.umask(0)
         try:
-            if remote:
-                remote_odb = self.cloud.get_remote_odb(name=remote)
-                oid = fs.info(fs_path)["dvc_info"]["md5"]
-                fs = remote_odb.fs
-                fs_path = remote_odb.oid_to_path(oid)
+            os.makedirs(repos_dir, mode=0o777, exist_ok=True)
+        finally:
+            os.umask(umask)
 
-            with fs.open(
-                fs_path,
-                mode=mode,
-                encoding=encoding,
-            ) as fobj:
-                yield fobj
-        except FileNotFoundError as exc:
-            raise FileMissingError(path) from exc
-        except IsADirectoryError as exc:
-            raise DvcIsADirectoryError(f"'{path}' is a directory") from exc
+        md5 = hashlib.md5(  # noqa: S324  # nosec B324, B303
+            str((root_dir, getpass.getuser())).encode()
+        )
+        repo_token = md5.hexdigest()
+        return os.path.join(repos_dir, repo_token)
 
     def close(self):
         self.scm.close()
         self.state.close()
+        if self._data_index is not None:
+            self._data_index.close()
 
-    def _reset_cached_indecies(self):
+    def _reset(self):
+        self.scm._reset()  # pylint: disable=protected-access
+        self.state.close()
         self.__dict__.pop("index", None)
         self.__dict__.pop("dvcignore", None)
         self.__dict__.pop("dvcfs", None)
         self.__dict__.pop("datafs", None)
 
-    def _reset(self):
-        self.state.close()
-        self.scm._reset()  # pylint: disable=protected-access
-        self._reset_cached_indecies()
-
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        self._reset()
-        self.scm.close()
+        self.close()

dvc/repo/add.py

@@ -148,7 +148,7 @@ VALIDATORS = (
 @validate(*VALIDATORS)
 @locked
 @scm_context
-def add(  # noqa: C901
+def add(
     repo: "Repo",
     targets: "TargetType",
     recursive: bool = False,

dvc/repo/artifacts.py

@@ -0,0 +1,67 @@
+import logging
+import os
+import re
+from typing import TYPE_CHECKING, Dict
+
+from dvc.annotations import Artifact
+from dvc.dvcfile import FileMixin
+from dvc.utils import relpath
+
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+
+logger = logging.getLogger(__name__)
+
+
+NAME_RE = re.compile(r"^[a-z]([a-z0-9-]*[a-z0-9])?$")
+
+
+def name_is_compatible(name: str) -> bool:
+    return bool(NAME_RE.search(name))
+
+
+def check_name_format(name: str) -> None:
+    if not name_is_compatible(name):
+        logger.warning(
+            "Can't use '%s' as artifact name (ID)."
+            " You can use letters and numbers, and use '-' as separator"
+            " (but not at the start or end). The first character must be a letter.",
+            name,
+        )
+
+
+class ArtifactsFile(FileMixin):
+    from dvc.schema import SINGLE_ARTIFACT_SCHEMA as SCHEMA
+
+    def dump(self, stage, **kwargs):
+        raise NotImplementedError
+
+    def merge(self, ancestor, other, allowed=None):
+        raise NotImplementedError
+
+
+class Artifacts:
+    def __init__(self, repo: "Repo") -> None:
+        self.repo = repo
+
+    def read(self) -> Dict[str, Dict[str, Artifact]]:
+        artifacts: Dict[str, Dict[str, Artifact]] = {}
+        for (
+            dvcfile,
+            dvcfile_artifacts,
+        ) in self.repo.index._artifacts.items():  # pylint: disable=protected-access
+            # read the artifacts.yaml file if needed
+            if isinstance(dvcfile_artifacts, str):
+                dvcfile_artifacts = ArtifactsFile(
+                    self.repo,
+                    os.path.join(os.path.dirname(dvcfile), dvcfile_artifacts),
+                    verify=False,
+                ).load()
+            if not dvcfile_artifacts:
+                continue
+            dvcyaml = relpath(dvcfile, self.repo.root_dir)
+            artifacts[dvcyaml] = {}
+            for name, value in dvcfile_artifacts.items():
+                check_name_format(name)
+                artifacts[dvcyaml][name] = Artifact(**value)
+        return artifacts

dvc/repo/brancher.py

@@ -1,7 +1,17 @@
-from typing import Optional
+import logging
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Iterator, Optional, Tuple
 
+from scmrepo.git import Git
+
+from dvc.exceptions import NotDvcRepoError
 from dvc.scm import iter_revs
 
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+
+logger = logging.getLogger(__name__)
+
 
 def brancher(  # noqa: E302
     self,
@@ -60,6 +70,7 @@ def brancher(  # noqa: E302
 
     scm = self.scm
 
+    logger.trace("switching fs to workspace")  # type: ignore[attr-defined]
     self.fs = LocalFileSystem(url=self.root_dir)
     yield "workspace"
 
@@ -79,21 +90,68 @@ def brancher(  # noqa: E302
     )
 
     try:
-        from dvc.fs import GitFileSystem
-
         for sha, names in found_revs.items():
-            self._reset_cached_indecies()  # pylint: disable=W0212
-            self.fs = GitFileSystem(scm=scm, rev=sha)
-            self.root_dir = self.fs.path.join("/", *repo_root_parts)
-
-            if cwd_parts:
-                cwd = self.fs.path.join("/", *cwd_parts)  # type: ignore[unreachable]
-                self.fs.path.chdir(cwd)
-
-            # ignore revs that don't contain repo root
-            # (i.e. revs from before a subdir=True repo was init'ed)
-            if self.fs.exists(self.root_dir):
+            try:
+                _switch_fs(self, sha, repo_root_parts, cwd_parts)
                 yield sha if sha_only else ",".join(names)
+            except NotDvcRepoError:
+                # ignore revs that don't contain repo root
+                # (i.e. revs from before a subdir=True repo was init'ed)
+                pass
     finally:
         self.fs = saved_fs
         self.root_dir = saved_root
+
+
+def _switch_fs(
+    repo: "Repo",
+    rev: str,
+    repo_root_parts: Tuple[str, ...],
+    cwd_parts: Tuple[str, ...],
+):
+    from dvc.fs import GitFileSystem, LocalFileSystem
+
+    if rev == "workspace":
+        logger.trace("switching fs to workspace")  # type: ignore[attr-defined]
+        repo.fs = LocalFileSystem(url=repo.root_dir)
+        return
+
+    logger.trace("switching fs to revision %s", rev[:7])  # type: ignore[attr-defined]
+    assert isinstance(repo.scm, Git)
+    fs = GitFileSystem(scm=repo.scm, rev=rev)
+    root_dir = repo.fs.path.join("/", *repo_root_parts)
+    if not fs.exists(root_dir):
+        raise NotDvcRepoError(f"Commit '{rev[:7]}' does not contain a DVC repo")
+
+    repo.fs = fs
+    repo.root_dir = root_dir
+
+    if cwd_parts:
+        cwd = repo.fs.path.join("/", *cwd_parts)
+        repo.fs.path.chdir(cwd)
+
+
+@contextmanager
+def switch(repo: "Repo", rev: str) -> Iterator[str]:
+    """Switch to a specific revision."""
+    from dvc.scm import resolve_rev
+
+    if rev != "workspace":
+        rev = resolve_rev(repo.scm, rev)
+
+    repo_root_parts = ()
+    if repo.fs.path.isin(repo.root_dir, repo.scm.root_dir):
+        repo_root_parts = repo.fs.path.relparts(repo.root_dir, repo.scm.root_dir)
+
+    cwd_parts = ()
+    if repo.fs.path.isin(repo.fs.path.getcwd(), repo.scm.root_dir):
+        cwd_parts = repo.fs.path.relparts(repo.fs.path.getcwd(), repo.scm.root_dir)
+
+    saved_fs = repo.fs
+    saved_root = repo.root_dir
+    try:
+        _switch_fs(repo, rev, repo_root_parts, cwd_parts)
+        yield rev
+    finally:
+        repo.fs = saved_fs
+        repo.root_dir = saved_root

dvc/repo/commit.py

@@ -43,6 +43,7 @@ def commit(
     force=False,
     allow_missing=False,
     data_only=False,
+    relink=True,
 ):
     stages_info = [
         info
@@ -60,6 +61,10 @@ def commit(
             if any(changes):
                 prompt_to_commit(stage, changes, force=force)
                 stage.save(allow_missing=allow_missing)
-        stage.commit(filter_info=stage_info.filter_info, allow_missing=allow_missing)
+        stage.commit(
+            filter_info=stage_info.filter_info,
+            allow_missing=allow_missing,
+            relink=relink,
+        )
         stage.dump(update_pipeline=False)
     return [s.stage for s in stages_info]

dvc/repo/data.py

@@ -13,17 +13,20 @@ from typing import (
     Union,
 )
 
+from dvc.ui import ui
+
 if TYPE_CHECKING:
     from dvc.repo import Repo
     from dvc.scm import Git, NoSCM
     from dvc_data.index import DataIndex
+    from dvc_data.index.diff import Change
 
 
 def posixpath_to_os_path(path: str) -> str:
     return path.replace(posixpath.sep, os.path.sep)
 
 
-def _adapt_typ(typ):
+def _adapt_typ(typ: str) -> str:
     from dvc_data.index.diff import ADD, DELETE, MODIFY
 
     if typ == MODIFY:
@@ -38,7 +41,7 @@ def _adapt_typ(typ):
     return typ
 
 
-def _adapt_path(change):
+def _adapt_path(change: "Change") -> str:
     isdir = False
     if change.new and change.new.meta:
         isdir = change.new.meta.isdir
@@ -54,9 +57,12 @@ def _diff(
     old: "DataIndex",
     new: "DataIndex",
     *,
-    granular: Optional[bool] = False,
-    with_missing: Optional[bool] = False,
+    granular: bool = False,
+    not_in_cache: bool = False,
+    not_in_remote: bool = False,
+    remote_refresh: bool = False,
 ) -> Dict[str, List[str]]:
+    from dvc_data.index import StorageError
     from dvc_data.index.diff import UNCHANGED, UNKNOWN, diff
 
     ret: Dict[str, List[str]] = {}
@@ -91,14 +97,27 @@ def _diff(
             continue
 
         if (
-            with_missing
+            not_in_cache
             and change.old
             and change.old.hash_info
-            and not old.storage_map[change.key].cache.exists(change.old.hash_info.value)
+            and not old.storage_map.cache_exists(change.old)
         ):
             # NOTE: emulating previous behaviour
             _add_change("not_in_cache", change)
 
+        try:
+            if (
+                not_in_remote
+                and change.old
+                and change.old.hash_info
+                and not old.storage_map.remote_exists(
+                    change.old, refresh=remote_refresh
+                )
+            ):
+                _add_change("not_in_remote", change)
+        except StorageError:
+            pass
+
         _add_change(change.typ, change)
 
     return ret
@@ -143,14 +162,19 @@ def _git_info(scm: Union["Git", "NoSCM"], untracked_files: str = "all") -> GitIn
 def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]:
     from .index import build_data_index
 
-    workspace = build_data_index(repo.index, repo.root_dir, repo.fs, compute_hash=True)
+    with ui.status("Building workspace index"):
+        workspace = build_data_index(
+            repo.index, repo.root_dir, repo.fs, compute_hash=True
+        )
 
-    return _diff(
-        repo.index.data["repo"],
-        workspace,
-        with_missing=True,
-        **kwargs,
-    )
+    with ui.status("Calculating diff between index/workspace"):
+        return _diff(
+            repo.index.data["repo"],
+            workspace,
+            not_in_cache=True,
+            not_in_remote=True,
+            **kwargs,
+        )
 
 
 def _diff_head_to_index(
@@ -158,17 +182,16 @@ def _diff_head_to_index(
 ) -> Dict[str, List[str]]:
     index = repo.index.data["repo"]
 
-    for rev in repo.brancher(revs=[head]):
-        if rev == "workspace":
-            continue
-
+    with repo.switch(head):
         head_index = repo.index.data["repo"]
 
-    return _diff(head_index, index, **kwargs)
+    with ui.status("Calculating diff between head/index"):
+        return _diff(head_index, index, **kwargs)
 
 
 class Status(TypedDict):
     not_in_cache: List[str]
+    not_in_remote: List[str]
     committed: Dict[str, List[str]]
     uncommitted: Dict[str, List[str]]
     untracked: List[str]
@@ -195,12 +218,18 @@ def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> List[str]
     return [repo.fs.path.relpath(file, start) for file in files]
 
 
-def status(repo: "Repo", untracked_files: str = "no", **kwargs: Any) -> Status:
+def status(
+    repo: "Repo",
+    untracked_files: str = "no",
+    **kwargs: Any,
+) -> Status:
     from dvc.scm import NoSCMError, SCMError
 
     head = kwargs.pop("head", "HEAD")
-    uncommitted_diff = _diff_index_to_wtree(repo, **kwargs)
-    not_in_cache = uncommitted_diff.pop("not_in_cache", [])
+    uncommitted_diff = _diff_index_to_wtree(
+        repo,
+        **kwargs,
+    )
     unchanged = set(uncommitted_diff.pop("unchanged", []))
 
     try:
@@ -215,7 +244,8 @@ def status(repo: "Repo", untracked_files: str = "no", **kwargs: Any) -> Status:
     untracked = _transform_git_paths_to_dvc(repo, untracked)
     # order matters here
     return Status(
-        not_in_cache=not_in_cache,
+        not_in_cache=uncommitted_diff.pop("not_in_cache", []),
+        not_in_remote=uncommitted_diff.pop("not_in_remote", []),
         committed=committed_diff,
         uncommitted=uncommitted_diff,
         untracked=untracked,

dvc/repo/diff.py

@@ -1,39 +1,103 @@
+import errno
 import logging
 import os
 from collections import defaultdict
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Dict,
-    Iterable,
-    Iterator,
-    List,
-    Optional,
-    Set,
-    Tuple,
-    cast,
-)
+from typing import Dict, List, Optional
 
-from dvc.exceptions import PathMissingError
 from dvc.repo import locked
-from dvc.utils.collections import ensure_list
-
-if TYPE_CHECKING:
-    from dvc.fs import DVCFileSystem, FileSystem
-    from dvc.output import Output
-    from dvc.repo import Repo
-    from dvc_data.hashfile.obj import HashFile
-    from dvc_data.hashfile.tree import Tree
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
 
+def _path(entry):
+    if entry and entry.meta and entry.meta.isdir:
+        return os.path.join(*entry.key, "")
+    return os.path.join(*entry.key)
+
+
+def _hash(entry):
+    if entry and entry.hash_info:
+        return entry.hash_info.value
+    return None
+
+
+def _diff(old, new, with_missing=False):
+    from dvc_data.index.diff import ADD, DELETE, MODIFY, RENAME
+    from dvc_data.index.diff import diff as idiff
+
+    ret: "Dict[str, List[Dict]]" = {
+        "added": [],
+        "deleted": [],
+        "modified": [],
+        "renamed": [],
+        "not in cache": [],
+    }
+
+    for change in idiff(
+        old,
+        new,
+        with_renames=True,
+        hash_only=True,
+    ):
+        if change.typ == ADD:
+            ret["added"].append(
+                {
+                    "path": _path(change.new),
+                    "hash": _hash(change.new),
+                }
+            )
+        elif change.typ == DELETE:
+            ret["deleted"].append(
+                {
+                    "path": _path(change.old),
+                    "hash": _hash(change.old),
+                }
+            )
+        elif change.typ == MODIFY:
+            ret["modified"].append(
+                {
+                    "path": _path(change.old),
+                    "hash": {
+                        "old": _hash(change.old),
+                        "new": _hash(change.new),
+                    },
+                }
+            )
+        elif change.typ == RENAME:
+            ret["renamed"].append(
+                {
+                    "path": {
+                        "old": _path(change.old),
+                        "new": _path(change.new),
+                    },
+                    "hash": _hash(change.old),
+                }
+            )
+
+        if (
+            with_missing
+            and change.old
+            and change.old.hash_info
+            and not old.storage_map.cache_exists(change.old)
+        ):
+            ret["not in cache"].append(
+                {
+                    "path": _path(change.old),
+                    "hash": _hash(change.old),
+                }
+            )
+
+    return ret if any(ret.values()) else {}
+
+
 @locked
 def diff(
     self,
     a_rev: str = "HEAD",
     b_rev: Optional[str] = None,
     targets: Optional[List[str]] = None,
+    recursive: bool = False,
 ):
     """
     By default, it compares the workspace with the last commit's fs.
@@ -42,231 +106,58 @@ def diff(
     the concept of `index`, but it keeps the same interface, thus,
     `dvc diff` would be the same as `dvc diff HEAD`.
     """
-
     if self.scm.no_commits:
         return {}
 
-    from dvc.fs.dvc import DVCFileSystem
+    indexes = {}
+    missing_targets = defaultdict(set)
+    with_missing = False
+    if not b_rev:
+        b_rev = "workspace"
+        with_missing = True
 
-    dvcfs = DVCFileSystem(repo=self)
-    targets = [dvcfs.from_os_path(target) for target in ensure_list(targets)]
-
-    b_rev = b_rev if b_rev else "workspace"
-    results = {}
-    missing_targets = {}
     for rev in self.brancher(revs=[a_rev, b_rev]):
-        if rev == "workspace" and rev != b_rev:
+        if rev == "workspace" and b_rev != "workspace":
             # brancher always returns workspace, but we only need to compute
             # workspace paths/checksums if b_rev was None
             continue
 
-        targets_paths = None
-        if targets:
-            # convert targets to paths, and capture any missing targets
-            targets_paths, missing_targets[rev] = _targets_to_paths(dvcfs, targets)
+        def onerror(target, _exc):
+            # pylint: disable-next=cell-var-from-loop
+            missing_targets[rev].add(target)  # noqa: B023
 
-        results[rev] = _paths_checksums(self, targets_paths)
-
-    if targets:
-        # check for overlapping missing targets between a_rev and b_rev
-        for target in set(missing_targets[a_rev]) & set(missing_targets[b_rev]):
-            raise PathMissingError(target, self)
-
-    old = results[a_rev]
-    new = results[b_rev]
-
-    # Compare paths between the old and new fs.
-    # set() efficiently converts dict keys to a set
-    added = sorted(set(new) - set(old))
-    deleted_or_missing = set(old) - set(new)
-    if b_rev == "workspace":
-        # missing status is only applicable when diffing local workspace
-        # against a commit
-        missing = sorted(_filter_missing(dvcfs, deleted_or_missing))
-    else:
-        missing = []
-    deleted = sorted(deleted_or_missing - set(missing))
-    modified = sorted(set(old) & set(new))
-
-    # Cases when file was changed and renamed are resulted
-    # in having deleted and added record
-    # To cover such cases we need to change hashing function
-    # to produce rolling/chunking hash
-
-    renamed = _calculate_renamed(new, old, added, deleted)
-
-    for renamed_item in renamed:
-        added.remove(renamed_item["path"]["new"])
-        deleted.remove(renamed_item["path"]["old"])
-
-    ret = {
-        "added": [{"path": path, "hash": new[path]} for path in added],
-        "deleted": [{"path": path, "hash": old[path]} for path in deleted],
-        "modified": [
-            {"path": path, "hash": {"old": old[path], "new": new[path]}}
-            for path in modified
-            if old[path] != new[path]
-        ],
-        "renamed": renamed,
-        "not in cache": [{"path": path, "hash": old[path]} for path in missing],
-    }
-
-    return ret if any(ret.values()) else {}
-
-
-def _paths_checksums(
-    repo: "Repo", targets: Optional[List[str]]
-) -> Dict[str, Optional[str]]:
-    """
-    A dictionary of checksums addressed by relpaths collected from
-    the current fs outputs.
-
-    To help distinguish between a directory and a file output,
-    the former one will come with a trailing slash in the path:
-
-        directory: "data/"
-        file:      "data"
-    """
-
-    return dict(_output_paths(repo, targets))
-
-
-def _output_paths(  # noqa: C901
-    repo: "Repo", targets: Optional[List[str]]
-) -> Iterable[Tuple[str, Optional[str]]]:
-    from dvc.fs import LocalFileSystem
-    from dvc_data.hashfile.build import build
-
-    on_working_fs = isinstance(repo.fs, LocalFileSystem)
-
-    def _exists(output: "Output") -> bool:
-        if on_working_fs:
-            return output.exists
-        return True
-
-    def _to_path(output: "Output") -> str:
-        relparts = output.fs.path.relparts(output.fs_path)
-        base = os.path.join(*relparts)
-        if output.is_dir_checksum:
-            return os.path.join(base, "")
-        return base
-
-    for output in repo.index.outs:
-        if not _exists(output):
-            continue
-
-        yield_output = targets is None or any(
-            output.fs.path.isin_or_eq(output.fs_path, target) for target in targets
+        view = self.index.targets_view(
+            targets,
+            onerror=onerror,
+            recursive=recursive,
         )
 
-        obj: Optional["HashFile"]
-        if on_working_fs:
-            _, _, obj = build(
-                repo.cache.repo,
-                output.fs_path,
-                repo.cache.repo.fs,
-                "md5",
-                dry_run=True,
-                ignore=output.dvcignore,
-            )
-            hash_info = obj.hash_info
-        else:
-            hash_info = output.hash_info
-            obj = output.get_obj()
+        if rev == "workspace":
+            from .index import build_data_index
 
-        if yield_output:
-            yield _to_path(output), hash_info.value
-
-        if not obj:
-            continue
-
-        if output.is_dir_checksum and (
-            yield_output
-            or (
-                isinstance(targets, list)
-                and any(
-                    output.fs.path.isin(target, output.fs_path) for target in targets
+            with ui.status("Building workspace index"):
+                data = build_data_index(
+                    view,
+                    self.root_dir,
+                    self.fs,
+                    compute_hash=True,
                 )
-            )
-        ):
-            obj = cast("Tree", obj)
-            yield from _dir_output_paths(output.fs, output.fs_path, obj, targets)
-
-
-def _dir_output_paths(
-    fs: "FileSystem",
-    fs_path: str,
-    obj: "Tree",
-    targets: Optional[Iterable[str]] = None,
-) -> Iterator[Tuple[str, str]]:
-    base = os.path.join(*fs.path.relparts(fs_path))
-    for key, _, oid in obj:
-        fname = fs.path.join(fs_path, *key)
-        if targets is None or any(
-            fs.path.isin_or_eq(fname, target) for target in targets
-        ):
-            # pylint: disable=no-member
-            yield os.path.join(base, *key), oid.value
-
-
-def _filter_missing(dvcfs: "DVCFileSystem", paths: Set[str]) -> Iterator[str]:
-    for path in paths:
-        fs_path = dvcfs.from_os_path(path)
-        try:
-            info = dvcfs.info(fs_path)
-            dvc_info = info.get("dvc_info", {})
-            entry = dvc_info.get("entry")
-            if (
-                entry
-                and info["type"] == "directory"
-                and not dvcfs.repo.cache.local.exists(entry.hash_info.value)
-            ):
-                yield path
-        except FileNotFoundError:
-            pass
-
-
-def _targets_to_paths(
-    dvcfs: "DVCFileSystem", targets: List[str]
-) -> Tuple[List[str], List[str]]:
-    paths: List[str] = []
-    missing: List[str] = []
-
-    if not targets:
-        return paths, missing
-
-    for target in targets:
-        if dvcfs.exists(target):
-            paths.append(dvcfs.repo.fs.path.abspath(target))
         else:
-            missing.append(target)
+            data = view.data["repo"]
 
-    return paths, missing
+        assert rev not in indexes
+        indexes[rev] = data
 
+    if targets:
+        old_missing = missing_targets.get(a_rev, set())
+        new_missing = missing_targets.get(b_rev, set())
 
-def _calculate_renamed(
-    new: Dict[str, Any],
-    old: Dict[str, Any],
-    added: Iterable[str],
-    deleted: Iterable[str],
-) -> List[Dict[str, Dict[str, str]]]:
-    old_inverted: Dict[str, List[str]] = defaultdict(list)
-    # It is needed to be dict of lists to cover cases
-    # when repo has paths with same hash
-    for path, path_hash in old.items():
-        old_inverted[path_hash].append(path)
+        # check for overlapping missing targets between a_rev and b_rev
+        for target in old_missing & new_missing:
+            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), target)
 
-    renamed = []
-    for path in added:
-        path_hash = new[path]
-        old_paths = old_inverted[path_hash]
-        try:
-            iterator = enumerate(old_paths)
-            index = next(idx for idx, path in iterator if path in deleted)
-        except StopIteration:
-            continue
+    old = indexes[a_rev]
+    new = indexes[b_rev]
 
-        old_path = old_paths.pop(index)
-        renamed.append({"path": {"old": old_path, "new": path}, "hash": path_hash})
-
-    return renamed
+    with ui.status("Calculating diff"):
+        return _diff(old, new, with_missing=with_missing)

dvc/repo/experiments/__init__.py

@@ -1,8 +1,7 @@
 import logging
 import os
 import re
-import time
-from typing import TYPE_CHECKING, Dict, Iterable, Optional
+from typing import TYPE_CHECKING, Dict, Iterable, List, Optional
 
 from funcy import chain, first
 
@@ -11,17 +10,15 @@ from dvc.ui import ui
 from dvc.utils import relpath
 from dvc.utils.objects import cached_property
 
+from .cache import ExpCache
 from .exceptions import (
     BaselineMismatchError,
     ExperimentExistsError,
     InvalidExpRefError,
     MultipleBranchError,
 )
-from .executor.base import BaseExecutor
-from .queue.celery import LocalCeleryQueue
-from .queue.tempdir import TempDirQueue
-from .queue.workspace import WorkspaceQueue
 from .refs import (
+    APPLY_STASH,
     CELERY_FAILED_STASH,
     CELERY_STASH,
     EXEC_APPLY,
@@ -31,10 +28,14 @@ from .refs import (
     WORKSPACE_STASH,
     ExpRefInfo,
 )
+from .stash import ApplyStash
 from .utils import check_ref_format, exp_refs_by_rev, unlocked_repo
 
 if TYPE_CHECKING:
     from .queue.base import BaseStashQueue, QueueEntry
+    from .queue.celery import LocalCeleryQueue
+    from .queue.tempdir import TempDirQueue
+    from .queue.workspace import WorkspaceQueue
     from .stash import ExpStashEntry
 
 logger = logging.getLogger(__name__)
@@ -62,6 +63,11 @@ class Experiments:
 
     @property
     def scm(self):
+        from dvc.scm import SCMError
+
+        if self.repo.scm.no_commits:
+            raise SCMError("Empty Git repo. Add a commit to use experiments.")
+
         return self.repo.scm
 
     @cached_property
@@ -70,22 +76,38 @@ class Experiments:
 
     @cached_property
     def args_file(self) -> str:
+        from .executor.base import BaseExecutor
+
         return os.path.join(self.repo.tmp_dir, BaseExecutor.PACKED_ARGS_FILE)
 
     @cached_property
-    def workspace_queue(self) -> WorkspaceQueue:
+    def workspace_queue(self) -> "WorkspaceQueue":
+        from .queue.workspace import WorkspaceQueue
+
         return WorkspaceQueue(self.repo, WORKSPACE_STASH)
 
     @cached_property
-    def tempdir_queue(self) -> TempDirQueue:
+    def tempdir_queue(self) -> "TempDirQueue":
+        from .queue.tempdir import TempDirQueue
+
         # NOTE: tempdir and workspace stash is shared since both
         # implementations immediately push -> pop (queue length is only 0 or 1)
         return TempDirQueue(self.repo, WORKSPACE_STASH)
 
     @cached_property
-    def celery_queue(self) -> LocalCeleryQueue:
+    def celery_queue(self) -> "LocalCeleryQueue":
+        from .queue.celery import LocalCeleryQueue
+
         return LocalCeleryQueue(self.repo, CELERY_STASH, CELERY_FAILED_STASH)
 
+    @cached_property
+    def apply_stash(self) -> ApplyStash:
+        return ApplyStash(self.scm, APPLY_STASH)
+
+    @cached_property
+    def cache(self) -> ExpCache:
+        return ExpCache(self.repo)
+
     @property
     def stash_revs(self) -> Dict[str, "ExpStashEntry"]:
         revs = {}
@@ -96,24 +118,15 @@ class Experiments:
     def reproduce_one(
         self,
         tmp_dir: bool = False,
-        machine: Optional[str] = None,
+        copy_paths: Optional[List[str]] = None,
         **kwargs,
     ):
         """Reproduce and checkout a single (standalone) experiment."""
-        if not (tmp_dir or machine):
-            staged, _, _ = self.scm.status(untracked_files="no")
-            if staged:
-                logger.warning(
-                    "Your workspace contains staged Git changes which will be "
-                    "unstaged before running this experiment."
-                )
-                self.scm.reset()
-
         exp_queue: "BaseStashQueue" = (
             self.tempdir_queue if tmp_dir else self.workspace_queue
         )
         self.queue_one(exp_queue, **kwargs)
-        results = self._reproduce_queue(exp_queue)
+        results = self._reproduce_queue(exp_queue, copy_paths=copy_paths)
         exp_rev = first(results)
         if exp_rev is not None:
             self._log_reproduced(results, tmp_dir=tmp_dir)
@@ -192,8 +205,7 @@ class Experiments:
             )
             for entry in entries:
                 # wait for task execution to start
-                while not self.celery_queue.proc.get(entry.stash_rev):
-                    time.sleep(1)
+                self.celery_queue.wait_for_start(entry, sleep_interval=1)
                 self.celery_queue.follow(entry)
                 # wait for task collection to complete
                 try:
@@ -336,7 +348,9 @@ class Experiments:
         self.scm.remove_ref(EXEC_APPLY)
 
     @unlocked_repo
-    def _reproduce_queue(self, queue: "BaseStashQueue", **kwargs) -> Dict[str, str]:
+    def _reproduce_queue(
+        self, queue: "BaseStashQueue", copy_paths: Optional[List[str]] = None, **kwargs
+    ) -> Dict[str, str]:
         """Reproduce queued experiments.
 
         Arguments:
@@ -346,7 +360,7 @@ class Experiments:
             dict mapping successfully reproduced experiment revs to their
             results.
         """
-        exec_results = queue.reproduce()
+        exec_results = queue.reproduce(copy_paths=copy_paths)
 
         results: Dict[str, str] = {}
         for _, exp_result in exec_results.items():
@@ -417,7 +431,7 @@ class Experiments:
                 try:
                     name = ExpRefInfo.from_ref(ref).name
                 except InvalidExpRefError:
-                    pass  # noqa: S110
+                    pass
             if not name:
                 if rev in self.stash_revs:
                     name = self.stash_revs[rev].name
@@ -493,3 +507,8 @@ class Experiments:
         from dvc.repo.experiments.remove import remove
 
         return remove(self.repo, *args, **kwargs)
+
+    def clean(self, *args, **kwargs):
+        from dvc.repo.experiments.clean import clean
+
+        return clean(self.repo, *args, **kwargs)

dvc/repo/experiments/apply.py

@@ -1,14 +1,14 @@
 import logging
 import os
-from contextlib import contextmanager
 from typing import TYPE_CHECKING, Optional
 
 from dvc.repo import locked
 from dvc.repo.scm_context import scm_context
 from dvc.scm import Git
+from dvc.ui import ui
 from dvc.utils.fs import remove
 
-from .exceptions import ApplyConflictError, BaselineMismatchError, InvalidExpRevError
+from .exceptions import BaselineMismatchError, InvalidExpRevError
 from .executor.base import BaseExecutor
 from .refs import EXEC_APPLY
 
@@ -21,11 +21,9 @@ logger = logging.getLogger(__name__)
 
 @locked
 @scm_context
-def apply(repo: "Repo", rev: str, force: bool = True, **kwargs):  # noqa: C901
-    from scmrepo.exceptions import SCMError as _SCMError
-
+def apply(repo: "Repo", rev: str, **kwargs):  # noqa: C901
     from dvc.repo.checkout import checkout as dvc_checkout
-    from dvc.scm import GitMergeError, RevError, resolve_rev
+    from dvc.scm import RevError, resolve_rev
 
     exps: "Experiments" = repo.experiments
 
@@ -34,7 +32,6 @@ def apply(repo: "Repo", rev: str, force: bool = True, **kwargs):  # noqa: C901
     assert isinstance(repo.scm, Git)
     try:
         exp_rev = resolve_rev(repo.scm, rev)
-        exps.check_baseline(exp_rev)
     except RevError as exc:
         (
             exp_ref_info,
@@ -42,14 +39,9 @@ def apply(repo: "Repo", rev: str, force: bool = True, **kwargs):  # noqa: C901
         ) = exps.celery_queue.get_ref_and_entry_by_names(
             rev
         )[rev]
-        baseline_sha = repo.scm.get_rev()
         if exp_ref_info:
-            if baseline_sha != exp_ref_info.baseline_sha:
-                raise InvalidExpRevError(rev)  # noqa: B904
             exp_rev = repo.scm.get_ref(str(exp_ref_info))
         elif queue_entry:
-            if queue_entry.baseline_rev != baseline_sha:
-                raise InvalidExpRevError(rev)
             exp_rev = queue_entry.stash_rev
             is_stash = True
         else:
@@ -57,68 +49,22 @@ def apply(repo: "Repo", rev: str, force: bool = True, **kwargs):  # noqa: C901
     except BaselineMismatchError as exc:
         raise InvalidExpRevError(rev) from exc
 
-    # NOTE: we don't use scmrepo's stash_workspace() here since we need
-    # finer control over the merge behavior when we unstash everything
-    with _apply_workspace(repo, rev, force):
-        try:
-            repo.scm.merge(exp_rev, commit=False, squash=True)
-        except _SCMError as exc:
-            raise GitMergeError(str(exc), scm=repo.scm)  # noqa: B904
-
-    repo.scm.reset()
-
-    if is_stash:
-        assert repo.tmp_dir is not None
-        args_path = os.path.join(repo.tmp_dir, BaseExecutor.PACKED_ARGS_FILE)
-        if os.path.exists(args_path):
-            remove(args_path)
-
+    _apply(repo, exp_rev, name=rev, is_stash=is_stash)
     dvc_checkout(repo, **kwargs)
 
     repo.scm.set_ref(EXEC_APPLY, exp_rev)
-    logger.info(
-        "Changes for experiment '%s' have been applied to your current workspace.",
-        rev,
+    ui.write(
+        f"Changes for experiment '{rev}' have been applied to your current workspace.",
     )
 
 
-@contextmanager
-def _apply_workspace(repo: "Repo", rev: str, force: bool):
-    from scmrepo.exceptions import MergeConflictError
-    from scmrepo.exceptions import SCMError as _SCMError
+def _apply(repo: "Repo", rev: str, name: Optional[str] = None, is_stash: bool = False):
+    exps: "Experiments" = repo.experiments
 
-    assert isinstance(repo.scm, Git)
-    if repo.scm.is_dirty(untracked_files=True):
-        logger.debug("Stashing workspace")
-        stash_rev: Optional[str] = repo.scm.stash.push(include_untracked=True)
-    else:
-        stash_rev = None
-    try:
-        yield
-    except Exception:  # pylint: disable=broad-except
-        if stash_rev:
-            _clean_and_pop(repo.scm)
-        raise
-    if not stash_rev:
-        return
-
-    try:
-        repo.scm.reset()
-        repo.scm.stash.apply(stash_rev, skip_conflicts=force)
-        repo.scm.stash.drop()
-    except (MergeConflictError, _SCMError) as exc:
-        _clean_and_pop(repo.scm)
-        raise ApplyConflictError(rev) from exc
-    except Exception:  # pylint: disable=broad-except
-        _clean_and_pop(repo.scm)
-        raise
-
-
-def _clean_and_pop(scm: "Git"):
-    """Revert any changes and pop the last stash entry."""
-    scm.reset(hard=True)
-    if scm.is_dirty(untracked_files=True):
-        # drop any changes to untracked files before popping stash
-        scm.stash.push(include_untracked=True)
-        scm.stash.drop()
-    scm.stash.pop()
+    with exps.apply_stash.preserve_workspace(rev, name=name):
+        with repo.scm.detach_head(rev, force=True):
+            if is_stash:
+                assert repo.tmp_dir is not None
+                args_path = os.path.join(repo.tmp_dir, BaseExecutor.PACKED_ARGS_FILE)
+                if os.path.exists(args_path):
+                    remove(args_path)

dvc/repo/experiments/branch.py

@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)
 
 @locked
 @scm_context
-def branch(repo, exp_rev, branch_name, *args, **kwargs):
+def branch(repo, exp_rev, branch_name=None, **kwargs):
     from dvc.scm import resolve_rev
 
     try:
@@ -46,6 +46,8 @@ def branch(repo, exp_rev, branch_name, *args, **kwargs):
     if not ref_info:
         raise InvalidExpRevError(exp_rev)
 
+    branch_name = branch_name or f"{ref_info.name}-branch"
+
     branch_ref = f"refs/heads/{branch_name}"
     if repo.scm.get_ref(branch_ref):
         raise InvalidArgumentError(f"Git branch '{branch_name}' already exists.")

dvc/repo/experiments/cache.py

@@ -0,0 +1,72 @@
+import logging
+import os
+from typing import TYPE_CHECKING, Optional, Union
+
+from dvc.fs import localfs
+from dvc_objects.db import ObjectDB
+
+from .serialize import DeserializeError, SerializableError, SerializableExp
+from .utils import EXEC_TMP_DIR
+
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+
+logger = logging.getLogger(__name__)
+
+
+class ExpCache:
+    """Serialized experiment state cache.
+
+    ODB with git SHAs as keys. Objects can be either SerializableExp or
+    SerializableError.
+    """
+
+    CACHE_DIR = os.path.join(EXEC_TMP_DIR, "cache")
+
+    def __init__(self, repo: "Repo"):
+        path = os.path.join(repo.tmp_dir, self.CACHE_DIR)
+        self.odb = ObjectDB(localfs, path)
+
+    def delete(self, rev: str):
+        self.odb.delete(rev)
+
+    def put(
+        self,
+        exp: Union[SerializableExp, SerializableError],
+        rev: Optional[str] = None,
+        force: bool = False,
+    ):
+        rev = rev or getattr(exp, "rev", None)
+        assert rev
+        assert rev != "workspace"
+        if force or not self.odb.exists(rev):
+            try:
+                self.delete(rev)
+            except FileNotFoundError:
+                pass
+            self.odb.add_bytes(rev, exp.as_bytes())
+            logger.trace(  # type: ignore[attr-defined]
+                "ExpCache: cache put '%s'", rev[:7]
+            )
+
+    def get(self, rev: str) -> Optional[Union[SerializableExp, SerializableError]]:
+        obj = self.odb.get(rev)
+        try:
+            with obj.fs.open(obj.path, "rb") as fobj:
+                data = fobj.read()
+        except FileNotFoundError:
+            logger.trace(  # type: ignore[attr-defined]
+                "ExpCache: cache miss '%s'", rev[:7]
+            )
+            return None
+        for typ in (SerializableExp, SerializableError):
+            try:
+                exp = typ.from_bytes(data)  # type: ignore[attr-defined]
+                logger.trace(  # type: ignore[attr-defined]
+                    "ExpCache: cache load '%s'", rev[:7]
+                )
+                return exp
+            except DeserializeError:
+                continue
+        logger.debug("ExpCache: unknown object type for '%s'", rev)
+        return None

dvc/repo/experiments/clean.py

@@ -0,0 +1,12 @@
+from typing import TYPE_CHECKING
+
+from dvc.ui import ui
+
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+
+
+def clean(repo: "Repo"):
+    ui.write("Cleaning up dvc-task messages...")
+    repo.experiments.celery_queue.celery.clean()
+    ui.write("Done!")

dvc/repo/experiments/exceptions.py

@@ -66,18 +66,6 @@ class MultipleBranchError(DvcException):
         self.ref_infos = ref_infos
 
 
-class ApplyConflictError(InvalidArgumentError):
-    def __init__(self, name):
-        super().__init__(
-            f"Experiment '{name}' cannot be applied to because your current "
-            "workspace contains changes which would be overwritten. Either "
-            "'git stash' your current changes before applying this "
-            "experiment, or re-run this command with '--force' to overwrite "
-            "your current changes."
-        )
-        self.name = name
-
-
 class AmbiguousExpRefInfo(InvalidArgumentError):
     def __init__(
         self,

dvc/repo/experiments/executor/base.py

@@ -1,6 +1,7 @@
 import logging
 import os
-import pickle
+import pickle  # nosec B403
+import shutil
 from abc import ABC, abstractmethod
 from contextlib import contextmanager
 from dataclasses import asdict, dataclass
@@ -21,6 +22,7 @@ from typing import (
     Union,
 )
 
+from funcy import get_in
 from scmrepo.exceptions import SCMError
 
 from dvc.env import DVC_EXP_AUTO_PUSH, DVC_EXP_GIT_REMOTE
@@ -33,6 +35,8 @@ from dvc.repo.experiments.refs import (
     ExpRefInfo,
 )
 from dvc.repo.experiments.utils import to_studio_params
+from dvc.repo.metrics.show import _collect_top_level_metrics
+from dvc.repo.params.show import _collect_top_level_params
 from dvc.stage.serialize import to_lockfile
 from dvc.ui import ui
 from dvc.utils import dict_sha256, env2bool, relpath
@@ -255,6 +259,72 @@ class BaseExecutor(ABC):
             **kwargs,
         )
 
+    @classmethod
+    def save(
+        cls,
+        info: "ExecutorInfo",
+        force: bool = False,
+        include_untracked: Optional[List[str]] = None,
+    ) -> ExecutorResult:
+        from dvc.dvcfile import LOCK_FILE
+        from dvc.repo import Repo
+
+        exp_hash: Optional[str] = None
+        exp_ref: Optional[ExpRefInfo] = None
+
+        dvc = Repo(os.path.join(info.root_dir, info.dvc_dir))
+        old_cwd = os.getcwd()
+        if info.wdir:
+            os.chdir(os.path.join(dvc.scm.root_dir, info.wdir))
+        else:
+            os.chdir(dvc.root_dir)
+
+        include_untracked = include_untracked or []
+        include_untracked.extend(_collect_top_level_metrics(dvc))
+        include_untracked.extend(_collect_top_level_params(dvc))
+        include_untracked.extend(
+            dvc.index._plot_sources  # pylint: disable=protected-access
+        )
+        # dvc repro automatically stages dvc.lock. Running redundant `git add`
+        # on it causes an error when exiting the detached head context.
+        if LOCK_FILE in dvc.scm.untracked_files():
+            include_untracked.append(LOCK_FILE)
+
+        try:
+            stages = dvc.commit([], force=True, relink=False)
+            exp_hash = cls.hash_exp(stages)
+            if include_untracked:
+                dvc.scm.add(include_untracked)
+            cls.commit(
+                dvc.scm,  # type: ignore[arg-type]
+                exp_hash,
+                exp_name=info.name,
+                force=force,
+            )
+            ref: Optional[str] = dvc.scm.get_ref(EXEC_BRANCH, follow=False)
+            exp_ref = ExpRefInfo.from_ref(ref) if ref else None
+            untracked = dvc.scm.untracked_files()
+            if untracked:
+                logger.warning(
+                    "The following untracked files were present in "
+                    "the workspace before saving but "
+                    "will not be included in the experiment commit:\n"
+                    "\t%s",
+                    ", ".join(untracked),
+                )
+            info.result_hash = exp_hash
+            info.result_ref = ref
+            info.result_force = False
+            info.status = TaskStatus.SUCCESS
+        except DvcException:
+            info.status = TaskStatus.FAILED
+            raise
+        finally:
+            dvc.close()
+            os.chdir(old_cwd)
+
+        return ExecutorResult(ref, exp_ref, info.result_force)
+
     @staticmethod
     def hash_exp(stages: Iterable["PipelineStage"]) -> str:
         from dvc.stage import PipelineStage
@@ -265,7 +335,7 @@ class BaseExecutor(ABC):
                 exp_data.update(to_lockfile(stage))
         return dict_sha256(exp_data)
 
-    def cleanup(self, infofile: str):
+    def cleanup(self, infofile: Optional[str] = None):
         if infofile is not None:
             info = ExecutorInfo.load_json(infofile)
             if info.status < TaskStatus.FAILED:
@@ -292,7 +362,7 @@ class BaseExecutor(ABC):
     @staticmethod
     def unpack_repro_args(path):
         with open(path, "rb") as fobj:
-            data = pickle.load(fobj)
+            data = pickle.load(fobj)  # noqa: S301 # nosec B301
         return data["args"], data["kwargs"]
 
     def fetch_exps(
@@ -382,6 +452,7 @@ class BaseExecutor(ABC):
         infofile: Optional[str] = None,
         log_errors: bool = True,
         log_level: Optional[int] = None,
+        copy_paths: Optional[List[str]] = None,
         **kwargs,
     ) -> "ExecutorResult":
         """Run dvc repro and return the result.
@@ -418,6 +489,7 @@ class BaseExecutor(ABC):
             info,
             infofile,
             log_errors=log_errors,
+            copy_paths=copy_paths,
             **kwargs,
         ) as dvc:
             if auto_push:
@@ -540,6 +612,7 @@ class BaseExecutor(ABC):
         info: "ExecutorInfo",
         infofile: Optional[str] = None,
         log_errors: bool = True,
+        copy_paths: Optional[List[str]] = None,
         **kwargs,
     ):
         from dvc_studio_client.post_live_metrics import post_live_metrics
@@ -547,55 +620,59 @@ class BaseExecutor(ABC):
         from dvc.repo import Repo
         from dvc.stage.monitor import CheckpointKilledError
 
-        dvc = Repo(os.path.join(info.root_dir, info.dvc_dir))
-        info.status = TaskStatus.RUNNING
-        if infofile is not None:
-            info.dump_json(infofile)
-        if cls.QUIET:
-            dvc.scm_context.quiet = cls.QUIET
-        old_cwd = os.getcwd()
-        if info.wdir:
-            os.chdir(os.path.join(dvc.scm.root_dir, info.wdir))
-        else:
-            os.chdir(dvc.root_dir)
-
-        try:
-            post_live_metrics(
-                "start",
-                info.baseline_rev,
-                info.name,
-                "dvc",
-                params=to_studio_params(dvc.params.show()),
-            )
-            logger.debug("Running repro in '%s'", os.getcwd())
-            yield dvc
-            info.status = TaskStatus.SUCCESS
-        except CheckpointKilledError:
-            info.status = TaskStatus.FAILED
-            raise
-        except DvcException:
-            if log_errors:
-                logger.exception("")
-            info.status = TaskStatus.FAILED
-            raise
-        except Exception:
-            if log_errors:
-                logger.exception("unexpected error")
-            info.status = TaskStatus.FAILED
-            raise
-        finally:
-            post_live_metrics(
-                "done",
-                info.baseline_rev,
-                info.name,
-                "dvc",
-                experiment_rev=dvc.experiments.scm.get_ref(EXEC_BRANCH),
-            )
-
+        with Repo(os.path.join(info.root_dir, info.dvc_dir)) as dvc:
+            info.status = TaskStatus.RUNNING
             if infofile is not None:
                 info.dump_json(infofile)
-            dvc.close()
-            os.chdir(old_cwd)
+            if cls.QUIET:
+                dvc.scm_context.quiet = cls.QUIET
+            old_cwd = os.getcwd()
+
+            for path in copy_paths or []:
+                cls._copy_path(os.path.realpath(path), os.path.join(dvc.root_dir, path))
+
+            if info.wdir:
+                os.chdir(os.path.join(dvc.scm.root_dir, info.wdir))
+            else:
+                os.chdir(dvc.root_dir)
+
+            try:
+                post_live_metrics(
+                    "start",
+                    info.baseline_rev,
+                    info.name,
+                    "dvc",
+                    params=to_studio_params(dvc.params.show()),
+                )
+                logger.debug("Running repro in '%s'", os.getcwd())
+                yield dvc
+                info.status = TaskStatus.SUCCESS
+            except CheckpointKilledError:
+                info.status = TaskStatus.FAILED
+                raise
+            except DvcException:
+                if log_errors:
+                    logger.exception("")
+                info.status = TaskStatus.FAILED
+                raise
+            except Exception:
+                if log_errors:
+                    logger.exception("unexpected error")
+                info.status = TaskStatus.FAILED
+                raise
+            finally:
+                post_live_metrics(
+                    "done",
+                    info.baseline_rev,
+                    info.name,
+                    "dvc",
+                    experiment_rev=dvc.experiments.scm.get_ref(EXEC_BRANCH),
+                    metrics=get_in(dvc.metrics.show(), ["", "data"]),
+                )
+
+                if infofile is not None:
+                    info.dump_json(infofile)
+                os.chdir(old_cwd)
 
     @classmethod
     def _repro_args(cls, dvc):
@@ -723,6 +800,20 @@ class BaseExecutor(ABC):
         if level is not None:
             dvc_logger.setLevel(level)
 
+    @staticmethod
+    def _copy_path(src, dst):
+        try:
+            if os.path.isfile(src):
+                shutil.copy(src, dst)
+            elif os.path.isdir(src):
+                shutil.copytree(src, dst)
+            else:
+                raise DvcException(
+                    f"Unable to copy '{src}'. It is not a file or directory."
+                )
+        except OSError as exc:
+            raise DvcException(f"Unable to copy '{src}' to '{dst}'.") from exc
+
     @contextmanager
     def set_temp_refs(self, scm: "Git", temp_dict: Dict[str, str]):
         try:

dvc/repo/experiments/executor/local.py

@@ -4,8 +4,8 @@ from contextlib import ExitStack
 from tempfile import mkdtemp
 from typing import TYPE_CHECKING, Optional, Union
 
+from configobj import ConfigObj
 from funcy import retry
-from scmrepo.exceptions import SCMError as _SCMError
 from shortuuid import uuid
 
 from dvc.lock import LockError
@@ -20,7 +20,7 @@ from dvc.repo.experiments.refs import (
     EXPS_TEMP,
 )
 from dvc.repo.experiments.utils import EXEC_TMP_DIR, get_exp_rwlock
-from dvc.scm import SCM, Git, GitMergeError
+from dvc.scm import SCM, Git
 from dvc.utils.fs import remove
 from dvc.utils.objects import cached_property
 
@@ -49,7 +49,7 @@ class BaseLocalExecutor(BaseExecutor):
     def scm(self) -> Union["Git", "NoSCM"]:
         return SCM(self.root_dir)
 
-    def cleanup(self, infofile: str):
+    def cleanup(self, infofile: Optional[str] = None):
         self.scm.close()
         del self.scm
         super().cleanup(infofile)
@@ -134,28 +134,32 @@ class TempDirExecutor(BaseLocalExecutor):
         self.scm.checkout(head, detach=True)
         merge_rev = self.scm.get_ref(EXEC_MERGE)
 
-        try:
-            self.scm.merge(merge_rev, squash=True, commit=False)
-        except _SCMError as exc:
-            raise GitMergeError(str(exc), scm=self.scm)  # noqa: B904
+        self.scm.stash.apply(merge_rev)
+        self._update_config(repo.config.read("local"))
 
-    def _config(self, cache_dir):
+    def _update_config(self, update):
         local_config = os.path.join(
             self.root_dir,
             self.dvc_dir,
             "config.local",
         )
         logger.debug("Writing experiments local config '%s'", local_config)
-        with open(local_config, "w", encoding="utf-8") as fobj:
-            fobj.write(f"[cache]\n    dir = {cache_dir}")
+        if os.path.exists(local_config):
+            conf_obj = ConfigObj(local_config)
+            conf_obj.merge(update)
+        else:
+            conf_obj = ConfigObj(update)
+        if conf_obj:
+            with open(local_config, "wb") as fobj:
+                conf_obj.write(fobj)
 
     def init_cache(
         self, repo: "Repo", rev: str, run_cache: bool = True  # noqa: ARG002
     ):
         """Initialize DVC cache."""
-        self._config(repo.cache.repo.path)
+        self._update_config({"cache": {"dir": repo.cache.repo.path}})
 
-    def cleanup(self, infofile: str):
+    def cleanup(self, infofile: Optional[str] = None):
         super().cleanup(infofile)
         logger.debug("Removing tmpdir '%s'", self.root_dir)
         remove(self.root_dir)
@@ -229,10 +233,7 @@ class WorkspaceExecutor(BaseLocalExecutor):
                 )
             )
             merge_rev = self.scm.get_ref(EXEC_MERGE)
-            try:
-                self.scm.merge(merge_rev, squash=True, commit=False)
-            except _SCMError as exc:
-                raise GitMergeError(str(exc), scm=self.scm)  # noqa: B904
+            self.scm.stash.apply(merge_rev)
             if branch:
                 self.scm.set_ref(EXEC_BRANCH, branch, symbolic=True)
             elif scm.get_ref(EXEC_BRANCH):
@@ -241,9 +242,10 @@ class WorkspaceExecutor(BaseLocalExecutor):
     def init_cache(self, repo: "Repo", rev: str, run_cache: bool = True):
         pass
 
-    def cleanup(self, infofile: str):
+    def cleanup(self, infofile: Optional[str] = None):
         super().cleanup(infofile)
-        remove(os.path.dirname(infofile))
+        if infofile:
+            remove(os.path.dirname(infofile))
         with self._detach_stack:
             self.scm.remove_ref(EXEC_BASELINE)
             self.scm.remove_ref(EXEC_MERGE)

dvc/repo/experiments/executor/ssh.py

@@ -3,7 +3,7 @@ import os
 import posixpath
 import sys
 from contextlib import contextmanager
-from typing import TYPE_CHECKING, Callable, Iterable, Optional
+from typing import TYPE_CHECKING, Callable, Iterable, List, Optional
 
 from dvc_ssh import SSHFileSystem
 from funcy import first
@@ -169,7 +169,7 @@ class SSHExecutor(BaseExecutor):
             head = EXEC_BRANCH if branch else EXEC_HEAD
             self._ssh_cmd(fs, f"git checkout {head}")
             merge_rev = scm.get_ref(EXEC_MERGE)
-            self._ssh_cmd(fs, f"git merge --squash --no-commit {merge_rev}")
+            self._ssh_cmd(fs, f"git stash apply {merge_rev}")
 
             if self._setup_script:
                 self._init_setup_script(fs)
@@ -242,6 +242,7 @@ class SSHExecutor(BaseExecutor):
         infofile: Optional[str] = None,
         log_errors: bool = True,
         log_level: Optional[int] = None,
+        copy_paths: Optional[List[str]] = None,  # noqa: ARG003
         **kwargs,
     ) -> "ExecutorResult":
         """Reproduce an experiment on a remote machine over SSH.

dvc/repo/experiments/init.py

@@ -21,9 +21,9 @@ from dvc.exceptions import DvcException
 from dvc.stage import PipelineStage
 
 if TYPE_CHECKING:
+    from dvc.dependency import Dependency
     from dvc.dvcfile import ProjectFile, SingleStageFile
     from dvc.repo import Repo
-    from dvc.dependency import Dependency
 
 from dvc.ui import ui
 
@@ -218,13 +218,12 @@ def init(
             provided=overrides,
             stream=stream,
         )
+    elif "live" in overrides:
+        # suppress `metrics`/`plots` if live is selected.
+        defaults.pop("metrics", None)
+        defaults.pop("plots", None)
     else:
-        if "live" in overrides:
-            # suppress `metrics`/`plots` if live is selected.
-            defaults.pop("metrics", None)
-            defaults.pop("plots", None)
-        else:
-            defaults.pop("live", None)  # suppress live otherwise
+        defaults.pop("live", None)  # suppress live otherwise
 
     context: Dict[str, str] = {**defaults, **overrides}
     assert "cmd" in context

dvc/repo/experiments/push.py

@@ -1,35 +1,88 @@
 import logging
-from typing import Iterable, List, Mapping, Optional, Set, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Set,
+    Union,
+)
 
-from funcy import group_by
+from funcy import compact, group_by
 from scmrepo.git.backend.base import SyncStatus
 
+from dvc.exceptions import DvcException
 from dvc.repo import locked
 from dvc.repo.scm_context import scm_context
-from dvc.scm import TqdmGit, iter_revs
-from dvc.ui import ui
+from dvc.scm import Git, TqdmGit, iter_revs
+from dvc.utils import env2bool
 
 from .exceptions import UnresolvedExpNamesError
 from .refs import ExpRefInfo
 from .utils import exp_commits, exp_refs, exp_refs_by_baseline, resolve_name
 
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+
 logger = logging.getLogger(__name__)
 
 
+class UploadError(DvcException):
+    def __init__(self, msg, result):
+        self.result = result
+        super().__init__(msg)
+
+
+def notify_refs_to_studio(
+    repo: "Repo", git_remote: str, **refs: List[str]
+) -> Optional[str]:
+    config = repo.config["feature"]
+    refs = compact(refs)
+    if not refs or env2bool("DVC_TEST"):
+        return None
+
+    if not (config.get("studio_token") or config["push_exp_to_studio"]):
+        logger.debug(
+            "Either feature.studio_token or feature.push_exp_to_studio config "
+            "needs to be set."
+        )
+        return None
+
+    import os
+
+    token = os.environ.get("STUDIO_TOKEN") or config.get("studio_token")
+    if not token:
+        logger.debug("Studio token not found.")
+        return None
+
+    from dulwich.porcelain import get_remote_repo
+
+    from dvc.utils import studio
+
+    _, repo_url = get_remote_repo(repo.scm.dulwich.repo, git_remote)
+    studio_url = config.get("studio_url")
+    d = studio.notify_refs(repo_url, token, base_url=studio_url, **refs)
+    return d.get("url")
+
+
 @locked
 @scm_context
 def push(  # noqa: C901
-    repo,
+    repo: "Repo",
     git_remote: str,
     exp_names: Union[Iterable[str], str],
-    all_commits=False,
+    all_commits: bool = False,
     rev: Optional[str] = None,
-    num=1,
+    num: int = 1,
     force: bool = False,
     push_cache: bool = False,
-    **kwargs,
-) -> Iterable[str]:
+    **kwargs: Any,
+) -> Dict[str, Any]:
     exp_ref_set: Set["ExpRefInfo"] = set()
+    assert isinstance(repo.scm, Git)
     if all_commits:
         exp_ref_set.update(exp_refs(repo.scm))
 
@@ -57,23 +110,29 @@ def push(  # noqa: C901
                 exp_ref_set.update(ref_info_list)
 
     push_result = _push(repo, git_remote, exp_ref_set, force)
-    if push_result[SyncStatus.DIVERGED]:
-        diverged_refs = [ref.name for ref in push_result[SyncStatus.DIVERGED]]
-        ui.warn(
-            f"Local experiment '{diverged_refs}' has diverged from remote "
-            "experiment with the same name. To override the remote experiment "
-            "re-run with '--force'."
-        )
+
+    refs = {
+        status.name.lower(): [ref.name for ref in ref_list]
+        for status, ref_list in push_result.items()
+    }
+    result: Dict[str, Any] = {**refs, "uploaded": 0}
+
+    pushed_refs_info = (
+        push_result[SyncStatus.UP_TO_DATE] + push_result[SyncStatus.SUCCESS]
+    )
     if push_cache:
-        push_cache_ref = (
-            push_result[SyncStatus.UP_TO_DATE] + push_result[SyncStatus.SUCCESS]
-        )
-        _push_cache(repo, push_cache_ref, **kwargs)
-    return [ref.name for ref in push_result[SyncStatus.SUCCESS]]
+        try:
+            result["uploaded"] = _push_cache(repo, pushed_refs_info, **kwargs)
+        except Exception as exc:  # noqa: BLE001, pylint: disable=broad-except
+            raise UploadError("failed to push cache", result) from exc
+
+    pushed_refs = [str(r) for r in pushed_refs_info]
+    url = notify_refs_to_studio(repo, git_remote, pushed=pushed_refs)
+    return {**result, "url": url}
 
 
 def _push(
-    repo,
+    repo: "Repo",
     git_remote: str,
     refs: Iterable["ExpRefInfo"],
     force: bool,
@@ -83,7 +142,7 @@ def _push(
     from dvc.scm import GitAuthError
 
     refspec_list = [f"{exp_ref}:{exp_ref}" for exp_ref in refs]
-    logger.debug("git push experiment '%s' -> '%s'", refspec_list, git_remote)
+    logger.debug("git push experiment %s -> '%s'", refspec_list, git_remote)
 
     with TqdmGit(desc="Pushing git refs") as pbar:
         try:
@@ -105,14 +164,15 @@ def _push(
 
 
 def _push_cache(
-    repo,
+    repo: "Repo",
     refs: Union[ExpRefInfo, Iterable["ExpRefInfo"]],
-    dvc_remote=None,
-    jobs=None,
-    run_cache=False,
-):
+    dvc_remote: Optional[str] = None,
+    jobs: Optional[int] = None,
+    run_cache: bool = False,
+) -> int:
     if isinstance(refs, ExpRefInfo):
         refs = [refs]
+    assert isinstance(repo.scm, Git)
     revs = list(exp_commits(repo.scm, refs))
     logger.debug("dvc push experiment '%s'", refs)
-    repo.push(jobs=jobs, remote=dvc_remote, run_cache=run_cache, revs=revs)
+    return repo.push(jobs=jobs, remote=dvc_remote, run_cache=run_cache, revs=revs)

dvc/repo/experiments/queue/base.py

@@ -17,6 +17,8 @@ from typing import (
     Union,
 )
 
+from dvc_studio_client.env import STUDIO_REPO_URL
+from dvc_studio_client.post_live_metrics import get_studio_repo_url
 from funcy import retry
 
 from dvc.dependency import ParamsDependency
@@ -250,7 +252,9 @@ class BaseStashQueue(ABC):
         """Iterate over items which been failed."""
 
     @abstractmethod
-    def reproduce(self) -> Mapping[str, Mapping[str, str]]:
+    def reproduce(
+        self, copy_paths: Optional[List[str]] = None
+    ) -> Mapping[str, Mapping[str, str]]:
         """Reproduce queued experiments sequentially."""
 
     @abstractmethod
@@ -295,7 +299,7 @@ class BaseStashQueue(ABC):
                 output.
         """
 
-    def _stash_exp(  # noqa: C901
+    def _stash_exp(  # noqa: PLR0915, C901
         self,
         *args,
         params: Optional[Dict[str, List[str]]] = None,
@@ -323,12 +327,12 @@ class BaseStashQueue(ABC):
         .. _Hydra Override:
             https://hydra.cc/docs/next/advanced/override_grammar/basic/
         """
-        with self.scm.detach_head(client="dvc") as orig_head:
-            stash_head = orig_head
-            if baseline_rev is None:
-                baseline_rev = orig_head
+        with self.scm.stash_workspace(reinstate_index=True) as workspace:
+            with self.scm.detach_head(client="dvc") as orig_head:
+                stash_head = orig_head
+                if baseline_rev is None:
+                    baseline_rev = orig_head
 
-            with self.scm.stash_workspace() as workspace:
                 try:
                     if workspace:
                         self.stash.apply(workspace)
@@ -415,6 +419,10 @@ class BaseStashQueue(ABC):
                     run_env[DVC_EXP_NAME] = name
                     if resume_rev:
                         run_env[DVCLIVE_RESUME] = "1"
+                    studio_repo_url = get_studio_repo_url()
+                    if studio_repo_url is not None:
+                        run_env[STUDIO_REPO_URL] = studio_repo_url
+
                     self._pack_args(*args, run_env=run_env, **kwargs)
 
                     # save experiment as a stash commit
@@ -472,6 +480,7 @@ class BaseStashQueue(ABC):
                 force=True,
                 allow_missing=True,
                 data_only=True,
+                relink=False,
             )
 
     @staticmethod
@@ -489,7 +498,7 @@ class BaseStashQueue(ABC):
         return msg
 
     def _pack_args(self, *args, **kwargs) -> None:
-        import pickle
+        import pickle  # nosec B403
 
         if os.path.exists(self.args_file) and self.scm.is_tracked(self.args_file):
             logger.warning(
@@ -504,7 +513,7 @@ class BaseStashQueue(ABC):
             )
             with open(self.args_file, "rb") as fobj:
                 try:
-                    data = pickle.load(fobj)
+                    data = pickle.load(fobj)  # noqa: S301  # nosec B301
                 except Exception:  # noqa: BLE001, pylint: disable=broad-except
                     data = {}
             extra = int(data.get("extra", 0)) + 1

dvc/repo/experiments/queue/celery.py

@@ -90,7 +90,7 @@ class LocalCeleryQueue(BaseStashQueue):
                 "dvc_task.proc.tasks",
             ],
         )
-        app.conf.update({"task_acks_late": True})
+        app.conf.update({"task_acks_late": True, "result_expires": None})
         return app
 
     @cached_property
@@ -138,6 +138,9 @@ class LocalCeleryQueue(BaseStashQueue):
         wdir_hash = hashlib.sha256(self.wdir.encode("utf-8")).hexdigest()[:6]
         node_name = f"dvc-exp-{wdir_hash}-{num}@localhost"
         cmd = ["exp", "queue-worker", node_name]
+        if num == 1:
+            # automatically run celery cleanup when primary worker shuts down
+            cmd.append("--clean")
         name = f"dvc-exp-worker-{num}"
 
         logger.debug("start a new worker: %s, node: %s", name, node_name)
@@ -171,11 +174,13 @@ class LocalCeleryQueue(BaseStashQueue):
 
         return started
 
-    def put(self, *args, **kwargs) -> QueueEntry:
+    def put(
+        self, *args, copy_paths: Optional[List[str]] = None, **kwargs
+    ) -> QueueEntry:
         """Stash an experiment and add it to the queue."""
         with get_exp_rwlock(self.repo, writes=["workspace", CELERY_STASH]):
             entry = self._stash_exp(*args, **kwargs)
-        self.celery.signature(run_exp.s(entry.asdict())).delay()
+        self.celery.signature(run_exp.s(entry.asdict(), copy_paths=copy_paths)).delay()
         return entry
 
     # NOTE: Queue consumption should not be done directly. Celery worker(s)
@@ -247,7 +252,9 @@ class LocalCeleryQueue(BaseStashQueue):
             if exp_result is None:
                 yield QueueDoneResult(queue_entry, exp_result)
 
-    def reproduce(self) -> Mapping[str, Mapping[str, str]]:
+    def reproduce(
+        self, copy_paths: Optional[List[str]] = None
+    ) -> Mapping[str, Mapping[str, str]]:
         raise NotImplementedError
 
     def _load_info(self, rev: str) -> ExecutorInfo:
@@ -292,6 +299,28 @@ class LocalCeleryQueue(BaseStashQueue):
         # out of the active task list, and needs to be loaded here.
         return self._get_done_result(entry, timeout)
 
+    def wait(self, revs: Collection[str], **kwargs) -> None:
+        """Block until the specified tasks have completed."""
+        revs = [revs] if isinstance(revs, str) else revs
+        results = self.match_queue_entry_by_name(
+            revs, self.iter_queued(), self.iter_done(), self.iter_failed()
+        )
+        for entry in results.values():
+            if not entry:
+                continue
+            self.wait_for_start(entry, **kwargs)
+            try:
+                self.get_result(entry)
+            except FileNotFoundError:
+                pass
+
+    def wait_for_start(self, entry: QueueEntry, sleep_interval: float = 0.001) -> None:
+        """Block until the specified task has been started."""
+        import time
+
+        while not self.proc.get(entry.stash_rev):
+            time.sleep(sleep_interval)
+
     def _get_running_task_ids(self) -> Set[str]:
         running_task_ids: Set[str] = set()
         active_workers = self.worker_status()
@@ -336,7 +365,8 @@ class LocalCeleryQueue(BaseStashQueue):
                         task_id,
                         remained_entries[entry],
                     )
-                    self.celery.backend.mark_as_failure(task_id, None)
+                    backend = self.celery.backend
+                    backend.mark_as_failure(task_id, None)  # type: ignore[attr-defined]
 
         if remained_revs:
             raise CannotKillTasksError(remained_revs)

dvc/repo/experiments/queue/remove.py

@@ -9,7 +9,7 @@ if TYPE_CHECKING:
     from dvc.repo.experiments.stash import ExpStashEntry
 
 
-def remove_tasks(  # noqa: C901
+def remove_tasks(  # noqa: C901, PLR0912
     celery_queue: "LocalCeleryQueue",
     queue_entries: Iterable["QueueEntry"],
 ):
@@ -40,7 +40,7 @@ def remove_tasks(  # noqa: C901
             msg,
             queue_entry,
         ) in celery_queue._iter_queued():  # pylint: disable=protected-access
-            if queue_entry.stash_rev in stash_revs:
+            if queue_entry.stash_rev in stash_revs and msg.delivery_tag:
                 celery_queue.celery.reject(msg.delivery_tag)
     finally:
         celery_queue.stash.remove_revs(list(stash_revs.values()))
@@ -56,7 +56,8 @@ def remove_tasks(  # noqa: C901
             result: AsyncResult = AsyncResult(task_id)
             if result is not None:
                 result.forget()
-            celery_queue.celery.purge(msg.delivery_tag)
+            if msg.delivery_tag:
+                celery_queue.celery.purge(msg.delivery_tag)
     finally:
         if celery_queue.failed_stash:
             celery_queue.failed_stash.remove_revs(failed_stash_revs)

dvc/repo/experiments/queue/tasks.py

@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, Any, Dict
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
 from celery import shared_task
 from celery.utils.log import get_task_logger
@@ -28,17 +28,17 @@ def setup_exp(entry_dict: Dict[str, Any]) -> "BaseExecutor":
     from dvc.repo import Repo
 
     entry = QueueEntry.from_dict(entry_dict)
-    repo = Repo(entry.dvc_root)
-    # TODO: split executor.init_cache into separate subtask - we can release
-    # exp.scm_lock before DVC push
-    executor = BaseStashQueue.init_executor(
-        repo.experiments,
-        entry,
-        TempDirExecutor,
-        location="dvc-task",
-    )
-    infofile = repo.experiments.celery_queue.get_infofile_path(entry.stash_rev)
-    executor.info.dump_json(infofile)
+    with Repo(entry.dvc_root) as repo:
+        # TODO: split executor.init_cache into separate subtask - we can release
+        # exp.scm_lock before DVC push
+        executor = BaseStashQueue.init_executor(
+            repo.experiments,
+            entry,
+            TempDirExecutor,
+            location="dvc-task",
+        )
+        infofile = repo.experiments.celery_queue.get_infofile_path(entry.stash_rev)
+        executor.info.dump_json(infofile)
     return executor
 
 
@@ -59,23 +59,23 @@ def collect_exp(
     from dvc.repo import Repo
 
     entry = QueueEntry.from_dict(entry_dict)
-    repo = Repo(entry.dvc_root)
-    celery_queue = repo.experiments.celery_queue
-    infofile = celery_queue.get_infofile_path(entry.stash_rev)
-    executor_info = ExecutorInfo.load_json(infofile)
-    logger.debug("Collecting experiment info '%s'", str(executor_info))
-    executor = TempDirExecutor.from_info(executor_info)
-    exec_result = executor_info.result
-    try:
-        if exec_result is not None:
-            BaseStashQueue.collect_executor(repo.experiments, executor, exec_result)
-        else:
-            logger.debug("Experiment failed (Exec result was None)")
-            celery_queue.stash_failed(entry)
-    except Exception:  # pylint: disable=broad-except
-        # Log exceptions but do not re-raise so that task chain execution
-        # continues
-        logger.exception("Failed to collect experiment")
+    with Repo(entry.dvc_root) as repo:
+        celery_queue = repo.experiments.celery_queue
+        infofile = celery_queue.get_infofile_path(entry.stash_rev)
+        executor_info = ExecutorInfo.load_json(infofile)
+        logger.debug("Collecting experiment info '%s'", str(executor_info))
+        executor = TempDirExecutor.from_info(executor_info)
+        exec_result = executor_info.result
+        try:
+            if exec_result is not None:
+                BaseStashQueue.collect_executor(repo.experiments, executor, exec_result)
+            else:
+                logger.debug("Experiment failed (Exec result was None)")
+                celery_queue.stash_failed(entry)
+        except Exception:  # pylint: disable=broad-except
+            # Log exceptions but do not re-raise so that task chain execution
+            # continues
+            logger.exception("Failed to collect experiment")
     return executor.root_dir
 
 
@@ -91,7 +91,7 @@ def cleanup_exp(executor: TempDirExecutor, infofile: str) -> None:
 
 
 @shared_task
-def run_exp(entry_dict: Dict[str, Any]) -> None:
+def run_exp(entry_dict: Dict[str, Any], copy_paths: Optional[List[str]] = None) -> None:
     """Run a full experiment.
 
     Experiment subtasks are executed inline as one atomic operation.
@@ -102,12 +102,15 @@ def run_exp(entry_dict: Dict[str, Any]) -> None:
     from dvc.repo import Repo
 
     entry = QueueEntry.from_dict(entry_dict)
-    repo = Repo(entry.dvc_root)
-    queue = repo.experiments.celery_queue
-    infofile = queue.get_infofile_path(entry.stash_rev)
+    with Repo(entry.dvc_root) as repo:
+        queue = repo.experiments.celery_queue
+        infofile = queue.get_infofile_path(entry.stash_rev)
     executor = setup_exp.s(entry_dict)()
     try:
         cmd = ["dvc", "exp", "exec-run", "--infofile", infofile]
+        if copy_paths:
+            for path in copy_paths:
+                cmd.extend(["--copy-paths", path])
         proc_dict = queue.proc.run_signature(cmd, name=entry.stash_rev)()
         collect_exp.s(proc_dict, entry_dict)()
     finally:

dvc/repo/experiments/queue/tempdir.py

@@ -1,7 +1,7 @@
 import logging
 import os
 from collections import defaultdict
-from typing import TYPE_CHECKING, Dict, Generator, Optional
+from typing import TYPE_CHECKING, Dict, Generator, List, Optional
 
 from funcy import first
 
@@ -92,7 +92,11 @@ class TempDirQueue(WorkspaceQueue):
                 )
 
     def _reproduce_entry(
-        self, entry: QueueEntry, executor: "BaseExecutor"
+        self,
+        entry: QueueEntry,
+        executor: "BaseExecutor",
+        copy_paths: Optional[List[str]] = None,
+        **kwargs,
     ) -> Dict[str, Dict[str, str]]:
         from dvc.stage.monitor import CheckpointKilledError
 
@@ -107,6 +111,7 @@ class TempDirQueue(WorkspaceQueue):
                 infofile=infofile,
                 log_level=logger.getEffectiveLevel(),
                 log_errors=True,
+                copy_paths=copy_paths,
             )
             if not exec_result.exp_hash:
                 raise DvcException(f"Failed to reproduce experiment '{rev[:7]}'")

dvc/repo/experiments/queue/workspace.py

@@ -33,6 +33,7 @@ class WorkspaceQueue(BaseStashQueue):
     _EXEC_NAME: Optional[str] = "workspace"
 
     def put(self, *args, **kwargs) -> QueueEntry:
+        kwargs.pop("copy_paths", None)
         with get_exp_rwlock(self.repo, writes=["workspace", WORKSPACE_STASH]):
             return self._stash_exp(*args, **kwargs)
 
@@ -81,19 +82,24 @@ class WorkspaceQueue(BaseStashQueue):
     def iter_success(self) -> Generator["QueueDoneResult", None, None]:
         raise NotImplementedError
 
-    def reproduce(self) -> Dict[str, Dict[str, str]]:
+    def reproduce(
+        self, copy_paths: Optional[List[str]] = None
+    ) -> Dict[str, Dict[str, str]]:
         results: Dict[str, Dict[str, str]] = defaultdict(dict)
         try:
             while True:
                 entry, executor = self.get()
-                results.update(self._reproduce_entry(entry, executor))
+                results.update(
+                    self._reproduce_entry(entry, executor, copy_paths=copy_paths)
+                )
         except ExpQueueEmptyError:
             pass
         return results
 
     def _reproduce_entry(
-        self, entry: QueueEntry, executor: "BaseExecutor"
+        self, entry: QueueEntry, executor: "BaseExecutor", **kwargs
     ) -> Dict[str, Dict[str, str]]:
+        kwargs.pop("copy_paths", None)
         from dvc.stage.monitor import CheckpointKilledError
 
         results: Dict[str, Dict[str, str]] = defaultdict(dict)

dvc/repo/experiments/refs.py

@@ -7,6 +7,9 @@ from .exceptions import InvalidExpRefError
 EXPS_NAMESPACE = "refs/exps"
 EXPS_STASH = f"{EXPS_NAMESPACE}/stash"
 WORKSPACE_STASH = EXPS_STASH
+APPLY_NAMESPACE = f"{EXPS_NAMESPACE}/apply"
+APPLY_HEAD = f"{APPLY_NAMESPACE}/ORIG_HEAD"
+APPLY_STASH = f"{APPLY_NAMESPACE}/stash"
 CELERY_STASH = f"{EXPS_NAMESPACE}/celery/stash"
 CELERY_FAILED_STASH = f"{EXPS_NAMESPACE}/celery/failed"
 EXEC_NAMESPACE = f"{EXPS_NAMESPACE}/exec"
@@ -18,6 +21,7 @@ EXEC_HEAD = f"{EXEC_NAMESPACE}/EXEC_HEAD"
 EXEC_MERGE = f"{EXEC_NAMESPACE}/EXEC_MERGE"
 EXPS_TEMP = f"{EXPS_NAMESPACE}/temp"
 STASHES = {WORKSPACE_STASH, CELERY_STASH}
+ITER_SKIP_NAMESPACES = {APPLY_NAMESPACE, EXEC_NAMESPACE}
 
 
 class ExpRefInfo:

dvc/repo/experiments/remove.py

@@ -21,7 +21,7 @@ logger = logging.getLogger(__name__)
 
 @locked
 @scm_context
-def remove(  # noqa: C901
+def remove(  # noqa: C901, PLR0912
     repo: "Repo",
     exp_names: Union[None, str, List[str]] = None,
     rev: Optional[str] = None,
@@ -39,13 +39,6 @@ def remove(  # noqa: C901
         removed.extend(celery_queue.clear(queued=True))
 
     assert isinstance(repo.scm, Git)
-    if all_commits:
-        removed.extend(
-            _remove_commited_exps(
-                repo.scm, list(exp_refs(repo.scm, git_remote)), git_remote
-            )
-        )
-        return removed
 
     exp_ref_list: List["ExpRefInfo"] = []
     queue_entry_list: List["QueueEntry"] = []
@@ -65,11 +58,14 @@ def remove(  # noqa: C901
                 queue_entry_list.append(result.queue_entry)
 
         if remained:
-            raise UnresolvedExpNamesError(remained)
+            raise UnresolvedExpNamesError(remained, git_remote=git_remote)
     elif rev:
         exp_ref_dict = _resolve_exp_by_baseline(repo, rev, num, git_remote)
         removed.extend(exp_ref_dict.keys())
         exp_ref_list.extend(exp_ref_dict.values())
+    elif all_commits:
+        exp_ref_list.extend(exp_refs(repo.scm, git_remote))
+        removed = [ref.name for ref in exp_ref_list]
 
     if exp_ref_list:
         _remove_commited_exps(repo.scm, exp_ref_list, git_remote)
@@ -79,15 +75,22 @@ def remove(  # noqa: C901
 
         remove_tasks(celery_queue, queue_entry_list)
 
+    if git_remote:
+        from .push import notify_refs_to_studio
+
+        removed_refs = [str(r) for r in exp_ref_list]
+        notify_refs_to_studio(repo, git_remote, removed=removed_refs)
     return removed
 
 
 def _resolve_exp_by_baseline(
-    repo,
+    repo: "Repo",
     rev: str,
     num: int,
     git_remote: Optional[str] = None,
 ) -> Dict[str, "ExpRefInfo"]:
+    assert isinstance(repo.scm, Git)
+
     commit_ref_dict: Dict[str, "ExpRefInfo"] = {}
     rev_dict = iter_revs(repo.scm, [rev], num)
     rev_set = set(rev_dict.keys())

dvc/repo/experiments/run.py

@@ -11,7 +11,7 @@ logger = logging.getLogger(__name__)
 
 
 @locked
-def run(  # noqa: C901
+def run(  # noqa: C901, PLR0912
     repo,
     targets: Optional[Iterable[str]] = None,
     params: Optional[Iterable[str]] = None,
@@ -19,6 +19,7 @@ def run(  # noqa: C901
     jobs: int = 1,
     tmp_dir: bool = False,
     queue: bool = False,
+    copy_paths: Optional[Iterable[str]] = None,
     **kwargs,
 ) -> Dict[str, str]:
     """Reproduce the specified targets as an experiment.
@@ -36,6 +37,18 @@ def run(  # noqa: C901
         from dvc.utils.hydra import to_hydra_overrides
 
         path_overrides = to_path_overrides(params)
+
+        if tmp_dir or queue:
+            untracked = repo.scm.untracked_files()
+            for path in path_overrides:
+                if path in untracked:
+                    logger.debug(
+                        "'%s' is currently untracked but will be modified by DVC. "
+                        "Adding it to git.",
+                        path,
+                    )
+                    repo.scm.add([path])
+
         hydra_sweep = any(
             x.is_sweep_override()
             for param_file in path_overrides
@@ -57,7 +70,11 @@ def run(  # noqa: C901
 
     if not queue:
         return repo.experiments.reproduce_one(
-            targets=targets, params=path_overrides, tmp_dir=tmp_dir, **kwargs
+            targets=targets,
+            params=path_overrides,
+            tmp_dir=tmp_dir,
+            copy_paths=copy_paths,
+            **kwargs,
         )
 
     if hydra_sweep:
@@ -78,6 +95,7 @@ def run(  # noqa: C901
             repo.experiments.celery_queue,
             targets=targets,
             params=sweep_overrides,
+            copy_paths=copy_paths,
             **kwargs,
         )
         if sweep_overrides:

dvc/repo/experiments/save.py

@@ -2,11 +2,7 @@ import logging
 import os
 from typing import TYPE_CHECKING, List, Optional
 
-from dvc.scm import Git
-
-from .exceptions import ExperimentExistsError
-from .refs import ExpRefInfo
-from .utils import check_ref_format, get_random_exp_name
+from funcy import first
 
 if TYPE_CHECKING:
     from dvc.repo import Repo
@@ -15,34 +11,6 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-def _save_experiment(
-    repo: "Repo",
-    baseline_rev: str,
-    force: bool,
-    name: Optional[str],
-    include_untracked: Optional[List[str]],
-) -> str:
-    repo.commit([], force=True)
-
-    name = name or get_random_exp_name(repo.scm, baseline_rev)
-    ref_info = ExpRefInfo(baseline_rev, name)
-    check_ref_format(repo.scm.dulwich, ref_info)
-    ref = str(ref_info)
-    if repo.scm.get_ref(ref) and not force:
-        raise ExperimentExistsError(ref_info.name, command="save")
-
-    assert isinstance(repo.scm, Git)
-
-    repo.scm.add([], update=True)
-    if include_untracked:
-        repo.scm.add(include_untracked)
-    repo.scm.commit(f"dvc: commit experiment {name}", no_verify=True)
-    exp_rev = repo.scm.get_rev()
-    repo.scm.set_ref(ref, exp_rev, old_ref=None)
-
-    return exp_rev
-
-
 def save(
     repo: "Repo",
     name: Optional[str] = None,
@@ -55,32 +23,16 @@ def save(
     """
     logger.debug("Saving workspace in %s", os.getcwd())
 
-    assert isinstance(repo.scm, Git)
+    queue = repo.experiments.workspace_queue
+    entry = repo.experiments.new(queue=queue, name=name, force=force)
+    executor = queue.init_executor(repo.experiments, entry)
 
-    _, _, untracked = repo.scm.status()
-    if include_untracked:
-        untracked = [file for file in untracked if file not in include_untracked]
-    if untracked:
-        logger.warning(
-            (
-                "The following untracked files were present in "
-                "the workspace before saving but "
-                "will not be included in the experiment commit:\n"
-                "\t%s"
-            ),
-            ", ".join(untracked),
+    try:
+        save_result = executor.save(
+            executor.info, force=force, include_untracked=include_untracked
         )
+        result = queue.collect_executor(repo.experiments, executor, save_result)
+    finally:
+        executor.cleanup()
 
-    with repo.scm.detach_head(client="dvc") as orig_head:
-        with repo.scm.stash_workspace() as workspace:
-            try:
-                if workspace is not None:
-                    repo.scm.stash.apply(workspace)
-
-                exp_rev = _save_experiment(
-                    repo, orig_head, force, name, include_untracked
-                )
-            finally:
-                repo.scm.reset(hard=True)
-
-    return exp_rev
+    return first(result)

dvc/repo/experiments/serialize.py

@@ -0,0 +1,164 @@
+import json
+from dataclasses import asdict, dataclass, field
+from datetime import datetime
+from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
+
+from dvc.exceptions import DvcException
+from dvc.repo.metrics.show import _gather_metrics
+from dvc.repo.params.show import _gather_params
+from dvc.utils import onerror_collect, relpath
+
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+
+
+class DeserializeError(DvcException):
+    pass
+
+
+class _ISOEncoder(json.JSONEncoder):
+    def default(self, o: object) -> Any:
+        if isinstance(o, datetime):
+            return o.isoformat()
+        return super().default(o)
+
+
+@dataclass(frozen=True)
+class SerializableExp:
+    """Serializable experiment state."""
+
+    rev: str
+    timestamp: Optional[datetime] = None
+    params: Dict[str, Any] = field(default_factory=dict)
+    metrics: Dict[str, Any] = field(default_factory=dict)
+    deps: Dict[str, "_ExpDep"] = field(default_factory=dict)
+    outs: Dict[str, "_ExpOut"] = field(default_factory=dict)
+    status: Optional[str] = None
+    executor: Optional[str] = None
+    error: Optional["SerializableError"] = None
+
+    @classmethod
+    def from_repo(
+        cls,
+        repo: "Repo",
+        rev: Optional[str] = None,
+        onerror: Optional[Callable] = None,
+        **kwargs,
+    ) -> "SerializableExp":
+        """Returns a SerializableExp from the current repo state.
+
+        Params, metrics, deps, outs are filled via repo fs/index, all other fields
+        should be passed via kwargs.
+        """
+        from dvc.dependency import ParamsDependency, RepoDependency
+
+        if not onerror:
+            onerror = onerror_collect
+
+        rev = rev or repo.get_rev()
+        assert rev
+        status: Optional[str] = kwargs.get("status")
+        # NOTE: _gather_params/_gather_metrics return defaultdict which is not
+        # supported in dataclasses.asdict() on all python releases
+        # see https://bugs.python.org/issue35540
+        params = dict(_gather_params(repo, onerror=onerror))
+        if status and status.lower() in ("queued", "failed"):
+            metrics: Dict[str, Any] = {}
+        else:
+            metrics = dict(
+                _gather_metrics(
+                    repo,
+                    targets=None,
+                    rev=rev[:7],
+                    recursive=False,
+                    onerror=onerror_collect,
+                )
+            )
+        return cls(
+            rev=rev,
+            params=params,
+            metrics=metrics,
+            deps={
+                relpath(dep.fs_path, repo.root_dir): _ExpDep(
+                    hash=dep.hash_info.value if dep.hash_info else None,
+                    size=dep.meta.size if dep.meta else None,
+                    nfiles=dep.meta.nfiles if dep.meta else None,
+                )
+                for dep in repo.index.deps
+                if not isinstance(dep, (ParamsDependency, RepoDependency))
+            },
+            outs={
+                relpath(out.fs_path, repo.root_dir): _ExpOut(
+                    hash=out.hash_info.value if out.hash_info else None,
+                    size=out.meta.size if out.meta else None,
+                    nfiles=out.meta.nfiles if out.meta else None,
+                    use_cache=out.use_cache,
+                    is_data_source=out.stage.is_data_source,
+                )
+                for out in repo.index.outs
+                if not (out.is_metric or out.is_plot)
+            },
+            **kwargs,
+        )
+
+    def dumpd(self) -> Dict[str, Any]:
+        return asdict(self)
+
+    def as_bytes(self) -> bytes:
+        return _ISOEncoder().encode(self.dumpd()).encode("utf-8")
+
+    @classmethod
+    def from_bytes(cls, data: bytes):
+        try:
+            parsed = json.loads(data)
+            if "timestamp" in parsed:
+                parsed["timestamp"] = datetime.fromisoformat(parsed["timestamp"])
+            return cls(**parsed)
+        except (TypeError, json.JSONDecodeError) as exc:
+            raise DeserializeError("failed to load SerializableExp") from exc
+
+    @property
+    def contains_error(self) -> bool:
+        return (
+            self.error is not None
+            or self.params.get("error")
+            or any(value.get("error") for value in self.params.values())
+            or self.metrics.get("error")
+            or any(value.get("error") for value in self.metrics.values())
+        )
+
+
+@dataclass(frozen=True)
+class _ExpDep:
+    hash: Optional[str]  # noqa: A003
+    size: Optional[int]
+    nfiles: Optional[int]
+
+
+@dataclass(frozen=True)
+class _ExpOut:
+    hash: Optional[str]  # noqa: A003
+    size: Optional[int]
+    nfiles: Optional[int]
+    use_cache: bool
+    is_data_source: bool
+
+
+@dataclass(frozen=True)
+class SerializableError:
+    msg: str
+    type: str = ""  # noqa: A003
+
+    def dumpd(self) -> Dict[str, Any]:
+        return asdict(self)
+
+    def as_bytes(self) -> bytes:
+        return json.dumps(self.dumpd()).encode("utf-8")
+
+    @classmethod
+    def from_bytes(cls, data: bytes):
+        try:
+            parsed = json.loads(data)
+            return cls(**parsed)
+        except (TypeError, json.JSONDecodeError) as exc:
+            raise DeserializeError("failed to load SerializableError") from exc

dvc/repo/experiments/show.py

@@ -18,12 +18,11 @@ from typing import (
 
 from scmrepo.exceptions import SCMError as InnerScmError
 
-from dvc.repo.metrics.show import _gather_metrics
-from dvc.repo.params.show import _gather_params
+from dvc.exceptions import DvcException
 from dvc.scm import Git, SCMError, iter_revs, resolve_rev
-from dvc.utils import error_handler, onerror_collect, relpath
 
 from .refs import ExpRefInfo
+from .serialize import SerializableError, SerializableExp
 
 if TYPE_CHECKING:
     from scmrepo.git.objects import GitCommit
@@ -40,85 +39,116 @@ class ExpStatus(Enum):
     Failed = 3
 
 
+class _CachedError(DvcException):
+    def __init__(self, msg, typ, *args):
+        super().__init__(msg, *args)
+        self.typ = typ
+
+
 def _is_scm_error(collected_exp: Dict[str, Any]) -> bool:
     if "error" in collected_exp and (
-        isinstance(collected_exp["error"], (SCMError, InnerScmError))
+        isinstance(collected_exp["error"], (_CachedError, SCMError, InnerScmError))
     ):
         return True
     return False
 
 
-def _show_onerror_collect(result: Dict, exception: Exception, *args, **kwargs):
-    onerror_collect(result, exception, *args, **kwargs)
-    result["data"] = {}
+def _format_exp(exp: SerializableExp) -> Dict[str, Any]:
+    # SerializableExp always includes error but we need to strip it from show
+    # output when it is false-y to maintain compatibility w/tools that consume
+    # json output and assume that "error" key presence means there was an error
+    exp_dict = exp.dumpd()
+    if "error" in exp_dict and not exp_dict["error"]:
+        del exp_dict["error"]
+    return {"data": exp_dict}
+
+
+def _format_error(error: SerializableError):
+    msg = error.msg or "None"
+    return {"data": {}, "error": _CachedError(msg, error.type)}
 
 
-@error_handler
 def collect_experiment_commit(
     repo: "Repo",
     exp_rev: str,
     status: ExpStatus = ExpStatus.Success,
-    param_deps=False,
-    running: Optional[Dict[str, Any]] = None,
-    onerror: Optional[Callable] = None,
+    param_deps: bool = False,
+    force: bool = False,
+    **kwargs,
 ) -> Dict[str, Any]:
-    from dvc.dependency import ParamsDependency, RepoDependency
+    cache = repo.experiments.cache
+    # TODO: support filtering serialized exp when param_deps is set
+    if exp_rev != "workspace" and not (force or param_deps):
+        cached_exp = cache.get(exp_rev)
+        if cached_exp:
+            if status == ExpStatus.Running or (
+                isinstance(cached_exp, SerializableExp)
+                and cached_exp.status == ExpStatus.Running.name
+            ):
+                # expire cached queued exp entry once we start running it
+                cache.delete(exp_rev)
+            elif isinstance(cached_exp, SerializableError):
+                return _format_error(cached_exp)
+            else:
+                return _format_exp(cached_exp)
+    try:
+        exp = _collect_from_repo(
+            repo,
+            exp_rev,
+            status=status,
+            param_deps=param_deps,
+            force=force,
+            **kwargs,
+        )
+        if not (
+            exp_rev == "workspace"
+            or status == ExpStatus.Running
+            or param_deps
+            or exp.contains_error
+        ):
+            cache.put(exp, force=True)
+        return _format_exp(exp)
+    except Exception as exc:  # noqa: BLE001, pylint: disable=broad-except
+        logger.debug("", exc_info=True)
+        error = SerializableError(str(exc), type(exc).__name__)
+        return _format_error(error)
 
-    result: Dict[str, Any] = defaultdict(dict)
+
+def _collect_from_repo(
+    repo: "Repo",
+    exp_rev: str,
+    status: ExpStatus = ExpStatus.Success,
+    running: Optional[Dict[str, Any]] = None,
+    **kwargs,
+) -> "SerializableExp":
     running = running or {}
-    for rev in repo.brancher(revs=[exp_rev]):
+    with repo.switch(exp_rev) as rev:
         if rev == "workspace":
-            if exp_rev != "workspace":
-                continue
-            result["timestamp"] = None
+            timestamp: Optional[datetime] = None
         else:
             commit = repo.scm.resolve_commit(rev)
-            result["timestamp"] = datetime.fromtimestamp(commit.commit_time)
+            timestamp = datetime.fromtimestamp(commit.commit_time)
 
-        params = _gather_params(repo, targets=None, deps=param_deps, onerror=onerror)
-        if params:
-            result["params"] = params
-
-        result["deps"] = {
-            relpath(dep.fs_path, repo.root_dir): {
-                "hash": dep.hash_info.value,
-                "size": dep.meta.size,
-                "nfiles": dep.meta.nfiles,
-            }
-            for dep in repo.index.deps
-            if not isinstance(dep, (ParamsDependency, RepoDependency))
-        }
-        result["outs"] = {
-            relpath(out.fs_path, repo.root_dir): {
-                "hash": out.hash_info.value,
-                "size": out.meta.size,
-                "nfiles": out.meta.nfiles,
-                "use_cache": out.use_cache,
-                "is_data_source": out.stage.is_data_source,
-            }
-            for out in repo.index.outs
-            if not (out.is_metric or out.is_plot)
-        }
-
-        result["status"] = status.name
         if status == ExpStatus.Running:
-            result["executor"] = running.get(exp_rev, {}).get("location", None)
+            executor: Optional[str] = running.get(exp_rev, {}).get("location", None)
         else:
-            result["executor"] = None
+            executor = None
 
         if status == ExpStatus.Failed:
-            result["error"] = {
-                "msg": "Experiment run failed.",
-                "type": "",
-            }
-
-        if status not in {ExpStatus.Queued, ExpStatus.Failed}:
-            vals = _gather_metrics(
-                repo, targets=None, rev=rev, recursive=False, onerror=onerror
+            error: Optional["SerializableError"] = SerializableError(
+                "Experiment run failed."
             )
-            result["metrics"] = vals
+        else:
+            error = None
 
-    return result
+        return SerializableExp.from_repo(
+            repo,
+            rev=exp_rev,
+            timestamp=timestamp,
+            status=status.name,
+            executor=executor,
+            error=error,
+        )
 
 
 def _collect_complete_experiment(
@@ -170,6 +200,7 @@ def _collect_branch(
     repo: "Repo",
     baseline: str,
     running: Dict[str, Any],
+    refs: Optional[Iterable[str]] = None,
     **kwargs,
 ) -> Dict[str, Dict[str, Any]]:
     results: Dict[str, Dict[str, Any]] = OrderedDict()
@@ -188,7 +219,11 @@ def _collect_branch(
     commits: List[Tuple[str, "GitCommit", str, List[str]]] = []
 
     assert isinstance(repo.scm, Git)
-    for ref in repo.scm.iter_refs(base=str(ref_info)):
+    if refs:
+        ref_it = (ref for ref in iter(refs) if ref.startswith(str(ref_info)))
+    else:
+        ref_it = repo.scm.iter_refs(base=str(ref_info))
+    for ref in ref_it:
         try:
             commit = repo.scm.resolve_commit(ref)
             exp_rev = resolve_rev(repo.scm, ref)
@@ -217,19 +252,22 @@ def _collect_branch(
 
 
 def get_branch_names(
-    scm: "Git", baseline_set: Iterable[str]
+    scm: "Git", baseline_set: Iterable[str], refs: Optional[Iterable[str]] = None
 ) -> Dict[str, Optional[str]]:
     names: Dict[str, Optional[str]] = {}
-    for base in [
+    bases = [
         f"refs/exps/{baseline[:2]}/{baseline[2:]}/" for baseline in baseline_set
-    ] + ["refs/heads/", "refs/tags/"]:
-        for ref in scm.iter_refs(base=base):
-            if ref:
+    ] + ["refs/heads/", "refs/tags/"]
+    ref_it = iter(refs) if refs else scm.iter_refs()
+    for ref in ref_it:
+        for base in bases:
+            if ref.startswith(base):
                 try:
                     rev = scm.get_ref(ref)
                     names[rev] = ref[len(base) :]
                 except KeyError:
-                    logger.debug("unreosolved ref %s", ref)
+                    logger.debug("unresolved ref %s", ref)
+                break
     logger.debug("found refs for revs: %s", names)
     return names
 
@@ -377,13 +415,12 @@ def move_properties_to_head(result: Dict[str, Dict[str, Dict[str, Any]]]):
                 if rev_result["status"] == ExpStatus.Running.name:
                     head["status"] = ExpStatus.Running.name
                     rev_result["status"] = ExpStatus.Success.name
-            else:
-                if rev_result["checkpoint_tip"] == rev:
-                    head = rev_result
-                    checkpoint = True
+            elif rev_result["checkpoint_tip"] == rev:
+                head = rev_result
+                checkpoint = True
 
 
-def show(
+def show(  # noqa: PLR0913
     repo: "Repo",
     all_branches=False,
     all_tags=False,
@@ -396,12 +433,11 @@ def show(
     param_deps=False,
     onerror: Optional[Callable] = None,
     fetch_running: bool = True,
+    force: bool = False,
 ):
     if repo.scm.no_commits:
         return {}
 
-    onerror = onerror or _show_onerror_collect
-
     res: Dict[str, Dict] = defaultdict(OrderedDict)
 
     if not any([revs, all_branches, all_tags, all_commits]):
@@ -415,7 +451,8 @@ def show(
     found_revs.update(
         iter_revs(repo.scm, revs, num, all_branches, all_tags, all_commits)
     )
-    branch_names = get_branch_names(repo.scm, found_revs)
+    cached_refs = list(repo.scm.iter_refs())
+    branch_names = get_branch_names(repo.scm, found_revs, refs=cached_refs)
 
     running: Dict[str, Dict] = repo.experiments.get_running_exps(
         fetch_refs=fetch_running
@@ -427,7 +464,7 @@ def show(
             found_revs,
             running,
             param_deps=param_deps,
-            onerror=onerror,
+            force=force,
         )
         if not hide_queued
         else {}
@@ -439,6 +476,7 @@ def show(
         running,
         param_deps=param_deps,
         onerror=onerror,
+        force=force,
     )
 
     failed_experiments = (
@@ -448,6 +486,7 @@ def show(
             running,
             param_deps=param_deps,
             onerror=onerror,
+            force=force,
         )
         if not hide_failed
         else {}
@@ -460,6 +499,8 @@ def show(
             running=running,
             param_deps=param_deps,
             onerror=onerror,
+            force=force,
+            refs=cached_refs,
         )
 
     update_new(res, failed_experiments)

dvc/repo/experiments/stash.py

@@ -1,8 +1,18 @@
+import logging
 import re
-from typing import Dict, Iterable, NamedTuple, Optional
+from contextlib import contextmanager
+from typing import Dict, Iterable, Iterator, NamedTuple, Optional
 
 from scmrepo.git import Stash
 
+from dvc.exceptions import DvcException
+from dvc_objects.fs.local import localfs
+from dvc_objects.fs.utils import as_atomic
+
+from .refs import APPLY_HEAD, APPLY_STASH
+
+logger = logging.getLogger(__name__)
+
 
 class ExpStashEntry(NamedTuple):
     """Experiment stash entry.
@@ -72,3 +82,122 @@ class ExpStash(Stash):
             reverse=True,
         ):
             self.drop(index)
+
+
+class ApplyStashEntry(NamedTuple):
+    """Apply stash entry.
+
+    stash_index: Stash index for this entry. Can be None if this commit
+        is not pushed onto the stash ref.
+    head_rev: HEAD Git commit prior to exp apply.
+    rev: Applied experiment commit.
+    name: Optional applied exp name.
+    """
+
+    stash_index: Optional[int]
+    head_rev: str
+    rev: str
+    name: Optional[str]
+
+
+class ApplyStash(Stash):
+    DEFAULT_STASH = APPLY_STASH
+    MESSAGE_FORMAT = "dvc-exp-apply:{head_rev}:{rev}:{name}"
+    MESSAGE_RE = re.compile(
+        r"(?:commit: )"
+        r"dvc-exp-apply:(?P<head_rev>[0-9a-f]+):(?P<rev>[0-9a-f]+)"
+        r":(?P<name>[^~^:\\?\[\]*]*)"
+    )
+
+    @property
+    def stash_revs(self) -> Dict[str, ApplyStashEntry]:
+        revs = {}
+        for i, entry in enumerate(self):
+            msg = entry.message.decode("utf-8").strip()
+            m = self.MESSAGE_RE.match(msg)
+            if m:
+                revs[entry.new_sha.decode("utf-8")] = ApplyStashEntry(
+                    i,
+                    m.group("head_rev"),
+                    m.group("rev"),
+                    m.group("name"),
+                )
+        return revs
+
+    @classmethod
+    def format_message(
+        cls,
+        head_rev: str,
+        rev: str,
+        name: Optional[str] = None,
+    ) -> str:
+        return cls.MESSAGE_FORMAT.format(
+            head_rev=head_rev, rev=rev, name=name if name else ""
+        )
+
+    @contextmanager
+    def preserve_workspace(
+        self, rev: str, name: Optional[str] = None
+    ) -> Iterator[Optional[str]]:
+        if len(self):
+            logger.debug("Clearing existing exp-apply stash")
+            self.clear()
+        head = self.scm.get_rev()
+        self.scm.set_ref(APPLY_HEAD, head)
+        message = self.format_message(head, rev, name=name)
+        stash_rev = self.push(message=message, include_untracked=True)
+        try:
+            yield stash_rev
+            if stash_rev:
+                self._apply_difference(stash_rev, rev)
+        except Exception:  # pylint: disable=broad-except
+            self.revert_workspace()
+            raise
+
+    def _apply_difference(self, stash_rev: str, rev: str):
+        """Selectively apply changes from stash_rev.
+
+        Only changes to files from left which do not exist in right will be applied.
+        """
+        self._copy_difference(stash_rev, rev)
+        commit = self.scm.resolve_commit(stash_rev)
+        for parent_rev in commit.parents:
+            parent_commit = self.scm.resolve_commit(parent_rev)
+            if parent_commit.message.startswith("untracked files on "):
+                self._copy_difference(parent_rev, rev)
+
+    def _copy_difference(self, left_rev: str, right_rev: str):
+        left_fs = self.scm.get_fs(left_rev)
+        right_fs = self.scm.get_fs(right_rev)
+        paths = [path for path in left_fs.find("/") if not right_fs.exists(path)]
+        dest_paths = [
+            localfs.path.join(self.scm.root_dir, left_fs.path.relpath(path, "/"))
+            for path in paths
+        ]
+        for src, dest in zip(paths, dest_paths):
+            with as_atomic(localfs, dest, create_parents=True) as tmp_file:
+                left_fs.get_file(src, tmp_file)
+
+    def revert_workspace(self):
+        apply_head = self.scm.get_ref(self.ref)
+        head = self.scm.get_rev()
+        if apply_head != head:
+            raise DvcException(
+                f"Cannot revert workspace, current HEAD '{head[:7]}' does not match the"
+                f" pre-apply HEAD '{apply_head[:7]}'"
+            )
+        self.scm.reset(hard=True)
+        if len(self):
+            # In the event that the apply-stash and current workspace contain
+            # conflicting untracked files, we do:
+            #   1. stash the current untracked files
+            #   2. restore/pop the apply-stash (with untracked files)
+            #   3. restore/pop the untracked files from (1) and ignore any conflicts
+            #      (forcefully reverting to the apply-stash version)
+            workspace_rev = self.scm.stash.push(include_untracked=True)
+            try:
+                self.pop()
+            finally:
+                if workspace_rev:
+                    self.scm.stash.pop(skip_conflicts=True)
+        self.scm.remove_ref(self.ref)

dvc/repo/fetch.py

@@ -19,7 +19,7 @@ logger = logging.getLogger(__name__)
 
 
 @locked
-def fetch(  # noqa: C901
+def fetch(  # noqa: C901, PLR0913
     self,
     targets=None,
     jobs=None,