Update dvc/commands/imp_url.py

Per https://github.com/iterative/dvc.org/pull/3934

.github/workflows/tests.yaml

@@ -8,11 +8,6 @@ on:
     - cron: '5 1 * * *'  # every day at 01:05
   workflow_dispatch:
 
-env:
-  DVC_TEST: "true"
-  HOMEBREW_NO_AUTO_UPDATE: 1
-  SHELL: /bin/bash
-
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
@@ -71,8 +66,11 @@ jobs:
         pip install --upgrade pip wheel
         pip install -e ".[dev]"
     - name: run tests
-      timeout-minutes: 20
-      run: pytest -nauto --durations 100 --cov --cov-report=xml --cov-report=term -k "${{ matrix.pytest-filter }}"
+      timeout-minutes: 30
+      run: >
+        pytest -nauto --timeout=300 --durations=100
+        --cov --cov-report=xml --cov-report=term
+        -k "${{ matrix.pytest-filter }}"
     - name: upload coverage report
       uses: codecov/codecov-action@v3
       with:

dvc/cli/parser.py

@@ -2,8 +2,8 @@
 import argparse
 import logging
 import os
-import sys
 
+from dvc import __version__
 from dvc.commands import (
     add,
     cache,
@@ -128,17 +128,6 @@ class DvcParser(argparse.ArgumentParser):
         return args
 
 
-class VersionAction(argparse.Action):  # pragma: no cover
-    # pylint: disable=too-few-public-methods
-    """Shows DVC version and exits."""
-
-    def __call__(self, parser, namespace, values, option_string=None):
-        from dvc import __version__
-
-        print(__version__)
-        sys.exit(0)
-
-
 def get_parent_parser():
     """Create instances of a parser containing common arguments shared among
     all the commands.
@@ -186,13 +175,11 @@ def get_main_parser():
         help="Show this help message and exit.",
     )
 
-    # NOTE: On some python versions action='version' prints to stderr
-    # instead of stdout https://bugs.python.org/issue18920
     parser.add_argument(
         "-V",
         "--version",
-        action=VersionAction,
-        nargs=0,
+        action="version",
+        version=__version__,
         help="Show program's version.",
     )
 

dvc/commands/experiments/__init__.py

@@ -18,6 +18,7 @@ from dvc.commands.experiments import (
     queue_worker,
     remove,
     run,
+    save,
     show,
 )
 
@@ -34,6 +35,7 @@ SUB_COMMANDS = [
     queue_worker,
     remove,
     run,
+    save,
     show,
 ]
 

dvc/commands/experiments/ls.py

@@ -3,6 +3,7 @@ import logging
 
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
@@ -16,22 +17,13 @@ class CmdExperimentsList(CmdBase):
             num=self.args.num,
             git_remote=self.args.git_remote,
         )
-        tags = self.repo.scm.describe(exps)
-        remained = {baseline for baseline, tag in tags.items() if tag is None}
-        base = "refs/heads/"
-        ref_heads = self.repo.scm.describe(remained, base=base)
 
         for baseline in exps:
-            name = (
-                tags[baseline]
-                or ref_heads[baseline][len(base) :]
-                or baseline[:7]
-            )
             if not name_only:
-                print(f"{name}:")
+                ui.write(f"{baseline}:")
             for exp_name in exps[baseline]:
                 indent = "" if name_only else "\t"
-                print(f"{indent}{exp_name}")
+                ui.write(f"{indent}{exp_name}")
 
         return 0
 

dvc/commands/experiments/pull.py

@@ -68,7 +68,7 @@ def add_parser(experiments_subparsers, parent_parser):
         "-f",
         "--force",
         action="store_true",
-        help="Replace local experiment already exists.",
+        help="Replace local experiment if it already exists.",
     )
     experiments_pull_parser.add_argument(
         "--no-cache",

dvc/commands/experiments/save.py

@@ -0,0 +1,79 @@
+import argparse
+import logging
+
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.exceptions import DvcException
+from dvc.ui import ui
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsSave(CmdBase):
+    def run(self):
+
+        try:
+            ref = self.repo.experiments.save(
+                name=self.args.name,
+                force=self.args.force,
+                include_untracked=self.args.include_untracked,
+            )
+        except DvcException:
+            logger.exception("failed to save experiment")
+            return 1
+
+        if self.args.json:
+            ui.write_json({"ref": ref})
+        else:
+            name = self.repo.experiments.get_exact_name([ref])[ref]
+            ui.write(f"Experiment has been saved as: {name}")
+            ui.write(
+                "\nTo promote an experiment to a Git branch run:\n\n"
+                "\tdvc exp branch <exp> <branch>\n"
+            )
+
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+    EXPERIMENTS_SAVE_HELP = "Save current workspace as a dvc experiment."
+    save_parser = experiments_subparsers.add_parser(
+        "save",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_SAVE_HELP, "exp/save"),
+        help=EXPERIMENTS_SAVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    save_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Replace experiment if it already exists.",
+    )
+    save_parser.add_argument(
+        "--json",
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    save_parser.add_argument(
+        "-n",
+        "--name",
+        default=None,
+        help=(
+            "Human-readable experiment name. If not specified, a name will "
+            "be auto-generated."
+        ),
+        metavar="<name>",
+    )
+    save_parser.add_argument(
+        "-I",
+        "--include-untracked",
+        action="append",
+        default=[],
+        help="List of untracked paths to include in the experiment.",
+        metavar="<path>",
+    )
+    save_parser.set_defaults(func=CmdExperimentsSave)

dvc/commands/get.py

@@ -4,6 +4,7 @@ import logging
 from dvc.cli import completion
 from dvc.cli.command import CmdBaseNoRepo
 from dvc.exceptions import DvcException
+from dvc.scm import CloneError
 
 from ..cli.utils import append_doc_link
 
@@ -44,6 +45,9 @@ class CmdGet(CmdBaseNoRepo):
                 jobs=self.args.jobs,
             )
             return 0
+        except CloneError:
+            logger.exception(f"failed to get '{self.args.path}'")
+            return 1
         except DvcException:
             logger.exception(
                 "failed to get '{}' from '{}'".format(

dvc/commands/imp.py

@@ -5,6 +5,7 @@ from dvc.cli import completion
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
 from dvc.exceptions import DvcException
+from dvc.scm import CloneError
 
 logger = logging.getLogger(__name__)
 
@@ -26,6 +27,9 @@ class CmdImport(CmdBase):
                 meta=self.args.meta,
                 jobs=self.args.jobs,
             )
+        except CloneError:
+            logger.exception(f"failed to import '{self.args.path}'")
+            return 1
         except DvcException:
             logger.exception(
                 "failed to import '{}' from '{}'.".format(

dvc/commands/imp_url.py

@@ -73,6 +73,18 @@ def add_parser(subparsers, parent_parser):
         help="Specify name of the .dvc file this command will generate.",
         metavar="<filename>",
     ).complete = completion.DIR
+    import_parser.add_argument(
+        "--to-remote",
+        action="store_true",
+        default=False,
+        help="Download it directly to the remote",
+    )
+    import_parser.add_argument(
+        "-r",
+        "--remote",
+        help="Remote storage to download to",
+        metavar="<name>",
+    )
     no_download_exec_group = import_parser.add_mutually_exclusive_group()
     no_download_exec_group.add_argument(
         "--no-exec",
@@ -87,18 +99,6 @@ def add_parser(subparsers, parent_parser):
         help="Create .dvc file including target data hash value(s)"
         " but do not actually download the file(s).",
     )
-    import_parser.add_argument(
-        "--to-remote",
-        action="store_true",
-        default=False,
-        help="Download it directly to the remote",
-    )
-    import_parser.add_argument(
-        "-r",
-        "--remote",
-        help="Remote storage to download to",
-        metavar="<name>",
-    )
     import_parser.add_argument(
         "-j",
         "--jobs",

dvc/commands/queue/kill.py

@@ -11,13 +11,18 @@ class CmdQueueKill(CmdBase):
     """Kill exp task in queue."""
 
     def run(self):
-        self.repo.experiments.celery_queue.kill(revs=self.args.task)
+        self.repo.experiments.celery_queue.kill(
+            revs=self.args.task, force=self.args.force
+        )
 
         return 0
 
 
 def add_parser(queue_subparsers, parent_parser):
-    QUEUE_KILL_HELP = "Kill actively running experiment queue tasks."
+    QUEUE_KILL_HELP = (
+        "Gracefully interrupt running experiment queue tasks "
+        "(equivalent to Ctrl-C)"
+    )
     queue_kill_parser = queue_subparsers.add_parser(
         "kill",
         parents=[parent_parser],
@@ -25,6 +30,13 @@ def add_parser(queue_subparsers, parent_parser):
         help=QUEUE_KILL_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    queue_kill_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Forcefully and immediately kill running experiment queue tasks",
+    )
     queue_kill_parser.add_argument(
         "task",
         nargs="*",

dvc/commands/update.py

@@ -31,7 +31,8 @@ class CmdUpdate(CmdBase):
 def add_parser(subparsers, parent_parser):
     UPDATE_HELP = (
         "Update data artifact imported (via dvc import or dvc import-url) "
-        "from an external DVC repository or URL."
+        "from an external DVC repository or URL, or update data from a "
+        "DVC worktree remote."
     )
     update_parser = subparsers.add_parser(
         "update",

dvc/config_schema.py

@@ -97,6 +97,8 @@ HTTP_COMMON = {
     "ask_password": Bool,
     "ssl_verify": Any(Bool, str),
     "method": str,
+    "connect_timeout": All(Coerce(float), Range(0, min_included=True)),
+    "read_timeout": All(Coerce(float), Range(0, min_included=True)),
     Optional("verify", default=False): Bool,
 }
 WEBDAV_COMMON = {

dvc/data_cloud.py

@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Iterable, Optional
 
 from funcy import cached_property
 
+from dvc.config import NoRemoteError, RemoteConfigError
 from dvc_data.hashfile.db import get_index
 
 if TYPE_CHECKING:
@@ -23,6 +24,14 @@ class Remote:
 
         self.worktree = config.pop("worktree", False)
         self.config = config
+        if self.worktree:
+            version_aware = self.config.get("version_aware")
+            if version_aware is False:
+                raise RemoteConfigError(
+                    "worktree remotes require version_aware cloud"
+                )
+            if version_aware is None:
+                self.fs.version_aware = True
 
     @cached_property
     def odb(self):
@@ -54,8 +63,6 @@ class DataCloud:
         name: Optional[str] = None,
         command: str = "<command>",
     ) -> "Remote":
-        from dvc.config import NoRemoteError
-
         if not name:
             name = self.repo.config["core"].get("remote")
 
@@ -88,6 +95,10 @@ class DataCloud:
         command: str = "<command>",
     ) -> "HashFileDB":
         remote = self.get_remote(name=name, command=command)
+        if remote.fs.version_aware or remote.worktree:
+            raise NoRemoteError(
+                f"'{command}' is unsupported for cloud versioned remotes"
+            )
         return remote.odb
 
     def _log_missing(self, status: "CompareStatusResult"):

dvc/dvcfile.py

@@ -1,7 +1,18 @@
 import contextlib
 import logging
 import os
-from typing import TYPE_CHECKING, Any, Callable, Tuple, TypeVar, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    List,
+    Tuple,
+    TypeVar,
+    Union,
+)
+
+from funcy import cached_property
 
 from dvc.exceptions import DvcException
 from dvc.parsing.versions import LOCKFILE_VERSION, SCHEMA_KWD
@@ -180,6 +191,10 @@ class SingleStageFile(FileMixin):
     from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
     from dvc.stage.loader import SingleStageLoader as LOADER
 
+    metrics: List[str] = []
+    plots: Dict[str, Any] = {}
+    params: List[str] = []
+
     @property
     def stage(self):
         data, raw = self._load()
@@ -223,6 +238,11 @@ class PipelineFile(FileMixin):
     def _lockfile(self):
         return Lockfile(self.repo, os.path.splitext(self.path)[0] + ".lock")
 
+    def _reset(self):
+        self.__dict__.pop("contents", None)
+        self.__dict__.pop("lockfile_contents", None)
+        self.__dict__.pop("resolver", None)
+
     def dump(
         self, stage, update_pipeline=True, update_lock=True, **kwargs
     ):  # pylint: disable=arguments-differ
@@ -278,11 +298,36 @@ class PipelineFile(FileMixin):
             "PipelineFile has multiple stages. Please specify it's name."
         )
 
+    @cached_property
+    def contents(self):
+        return self._load()[0]
+
+    @cached_property
+    def lockfile_contents(self):
+        return self._lockfile.load()
+
+    @cached_property
+    def resolver(self):
+        from .parsing import DataResolver
+
+        wdir = self.repo.fs.path.parent(self.path)
+        return DataResolver(self.repo, wdir, self.contents)
+
     @property
     def stages(self):
-        data, _ = self._load()
-        lockfile_data = self._lockfile.load()
-        return self.LOADER(self, data, lockfile_data)
+        return self.LOADER(self, self.contents, self.lockfile_contents)
+
+    @property
+    def metrics(self):
+        return self.contents.get("metrics", [])
+
+    @property
+    def plots(self):
+        return self.contents.get("plots", {})
+
+    @property
+    def params(self):
+        return self.contents.get("params", [])
 
     def remove(self, force=False):
         if not force:

dvc/env.py

@@ -4,6 +4,8 @@ DVC_PAGER = "DVC_PAGER"
 DVC_ROOT = "DVC_ROOT"
 DVCLIVE_RESUME = "DVCLIVE_RESUME"
 DVC_IGNORE_ISATTY = "DVC_IGNORE_ISATTY"
+DVC_EXP_BASELINE_REV = "DVC_EXP_BASELINE_REV"
+DVC_EXP_NAME = "DVC_EXP_NAME"
 DVC_EXP_GIT_REMOTE = "DVC_EXP_GIT_REMOTE"
 DVC_EXP_AUTO_PUSH = "DVC_EXP_AUTO_PUSH"
 DVC_NO_ANALYTICS = "DVC_NO_ANALYTICS"

dvc/fs/dvc.py

@@ -276,43 +276,38 @@ class _DVCFileSystem(AbstractFileSystem):  # pylint:disable=abstract-method
         key = self._get_key_from_relative(path)
         repo, dvc_fs, subkey = self._get_subrepo_info(key)
 
-        names = set()
+        dvc_infos = {}
         if dvc_fs:
             with suppress(FileNotFoundError):
                 dvc_path = _get_dvc_path(dvc_fs, subkey)
-                for entry in dvc_fs.ls(dvc_path, detail=False):
-                    names.add(dvc_fs.path.name(entry))
+                for info in dvc_fs.ls(dvc_path, detail=True):
+                    dvc_infos[dvc_fs.path.name(info["name"])] = info
 
+        fs_infos = {}
         ignore_subrepos = kwargs.get("ignore_subrepos", True)
         if not dvc_only:
             fs = self.repo.fs
             fs_path = self._from_key(key)
             try:
-                for entry in repo.dvcignore.ls(
-                    fs, fs_path, detail=False, ignore_subrepos=ignore_subrepos
+                for info in repo.dvcignore.ls(
+                    fs, fs_path, detail=True, ignore_subrepos=ignore_subrepos
                 ):
-                    names.add(fs.path.name(entry))
+                    fs_infos[fs.path.name(info["name"])] = info
             except (FileNotFoundError, NotADirectoryError):
                 pass
 
         dvcfiles = kwargs.get("dvcfiles", False)
-        if not dvcfiles:
-            names = (name for name in names if not _is_dvc_file(name))
 
         infos = []
         paths = []
+        names = set(dvc_infos.keys()) | set(fs_infos.keys())
         for name in names:
-            entry_path = self.path.join(path, name)
-            entry_key = key + (name,)
-            try:
-                info = self._info(
-                    entry_key,
-                    entry_path,
-                    ignore_subrepos=ignore_subrepos,
-                    check_ignored=False,
-                )
-            except FileNotFoundError:
+            if not dvcfiles and _is_dvc_file(name):
                 continue
+
+            entry_path = self.path.join(path, name)
+            info = _merge_info(repo, fs_infos.get(name), dvc_infos.get(name))
+            info["name"] = entry_path
             infos.append(info)
             paths.append(entry_path)
 

dvc/output.py

@@ -90,6 +90,7 @@ def loadd_from(stage, d_list):
         remote = d.pop(Output.PARAM_REMOTE, None)
         annot = {field: d.pop(field, None) for field in ANNOTATION_FIELDS}
         files = d.pop(Output.PARAM_FILES, None)
+        push = d.pop(Output.PARAM_PUSH, True)
         ret.append(
             _get(
                 stage,
@@ -103,6 +104,7 @@ def loadd_from(stage, d_list):
                 remote=remote,
                 **annot,
                 files=files,
+                push=push,
             )
         )
     return ret
@@ -117,6 +119,7 @@ def loads_from(
     persist=False,
     checkpoint=False,
     remote=None,
+    push=True,
 ):
     return [
         _get(
@@ -129,6 +132,7 @@ def loads_from(
             persist=persist,
             checkpoint=checkpoint,
             remote=remote,
+            push=push,
         )
         for s in s_list
     ]
@@ -184,6 +188,7 @@ def load_from_pipeline(stage, data, typ="outs"):
                 Output.PARAM_PERSIST,
                 Output.PARAM_CHECKPOINT,
                 Output.PARAM_REMOTE,
+                Output.PARAM_PUSH,
                 *ANNOTATION_FIELDS,
             ],
         )
@@ -251,6 +256,7 @@ class Output:
     PARAM_PLOT_HEADER = "header"
     PARAM_PERSIST = "persist"
     PARAM_REMOTE = "remote"
+    PARAM_PUSH = "push"
 
     METRIC_SCHEMA = Any(
         None,
@@ -284,6 +290,7 @@ class Output:
         repo=None,
         fs_config=None,
         files: List[Dict[str, Any]] = None,
+        push: bool = True,
     ):
         self.annot = Annotation(
             desc=desc, type=type, labels=labels or [], meta=meta or {}
@@ -338,6 +345,7 @@ class Output:
         self.plot = False if self.IS_DEPENDENCY else plot
         self.persist = persist
         self.checkpoint = checkpoint
+        self.can_push = push
 
         self.fs_path = self._parse_path(self.fs, fs_path)
         self.obj: Optional["HashFile"] = None
@@ -776,6 +784,9 @@ class Output:
             if self.remote:
                 ret[self.PARAM_REMOTE] = self.remote
 
+            if not self.can_push:
+                ret[self.PARAM_PUSH] = self.can_push
+
         if (
             (not self.IS_DEPENDENCY or self.stage.is_import)
             and self.hash_info.isdir
@@ -1032,9 +1043,15 @@ class Output:
         if not self.use_cache:
             return {}
 
+        push: bool = kwargs.pop("push", False)
         if self.stage.is_repo_import:
+            if push:
+                return {}
             return self.get_used_external(**kwargs)
 
+        if push and not self.can_push:
+            return {}
+
         if not self.hash_info:
             msg = (
                 "Output '{}'({}) is missing version info. "
@@ -1180,6 +1197,34 @@ class Output:
     def is_plot(self) -> bool:
         return bool(self.plot)
 
+    def restore_fields(self, other: "Output"):
+        """Restore attributes that need to be preserved when serialized."""
+        self.annot = other.annot
+        self.remote = other.remote
+        self.can_push = other.can_push
+
+    def merge_version_meta(self, other: "Output"):
+        """Merge version meta for files which are unchanged from other."""
+        if not self.hash_info:
+            return
+        if self.hash_info.isdir:
+            return self._merge_dir_version_meta(other)
+        if self.hash_info != other.hash_info:
+            return
+        self.meta = other.meta
+
+    def _merge_dir_version_meta(self, other: "Output"):
+        from dvc_data.hashfile.tree import update_meta
+
+        if not self.obj or not other.hash_info.isdir:
+            return
+        other_obj = other.obj if other.obj is not None else other.get_obj()
+        assert isinstance(self.obj, Tree) and isinstance(other_obj, Tree)
+        updated = update_meta(self.obj, other_obj)
+        assert updated.hash_info == self.obj.hash_info
+        self.obj = updated
+        self.files = updated.as_list(with_meta=True)
+
 
 META_SCHEMA = {
     Meta.PARAM_SIZE: int,
@@ -1209,5 +1254,6 @@ SCHEMA = {
     Output.PARAM_CACHE: bool,
     Output.PARAM_METRIC: Output.METRIC_SCHEMA,
     Output.PARAM_REMOTE: str,
+    Output.PARAM_PUSH: bool,
     Output.PARAM_FILES: [DIR_FILES_SCHEMA],
 }

dvc/parsing/interpolate.py

@@ -1,3 +1,4 @@
+import os
 import re
 import typing
 from collections.abc import Iterable, Mapping
@@ -70,6 +71,16 @@ def embrace(s: str):
     return BRACE_OPEN + s + BRACE_CLOSE
 
 
+def escape_str(value):
+    if os.name == "nt":
+        from subprocess import list2cmdline
+
+        return list2cmdline([value])
+    from shlex import quote
+
+    return quote(value)
+
+
 @singledispatch
 def to_str(obj) -> str:
     return str(obj)
@@ -97,12 +108,12 @@ def _(obj: dict):
                     result += f"--no-{k} "
 
         elif isinstance(v, str):
-            result += f"--{k} '{v}' "
+            result += f"--{k} {escape_str(v)} "
 
         elif isinstance(v, Iterable):
             for n, i in enumerate(v):
                 if isinstance(i, str):
-                    i = f"'{i}'"
+                    i = escape_str(i)
                 elif isinstance(i, Iterable):
                     raise ParseError(
                         f"Cannot interpolate nested iterable in '{k}'"

dvc/render/convert.py

@@ -14,14 +14,14 @@ from dvc.render.converter.vega import VegaConverter
 
 
 def _get_converter(
-    renderer_class, props
+    renderer_class, renderer_id, props, data
 ) -> Union[VegaConverter, ImageConverter]:
     from dvc_render import ImageRenderer, VegaRenderer
 
     if renderer_class.TYPE == VegaRenderer.TYPE:
-        return VegaConverter(props)
+        return VegaConverter(renderer_id, data, props)
     if renderer_class.TYPE == ImageRenderer.TYPE:
-        return ImageConverter(props)
+        return ImageConverter(renderer_id, data, props)
 
     raise ValueError(f"Invalid renderer class {renderer_class}")
 

dvc/render/converter/__init__.py

@@ -1,9 +1,19 @@
-from typing import Dict, Optional
+from typing import Any, Dict, List, Optional, Tuple
 
 
 class Converter:
-    def __init__(self, plot_properties: Optional[Dict] = None):
-        self.plot_properties = plot_properties or {}
+    def __init__(
+        self,
+        plot_id: str,
+        data: Dict[str, Any] = None,
+        properties: Optional[Dict] = None,
+    ):
+        self.plot_id = plot_id
+        self.properties = properties or {}
+        self.data = data or {}
 
-    def convert(self, data, revision: str, filename: str, **kwargs):
+    def convert(self) -> Tuple[List[Tuple[str, str, Any]], Dict]:
+        raise NotImplementedError
+
+    def flat_datapoints(self, revision: str) -> Tuple[List[Dict], Dict]:
         raise NotImplementedError

dvc/render/converter/image.py

@@ -1,6 +1,6 @@
 import base64
 import os
-from typing import TYPE_CHECKING, Dict, List, Tuple
+from typing import TYPE_CHECKING, Any, Dict, List, Tuple
 
 from dvc.render import FILENAME_FIELD, REVISION_FIELD, SRC_FIELD
 
@@ -34,25 +34,33 @@ class ImageConverter(Converter):
         base64_str = base64.b64encode(image_data).decode()
         return f"data:image;base64,{base64_str}"
 
-    def convert(
-        self, data, revision: str, filename: str, **kwargs
-    ) -> Tuple[List[Dict], Dict]:
+    def convert(self) -> Tuple[List[Tuple[str, str, Any]], Dict]:
+        datas = []
+        for filename, image_data in self.data.items():
+            datas.append((filename, "", image_data))
+        return datas, self.properties
+
+    def flat_datapoints(self, revision: str) -> Tuple[List[Dict], Dict]:
         """
         Convert the DVC Plots content to DVC Render datapoints.
         Return both generated datapoints and updated properties.
         """
-        path = self.plot_properties.get("out")
-        if path:
-            if not os.path.isdir(path):
-                os.makedirs(path, exist_ok=True)
-            src = self._write_image(
-                os.path.abspath(path), revision, filename, data
-            )
-        else:
-            src = self._encode_image(data)
-        datapoint = {
-            REVISION_FIELD: revision,
-            FILENAME_FIELD: filename,
-            SRC_FIELD: src,
-        }
-        return [datapoint], self.plot_properties
+        path = self.properties.get("out")
+        datapoints = []
+        datas, properties = self.convert()
+        for filename, _, image_data in datas:
+            if path:
+                if not os.path.isdir(path):
+                    os.makedirs(path, exist_ok=True)
+                src = self._write_image(
+                    os.path.abspath(path), revision, filename, image_data
+                )
+            else:
+                src = self._encode_image(image_data)
+            datapoint = {
+                REVISION_FIELD: revision,
+                FILENAME_FIELD: filename,
+                SRC_FIELD: src,
+            }
+            datapoints.append(datapoint)
+        return datapoints, properties

dvc/render/converter/vega.py

@@ -1,90 +1,100 @@
 from copy import deepcopy
-from functools import partial
-from typing import Dict, Iterable, List, Optional, Set, Union
+from typing import Any, Dict, Iterable, List, Tuple, Union
 
-from funcy import first, project
+from funcy import first, last
 
 from dvc.exceptions import DvcException
-from dvc.render import (
-    FILENAME_FIELD,
-    INDEX_FIELD,
-    REVISION_FIELD,
-    VERSION_FIELD,
-)
+from dvc.render import FILENAME_FIELD, INDEX_FIELD, VERSION_FIELD
 
 from . import Converter
 
 
-class FieldsNotFoundError(DvcException):
-    def __init__(self, expected_fields, found_fields):
-        expected_str = ", ".join(expected_fields)
+class FieldNotFoundError(DvcException):
+    def __init__(self, expected_field, found_fields):
         found_str = ", ".join(found_fields)
         super().__init__(
-            f"Could not find all provided fields ('{expected_str}') "
+            f"Could not find provided field ('{expected_field}') "
             f"in data fields ('{found_str}')."
         )
 
 
-class PlotDataStructureError(DvcException):
-    def __init__(self):
-        super().__init__(
-            "Plot data extraction failed. Please see "
-            "https://man.dvc.org/plots for supported data formats."
-        )
+def _lists(blob: Union[Dict, List]) -> Iterable[List]:
+    if isinstance(blob, list):
+        yield blob
+    else:
+        for _, value in blob.items():
+            if isinstance(value, dict):
+                yield from _lists(value)
+            elif isinstance(value, list):
+                yield value
 
 
-def _filter_fields(
-    datapoints: List[Dict], fields: Set, **kwargs
-) -> List[Dict]:
-    if not fields:
-        return datapoints
-    assert isinstance(fields, set)
-
-    new_data = []
-    for data_point in datapoints:
-        keys = set(data_point.keys())
-        if not fields <= keys:
-            raise FieldsNotFoundError(fields, keys)
-
-        new_data.append(project(data_point, fields))
-
-    return new_data
+def _file_field(*args):
+    for axis_def in args:
+        if axis_def is not None:
+            for file, val in axis_def.items():
+                if isinstance(val, str):
+                    yield file, val
+                elif isinstance(val, list):
+                    for field in val:
+                        yield file, field
 
 
-def _lists(dictionary: Dict):
-    for _, value in dictionary.items():
-        if isinstance(value, dict):
-            yield from _lists(value)
-        elif isinstance(value, list):
-            yield value
+def _find(
+    filename: str,
+    field: str,
+    data_series: List[Tuple[str, str, Any]],
+):
+    for data_file, data_field, data in data_series:
+        if data_file == filename and data_field == field:
+            return data_file, data_field, data
+    return None
 
 
-def _find_first_list(
-    data: Union[Dict, List], fields: Set, **kwargs
-) -> List[Dict]:
-    fields = fields or set()
-
-    if not isinstance(data, dict):
-        return data
-
-    for lst in _lists(data):
-        if (
-            all(isinstance(dp, dict) for dp in lst)
-            # if fields is empty, it will match any set
-            and set(first(lst).keys()) & fields == fields
-        ):
-            return lst
-
-    raise PlotDataStructureError()
+def _verify_field(file2datapoints: Dict[str, List], filename: str, field: str):
+    if filename in file2datapoints:
+        datapoint = first(file2datapoints[filename])
+        if field not in datapoint:
+            raise FieldNotFoundError(field, datapoint.keys())
+    return
 
 
-def _append_index(datapoints: List[Dict], **kwargs) -> List[Dict]:
-    if INDEX_FIELD in first(datapoints).keys():
-        return datapoints
+def _get_xs(properties: Dict, file2datapoints: Dict[str, List[Dict]]):
+    x = properties.get("x", None)
+    if x is not None and isinstance(x, dict):
+        for filename, field in _file_field(x):
+            _verify_field(file2datapoints, filename, field)
+            yield filename, field
 
-    for index, data_point in enumerate(datapoints):
-        data_point[INDEX_FIELD] = index
-    return datapoints
+
+def _get_ys(properties, file2datapoints: Dict[str, List[Dict]]):
+    y = properties.get("y", None)
+    if y is not None:
+        for filename, field in _file_field(y):
+            _verify_field(file2datapoints, filename, field)
+            yield filename, field
+
+
+def _is_datapoints(lst: List[Dict]):
+    """
+    check if dict keys match, datapoints with different keys mgiht lead
+    to unexpected behavior
+    """
+
+    return all(isinstance(item, dict) for item in lst) and set(
+        first(lst).keys()
+    ) == {key for keys in lst for key in keys}
+
+
+def get_datapoints(file_content: Dict):
+    result: List[Dict[str, Any]] = []
+    for lst in _lists(file_content):
+        if _is_datapoints(lst):
+            for index, datapoint in enumerate(lst):
+                if len(result) <= index:
+                    result.append({})
+                result[index].update(datapoint)
+    return result
 
 
 class VegaConverter(Converter):
@@ -95,228 +105,230 @@ class VegaConverter(Converter):
     ('x', 'y') it will attempt to fill in the blanks.
     """
 
-    def __init__(self, plot_properties: Optional[Dict] = None):
-        super().__init__(plot_properties)
+    def __init__(
+        self, plot_id: str, data: Dict = None, properties: Dict = None
+    ):
+        super().__init__(plot_id, data, properties)
+        self.plot_id = plot_id
         self.inferred_properties: Dict = {}
 
-        self.steps = []
+        # TODO we should be handling that in `convert`,
+        #      to avoid stateful `self.inferred_properties`
+        self._infer_x_y()
 
-        self._infer_x()
-        self._infer_fields()
+    def _infer_y_from_data(self):
+        if self.plot_id in self.data:
+            for lst in _lists(self.data[self.plot_id]):
+                if all(isinstance(item, dict) for item in lst):
+                    datapoint = first(lst)
+                    field = last(datapoint.keys())
+                    self.inferred_properties["y"] = {self.plot_id: field}
+                    break
 
-        self.steps.append(
-            (
-                "find_data",
-                partial(
-                    _find_first_list,
-                    fields=self.inferred_properties.get("fields", set())
-                    - {INDEX_FIELD},
-                ),
-            )
-        )
+    def _infer_x_y(self):
+        x = self.properties.get("x", None)
+        y = self.properties.get("y", None)
 
-        if not self.plot_properties.get("x", None):
-            self.steps.append(("append_index", partial(_append_index)))
-
-        self.steps.append(
-            (
-                "filter_fields",
-                partial(
-                    _filter_fields,
-                    fields=self.inferred_properties.get("fields", set()),
-                ),
-            )
-        )
-        self.steps.append(
-            (
-                "infer_y",
-                partial(
-                    self._infer_y,
-                ),
-            )
-        )
-
-        self.steps.append(
-            (
-                "generate_y",
-                partial(
-                    self._generate_y_values,
-                ),
-            )
-        )
-
-    def _infer_x(self):
-        if not self.plot_properties.get("x", None):
-            self.inferred_properties["x"] = INDEX_FIELD
-
-    def skip_step(self, name: str):
-        self.steps = [(_name, fn) for _name, fn in self.steps if _name != name]
-
-    def _infer_fields(self):
-        fields = self.plot_properties.get("fields", set())
-        if fields:
-            fields = {
-                *fields,
-                self.plot_properties.get("x", None),
-                self.plot_properties.get("y", None),
-                self.inferred_properties.get("x", None),
-            } - {None}
-            self.inferred_properties["fields"] = fields
-
-    def _infer_y(self, datapoints: List[Dict], **kwargs):
-        if "y" not in self.plot_properties:
-            data_fields = list(first(datapoints))
-            skip = (
-                REVISION_FIELD,
-                self.plot_properties.get("x", None)
-                or self.inferred_properties.get("x"),
-                FILENAME_FIELD,
-                VERSION_FIELD,
-            )
-            inferred_y = first(
-                f for f in reversed(data_fields) if f not in skip
-            )
-            if "y" in self.inferred_properties:
-                previous_y = self.inferred_properties["y"]
-                if previous_y != inferred_y:
-                    raise DvcException(
-                        f"Inferred y ('{inferred_y}' value does not match"
-                        f"previously matched one ('f{previous_y}')."
-                    )
+        # Infer x.
+        if isinstance(x, str):
+            self.inferred_properties["x"] = {}
+            # If multiple y files, duplicate x for each file.
+            if isinstance(y, dict):
+                for file, fields in y.items():
+                    # Duplicate x for each y.
+                    if isinstance(fields, list):
+                        self.inferred_properties["x"][file] = [x] * len(fields)
+                    else:
+                        self.inferred_properties["x"][file] = x
+            # Otherwise use plot ID as file.
             else:
-                self.inferred_properties["y"] = inferred_y
-        return datapoints
+                self.inferred_properties["x"][self.plot_id] = x
+
+        # Infer y.
+        if y is None:
+            self._infer_y_from_data()
+        # If y files not provided, use plot ID as file.
+        elif not isinstance(y, dict):
+            self.inferred_properties["y"] = {self.plot_id: y}
+
+    def _find_datapoints(self):
+        result = {}
+        for file, content in self.data.items():
+            result[file] = get_datapoints(content)
+
+        return result
+
+    @staticmethod
+    def infer_y_label(properties):
+        y_label = properties.get("y_label", None)
+        if y_label is None:
+            y = properties.get("y", None)
+            if isinstance(y, dict):
+                fields = {field for _, field in _file_field(y)}
+                if len(fields) == 1:
+                    y_label = first(fields)
+                else:
+                    y_label = "y"
+            elif isinstance(y, list):
+                y_label = "y"
+            elif isinstance(y, str):
+                y_label = y
+
+        return y_label
+
+    @staticmethod
+    def infer_x_label(properties):
+        x_label = properties.get("x_label", None)
+        if x_label is None:
+            x = properties.get("x", None)
+            if isinstance(x, dict):
+                fields = {field for _, field in _file_field(x)}
+                if len(fields) == 1:
+                    x_label = first(fields)
+                else:
+                    x_label = "x"
+            else:
+                x_label = INDEX_FIELD
+        return x_label
+
+    def flat_datapoints(self, revision):
+
+        file2datapoints, properties = self.convert()
+
+        props_update = {}
+
+        xs = list(_get_xs(properties, file2datapoints))
+
+        # assign "step" if no x provided
+        if not xs:
+            x_file, x_field = (
+                None,
+                INDEX_FIELD,
+            )
+        else:
+            x_file, x_field = xs[0]
+        props_update["x"] = x_field
+
+        ys = list(_get_ys(properties, file2datapoints))
+
+        num_xs = len(xs)
+        num_ys = len(ys)
+        if num_xs > 1 and num_xs != num_ys:
+            raise DvcException(
+                f"Cannot have different number of x and y data sources. Found "
+                f"{num_xs} x and {num_ys} y data sources."
+            )
+
+        all_datapoints = []
+        all_y_fields = {y_field for _, y_field in ys}
+
+        # override to unified y field name if there are different y fields
+        if len(all_y_fields) > 1:
+            props_update["y"] = "dvc_inferred_y_value"
+        else:
+            props_update["y"] = first(all_y_fields)
+
+        for i, (y_file, y_field) in enumerate(ys):
+            if num_xs > 1:
+                x_file, x_field = xs[i]
+            datapoints = deepcopy(file2datapoints.get(y_file, []))
+
+            if props_update.get("y", None) == "dvc_inferred_y_value":
+                _update_from_field(
+                    datapoints,
+                    field="dvc_inferred_y_value",
+                    source_field=y_field,
+                )
+
+            if x_field == INDEX_FIELD and x_file is None:
+                _update_from_index(datapoints, INDEX_FIELD)
+            else:
+                x_datapoints = file2datapoints.get(x_file, [])
+                try:
+                    _update_from_field(
+                        datapoints,
+                        field=x_field,
+                        source_datapoints=x_datapoints,
+                    )
+                except IndexError:
+                    raise DvcException(
+                        f"Cannot join '{x_field}' from '{x_file}' and "
+                        f"'{y_field}' from '{y_file}'. "
+                        "They have to have same length."
+                    )
+
+            _update_all(
+                datapoints,
+                update_dict={
+                    VERSION_FIELD: {
+                        "revision": revision,
+                        FILENAME_FIELD: y_file,
+                        "field": y_field,
+                    }
+                },
+            )
+
+            all_datapoints.extend(datapoints)
+
+        if not all_datapoints:
+            return [], {}
+
+        properties = {**properties, **props_update}
+
+        return all_datapoints, properties
 
     def convert(
         self,
-        data,
-        revision: str,
-        filename: str,
-        skip: List = None,
-        **kwargs,
     ):
         """
         Convert the data. Fill necessary fields ('x', 'y') and return both
-        generated datapoints and updated properties.
+        generated datapoints and updated properties. If `x` is not provided,
+        leave it as None, fronteds should handle it.
+
+        NOTE: Studio uses this method.
+              The only thing studio FE handles is filling `x` and `y`.
+              `x/y_label` should be filled here.
+
+              Datapoints are not stripped according to config, because users
+              might be utilizing other fields in their custom plots.
         """
-        if not skip:
-            skip = []
+        datapoints = self._find_datapoints()
+        properties = {**self.properties, **self.inferred_properties}
 
-        processed = deepcopy(data)
+        properties["y_label"] = self.infer_y_label(properties)
+        properties["x_label"] = self.infer_x_label(properties)
 
-        for step_name, step in self.steps:
-            if step_name not in skip:
-                processed = step(  # type: ignore
-                    processed,
-                    revision=revision,
-                    filename=filename,
-                )
+        return datapoints, properties
 
-        return processed, {**self.plot_properties, **self.inferred_properties}
 
-    def _generate_y_values(  # noqa: C901
-        self,
-        datapoints: List[Dict],
-        revision: str,
-        filename: str,
-        **kwargs,
-    ) -> List[Dict]:
+def _update_from_field(
+    target_datapoints: List[Dict],
+    field: str,
+    source_datapoints: List[Dict] = None,
+    source_field: str = None,
+):
+    if source_datapoints is None:
+        source_datapoints = target_datapoints
+    if source_field is None:
+        source_field = field
 
-        y_values = self.plot_properties.get(
-            "y", None
-        ) or self.inferred_properties.get("y", None)
+    if len(source_datapoints) != len(target_datapoints):
+        raise IndexError(
+            "Source and target datapoints must have the same length"
+        )
 
-        assert y_values is not None
+    for index, datapoint in enumerate(target_datapoints):
+        source_datapoint = source_datapoints[index]
+        if source_field in source_datapoint:
+            datapoint[field] = source_datapoint[source_field]
 
-        result = []
-        properties_update = {}
 
-        def _add_version_info(datapoint, version_info):
-            tmp = datapoint.copy()
-            tmp[VERSION_FIELD] = version_info
-            return tmp
+def _update_from_index(datapoints: List[Dict], new_field: str):
+    for index, datapoint in enumerate(datapoints):
+        datapoint[new_field] = index
 
-        def _version_info(revision, filename=None, field=None):
-            res = {"revision": revision}
-            if filename is not None:
-                res["filename"] = filename
-            if field is not None:
-                res["field"] = field
-            return res
 
-        def _generate_y(datapoint, field):
-            tmp = datapoint.copy()
-            tmp["dvc_inferred_y_value"] = datapoint[field]
-            tmp = _add_version_info(
-                tmp, _version_info(revision, filename, field)
-            )
-            if (
-                "y_label" not in properties_update
-                and "y_label" not in self.plot_properties
-            ):
-                properties_update["y_label"] = "y"
+def _update_all(datapoints: List[Dict], update_dict: Dict):
 
-            properties_update["y"] = "dvc_inferred_y_value"
-
-            return tmp
-
-        if isinstance(y_values, str):
-            for datapoint in datapoints:
-                result.append(
-                    _add_version_info(
-                        datapoint, _version_info(revision, filename, y_values)
-                    )
-                )
-
-        if isinstance(y_values, list):
-            for datapoint in datapoints:
-                for y_val in y_values:
-                    if y_val in datapoint:
-                        result.append(_generate_y(datapoint, y_val))
-
-        if isinstance(y_values, dict):
-
-            def _to_set(values: Iterable):
-                result = set()
-                for val in values:
-                    if isinstance(val, list):
-                        for elem in val:
-                            result.add(elem)
-                    else:
-                        result.add(val)
-
-                return result
-
-            all_fields = _to_set(y_values.values())
-            if (
-                all([isinstance(field, str) for field in all_fields])
-                and len(all_fields) == 1
-            ):
-                # if we use the same field from all files,
-                # we dont have to generate it
-                y_field = all_fields.pop()
-                for datapoint in datapoints:
-                    result.append(
-                        _add_version_info(
-                            datapoint,
-                            _version_info(revision, filename, y_field),
-                        )
-                    )
-                properties_update.update({"y": y_field})
-            else:
-                for def_filename, val in y_values.items():
-                    if isinstance(val, str):
-                        fields = [val]
-                    if isinstance(val, list):
-                        fields = val
-                    for datapoint in datapoints:
-                        for field in fields:
-                            if field in datapoint and def_filename in filename:
-                                result.append(_generate_y(datapoint, field))
-
-        self.inferred_properties = {
-            **self.inferred_properties,
-            **properties_update,
-        }
-
-        return result
+    for datapoint in datapoints:
+        datapoint.update(update_dict)

dvc/render/match.py

@@ -42,7 +42,7 @@ class PlotsData:
         return dict(groups)
 
     def get_definition_data(self, target_files, rev):
-        result = []
+        result = {}
         for file in target_files:
             file_content = (
                 self.data.get(rev, {})
@@ -52,7 +52,7 @@ class PlotsData:
                 .get("data", {})
             )
             if file_content:
-                result.append((file, file_content))
+                result[file] = file_content
 
         return result
 
@@ -91,15 +91,14 @@ def match_defs_renderers(
                 renderer_cls = VegaRenderer
                 renderer_id = plot_id
 
-            converter = _get_converter(renderer_cls, props)
+            converter = _get_converter(
+                renderer_cls, inner_id, props, definitions_data
+            )
 
-            for filename, plot_data in definitions_data:
-                dps, final_props = converter.convert(
-                    revision=rev,
-                    filename=filename,
-                    data=plot_data,
-                )
-                plot_datapoints.extend(dps)
+            dps, rev_props = converter.flat_datapoints(rev)
+            if not final_props and rev_props:
+                final_props = rev_props
+            plot_datapoints.extend(dps)
 
         if "title" not in final_props:
             final_props["title"] = renderer_id

dvc/repo/__init__.py

@@ -401,6 +401,7 @@ class Repo:
         used_run_cache=None,
         revs=None,
         num=1,
+        push: bool = False,
     ):
         """Get the stages related to the given target and collect
         the `info` of its outputs.
@@ -435,6 +436,7 @@ class Repo:
                 jobs=jobs,
                 recursive=recursive,
                 with_deps=with_deps,
+                push=push,
             ).items():
                 used[odb].update(objs)
 
@@ -534,11 +536,16 @@ class Repo:
         self.scm.close()
         self.state.close()
 
+    def _reset_cached_indecies(self):
+        self.__dict__.pop("index", None)
+        self.__dict__.pop("dvcignore", None)
+        self.__dict__.pop("dvcfs", None)
+        self.__dict__.pop("datafs", None)
+
     def _reset(self):
         self.state.close()
         self.scm._reset()  # pylint: disable=protected-access
-        self.__dict__.pop("index", None)
-        self.__dict__.pop("dvcignore", None)
+        self._reset_cached_indecies()
 
     def __enter__(self):
         return self

dvc/repo/add.py

@@ -187,7 +187,7 @@ def add(  # noqa: C901
                 stage.transfer(source, to_remote=to_remote, odb=odb, **kwargs)
             else:
                 try:
-                    stage.save()
+                    stage.save(merge_versioned=True)
                     if not no_commit:
                         stage.commit()
                 except CacheLinkError:

dvc/repo/brancher.py

@@ -77,7 +77,7 @@ def brancher(  # noqa: E302
         from dvc.fs import GitFileSystem
 
         for sha, names in found_revs.items():
-            self.__dict__.pop("index", None)
+            self._reset_cached_indecies()  # pylint: disable=W0212
             self.fs = GitFileSystem(scm=scm, rev=sha)
             self.root_dir = self.fs.path.join("/", *repo_root_parts)
 

dvc/repo/experiments/__init__.py

@@ -32,7 +32,7 @@ from .refs import (
     ExpRefInfo,
 )
 from .stash import ExpStashEntry
-from .utils import exp_refs_by_rev, scm_locked, unlocked_repo
+from .utils import check_ref_format, exp_refs_by_rev, unlocked_repo
 
 logger = logging.getLogger(__name__)
 
@@ -50,18 +50,12 @@ class Experiments:
     )
 
     def __init__(self, repo):
-        from dvc.lock import make_lock
         from dvc.scm import NoSCMError
 
         if repo.config["core"].get("no_scm", False):
             raise NoSCMError
 
         self.repo = repo
-        self.scm_lock = make_lock(
-            os.path.join(self.repo.tmp_dir, "exp_scm_lock"),
-            tmp_dir=self.repo.tmp_dir,
-            hardlink_lock=repo.config["core"].get("hardlink_lock", False),
-        )
 
     @property
     def scm(self):
@@ -186,8 +180,7 @@ class Experiments:
         if not entries:
             return results
 
-        # TODO: re-enable --jobs concurrency
-        self.celery_queue.spawn_worker()
+        self.celery_queue.start_workers(count=kwargs.get("jobs", 1))
         failed = []
         try:
             ui.write(
@@ -242,18 +235,6 @@ class Experiments:
             "\tdvc exp branch <exp> <branch>\n"
         )
 
-    def _validate_new_ref(self, exp_ref: ExpRefInfo):
-        from .utils import check_ref_format
-
-        if not exp_ref.name:
-            return
-
-        check_ref_format(self.scm, exp_ref)
-
-        if self.scm.get_ref(str(exp_ref)):
-            raise ExperimentExistsError(exp_ref.name)
-
-    @scm_locked
     def new(
         self,
         queue: BaseStashQueue,
@@ -272,13 +253,13 @@ class Experiments:
 
         name = kwargs.get("name", None)
         baseline_sha = kwargs.get("baseline_rev") or self.repo.scm.get_rev()
-        exp_ref = ExpRefInfo(baseline_sha=baseline_sha, name=name)
 
-        try:
-            self._validate_new_ref(exp_ref)
-        except ExperimentExistsError as err:
-            if not (kwargs.get("force", False) or kwargs.get("reset", False)):
-                raise err
+        if name:
+            exp_ref = ExpRefInfo(baseline_sha=baseline_sha, name=name)
+            check_ref_format(self.scm, exp_ref)
+            force = kwargs.get("force", False)
+            if self.scm.get_ref(str(exp_ref)) and not force:
+                raise ExperimentExistsError(exp_ref.name)
 
         return queue.put(*args, **kwargs)
 
@@ -487,6 +468,11 @@ class Experiments:
 
         return run(self.repo, *args, **kwargs)
 
+    def save(self, *args, **kwargs):
+        from dvc.repo.experiments.save import save
+
+        return save(self.repo, *args, **kwargs)
+
     def gc(self, *args, **kwargs):
         from dvc.repo.experiments.gc import gc
 

dvc/repo/experiments/apply.py

@@ -16,9 +16,10 @@ from .executor.base import BaseExecutor
 from .refs import EXEC_APPLY
 
 if TYPE_CHECKING:
-    from scmrepo import Git
+    from scmrepo.git import Git
 
     from dvc.repo import Repo
+    from dvc.repo.experiments import Experiments
 
 logger = logging.getLogger(__name__)
 
@@ -31,20 +32,33 @@ def apply(repo: "Repo", rev: str, force: bool = True, **kwargs):
     from dvc.repo.checkout import checkout as dvc_checkout
     from dvc.scm import GitMergeError, RevError, resolve_rev
 
-    exps = repo.experiments
+    exps: "Experiments" = repo.experiments
+
+    is_stash: bool = False
 
     try:
         exp_rev = resolve_rev(repo.scm, rev)
         exps.check_baseline(exp_rev)
-    except (RevError, BaselineMismatchError) as exc:
+    except RevError as exc:
+        (
+            exp_ref_info,
+            queue_entry,
+        ) = exps.celery_queue.get_ref_and_entry_by_names(rev)[rev]
+        baseline_sha = repo.scm.get_rev()
+        if exp_ref_info:
+            if baseline_sha != exp_ref_info.baseline_sha:
+                raise InvalidExpRevError(rev)
+            exp_rev = repo.scm.get_ref(str(exp_ref_info))
+        elif queue_entry:
+            if queue_entry.baseline_rev != baseline_sha:
+                raise InvalidExpRevError(rev)
+            exp_rev = queue_entry.stash_rev
+            is_stash = True
+        else:
+            raise InvalidExpRevError(rev) from exc
+    except BaselineMismatchError as exc:
         raise InvalidExpRevError(rev) from exc
 
-    stash_rev = exp_rev in exps.stash_revs
-    if not stash_rev and not exps.get_branch_by_rev(
-        exp_rev, allow_multiple=True
-    ):
-        raise InvalidExpRevError(exp_rev)
-
     # NOTE: we don't use scmrepo's stash_workspace() here since we need
     # finer control over the merge behavior when we unstash everything
     with _apply_workspace(repo, rev, force):
@@ -55,7 +69,7 @@ def apply(repo: "Repo", rev: str, force: bool = True, **kwargs):
 
     repo.scm.reset()
 
-    if stash_rev:
+    if is_stash:
         args_path = os.path.join(repo.tmp_dir, BaseExecutor.PACKED_ARGS_FILE)
         if os.path.exists(args_path):
             remove(args_path)

dvc/repo/experiments/exceptions.py

@@ -6,12 +6,6 @@ if TYPE_CHECKING:
     from .refs import ExpRefInfo
 
 
-class UnchangedExperimentError(DvcException):
-    def __init__(self, rev):
-        super().__init__(f"Experiment unchanged from '{rev[:7]}'.")
-        self.rev = rev
-
-
 class BaselineMismatchError(DvcException):
     def __init__(self, rev, expected):
         if hasattr(rev, "hexsha"):
@@ -25,13 +19,13 @@ class BaselineMismatchError(DvcException):
 
 
 class ExperimentExistsError(DvcException):
-    def __init__(self, name: str):
+    def __init__(self, name: str, command: str = "run"):
         msg = (
             "Experiment conflicts with existing experiment "
             f"'{name}'. To overwrite the existing experiment run:\n\n"
-            "\tdvc exp run -f ...\n\n"
-            "To run this experiment with a different name run:\n\n"
-            f"\tdvc exp run -n <new_name> ...\n"
+            f"\tdvc exp {command} -f ...\n\n"
+            f"To {command} this experiment with a different name run:\n\n"
+            f"\tdvc exp {command} -n <new_name> ...\n"
         )
         super().__init__(msg)
         self.name = name

dvc/repo/experiments/executor/base.py

@@ -12,6 +12,7 @@ from typing import (
     Callable,
     Dict,
     Iterable,
+    List,
     NamedTuple,
     Optional,
     Tuple,
@@ -29,22 +30,8 @@ from dvc.ui import ui
 from dvc.utils import dict_sha256, env2bool, relpath
 from dvc.utils.fs import remove
 
-from ..exceptions import (
-    CheckpointExistsError,
-    ExperimentExistsError,
-    UnchangedExperimentError,
-)
-from ..refs import (
-    EXEC_BASELINE,
-    EXEC_BRANCH,
-    EXEC_CHECKPOINT,
-    EXEC_HEAD,
-    EXEC_MERGE,
-    EXEC_NAMESPACE,
-    EXPS_NAMESPACE,
-    EXPS_STASH,
-    ExpRefInfo,
-)
+from ..exceptions import CheckpointExistsError, ExperimentExistsError
+from ..refs import EXEC_BASELINE, EXEC_BRANCH, EXEC_CHECKPOINT, ExpRefInfo
 
 if TYPE_CHECKING:
     from queue import Queue
@@ -58,10 +45,6 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-EXEC_TMP_DIR = "exps"
-EXEC_PID_DIR = "run"
-
-
 class ExecutorResult(NamedTuple):
     exp_hash: Optional[str]
     ref_info: Optional["ExpRefInfo"]
@@ -172,6 +155,7 @@ class BaseExecutor(ABC):
     @abstractmethod
     def init_git(
         self,
+        repo: "Repo",
         scm: "Git",
         stash_rev: str,
         entry: "ExpStashEntry",
@@ -319,6 +303,7 @@ class BaseExecutor(ABC):
     def fetch_exps(
         self,
         dest_scm: "Git",
+        refs: List[str],
         force: bool = False,
         on_diverged: Callable[[str, bool], None] = None,
         **kwargs,
@@ -327,26 +312,19 @@ class BaseExecutor(ABC):
 
         Args:
             dest_scm: Destination Git instance.
+            refs: reference names to be fetched from the remotes.
             force: If True, diverged refs will be overwritten
             on_diverged: Callback in the form on_diverged(ref, is_checkpoint)
                 to be called when an experiment ref has diverged.
 
         Extra kwargs will be passed into the remote git client.
         """
-        from ..utils import iter_remote_refs
 
-        refs = []
-        has_checkpoint = False
-        for ref in iter_remote_refs(
-            dest_scm,
-            self.git_url,
-            base=EXPS_NAMESPACE,
-            **kwargs,
-        ):
-            if ref == EXEC_CHECKPOINT:
-                has_checkpoint = True
-            elif not ref.startswith(EXEC_NAMESPACE) and ref != EXPS_STASH:
-                refs.append(ref)
+        if EXEC_CHECKPOINT in refs:
+            refs.remove(EXEC_CHECKPOINT)
+            has_checkpoint = True
+        else:
+            has_checkpoint = False
 
         def on_diverged_ref(orig_ref: str, new_rev: str):
             if force:
@@ -514,7 +492,6 @@ class BaseExecutor(ABC):
                     info,
                     stages,
                     exp_hash,
-                    checkpoint_reset,
                     auto_push,
                     git_remote,
                     repro_force,
@@ -535,29 +512,20 @@ class BaseExecutor(ABC):
         info,
         stages,
         exp_hash,
-        checkpoint_reset,
         auto_push,
         git_remote,
         repro_force,
     ) -> Tuple[Optional[str], Optional["ExpRefInfo"], bool]:
-        try:
-            is_checkpoint = any(stage.is_checkpoint for stage in stages)
-            if is_checkpoint and checkpoint_reset:
-                # For reset checkpoint stages, we need to force
-                # overwriting existing checkpoint refs even though
-                # repro may not have actually been run with --force
-                repro_force = True
-            cls.commit(
-                dvc.scm,
-                exp_hash,
-                exp_name=info.name,
-                force=repro_force,
-                checkpoint=is_checkpoint,
-            )
-            if auto_push:
-                cls._auto_push(dvc, dvc.scm, git_remote)
-        except UnchangedExperimentError:
-            pass
+        is_checkpoint = any(stage.is_checkpoint for stage in stages)
+        cls.commit(
+            dvc.scm,
+            exp_hash,
+            exp_name=info.name,
+            force=repro_force,
+            checkpoint=is_checkpoint,
+        )
+        if auto_push:
+            cls._auto_push(dvc, dvc.scm, git_remote)
         ref: Optional[str] = dvc.scm.get_ref(EXEC_BRANCH, follow=False)
         exp_ref: Optional["ExpRefInfo"] = (
             ExpRefInfo.from_ref(ref) if ref else None
@@ -668,18 +636,15 @@ class BaseExecutor(ABC):
         unchanged: Iterable["PipelineStage"],
         stages: Iterable["PipelineStage"],
     ):
-        try:
-            exp_hash = cls.hash_exp(list(stages) + list(unchanged))
-            exp_rev = cls.commit(
-                scm, exp_hash, exp_name=name, force=force, checkpoint=True
-            )
+        exp_hash = cls.hash_exp(list(stages) + list(unchanged))
+        exp_rev = cls.commit(
+            scm, exp_hash, exp_name=name, force=force, checkpoint=True
+        )
 
-            if env2bool(DVC_EXP_AUTO_PUSH):
-                git_remote = os.getenv(DVC_EXP_GIT_REMOTE)
-                cls._auto_push(dvc, scm, git_remote)
-            ui.write(f"Checkpoint experiment iteration '{exp_rev[:7]}'.")
-        except UnchangedExperimentError:
-            pass
+        if env2bool(DVC_EXP_AUTO_PUSH):
+            git_remote = os.getenv(DVC_EXP_GIT_REMOTE)
+            cls._auto_push(dvc, scm, git_remote)
+        ui.write(f"Checkpoint experiment iteration '{exp_rev[:7]}'.")
 
     @classmethod
     def commit(
@@ -694,7 +659,6 @@ class BaseExecutor(ABC):
         rev = scm.get_rev()
         if not scm.is_dirty(untracked_files=False):
             logger.debug("No changes to commit")
-            raise UnchangedExperimentError(rev)
 
         check_conflict = False
         branch = scm.get_ref(EXEC_BRANCH, follow=False)
@@ -755,21 +719,12 @@ class BaseExecutor(ABC):
             dvc_logger.setLevel(level)
 
     @contextmanager
-    def set_exec_refs(
-        self, scm: "Git", stash_rev: str, entry: "ExpStashEntry"
-    ):
+    def set_temp_refs(self, scm: "Git", temp_dict: Dict[str, str]):
         try:
-            # Executor will be initialized with an empty git repo that
-            # we populate by pushing:
-            #   EXEC_HEAD - the base commit for this experiment
-            #   EXEC_MERGE - the unmerged changes (from our stash)
-            #       to be reproduced
-            #   EXEC_BASELINE - the baseline commit for this experiment
-            scm.set_ref(EXEC_HEAD, entry.head_rev)
-            scm.set_ref(EXEC_MERGE, stash_rev)
-            scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
+            for ref, rev in temp_dict.items():
+                scm.set_ref(ref, rev)
             yield
         finally:
-            for ref in (EXEC_HEAD, EXEC_MERGE, EXEC_BASELINE):
+            for ref in temp_dict:
                 if scm.get_ref(ref):
                     scm.remove_ref(ref)

dvc/repo/experiments/executor/local.py

@@ -4,9 +4,11 @@ from contextlib import ExitStack
 from tempfile import mkdtemp
 from typing import TYPE_CHECKING, Optional
 
-from funcy import cached_property
+from funcy import cached_property, retry
 from scmrepo.exceptions import SCMError as _SCMError
+from shortuuid import uuid
 
+from dvc.lock import LockError
 from dvc.scm import SCM, GitMergeError
 from dvc.utils.fs import makedirs, remove
 
@@ -18,15 +20,17 @@ from ..refs import (
     EXEC_HEAD,
     EXEC_MERGE,
     EXEC_NAMESPACE,
+    EXPS_TEMP,
+    ExpRefInfo,
 )
-from .base import EXEC_TMP_DIR, BaseExecutor, TaskStatus
+from ..utils import EXEC_TMP_DIR, get_exp_rwlock
+from .base import BaseExecutor, TaskStatus
 
 if TYPE_CHECKING:
     from scmrepo.git import Git
 
     from dvc.repo import Repo
 
-    from ..refs import ExpRefInfo
     from ..stash import ExpStashEntry
 
 logger = logging.getLogger(__name__)
@@ -67,8 +71,10 @@ class TempDirExecutor(BaseLocalExecutor):
     QUIET = True
     DEFAULT_LOCATION = "tempdir"
 
+    @retry(180, errors=LockError, timeout=1)
     def init_git(
         self,
+        repo: "Repo",
         scm: "Git",
         stash_rev: str,
         entry: "ExpStashEntry",
@@ -85,18 +91,43 @@ class TempDirExecutor(BaseLocalExecutor):
         if infofile:
             self.info.dump_json(infofile)
 
-        with self.set_exec_refs(scm, stash_rev, entry):
-            refspec = f"{EXEC_NAMESPACE}/"
-            push_refspec(scm, self.git_url, refspec, refspec)
+        temp_head = f"{EXPS_TEMP}/head-{uuid()}"
+        temp_merge = f"{EXPS_TEMP}/merge-{uuid()}"
+        temp_baseline = f"{EXPS_TEMP}/baseline-{uuid()}"
 
-        if branch:
-            push_refspec(scm, self.git_url, branch, branch)
-            self.scm.set_ref(EXEC_BRANCH, branch, symbolic=True)
-        elif self.scm.get_ref(EXEC_BRANCH):
-            self.scm.remove_ref(EXEC_BRANCH)
+        temp_ref_dict = {
+            temp_head: entry.head_rev,
+            temp_merge: stash_rev,
+            temp_baseline: entry.baseline_rev,
+        }
+        with get_exp_rwlock(
+            repo, writes=[temp_head, temp_merge, temp_baseline]
+        ), self.set_temp_refs(scm, temp_ref_dict):
+            # Executor will be initialized with an empty git repo that
+            # we populate by pushing:
+            #   EXEC_HEAD - the base commit for this experiment
+            #   EXEC_MERGE - the unmerged changes (from our stash)
+            #       to be reproduced
+            #   EXEC_BASELINE - the baseline commit for this experiment
+            refspec = [
+                (temp_head, EXEC_HEAD),
+                (temp_merge, EXEC_MERGE),
+                (temp_baseline, EXEC_BASELINE),
+            ]
+
+            if branch:
+                refspec.append((branch, branch))
+                with get_exp_rwlock(repo, reads=[branch]):
+                    push_refspec(scm, self.git_url, refspec)
+                self.scm.set_ref(EXEC_BRANCH, branch, symbolic=True)
+            else:
+                push_refspec(scm, self.git_url, refspec)
+                if self.scm.get_ref(EXEC_BRANCH):
+                    self.scm.remove_ref(EXEC_BRANCH)
+
+            if self.scm.get_ref(EXEC_CHECKPOINT):
+                self.scm.remove_ref(EXEC_CHECKPOINT)
 
-        if self.scm.get_ref(EXEC_CHECKPOINT):
-            self.scm.remove_ref(EXEC_CHECKPOINT)
         # checkout EXEC_HEAD and apply EXEC_MERGE on top of it without
         # committing
         head = EXEC_BRANCH if branch else EXEC_HEAD
@@ -167,8 +198,10 @@ class WorkspaceExecutor(BaseLocalExecutor):
         logger.debug("Init workspace executor in '%s'", root_dir)
         return executor
 
+    @retry(180, errors=LockError, timeout=1)
     def init_git(
         self,
+        repo: "Repo",
         scm: "Git",
         stash_rev: str,
         entry: "ExpStashEntry",
@@ -179,25 +212,26 @@ class WorkspaceExecutor(BaseLocalExecutor):
         if infofile:
             self.info.dump_json(infofile)
 
-        scm.set_ref(EXEC_HEAD, entry.head_rev)
-        scm.set_ref(EXEC_MERGE, stash_rev)
-        scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
-        self._detach_stack.enter_context(
-            self.scm.detach_head(
-                self.scm.get_ref(EXEC_HEAD),
-                force=True,
-                client="dvc",
+        with get_exp_rwlock(repo, writes=[EXEC_NAMESPACE]):
+            scm.set_ref(EXEC_HEAD, entry.head_rev)
+            scm.set_ref(EXEC_MERGE, stash_rev)
+            scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
+            self._detach_stack.enter_context(
+                self.scm.detach_head(
+                    self.scm.get_ref(EXEC_HEAD),
+                    force=True,
+                    client="dvc",
+                )
             )
-        )
-        merge_rev = self.scm.get_ref(EXEC_MERGE)
-        try:
-            self.scm.merge(merge_rev, squash=True, commit=False)
-        except _SCMError as exc:
-            raise GitMergeError(str(exc), scm=self.scm)
-        if branch:
-            self.scm.set_ref(EXEC_BRANCH, branch, symbolic=True)
-        elif scm.get_ref(EXEC_BRANCH):
-            self.scm.remove_ref(EXEC_BRANCH)
+            merge_rev = self.scm.get_ref(EXEC_MERGE)
+            try:
+                self.scm.merge(merge_rev, squash=True, commit=False)
+            except _SCMError as exc:
+                raise GitMergeError(str(exc), scm=self.scm)
+            if branch:
+                self.scm.set_ref(EXEC_BRANCH, branch, symbolic=True)
+            elif scm.get_ref(EXEC_BRANCH):
+                self.scm.remove_ref(EXEC_BRANCH)
 
     def init_cache(self, repo: "Repo", rev: str, run_cache: bool = True):
         pass

dvc/repo/experiments/executor/ssh.py

@@ -9,6 +9,7 @@ from dvc_ssh import SSHFileSystem
 from funcy import first
 
 from ..refs import (
+    EXEC_BASELINE,
     EXEC_BRANCH,
     EXEC_CHECKPOINT,
     EXEC_HEAD,
@@ -119,6 +120,7 @@ class SSHExecutor(BaseExecutor):
 
     def init_git(
         self,
+        repo: "Repo",
         scm: "Git",
         stash_rev: str,
         entry: "ExpStashEntry",
@@ -147,12 +149,18 @@ class SSHExecutor(BaseExecutor):
             # (see https://github.com/iterative/dvc/issues/6508)
             kwargs = self._git_client_args(fs)
 
-            with self.set_exec_refs(scm, stash_rev, entry):
-                refspec = f"{EXEC_NAMESPACE}/"
-                push_refspec(scm, self.git_url, refspec, refspec, **kwargs)
+            ref_dict = {
+                EXEC_HEAD: entry.head_rev,
+                EXEC_MERGE: stash_rev,
+                EXEC_BASELINE: entry.baseline_rev,
+            }
+            with self.set_temp_refs(scm, ref_dict):
+                exec_namespace = f"{EXEC_NAMESPACE}/"
+                refspec = [(exec_namespace, exec_namespace)]
+                push_refspec(scm, self.git_url, refspec, **kwargs)
 
             if branch:
-                push_refspec(scm, self.git_url, branch, branch, **kwargs)
+                push_refspec(scm, self.git_url, [(branch, branch)], **kwargs)
                 self._ssh_cmd(fs, f"git symbolic-ref {EXEC_BRANCH} {branch}")
             else:
                 self._ssh_cmd(

dvc/repo/experiments/ls.py

@@ -6,7 +6,7 @@ from dvc.repo.scm_context import scm_context
 from dvc.scm import iter_revs
 from dvc.types import Optional
 
-from .utils import exp_refs, exp_refs_by_baseline
+from .utils import exp_refs_by_baseline
 
 logger = logging.getLogger(__name__)
 
@@ -20,19 +20,22 @@ def ls(
     num: int = 1,
     git_remote: Optional[str] = None,
 ):
-    results = defaultdict(list)
-    if all_commits:
-        gen = exp_refs(repo.scm, git_remote)
-        for info in gen:
-            results[info.baseline_sha].append(info.name)
-        return results
-
-    rev = rev or "HEAD"
-
-    revs = iter_revs(repo.scm, [rev], num)
-    rev_set = set(revs.keys())
+    rev_set = None
+    if not all_commits:
+        revs = iter_revs(repo.scm, [rev or "HEAD"], num)
+        rev_set = set(revs.keys())
     ref_info_dict = exp_refs_by_baseline(repo.scm, rev_set, git_remote)
-    for rev, ref_info_list in ref_info_dict.items():
-        results[rev] = [ref_info.name for ref_info in ref_info_list]
+
+    tags = repo.scm.describe(ref_info_dict.keys())
+    remained = {baseline for baseline, tag in tags.items() if tag is None}
+    base = "refs/heads"
+    ref_heads = repo.scm.describe(remained, base=base)
+
+    results = defaultdict(list)
+    for baseline in ref_info_dict:
+        name = baseline[:7]
+        if tags[baseline] or ref_heads[baseline]:
+            name = tags[baseline] or ref_heads[baseline][len(base) + 1 :]
+        results[name] = [info.name for info in ref_info_dict[baseline]]
 
     return results

dvc/repo/experiments/queue/base.py

@@ -20,22 +20,24 @@ from typing import (
 from funcy import cached_property, retry
 
 from dvc.dependency import ParamsDependency
-from dvc.env import DVCLIVE_RESUME
+from dvc.env import DVC_EXP_BASELINE_REV, DVC_EXP_NAME, DVCLIVE_RESUME
 from dvc.exceptions import DvcException
 from dvc.lock import LockError
 from dvc.ui import ui
 
 from ..exceptions import CheckpointExistsError, ExperimentExistsError
-from ..executor.base import (
-    EXEC_PID_DIR,
-    EXEC_TMP_DIR,
-    BaseExecutor,
-    ExecutorResult,
-)
+from ..executor.base import BaseExecutor, ExecutorResult
 from ..executor.local import WorkspaceExecutor
 from ..refs import ExpRefInfo
 from ..stash import ExpStash, ExpStashEntry
-from ..utils import exp_refs_by_rev, scm_locked
+from ..utils import (
+    EXEC_PID_DIR,
+    EXEC_TMP_DIR,
+    exp_refs_by_rev,
+    get_exp_rwlock,
+    get_random_exp_name,
+)
+from .utils import get_remote_executor_refs
 
 if TYPE_CHECKING:
     from scmrepo.git import Git
@@ -84,6 +86,11 @@ class QueueDoneResult(NamedTuple):
     result: Optional[ExecutorResult]
 
 
+class ExpRefAndQueueEntry(NamedTuple):
+    exp_ref_info: Optional["ExpRefInfo"]
+    queue_entry: Optional["QueueEntry"]
+
+
 class BaseStashQueue(ABC):
     """Naive Git-stash based experiment queue.
 
@@ -285,7 +292,7 @@ class BaseStashQueue(ABC):
                 output.
         """
 
-    def _stash_exp(
+    def _stash_exp(  # noqa: C901
         self,
         *args,
         params: Optional[Dict[str, List[str]]] = None,
@@ -400,7 +407,14 @@ class BaseStashQueue(ABC):
                             )
 
                     # save additional repro command line arguments
-                    run_env = {DVCLIVE_RESUME: "1"} if resume_rev else {}
+                    run_env = {
+                        DVC_EXP_BASELINE_REV: baseline_rev,
+                    }
+                    if not name:
+                        name = get_random_exp_name(self.scm, baseline_rev)
+                    run_env[DVC_EXP_NAME] = name
+                    if resume_rev:
+                        run_env[DVCLIVE_RESUME] = "1"
                     self._pack_args(*args, run_env=run_env, **kwargs)
 
                     # save experiment as a stash commit
@@ -555,28 +569,39 @@ class BaseStashQueue(ABC):
 
     @staticmethod
     @retry(180, errors=LockError, timeout=1)
-    @scm_locked
+    def get_stash_entry(
+        exp: "Experiments",
+        queue_entry: QueueEntry,
+    ) -> "ExpStashEntry":
+        stash = ExpStash(exp.scm, queue_entry.stash_ref)
+        stash_rev = queue_entry.stash_rev
+        with get_exp_rwlock(exp.repo, writes=[queue_entry.stash_ref]):
+            stash_entry = stash.stash_revs.get(
+                stash_rev,
+                ExpStashEntry(None, stash_rev, stash_rev, None, None),
+            )
+            if stash_entry.stash_index is not None:
+                stash.drop(stash_entry.stash_index)
+        return stash_entry
+
+    @classmethod
     def init_executor(
+        cls,
         exp: "Experiments",
         queue_entry: QueueEntry,
         executor_cls: Type[BaseExecutor] = WorkspaceExecutor,
         **kwargs,
     ) -> BaseExecutor:
-        scm = exp.scm
-        stash = ExpStash(scm, queue_entry.stash_ref)
-        stash_rev = queue_entry.stash_rev
-        stash_entry = stash.stash_revs.get(
-            stash_rev,
-            ExpStashEntry(None, stash_rev, stash_rev, None, None),
-        )
-        if stash_entry.stash_index is not None:
-            stash.drop(stash_entry.stash_index)
+        stash_entry = cls.get_stash_entry(exp, queue_entry)
+
         executor = executor_cls.from_stash_entry(
             exp.repo, stash_entry, **kwargs
         )
 
+        stash_rev = queue_entry.stash_rev
         infofile = exp.celery_queue.get_infofile_path(stash_rev)
         executor.init_git(
+            exp.repo,
             exp.repo.scm,
             stash_rev,
             stash_entry,
@@ -597,7 +622,6 @@ class BaseStashQueue(ABC):
 
     @staticmethod
     @retry(180, errors=LockError, timeout=1)
-    @scm_locked
     def collect_git(
         exp: "Experiments",
         executor: BaseExecutor,
@@ -611,16 +635,20 @@ class BaseStashQueue(ABC):
                 raise CheckpointExistsError(ref_info.name)
             raise ExperimentExistsError(ref_info.name)
 
-        for ref in executor.fetch_exps(
-            exp.scm,
-            force=exec_result.force,
-            on_diverged=on_diverged,
-        ):
-            exp_rev = exp.scm.get_ref(ref)
-            if exp_rev:
-                assert exec_result.exp_hash
-                logger.debug("Collected experiment '%s'.", exp_rev[:7])
-                results[exp_rev] = exec_result.exp_hash
+        refs = get_remote_executor_refs(exp.scm, executor.git_url)
+
+        with get_exp_rwlock(exp.repo, writes=refs):
+            for ref in executor.fetch_exps(
+                exp.scm,
+                refs,
+                force=exec_result.force,
+                on_diverged=on_diverged,
+            ):
+                exp_rev = exp.scm.get_ref(ref)
+                if exp_rev:
+                    assert exec_result.exp_hash
+                    logger.debug("Collected experiment '%s'.", exp_rev[:7])
+                    results[exp_rev] = exec_result.exp_hash
 
         return results
 

dvc/repo/experiments/queue/celery.py

@@ -12,10 +12,11 @@ from typing import (
     NamedTuple,
     Optional,
     Set,
+    Union,
 )
 
 from celery.result import AsyncResult
-from funcy import cached_property
+from funcy import cached_property, first
 from kombu.message import Message
 
 from dvc.daemon import daemonize
@@ -23,8 +24,17 @@ from dvc.exceptions import DvcException
 from dvc.ui import ui
 
 from ..exceptions import UnresolvedQueueExpNamesError
-from ..executor.base import EXEC_TMP_DIR, ExecutorInfo, ExecutorResult
-from .base import BaseStashQueue, QueueDoneResult, QueueEntry, QueueGetResult
+from ..executor.base import ExecutorInfo, ExecutorResult
+from ..refs import CELERY_STASH
+from ..utils import EXEC_TMP_DIR, get_exp_rwlock
+from .base import (
+    BaseStashQueue,
+    ExpRefAndQueueEntry,
+    QueueDoneResult,
+    QueueEntry,
+    QueueGetResult,
+)
+from .exceptions import CannotKillTasksError
 from .tasks import run_exp
 from .utils import fetch_running_exp_from_temp_dir
 
@@ -33,6 +43,8 @@ if TYPE_CHECKING:
     from dvc_task.proc.manager import ProcessManager
     from dvc_task.worker import TemporaryWorker
 
+    from ..refs import ExpRefInfo
+
 logger = logging.getLogger(__name__)
 
 
@@ -54,8 +66,6 @@ class LocalCeleryQueue(BaseStashQueue):
 
     CELERY_DIR = "celery"
 
-    _shutdown_task_ids: Set[str] = set()
-
     @cached_property
     def wdir(self) -> str:
         return os.path.join(self.repo.tmp_dir, EXEC_TMP_DIR, self.CELERY_DIR)
@@ -113,7 +123,7 @@ class LocalCeleryQueue(BaseStashQueue):
             timeout=10,
         )
 
-    def spawn_worker(self, num: int = 1):
+    def _spawn_worker(self, num: int = 1):
         """spawn one single worker to process to queued tasks.
 
         Argument:
@@ -156,14 +166,15 @@ class LocalCeleryQueue(BaseStashQueue):
             if node_name in active_worker:
                 logger.debug(f"Exp queue worker {node_name} already exist")
                 continue
-            self.spawn_worker(num)
+            self._spawn_worker(num)
             started += 1
 
         return started
 
     def put(self, *args, **kwargs) -> QueueEntry:
         """Stash an experiment and add it to the queue."""
-        entry = self._stash_exp(*args, **kwargs)
+        with get_exp_rwlock(self.repo, writes=["workspace", CELERY_STASH]):
+            entry = self._stash_exp(*args, **kwargs)
         self.celery.signature(run_exp.s(entry.asdict())).delay()
         return entry
 
@@ -286,33 +297,91 @@ class LocalCeleryQueue(BaseStashQueue):
         # out of the active task list, and needs to be loaded here.
         return self._get_done_result(entry, timeout)
 
-    def kill(self, revs: Collection[str]) -> None:
-        to_kill: Set[QueueEntry] = set()
+    def _get_running_task_ids(self) -> Set[str]:
+        running_task_ids: Set[str] = set()
+        active_workers = self.worker_status()
+        for _, tasks in active_workers.items():
+            task = first(tasks)
+            if task:
+                running_task_ids.add(task["id"])
+        return running_task_ids
+
+    def _try_to_kill_tasks(
+        self, to_kill: Dict[QueueEntry, str], force: bool
+    ) -> Dict[QueueEntry, str]:
+        fail_to_kill_entries: Dict[QueueEntry, str] = {}
+        for queue_entry, rev in to_kill.items():
+            try:
+                if force:
+                    self.proc.kill(queue_entry.stash_rev)
+                else:
+                    self.proc.interrupt(queue_entry.stash_rev)
+                logger.debug(f"Task {rev} had been killed.")
+            except ProcessLookupError:
+                fail_to_kill_entries[queue_entry] = rev
+        return fail_to_kill_entries
+
+    def _mark_inactive_tasks_failure(self, remained_entries):
+        remained_revs: List[str] = []
+        running_ids = self._get_running_task_ids()
+        logger.debug(f"Current running tasks ids: {running_ids}.")
+        for msg, entry in self._iter_processed():
+            if entry not in remained_entries:
+                continue
+            task_id = msg.headers["id"]
+            if task_id in running_ids:
+                remained_revs.append(remained_entries[entry])
+            else:
+                result: AsyncResult = AsyncResult(task_id)
+                if not result.ready():
+                    logger.debug(
+                        f"Task id {task_id} rev {remained_entries[entry]} "
+                        "marked as failure."
+                    )
+                    self.celery.backend.mark_as_failure(task_id, None)
+
+        if remained_revs:
+            raise CannotKillTasksError(remained_revs)
+
+    def _kill_entries(self, entries: Dict[QueueEntry, str], force: bool):
+        logger.debug(
+            "Found active tasks: '%s' to kill",
+            list(entries.values()),
+        )
+        inactive_entries: Dict[QueueEntry, str] = self._try_to_kill_tasks(
+            entries, force
+        )
+
+        if inactive_entries:
+            self._mark_inactive_tasks_failure(inactive_entries)
+
+    def kill(self, revs: Collection[str], force: bool = False) -> None:
         name_dict: Dict[
             str, Optional[QueueEntry]
         ] = self.match_queue_entry_by_name(set(revs), self.iter_active())
 
-        missing_rev: List[str] = []
+        missing_revs: List[str] = []
+        to_kill: Dict[QueueEntry, str] = {}
         for rev, queue_entry in name_dict.items():
             if queue_entry is None:
-                missing_rev.append(rev)
+                missing_revs.append(rev)
             else:
-                to_kill.add(queue_entry)
+                to_kill[queue_entry] = rev
 
-        if missing_rev:
-            raise UnresolvedQueueExpNamesError(missing_rev)
+        if missing_revs:
+            raise UnresolvedQueueExpNamesError(missing_revs)
 
-        for queue_entry in to_kill:
-            self.proc.kill(queue_entry.stash_rev)
+        if to_kill:
+            self._kill_entries(to_kill, force)
 
     def shutdown(self, kill: bool = False):
         self.celery.control.shutdown()
         if kill:
-            for _, task_entry in self._iter_active_tasks():
-                try:
-                    self.proc.kill(task_entry.stash_rev)
-                except ProcessLookupError:
-                    continue
+            to_kill: Dict[QueueEntry, str] = {}
+            for entry in self.iter_active():
+                to_kill[entry] = entry.name or entry.stash_rev
+            if to_kill:
+                self._kill_entries(to_kill, True)
 
     def follow(
         self,
@@ -359,10 +428,11 @@ class LocalCeleryQueue(BaseStashQueue):
         ) as fobj:
             ui.write(fobj.read())
 
-    def worker_status(self) -> Dict:
+    def worker_status(self) -> Dict[str, List[Dict]]:
         """Return the current active celery worker"""
         status = self.celery.control.inspect().active() or {}
         logger.debug(f"Worker status: {status}")
+
         return status
 
     def clear(self, *args, **kwargs):
@@ -389,3 +459,31 @@ class LocalCeleryQueue(BaseStashQueue):
                 )
             )
         return result
+
+    def get_ref_and_entry_by_names(
+        self,
+        exp_names: Union[str, List[str]],
+        git_remote: Optional[str] = None,
+    ) -> Dict[str, ExpRefAndQueueEntry]:
+        """Find finished ExpRefInfo or queued or failed QueueEntry by name"""
+        from ..utils import resolve_name
+
+        if isinstance(exp_names, str):
+            exp_names = [exp_names]
+        results: Dict[str, ExpRefAndQueueEntry] = {}
+
+        exp_ref_match: Dict[str, Optional["ExpRefInfo"]] = resolve_name(
+            self.scm, exp_names, git_remote
+        )
+        if not git_remote:
+            queue_entry_match: Dict[
+                str, Optional["QueueEntry"]
+            ] = self.match_queue_entry_by_name(
+                exp_names, self.iter_queued(), self.iter_done()
+            )
+
+        for exp_name in exp_names:
+            exp_ref = exp_ref_match[exp_name]
+            queue_entry = None if git_remote else queue_entry_match[exp_name]
+            results[exp_name] = ExpRefAndQueueEntry(exp_ref, queue_entry)
+        return results

dvc/repo/experiments/queue/exceptions.py

@@ -0,0 +1,12 @@
+from typing import Collection
+
+from dvc.exceptions import DvcException
+
+
+class CannotKillTasksError(DvcException):
+    def __init__(self, revs: Collection[str]):
+        rev_str = ",".join(revs)
+        super().__init__(
+            f"Task '{rev_str}' is initializing, please wait a few seconds "
+            "until the experiments start running to retry the kill operation."
+        )

dvc/repo/experiments/queue/tasks.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict
+from typing import TYPE_CHECKING, Any, Dict
 
 from celery import shared_task
 from celery.utils.log import get_task_logger
@@ -7,11 +7,15 @@ from ..executor.base import ExecutorInfo
 from ..executor.local import TempDirExecutor
 from .base import BaseStashQueue, QueueEntry
 
+if TYPE_CHECKING:
+    from ..executor.base import BaseExecutor
+
+
 logger = get_task_logger(__name__)
 
 
 @shared_task
-def setup_exp(entry_dict: Dict[str, Any]) -> TempDirExecutor:
+def setup_exp(entry_dict: Dict[str, Any]) -> "BaseExecutor":
     """Setup an experiment.
 
     Arguments:

dvc/repo/experiments/queue/tempdir.py

@@ -1,19 +1,21 @@
 import logging
 import os
+from collections import defaultdict
 from typing import TYPE_CHECKING, Dict, Generator, Optional
 
 from funcy import cached_property, first
 
+from dvc.exceptions import DvcException
+
 from ..exceptions import ExpQueueEmptyError
 from ..executor.base import (
-    EXEC_PID_DIR,
-    EXEC_TMP_DIR,
     BaseExecutor,
     ExecutorInfo,
     ExecutorResult,
     TaskStatus,
 )
 from ..executor.local import TempDirExecutor
+from ..utils import EXEC_PID_DIR, EXEC_TMP_DIR
 from .base import BaseStashQueue, QueueEntry, QueueGetResult
 from .utils import fetch_running_exp_from_temp_dir
 from .workspace import WorkspaceQueue
@@ -77,9 +79,8 @@ class TempDirQueue(WorkspaceQueue):
         for stash_rev in self.proc:
             infofile = self.get_infofile_path(stash_rev)
             executor_info = ExecutorInfo.load_json(infofile)
-            if (
-                not executor_info.status <= TaskStatus.SUCCESS
-                and os.path.exists(executor_info.root_dir)
+            if executor_info.status <= TaskStatus.SUCCESS and os.path.exists(
+                executor_info.root_dir
             ):
                 yield QueueEntry(
                     self.repo.root_dir,
@@ -92,6 +93,51 @@ class TempDirQueue(WorkspaceQueue):
                     None,
                 )
 
+    def _reproduce_entry(
+        self, entry: QueueEntry, executor: BaseExecutor
+    ) -> Dict[str, Dict[str, str]]:
+        from dvc.stage.monitor import CheckpointKilledError
+
+        results: Dict[str, Dict[str, str]] = defaultdict(dict)
+        exec_name = self._EXEC_NAME or entry.stash_rev
+        infofile = self.get_infofile_path(exec_name)
+        try:
+            rev = entry.stash_rev
+            exec_result = executor.reproduce(
+                info=executor.info,
+                rev=rev,
+                infofile=infofile,
+                log_level=logger.getEffectiveLevel(),
+                log_errors=True,
+            )
+            if not exec_result.exp_hash:
+                raise DvcException(
+                    f"Failed to reproduce experiment '{rev[:7]}'"
+                )
+            if exec_result.ref_info:
+                results[rev].update(
+                    self.collect_executor(
+                        self.repo.experiments, executor, exec_result
+                    )
+                )
+        except CheckpointKilledError:
+            results[rev].update(
+                self.collect_executor(
+                    self.repo.experiments, executor, exec_result
+                )
+            )
+
+            return results
+        except DvcException:
+            raise
+        except Exception as exc:
+            raise DvcException(
+                f"Failed to reproduce experiment '{rev[:7]}'"
+            ) from exc
+        finally:
+            executor.cleanup(infofile)
+        return results
+
     @staticmethod
     def collect_executor(
         exp: "Experiments",

dvc/repo/experiments/queue/utils.py

@@ -1,17 +1,40 @@
 import logging
-from typing import TYPE_CHECKING, Dict
+from typing import TYPE_CHECKING, Dict, List
 
 from scmrepo.exceptions import SCMError
 
 from ..executor.base import ExecutorInfo, TaskStatus
+from ..refs import EXEC_CHECKPOINT, EXEC_NAMESPACE, EXPS_NAMESPACE, EXPS_STASH
+from ..utils import get_exp_rwlock, iter_remote_refs
 
 logger = logging.getLogger(__name__)
 
 
 if TYPE_CHECKING:
+    from scmrepo.git import Git
+
     from .base import BaseStashQueue
 
 
+def get_remote_executor_refs(scm: "Git", remote_url: str) -> List[str]:
+    """Get result list refs from a remote repository
+
+    Args:
+        remote_url : remote executor's url
+    """
+    refs = []
+    for ref in iter_remote_refs(
+        scm,
+        remote_url,
+        base=EXPS_NAMESPACE,
+    ):
+        if ref == EXEC_CHECKPOINT or (
+            not ref.startswith(EXEC_NAMESPACE) and ref != EXPS_STASH
+        ):
+            refs.append(ref)
+    return refs
+
+
 def fetch_running_exp_from_temp_dir(
     queue: "BaseStashQueue", rev: str, fetch_refs: bool
 ) -> Dict[str, Dict]:
@@ -36,24 +59,27 @@ def fetch_running_exp_from_temp_dir(
         info = ExecutorInfo.from_dict(load_json(infofile))
     except OSError:
         return result
-    if info.status < TaskStatus.FAILED:
+    if info.status <= TaskStatus.RUNNING:
         result[rev] = info.asdict()
         if info.git_url and fetch_refs and info.status > TaskStatus.PREPARING:
 
             def on_diverged(_ref: str, _checkpoint: bool):
-                return False
+                return True
 
             executor = TempDirExecutor.from_info(info)
             try:
-                for ref in executor.fetch_exps(
-                    queue.scm,
-                    on_diverged=on_diverged,
-                ):
-                    logger.debug("Updated running experiment '%s'.", ref)
-                    last_rev = queue.scm.get_ref(ref)
-                    result[rev]["last"] = last_rev
-                    if last_rev:
-                        result[last_rev] = info.asdict()
+                refs = get_remote_executor_refs(queue.scm, executor.git_url)
+                with get_exp_rwlock(queue.repo, writes=refs):
+                    for ref in executor.fetch_exps(
+                        queue.scm,
+                        refs,
+                        on_diverged=on_diverged,
+                    ):
+                        logger.debug("Updated running experiment '%s'.", ref)
+                        last_rev = queue.scm.get_ref(ref)
+                        result[rev]["last"] = last_rev
+                        if last_rev:
+                            result[last_rev] = info.asdict()
             except (InvalidRemoteSCMRepo, SCMError):
                 # ignore stale info files
                 del result[rev]

dvc/repo/experiments/queue/workspace.py

@@ -21,7 +21,8 @@ from ..executor.base import (
     TaskStatus,
 )
 from ..executor.local import WorkspaceExecutor
-from ..refs import EXEC_BRANCH
+from ..refs import EXEC_BRANCH, WORKSPACE_STASH
+from ..utils import get_exp_rwlock, to_studio_params
 from .base import BaseStashQueue, QueueDoneResult, QueueEntry, QueueGetResult
 
 if TYPE_CHECKING:
@@ -34,7 +35,8 @@ class WorkspaceQueue(BaseStashQueue):
     _EXEC_NAME: Optional[str] = "workspace"
 
     def put(self, *args, **kwargs) -> QueueEntry:
-        return self._stash_exp(*args, **kwargs)
+        with get_exp_rwlock(self.repo, writes=["workspace", WORKSPACE_STASH]):
+            return self._stash_exp(*args, **kwargs)
 
     def get(self) -> QueueGetResult:
         revs = self.stash.stash_revs
@@ -94,6 +96,8 @@ class WorkspaceQueue(BaseStashQueue):
     def _reproduce_entry(
         self, entry: QueueEntry, executor: BaseExecutor
     ) -> Dict[str, Dict[str, str]]:
+        from dvc_studio_client.post_live_metrics import post_live_metrics
+
         from dvc.stage.monitor import CheckpointKilledError
 
         results: Dict[str, Dict[str, str]] = defaultdict(dict)
@@ -101,6 +105,13 @@ class WorkspaceQueue(BaseStashQueue):
         infofile = self.get_infofile_path(exec_name)
         try:
             rev = entry.stash_rev
+            post_live_metrics(
+                "start",
+                executor.info.baseline_rev,
+                executor.info.name,
+                "dvc",
+                params=to_studio_params(self.repo.params.show()),
+            )
             exec_result = executor.reproduce(
                 info=executor.info,
                 rev=rev,
@@ -128,6 +139,13 @@ class WorkspaceQueue(BaseStashQueue):
                 f"Failed to reproduce experiment '{rev[:7]}'"
             ) from exc
         finally:
+            post_live_metrics(
+                "done",
+                executor.info.baseline_rev,
+                executor.info.name,
+                "dvc",
+                experiment_rev=first(results.get(rev, {})),
+            )
             executor.cleanup(infofile)
         return results
 

dvc/repo/experiments/refs.py

@@ -16,6 +16,7 @@ EXEC_BRANCH = f"{EXEC_NAMESPACE}/EXEC_BRANCH"
 EXEC_BASELINE = f"{EXEC_NAMESPACE}/EXEC_BASELINE"
 EXEC_HEAD = f"{EXEC_NAMESPACE}/EXEC_HEAD"
 EXEC_MERGE = f"{EXEC_NAMESPACE}/EXEC_MERGE"
+EXPS_TEMP = f"{EXPS_NAMESPACE}/temp"
 STASHES = {WORKSPACE_STASH, CELERY_STASH}
 
 
@@ -23,9 +24,7 @@ class ExpRefInfo:
 
     namespace = EXPS_NAMESPACE
 
-    def __init__(
-        self, baseline_sha: Optional[str] = None, name: Optional[str] = None
-    ):
+    def __init__(self, baseline_sha: str, name: Optional[str] = None):
         self.baseline_sha = baseline_sha
         self.name: str = name if name else ""
 
@@ -33,7 +32,7 @@ class ExpRefInfo:
         return "/".join(self.parts)
 
     def __repr__(self):
-        baseline = f"'{self.baseline_sha}'" if self.baseline_sha else "None"
+        baseline = f"'{self.baseline_sha}'"
         name = f"'{self.name}'" if self.name else "None"
         return f"ExpRefInfo(baseline_sha={baseline}, name={name})"
 
@@ -41,11 +40,7 @@ class ExpRefInfo:
     def parts(self):
         return (
             (self.namespace,)
-            + (
-                (self.baseline_sha[:2], self.baseline_sha[2:])
-                if self.baseline_sha
-                else ()
-            )
+            + ((self.baseline_sha[:2], self.baseline_sha[2:]))
             + ((self.name,) if self.name else ())
         )
 
@@ -54,14 +49,13 @@ class ExpRefInfo:
         try:
             parts = ref.split("/")
             if (
-                len(parts) < 2
-                or len(parts) == 3
+                len(parts) < 4
                 or len(parts) > 5
                 or "/".join(parts[:2]) != EXPS_NAMESPACE
             ):
                 raise InvalidExpRefError(ref)
         except ValueError:
             raise InvalidExpRefError(ref)
-        baseline_sha = parts[2] + parts[3] if len(parts) >= 4 else None
+        baseline_sha = parts[2] + parts[3]
         name = parts[4] if len(parts) == 5 else None
         return cls(baseline_sha, name)

dvc/repo/experiments/remove.py

@@ -1,19 +1,12 @@
 import logging
-from typing import (
-    TYPE_CHECKING,
-    Dict,
-    Iterable,
-    List,
-    NamedTuple,
-    Optional,
-    Union,
-)
+from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Union
 
 from dvc.repo import locked
 from dvc.repo.scm_context import scm_context
 from dvc.scm import iter_revs
 
 from .exceptions import UnresolvedExpNamesError
+from .queue.base import QueueEntry
 from .refs import ExpRefInfo
 from .utils import exp_refs, exp_refs_by_baseline, push_refspec
 
@@ -22,58 +15,12 @@ if TYPE_CHECKING:
     from dvc.repo.experiments.queue.celery import LocalCeleryQueue
     from dvc.scm import Git
 
-    from .queue.base import QueueEntry
+    from .queue.base import ExpRefAndQueueEntry
 
 
 logger = logging.getLogger(__name__)
 
 
-class ExpRefAndQueueEntry(NamedTuple):
-    exp_ref_list: List["ExpRefInfo"]
-    queue_entry_list: List["QueueEntry"]
-    removed: List["str"]
-
-
-def _get_ref_and_entry_by_names(
-    exp_names: Union[str, List[str]],
-    scm: "Git",
-    celery_queue: "LocalCeleryQueue",
-    git_remote: Optional[str],
-) -> ExpRefAndQueueEntry:
-    from .utils import resolve_name
-
-    exp_ref_list: List["ExpRefInfo"] = []
-    queue_entry_list: List["QueueEntry"] = []
-    removed: List[str] = []
-    if isinstance(exp_names, str):
-        exp_names = [exp_names]
-    exp_ref_match: Dict[str, Optional["ExpRefInfo"]] = resolve_name(
-        scm, exp_names, git_remote
-    )
-    if not git_remote:
-        queue_entry_match: Dict[
-            str, Optional["QueueEntry"]
-        ] = celery_queue.match_queue_entry_by_name(
-            exp_names, celery_queue.iter_queued(), celery_queue.iter_done()
-        )
-
-    remained = []
-    for exp_name in exp_names:
-        exp_ref = exp_ref_match[exp_name]
-        queue_entry = None if git_remote else queue_entry_match[exp_name]
-        if exp_ref or queue_entry:
-            if exp_ref:
-                exp_ref_list.append(exp_ref)
-            if queue_entry:
-                queue_entry_list.append(queue_entry)
-            removed.append(exp_name)
-        else:
-            remained.append(exp_name)
-    if remained:
-        raise UnresolvedExpNamesError(remained)
-    return ExpRefAndQueueEntry(exp_ref_list, queue_entry_list, removed)
-
-
 @locked
 @scm_context
 def remove(
@@ -103,12 +50,22 @@ def remove(
     exp_ref_list: List["ExpRefInfo"] = []
     queue_entry_list: List["QueueEntry"] = []
     if exp_names:
-        result = _get_ref_and_entry_by_names(
-            exp_names, repo.scm, celery_queue, git_remote
-        )
-        removed.extend(result.removed)
-        exp_ref_list.extend(result.exp_ref_list)
-        queue_entry_list.extend(result.queue_entry_list)
+        results: Dict[
+            str, "ExpRefAndQueueEntry"
+        ] = celery_queue.get_ref_and_entry_by_names(exp_names, git_remote)
+        remained: List[str] = []
+        for name, result in results.items():
+            if not result.exp_ref_info and not result.queue_entry:
+                remained.append(name)
+                continue
+            removed.append(name)
+            if result.exp_ref_info:
+                exp_ref_list.append(result.exp_ref_info)
+            if result.queue_entry:
+                queue_entry_list.append(result.queue_entry)
+
+        if remained:
+            raise UnresolvedExpNamesError(remained)
     elif rev:
         exp_ref_dict = _resolve_exp_by_baseline(repo, rev, num, git_remote)
         removed.extend(exp_ref_dict.keys())
@@ -152,8 +109,7 @@ def _remove_commited_exps(
                 push_refspec(
                     scm,
                     remote,
-                    None,
-                    str(ref_info),
+                    [(None, str(ref_info))],
                     progress=pbar.update_git,
                 )
     else:

dvc/repo/experiments/save.py

@@ -0,0 +1,80 @@
+import logging
+import os
+from typing import TYPE_CHECKING, List, Optional
+
+from .exceptions import ExperimentExistsError
+from .refs import ExpRefInfo
+from .utils import check_ref_format, get_random_exp_name
+
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+
+
+logger = logging.getLogger(__name__)
+
+
+def _save_experiment(
+    repo: "Repo",
+    baseline_rev: str,
+    force: bool,
+    name: Optional[str],
+    include_untracked: Optional[List[str]],
+) -> str:
+    repo.commit([], force=True)
+
+    name = name or get_random_exp_name(repo.scm, baseline_rev)
+    ref_info = ExpRefInfo(baseline_rev, name)
+    check_ref_format(repo.scm.dulwich, ref_info)
+    ref = str(ref_info)
+    if repo.scm.get_ref(ref) and not force:
+        raise ExperimentExistsError(ref_info.name, command="save")
+
+    repo.scm.add([], update=True)
+    if include_untracked:
+        repo.scm.add(include_untracked)
+    repo.scm.commit(f"dvc: commit experiment {name}", no_verify=True)
+    exp_rev = repo.scm.get_rev()
+    repo.scm.set_ref(ref, exp_rev, old_ref=None)
+
+    return exp_rev
+
+
+def save(
+    repo: "Repo",
+    name: Optional[str] = None,
+    force: bool = False,
+    include_untracked: Optional[List[str]] = None,
+) -> Optional[str]:
+    """Save the current workspace status as an experiment.
+
+    Returns the saved experiment's SHAs.
+    """
+    logger.debug("Saving workspace in %s", os.getcwd())
+
+    _, _, untracked = repo.scm.status()
+    if include_untracked:
+        untracked = [
+            file for file in untracked if file not in include_untracked
+        ]
+    if untracked:
+        logger.warning(
+            "The following untracked files were present in "
+            "the workspace before saving but "
+            "will not be included in the experiment commit:\n"
+            "\t%s",
+            ", ".join(untracked),
+        )
+
+    with repo.scm.detach_head(client="dvc") as orig_head:
+        with repo.scm.stash_workspace() as workspace:
+            try:
+                if workspace is not None:
+                    repo.scm.stash.apply(workspace)
+
+                exp_rev = _save_experiment(
+                    repo, orig_head, force, name, include_untracked
+                )
+            finally:
+                repo.scm.reset(hard=True)
+
+    return exp_rev

dvc/repo/experiments/show.py

@@ -8,8 +8,10 @@ from typing import (
     Any,
     Callable,
     Dict,
+    Iterable,
     List,
     Optional,
+    Set,
     Tuple,
     Union,
 )
@@ -24,6 +26,7 @@ from dvc.utils import error_handler, onerror_collect, relpath
 from .refs import ExpRefInfo
 
 if TYPE_CHECKING:
+    from scmrepo.git import Git
     from scmrepo.git.objects import GitCommit
 
     from dvc.repo import Repo
@@ -134,6 +137,7 @@ def _collect_complete_experiment(
 
     checkpoint: bool = True if len(revs) > 1 else False
     prev = ""
+
     for rev in revs:
         status = ExpStatus.Running if rev in running else ExpStatus.Success
         collected_exp = collect_experiment_commit(
@@ -215,41 +219,55 @@ def _collect_branch(
     return results
 
 
-def get_names(repo: "Repo", result: Dict[str, Dict[str, Any]]):
+def get_branch_names(
+    scm: "Git", baseline_set: Iterable[str]
+) -> Dict[str, Optional[str]]:
+    names: Dict[str, Optional[str]] = {}
+    for base in [
+        f"refs/exps/{baseline[:2]}/{baseline[2:]}/"
+        for baseline in baseline_set
+    ] + ["refs/heads/", "refs/tags/"]:
+        for ref in scm.iter_refs(base=base):
+            if ref:
+                try:
+                    rev = scm.get_ref(ref)
+                    names[rev] = ref[len(base) :]
+                except KeyError:
+                    logger.debug("unreosolved ref %s", ref)
+    logger.debug("found refs for revs: %s", names)
+    return names
+
+
+def update_names(
+    repo: "Repo",
+    branch_names: Dict[str, Optional[str]],
+    result: Dict[str, Dict[str, Any]],
+):
 
     rev_set = set()
-    baseline_set = set()
     for baseline in result:
         for rev in result[baseline]:
             if rev == "baseline":
                 rev = baseline
-                baseline_set.add(rev)
             if rev != "workspace":
                 rev_set.add(rev)
 
-    names: Dict[str, Optional[str]] = {}
-    for base in ("refs/tags/", "refs/heads/"):
-        if rev_set:
-            names.update(
-                (rev, ref[len(base) :])
-                for rev, ref in repo.scm.describe(
-                    baseline_set, base=base
-                ).items()
-                if ref is not None
-            )
-            rev_set.difference_update(names.keys())
+    if rev_set:
+        rev_set.difference_update(branch_names.keys())
 
     exact_name = repo.experiments.get_exact_name(rev_set)
 
     for baseline, baseline_results in result.items():
+        name_set: Set[str] = set()
         for rev, rev_result in baseline_results.items():
             name: Optional[str] = None
             if rev == "baseline":
                 rev = baseline
                 if rev == "workspace":
                     continue
-            name = names.get(rev, None) or exact_name[rev]
-            if name:
+            name = branch_names.get(rev, None) or exact_name[rev]
+            if name and name not in name_set:
+                name_set.add(name)
                 rev_result["data"]["name"] = name
 
 
@@ -336,6 +354,40 @@ def update_new(
             to_dict[baseline][rev] = to_dict[baseline].get(rev, experiment)
 
 
+def move_properties_to_head(result: Dict[str, Dict[str, Dict[str, Any]]]):
+    for _, baseline_results in result.items():
+        checkpoint: bool = False
+        head: Dict[str, Any] = {}
+        for rev, rev_data in baseline_results.items():
+            if (
+                "data" not in rev_data
+                or rev_data["data"].get("checkpoint_tip", None) is None
+            ):
+                checkpoint = False
+                head = {}
+                continue
+
+            rev_result: Dict[str, Any] = rev_data["data"]
+            if (
+                checkpoint is True
+                and rev_result["checkpoint_tip"] == head["checkpoint_tip"]
+            ):
+                if "name" in rev_result and "name" not in head:
+                    head["name"] = rev_result["name"]
+                    del rev_result["name"]
+                if rev_result["executor"]:
+                    if not head["executor"]:
+                        head["executor"] = rev_result["executor"]
+                    rev_result["executor"] = None
+                if rev_result["status"] == ExpStatus.Running.name:
+                    head["status"] = ExpStatus.Running.name
+                    rev_result["status"] = ExpStatus.Success.name
+            else:
+                if rev_result["checkpoint_tip"] == rev:
+                    head = rev_result
+                    checkpoint = True
+
+
 def show(
     repo: "Repo",
     all_branches=False,
@@ -354,8 +406,7 @@ def show(
     if repo.scm.no_commits:
         return {}
 
-    if onerror is None:
-        onerror = _show_onerror_collect
+    onerror = onerror or _show_onerror_collect
 
     res: Dict[str, Dict] = defaultdict(OrderedDict)
 
@@ -368,6 +419,7 @@ def show(
     found_revs.update(
         iter_revs(repo.scm, revs, num, all_branches, all_tags, all_commits)
     )
+    branch_names = get_branch_names(repo.scm, found_revs)
 
     running: Dict[str, Dict] = repo.experiments.get_running_exps(
         fetch_refs=fetch_running
@@ -421,6 +473,8 @@ def show(
     update_new(res, queued_experiment)
 
     if not sha_only:
-        get_names(repo, res)
+        update_names(repo, branch_names, res)
+
+    move_properties_to_head(res)
 
     return res

dvc/repo/fetch.py

@@ -1,6 +1,6 @@
 import logging
 from contextlib import suppress
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, Optional, Sequence
 
 from dvc.config import NoRemoteError
 from dvc.exceptions import DownloadError
@@ -9,6 +9,7 @@ from dvc.fs import Schemes
 from . import locked
 
 if TYPE_CHECKING:
+    from dvc.cloud import Remote
     from dvc.repo import Repo
     from dvc.types import TargetType
     from dvc_data.hashfile.transfer import TransferResult
@@ -17,20 +18,6 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-def _fetch_worktree(repo, remote):
-    from dvc_data.index import save
-
-    index = repo.index.data["repo"]
-    for key, entry in index.iteritems():
-        entry.fs = remote.fs
-        entry.path = remote.fs.path.join(
-            remote.path,
-            *key,
-        )
-    save(index)
-    return len(index)
-
-
 @locked
 def fetch(
     self,
@@ -64,12 +51,14 @@ def fetch(
     if isinstance(targets, str):
         targets = [targets]
 
+    worktree_remote: Optional["Remote"] = None
     with suppress(NoRemoteError):
         _remote = self.cloud.get_remote(name=remote)
-        if _remote.worktree:
-            return _fetch_worktree(self, _remote)
+        if _remote.worktree or _remote.fs.version_aware:
+            worktree_remote = _remote
 
     failed_count = 0
+    transferred_count = 0
 
     try:
         if run_cache:
@@ -80,22 +69,36 @@ def fetch(
     no_remote_msg: Optional[str] = None
     result = TransferResult(set(), set())
     try:
-        d, f = _fetch(
-            self,
-            targets,
-            all_branches=all_branches,
-            all_tags=all_tags,
-            all_commits=all_commits,
-            with_deps=with_deps,
-            force=True,
-            remote=remote,
-            jobs=jobs,
-            recursive=recursive,
-            revs=revs,
-            odb=odb,
-        )
-        result.transferred.update(d)
-        result.failed.update(f)
+        if worktree_remote is not None:
+            transferred_count += _fetch_worktree(
+                self,
+                worktree_remote,
+                revs=revs,
+                all_branches=all_branches,
+                all_tags=all_tags,
+                all_commits=all_commits,
+                targets=targets,
+                jobs=jobs,
+                with_deps=with_deps,
+                recursive=recursive,
+            )
+        else:
+            d, f = _fetch(
+                self,
+                targets,
+                all_branches=all_branches,
+                all_tags=all_tags,
+                all_commits=all_commits,
+                with_deps=with_deps,
+                force=True,
+                remote=remote,
+                jobs=jobs,
+                recursive=recursive,
+                revs=revs,
+                odb=odb,
+            )
+            result.transferred.update(d)
+            result.failed.update(f)
     except NoRemoteError as exc:
         no_remote_msg = str(exc)
 
@@ -121,7 +124,8 @@ def fetch(
             logger.error(no_remote_msg)
         raise DownloadError(failed_count)
 
-    return len(result.transferred)
+    transferred_count += len(result.transferred)
+    return transferred_count
 
 
 def _fetch(
@@ -168,3 +172,29 @@ def _fetch(
             result.transferred.update(d)
             result.failed.update(f)
     return result
+
+
+def _fetch_worktree(
+    repo: "Repo",
+    remote: "Remote",
+    revs: Optional[Sequence[str]] = None,
+    all_branches: bool = False,
+    all_tags: bool = False,
+    all_commits: bool = False,
+    targets: Optional["TargetType"] = None,
+    jobs: Optional[int] = None,
+    **kwargs,
+) -> int:
+    from dvc.repo.worktree import fetch_worktree
+
+    downloaded = 0
+    for _ in repo.brancher(
+        revs=revs,
+        all_branches=all_branches,
+        all_tags=all_tags,
+        all_commits=all_commits,
+    ):
+        downloaded += fetch_worktree(
+            repo, remote, targets=targets, jobs=jobs, **kwargs
+        )
+    return downloaded

dvc/repo/imp_url.py

@@ -97,6 +97,9 @@ def imp_url(
     else:
         stage.run(jobs=jobs, no_download=no_download)
 
+    if not no_exec and stage.deps[0].fs.version_aware:
+        stage.outs[0].can_push = False
+
     stage.frozen = frozen
 
     dvcfile.dump(stage)

dvc/repo/index.py

@@ -1,4 +1,3 @@
-from contextlib import suppress
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -11,11 +10,10 @@ from typing import (
     Optional,
     Set,
     Tuple,
+    Union,
 )
 
-from funcy import cached_property, nullcontext
-
-from dvc.utils import dict_md5
+from funcy import cached_property
 
 if TYPE_CHECKING:
     from networkx import DiGraph
@@ -24,9 +22,10 @@ if TYPE_CHECKING:
     from dvc.dependency import Dependency, ParamsDependency
     from dvc.fs import FileSystem
     from dvc.output import Output
+    from dvc.repo import Repo
     from dvc.repo.stage import StageInfo, StageLoad
     from dvc.stage import Stage
-    from dvc.types import StrPath, TargetType
+    from dvc.types import TargetType
     from dvc_data.hashfile.hash_info import HashInfo
     from dvc_data.index import DataIndex, DataIndexKey, DataIndexView
     from dvc_objects.db import ObjectDB
@@ -67,14 +66,65 @@ class Index:
         if stages is not None:
             self.stages: List["Stage"] = stages
         self._collected_targets: Dict[int, List["StageInfo"]] = {}
+        self._metrics: Dict[str, List[str]] = {}
+        self._plots: Dict[str, Any] = {}
+        self._params: Dict[str, List[str]] = {}
 
     @cached_property
     def stages(self) -> List["Stage"]:  # pylint: disable=method-hidden
         # note that ideally we should be keeping this in a set as it is unique,
         # hashable and has no concept of orderliness on its own. But we depend
         # on this to be somewhat ordered for status/metrics/plots, etc.
+        return self._collect()
+
+    @cached_property
+    def _top_metrics(self):
+        self._collect()
+        return self._metrics
+
+    @cached_property
+    def _top_plots(self):
+        self._collect()
+        return self._plots
+
+    @cached_property
+    def _plot_sources(self):
+        from dvc.repo.plots import _collect_pipeline_files
+
+        sources = []
+        for data in _collect_pipeline_files(self.repo, [], {}).values():
+            for plot_id, props in data.get("data", {}).items():
+                if isinstance(props.get("y"), dict):
+                    sources.extend(props["y"])
+                    if isinstance(props.get("x"), dict):
+                        sources.extend(props["x"])
+                else:
+                    sources.append(plot_id)
+        return sources
+
+    @cached_property
+    def _top_params(self):
+        self._collect()
+        return self._params
+
+    def _collect(self):
+        if "stages" in self.__dict__:
+            return self.stages
+
         onerror = self.repo.stage_collection_error_handler
-        return self.stage_collector.collect_repo(onerror=onerror)
+
+        # pylint: disable=protected-access
+        (
+            stages,
+            metrics,
+            plots,
+            params,
+        ) = self.stage_collector._collect_all_from_repo(onerror=onerror)
+        self.stages = stages
+        self._metrics = metrics
+        self._plots = plots
+        self._params = params
+        return stages
 
     def __repr__(self) -> str:
         from dvc.fs import LocalFileSystem
@@ -84,38 +134,6 @@ class Index:
             rev = self.repo.get_rev()[:7]
         return f"Index({self.repo}, fs@{rev})"
 
-    def __len__(self) -> int:
-        return len(self.stages)
-
-    def __contains__(self, stage: "Stage") -> bool:
-        # as we are keeping stages inside a list, it might be slower.
-        return stage in self.stages
-
-    def __iter__(self) -> Iterator["Stage"]:
-        yield from self.stages
-
-    def __getitem__(self, item: str) -> "Stage":
-        """Get a stage by its addressing attribute."""
-        for stage in self:
-            if stage.addressing == item:
-                return stage
-        raise KeyError(f"{item} - available stages are {self.stages}")
-
-    def filter(self, filter_fn: Callable[["Stage"], bool]) -> "Index":
-        stages_it = filter(filter_fn, self)
-        return Index(self.repo, self.fs, stages=list(stages_it))
-
-    def slice(self, path: "StrPath") -> "Index":
-        from dvc.utils import relpath
-        from dvc.utils.fs import path_isin
-
-        target_path = relpath(path, self.repo.root_dir)
-
-        def is_stage_inside_path(stage: "Stage") -> bool:
-            return path_isin(stage.path_in_repo, target_path)
-
-        return self.filter(is_stage_inside_path)
-
     @staticmethod
     def _hash_targets(
         targets: Iterable[Optional[str]],
@@ -129,7 +147,7 @@ class Index:
             )
         )
 
-    def _collect_targets(
+    def collect_targets(
         self, targets: Optional["TargetType"], **kwargs: Any
     ) -> List["StageInfo"]:
         from itertools import chain
@@ -152,7 +170,7 @@ class Index:
 
     def targets_view(
         self,
-        targets: "TargetType",
+        targets: Optional["TargetType"],
         stage_filter: Optional[Callable[["Stage"], bool]] = None,
         outs_filter: Optional[Callable[["Output"], bool]] = None,
         **kwargs: Any,
@@ -177,14 +195,14 @@ class Index:
         """
         stage_infos = [
             stage_info
-            for stage_info in self._collect_targets(targets, **kwargs)
+            for stage_info in self.collect_targets(targets, **kwargs)
             if not stage_filter or stage_filter(stage_info.stage)
         ]
         return IndexView(self, stage_infos, outs_filter=outs_filter)
 
     @property
     def outs(self) -> Iterator["Output"]:
-        for stage in self:
+        for stage in self.stages:
             yield from stage.outs
 
     @property
@@ -207,7 +225,7 @@ class Index:
 
     @property
     def deps(self) -> Iterator["Dependency"]:
-        for stage in self:
+        for stage in self.stages:
             yield from stage.deps
 
     @property
@@ -268,11 +286,12 @@ class Index:
         force: bool = False,
         recursive: bool = False,
         jobs: int = None,
+        push: bool = False,
     ) -> "ObjectContainer":
         from collections import defaultdict
 
         used: "ObjectContainer" = defaultdict(set)
-        pairs = self._collect_targets(
+        pairs = self.collect_targets(
             targets, recursive=recursive, with_deps=with_deps
         )
         for stage, filter_info in pairs:
@@ -281,6 +300,7 @@ class Index:
                 force=force,
                 jobs=jobs,
                 filter_info=filter_info,
+                push=push,
             ).items():
                 used[odb].update(objs)
         return used
@@ -299,56 +319,10 @@ class Index:
     def add(self, stage: "Stage") -> "Index":
         return self.update([stage])
 
-    def remove(
-        self, stage: "Stage", ignore_not_existing: bool = False
-    ) -> "Index":
-        stages = self._discard_stage(
-            stage, ignore_not_existing=ignore_not_existing
-        )
-        return Index(self.repo, self.fs, stages=stages)
-
-    def discard(self, stage: "Stage") -> "Index":
-        return self.remove(stage, ignore_not_existing=True)
-
-    def difference(self, stages: Iterable["Stage"]) -> "Index":
-        stages_set = set(self.stages) - set(stages)
-        return Index(self.repo, self.fs, stages=list(stages_set))
-
-    def _discard_stage(
-        self, stage: "Stage", ignore_not_existing: bool = False
-    ) -> List["Stage"]:
-        stages = self.stages[:]
-        ctx = suppress(ValueError) if ignore_not_existing else nullcontext()
-        with ctx:
-            stages.remove(stage)
-        return stages
-
     def check_graph(self) -> None:
         if not getattr(self.repo, "_skip_graph_checks", False):
             self.graph  # pylint: disable=pointless-statement
 
-    def dumpd(self) -> Dict[str, Dict]:
-        def dump(stage: "Stage"):
-            key = stage.path_in_repo
-            try:
-                key += ":" + stage.name  # type: ignore[attr-defined]
-            except AttributeError:
-                pass
-            return key, stage.dumpd()
-
-        return dict(dump(stage) for stage in self)
-
-    @cached_property
-    def identifier(self) -> str:
-        """Unique identifier for the index.
-
-        We can use this to optimize and skip opening some indices
-        eg: on push/pull/fetch/gc --all-commits.
-
-        Currently, it is unique to the platform (windows vs posix).
-        """
-        return dict_md5(self.dumpd())
-
 
 class _DataPrefixes(NamedTuple):
     explicit: Set["DataIndexKey"]
@@ -371,30 +345,13 @@ class IndexView:
         self._stages = list({stage for stage, _ in stage_infos})
         self._outs_filter = outs_filter
 
-    def __len__(self) -> int:
-        return len(self._stages)
-
-    def __contains__(self, stage: "Stage") -> bool:
-        # as we are keeping stages inside a list, it might be slower.
-        return stage in self._stages
-
-    def __iter__(self) -> Iterator["Stage"]:
-        yield from self._stages
-
-    def __getitem__(self, item: str) -> "Stage":
-        """Get a stage by its addressing attribute."""
-        for stage in self:
-            if stage.addressing == item:
-                return stage
-        raise KeyError(f"{item} - available stages are {self.stages}")
-
     @property
     def stages(self):
         return self._stages
 
     @property
     def deps(self) -> Iterator["Dependency"]:
-        for stage in self:
+        for stage in self.stages:
             yield from stage.deps
 
     @property
@@ -427,10 +384,10 @@ class IndexView:
         return prefixes
 
     @cached_property
-    def data(self) -> "Dict[str, DataIndexView]":
+    def data(self) -> Dict[str, Union["DataIndex", "DataIndexView"]]:
         from functools import partial
 
-        from dvc_data.index import view
+        from dvc_data.index import DataIndex, view
 
         def key_filter(workspace: str, key: "DataIndexKey"):
             try:
@@ -442,34 +399,12 @@ class IndexView:
             except KeyError:
                 return False
 
-        data = {}
+        data: Dict[str, Union["DataIndex", "DataIndexView"]] = {}
         for workspace, data_index in self._index.data.items():
-            data_index.load()
-            data[workspace] = view(data_index, partial(key_filter, workspace))
+            if self._stages:
+                data[workspace] = view(
+                    data_index, partial(key_filter, workspace)
+                )
+            else:
+                data[workspace] = DataIndex()
         return data
-
-
-if __name__ == "__main__":
-    import logging
-
-    from funcy import log_durations
-
-    from dvc.logger import setup
-    from dvc.repo import Repo
-
-    setup(level=logging.TRACE)  # type: ignore[attr-defined]
-
-    repo = Repo()
-    index = Index(repo, repo.fs)
-    print(index)
-    with log_durations(print, "collecting stages"):
-        # pylint: disable=pointless-statement
-        print("no of stages", len(index.stages))
-    with log_durations(print, "building graph"):
-        index.graph  # pylint: disable=pointless-statement
-    with log_durations(print, "calculating hash"):
-        print(index.identifier)
-    with log_durations(print, "updating"):
-        index2 = index.update(index.stages)
-    with log_durations(print, "calculating hash"):
-        print(index2.identifier)

dvc/repo/ls.py

@@ -70,11 +70,12 @@ def _ls(
         if not recursive:
             files.update(dirs)
 
+        parts = fs.path.relparts(root, fs_path)
+        if parts == (".",):
+            parts = ()
+
         for name, entry in files.items():
-            entry_fs_path = fs.path.join(root, name)
-            relparts = fs.path.relparts(entry_fs_path, fs_path)
-            name = os.path.join(*relparts)
-            infos[name] = entry
+            infos[os.path.join(*parts, name)] = entry
 
         if not recursive:
             break

dvc/repo/metrics/diff.py

@@ -11,7 +11,10 @@ def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
     a_rev = a_rev or "HEAD"
     b_rev = b_rev or "workspace"
 
-    metrics = repo.metrics.show(*args, **kwargs, revs=[a_rev, b_rev])
+    metrics = repo.metrics.show(
+        *args, **kwargs, revs=[a_rev, b_rev], hide_workspace=False
+    )
+
     old = metrics.get(a_rev, {}).get("data", {})
     new = metrics.get(b_rev, {}).get("data", {})
 

dvc/repo/metrics/show.py

@@ -9,7 +9,12 @@ from dvc.output import Output
 from dvc.repo import locked
 from dvc.repo.collect import StrPaths, collect
 from dvc.scm import NoSCMError
-from dvc.utils import error_handler, errored_revisions, onerror_collect
+from dvc.utils import (
+    as_posix,
+    error_handler,
+    errored_revisions,
+    onerror_collect,
+)
 from dvc.utils.collections import ensure_list
 from dvc.utils.serialize import load_path
 
@@ -28,6 +33,17 @@ def _to_fs_paths(metrics: List[Output]) -> StrPaths:
     return result
 
 
+def _collect_top_level_metrics(repo):
+    top_metrics = repo.index._top_metrics  # pylint: disable=protected-access
+    for dvcfile, metrics in top_metrics.items():
+        wdir = repo.fs.path.relpath(
+            repo.fs.path.parent(dvcfile), repo.root_dir
+        )
+        for file in metrics:
+            path = repo.fs.path.join(wdir, as_posix(file))
+            yield repo.fs.path.normpath(path)
+
+
 def _collect_metrics(repo, targets, revision, recursive):
     metrics, fs_paths = collect(
         repo,
@@ -94,6 +110,7 @@ def _read_metrics(repo, metrics, rev, onerror=None):
 
 def _gather_metrics(repo, targets, rev, recursive, onerror=None):
     metrics = _collect_metrics(repo, targets, rev, recursive)
+    metrics.extend(_collect_top_level_metrics(repo))
     return _read_metrics(repo, metrics, rev, onerror=onerror)
 
 
@@ -107,6 +124,7 @@ def show(
     revs=None,
     all_commits=False,
     onerror=None,
+    hide_workspace=True,
 ):
     if onerror is None:
         onerror = onerror_collect
@@ -125,16 +143,17 @@ def show(
             repo, targets, rev, recursive, onerror=onerror
         )
 
-    # Hide workspace metrics if they are the same as in the active branch
-    try:
-        active_branch = repo.scm.active_branch()
-    except (SCMError, NoSCMError):
-        # SCMError - detached head
-        # NoSCMError - no repo case
-        pass
-    else:
-        if res.get("workspace") == res.get(active_branch):
-            res.pop("workspace", None)
+    if hide_workspace:
+        # Hide workspace metrics if they are the same as in the active branch
+        try:
+            active_branch = repo.scm.active_branch()
+        except (SCMError, NoSCMError):
+            # SCMError - detached head
+            # NoSCMError - no repo case
+            pass
+        else:
+            if res.get("workspace") == res.get(active_branch):
+                res.pop("workspace", None)
 
     errored = errored_revisions(res)
     if errored:

dvc/repo/params/diff.py

@@ -11,7 +11,9 @@ def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
     a_rev = a_rev or "HEAD"
     b_rev = b_rev or "workspace"
 
-    params = repo.params.show(*args, **kwargs, revs=[a_rev, b_rev])
+    params = repo.params.show(
+        *args, **kwargs, revs=[a_rev, b_rev], hide_workspace=False
+    )
 
     old = params.get(a_rev, {}).get("data", {})
     new = params.get(b_rev, {}).get("data", {})

dvc/repo/params/show.py

@@ -20,7 +20,12 @@ from dvc.repo.collect import collect
 from dvc.scm import NoSCMError
 from dvc.stage import PipelineStage
 from dvc.ui import ui
-from dvc.utils import error_handler, errored_revisions, onerror_collect
+from dvc.utils import (
+    as_posix,
+    error_handler,
+    errored_revisions,
+    onerror_collect,
+)
 from dvc.utils.collections import ensure_list
 from dvc.utils.serialize import load_path
 
@@ -35,6 +40,17 @@ def _is_params(dep: "Output"):
     return isinstance(dep, ParamsDependency)
 
 
+def _collect_top_level_params(repo):
+    top_params = repo.index._top_params  # pylint: disable=protected-access
+    for dvcfile, params in top_params.items():
+        wdir = repo.fs.path.relpath(
+            repo.fs.path.parent(dvcfile), repo.root_dir
+        )
+        for file in params:
+            path = repo.fs.path.join(wdir, as_posix(file))
+            yield repo.fs.path.normpath(path)
+
+
 def _collect_configs(
     repo: "Repo", rev, targets=None, deps=False, stages=None
 ) -> Tuple[List["Output"], List[str]]:
@@ -128,6 +144,7 @@ def show(
     deps=False,
     onerror: Callable = None,
     stages=None,
+    hide_workspace=True,
 ):
     if onerror is None:
         onerror = onerror_collect
@@ -149,16 +166,17 @@ def show(
         if params:
             res[branch] = params
 
-    # Hide workspace params if they are the same as in the active branch
-    try:
-        active_branch = repo.scm.active_branch()
-    except (SCMError, NoSCMError):
-        # SCMError - detached head
-        # NoSCMError - no repo case
-        pass
-    else:
-        if res.get("workspace") == res.get(active_branch):
-            res.pop("workspace", None)
+    if hide_workspace:
+        # Hide workspace params if they are the same as in the active branch
+        try:
+            active_branch = repo.scm.active_branch()
+        except (SCMError, NoSCMError):
+            # SCMError - detached head
+            # NoSCMError - no repo case
+            pass
+        else:
+            if res.get("workspace") == res.get(active_branch):
+                res.pop("workspace", None)
 
     errored = errored_revisions(res)
     if errored:
@@ -176,6 +194,7 @@ def _gather_params(
     param_outs, params_fs_paths = _collect_configs(
         repo, rev, targets=targets, deps=deps, stages=stages
     )
+    params_fs_paths.extend(_collect_top_level_params(repo=repo))
     params = _read_params(
         repo,
         params=param_outs,

dvc/repo/plots/__init__.py

@@ -19,7 +19,7 @@ from typing import (
 
 import dpath.options
 import dpath.util
-from funcy import cached_property, first, project
+from funcy import cached_property, distinct, first, project
 
 from dvc.exceptions import DvcException
 from dvc.utils import error_handler, errored_revisions, onerror_collect
@@ -314,16 +314,18 @@ def _get_data_targets(definitions: Dict):
 
 
 def infer_data_sources(plot_id, config=None):
-    def _deduplicate(lst: List):
-        return list({elem: None for elem in lst}.keys())
-
     y = config.get("y", None)
+
     if isinstance(y, dict):
         sources = list(y.keys())
     else:
         sources = [plot_id]
 
-    return _deduplicate(source for source in sources)
+    x = config.get("x", None)
+    if isinstance(x, dict):
+        sources.append(first(x.keys()))
+
+    return distinct(source for source in sources)
 
 
 def _matches(targets, config_file, plot_id):
@@ -437,36 +439,33 @@ def _resolve_definitions(
 
 
 def _collect_pipeline_files(repo, targets: List[str], props, onerror=None):
-    from dvc.dvcfile import PipelineFile
-
     result: Dict[str, Dict] = {}
-    dvcfiles = {stage.dvcfile for stage in repo.index.stages}
-    for dvcfile in dvcfiles:
-        if isinstance(dvcfile, PipelineFile):
-            dvcfile_path = _relpath(repo.dvcfs, dvcfile.path)
-            dvcfile_defs = dvcfile.load().get("plots", {})
-            dvcfile_defs_dict: Dict[str, Union[Dict, None]] = {}
-            if isinstance(dvcfile_defs, list):
-                for elem in dvcfile_defs:
-                    if isinstance(elem, str):
-                        dvcfile_defs_dict[elem] = None
-                    else:
-                        k, v = list(elem.items())[0]
-                        dvcfile_defs_dict[k] = v
-            else:
-                dvcfile_defs_dict = dvcfile_defs
-            resolved = _resolve_definitions(
-                repo.dvcfs,
-                targets,
-                props,
-                dvcfile_path,
-                dvcfile_defs_dict,
-                onerror=onerror,
-            )
-            dpath.util.merge(
-                result,
-                {dvcfile_path: resolved},
-            )
+    top_plots = repo.index._top_plots  # pylint: disable=protected-access
+    for dvcfile, plots_def in top_plots.items():
+        dvcfile_path = _relpath(repo.dvcfs, dvcfile)
+        dvcfile_defs_dict: Dict[str, Union[Dict, None]] = {}
+        if isinstance(plots_def, list):
+            for elem in plots_def:
+                if isinstance(elem, str):
+                    dvcfile_defs_dict[elem] = None
+                else:
+                    k, v = list(elem.items())[0]
+                    dvcfile_defs_dict[k] = v
+        else:
+            dvcfile_defs_dict = plots_def
+
+        resolved = _resolve_definitions(
+            repo.dvcfs,
+            targets,
+            props,
+            dvcfile_path,
+            dvcfile_defs_dict,
+            onerror=onerror,
+        )
+        dpath.util.merge(
+            result,
+            {dvcfile_path: resolved},
+        )
     return result
 
 

dvc/repo/push.py

@@ -1,56 +1,17 @@
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, Optional, Sequence
 
-from dvc.exceptions import UploadError
+from dvc.exceptions import InvalidArgumentError, UploadError
 
 from ..utils import glob_targets
 from . import locked
 
 if TYPE_CHECKING:
+    from dvc.cloud import Remote
+    from dvc.repo import Repo
+    from dvc.types import TargetType
     from dvc_objects.db.base import ObjectDB
 
 
-def _push_worktree(repo, remote):
-    from dvc_data.index import checkout
-    from dvc_data.index.save import build_tree
-
-    index = repo.index.data["repo"]
-    checkout(index, remote.path, remote.fs)
-
-    for stage in repo.index.stages:
-        for out in stage.outs:
-            if not out.use_cache:
-                continue
-
-            if not out.is_in_repo:
-                continue
-
-            workspace, key = out.index_key
-            index = repo.index.data[workspace]
-            entry = index[key]
-            if out.isdir():
-                old_tree = out.get_obj()
-                entry.hash_info = old_tree.hash_info
-                entry.meta = out.meta
-                for subkey, entry in index.iteritems(key):
-                    if entry.meta.isdir:
-                        continue
-                    fs_path = repo.fs.path.join(repo.root_dir, *subkey)
-                    _, hash_info = old_tree.get(
-                        repo.fs.path.relparts(fs_path, out.fs_path)
-                    )
-                    entry.hash_info = hash_info
-                tree_meta, new_tree = build_tree(index, key)
-                out.obj = new_tree
-                out.hash_info = new_tree.hash_info
-                out.meta = tree_meta
-            else:
-                out.hash_info = entry.hash_info
-                out.meta = entry.meta
-        stage.dvcfile.dump(stage, with_files=True, update_pipeline=False)
-
-    return len(index)
-
-
 @locked
 def push(
     self,
@@ -68,52 +29,91 @@ def push(
     odb: Optional["ObjectDB"] = None,
     include_imports=False,
 ):
-    _remote = self.cloud.get_remote(name=remote)
-    if _remote.worktree:
-        return _push_worktree(self, _remote)
 
+    worktree_remote: Optional["Remote"] = None
+    _remote = self.cloud.get_remote(name=remote)
+    if _remote.worktree or _remote.fs.version_aware:
+        worktree_remote = _remote
+
+    pushed = 0
     used_run_cache = (
         self.stage_cache.push(remote, odb=odb) if run_cache else []
     )
+    pushed += len(used_run_cache)
 
     if isinstance(targets, str):
         targets = [targets]
 
     expanded_targets = glob_targets(targets, glob=glob)
 
-    used = self.used_objs(
-        expanded_targets,
-        all_branches=all_branches,
-        all_tags=all_tags,
-        all_commits=all_commits,
-        with_deps=with_deps,
-        force=True,
-        remote=remote,
-        jobs=jobs,
-        recursive=recursive,
-        used_run_cache=used_run_cache,
-        revs=revs,
-    )
-
-    pushed = len(used_run_cache)
-    if odb:
-        all_ids = set()
-        for dest_odb, obj_ids in used.items():
-            if not include_imports and dest_odb and dest_odb.read_only:
-                continue
-            all_ids.update(obj_ids)
-        result = self.cloud.push(all_ids, jobs, remote=remote, odb=odb)
-        if result.failed:
-            raise UploadError(len(result.failed))
-        pushed += len(result.transferred)
+    if worktree_remote is not None:
+        pushed += _push_worktree(
+            self,
+            worktree_remote,
+            revs=revs,
+            all_branches=all_branches,
+            all_tags=all_tags,
+            all_commits=all_commits,
+            targets=expanded_targets,
+            jobs=jobs,
+            with_deps=with_deps,
+            recursive=recursive,
+        )
     else:
-        for dest_odb, obj_ids in used.items():
-            if dest_odb and dest_odb.read_only:
-                continue
-            result = self.cloud.push(
-                obj_ids, jobs, remote=remote, odb=odb or dest_odb
-            )
+        used = self.used_objs(
+            expanded_targets,
+            all_branches=all_branches,
+            all_tags=all_tags,
+            all_commits=all_commits,
+            with_deps=with_deps,
+            force=True,
+            remote=remote,
+            jobs=jobs,
+            recursive=recursive,
+            used_run_cache=used_run_cache,
+            revs=revs,
+            push=True,
+        )
+
+        if odb:
+            all_ids = set()
+            for dest_odb, obj_ids in used.items():
+                if not include_imports and dest_odb and dest_odb.read_only:
+                    continue
+                all_ids.update(obj_ids)
+            result = self.cloud.push(all_ids, jobs, remote=remote, odb=odb)
             if result.failed:
                 raise UploadError(len(result.failed))
             pushed += len(result.transferred)
+        else:
+            for dest_odb, obj_ids in used.items():
+                if dest_odb and dest_odb.read_only:
+                    continue
+                result = self.cloud.push(
+                    obj_ids, jobs, remote=remote, odb=odb or dest_odb
+                )
+                if result.failed:
+                    raise UploadError(len(result.failed))
+                pushed += len(result.transferred)
     return pushed
+
+
+def _push_worktree(
+    repo: "Repo",
+    remote: "Remote",
+    revs: Optional[Sequence[str]] = None,
+    all_branches: bool = False,
+    all_tags: bool = False,
+    all_commits: bool = False,
+    targets: Optional["TargetType"] = None,
+    jobs: Optional[int] = None,
+    **kwargs,
+) -> int:
+    from dvc.repo.worktree import push_worktree
+
+    if revs or all_branches or all_tags or all_commits:
+        raise InvalidArgumentError(
+            "Multiple rev push is unsupported for cloud versioned remotes"
+        )
+
+    return push_worktree(repo, remote, targets=targets, jobs=jobs, **kwargs)

dvc/repo/reproduce.py

@@ -1,13 +1,14 @@
 import logging
-import typing
 from functools import partial
+from typing import TYPE_CHECKING, Iterator
 
 from dvc.exceptions import DvcException, ReproductionError
 from dvc.repo.scm_context import scm_context
+from dvc.stage.exceptions import CheckpointKilledError
 
 from . import locked
 
-if typing.TYPE_CHECKING:
+if TYPE_CHECKING:
     from dvc.stage import Stage
 
     from . import Repo
@@ -58,7 +59,7 @@ def _dump_stage(stage):
     dvcfile.dump(stage, update_pipeline=False)
 
 
-def _get_stage_files(stage: "Stage") -> typing.Iterator[str]:
+def _get_stage_files(stage: "Stage") -> Iterator[str]:
     yield stage.dvcfile.relpath
     for dep in stage.deps:
         if (
@@ -183,7 +184,8 @@ def _reproduce_stages(
     # `ret` is used to add a cosmetic newline.
     ret = []
     checkpoint_func = kwargs.pop("checkpoint_func", None)
-    for stage in steps:
+
+    for i, stage in enumerate(steps):
         if ret:
             logger.info("")
 
@@ -192,8 +194,6 @@ def _reproduce_stages(
                 _repro_callback, checkpoint_func, unchanged
             )
 
-        from dvc.stage.monitor import CheckpointKilledError
-
         try:
             ret = _reproduce_stage(stage, **kwargs)
 
@@ -210,7 +210,18 @@ def _reproduce_stages(
             if ret:
                 result.extend(ret)
         except CheckpointKilledError:
-            raise
+            result.append(stage)
+            logger.warning(
+                "Checkpoint stage '%s' was interrupted remaining stages in "
+                "pipeline will not be reproduced.",
+                stage.addressing,
+            )
+            logger.warning(
+                "skipped stages '%s'",
+                ", ".join(s.addressing for s in steps[i + 1 :]),
+            )
+
+            break
         except Exception as exc:
             raise ReproductionError(stage.addressing) from exc
 

dvc/repo/stage.py

@@ -23,6 +23,7 @@ from dvc.exceptions import (
     OutputNotFoundError,
 )
 from dvc.repo import lock_repo
+from dvc.ui import ui
 from dvc.utils import as_posix, parse_target, relpath
 
 logger = logging.getLogger(__name__)
@@ -139,7 +140,14 @@ class StageLoad:
         )
         with self.repo.scm_context:
             stage.dump(update_lock=update_lock)
-            stage.ignore_outs()
+            try:
+                stage.ignore_outs()
+            except FileNotFoundError as exc:
+                ui.warn(
+                    f"Could not create .gitignore entry in {exc.filename}."
+                    " DVC will attempt to create .gitignore entry again when"
+                    " the stage is run."
+                )
 
         return stage
 
@@ -361,7 +369,7 @@ class StageLoad:
             glob: Use `target` as a pattern to match stages in a file.
         """
         if not target:
-            return list(graph) if graph else list(self.repo.index)
+            return list(graph) if graph else self.repo.index.stages
 
         if recursive and self.fs.isdir(target):
             from dvc.repo.graph import collect_inside_path
@@ -402,7 +410,7 @@ class StageLoad:
             (see `collect()` for other arguments)
         """
         if not target:
-            return [StageInfo(stage) for stage in self.repo.index]
+            return [StageInfo(stage) for stage in self.repo.index.stages]
 
         target = as_posix(target)
 
@@ -502,3 +510,92 @@ class StageLoad:
 
     def collect_repo(self, onerror: Callable[[str, Exception], None] = None):
         return list(self._collect_repo(onerror))
+
+    def _load_file(self, path):
+        from dvc.dvcfile import Dvcfile
+        from dvc.stage.loader import SingleStageLoader, StageLoader
+
+        path = self._get_filepath(path)
+        dvcfile = Dvcfile(self.repo, path)
+        stages = dvcfile.stages  # type: ignore
+
+        if isinstance(stages, SingleStageLoader):
+            stages_ = [stages[None]]
+        else:
+            assert isinstance(stages, StageLoader)
+            keys = self._get_keys(stages)
+            stages_ = [stages[key] for key in keys]
+        return stages_, dvcfile
+
+    def _collect_all_from_repo(
+        self, onerror: Callable[[str, Exception], None] = None
+    ):
+        """Collects all of the stages present in the DVC repo.
+
+        Args:
+            onerror (optional): callable that will be called with two args:
+                the filepath whose collection failed and the exc instance.
+                It can report the error to continue with the collection
+                (and, skip failed ones), or raise the exception to abort
+                the collection.
+        """
+        from dvc.dvcfile import is_valid_filename
+        from dvc.fs import LocalFileSystem
+
+        scm = self.repo.scm
+        sep = self.fs.sep
+        outs: Set[str] = set()
+
+        is_local_fs = isinstance(self.fs, LocalFileSystem)
+
+        def is_ignored(path):
+            # apply only for the local fs
+            return is_local_fs and scm.is_ignored(path)
+
+        def is_dvcfile_and_not_ignored(root, file):
+            return is_valid_filename(file) and not is_ignored(
+                f"{root}{sep}{file}"
+            )
+
+        def is_out_or_ignored(root, directory):
+            dir_path = f"{root}{sep}{directory}"
+            # trailing slash needed to check if a directory is gitignored
+            return dir_path in outs or is_ignored(f"{dir_path}{sep}")
+
+        walk_iter = self.repo.dvcignore.walk(self.fs, self.repo.root_dir)
+        if logger.isEnabledFor(logging.TRACE):  # type: ignore[attr-defined]
+            walk_iter = log_walk(walk_iter)
+
+        stages = []
+        metrics = {}
+        plots = {}
+        params = {}
+
+        for root, dirs, files in walk_iter:
+            dvcfile_filter = partial(is_dvcfile_and_not_ignored, root)
+            for file in filter(dvcfile_filter, files):
+                file_path = self.fs.path.join(root, file)
+                try:
+                    new_stages, dvcfile = self._load_file(file_path)
+                except DvcException as exc:
+                    if onerror:
+                        onerror(relpath(file_path), exc)
+                        continue
+                    raise
+
+                stages.extend(new_stages)
+                if new_metrics := dvcfile.metrics:
+                    metrics[file_path] = new_metrics
+                if new_plots := dvcfile.plots:
+                    plots[file_path] = new_plots
+                if new_params := dvcfile.params:
+                    params[file_path] = new_params
+
+                outs.update(
+                    out.fspath
+                    for stage in new_stages
+                    for out in stage.outs
+                    if out.protocol == "local"
+                )
+            dirs[:] = [d for d in dirs if not is_out_or_ignored(root, d)]
+        return stages, metrics, plots, params

dvc/repo/status.py

@@ -89,6 +89,7 @@ def _cloud_status(
         remote=remote,
         jobs=jobs,
         recursive=recursive,
+        push=True,
     )
 
     ret = {}

dvc/repo/update.py

@@ -1,7 +1,13 @@
+from typing import TYPE_CHECKING, List
+
 from dvc.exceptions import InvalidArgumentError
+from dvc.stage.exceptions import StageUpdateError
 
 from . import locked
 
+if TYPE_CHECKING:
+    from dvc.repo.stage import StageInfo
+
 
 @locked
 def update(
@@ -15,6 +21,7 @@ def update(
     jobs=None,
 ):
     from ..dvcfile import Dvcfile
+    from .worktree import update_worktree_stages
 
     if not targets:
         targets = [None]
@@ -28,15 +35,21 @@ def update(
         )
 
     if not to_remote and remote:
-        raise InvalidArgumentError(
-            "--remote can't be used without --to-remote"
-        )
+        if not self.cloud.get_remote(name=remote).worktree:
+            raise InvalidArgumentError(
+                "--remote can't be used without --to-remote"
+            )
 
-    stages = set()
-    for target in targets:
-        stages.update(self.stage.collect(target, recursive=recursive))
+    import_stages = set()
+    other_stage_infos: List["StageInfo"] = []
 
-    for stage in stages:
+    for stage_info in self.index.collect_targets(targets, recursive=recursive):
+        if stage_info.stage.is_import:
+            import_stages.add(stage_info.stage)
+        else:
+            other_stage_infos.append(stage_info)
+
+    for stage in import_stages:
         stage.update(
             rev,
             to_remote=to_remote,
@@ -47,4 +60,29 @@ def update(
         dvcfile = Dvcfile(self, stage.path)
         dvcfile.dump(stage)
 
+    if other_stage_infos:
+        remote_obj = self.cloud.get_remote(name=remote)
+        if not remote_obj.worktree:
+            raise StageUpdateError(other_stage_infos[0].stage.relpath)
+        if rev:
+            raise InvalidArgumentError(
+                "--rev can't be used with worktree update"
+            )
+        if no_download:
+            raise InvalidArgumentError(
+                "--no-download can't be used with worktree update"
+            )
+        if to_remote:
+            raise InvalidArgumentError(
+                "--to-remote can't be used with worktree update"
+            )
+        update_worktree_stages(
+            self,
+            other_stage_infos,
+            remote_obj,
+        )
+
+    stages = import_stages | {
+        stage_info.stage for stage_info in other_stage_infos
+    }
     return list(stages)

dvc/repo/worktree.py

@@ -0,0 +1,283 @@
+import logging
+from functools import partial
+from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Union, cast
+
+from dvc.fs.callbacks import Callback
+
+if TYPE_CHECKING:
+    from dvc.cloud import Remote
+    from dvc.index import Index, IndexView
+    from dvc.output import Output
+    from dvc.repo import Repo
+    from dvc.repo.stage import StageInfo
+    from dvc.stage import Stage
+    from dvc.types import TargetType
+    from dvc_data.hashfile.meta import Meta
+    from dvc_data.hashfile.tree import Tree
+    from dvc_data.index import DataIndex, DataIndexView
+    from dvc_objects.fs.base import FileSystem
+
+logger = logging.getLogger(__name__)
+
+
+# for files, if our version's checksum (etag) matches the latest remote
+# checksum, we do not need to push, even if the version IDs don't match
+def _meta_checksum(fs: "FileSystem", meta: "Meta") -> Any:
+    if not meta or meta.isdir:
+        return meta
+    return getattr(meta, fs.PARAM_CHECKSUM)
+
+
+def worktree_view(
+    index: "Index",
+    targets: Optional["TargetType"] = None,
+    push: bool = False,
+    **kwargs,
+) -> "IndexView":
+    """Return view of data that can be stored in worktree remotes.
+
+    Args:
+        targets: Optional targets.
+        push: Whether the view should be restricted to pushable data only.
+
+    Additional kwargs will be passed into target collection.
+    """
+
+    def stage_filter(stage: "Stage") -> bool:
+        if push and stage.is_repo_import:
+            return False
+        return True
+
+    def outs_filter(out: "Output") -> bool:
+        if (
+            not out.is_in_repo
+            or not out.use_cache
+            or (push and not out.can_push)
+        ):
+            return False
+        return True
+
+    return index.targets_view(
+        targets,
+        stage_filter=stage_filter,
+        outs_filter=outs_filter,
+        **kwargs,
+    )
+
+
+def fetch_worktree(
+    repo: "Repo",
+    remote: "Remote",
+    targets: Optional["TargetType"] = None,
+    jobs: Optional[int] = None,
+    **kwargs,
+) -> int:
+    from dvc_data.index import save
+
+    view = worktree_view(repo.index, targets=targets, **kwargs)
+    index = view.data["repo"]
+    for key, entry in index.iteritems():
+        entry.fs = remote.fs
+        entry.path = remote.fs.path.join(
+            remote.path,
+            *key,
+        )
+    total = len(index)
+    with Callback.as_tqdm_callback(
+        unit="file", desc="Fetch", disable=total == 0
+    ) as cb:
+        cb.set_size(total)
+        return save(index, callback=cb, jobs=jobs)
+
+
+def push_worktree(
+    repo: "Repo",
+    remote: "Remote",
+    targets: Optional["TargetType"] = None,
+    jobs: Optional[int] = None,
+    **kwargs,
+) -> int:
+    from dvc_data.index import checkout
+
+    view = worktree_view(
+        repo.index,
+        push=True,
+        targets=targets,
+        **kwargs,
+    )
+    new_index = view.data["repo"]
+    if remote.worktree:
+        logger.debug("indexing latest worktree for '%s'", remote.path)
+        old_index = _build_worktree_index(repo, remote, view)
+        logger.debug("Pushing worktree changes to '%s'", remote.path)
+    else:
+        old_index = None
+        logger.debug("Pushing version-aware files to '%s'", remote.path)
+
+    if remote.worktree:
+        diff_kwargs: Dict[str, Any] = {
+            "meta_only": True,
+            "meta_cmp_key": partial(_meta_checksum, remote.fs),
+        }
+    else:
+        diff_kwargs = {}
+
+    total = len(new_index)
+    with Callback.as_tqdm_callback(
+        unit="file", desc="Push", disable=total == 0
+    ) as cb:
+        cb.set_size(total)
+        pushed = checkout(
+            new_index,
+            remote.path,
+            remote.fs,
+            old=old_index,
+            delete=remote.worktree,
+            callback=cb,
+            latest_only=remote.worktree,
+            jobs=jobs,
+            **diff_kwargs,
+        )
+    if pushed:
+        for stage in view.stages:
+            for out in stage.outs:
+                workspace, _key = out.index_key
+                _update_out_meta(out, repo.index.data[workspace])
+            stage.dvcfile.dump(stage, with_files=True, update_pipeline=False)
+    return pushed
+
+
+def _build_worktree_index(
+    repo: "Repo", remote: "Remote", view: "IndexView"
+) -> "DataIndex":
+    from dvc_data.index import DataIndex
+    from dvc_data.index.build import build_entries
+
+    index = DataIndex()
+    for out in view.outs:
+        _workspace, key = out.index_key
+        parts = out.fs.path.relparts(out.fs_path, repo.root_dir)
+        path = remote.fs.path.join(remote.path, *parts)
+        for entry in build_entries(path, remote.fs):
+            if not entry.key or entry.key == ("",):
+                entry.key = key
+            else:
+                entry.key = key + entry.key
+            index.add(entry)
+    return index
+
+
+def _update_out_meta(
+    out: "Output", index: Union["DataIndex", "DataIndexView"]
+):
+    from dvc_data.index.save import build_tree
+
+    _, key = out.index_key
+    entry = index[key]
+    repo = out.stage.repo
+    if out.isdir():
+        old_tree = cast("Tree", out.get_obj())
+        entry.hash_info = old_tree.hash_info
+        entry.meta = out.meta
+        for subkey, entry in index.iteritems(key):
+            if entry.meta.isdir:
+                continue
+            fs_path = repo.fs.path.join(repo.root_dir, *subkey)
+            meta, hash_info = old_tree.get(
+                repo.fs.path.relparts(fs_path, out.fs_path)
+            )
+            entry.hash_info = hash_info
+            if meta.version_id is not None:
+                # preserve existing version IDs for unchanged files in
+                # this dir (entry will have the latest remote version
+                # ID after checkout)
+                entry.meta = meta
+        tree_meta, new_tree = build_tree(index, key)
+        out.obj = new_tree
+        out.hash_info = new_tree.hash_info
+        out.meta = tree_meta
+    else:
+        if entry.hash_info:
+            out.hash_info = entry.hash_info
+        if out.meta.version_id is None:
+            out.meta = entry.meta
+
+
+def update_worktree_stages(
+    repo: "Repo",
+    stage_infos: Iterable["StageInfo"],
+    remote: "Remote",
+):
+    from dvc.repo.index import IndexView
+    from dvc_data.index import build
+
+    def outs_filter(out: "Output") -> bool:
+        return out.is_in_repo and out.use_cache and out.can_push
+
+    view = IndexView(
+        repo.index,
+        stage_infos,
+        outs_filter=outs_filter,
+    )
+    local_index = view.data["repo"]
+    logger.debug("indexing latest worktree for '%s'", remote.path)
+    remote_index = build(remote.path, remote.fs)
+    for stage in view.stages:
+        for out in stage.outs:
+            _workspace, key = out.index_key
+            if key not in remote_index:
+                logger.warning(
+                    "Could not update '%s', it does not exist in the remote",
+                    out,
+                )
+                continue
+            entry = remote_index[key]
+            if entry.meta and entry.meta.isdir:
+                if not any(
+                    subkey != key and subentry.meta and not subentry.meta.isdir
+                    for subkey, subentry in remote_index.iteritems(key)
+                ):
+                    logger.warning(
+                        "Could not update '%s', directory is empty in the "
+                        "remote",
+                        out,
+                    )
+                    continue
+            _fetch_out_changes(out, local_index, remote_index, remote)
+        stage.save(merge_versioned=True)
+        for out in stage.outs:
+            _update_out_meta(out, remote_index)
+        stage.dvcfile.dump(stage, with_files=True, update_pipeline=False)
+
+
+def _fetch_out_changes(
+    out: "Output",
+    local_index: "DataIndex",
+    remote_index: "DataIndex",
+    remote: "Remote",
+):
+    from dvc_data.index import DataIndex, checkout
+
+    _, key = out.index_key
+    old = DataIndex()
+    new = DataIndex()
+    for _, entry in local_index.iteritems(key):
+        old.add(entry)
+    for _, entry in remote_index.iteritems(key):
+        new.add(entry)
+    total = len(new)
+    with Callback.as_tqdm_callback(
+        unit="file", desc=f"Updating '{out}'", disable=total == 0
+    ) as cb:
+        cb.set_size(total)
+        checkout(
+            new,
+            out.repo.root_dir,
+            out.fs,
+            old=old,
+            delete=True,
+            update_meta=False,
+            meta_only=True,
+            meta_cmp_key=partial(_meta_checksum, remote.fs),
+            callback=cb,
+        )

dvc/rwlock.py

@@ -4,6 +4,7 @@ import os
 from collections import defaultdict
 from contextlib import contextmanager
 
+import psutil
 from voluptuous import Invalid, Optional, Required, Schema
 
 from .exceptions import DvcException
@@ -72,35 +73,67 @@ def _infos_to_str(infos):
     )
 
 
-def _check_blockers(lock, info, *, mode, waiters):
+def _check_blockers(tmp_dir, lock, info, *, mode, waiters):
     from .lock import LockError
 
-    for waiter_path in waiters:
-        blockers = [
-            blocker
-            for path, infos in lock[mode].items()
-            if localfs.path.overlaps(waiter_path, path)
-            for blocker in (infos if isinstance(infos, list) else [infos])
-            if blocker != info
-        ]
+    non_existing_pid = set()
 
-        if not blockers:
+    blockers = []
+    to_release = defaultdict(list)
+    for path, infos in lock[mode].items():
+        for waiter_path in waiters:
+            if localfs.path.overlaps(waiter_path, path):
+                break
+        else:
             continue
 
+        infos = infos if isinstance(infos, list) else [infos]
+        for blocker in infos:
+            if blocker == info:
+                continue
+
+            pid = int(blocker["pid"])
+
+            if pid in non_existing_pid:
+                pass
+            elif psutil.pid_exists(pid):
+                blockers.append(blocker)
+                continue
+            else:
+                non_existing_pid.add(pid)
+                cmd = blocker["cmd"]
+                logger.warning(
+                    "Process '%s' with (Pid %s), in RWLock-file '%s'"
+                    " had been killed. Auto remove it from the lock file.",
+                    cmd,
+                    pid,
+                    relpath(path),
+                )
+            to_release[json.dumps(blocker, sort_keys=True)].append(path)
+
+    if to_release:
+        for info_json, path_list in to_release.items():
+            info = json.loads(info_json)
+            if mode == "read":
+                _release_read(lock, info, path_list)
+            elif mode == "write":
+                _release_write(lock, info, path_list)
+
+    if blockers:
         raise LockError(
-            "'{path}' is busy, it is being blocked by:\n"
-            "{blockers}\n"
+            f"'{waiter_path}' is busy, it is being blocked by:\n"
+            f"{_infos_to_str(blockers)}\n"
             "\n"
             "If there are no processes with such PIDs, you can manually "
-            "remove '.dvc/tmp/rwlock' and try again.".format(
-                path=waiter_path, blockers=_infos_to_str(blockers)
-            )
+            f"remove '{tmp_dir}/rwlock' and try again."
         )
 
 
 def _acquire_read(lock, info, paths):
     changes = []
 
+    lock["read"] = lock.get("read", defaultdict(list))
+
     for path in paths:
         readers = lock["read"][path]
         if info in readers:
@@ -115,6 +148,8 @@ def _acquire_read(lock, info, paths):
 def _acquire_write(lock, info, paths):
     changes = []
 
+    lock["write"] = lock.get("write", defaultdict(dict))
+
     for path in paths:
         if lock["write"][path] == info:
             continue
@@ -171,8 +206,10 @@ def rwlock(tmp_dir, fs, cmd, read, write, hardlink):
 
     with _edit_rwlock(tmp_dir, fs, hardlink) as lock:
 
-        _check_blockers(lock, info, mode="write", waiters=read + write)
-        _check_blockers(lock, info, mode="read", waiters=write)
+        _check_blockers(
+            tmp_dir, lock, info, mode="write", waiters=read + write
+        )
+        _check_blockers(tmp_dir, lock, info, mode="read", waiters=write)
 
         rchanges = _acquire_read(lock, info, read)
         wchanges = _acquire_write(lock, info, write)

dvc/schema.py

@@ -51,6 +51,7 @@ OUT_PSTAGE_DETAILED_SCHEMA = {
         Output.PARAM_PERSIST: bool,
         Output.PARAM_CHECKPOINT: bool,
         Output.PARAM_REMOTE: str,
+        Output.PARAM_PUSH: bool,
     }
 }
 
@@ -103,7 +104,7 @@ def either_or(primary, fallback, fallback_includes=None):
 
 
 PLOT_DEFINITION = {
-    Output.PARAM_PLOT_X: str,
+    Output.PARAM_PLOT_X: Any(str, {str: str}),
     Output.PARAM_PLOT_Y: Any(str, [str], {str: Any(str, [str])}),
     Output.PARAM_PLOT_X_LABEL: str,
     Output.PARAM_PLOT_Y_LABEL: str,
@@ -122,6 +123,8 @@ MULTI_STAGE_SCHEMA = {
     PLOTS: Any(SINGLE_PLOT_SCHEMA, [Any(str, SINGLE_PLOT_SCHEMA)]),
     STAGES: SINGLE_PIPELINE_STAGE_SCHEMA,
     VARS_KWD: VARS_SCHEMA,
+    StageParams.PARAM_PARAMS: [str],
+    StageParams.PARAM_METRICS: [str],
 }
 
 COMPILED_SINGLE_STAGE_SCHEMA = Schema(SINGLE_STAGE_SCHEMA)

dvc/scm.py

@@ -147,7 +147,7 @@ def clone(url: str, to_path: str, **kwargs):
                 fetch_all_exps(git, url, progress=pbar.update_git)
             return git
         except InternalCloneError as exc:
-            raise CloneError(str(exc))
+            raise CloneError("SCM error") from exc
 
 
 def resolve_rev(scm: "Git", rev: str) -> str:

dvc/stage/__init__.py

@@ -113,10 +113,11 @@ def restore_fields(stage):
     stage.meta = old.meta
     stage.desc = old.desc
 
-    old_fields = {out.def_path: (out.annot, out.remote) for out in old.outs}
+    old_outs = {out.def_path: out for out in old.outs}
     for out in stage.outs:
-        if out_fields := old_fields.get(out.def_path, None):
-            out.annot, out.remote = out_fields
+        old_out = old_outs.get(out.def_path, None)
+        if old_out is not None:
+            out.restore_fields(old_out)
 
 
 class Stage(params.StageParams):
@@ -462,13 +463,21 @@ class Stage(params.StageParams):
         logger.debug("Computed %s md5: '%s'", self, m)
         return m
 
-    def save(self, allow_missing=False):
+    def save(
+        self,
+        allow_missing: bool = False,
+        merge_versioned: bool = False,
+        run_cache: bool = True,
+    ):
         self.save_deps(allow_missing=allow_missing)
 
-        self.save_outs(allow_missing=allow_missing)
+        self.save_outs(
+            allow_missing=allow_missing, merge_versioned=merge_versioned
+        )
         self.md5 = self.compute_md5()
 
-        self.repo.stage_cache.save(self)
+        if run_cache:
+            self.repo.stage_cache.save(self)
 
     def save_deps(self, allow_missing=False):
         from dvc.dependency.base import DependencyDoesNotExistError
@@ -480,15 +489,40 @@ class Stage(params.StageParams):
                 if not allow_missing:
                     raise
 
-    def save_outs(self, allow_missing=False):
+    def save_outs(
+        self, allow_missing: bool = False, merge_versioned: bool = False
+    ):
         from dvc.output import OutputDoesNotExistError
 
+        from .exceptions import StageFileDoesNotExistError, StageNotFound
+
+        if merge_versioned:
+            try:
+                old = self.reload()
+                old_outs = {out.def_path: out for out in old.outs}
+                merge_versioned = any(
+                    (
+                        out.files is not None
+                        or (
+                            out.meta is not None
+                            and out.meta.version_id is not None
+                        )
+                    )
+                    for out in old_outs.values()
+                )
+            except (StageFileDoesNotExistError, StageNotFound):
+                merge_versioned = False
+
         for out in self.outs:
             try:
                 out.save()
             except OutputDoesNotExistError:
                 if not (allow_missing or out.checkpoint):
                     raise
+            if merge_versioned:
+                old_out = old_outs.get(out.def_path)
+                if old_out is not None:
+                    out.merge_version_meta(old_out)
 
     def ignore_outs(self):
         for out in self.outs:
@@ -556,7 +590,9 @@ class Stage(params.StageParams):
         if not dry:
             if kwargs.get("checkpoint_func", None) or no_download:
                 allow_missing = True
-            self.save(allow_missing=allow_missing)
+
+            self.save(allow_missing=allow_missing, run_cache=not no_commit)
+
             if no_download:
                 self.ignore_outs()
             if not no_commit:
@@ -747,6 +783,7 @@ class PipelineStage(Stage):
         return f"{super().addressing}:{self.name}"
 
     def reload(self):
+        self.dvcfile._reset()  # pylint: disable=protected-access
         return self.dvcfile.stages[self.name]
 
     def _status_stage(self, ret):

dvc/stage/cache.py

@@ -180,7 +180,7 @@ class StageCache:
         dump_yaml(tmp, cache)
         self.repo.odb.local.move(tmp, path)
 
-    def restore(self, stage, run_cache=True, pull=False):
+    def restore(self, stage, run_cache=True, pull=False, dry=False):
         from .serialize import to_single_stage_lockfile
 
         if not _can_hash(stage):
@@ -195,14 +195,15 @@ class StageCache:
         else:
             if not run_cache:  # backward compatibility
                 raise RunCacheNotFoundError(stage)
-            stage.save_deps()
+            if not dry:
+                stage.save_deps()
             cache = self._load(stage)
             if not cache:
                 raise RunCacheNotFoundError(stage)
 
         cached_stage = self._create_stage(cache, wdir=stage.wdir)
 
-        if pull:
+        if pull and not dry:
             for objs in cached_stage.get_used_objs().values():
                 self.repo.cloud.pull(objs)
 
@@ -213,7 +214,8 @@ class StageCache:
             "Stage '%s' is cached - skipping run, checking out outputs",
             stage.addressing,
         )
-        cached_stage.checkout()
+        if not dry:
+            cached_stage.checkout()
 
     def transfer(self, from_odb, to_odb):
         from dvc.fs import HTTPFileSystem, LocalFileSystem
@@ -265,11 +267,15 @@ class StageCache:
         return ret
 
     def push(self, remote: Optional[str], odb: Optional["ObjectDB"] = None):
-        dest_odb = odb or self.repo.cloud.get_remote_odb(remote)
+        dest_odb = odb or self.repo.cloud.get_remote_odb(
+            remote, "push --run-cache"
+        )
         return self.transfer(self.repo.odb.local, dest_odb)
 
     def pull(self, remote: Optional[str], odb: Optional["ObjectDB"] = None):
-        odb = odb or self.repo.cloud.get_remote_odb(remote)
+        odb = odb or self.repo.cloud.get_remote_odb(
+            remote, "fetch --run-cache"
+        )
         return self.transfer(odb, self.repo.odb.local)
 
     def get_used_objs(self, used_run_cache, *args, **kwargs):

dvc/stage/exceptions.py

@@ -9,6 +9,14 @@ class StageCmdFailedError(DvcException):
         super().__init__(msg)
 
 
+class CheckpointKilledError(DvcException):
+    def __init__(self, cmd, status=None):
+        msg = f"failed to finish: {cmd}"
+        if status is not None:
+            msg += f", exited with {status}"
+        super().__init__(msg)
+
+
 class StageFileDoesNotExistError(DvcException):
     DVC_IGNORED = "is dvc-ignored"
     DOES_NOT_EXIST = "does not exist"

dvc/stage/imports.py

@@ -21,22 +21,20 @@ def update_import(
     stage, rev=None, to_remote=False, remote=None, no_download=None, jobs=None
 ):
     stage.deps[0].update(rev=rev)
-    outs = stage.outs
-    deps = stage.deps
 
     frozen = stage.frozen
     stage.frozen = False
+    changed = stage.changed()
 
-    if stage.outs:
-        stage.outs[0].clear()
     try:
         if to_remote:
             _update_import_on_remote(stage, remote, jobs)
         else:
             stage.reproduce(no_download=no_download, jobs=jobs)
     finally:
-        if deps == stage.deps:
-            stage.outs = outs
+        if no_download and changed:
+            # Avoid retaining stale information
+            stage.outs[0].clear()
         stage.frozen = frozen
 
 

dvc/stage/loader.py

@@ -6,7 +6,7 @@ from itertools import chain
 from funcy import cached_property, get_in, lcat, once, project
 
 from dvc import dependency, output
-from dvc.parsing import FOREACH_KWD, JOIN, DataResolver, EntryNotFound
+from dvc.parsing import FOREACH_KWD, JOIN, EntryNotFound
 from dvc.parsing.versions import LOCKFILE_VERSION
 from dvc_data.hashfile.hash_info import HashInfo
 from dvc_data.hashfile.meta import Meta
@@ -20,8 +20,14 @@ logger = logging.getLogger(__name__)
 
 
 class StageLoader(Mapping):
-    def __init__(self, dvcfile, data, lockfile_data=None):
+    def __init__(
+        self,
+        dvcfile,
+        data,
+        lockfile_data=None,
+    ):
         self.dvcfile = dvcfile
+        self.resolver = self.dvcfile.resolver
         self.data = data or {}
         self.stages_data = self.data.get("stages", {})
         self.repo = self.dvcfile.repo
@@ -33,11 +39,6 @@ class StageLoader(Mapping):
         else:
             self._lockfile_data = lockfile_data.get("stages", {})
 
-    @cached_property
-    def resolver(self):
-        wdir = self.repo.fs.path.parent(self.dvcfile.path)
-        return DataResolver(self.repo, wdir, self.data)
-
     @cached_property
     def lockfile_data(self):
         if not self._lockfile_data:

dvc/stage/monitor.py

@@ -7,7 +7,7 @@ from dataclasses import dataclass
 from typing import TYPE_CHECKING, Callable, List
 
 from dvc.stage.decorators import relock_repo
-from dvc.stage.exceptions import StageCmdFailedError
+from dvc.stage.exceptions import CheckpointKilledError
 
 if TYPE_CHECKING:
     from dvc.stage import Stage
@@ -16,17 +16,13 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-class CheckpointKilledError(StageCmdFailedError):
-    pass
-
-
 @dataclass
 class MonitorTask:
     stage: "Stage"
     execute: Callable
     proc: subprocess.Popen
     done: threading.Event = threading.Event()
-    killed: threading.Event = threading.Event()
+    updated: threading.Event = threading.Event()
 
     @property
     def name(self) -> str:
@@ -77,7 +73,7 @@ class CheckpointTask(MonitorTask):
 class Monitor:
     AWAIT: float = 1.0
 
-    def __init__(self, tasks: List[MonitorTask]):
+    def __init__(self, tasks: List[CheckpointTask]):
         self.done = threading.Event()
         self.tasks = tasks
         self.monitor_thread = threading.Thread(
@@ -113,6 +109,7 @@ class Monitor:
                 if os.path.exists(task.signal_path):
                     try:
                         task.execute()
+                        task.updated.set()
                     except Exception:  # pylint: disable=broad-except
                         logger.exception(
                             "Error running '%s' task, '%s' will be aborted",
@@ -120,11 +117,11 @@ class Monitor:
                             task.stage,
                         )
                         Monitor.kill(task.proc)
-                        task.killed.set()
                     finally:
                         logger.debug(
                             "Removing signal file for '%s' task", task.name
                         )
                         os.remove(task.signal_path)
+
             if done.wait(Monitor.AWAIT):
                 return

dvc/stage/run.py

@@ -3,13 +3,17 @@ import os
 import signal
 import subprocess
 import threading
+from typing import TYPE_CHECKING, List
 
-from dvc.stage.monitor import Monitor
+from dvc.stage.monitor import CheckpointTask, Monitor
 from dvc.utils import fix_env
 
 from .decorators import unlocked_repo
 from .exceptions import StageCmdFailedError
 
+if TYPE_CHECKING:
+    from dvc.stage import Stage
+
 logger = logging.getLogger(__name__)
 
 
@@ -74,14 +78,15 @@ def get_executable():
     return (os.getenv("SHELL") or "/bin/sh") if os.name != "nt" else None
 
 
-def _run(stage, executable, cmd, checkpoint_func, **kwargs):
+def _run(stage: "Stage", executable, cmd, checkpoint_func, **kwargs):
+    # pylint: disable=protected-access
     main_thread = isinstance(
         threading.current_thread(),
-        threading._MainThread,  # pylint: disable=protected-access
+        threading._MainThread,  # type: ignore[attr-defined]
     )
+    old_handler = None
 
     exec_cmd = _make_cmd(executable, cmd)
-    old_handler = None
 
     try:
         p = subprocess.Popen(exec_cmd, **kwargs)
@@ -98,7 +103,7 @@ def _run(stage, executable, cmd, checkpoint_func, **kwargs):
 
         if p.returncode != 0:
             for t in tasks:
-                if t.killed.is_set():
+                if t.updated.is_set():
                     raise t.error_cls(cmd, p.returncode)
             raise StageCmdFailedError(cmd, p.returncode)
     finally:
@@ -106,12 +111,10 @@ def _run(stage, executable, cmd, checkpoint_func, **kwargs):
             signal.signal(signal.SIGINT, old_handler)
 
 
-def _get_monitor_tasks(stage, checkpoint_func, proc):
+def _get_monitor_tasks(stage, checkpoint_func, proc) -> List[CheckpointTask]:
 
     result = []
     if checkpoint_func:
-        from .monitor import CheckpointTask
-
         result.append(CheckpointTask(stage, checkpoint_func, proc))
 
     return result
@@ -139,14 +142,16 @@ def cmd_run(stage, dry=False, checkpoint_func=None, run_env=None):
 def run_stage(
     stage, dry=False, force=False, checkpoint_func=None, run_env=None, **kwargs
 ):
-    if not (dry or force or checkpoint_func):
+    if not (force or checkpoint_func):
         from .cache import RunCacheNotFoundError
 
         try:
-            stage.repo.stage_cache.restore(stage, **kwargs)
-            return
+            stage.repo.stage_cache.restore(stage, dry=dry, **kwargs)
+            if not dry:
+                return
         except RunCacheNotFoundError:
-            stage.save_deps()
+            if not dry:
+                stage.save_deps()
 
     run = cmd_run if dry else unlocked_repo(cmd_run)
     run(stage, dry=dry, checkpoint_func=checkpoint_func, run_env=run_env)

dvc/stage/serialize.py

@@ -29,6 +29,7 @@ PARAM_PERSIST = Output.PARAM_PERSIST
 PARAM_CHECKPOINT = Output.PARAM_CHECKPOINT
 PARAM_DESC = Annotation.PARAM_DESC
 PARAM_REMOTE = Output.PARAM_REMOTE
+PARAM_PUSH = Output.PARAM_PUSH
 
 DEFAULT_PARAMS_FILE = ParamsDependency.DEFAULT_PARAMS_FILE
 
@@ -54,6 +55,8 @@ def _get_flags(out):
         yield from out.plot.items()
     if out.remote:
         yield PARAM_REMOTE, out.remote
+    if not out.can_push:
+        yield PARAM_PUSH, False
 
 
 def _serialize_out(out):

dvc/testing/api_tests.py

@@ -13,14 +13,22 @@ class TestAPI:
         assert api.get_url("foo") == expected_url
 
     def test_open(self, tmp_dir, dvc, remote):
-        tmp_dir.dvc_gen("foo", "foo-text")
+        tmp_dir.dvc_gen(
+            {
+                "foo": "foo-text",
+                "dir": {"bar": "bar-text"},
+            }
+        )
         dvc.push()
 
         # Remove cache to force download
         remove(dvc.odb.local.path)
 
-        with api.open("foo") as fd:
-            assert fd.read() == "foo-text"
+        with api.open("foo") as fobj:
+            assert fobj.read() == "foo-text"
+
+        with api.open("dir/bar") as fobj:
+            assert fobj.read() == "bar-text"
 
     @pytest.mark.parametrize(
         "clear_cache", [True, False], ids=["cache", "no_cache"]

dvc/ui/__init__.py

@@ -270,7 +270,9 @@ class Console:
 
         path = Path(file).resolve()
         url = (
-            relpath(path) if "Microsoft" in uname().release else path.as_uri()
+            relpath(path)
+            if "microsoft" in uname().release.lower()
+            else path.as_uri()
         )
 
         opened = webbrowser.open(url)

dvc/utils/hydra.py

@@ -1,27 +1,9 @@
-import sys
 from pathlib import Path
 from typing import TYPE_CHECKING, List
 
-try:
-    from hydra import compose, initialize_config_dir
-    from hydra._internal.config_loader_impl import ConfigLoaderImpl
-    from hydra._internal.core_plugins.basic_sweeper import BasicSweeper
-    from hydra.core.override_parser.overrides_parser import OverridesParser
-    from hydra.core.override_parser.types import ValueType
-    from hydra.errors import ConfigCompositionException, OverrideParseException
-    from omegaconf import OmegaConf
-
-    hydra_compatible = True
-except ValueError:
-    if sys.version_info >= (3, 11):
-        hydra_compatible = False
-    else:
-        raise
-
-from dvc.exceptions import DvcException, InvalidArgumentError
+from dvc.exceptions import InvalidArgumentError
 
 from .collections import merge_dicts, remove_missing_keys, to_omegaconf
-from .serialize import DUMPERS, MODIFIERS
 
 if TYPE_CHECKING:
     from dvc.types import StrPath
@@ -46,16 +28,20 @@ def compose_and_dump(
     .. _Hydra Override:
         https://hydra.cc/docs/advanced/override_grammar/basic/
     """
-    if not hydra_compatible:
-        raise DvcException(
-            "hydra functionality is not supported in Python >= 3.11"
-        )
+    from hydra import compose, initialize_config_dir
+    from omegaconf import OmegaConf
+
+    from .serialize import DUMPERS
 
     with initialize_config_dir(config_dir, version_base=None):
         cfg = compose(config_name=config_name, overrides=overrides)
 
-    dumper = DUMPERS[Path(output_file).suffix.lower()]
-    dumper(output_file, OmegaConf.to_object(cfg))
+    suffix = Path(output_file).suffix.lower()
+    if suffix not in [".yml", ".yaml"]:
+        dumper = DUMPERS[suffix]
+        dumper(output_file, OmegaConf.to_object(cfg))
+    else:
+        Path(output_file).write_text(OmegaConf.to_yaml(cfg), encoding="utf-8")
 
 
 def apply_overrides(path: "StrPath", overrides: List[str]) -> None:
@@ -67,10 +53,11 @@ def apply_overrides(path: "StrPath", overrides: List[str]) -> None:
     .. _Hydra Override:
         https://hydra.cc/docs/next/advanced/override_grammar/basic/
     """
-    if not hydra_compatible:
-        raise DvcException(
-            "hydra functionality is not supported in Python >= 3.11"
-        )
+    from hydra._internal.config_loader_impl import ConfigLoaderImpl
+    from hydra.errors import ConfigCompositionException, OverrideParseException
+    from omegaconf import OmegaConf
+
+    from .serialize import MODIFIERS
 
     suffix = Path(path).suffix.lower()
 
@@ -97,10 +84,7 @@ def apply_overrides(path: "StrPath", overrides: List[str]) -> None:
 
 
 def to_hydra_overrides(path_overrides):
-    if not hydra_compatible:
-        raise DvcException(
-            "hydra functionality is not supported in Python >= 3.11"
-        )
+    from hydra.core.override_parser.overrides_parser import OverridesParser
 
     parser = OverridesParser.create()
     return parser.parse_overrides(overrides=path_overrides)
@@ -113,10 +97,8 @@ def dict_product(dicts):
 
 
 def get_hydra_sweeps(path_overrides):
-    if not hydra_compatible:
-        raise DvcException(
-            "hydra functionality is not supported in Python >= 3.11"
-        )
+    from hydra._internal.core_plugins.basic_sweeper import BasicSweeper
+    from hydra.core.override_parser.types import ValueType
 
     path_sweeps = {}
     for path, overrides in path_overrides.items():

dvc/utils/plots.py

@@ -1,2 +1,6 @@
-def get_plot_id(plot_id: str, config_file_path: str = ""):
-    return f"{config_file_path}::{plot_id}" if config_file_path else plot_id
+def get_plot_id(config_plot_id: str, config_file_path: str = ""):
+    return (
+        f"{config_file_path}::{config_plot_id}"
+        if config_file_path
+        else config_plot_id
+    )

pyproject.toml

@@ -50,14 +50,15 @@ dependencies = [
     "rich>=10.13.0",
     "pyparsing>=2.4.7",
     "typing-extensions>=3.7.4",
-    "scmrepo==0.1.3",
-    "dvc-render==0.0.12",
-    "dvc-task==0.1.5",
-    "dvclive>=1.0",
-    "dvc-data==0.25.3",
-    "dvc-http==2.27.2",
+    "scmrepo==0.1.5",
+    "dvc-render==0.0.17",
+    "dvc-task==0.1.9",
+    "dvclive>=1.2.2",
+    "dvc-data==0.28.6",
+    "dvc-http",
     "hydra-core>=1.1.0",
     "iterative-telemetry==0.0.6",
+    "dvc-studio-client>=0.1.1"
 ]
 dynamic = ["version"]
 classifiers = [
@@ -70,10 +71,10 @@ classifiers = [
 ]
 
 [project.optional-dependencies]
-azure = ["dvc-azure==2.20.4"]
+azure = ["dvc-azure==2.20.5"]
 gdrive = ["dvc-gdrive==2.19.1"]
 gs = ["dvc-gs==2.20.0"]
-hdfs = ["dvc-hdfs==2.19.0; python_version < '3.11'"]
+hdfs = ["dvc-hdfs==2.19.0"]
 oss = ["dvc-oss==2.19.0"]
 s3 = ["dvc-s3==2.21.0"]
 ssh = ["dvc-ssh==2.20.0"]
@@ -90,30 +91,28 @@ tests = [
     # Test requirements
     "pytest==7.2.0",
     "pytest-cov==4.0.0",
-    "pytest-xdist==3.0.2",
+    "pytest-xdist==3.1.0",
     "pytest-mock==3.10.0",
     "pytest-lazy-fixture==0.6.3",
     # https://github.com/docker/docker-py/issues/2902
     "pytest-docker==0.11.0; python_version < '3.10' or sys_platform != 'win32'",
     "flaky==3.7.0",
     "pytest-timeout==2.1.0",
-    "filelock==3.8.0",
-    # optional dependencies
-    "dvclive>=1.0",
-    "pywin32>=225; sys_platform == 'win32'",
+    "filelock==3.9.0",
     "beautifulsoup4==4.11.1",
+    "pywin32>=225; sys_platform == 'win32'",  # optional dependency
     # pylint requirements
-    "pylint==2.15.5",
+    "pylint==2.15.10",
     # we use this to suppress pytest-related false positives in our tests.
     "pylint-pytest==1.1.2",
     # we use this to suppress some messages in tests, eg: foo/bar naming,
     # and, protected method calls in our tests
     "pylint-plugin-utils==0.7",
     # type-checking
-    "mypy==0.990",
-    "types-requests>=2.27.15",
-    "types-tabulate>=0.8.6",
-    "types-toml>=0.10.4",
+    "mypy==0.991",
+    "types-requests",
+    "types-tabulate",
+    "types-toml",
 ]
 all = ["dvc[azure,gdrive,gs,hdfs,oss,s3,ssh,webdav,webhdfs]"]
 dev = ["dvc[azure,gdrive,gs,hdfs,oss,s3,ssh,webdav,webhdfs,tests]"]
@@ -163,7 +162,7 @@ line_length = 79
 
 [tool.pytest.ini_options]
 log_level = "debug"
-addopts = "-ra -p no:hydra_pytest --cov-config pyproject.toml"
+addopts = "-ra --cov-config pyproject.toml"
 markers = [
     "needs_internet: Might need network access for the tests",
     "vscode: Tests verifying contract between DVC and VSCode plugin",

scripts/build-requirements.txt

@@ -1,3 +1,3 @@
 # NOTE: using strict version due to
 # https://github.com/iterative/dvc/issues/7949
-PyInstaller==5.6.2
+PyInstaller==5.7.0

scripts/fpm/build.py

@@ -44,6 +44,10 @@ if args.pkg == "osxpkg":
         ]
     )
 else:
+    if args.pkg == "rpm":
+        # https://github.com/jordansissel/fpm/issues/1503
+        flags.extend(["--rpm-rpmbuild-define", "_build_id_links none"])
+
     bash_dir = build / "etc" / "bash_completion.d"
     dirs = ["usr", "etc"]
     flags.extend(["--depends", "git >= 1.7.0"])  # needed for gitpython

scripts/pyinstaller/hooks/hook-asyncssh.py

@@ -0,0 +1 @@
+hiddenimports = ["win32timezone"]

tests/conftest.py

@@ -1,3 +1,4 @@
+import gc
 import json
 import os
 import sys
@@ -250,12 +251,16 @@ def run_copy_metrics(tmp_dir, copy_script):
 
 
 @pytest.fixture(autouse=True)
-def mock_hydra_conf(mocker):
-    if sys.version_info < (3, 11):
+def gc_collect_on_dvc_close_on_win_311(mocker):
+    if sys.version_info < (3, 11) and os.name != "nt":
         return
 
-    # `hydra.conf` fails to import in 3.11, it raises ValueError due to changes
-    # in dataclasses. See https://github.com/python/cpython/pull/29867.
-    # NOTE: using sentinel here so that any imports from `hydra.conf`
-    # return a mock.
-    sys.modules["hydra.conf"] = mocker.sentinel
+    from dvc.repo import Repo
+
+    close = Repo.close
+
+    def wrapped(repo):
+        close(repo)
+        gc.collect()
+
+    mocker.patch("dvc.repo.Repo.close", wrapped)

tests/func/experiments/__init__.py

@@ -0,0 +1,14 @@
+import json
+
+import pytest
+
+pytestmark = pytest.mark.xfail(
+    raises=(
+        # looks like dvc-task is not saving json atomically
+        json.JSONDecodeError,
+        # needs investigation, maybe a file locking issue?
+        PermissionError,
+    ),
+    strict=False,
+    reason="See https://github.com/iterative/dvc/issues/8570",
+)

tests/func/experiments/conftest.py

@@ -3,6 +3,7 @@ import pytest
 from tests.unit.repo.experiments.conftest import (  # noqa, pylint disable=unused-argument
     checkpoint_stage,
     exp_stage,
+    failed_checkpoint_stage,
     failed_exp_stage,
     test_queue,
 )

tests/func/experiments/executor/test_ssh.py

@@ -38,7 +38,7 @@ def test_init_from_stash(tmp_dir, scm, dvc, machine_instance, mocker):
 
 @pytest.mark.needs_internet
 @pytest.mark.parametrize("cloud", [pytest.lazy_fixture("git_ssh")])
-def test_init_git(tmp_dir, scm, cloud, mocker):
+def test_init_git(tmp_dir, dvc, scm, cloud, mocker):
     tmp_dir.scm_gen({"foo": "foo", "dir": {"bar": "bar"}}, commit="init")
     baseline_rev = scm.get_rev()
     tmp_dir.gen("foo", "stashed")
@@ -59,7 +59,7 @@ def test_init_git(tmp_dir, scm, cloud, mocker):
         fs_factory=partial(_ssh_factory, cloud),
     )
     infofile = str((root_url / "foo.run").path)
-    executor.init_git(scm, rev, mock, infofile=infofile)
+    executor.init_git(dvc, scm, rev, mock, infofile=infofile)
     assert root_url.path == executor._repo_abspath
 
     fs = cloud._ssh

tests/func/experiments/test_checkpoints.py

@@ -90,14 +90,11 @@ def test_resume_checkpoint(
 def test_reset_checkpoint(
     tmp_dir, scm, dvc, checkpoint_stage, caplog, workspace
 ):
-    dvc.experiments.run(
-        checkpoint_stage.addressing, name="foo", tmp_dir=not workspace
-    )
+    dvc.experiments.run(checkpoint_stage.addressing, tmp_dir=not workspace)
 
     results = dvc.experiments.run(
         checkpoint_stage.addressing,
         params=["foo=2"],
-        name="foo",
         tmp_dir=not workspace,
         reset=True,
     )
@@ -228,7 +225,8 @@ def test_auto_push_during_iterations(
     ref_info = first(exp_refs_by_rev(scm, exp))
     assert git_upstream.tmp_dir.scm.get_ref(str(ref_info)) == exp
 
-    assert auto_push_spy.call_count == 2
+    # Assert 3 pushes: 2 checkpoints and final commit
+    assert auto_push_spy.call_count == 3
     assert auto_push_spy.call_args[0][2] == remote
 
 
@@ -275,3 +273,23 @@ def test_auto_push_self_remote(
             "automatically be pushed to the default DVC remote (if any) "
             "on each experiment commit." in caplog.text
         )
+
+
+def test_tmp_dir_failed_checkpoint(
+    tmp_dir, scm, dvc, failed_checkpoint_stage, caplog
+):
+    baseline = scm.get_rev()
+
+    results = dvc.experiments.run(
+        failed_checkpoint_stage.addressing, params=["foo=2"], tmp_dir=True
+    )
+    exp = first(results)
+
+    result = dvc.experiments.show()[baseline]
+    # Assert 4 rows: baseline, 2 checkpoints, and final commit
+    assert len(result) == 4
+    assert exp in result
+    assert (
+        "Checkpoint stage 'failed-checkpoint-file' was interrupted remaining "
+        "stages in pipeline will not be reproduced." in caplog.text
+    )

tests/func/experiments/test_experiments.py

@@ -2,13 +2,18 @@ import itertools
 import logging
 import os
 import stat
+from textwrap import dedent
 
+import dulwich.errors
+import dulwich.file
 import pytest
 from funcy import first
 
 from dvc.dvcfile import PIPELINE_FILE
 from dvc.exceptions import ReproductionError
+from dvc.repo.experiments.exceptions import ExperimentExistsError
 from dvc.repo.experiments.queue.base import BaseStashQueue
+from dvc.repo.experiments.refs import CELERY_STASH
 from dvc.repo.experiments.utils import exp_refs_by_rev
 from dvc.scm import resolve_rev
 from dvc.stage.exceptions import StageFileDoesNotExistError
@@ -43,8 +48,6 @@ def test_new_simple(tmp_dir, scm, dvc, exp_stage, mocker, name, workspace):
 
 
 def test_experiment_exists(tmp_dir, scm, dvc, exp_stage, mocker, workspace):
-    from dvc.repo.experiments.exceptions import ExperimentExistsError
-
     dvc.experiments.run(
         exp_stage.addressing,
         name="foo",
@@ -114,12 +117,12 @@ def test_apply(tmp_dir, scm, dvc, exp_stage):
     exp_a = first(results)
 
     results = dvc.experiments.run(
-        exp_stage.addressing, params=["foo=3"], tmp_dir=True
+        exp_stage.addressing, params=["foo=3"], tmp_dir=True, name="foo"
     )
     exp_b = first(results)
 
     with pytest.raises(InvalidArgumentError):
-        dvc.experiments.apply("foo")
+        dvc.experiments.apply("bar")
 
     dvc.experiments.apply(exp_a)
     assert (tmp_dir / "params.yaml").read_text().strip() == "foo: 2"
@@ -131,11 +134,42 @@ def test_apply(tmp_dir, scm, dvc, exp_stage):
         assert (tmp_dir / "params.yaml").read_text().strip() == "foo: 2"
         assert (tmp_dir / "metrics.yaml").read_text().strip() == "foo: 2"
 
-    dvc.experiments.apply(exp_b)
+    dvc.experiments.apply("foo")
     assert (tmp_dir / "params.yaml").read_text().strip() == "foo: 3"
     assert (tmp_dir / "metrics.yaml").read_text().strip() == "foo: 3"
 
 
+@pytest.mark.xfail(
+    raises=(
+        dulwich.errors.CommitError,
+        dulwich.file.FileLocked,
+    ),
+    strict=False,
+    reason="See https://github.com/iterative/dvc/issues/8570",
+)
+def test_apply_failed(tmp_dir, scm, dvc, failed_exp_stage, test_queue):
+    from dvc.exceptions import InvalidArgumentError
+
+    dvc.experiments.run(
+        failed_exp_stage.addressing, params=["foo=2"], queue=True
+    )
+    exp_rev = dvc.experiments.scm.resolve_rev(f"{CELERY_STASH}@{{0}}")
+
+    dvc.experiments.run(
+        failed_exp_stage.addressing, params=["foo=3"], queue=True, name="foo"
+    )
+    dvc.experiments.run(run_all=True)
+
+    with pytest.raises(InvalidArgumentError):
+        dvc.experiments.apply("bar")
+
+    dvc.experiments.apply(exp_rev)
+    assert (tmp_dir / "params.yaml").read_text().strip() == "foo: 2"
+
+    dvc.experiments.apply("foo")
+    assert (tmp_dir / "params.yaml").read_text().strip() == "foo: 3"
+
+
 def test_apply_queued(tmp_dir, scm, dvc, exp_stage):
     from dvc.repo.experiments.exceptions import ApplyConflictError
 
@@ -183,7 +217,6 @@ def test_apply_untracked(tmp_dir, scm, dvc, exp_stage):
 
 
 def test_get_baseline(tmp_dir, scm, dvc, exp_stage):
-    from dvc.repo.experiments.refs import CELERY_STASH
 
     init_rev = scm.get_rev()
     assert dvc.experiments.get_baseline(init_rev) is None
@@ -428,22 +461,28 @@ def test_list(tmp_dir, scm, dvc, exp_stage):
     ref_info_b = first(exp_refs_by_rev(scm, exp_b))
 
     tmp_dir.scm_gen("new", "new", commit="new")
-    baseline_c = scm.get_rev()
     results = dvc.experiments.run(exp_stage.addressing, params=["foo=4"])
     exp_c = first(results)
     ref_info_c = first(exp_refs_by_rev(scm, exp_c))
 
-    assert dvc.experiments.ls() == {baseline_c: [ref_info_c.name]}
+    assert dvc.experiments.ls() == {"master": [ref_info_c.name]}
 
     exp_list = dvc.experiments.ls(rev=ref_info_a.baseline_sha)
     assert {key: set(val) for key, val in exp_list.items()} == {
-        baseline_a: {ref_info_a.name, ref_info_b.name}
+        baseline_a[:7]: {ref_info_a.name, ref_info_b.name}
     }
 
     exp_list = dvc.experiments.ls(all_commits=True)
     assert {key: set(val) for key, val in exp_list.items()} == {
-        baseline_a: {ref_info_a.name, ref_info_b.name},
-        baseline_c: {ref_info_c.name},
+        baseline_a[:7]: {ref_info_a.name, ref_info_b.name},
+        "master": {ref_info_c.name},
+    }
+
+    scm.checkout("branch", True)
+    exp_list = dvc.experiments.ls(all_commits=True)
+    assert {key: set(val) for key, val in exp_list.items()} == {
+        baseline_a[:7]: {ref_info_a.name, ref_info_b.name},
+        "branch": {ref_info_c.name},
     }
 
 
@@ -676,3 +715,36 @@ def test_experiments_workspace_not_log_exception(caplog, dvc, scm):
             dvc.experiments.run()
 
     assert not caplog.text
+
+
+def test_run_env(tmp_dir, dvc, scm, mocker):
+    dump_run_env = dedent(
+        """\
+        import os
+        from dvc.env import DVC_EXP_BASELINE_REV, DVC_EXP_NAME
+        with open("DVC_EXP_BASELINE_REV", "w") as f:
+            f.write(os.environ.get(DVC_EXP_BASELINE_REV, ""))
+        with open("DVC_EXP_NAME", "w") as f:
+            f.write(os.environ.get(DVC_EXP_NAME, ""))
+        """
+    )
+    (tmp_dir / "dump_run_env.py").write_text(dump_run_env)
+    baseline = scm.get_rev()
+    dvc.stage.add(
+        cmd="python dump_run_env.py",
+        name="run_env",
+    )
+    dvc.experiments.run()
+    assert (tmp_dir / "DVC_EXP_BASELINE_REV").read_text().strip() == baseline
+    assert (tmp_dir / "DVC_EXP_NAME").read_text().strip() != ""
+
+    dvc.experiments.run(name="foo")
+    assert (tmp_dir / "DVC_EXP_BASELINE_REV").read_text().strip() == baseline
+    assert (tmp_dir / "DVC_EXP_NAME").read_text().strip() == "foo"
+
+
+def test_experiment_unchanged(tmp_dir, scm, dvc, exp_stage):
+    dvc.experiments.run(exp_stage.addressing)
+    dvc.experiments.run(exp_stage.addressing)
+
+    assert len(dvc.experiments.ls()["master"]) == 2

tests/func/experiments/test_remote.py

@@ -151,7 +151,6 @@ def test_list_remote(tmp_dir, scm, dvc, git_downstream, exp_stage, use_url):
     ref_info_b = first(exp_refs_by_rev(scm, exp_b))
 
     tmp_dir.scm_gen("new", "new", commit="new")
-    baseline_c = scm.get_rev()
     results = dvc.experiments.run(exp_stage.addressing, params=["foo=4"])
     exp_c = first(results)
     ref_info_c = first(exp_refs_by_rev(scm, exp_c))
@@ -165,13 +164,13 @@ def test_list_remote(tmp_dir, scm, dvc, git_downstream, exp_stage, use_url):
     git_downstream.tmp_dir.scm.fetch_refspecs(remote, ["master:master"])
     exp_list = downstream_exp.ls(rev=baseline_a, git_remote=remote)
     assert {key: set(val) for key, val in exp_list.items()} == {
-        baseline_a: {ref_info_a.name, ref_info_b.name}
+        baseline_a[:7]: {ref_info_a.name, ref_info_b.name}
     }
 
     exp_list = downstream_exp.ls(all_commits=True, git_remote=remote)
     assert {key: set(val) for key, val in exp_list.items()} == {
-        baseline_a: {ref_info_a.name, ref_info_b.name},
-        baseline_c: {ref_info_c.name},
+        baseline_a[:7]: {ref_info_a.name, ref_info_b.name},
+        "master": {ref_info_c.name},
     }
 
 

tests/func/experiments/test_save.py

@@ -0,0 +1,96 @@
+import pytest
+from funcy import first
+
+from dvc.repo.experiments.exceptions import (
+    ExperimentExistsError,
+    InvalidArgumentError,
+)
+from dvc.repo.experiments.utils import exp_refs_by_rev
+from dvc.scm import resolve_rev
+
+
+@pytest.fixture
+def modified_exp_stage(exp_stage, tmp_dir):
+    with open(tmp_dir / "copy.py", "a", encoding="utf-8") as fh:
+        fh.write("\n# dummy change")
+    yield
+
+
+def test_exp_save_unchanged(tmp_dir, dvc, scm, exp_stage):
+    dvc.experiments.save()
+
+
+@pytest.mark.parametrize("name", (None, "test"))
+def test_exp_save(tmp_dir, dvc, scm, exp_stage, name, modified_exp_stage):
+    baseline = scm.get_rev()
+
+    exp = dvc.experiments.save(name=name)
+    ref_info = first(exp_refs_by_rev(scm, exp))
+    assert ref_info and ref_info.baseline_sha == baseline
+
+    exp_name = name if name else ref_info.name
+    assert dvc.experiments.get_exact_name([exp])[exp] == exp_name
+    assert resolve_rev(scm, exp_name) == exp
+
+
+def test_exp_save_overwrite_experiment(
+    tmp_dir, dvc, scm, exp_stage, modified_exp_stage
+):
+    name = "dummy"
+    dvc.experiments.save(name=name)
+
+    with pytest.raises(ExperimentExistsError):
+        dvc.experiments.save(name=name)
+
+    dvc.experiments.save(name=name, force=True)
+
+
+@pytest.mark.parametrize(
+    "name",
+    (
+        "invalid/name",
+        "invalid..name",
+        "invalid~name",
+        "invalid?name",
+        "invalidname.",
+    ),
+)
+def test_exp_save_invalid_name(tmp_dir, dvc, scm, exp_stage, name):
+    with pytest.raises(InvalidArgumentError):
+        dvc.experiments.save(name=name, force=True)
+
+
+def test_exp_save_after_commit(tmp_dir, dvc, scm, exp_stage):
+    baseline = scm.get_rev()
+    dvc.experiments.save(name="exp-1", force=True)
+
+    tmp_dir.scm_gen({"new_file": "new_file"}, commit="new baseline")
+    dvc.experiments.save(name="exp-2", force=True)
+
+    all_exps = dvc.experiments.ls(all_commits=True)
+    assert all_exps[baseline[:7]] == ["exp-1"]
+    assert all_exps["master"] == ["exp-2"]
+
+
+def test_exp_save_with_staged_changes(tmp_dir, dvc, scm):
+    tmp_dir.gen({"new_file": "new_file"})
+    scm.add("new_file")
+
+    dvc.experiments.save(name="exp")
+
+    _, _, unstaged = scm.status()
+    assert "new_file" in unstaged
+
+
+def test_exp_save_include_untracked(tmp_dir, dvc, scm, exp_stage):
+    new_file = tmp_dir / "new_file"
+    for i in range(2):
+        new_file.write_text(f"exp-{i}")
+        dvc.experiments.save(name=f"exp-{i}", include_untracked=["new_file"])
+
+    _, _, unstaged = scm.status()
+    assert "new_file" in unstaged
+    assert new_file.read_text() == f"exp-{i}"
+
+    dvc.experiments.apply("exp-0")
+    assert new_file.read_text() == "exp-0"

tests/func/experiments/test_set_params.py

@@ -1,5 +1,3 @@
-import sys
-
 import pytest
 
 from dvc.exceptions import InvalidArgumentError
@@ -21,18 +19,7 @@ def test_modify_params(params_repo, dvc, changes, expected):
         assert fobj.read().strip() == expected
 
 
-@pytest.mark.parametrize(
-    "hydra_enabled",
-    [
-        pytest.param(
-            True,
-            marks=pytest.mark.skipif(
-                sys.version_info >= (3, 11), reason="unsupported on 3.11"
-            ),
-        ),
-        False,
-    ],
-)
+@pytest.mark.parametrize("hydra_enabled", [True, False])
 @pytest.mark.parametrize(
     "config_dir,config_name",
     [
@@ -93,12 +80,7 @@ def test_hydra_compose_and_dump(
     "hydra_enabled,overrides,expected",
     [
         (
-            pytest.param(
-                True,
-                marks=pytest.mark.xfail(
-                    sys.version_info >= (3, 11), reason="unsupported on 3.11"
-                ),
-            ),
+            True,
             ["db=mysql,postgresql"],
             [
                 {"params.yaml": ["db=mysql"]},

tests/func/experiments/test_show.py

@@ -10,15 +10,17 @@ from scmrepo.exceptions import SCMError
 
 from dvc.cli import main
 from dvc.repo.experiments.executor.base import (
-    EXEC_PID_DIR,
-    EXEC_TMP_DIR,
     BaseExecutor,
     ExecutorInfo,
     TaskStatus,
 )
 from dvc.repo.experiments.queue.base import QueueEntry
 from dvc.repo.experiments.refs import CELERY_STASH, ExpRefInfo
-from dvc.repo.experiments.utils import exp_refs_by_rev
+from dvc.repo.experiments.utils import (
+    EXEC_PID_DIR,
+    EXEC_TMP_DIR,
+    exp_refs_by_rev,
+)
 from dvc.utils import relpath
 from dvc.utils.fs import makedirs
 from dvc.utils.serialize import YAMLFileCorruptedError
@@ -218,6 +220,7 @@ def test_show_failed_experiment(tmp_dir, scm, dvc, failed_exp_stage):
                 "msg": "Experiment run failed.",
                 "type": "",
             },
+            "name": ANY,
         }
     }
 
@@ -242,7 +245,8 @@ def test_show_checkpoint(
     exp_rev = first(results)
 
     results = dvc.experiments.show()[baseline_rev]
-    assert len(results) == checkpoint_stage.iterations + 1
+    # Assert 4 rows: baseline, 2 checkpoints, and final commit
+    assert len(results) == checkpoint_stage.iterations + 2
 
     checkpoints = []
     for rev, exp in results.items():
@@ -686,14 +690,14 @@ def test_show_only_changed(tmp_dir, dvc, scm, capsys, copy_script):
     params_file = tmp_dir / "params.yaml"
     params_data = {
         "foo": 1,
-        "bar": 1,
+        "goobar": 1,
     }
     (tmp_dir / params_file).dump(params_data)
 
     dvc.run(
         cmd="python copy.py params.yaml metrics.yaml",
         metrics_no_cache=["metrics.yaml"],
-        params=["foo", "bar"],
+        params=["foo", "goobar"],
         name="copy-file",
         deps=["copy.py"],
     )
@@ -714,18 +718,18 @@ def test_show_only_changed(tmp_dir, dvc, scm, capsys, copy_script):
     capsys.readouterr()
     assert main(["exp", "show"]) == 0
     cap = capsys.readouterr()
-    assert "bar" in cap.out
+    assert "goobar" in cap.out
 
     capsys.readouterr()
     assert main(["exp", "show", "--only-changed"]) == 0
     cap = capsys.readouterr()
-    assert "bar" not in cap.out
+    assert "goobar" not in cap.out
 
     capsys.readouterr()
     assert main(["exp", "show", "--only-changed", "--keep=.*bar"]) == 0
     cap = capsys.readouterr()
-    assert "params.yaml:bar" in cap.out
-    assert "metrics.yaml:bar" in cap.out
+    assert "params.yaml:goobar" in cap.out
+    assert "metrics.yaml:goobar" in cap.out
 
 
 def test_show_parallel_coordinates(
@@ -1046,7 +1050,8 @@ def test_show_checkpoint_error(tmp_dir, scm, dvc, checkpoint_stage, mocker):
     exp_ref = str(first(exp_refs_by_rev(scm, exp_rev)))
 
     results = dvc.experiments.show()[baseline_rev]
-    assert len(results) == checkpoint_stage.iterations + 1
+    # Assert 4 rows: baseline, 2 checkpoints, and final commit
+    assert len(results) == checkpoint_stage.iterations + 2
 
     checkpoints = {}
     for rev in results:

tests/func/experiments/test_utils.py

@@ -0,0 +1,67 @@
+from dvc_studio_client import env
+from funcy import first
+
+
+def test_generate_random_exp_name(tmp_dir, dvc, scm, exp_stage, mocker):
+    mocker.patch(
+        "dvc.repo.experiments.utils.random.choice",
+        side_effect=[
+            0,
+            0,
+            0,
+            0,
+            1,
+            1,
+            0,
+            0,
+        ],
+    )
+
+    ref = first(dvc.experiments.run(exp_stage.addressing, params=["foo=1"]))
+    assert dvc.experiments.get_exact_name([ref])[ref] == "0-0"
+
+    # Causes 1 retry
+    ref = first(dvc.experiments.run(exp_stage.addressing, params=["foo=2"]))
+    assert dvc.experiments.get_exact_name([ref])[ref] == "1-1"
+
+    tmp_dir.scm_gen({"foo": "bar"}, commit="foo")
+    # Can use same name because of different baseline_rev
+    ref = first(dvc.experiments.run(exp_stage.addressing, params=["foo=1"]))
+    assert dvc.experiments.get_exact_name([ref])[ref] == "0-0"
+
+
+def test_post_to_studio(tmp_dir, dvc, scm, exp_stage, mocker, monkeypatch):
+    valid_response = mocker.MagicMock()
+    valid_response.status_code = 200
+    mocked_post = mocker.patch("requests.post", return_value=valid_response)
+
+    monkeypatch.setenv(env.STUDIO_ENDPOINT, "https://0.0.0.0")
+    monkeypatch.setenv(env.STUDIO_REPO_URL, "STUDIO_REPO_URL")
+    monkeypatch.setenv(env.STUDIO_TOKEN, "STUDIO_TOKEN")
+
+    baseline_sha = scm.get_rev()
+    exp_rev = first(
+        dvc.experiments.run(exp_stage.addressing, params=["foo=1"])
+    )
+    name = dvc.experiments.get_exact_name([exp_rev])[exp_rev]
+    assert mocked_post.call_count == 2
+
+    start_call, done_call = mocked_post.call_args_list
+
+    assert start_call.kwargs["json"] == {
+        "type": "start",
+        "repo_url": "STUDIO_REPO_URL",
+        "baseline_sha": baseline_sha,
+        "name": name,
+        "params": {"params.yaml": {"foo": 1}},
+        "client": "dvc",
+    }
+
+    assert done_call.kwargs["json"] == {
+        "type": "done",
+        "repo_url": "STUDIO_REPO_URL",
+        "baseline_sha": baseline_sha,
+        "name": name,
+        "client": "dvc",
+        "experiment_rev": exp_rev,
+    }

tests/func/metrics/test_diff.py

@@ -1,6 +1,10 @@
 import json
+from os.path import join
+
+import pytest
 
 from dvc.cli import main
+from dvc.utils import relpath
 
 
 def test_metrics_diff_simple(tmp_dir, scm, dvc, run_copy_metrics):
@@ -206,3 +210,45 @@ def test_metrics_diff_non_metrics(tmp_dir, scm, dvc):
     assert result == {
         "some_file.yaml": {"foo": {"old": 1, "new": 3, "diff": 2}}
     }
+
+
+@pytest.mark.parametrize(
+    "dvcfile, metrics_file",
+    [
+        ("dvc.yaml", "my_metrics.yaml"),
+        ("dir/dvc.yaml", "my_metrics.yaml"),
+        ("dir/dvc.yaml", join("..", "my_metrics.yaml")),
+    ],
+)
+def test_diff_top_level_metrics(tmp_dir, dvc, scm, dvcfile, metrics_file):
+    directory = (tmp_dir / dvcfile).parent
+    directory.mkdir(exist_ok=True)
+    (tmp_dir / dvcfile).dump({"metrics": [metrics_file]})
+
+    metrics_file = directory / metrics_file
+    metrics_file.dump({"foo": 3})
+    scm.add_commit([metrics_file, tmp_dir / dvcfile], message="add metrics")
+
+    metrics_file.dump({"foo": 5})
+    assert dvc.metrics.diff() == {
+        relpath(directory / metrics_file): {
+            "foo": {"diff": 2, "new": 5, "old": 3}
+        }
+    }
+
+
+def test_metrics_diff_active_branch_unchanged(
+    tmp_dir, scm, dvc, run_copy_metrics
+):
+    def _gen(val):
+        metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}}
+        (tmp_dir / "m_temp.yaml").dump(metrics)
+        run_copy_metrics(
+            "m_temp.yaml", "m.yaml", metrics=["m.yaml"], commit=str(val)
+        )
+
+    _gen(1)
+    _gen(2)
+    _gen(1)
+
+    assert dvc.metrics.diff(a_rev=tmp_dir.scm.active_branch()) == {}

tests/func/params/test_diff.py

@@ -1,3 +1,5 @@
+from os.path import join
+
 import pytest
 
 from dvc.utils import relpath
@@ -241,3 +243,36 @@ def test_diff_without_targets_specified(tmp_dir, dvc, scm, file):
             "y": {"new": "100", "old": None},
         }
     }
+
+
+@pytest.mark.parametrize(
+    "dvcfile, params_file",
+    [
+        ("dvc.yaml", "my_params.yaml"),
+        ("dir/dvc.yaml", "my_params.yaml"),
+        ("dir/dvc.yaml", join("..", "my_params.yaml")),
+    ],
+)
+def test_diff_top_level_params(tmp_dir, dvc, scm, dvcfile, params_file):
+    directory = (tmp_dir / dvcfile).parent
+    directory.mkdir(exist_ok=True)
+    (tmp_dir / dvcfile).dump({"params": [params_file]})
+
+    params_file = directory / params_file
+    params_file.dump({"foo": 3})
+    scm.add_commit([params_file, tmp_dir / dvcfile], message="add params")
+
+    params_file.dump({"foo": 5})
+    assert dvc.params.diff() == {
+        relpath(directory / params_file): {
+            "foo": {"diff": 2, "new": 5, "old": 3}
+        }
+    }
+
+
+def test_diff_active_branch_no_changes(tmp_dir, scm, dvc):
+    tmp_dir.gen("params.yaml", "foo: bar")
+    dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
+    scm.add(["params.yaml", "Dvcfile"])
+    scm.commit("bar")
+    assert dvc.params.diff(a_rev=tmp_dir.scm.active_branch()) == {}

tests/func/parsing/test_interpolated_entry.py

@@ -6,6 +6,7 @@ import pytest
 from dvc.dependency import _merge_params
 from dvc.parsing import DEFAULT_PARAMS_FILE, DataResolver
 from dvc.parsing.context import recurse_not_a_node
+from dvc.parsing.interpolate import escape_str
 
 from . import (
     CONTEXT_DATA,
@@ -272,14 +273,17 @@ def test_cmd_dict(tmp_dir, dvc, bool_config, list_config):
         if list_config:
             conf["parsing"]["list"] = list_config
 
+    string = "spaced string"
+    mixed_quote_string = "quote\"'d"
     data = {
         "dict": {
             "foo": "foo",
             "bar": 2,
-            "string": "spaced string",
+            "string": string,
+            "mixed_quote_string": mixed_quote_string,
             "bool": True,
             "bool-false": False,
-            "list": [1, 2, "foo"],
+            "list": [1, 2, "foo", mixed_quote_string],
             "nested": {"foo": "foo"},
         }
     }
@@ -296,9 +300,10 @@ def test_cmd_dict(tmp_dir, dvc, bool_config, list_config):
         bool_resolved = " --bool --no-bool-false"
 
     if list_config is None or list_config == "nargs":
-        list_resolved = " --list 1 2 'foo'"
+        list_resolved = f" --list 1 2 foo {escape_str(mixed_quote_string)}"
     else:
-        list_resolved = " --list 1 --list 2 --list 'foo'"
+        list_resolved = " --list 1 --list 2 --list foo"
+        list_resolved += f" --list {escape_str(mixed_quote_string)}"
 
     assert_stage_equal(
         resolver.resolve(),
@@ -306,11 +311,12 @@ def test_cmd_dict(tmp_dir, dvc, bool_config, list_config):
             "stages": {
                 "stage1": {
                     "cmd": "python script.py"
-                    " --foo 'foo' --bar 2"
-                    " --string 'spaced string'"
+                    " --foo foo --bar 2"
+                    f" --string {escape_str(string)}"
+                    f" --mixed_quote_string {escape_str(mixed_quote_string)}"
                     f"{bool_resolved}"
                     f"{list_resolved}"
-                    " --nested.foo 'foo'"
+                    " --nested.foo foo"
                 }
             }
         },

tests/func/plots/test_show.py

@@ -336,16 +336,15 @@ def test_collect_non_existing_dir(tmp_dir, dvc, run_copy_metrics):
 @pytest.mark.parametrize(
     "plot_config,expected_datafiles",
     [
-        # TODO - enable providing data files for x
-        # (
-        #     {
-        #         "comparison": {
-        #             "x": {"data1.json": "a"},
-        #             "y": {"sub/dir/data2.json": "b"},
-        #         }
-        #     },
-        #     ["data1.json", os.path.join("sub", "dir", "data2.json")],
-        # ),
+        (
+            {
+                "comparison": {
+                    "x": {"data1.json": "a"},
+                    "y": {"sub/dir/data2.json": "b"},
+                }
+            },
+            ["data1.json", os.path.join("sub", "dir", "data2.json")],
+        ),
         (
             {"data1.json": {"x": "c", "y": "a", "title": "File as key test"}},
             ["data1.json"],

tests/func/test_data_cloud.py

@@ -478,7 +478,7 @@ def test_push_pull_all(tmp_dir, scm, dvc, local_remote, key, expected):
 
 def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, local_remote):
     tmp_dir.dvc_gen("foo", "foo")
-    run_copy("foo", "bar", no_commit=True, name="copy-foo-bar")
+    run_copy("foo", "bar", name="copy-foo-bar")
 
     dvc.push("copy-foo-bar")
     assert len(recurse_list_dir(local_remote.url)) == 1

tests/func/test_import.py

@@ -304,7 +304,7 @@ def test_pull_imported_directory_stage(tmp_dir, dvc, erepo_dir):
     dvc.imp(os.fspath(erepo_dir), "dir", "dir_imported")
 
     remove("dir_imported")
-    remove(dvc.odb.local.path)
+    dvc.odb.local.clear()
 
     dvc.pull(["dir_imported.dvc"])
 
@@ -320,7 +320,7 @@ def test_pull_wildcard_imported_directory_stage(tmp_dir, dvc, erepo_dir):
     dvc.imp(os.fspath(erepo_dir), "dir123", "dir_imported123")
 
     remove("dir_imported123")
-    remove(dvc.odb.local.path)
+    dvc.odb.local.clear()
 
     dvc.pull(["dir_imported*.dvc"], glob=True)
 
@@ -546,9 +546,8 @@ def test_pull_imported_stage_from_subrepos(
     dvc.imp(os.fspath(erepo_dir), path, out="out")
 
     # clean everything
-    remove(dvc.odb.local.path)
+    dvc.odb.local.clear()
     remove("out")
-    makedirs(dvc.odb.local.path)
 
     stats = dvc.pull(["out.dvc"])
 
@@ -633,7 +632,7 @@ def test_chained_import(tmp_dir, dvc, make_tmp_dir, erepo_dir, local_cloud):
     assert (dst / "foo").read_text() == "foo"
     assert (dst / "bar").read_text() == "bar"
 
-    remove(dvc.odb.local.path)
+    dvc.odb.local.clear()
     remove("dir_imported")
 
     # pulled objects should come from the original upstream repo's remote,

tests/func/test_ls.py

@@ -587,4 +587,10 @@ def test_broken_symlink(tmp_dir, dvc):
             "isexec": False,
             "path": ".dvcignore",
         },
+        {
+            "isout": False,
+            "isdir": False,
+            "isexec": False,
+            "path": "link",
+        },
     ]

tests/func/test_repo_index.py

@@ -1,12 +1,10 @@
-import os
 from itertools import chain
 
 import pytest
 from pygtrie import Trie
 
 from dvc.repo.index import Index
-from dvc.stage import PipelineStage, Stage
-from dvc.utils import relpath
+from dvc.stage import Stage
 
 
 def test_index(tmp_dir, scm, dvc, run_copy):
@@ -17,15 +15,11 @@ def test_index(tmp_dir, scm, dvc, run_copy):
     index = Index(dvc)
     assert index.fs == dvc.fs
 
-    assert len(index) == len(index.stages) == 2
-    assert set(index.stages) == set(index) == {stage1, stage2}
-    assert stage1 in index
-    assert stage2 in index
+    assert set(index.stages) == {stage1, stage2}
 
     assert index.outs_graph
     assert index.graph
     assert isinstance(index.outs_trie, Trie)
-    assert index.identifier
     index.check_graph()
 
 
@@ -42,41 +36,6 @@ def test_repr(tmp_dir, scm, dvc):
     assert repr(Index(dvc)) == f"Index({dvc}, fs@{rev[:7]})"
 
 
-def test_filter_index(tmp_dir, dvc, run_copy):
-    tmp_dir.dvc_gen("foo", "foo")
-    stage2 = run_copy("foo", "bar", name="copy-foo-bar")
-
-    def filter_pipeline(stage):
-        return bool(stage.cmd)
-
-    filtered_index = Index(dvc).filter(filter_pipeline)
-    assert list(filtered_index) == [stage2]
-
-
-def test_slice_index(tmp_dir, dvc):
-    tmp_dir.gen({"dir1": {"foo": "foo"}, "dir2": {"bar": "bar"}})
-    with (tmp_dir / "dir1").chdir():
-        (stage1,) = dvc.add("foo")
-    with (tmp_dir / "dir2").chdir():
-        (stage2,) = dvc.add("bar")
-
-    index = Index(dvc)
-
-    sliced = index.slice("dir1")
-    assert set(sliced) == {stage1}
-    assert sliced.stages is not index.stages  # sanity check
-
-    sliced = index.slice(tmp_dir / "dir1")
-    assert set(sliced) == {stage1}
-
-    sliced = index.slice("dir2")
-    assert set(sliced) == {stage2}
-
-    with (tmp_dir / "dir1").chdir():
-        sliced = index.slice(relpath(tmp_dir / "dir2"))
-        assert set(sliced) == {stage2}
-
-
 def outputs_equal(actual, expected):
     actual, expected = list(actual), list(expected)
 
@@ -147,7 +106,7 @@ def test_add_update(dvc):
     dup_stage2 = Stage(dvc, path="path2")
     dup_index = index.update([dup_stage1, dup_stage2])
     assert not index.stages
-    assert len(new_index) == 1
+    assert len(new_index.stages) == 1
     assert new_index.stages[0] is new_stage
     assert set(map(id, dup_index.stages)) == {id(dup_stage1), id(dup_stage2)}
 
@@ -165,67 +124,6 @@ def assert_index_equal(first, second, strict=True, ordered=True):
         ), "Index is not strictly equal"
 
 
-def test_discard_remove(dvc):
-    stage = Stage(dvc, path="path1")
-    index = Index(dvc, stages=[stage])
-
-    assert list(index.discard(Stage(dvc, "path2"))) == list(index)
-    new_index = index.discard(stage)
-    assert len(new_index) == 0
-
-    with pytest.raises(ValueError):
-        index.remove(Stage(dvc, "path2"))
-    assert index.stages == [stage]
-    assert list(index.remove(stage)) == []
-
-
-def test_difference(dvc):
-    stages = [Stage(dvc, path=f"path{i}") for i in range(10)]
-    index = Index(dvc, stages=stages)
-
-    new_index = index.difference([*stages[:5], Stage(dvc, path="path100")])
-    assert index.stages == stages
-    assert set(new_index) == set(stages[5:])
-
-
-def test_dumpd(dvc):
-    stages = [
-        PipelineStage(dvc, "dvc.yaml", name="stage1"),
-        Stage(dvc, "path"),
-    ]
-    index = Index(dvc, stages=stages)
-    assert index.dumpd() == {"dvc.yaml:stage1": {}, "path": {}}
-    assert index.identifier == "d43da84e9001540c26abf2bf4541c275"
-
-
-def test_unique_identifier(tmp_dir, dvc, scm, run_copy):
-    dvc.scm_context.autostage = True
-    tmp_dir.dvc_gen("foo", "foo")
-    run_copy("foo", "bar", name="copy-foo-bar")
-
-    revs = []
-    n_commits = 5
-    for i in range(n_commits):
-        # create a few empty commits
-        scm.commit(f"commit {i}")
-        revs.append(scm.get_rev())
-    assert len(set(revs)) == n_commits  # the commit revs should be unique
-
-    ids = []
-    for _ in dvc.brancher(revs=revs):
-        index = Index(dvc)
-        assert index.stages
-        ids.append(index.identifier)
-
-    # we get "workspace" as well from the brancher by default
-    assert len(revs) + 1 == len(ids)
-    possible_ids = {
-        True: "dd9cf3a3dc2caac5835e64f258ac2d7b",
-        False: "b446f3cdc35539884dcc474d90ac6286",
-    }
-    assert set(ids) == {possible_ids[os.name == "posix"]}
-
-
 def test_skip_graph_checks(dvc, mocker):
     # See https://github.com/iterative/dvc/issues/2671 for more info
     mock_build_graph = mocker.spy(Index.graph, "fget")
@@ -293,18 +191,6 @@ def test_used_objs(tmp_dir, scm, dvc, run_copy, rev):
     }
 
 
-def test_getitem(tmp_dir, dvc, run_copy):
-    (stage1,) = tmp_dir.dvc_gen("foo", "foo")
-    stage2 = run_copy("foo", "bar", name="copy-foo-bar")
-
-    index = Index(dvc)
-    assert index["foo.dvc"] == stage1
-    assert index["copy-foo-bar"] == stage2
-
-    with pytest.raises(KeyError):
-        _ = index["no-valid-stage-name"]
-
-
 def test_view_granular_dir(tmp_dir, scm, dvc, run_copy):
     tmp_dir.dvc_gen(
         {"dir": {"subdir": {"in-subdir": "in-subdir"}, "in-dir": "in-dir"}},

tests/func/test_repro.py

@@ -776,7 +776,8 @@ def test_cmd_repro(tmp_dir, copy_script, run_stage):
 
 
 @pytest.mark.skipif(os.name == "nt", reason="not on nt")
-def test_repro_shell(tmp_dir, dvc):
+def test_repro_shell(tmp_dir, monkeypatch, dvc):
+    monkeypatch.setenv("SHELL", "/bin/sh")
     dvc.run(
         fname="shell.txt.dvc",
         outs=["shell.txt"],
@@ -842,7 +843,8 @@ def test_repro_no_commit(tmp_dir, dvc, copy_script, run_stage):
     remove(dvc.odb.local.path)
     ret = main(["repro", stage.addressing, "--no-commit"])
     assert ret == 0
-    assert os.listdir(dvc.odb.local.path) == ["runs"]
+    # run-cache should be skipped if `-no-commit`.
+    assert not os.path.isdir(dvc.odb.local.path)
 
 
 class TestReproAlreadyCached:

tests/func/test_run_cache.py

@@ -33,7 +33,7 @@ def test_restore(tmp_dir, dvc, run_copy, mocker):
 
     (stage,) = dvc.reproduce("copy-foo-bar")
 
-    mock_restore.assert_called_once_with(stage)
+    mock_restore.assert_called_once_with(stage, dry=False)
     mock_run.assert_not_called()
     assert (tmp_dir / "bar").exists() and not (tmp_dir / "foo").unlink()
     assert (tmp_dir / PIPELINE_LOCK).exists()
@@ -100,7 +100,7 @@ def test_memory_for_multiple_runs_of_same_stage(
     assert (tmp_dir / PIPELINE_LOCK).exists()
     assert (tmp_dir / "bar").read_text() == "foobar"
     mock_run.assert_not_called()
-    mock_restore.assert_called_once_with(stage)
+    mock_restore.assert_called_once_with(stage, dry=False)
     mock_restore.reset_mock()
 
     (tmp_dir / PIPELINE_LOCK).unlink()
@@ -109,7 +109,7 @@ def test_memory_for_multiple_runs_of_same_stage(
 
     assert (tmp_dir / "bar").read_text() == "foo"
     mock_run.assert_not_called()
-    mock_restore.assert_called_once_with(stage)
+    mock_restore.assert_called_once_with(stage, dry=False)
     assert (tmp_dir / "bar").exists() and not (tmp_dir / "foo").unlink()
     assert (tmp_dir / PIPELINE_LOCK).exists()
 
@@ -138,7 +138,7 @@ def test_memory_runs_of_multiple_stages(tmp_dir, dvc, run_copy, mocker):
     assert (tmp_dir / "foo.bak").read_text() == "foo"
     assert (tmp_dir / PIPELINE_LOCK).exists()
     mock_run.assert_not_called()
-    mock_restore.assert_called_once_with(stage)
+    mock_restore.assert_called_once_with(stage, dry=False)
     mock_restore.reset_mock()
 
     (stage,) = dvc.reproduce("backup-bar")
@@ -146,7 +146,7 @@ def test_memory_runs_of_multiple_stages(tmp_dir, dvc, run_copy, mocker):
     assert (tmp_dir / "bar.bak").read_text() == "bar"
     assert (tmp_dir / PIPELINE_LOCK).exists()
     mock_run.assert_not_called()
-    mock_restore.assert_called_once_with(stage)
+    mock_restore.assert_called_once_with(stage, dry=False)
 
 
 def test_restore_pull(tmp_dir, dvc, run_copy, mocker, local_remote):
@@ -168,7 +168,7 @@ def test_restore_pull(tmp_dir, dvc, run_copy, mocker, local_remote):
 
     (stage,) = dvc.reproduce("copy-foo-bar", pull=True)
 
-    mock_restore.assert_called_once_with(stage, pull=True)
+    mock_restore.assert_called_once_with(stage, pull=True, dry=False)
     mock_run.assert_not_called()
     assert mock_checkout.call_count == 2
     assert (tmp_dir / "bar").exists() and not (tmp_dir / "foo").unlink()