stage add: don't fail if unable to create .gitignore

Bumps [pylint](https://github.com/PyCQA/pylint) from 2.15.5 to 2.15.7.
- [Release notes](https://github.com/PyCQA/pylint/releases)
- [Commits](https://github.com/PyCQA/pylint/compare/v2.15.5...v2.15.7)

---
updated-dependencies:
- dependency-name: pylint
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <[email protected]>

dvc/cli/parser.py

@@ -2,8 +2,8 @@
 import argparse
 import logging
 import os
-import sys
 
+from dvc import __version__
 from dvc.commands import (
     add,
     cache,
@@ -128,17 +128,6 @@ class DvcParser(argparse.ArgumentParser):
         return args
 
 
-class VersionAction(argparse.Action):  # pragma: no cover
-    # pylint: disable=too-few-public-methods
-    """Shows DVC version and exits."""
-
-    def __call__(self, parser, namespace, values, option_string=None):
-        from dvc import __version__
-
-        print(__version__)
-        sys.exit(0)
-
-
 def get_parent_parser():
     """Create instances of a parser containing common arguments shared among
     all the commands.
@@ -186,13 +175,11 @@ def get_main_parser():
         help="Show this help message and exit.",
     )
 
-    # NOTE: On some python versions action='version' prints to stderr
-    # instead of stdout https://bugs.python.org/issue18920
     parser.add_argument(
         "-V",
         "--version",
-        action=VersionAction,
-        nargs=0,
+        action="version",
+        version=__version__,
         help="Show program's version.",
     )
 

dvc/commands/experiments/__init__.py

@@ -18,6 +18,7 @@ from dvc.commands.experiments import (
     queue_worker,
     remove,
     run,
+    save,
     show,
 )
 
@@ -34,6 +35,7 @@ SUB_COMMANDS = [
     queue_worker,
     remove,
     run,
+    save,
     show,
 ]
 

dvc/commands/experiments/ls.py

@@ -3,6 +3,7 @@ import logging
 
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
@@ -16,22 +17,13 @@ class CmdExperimentsList(CmdBase):
             num=self.args.num,
             git_remote=self.args.git_remote,
         )
-        tags = self.repo.scm.describe(exps)
-        remained = {baseline for baseline, tag in tags.items() if tag is None}
-        base = "refs/heads/"
-        ref_heads = self.repo.scm.describe(remained, base=base)
 
         for baseline in exps:
-            name = (
-                tags[baseline]
-                or ref_heads[baseline][len(base) :]
-                or baseline[:7]
-            )
             if not name_only:
-                print(f"{name}:")
+                ui.write(f"{baseline}:")
             for exp_name in exps[baseline]:
                 indent = "" if name_only else "\t"
-                print(f"{indent}{exp_name}")
+                ui.write(f"{indent}{exp_name}")
 
         return 0
 

dvc/commands/experiments/save.py

@@ -0,0 +1,79 @@
+import argparse
+import logging
+
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.exceptions import DvcException
+from dvc.ui import ui
+
+logger = logging.getLogger(__name__)
+
+
+class CmdExperimentsSave(CmdBase):
+    def run(self):
+
+        try:
+            ref = self.repo.experiments.save(
+                name=self.args.name,
+                force=self.args.force,
+                include_untracked=self.args.include_untracked,
+            )
+        except DvcException:
+            logger.exception("failed to save experiment")
+            return 1
+
+        if self.args.json:
+            ui.write_json({"ref": ref})
+        else:
+            name = self.repo.experiments.get_exact_name([ref])[ref]
+            ui.write(f"Experiment has been saved as: {name}")
+            ui.write(
+                "\nTo promote an experiment to a Git branch run:\n\n"
+                "\tdvc exp branch <exp> <branch>\n"
+            )
+
+        return 0
+
+
+def add_parser(experiments_subparsers, parent_parser):
+    EXPERIMENTS_SAVE_HELP = "Save current workspace as a dvc experiment."
+    save_parser = experiments_subparsers.add_parser(
+        "save",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_SAVE_HELP, "exp/save"),
+        help=EXPERIMENTS_SAVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    save_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Save even if hash value for dependencies/outputs changed.",
+    )
+    save_parser.add_argument(
+        "--json",
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    save_parser.add_argument(
+        "-n",
+        "--name",
+        default=None,
+        help=(
+            "Human-readable experiment name. If not specified, a name will "
+            "be auto-generated."
+        ),
+        metavar="<name>",
+    )
+    save_parser.add_argument(
+        "-I",
+        "--include-untracked",
+        action="append",
+        default=[],
+        help="List of untracked paths to include in the experiment.",
+        metavar="<path>",
+    )
+    save_parser.set_defaults(func=CmdExperimentsSave)

dvc/data_cloud.py

@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Iterable, Optional
 
 from funcy import cached_property
 
+from dvc.config import RemoteConfigError
 from dvc_data.hashfile.db import get_index
 
 if TYPE_CHECKING:
@@ -23,6 +24,14 @@ class Remote:
 
         self.worktree = config.pop("worktree", False)
         self.config = config
+        if self.worktree:
+            version_aware = self.config.get("version_aware")
+            if version_aware is False:
+                raise RemoteConfigError(
+                    "worktree remotes require version_aware cloud"
+                )
+            if version_aware is None:
+                self.fs.version_aware = True
 
     @cached_property
     def odb(self):

dvc/env.py

@@ -4,6 +4,8 @@ DVC_PAGER = "DVC_PAGER"
 DVC_ROOT = "DVC_ROOT"
 DVCLIVE_RESUME = "DVCLIVE_RESUME"
 DVC_IGNORE_ISATTY = "DVC_IGNORE_ISATTY"
+DVC_EXP_BASELINE_REV = "DVC_EXP_BASELINE_REV"
+DVC_EXP_NAME = "DVC_EXP_NAME"
 DVC_EXP_GIT_REMOTE = "DVC_EXP_GIT_REMOTE"
 DVC_EXP_AUTO_PUSH = "DVC_EXP_AUTO_PUSH"
 DVC_NO_ANALYTICS = "DVC_NO_ANALYTICS"

dvc/output.py

@@ -90,6 +90,7 @@ def loadd_from(stage, d_list):
         remote = d.pop(Output.PARAM_REMOTE, None)
         annot = {field: d.pop(field, None) for field in ANNOTATION_FIELDS}
         files = d.pop(Output.PARAM_FILES, None)
+        push = d.pop(Output.PARAM_PUSH, True)
         ret.append(
             _get(
                 stage,
@@ -103,6 +104,7 @@ def loadd_from(stage, d_list):
                 remote=remote,
                 **annot,
                 files=files,
+                push=push,
             )
         )
     return ret
@@ -117,6 +119,7 @@ def loads_from(
     persist=False,
     checkpoint=False,
     remote=None,
+    push=True,
 ):
     return [
         _get(
@@ -129,6 +132,7 @@ def loads_from(
             persist=persist,
             checkpoint=checkpoint,
             remote=remote,
+            push=push,
         )
         for s in s_list
     ]
@@ -184,6 +188,7 @@ def load_from_pipeline(stage, data, typ="outs"):
                 Output.PARAM_PERSIST,
                 Output.PARAM_CHECKPOINT,
                 Output.PARAM_REMOTE,
+                Output.PARAM_PUSH,
                 *ANNOTATION_FIELDS,
             ],
         )
@@ -251,6 +256,7 @@ class Output:
     PARAM_PLOT_HEADER = "header"
     PARAM_PERSIST = "persist"
     PARAM_REMOTE = "remote"
+    PARAM_PUSH = "push"
 
     METRIC_SCHEMA = Any(
         None,
@@ -284,6 +290,7 @@ class Output:
         repo=None,
         fs_config=None,
         files: List[Dict[str, Any]] = None,
+        push: bool = True,
     ):
         self.annot = Annotation(
             desc=desc, type=type, labels=labels or [], meta=meta or {}
@@ -338,6 +345,7 @@ class Output:
         self.plot = False if self.IS_DEPENDENCY else plot
         self.persist = persist
         self.checkpoint = checkpoint
+        self.can_push = push
 
         self.fs_path = self._parse_path(self.fs, fs_path)
         self.obj: Optional["HashFile"] = None
@@ -776,6 +784,9 @@ class Output:
             if self.remote:
                 ret[self.PARAM_REMOTE] = self.remote
 
+            if not self.can_push:
+                ret[self.PARAM_PUSH] = self.can_push
+
         if (
             (not self.IS_DEPENDENCY or self.stage.is_import)
             and self.hash_info.isdir
@@ -1032,9 +1043,15 @@ class Output:
         if not self.use_cache:
             return {}
 
+        push: bool = kwargs.pop("push", False)
         if self.stage.is_repo_import:
+            if push:
+                return {}
             return self.get_used_external(**kwargs)
 
+        if push and not self.can_push:
+            return {}
+
         if not self.hash_info:
             msg = (
                 "Output '{}'({}) is missing version info. "
@@ -1180,6 +1197,34 @@ class Output:
     def is_plot(self) -> bool:
         return bool(self.plot)
 
+    def restore_fields(self, other: "Output"):
+        """Restore attributes that need to be preserved when serialized."""
+        self.annot = other.annot
+        self.remote = other.remote
+        self.can_push = other.can_push
+
+    def merge_version_meta(self, other: "Output"):
+        """Merge version meta for files which are unchanged from other."""
+        if not self.hash_info:
+            return
+        if self.hash_info.isdir:
+            return self._merge_dir_version_meta(other)
+        if self.hash_info != other.hash_info:
+            return
+        self.meta = other.meta
+
+    def _merge_dir_version_meta(self, other: "Output"):
+        from dvc_data.hashfile.tree import update_meta
+
+        if not self.obj or not other.hash_info.isdir:
+            return
+        other_obj = other.obj if other.obj is not None else other.get_obj()
+        assert isinstance(self.obj, Tree) and isinstance(other_obj, Tree)
+        updated = update_meta(self.obj, other_obj)
+        assert updated.hash_info == self.obj.hash_info
+        self.obj = updated
+        self.files = updated.as_list(with_meta=True)
+
 
 META_SCHEMA = {
     Meta.PARAM_SIZE: int,
@@ -1209,5 +1254,6 @@ SCHEMA = {
     Output.PARAM_CACHE: bool,
     Output.PARAM_METRIC: Output.METRIC_SCHEMA,
     Output.PARAM_REMOTE: str,
+    Output.PARAM_PUSH: bool,
     Output.PARAM_FILES: [DIR_FILES_SCHEMA],
 }

dvc/parsing/interpolate.py

@@ -1,3 +1,4 @@
+import os
 import re
 import typing
 from collections.abc import Iterable, Mapping
@@ -70,6 +71,16 @@ def embrace(s: str):
     return BRACE_OPEN + s + BRACE_CLOSE
 
 
+def escape_str(value):
+    if os.name == "nt":
+        from subprocess import list2cmdline
+
+        return list2cmdline([value])
+    from shlex import quote
+
+    return quote(value)
+
+
 @singledispatch
 def to_str(obj) -> str:
     return str(obj)
@@ -97,12 +108,12 @@ def _(obj: dict):
                     result += f"--no-{k} "
 
         elif isinstance(v, str):
-            result += f"--{k} '{v}' "
+            result += f"--{k} {escape_str(v)} "
 
         elif isinstance(v, Iterable):
             for n, i in enumerate(v):
                 if isinstance(i, str):
-                    i = f"'{i}'"
+                    i = escape_str(i)
                 elif isinstance(i, Iterable):
                     raise ParseError(
                         f"Cannot interpolate nested iterable in '{k}'"

dvc/render/convert.py

@@ -14,14 +14,14 @@ from dvc.render.converter.vega import VegaConverter
 
 
 def _get_converter(
-    renderer_class, props
+    renderer_class, renderer_id, props, data
 ) -> Union[VegaConverter, ImageConverter]:
     from dvc_render import ImageRenderer, VegaRenderer
 
     if renderer_class.TYPE == VegaRenderer.TYPE:
-        return VegaConverter(props)
+        return VegaConverter(renderer_id, data, props)
     if renderer_class.TYPE == ImageRenderer.TYPE:
-        return ImageConverter(props)
+        return ImageConverter(renderer_id, data, props)
 
     raise ValueError(f"Invalid renderer class {renderer_class}")
 

dvc/render/converter/__init__.py

@@ -1,9 +1,19 @@
-from typing import Dict, Optional
+from typing import Any, Dict, List, Optional, Tuple
 
 
 class Converter:
-    def __init__(self, plot_properties: Optional[Dict] = None):
-        self.plot_properties = plot_properties or {}
+    def __init__(
+        self,
+        plot_id: str,
+        data: Dict[str, Any] = None,
+        properties: Optional[Dict] = None,
+    ):
+        self.plot_id = plot_id
+        self.properties = properties or {}
+        self.data = data or {}
 
-    def convert(self, data, revision: str, filename: str, **kwargs):
+    def convert(self) -> Tuple[List[Tuple[str, str, Any]], Dict]:
+        raise NotImplementedError
+
+    def flat_datapoints(self, revision: str) -> Tuple[List[Dict], Dict]:
         raise NotImplementedError

dvc/render/converter/image.py

@@ -1,6 +1,6 @@
 import base64
 import os
-from typing import TYPE_CHECKING, Dict, List, Tuple
+from typing import TYPE_CHECKING, Any, Dict, List, Tuple
 
 from dvc.render import FILENAME_FIELD, REVISION_FIELD, SRC_FIELD
 
@@ -34,25 +34,33 @@ class ImageConverter(Converter):
         base64_str = base64.b64encode(image_data).decode()
         return f"data:image;base64,{base64_str}"
 
-    def convert(
-        self, data, revision: str, filename: str, **kwargs
-    ) -> Tuple[List[Dict], Dict]:
+    def convert(self) -> Tuple[List[Tuple[str, str, Any]], Dict]:
+        datas = []
+        for filename, image_data in self.data.items():
+            datas.append((filename, "", image_data))
+        return datas, self.properties
+
+    def flat_datapoints(self, revision: str) -> Tuple[List[Dict], Dict]:
         """
         Convert the DVC Plots content to DVC Render datapoints.
         Return both generated datapoints and updated properties.
         """
-        path = self.plot_properties.get("out")
-        if path:
-            if not os.path.isdir(path):
-                os.makedirs(path, exist_ok=True)
-            src = self._write_image(
-                os.path.abspath(path), revision, filename, data
-            )
-        else:
-            src = self._encode_image(data)
-        datapoint = {
-            REVISION_FIELD: revision,
-            FILENAME_FIELD: filename,
-            SRC_FIELD: src,
-        }
-        return [datapoint], self.plot_properties
+        path = self.properties.get("out")
+        datapoints = []
+        datas, properties = self.convert()
+        for filename, _, image_data in datas:
+            if path:
+                if not os.path.isdir(path):
+                    os.makedirs(path, exist_ok=True)
+                src = self._write_image(
+                    os.path.abspath(path), revision, filename, image_data
+                )
+            else:
+                src = self._encode_image(image_data)
+            datapoint = {
+                REVISION_FIELD: revision,
+                FILENAME_FIELD: filename,
+                SRC_FIELD: src,
+            }
+            datapoints.append(datapoint)
+        return datapoints, properties

dvc/render/converter/vega.py

@@ -1,90 +1,100 @@
 from copy import deepcopy
-from functools import partial
-from typing import Dict, Iterable, List, Optional, Set, Union
+from typing import Any, Dict, Iterable, List, Tuple, Union
 
-from funcy import first, project
+from funcy import first, last
 
 from dvc.exceptions import DvcException
-from dvc.render import (
-    FILENAME_FIELD,
-    INDEX_FIELD,
-    REVISION_FIELD,
-    VERSION_FIELD,
-)
+from dvc.render import FILENAME_FIELD, INDEX_FIELD, VERSION_FIELD
 
 from . import Converter
 
 
-class FieldsNotFoundError(DvcException):
-    def __init__(self, expected_fields, found_fields):
-        expected_str = ", ".join(expected_fields)
+class FieldNotFoundError(DvcException):
+    def __init__(self, expected_field, found_fields):
         found_str = ", ".join(found_fields)
         super().__init__(
-            f"Could not find all provided fields ('{expected_str}') "
+            f"Could not find provided field ('{expected_field}') "
             f"in data fields ('{found_str}')."
         )
 
 
-class PlotDataStructureError(DvcException):
-    def __init__(self):
-        super().__init__(
-            "Plot data extraction failed. Please see "
-            "https://man.dvc.org/plots for supported data formats."
-        )
+def _lists(blob: Union[Dict, List]) -> Iterable[List]:
+    if isinstance(blob, list):
+        yield blob
+    else:
+        for _, value in blob.items():
+            if isinstance(value, dict):
+                yield from _lists(value)
+            elif isinstance(value, list):
+                yield value
 
 
-def _filter_fields(
-    datapoints: List[Dict], fields: Set, **kwargs
-) -> List[Dict]:
-    if not fields:
-        return datapoints
-    assert isinstance(fields, set)
-
-    new_data = []
-    for data_point in datapoints:
-        keys = set(data_point.keys())
-        if not fields <= keys:
-            raise FieldsNotFoundError(fields, keys)
-
-        new_data.append(project(data_point, fields))
-
-    return new_data
+def _file_field(*args):
+    for axis_def in args:
+        if axis_def is not None:
+            for file, val in axis_def.items():
+                if isinstance(val, str):
+                    yield file, val
+                elif isinstance(val, list):
+                    for field in val:
+                        yield file, field
 
 
-def _lists(dictionary: Dict):
-    for _, value in dictionary.items():
-        if isinstance(value, dict):
-            yield from _lists(value)
-        elif isinstance(value, list):
-            yield value
+def _find(
+    filename: str,
+    field: str,
+    data_series: List[Tuple[str, str, Any]],
+):
+    for data_file, data_field, data in data_series:
+        if data_file == filename and data_field == field:
+            return data_file, data_field, data
+    return None
 
 
-def _find_first_list(
-    data: Union[Dict, List], fields: Set, **kwargs
-) -> List[Dict]:
-    fields = fields or set()
-
-    if not isinstance(data, dict):
-        return data
-
-    for lst in _lists(data):
-        if (
-            all(isinstance(dp, dict) for dp in lst)
-            # if fields is empty, it will match any set
-            and set(first(lst).keys()) & fields == fields
-        ):
-            return lst
-
-    raise PlotDataStructureError()
+def _verify_field(file2datapoints: Dict[str, List], filename: str, field: str):
+    if filename in file2datapoints:
+        datapoint = first(file2datapoints[filename])
+        if field not in datapoint:
+            raise FieldNotFoundError(field, datapoint.keys())
+    return
 
 
-def _append_index(datapoints: List[Dict], **kwargs) -> List[Dict]:
-    if INDEX_FIELD in first(datapoints).keys():
-        return datapoints
+def _get_xs(properties: Dict, file2datapoints: Dict[str, List[Dict]]):
+    x = properties.get("x", None)
+    if x is not None and isinstance(x, dict):
+        for filename, field in _file_field(x):
+            _verify_field(file2datapoints, filename, field)
+            yield filename, field
 
-    for index, data_point in enumerate(datapoints):
-        data_point[INDEX_FIELD] = index
-    return datapoints
+
+def _get_ys(properties, file2datapoints: Dict[str, List[Dict]]):
+    y = properties.get("y", None)
+    if y is not None:
+        for filename, field in _file_field(y):
+            _verify_field(file2datapoints, filename, field)
+            yield filename, field
+
+
+def _is_datapoints(lst: List[Dict]):
+    """
+    check if dict keys match, datapoints with different keys mgiht lead
+    to unexpected behavior
+    """
+
+    return all(isinstance(item, dict) for item in lst) and set(
+        first(lst).keys()
+    ) == {key for keys in lst for key in keys}
+
+
+def get_datapoints(file_content: Dict):
+    result: List[Dict[str, Any]] = []
+    for lst in _lists(file_content):
+        if _is_datapoints(lst):
+            for index, datapoint in enumerate(lst):
+                if len(result) <= index:
+                    result.append({})
+                result[index].update(datapoint)
+    return result
 
 
 class VegaConverter(Converter):
@@ -95,228 +105,225 @@ class VegaConverter(Converter):
     ('x', 'y') it will attempt to fill in the blanks.
     """
 
-    def __init__(self, plot_properties: Optional[Dict] = None):
-        super().__init__(plot_properties)
+    def __init__(
+        self, plot_id: str, data: Dict = None, properties: Dict = None
+    ):
+        super().__init__(plot_id, data, properties)
+        self.plot_id = plot_id
         self.inferred_properties: Dict = {}
 
-        self.steps = []
+        # TODO we should be handling that in `convert`,
+        #      to avoid stateful `self.inferred_properties`
+        self._infer_x_y()
 
-        self._infer_x()
-        self._infer_fields()
+    def _infer_y_from_data(self):
+        if self.plot_id in self.data:
+            for lst in _lists(self.data[self.plot_id]):
+                if all(isinstance(item, dict) for item in lst):
+                    datapoint = first(lst)
+                    field = last(datapoint.keys())
+                    self.inferred_properties["y"] = {self.plot_id: field}
+                    break
 
-        self.steps.append(
-            (
-                "find_data",
-                partial(
-                    _find_first_list,
-                    fields=self.inferred_properties.get("fields", set())
-                    - {INDEX_FIELD},
-                ),
-            )
-        )
+    def _infer_x_y(self):
+        def _infer_files(from_name, to_name):
+            from_value = self.properties.get(from_name, None)
+            to_value = self.properties.get(to_name, None)
 
-        if not self.plot_properties.get("x", None):
-            self.steps.append(("append_index", partial(_append_index)))
-
-        self.steps.append(
-            (
-                "filter_fields",
-                partial(
-                    _filter_fields,
-                    fields=self.inferred_properties.get("fields", set()),
-                ),
-            )
-        )
-        self.steps.append(
-            (
-                "infer_y",
-                partial(
-                    self._infer_y,
-                ),
-            )
-        )
-
-        self.steps.append(
-            (
-                "generate_y",
-                partial(
-                    self._generate_y_values,
-                ),
-            )
-        )
-
-    def _infer_x(self):
-        if not self.plot_properties.get("x", None):
-            self.inferred_properties["x"] = INDEX_FIELD
-
-    def skip_step(self, name: str):
-        self.steps = [(_name, fn) for _name, fn in self.steps if _name != name]
-
-    def _infer_fields(self):
-        fields = self.plot_properties.get("fields", set())
-        if fields:
-            fields = {
-                *fields,
-                self.plot_properties.get("x", None),
-                self.plot_properties.get("y", None),
-                self.inferred_properties.get("x", None),
-            } - {None}
-            self.inferred_properties["fields"] = fields
-
-    def _infer_y(self, datapoints: List[Dict], **kwargs):
-        if "y" not in self.plot_properties:
-            data_fields = list(first(datapoints))
-            skip = (
-                REVISION_FIELD,
-                self.plot_properties.get("x", None)
-                or self.inferred_properties.get("x"),
-                FILENAME_FIELD,
-                VERSION_FIELD,
-            )
-            inferred_y = first(
-                f for f in reversed(data_fields) if f not in skip
-            )
-            if "y" in self.inferred_properties:
-                previous_y = self.inferred_properties["y"]
-                if previous_y != inferred_y:
-                    raise DvcException(
-                        f"Inferred y ('{inferred_y}' value does not match"
-                        f"previously matched one ('f{previous_y}')."
-                    )
+            if isinstance(to_value, str):
+                self.inferred_properties[to_name] = {}
+                if isinstance(from_value, dict):
+                    for file in from_value.keys():
+                        self.inferred_properties[to_name][file] = to_value
+                else:
+                    self.inferred_properties[to_name][self.plot_id] = to_value
             else:
-                self.inferred_properties["y"] = inferred_y
-        return datapoints
+                self.inferred_properties[to_name] = to_value
+
+        _infer_files("y", "x")
+        _infer_files("x", "y")
+
+        if self.inferred_properties.get("y", None) is None:
+            self._infer_y_from_data()
+
+    def _find_datapoints(self):
+        result = {}
+        for file, content in self.data.items():
+            result[file] = get_datapoints(content)
+
+        return result
+
+    @staticmethod
+    def infer_y_label(properties):
+        y_label = properties.get("y_label", None)
+        if y_label is None:
+            y = properties.get("y", None)
+            if isinstance(y, dict):
+                fields = {field for _, field in _file_field(y)}
+                if len(fields) == 1:
+                    y_label = first(fields)
+                else:
+                    y_label = "y"
+            elif isinstance(y, list):
+                y_label = "y"
+            elif isinstance(y, str):
+                y_label = y
+
+        return y_label
+
+    @staticmethod
+    def infer_x_label(properties):
+        x_label = properties.get("x_label", None)
+        if x_label is None:
+            x = properties.get("x", None)
+            if isinstance(x, dict):
+                fields = {field for _, field in _file_field(x)}
+                if len(fields) == 1:
+                    x_label = first(fields)
+                else:
+                    x_label = "x"
+            else:
+                x_label = INDEX_FIELD
+        return x_label
+
+    def flat_datapoints(self, revision):
+
+        file2datapoints, properties = self.convert()
+
+        props_update = {}
+
+        xs = list(_get_xs(properties, file2datapoints))
+
+        # assign "step" if no x provided
+        if not xs:
+            x_file, x_field = (
+                None,
+                INDEX_FIELD,
+            )
+        else:
+            x_file, x_field = xs[0]
+        props_update["x"] = x_field
+
+        ys = list(_get_ys(properties, file2datapoints))
+
+        num_xs = len(xs)
+        num_ys = len(ys)
+        if num_xs > 1 and num_xs != num_ys:
+            raise DvcException(
+                f"Cannot have different number of x and y data sources. Found "
+                f"{num_xs} x and {num_ys} y data sources."
+            )
+
+        all_datapoints = []
+        all_y_fields = {y_field for _, y_field in ys}
+
+        # override to unified y field name if there are different y fields
+        if len(all_y_fields) > 1:
+            props_update["y"] = "dvc_inferred_y_value"
+        else:
+            props_update["y"] = first(all_y_fields)
+
+        for i, (y_file, y_field) in enumerate(ys):
+            if num_xs > 1:
+                x_file, x_field = xs[i]
+            datapoints = deepcopy(file2datapoints.get(y_file, []))
+
+            if props_update.get("y", None) == "dvc_inferred_y_value":
+                _update_from_field(
+                    datapoints,
+                    field="dvc_inferred_y_value",
+                    source_field=y_field,
+                )
+
+            if x_field == INDEX_FIELD and x_file is None:
+                _update_from_index(datapoints, INDEX_FIELD)
+            else:
+                x_datapoints = file2datapoints.get(x_file, [])
+                try:
+                    _update_from_field(
+                        datapoints,
+                        field=x_field,
+                        source_datapoints=x_datapoints,
+                    )
+                except IndexError:
+                    raise DvcException(
+                        f"Cannot join '{x_field}' from '{x_file}' and "
+                        f"'{y_field}' from '{y_file}'. "
+                        "They have to have same length."
+                    )
+
+            _update_all(
+                datapoints,
+                update_dict={
+                    VERSION_FIELD: {
+                        "revision": revision,
+                        FILENAME_FIELD: y_file,
+                        "field": y_field,
+                    }
+                },
+            )
+
+            all_datapoints.extend(datapoints)
+
+        if not all_datapoints:
+            return [], {}
+
+        properties = {**properties, **props_update}
+
+        return all_datapoints, properties
 
     def convert(
         self,
-        data,
-        revision: str,
-        filename: str,
-        skip: List = None,
-        **kwargs,
     ):
         """
         Convert the data. Fill necessary fields ('x', 'y') and return both
-        generated datapoints and updated properties.
+        generated datapoints and updated properties. If `x` is not provided,
+        leave it as None, fronteds should handle it.
+
+        NOTE: Studio uses this method.
+              The only thing studio FE handles is filling `x` and `y`.
+              `x/y_label` should be filled here.
+
+              Datapoints are not stripped according to config, because users
+              might be utilizing other fields in their custom plots.
         """
-        if not skip:
-            skip = []
+        datapoints = self._find_datapoints()
+        properties = {**self.properties, **self.inferred_properties}
 
-        processed = deepcopy(data)
+        properties["y_label"] = self.infer_y_label(properties)
+        properties["x_label"] = self.infer_x_label(properties)
 
-        for step_name, step in self.steps:
-            if step_name not in skip:
-                processed = step(  # type: ignore
-                    processed,
-                    revision=revision,
-                    filename=filename,
-                )
+        return datapoints, properties
 
-        return processed, {**self.plot_properties, **self.inferred_properties}
 
-    def _generate_y_values(  # noqa: C901
-        self,
-        datapoints: List[Dict],
-        revision: str,
-        filename: str,
-        **kwargs,
-    ) -> List[Dict]:
+def _update_from_field(
+    target_datapoints: List[Dict],
+    field: str,
+    source_datapoints: List[Dict] = None,
+    source_field: str = None,
+):
+    if source_datapoints is None:
+        source_datapoints = target_datapoints
+    if source_field is None:
+        source_field = field
 
-        y_values = self.plot_properties.get(
-            "y", None
-        ) or self.inferred_properties.get("y", None)
+    if len(source_datapoints) != len(target_datapoints):
+        raise IndexError(
+            "Source and target datapoints must have the same length"
+        )
 
-        assert y_values is not None
+    for index, datapoint in enumerate(target_datapoints):
+        source_datapoint = source_datapoints[index]
+        if source_field in source_datapoint:
+            datapoint[field] = source_datapoint[source_field]
 
-        result = []
-        properties_update = {}
 
-        def _add_version_info(datapoint, version_info):
-            tmp = datapoint.copy()
-            tmp[VERSION_FIELD] = version_info
-            return tmp
+def _update_from_index(datapoints: List[Dict], new_field: str):
+    for index, datapoint in enumerate(datapoints):
+        datapoint[new_field] = index
 
-        def _version_info(revision, filename=None, field=None):
-            res = {"revision": revision}
-            if filename is not None:
-                res["filename"] = filename
-            if field is not None:
-                res["field"] = field
-            return res
 
-        def _generate_y(datapoint, field):
-            tmp = datapoint.copy()
-            tmp["dvc_inferred_y_value"] = datapoint[field]
-            tmp = _add_version_info(
-                tmp, _version_info(revision, filename, field)
-            )
-            if (
-                "y_label" not in properties_update
-                and "y_label" not in self.plot_properties
-            ):
-                properties_update["y_label"] = "y"
+def _update_all(datapoints: List[Dict], update_dict: Dict):
 
-            properties_update["y"] = "dvc_inferred_y_value"
-
-            return tmp
-
-        if isinstance(y_values, str):
-            for datapoint in datapoints:
-                result.append(
-                    _add_version_info(
-                        datapoint, _version_info(revision, filename, y_values)
-                    )
-                )
-
-        if isinstance(y_values, list):
-            for datapoint in datapoints:
-                for y_val in y_values:
-                    if y_val in datapoint:
-                        result.append(_generate_y(datapoint, y_val))
-
-        if isinstance(y_values, dict):
-
-            def _to_set(values: Iterable):
-                result = set()
-                for val in values:
-                    if isinstance(val, list):
-                        for elem in val:
-                            result.add(elem)
-                    else:
-                        result.add(val)
-
-                return result
-
-            all_fields = _to_set(y_values.values())
-            if (
-                all([isinstance(field, str) for field in all_fields])
-                and len(all_fields) == 1
-            ):
-                # if we use the same field from all files,
-                # we dont have to generate it
-                y_field = all_fields.pop()
-                for datapoint in datapoints:
-                    result.append(
-                        _add_version_info(
-                            datapoint,
-                            _version_info(revision, filename, y_field),
-                        )
-                    )
-                properties_update.update({"y": y_field})
-            else:
-                for def_filename, val in y_values.items():
-                    if isinstance(val, str):
-                        fields = [val]
-                    if isinstance(val, list):
-                        fields = val
-                    for datapoint in datapoints:
-                        for field in fields:
-                            if field in datapoint and def_filename in filename:
-                                result.append(_generate_y(datapoint, field))
-
-        self.inferred_properties = {
-            **self.inferred_properties,
-            **properties_update,
-        }
-
-        return result
+    for datapoint in datapoints:
+        datapoint.update(update_dict)

dvc/render/match.py

@@ -42,7 +42,7 @@ class PlotsData:
         return dict(groups)
 
     def get_definition_data(self, target_files, rev):
-        result = []
+        result = {}
         for file in target_files:
             file_content = (
                 self.data.get(rev, {})
@@ -52,7 +52,7 @@ class PlotsData:
                 .get("data", {})
             )
             if file_content:
-                result.append((file, file_content))
+                result[file] = file_content
 
         return result
 
@@ -91,15 +91,14 @@ def match_defs_renderers(
                 renderer_cls = VegaRenderer
                 renderer_id = plot_id
 
-            converter = _get_converter(renderer_cls, props)
+            converter = _get_converter(
+                renderer_cls, inner_id, props, definitions_data
+            )
 
-            for filename, plot_data in definitions_data:
-                dps, final_props = converter.convert(
-                    revision=rev,
-                    filename=filename,
-                    data=plot_data,
-                )
-                plot_datapoints.extend(dps)
+            dps, rev_props = converter.flat_datapoints(rev)
+            if not final_props and rev_props:
+                final_props = rev_props
+            plot_datapoints.extend(dps)
 
         if "title" not in final_props:
             final_props["title"] = renderer_id

dvc/repo/__init__.py

@@ -401,6 +401,7 @@ class Repo:
         used_run_cache=None,
         revs=None,
         num=1,
+        push: bool = False,
     ):
         """Get the stages related to the given target and collect
         the `info` of its outputs.
@@ -435,6 +436,7 @@ class Repo:
                 jobs=jobs,
                 recursive=recursive,
                 with_deps=with_deps,
+                push=push,
             ).items():
                 used[odb].update(objs)
 

dvc/repo/add.py

@@ -187,7 +187,7 @@ def add(  # noqa: C901
                 stage.transfer(source, to_remote=to_remote, odb=odb, **kwargs)
             else:
                 try:
-                    stage.save()
+                    stage.save(merge_versioned=True)
                     if not no_commit:
                         stage.commit()
                 except CacheLinkError:

dvc/repo/experiments/__init__.py

@@ -32,7 +32,7 @@ from .refs import (
     ExpRefInfo,
 )
 from .stash import ExpStashEntry
-from .utils import exp_refs_by_rev, scm_locked, unlocked_repo
+from .utils import exp_refs_by_rev, unlocked_repo
 
 logger = logging.getLogger(__name__)
 
@@ -50,18 +50,12 @@ class Experiments:
     )
 
     def __init__(self, repo):
-        from dvc.lock import make_lock
         from dvc.scm import NoSCMError
 
         if repo.config["core"].get("no_scm", False):
             raise NoSCMError
 
         self.repo = repo
-        self.scm_lock = make_lock(
-            os.path.join(self.repo.tmp_dir, "exp_scm_lock"),
-            tmp_dir=self.repo.tmp_dir,
-            hardlink_lock=repo.config["core"].get("hardlink_lock", False),
-        )
 
     @property
     def scm(self):
@@ -253,7 +247,6 @@ class Experiments:
         if self.scm.get_ref(str(exp_ref)):
             raise ExperimentExistsError(exp_ref.name)
 
-    @scm_locked
     def new(
         self,
         queue: BaseStashQueue,
@@ -279,7 +272,6 @@ class Experiments:
         except ExperimentExistsError as err:
             if not (kwargs.get("force", False) or kwargs.get("reset", False)):
                 raise err
-
         return queue.put(*args, **kwargs)
 
     def _resume_checkpoint(
@@ -487,6 +479,11 @@ class Experiments:
 
         return run(self.repo, *args, **kwargs)
 
+    def save(self, *args, **kwargs):
+        from dvc.repo.experiments.save import save
+
+        return save(self.repo, *args, **kwargs)
+
     def gc(self, *args, **kwargs):
         from dvc.repo.experiments.gc import gc
 

dvc/repo/experiments/executor/base.py

@@ -12,6 +12,7 @@ from typing import (
     Callable,
     Dict,
     Iterable,
+    List,
     NamedTuple,
     Optional,
     Tuple,
@@ -34,17 +35,7 @@ from ..exceptions import (
     ExperimentExistsError,
     UnchangedExperimentError,
 )
-from ..refs import (
-    EXEC_BASELINE,
-    EXEC_BRANCH,
-    EXEC_CHECKPOINT,
-    EXEC_HEAD,
-    EXEC_MERGE,
-    EXEC_NAMESPACE,
-    EXPS_NAMESPACE,
-    EXPS_STASH,
-    ExpRefInfo,
-)
+from ..refs import EXEC_BASELINE, EXEC_BRANCH, EXEC_CHECKPOINT, ExpRefInfo
 
 if TYPE_CHECKING:
     from queue import Queue
@@ -58,10 +49,6 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-EXEC_TMP_DIR = "exps"
-EXEC_PID_DIR = "run"
-
-
 class ExecutorResult(NamedTuple):
     exp_hash: Optional[str]
     ref_info: Optional["ExpRefInfo"]
@@ -172,6 +159,7 @@ class BaseExecutor(ABC):
     @abstractmethod
     def init_git(
         self,
+        repo: "Repo",
         scm: "Git",
         stash_rev: str,
         entry: "ExpStashEntry",
@@ -319,6 +307,7 @@ class BaseExecutor(ABC):
     def fetch_exps(
         self,
         dest_scm: "Git",
+        refs: List[str],
         force: bool = False,
         on_diverged: Callable[[str, bool], None] = None,
         **kwargs,
@@ -327,26 +316,19 @@ class BaseExecutor(ABC):
 
         Args:
             dest_scm: Destination Git instance.
+            refs: reference names to be fetched from the remotes.
             force: If True, diverged refs will be overwritten
             on_diverged: Callback in the form on_diverged(ref, is_checkpoint)
                 to be called when an experiment ref has diverged.
 
         Extra kwargs will be passed into the remote git client.
         """
-        from ..utils import iter_remote_refs
 
-        refs = []
-        has_checkpoint = False
-        for ref in iter_remote_refs(
-            dest_scm,
-            self.git_url,
-            base=EXPS_NAMESPACE,
-            **kwargs,
-        ):
-            if ref == EXEC_CHECKPOINT:
-                has_checkpoint = True
-            elif not ref.startswith(EXEC_NAMESPACE) and ref != EXPS_STASH:
-                refs.append(ref)
+        if EXEC_CHECKPOINT in refs:
+            refs.remove(EXEC_CHECKPOINT)
+            has_checkpoint = True
+        else:
+            has_checkpoint = False
 
         def on_diverged_ref(orig_ref: str, new_rev: str):
             if force:
@@ -755,21 +737,12 @@ class BaseExecutor(ABC):
             dvc_logger.setLevel(level)
 
     @contextmanager
-    def set_exec_refs(
-        self, scm: "Git", stash_rev: str, entry: "ExpStashEntry"
-    ):
+    def set_temp_refs(self, scm: "Git", temp_dict: Dict[str, str]):
         try:
-            # Executor will be initialized with an empty git repo that
-            # we populate by pushing:
-            #   EXEC_HEAD - the base commit for this experiment
-            #   EXEC_MERGE - the unmerged changes (from our stash)
-            #       to be reproduced
-            #   EXEC_BASELINE - the baseline commit for this experiment
-            scm.set_ref(EXEC_HEAD, entry.head_rev)
-            scm.set_ref(EXEC_MERGE, stash_rev)
-            scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
+            for ref, rev in temp_dict.items():
+                scm.set_ref(ref, rev)
             yield
         finally:
-            for ref in (EXEC_HEAD, EXEC_MERGE, EXEC_BASELINE):
+            for ref in temp_dict:
                 if scm.get_ref(ref):
                     scm.remove_ref(ref)

dvc/repo/experiments/executor/local.py

@@ -2,11 +2,14 @@ import logging
 import os
 from contextlib import ExitStack
 from tempfile import mkdtemp
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, List, Optional
 
-from funcy import cached_property
+from funcy import cached_property, retry
 from scmrepo.exceptions import SCMError as _SCMError
+from shortuuid import uuid
 
+from dvc.exceptions import DvcException
+from dvc.lock import LockError
 from dvc.scm import SCM, GitMergeError
 from dvc.utils.fs import makedirs, remove
 
@@ -18,16 +21,19 @@ from ..refs import (
     EXEC_HEAD,
     EXEC_MERGE,
     EXEC_NAMESPACE,
+    EXPS_TEMP,
+    ExpRefInfo,
 )
-from .base import EXEC_TMP_DIR, BaseExecutor, TaskStatus
+from ..utils import EXEC_TMP_DIR, get_exp_rwlock
+from .base import BaseExecutor, ExecutorResult, TaskStatus
 
 if TYPE_CHECKING:
     from scmrepo.git import Git
 
     from dvc.repo import Repo
 
-    from ..refs import ExpRefInfo
     from ..stash import ExpStashEntry
+    from .base import ExecutorInfo
 
 logger = logging.getLogger(__name__)
 
@@ -67,8 +73,10 @@ class TempDirExecutor(BaseLocalExecutor):
     QUIET = True
     DEFAULT_LOCATION = "tempdir"
 
+    @retry(180, errors=LockError, timeout=1)
     def init_git(
         self,
+        repo: "Repo",
         scm: "Git",
         stash_rev: str,
         entry: "ExpStashEntry",
@@ -85,18 +93,43 @@ class TempDirExecutor(BaseLocalExecutor):
         if infofile:
             self.info.dump_json(infofile)
 
-        with self.set_exec_refs(scm, stash_rev, entry):
-            refspec = f"{EXEC_NAMESPACE}/"
-            push_refspec(scm, self.git_url, refspec, refspec)
+        temp_head = f"{EXPS_TEMP}/head-{uuid()}"
+        temp_merge = f"{EXPS_TEMP}/merge-{uuid()}"
+        temp_baseline = f"{EXPS_TEMP}/baseline-{uuid()}"
 
-        if branch:
-            push_refspec(scm, self.git_url, branch, branch)
-            self.scm.set_ref(EXEC_BRANCH, branch, symbolic=True)
-        elif self.scm.get_ref(EXEC_BRANCH):
-            self.scm.remove_ref(EXEC_BRANCH)
+        temp_ref_dict = {
+            temp_head: entry.head_rev,
+            temp_merge: stash_rev,
+            temp_baseline: entry.baseline_rev,
+        }
+        with get_exp_rwlock(
+            repo, writes=[temp_head, temp_merge, temp_baseline]
+        ), self.set_temp_refs(scm, temp_ref_dict):
+            # Executor will be initialized with an empty git repo that
+            # we populate by pushing:
+            #   EXEC_HEAD - the base commit for this experiment
+            #   EXEC_MERGE - the unmerged changes (from our stash)
+            #       to be reproduced
+            #   EXEC_BASELINE - the baseline commit for this experiment
+            refspec = [
+                (temp_head, EXEC_HEAD),
+                (temp_merge, EXEC_MERGE),
+                (temp_baseline, EXEC_BASELINE),
+            ]
+
+            if branch:
+                refspec.append((branch, branch))
+                with get_exp_rwlock(repo, reads=[branch]):
+                    push_refspec(scm, self.git_url, refspec)
+                self.scm.set_ref(EXEC_BRANCH, branch, symbolic=True)
+            else:
+                push_refspec(scm, self.git_url, refspec)
+                if self.scm.get_ref(EXEC_BRANCH):
+                    self.scm.remove_ref(EXEC_BRANCH)
+
+            if self.scm.get_ref(EXEC_CHECKPOINT):
+                self.scm.remove_ref(EXEC_CHECKPOINT)
 
-        if self.scm.get_ref(EXEC_CHECKPOINT):
-            self.scm.remove_ref(EXEC_CHECKPOINT)
         # checkout EXEC_HEAD and apply EXEC_MERGE on top of it without
         # committing
         head = EXEC_BRANCH if branch else EXEC_HEAD
@@ -167,8 +200,10 @@ class WorkspaceExecutor(BaseLocalExecutor):
         logger.debug("Init workspace executor in '%s'", root_dir)
         return executor
 
+    @retry(180, errors=LockError, timeout=1)
     def init_git(
         self,
+        repo: "Repo",
         scm: "Git",
         stash_rev: str,
         entry: "ExpStashEntry",
@@ -179,25 +214,26 @@ class WorkspaceExecutor(BaseLocalExecutor):
         if infofile:
             self.info.dump_json(infofile)
 
-        scm.set_ref(EXEC_HEAD, entry.head_rev)
-        scm.set_ref(EXEC_MERGE, stash_rev)
-        scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
-        self._detach_stack.enter_context(
-            self.scm.detach_head(
-                self.scm.get_ref(EXEC_HEAD),
-                force=True,
-                client="dvc",
+        with get_exp_rwlock(repo, writes=[EXEC_NAMESPACE]):
+            scm.set_ref(EXEC_HEAD, entry.head_rev)
+            scm.set_ref(EXEC_MERGE, stash_rev)
+            scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
+            self._detach_stack.enter_context(
+                self.scm.detach_head(
+                    self.scm.get_ref(EXEC_HEAD),
+                    force=True,
+                    client="dvc",
+                )
             )
-        )
-        merge_rev = self.scm.get_ref(EXEC_MERGE)
-        try:
-            self.scm.merge(merge_rev, squash=True, commit=False)
-        except _SCMError as exc:
-            raise GitMergeError(str(exc), scm=self.scm)
-        if branch:
-            self.scm.set_ref(EXEC_BRANCH, branch, symbolic=True)
-        elif scm.get_ref(EXEC_BRANCH):
-            self.scm.remove_ref(EXEC_BRANCH)
+            merge_rev = self.scm.get_ref(EXEC_MERGE)
+            try:
+                self.scm.merge(merge_rev, squash=True, commit=False)
+            except _SCMError as exc:
+                raise GitMergeError(str(exc), scm=self.scm)
+            if branch:
+                self.scm.set_ref(EXEC_BRANCH, branch, symbolic=True)
+            elif scm.get_ref(EXEC_BRANCH):
+                self.scm.remove_ref(EXEC_BRANCH)
 
     def init_cache(self, repo: "Repo", rev: str, run_cache: bool = True):
         pass
@@ -213,3 +249,57 @@ class WorkspaceExecutor(BaseLocalExecutor):
             checkpoint = self.scm.get_ref(EXEC_CHECKPOINT)
             if checkpoint and checkpoint != self._orig_checkpoint:
                 self.scm.set_ref(EXEC_APPLY, checkpoint)
+
+    @classmethod
+    def save(
+        cls,
+        info: "ExecutorInfo",
+        force: bool = False,
+        include_untracked: Optional[List[str]] = None,
+    ) -> ExecutorResult:
+        from dvc.repo import Repo
+
+        exp_hash: Optional[str] = None
+        exp_ref: Optional[ExpRefInfo] = None
+
+        dvc = Repo(os.path.join(info.root_dir, info.dvc_dir))
+        old_cwd = os.getcwd()
+        if info.wdir:
+            os.chdir(os.path.join(dvc.scm.root_dir, info.wdir))
+        else:
+            os.chdir(dvc.root_dir)
+
+        try:
+            stages = dvc.commit([], force=force)
+            exp_hash = cls.hash_exp(stages)
+            if include_untracked:
+                dvc.scm.add(include_untracked)
+            cls.commit(
+                dvc.scm,
+                exp_hash,
+                exp_name=info.name,
+                force=force,
+            )
+            ref: Optional[str] = dvc.scm.get_ref(EXEC_BRANCH, follow=False)
+            exp_ref = ExpRefInfo.from_ref(ref) if ref else None
+            untracked = dvc.scm.untracked_files()
+            if untracked:
+                logger.warning(
+                    "The following untracked files were present in "
+                    "the workspace before saving but "
+                    "will not be included in the experiment commit:\n"
+                    "\t%s",
+                    ", ".join(untracked),
+                )
+            info.result_hash = exp_hash
+            info.result_ref = ref
+            info.result_force = False
+            info.status = TaskStatus.SUCCESS
+        except DvcException:
+            info.status = TaskStatus.FAILED
+            raise
+        finally:
+            dvc.close()
+            os.chdir(old_cwd)
+
+        return ExecutorResult(ref, exp_ref, info.result_force)

dvc/repo/experiments/executor/ssh.py

@@ -9,6 +9,7 @@ from dvc_ssh import SSHFileSystem
 from funcy import first
 
 from ..refs import (
+    EXEC_BASELINE,
     EXEC_BRANCH,
     EXEC_CHECKPOINT,
     EXEC_HEAD,
@@ -119,6 +120,7 @@ class SSHExecutor(BaseExecutor):
 
     def init_git(
         self,
+        repo: "Repo",
         scm: "Git",
         stash_rev: str,
         entry: "ExpStashEntry",
@@ -147,12 +149,18 @@ class SSHExecutor(BaseExecutor):
             # (see https://github.com/iterative/dvc/issues/6508)
             kwargs = self._git_client_args(fs)
 
-            with self.set_exec_refs(scm, stash_rev, entry):
-                refspec = f"{EXEC_NAMESPACE}/"
-                push_refspec(scm, self.git_url, refspec, refspec, **kwargs)
+            ref_dict = {
+                EXEC_HEAD: entry.head_rev,
+                EXEC_MERGE: stash_rev,
+                EXEC_BASELINE: entry.baseline_rev,
+            }
+            with self.set_temp_refs(scm, ref_dict):
+                exec_namespace = f"{EXEC_NAMESPACE}/"
+                refspec = [(exec_namespace, exec_namespace)]
+                push_refspec(scm, self.git_url, refspec, **kwargs)
 
             if branch:
-                push_refspec(scm, self.git_url, branch, branch, **kwargs)
+                push_refspec(scm, self.git_url, [(branch, branch)], **kwargs)
                 self._ssh_cmd(fs, f"git symbolic-ref {EXEC_BRANCH} {branch}")
             else:
                 self._ssh_cmd(

dvc/repo/experiments/ls.py

@@ -6,7 +6,7 @@ from dvc.repo.scm_context import scm_context
 from dvc.scm import iter_revs
 from dvc.types import Optional
 
-from .utils import exp_refs, exp_refs_by_baseline
+from .utils import exp_refs_by_baseline
 
 logger = logging.getLogger(__name__)
 
@@ -20,19 +20,22 @@ def ls(
     num: int = 1,
     git_remote: Optional[str] = None,
 ):
-    results = defaultdict(list)
-    if all_commits:
-        gen = exp_refs(repo.scm, git_remote)
-        for info in gen:
-            results[info.baseline_sha].append(info.name)
-        return results
-
-    rev = rev or "HEAD"
-
-    revs = iter_revs(repo.scm, [rev], num)
-    rev_set = set(revs.keys())
+    rev_set = None
+    if not all_commits:
+        revs = iter_revs(repo.scm, [rev or "HEAD"], num)
+        rev_set = set(revs.keys())
     ref_info_dict = exp_refs_by_baseline(repo.scm, rev_set, git_remote)
-    for rev, ref_info_list in ref_info_dict.items():
-        results[rev] = [ref_info.name for ref_info in ref_info_list]
+
+    tags = repo.scm.describe(ref_info_dict.keys())
+    remained = {baseline for baseline, tag in tags.items() if tag is None}
+    base = "refs/heads"
+    ref_heads = repo.scm.describe(remained, base=base)
+
+    results = defaultdict(list)
+    for baseline in ref_info_dict:
+        name = baseline[:7]
+        if tags[baseline] or ref_heads[baseline]:
+            name = tags[baseline] or ref_heads[baseline][len(base) + 1 :]
+        results[name] = [info.name for info in ref_info_dict[baseline]]
 
     return results

dvc/repo/experiments/queue/base.py

@@ -20,22 +20,18 @@ from typing import (
 from funcy import cached_property, retry
 
 from dvc.dependency import ParamsDependency
-from dvc.env import DVCLIVE_RESUME
+from dvc.env import DVC_EXP_BASELINE_REV, DVC_EXP_NAME, DVCLIVE_RESUME
 from dvc.exceptions import DvcException
 from dvc.lock import LockError
 from dvc.ui import ui
 
 from ..exceptions import CheckpointExistsError, ExperimentExistsError
-from ..executor.base import (
-    EXEC_PID_DIR,
-    EXEC_TMP_DIR,
-    BaseExecutor,
-    ExecutorResult,
-)
+from ..executor.base import BaseExecutor, ExecutorResult
 from ..executor.local import WorkspaceExecutor
 from ..refs import ExpRefInfo
 from ..stash import ExpStash, ExpStashEntry
-from ..utils import exp_refs_by_rev, scm_locked
+from ..utils import EXEC_PID_DIR, EXEC_TMP_DIR, exp_refs_by_rev, get_exp_rwlock
+from .utils import get_remote_executor_refs
 
 if TYPE_CHECKING:
     from scmrepo.git import Git
@@ -400,7 +396,13 @@ class BaseStashQueue(ABC):
                             )
 
                     # save additional repro command line arguments
-                    run_env = {DVCLIVE_RESUME: "1"} if resume_rev else {}
+                    run_env = {
+                        DVC_EXP_BASELINE_REV: baseline_rev,
+                    }
+                    if name:
+                        run_env[DVC_EXP_NAME] = name
+                    if resume_rev:
+                        run_env[DVCLIVE_RESUME] = "1"
                     self._pack_args(*args, run_env=run_env, **kwargs)
 
                     # save experiment as a stash commit
@@ -555,28 +557,39 @@ class BaseStashQueue(ABC):
 
     @staticmethod
     @retry(180, errors=LockError, timeout=1)
-    @scm_locked
+    def get_stash_entry(
+        exp: "Experiments",
+        queue_entry: QueueEntry,
+    ) -> "ExpStashEntry":
+        stash = ExpStash(exp.scm, queue_entry.stash_ref)
+        stash_rev = queue_entry.stash_rev
+        with get_exp_rwlock(exp.repo, writes=[queue_entry.stash_ref]):
+            stash_entry = stash.stash_revs.get(
+                stash_rev,
+                ExpStashEntry(None, stash_rev, stash_rev, None, None),
+            )
+            if stash_entry.stash_index is not None:
+                stash.drop(stash_entry.stash_index)
+        return stash_entry
+
+    @classmethod
     def init_executor(
+        cls,
         exp: "Experiments",
         queue_entry: QueueEntry,
         executor_cls: Type[BaseExecutor] = WorkspaceExecutor,
         **kwargs,
     ) -> BaseExecutor:
-        scm = exp.scm
-        stash = ExpStash(scm, queue_entry.stash_ref)
-        stash_rev = queue_entry.stash_rev
-        stash_entry = stash.stash_revs.get(
-            stash_rev,
-            ExpStashEntry(None, stash_rev, stash_rev, None, None),
-        )
-        if stash_entry.stash_index is not None:
-            stash.drop(stash_entry.stash_index)
+        stash_entry = cls.get_stash_entry(exp, queue_entry)
+
         executor = executor_cls.from_stash_entry(
             exp.repo, stash_entry, **kwargs
         )
 
+        stash_rev = queue_entry.stash_rev
         infofile = exp.celery_queue.get_infofile_path(stash_rev)
         executor.init_git(
+            exp.repo,
             exp.repo.scm,
             stash_rev,
             stash_entry,
@@ -597,7 +610,6 @@ class BaseStashQueue(ABC):
 
     @staticmethod
     @retry(180, errors=LockError, timeout=1)
-    @scm_locked
     def collect_git(
         exp: "Experiments",
         executor: BaseExecutor,
@@ -611,16 +623,20 @@ class BaseStashQueue(ABC):
                 raise CheckpointExistsError(ref_info.name)
             raise ExperimentExistsError(ref_info.name)
 
-        for ref in executor.fetch_exps(
-            exp.scm,
-            force=exec_result.force,
-            on_diverged=on_diverged,
-        ):
-            exp_rev = exp.scm.get_ref(ref)
-            if exp_rev:
-                assert exec_result.exp_hash
-                logger.debug("Collected experiment '%s'.", exp_rev[:7])
-                results[exp_rev] = exec_result.exp_hash
+        refs = get_remote_executor_refs(exp.scm, executor.git_url)
+
+        with get_exp_rwlock(exp.repo, writes=refs):
+            for ref in executor.fetch_exps(
+                exp.scm,
+                refs,
+                force=exec_result.force,
+                on_diverged=on_diverged,
+            ):
+                exp_rev = exp.scm.get_ref(ref)
+                if exp_rev:
+                    assert exec_result.exp_hash
+                    logger.debug("Collected experiment '%s'.", exp_rev[:7])
+                    results[exp_rev] = exec_result.exp_hash
 
         return results
 

dvc/repo/experiments/queue/celery.py

@@ -15,7 +15,7 @@ from typing import (
 )
 
 from celery.result import AsyncResult
-from funcy import cached_property
+from funcy import cached_property, first
 from kombu.message import Message
 
 from dvc.daemon import daemonize
@@ -23,8 +23,11 @@ from dvc.exceptions import DvcException
 from dvc.ui import ui
 
 from ..exceptions import UnresolvedQueueExpNamesError
-from ..executor.base import EXEC_TMP_DIR, ExecutorInfo, ExecutorResult
+from ..executor.base import ExecutorInfo, ExecutorResult
+from ..refs import CELERY_STASH
+from ..utils import EXEC_TMP_DIR, get_exp_rwlock
 from .base import BaseStashQueue, QueueDoneResult, QueueEntry, QueueGetResult
+from .exceptions import CannotKillTasksError
 from .tasks import run_exp
 from .utils import fetch_running_exp_from_temp_dir
 
@@ -163,7 +166,8 @@ class LocalCeleryQueue(BaseStashQueue):
 
     def put(self, *args, **kwargs) -> QueueEntry:
         """Stash an experiment and add it to the queue."""
-        entry = self._stash_exp(*args, **kwargs)
+        with get_exp_rwlock(self.repo, writes=["workspace", CELERY_STASH]):
+            entry = self._stash_exp(*args, **kwargs)
         self.celery.signature(run_exp.s(entry.asdict())).delay()
         return entry
 
@@ -286,24 +290,70 @@ class LocalCeleryQueue(BaseStashQueue):
         # out of the active task list, and needs to be loaded here.
         return self._get_done_result(entry, timeout)
 
+    def _get_running_task_ids(self) -> Set[str]:
+        running_task_ids: Set[str] = set()
+        active_workers = self.worker_status()
+        for _, tasks in active_workers.items():
+            task = first(tasks)
+            if task:
+                running_task_ids.add(task["id"])
+        return running_task_ids
+
+    def _try_to_kill_tasks(
+        self, to_kill: Dict[QueueEntry, str]
+    ) -> Dict[QueueEntry, str]:
+        fail_to_kill_entries: Dict[QueueEntry, str] = {}
+        for queue_entry, rev in to_kill.items():
+            try:
+                self.proc.kill(queue_entry.stash_rev)
+                logger.debug(f"Task {rev} had been killed.")
+            except ProcessLookupError:
+                fail_to_kill_entries[queue_entry] = rev
+        return fail_to_kill_entries
+
+    def _mark_inactive_tasks_failure(self, remained_entries):
+        remained_revs: List[str] = []
+        running_ids = self._get_running_task_ids()
+        logger.debug(f"Current running tasks ids: {running_ids}.")
+        for msg, entry in self._iter_processed():
+            if entry not in remained_entries:
+                continue
+            task_id = msg.headers["id"]
+            if task_id in running_ids:
+                remained_revs.append(remained_entries[entry])
+            else:
+                result: AsyncResult = AsyncResult(task_id)
+                if not result.ready():
+                    logger.debug(
+                        f"Task id {task_id} rev {remained_entries[entry]} "
+                        "marked as failure."
+                    )
+                    self.celery.backend.mark_as_failure(task_id, None)
+
+        if remained_revs:
+            raise CannotKillTasksError(remained_revs)
+
     def kill(self, revs: Collection[str]) -> None:
-        to_kill: Set[QueueEntry] = set()
         name_dict: Dict[
             str, Optional[QueueEntry]
         ] = self.match_queue_entry_by_name(set(revs), self.iter_active())
 
-        missing_rev: List[str] = []
+        to_kill: Dict[QueueEntry, str] = {}
+        missing_revs: List[str] = []
         for rev, queue_entry in name_dict.items():
             if queue_entry is None:
-                missing_rev.append(rev)
+                missing_revs.append(rev)
             else:
-                to_kill.add(queue_entry)
+                to_kill[queue_entry] = rev
+        if missing_revs:
+            raise UnresolvedQueueExpNamesError(missing_revs)
 
-        if missing_rev:
-            raise UnresolvedQueueExpNamesError(missing_rev)
+        fail_to_kill_entries: Dict[QueueEntry, str] = self._try_to_kill_tasks(
+            to_kill
+        )
 
-        for queue_entry in to_kill:
-            self.proc.kill(queue_entry.stash_rev)
+        if fail_to_kill_entries:
+            self._mark_inactive_tasks_failure(fail_to_kill_entries)
 
     def shutdown(self, kill: bool = False):
         self.celery.control.shutdown()
@@ -359,10 +409,11 @@ class LocalCeleryQueue(BaseStashQueue):
         ) as fobj:
             ui.write(fobj.read())
 
-    def worker_status(self) -> Dict:
+    def worker_status(self) -> Dict[str, List[Dict]]:
         """Return the current active celery worker"""
         status = self.celery.control.inspect().active() or {}
         logger.debug(f"Worker status: {status}")
+
         return status
 
     def clear(self, *args, **kwargs):

dvc/repo/experiments/queue/exceptions.py

@@ -0,0 +1,12 @@
+from typing import Collection
+
+from dvc.exceptions import DvcException
+
+
+class CannotKillTasksError(DvcException):
+    def __init__(self, revs: Collection[str]):
+        rev_str = ",".join(revs)
+        super().__init__(
+            f"Task '{rev_str}' is initializing, please wait a few seconds "
+            "until the experiments start running to retry the kill operation."
+        )

dvc/repo/experiments/queue/tasks.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict
+from typing import TYPE_CHECKING, Any, Dict
 
 from celery import shared_task
 from celery.utils.log import get_task_logger
@@ -7,11 +7,15 @@ from ..executor.base import ExecutorInfo
 from ..executor.local import TempDirExecutor
 from .base import BaseStashQueue, QueueEntry
 
+if TYPE_CHECKING:
+    from ..executor.base import BaseExecutor
+
+
 logger = get_task_logger(__name__)
 
 
 @shared_task
-def setup_exp(entry_dict: Dict[str, Any]) -> TempDirExecutor:
+def setup_exp(entry_dict: Dict[str, Any]) -> "BaseExecutor":
     """Setup an experiment.
 
     Arguments:

dvc/repo/experiments/queue/tempdir.py

@@ -6,14 +6,13 @@ from funcy import cached_property, first
 
 from ..exceptions import ExpQueueEmptyError
 from ..executor.base import (
-    EXEC_PID_DIR,
-    EXEC_TMP_DIR,
     BaseExecutor,
     ExecutorInfo,
     ExecutorResult,
     TaskStatus,
 )
 from ..executor.local import TempDirExecutor
+from ..utils import EXEC_PID_DIR, EXEC_TMP_DIR
 from .base import BaseStashQueue, QueueEntry, QueueGetResult
 from .utils import fetch_running_exp_from_temp_dir
 from .workspace import WorkspaceQueue
@@ -77,9 +76,8 @@ class TempDirQueue(WorkspaceQueue):
         for stash_rev in self.proc:
             infofile = self.get_infofile_path(stash_rev)
             executor_info = ExecutorInfo.load_json(infofile)
-            if (
-                not executor_info.status <= TaskStatus.SUCCESS
-                and os.path.exists(executor_info.root_dir)
+            if executor_info.status <= TaskStatus.SUCCESS and os.path.exists(
+                executor_info.root_dir
             ):
                 yield QueueEntry(
                     self.repo.root_dir,

dvc/repo/experiments/queue/utils.py

@@ -1,17 +1,40 @@
 import logging
-from typing import TYPE_CHECKING, Dict
+from typing import TYPE_CHECKING, Dict, List
 
 from scmrepo.exceptions import SCMError
 
 from ..executor.base import ExecutorInfo, TaskStatus
+from ..refs import EXEC_CHECKPOINT, EXEC_NAMESPACE, EXPS_NAMESPACE, EXPS_STASH
+from ..utils import get_exp_rwlock, iter_remote_refs
 
 logger = logging.getLogger(__name__)
 
 
 if TYPE_CHECKING:
+    from scmrepo.git import Git
+
     from .base import BaseStashQueue
 
 
+def get_remote_executor_refs(scm: "Git", remote_url: str) -> List[str]:
+    """Get result list refs from a remote repository
+
+    Args:
+        remote_url : remote executor's url
+    """
+    refs = []
+    for ref in iter_remote_refs(
+        scm,
+        remote_url,
+        base=EXPS_NAMESPACE,
+    ):
+        if ref == EXEC_CHECKPOINT or (
+            not ref.startswith(EXEC_NAMESPACE) and ref != EXPS_STASH
+        ):
+            refs.append(ref)
+    return refs
+
+
 def fetch_running_exp_from_temp_dir(
     queue: "BaseStashQueue", rev: str, fetch_refs: bool
 ) -> Dict[str, Dict]:
@@ -36,24 +59,27 @@ def fetch_running_exp_from_temp_dir(
         info = ExecutorInfo.from_dict(load_json(infofile))
     except OSError:
         return result
-    if info.status < TaskStatus.FAILED:
+    if info.status <= TaskStatus.RUNNING:
         result[rev] = info.asdict()
         if info.git_url and fetch_refs and info.status > TaskStatus.PREPARING:
 
             def on_diverged(_ref: str, _checkpoint: bool):
-                return False
+                return True
 
             executor = TempDirExecutor.from_info(info)
             try:
-                for ref in executor.fetch_exps(
-                    queue.scm,
-                    on_diverged=on_diverged,
-                ):
-                    logger.debug("Updated running experiment '%s'.", ref)
-                    last_rev = queue.scm.get_ref(ref)
-                    result[rev]["last"] = last_rev
-                    if last_rev:
-                        result[last_rev] = info.asdict()
+                refs = get_remote_executor_refs(queue.scm, executor.git_url)
+                with get_exp_rwlock(queue.repo, writes=refs):
+                    for ref in executor.fetch_exps(
+                        queue.scm,
+                        refs,
+                        on_diverged=on_diverged,
+                    ):
+                        logger.debug("Updated running experiment '%s'.", ref)
+                        last_rev = queue.scm.get_ref(ref)
+                        result[rev]["last"] = last_rev
+                        if last_rev:
+                            result[last_rev] = info.asdict()
             except (InvalidRemoteSCMRepo, SCMError):
                 # ignore stale info files
                 del result[rev]

dvc/repo/experiments/queue/workspace.py

@@ -21,7 +21,8 @@ from ..executor.base import (
     TaskStatus,
 )
 from ..executor.local import WorkspaceExecutor
-from ..refs import EXEC_BRANCH
+from ..refs import EXEC_BRANCH, WORKSPACE_STASH
+from ..utils import get_exp_rwlock
 from .base import BaseStashQueue, QueueDoneResult, QueueEntry, QueueGetResult
 
 if TYPE_CHECKING:
@@ -34,7 +35,8 @@ class WorkspaceQueue(BaseStashQueue):
     _EXEC_NAME: Optional[str] = "workspace"
 
     def put(self, *args, **kwargs) -> QueueEntry:
-        return self._stash_exp(*args, **kwargs)
+        with get_exp_rwlock(self.repo, writes=["workspace", WORKSPACE_STASH]):
+            return self._stash_exp(*args, **kwargs)
 
     def get(self) -> QueueGetResult:
         revs = self.stash.stash_revs

dvc/repo/experiments/refs.py

@@ -16,6 +16,7 @@ EXEC_BRANCH = f"{EXEC_NAMESPACE}/EXEC_BRANCH"
 EXEC_BASELINE = f"{EXEC_NAMESPACE}/EXEC_BASELINE"
 EXEC_HEAD = f"{EXEC_NAMESPACE}/EXEC_HEAD"
 EXEC_MERGE = f"{EXEC_NAMESPACE}/EXEC_MERGE"
+EXPS_TEMP = f"{EXPS_NAMESPACE}/temp"
 STASHES = {WORKSPACE_STASH, CELERY_STASH}
 
 
@@ -23,9 +24,7 @@ class ExpRefInfo:
 
     namespace = EXPS_NAMESPACE
 
-    def __init__(
-        self, baseline_sha: Optional[str] = None, name: Optional[str] = None
-    ):
+    def __init__(self, baseline_sha: str, name: Optional[str] = None):
         self.baseline_sha = baseline_sha
         self.name: str = name if name else ""
 
@@ -33,7 +32,7 @@ class ExpRefInfo:
         return "/".join(self.parts)
 
     def __repr__(self):
-        baseline = f"'{self.baseline_sha}'" if self.baseline_sha else "None"
+        baseline = f"'{self.baseline_sha}'"
         name = f"'{self.name}'" if self.name else "None"
         return f"ExpRefInfo(baseline_sha={baseline}, name={name})"
 
@@ -41,11 +40,7 @@ class ExpRefInfo:
     def parts(self):
         return (
             (self.namespace,)
-            + (
-                (self.baseline_sha[:2], self.baseline_sha[2:])
-                if self.baseline_sha
-                else ()
-            )
+            + ((self.baseline_sha[:2], self.baseline_sha[2:]))
             + ((self.name,) if self.name else ())
         )
 
@@ -54,14 +49,13 @@ class ExpRefInfo:
         try:
             parts = ref.split("/")
             if (
-                len(parts) < 2
-                or len(parts) == 3
+                len(parts) < 4
                 or len(parts) > 5
                 or "/".join(parts[:2]) != EXPS_NAMESPACE
             ):
                 raise InvalidExpRefError(ref)
         except ValueError:
             raise InvalidExpRefError(ref)
-        baseline_sha = parts[2] + parts[3] if len(parts) >= 4 else None
+        baseline_sha = parts[2] + parts[3]
         name = parts[4] if len(parts) == 5 else None
         return cls(baseline_sha, name)

dvc/repo/experiments/remove.py

@@ -152,8 +152,7 @@ def _remove_commited_exps(
                 push_refspec(
                     scm,
                     remote,
-                    None,
-                    str(ref_info),
+                    [(None, str(ref_info))],
                     progress=pbar.update_git,
                 )
     else:

dvc/repo/experiments/save.py

@@ -0,0 +1,44 @@
+import logging
+import os
+from typing import TYPE_CHECKING, List, Optional
+
+from funcy import first
+
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+
+
+logger = logging.getLogger(__name__)
+
+
+def save(
+    repo: "Repo",
+    name: Optional[str] = None,
+    force: bool = False,
+    include_untracked: Optional[List[str]] = None,
+) -> Optional[str]:
+    """Save the current workspace status as an experiment.
+
+    Returns the saved experiment's SHAs.
+    """
+    queue = repo.experiments.workspace_queue
+    logger.debug("Saving workspace in %s", os.getcwd())
+
+    staged, _, _ = repo.scm.status(untracked_files="no")
+    if staged:
+        logger.warning(
+            "Your workspace contains staged Git changes which will be "
+            "unstaged before saving this experiment."
+        )
+        repo.scm.reset()
+
+    entry = repo.experiments.new(queue=queue, name=name, force=force)
+    executor = queue.init_executor(repo.experiments, entry)
+
+    save_result = executor.save(
+        executor.info, force=force, include_untracked=include_untracked
+    )
+    result = queue.collect_executor(repo.experiments, executor, save_result)
+
+    exp_rev = first(result)
+    return exp_rev

dvc/repo/experiments/show.py

@@ -8,8 +8,10 @@ from typing import (
     Any,
     Callable,
     Dict,
+    Iterable,
     List,
     Optional,
+    Set,
     Tuple,
     Union,
 )
@@ -24,6 +26,7 @@ from dvc.utils import error_handler, onerror_collect, relpath
 from .refs import ExpRefInfo
 
 if TYPE_CHECKING:
+    from scmrepo.git import Git
     from scmrepo.git.objects import GitCommit
 
     from dvc.repo import Repo
@@ -134,6 +137,7 @@ def _collect_complete_experiment(
 
     checkpoint: bool = True if len(revs) > 1 else False
     prev = ""
+
     for rev in revs:
         status = ExpStatus.Running if rev in running else ExpStatus.Success
         collected_exp = collect_experiment_commit(
@@ -215,41 +219,55 @@ def _collect_branch(
     return results
 
 
-def get_names(repo: "Repo", result: Dict[str, Dict[str, Any]]):
+def get_branch_names(
+    scm: "Git", baseline_set: Iterable[str]
+) -> Dict[str, Optional[str]]:
+    names: Dict[str, Optional[str]] = {}
+    for base in [
+        f"refs/exps/{baseline[:2]}/{baseline[2:]}/"
+        for baseline in baseline_set
+    ] + ["refs/heads/", "refs/tags/"]:
+        for ref in scm.iter_refs(base=base):
+            if ref:
+                try:
+                    rev = scm.get_ref(ref)
+                    names[rev] = ref[len(base) :]
+                except KeyError:
+                    logger.debug("unreosolved ref %s", ref)
+    logger.debug("found refs for revs: %s", names)
+    return names
+
+
+def update_names(
+    repo: "Repo",
+    branch_names: Dict[str, Optional[str]],
+    result: Dict[str, Dict[str, Any]],
+):
 
     rev_set = set()
-    baseline_set = set()
     for baseline in result:
         for rev in result[baseline]:
             if rev == "baseline":
                 rev = baseline
-                baseline_set.add(rev)
             if rev != "workspace":
                 rev_set.add(rev)
 
-    names: Dict[str, Optional[str]] = {}
-    for base in ("refs/tags/", "refs/heads/"):
-        if rev_set:
-            names.update(
-                (rev, ref[len(base) :])
-                for rev, ref in repo.scm.describe(
-                    baseline_set, base=base
-                ).items()
-                if ref is not None
-            )
-            rev_set.difference_update(names.keys())
+    if rev_set:
+        rev_set.difference_update(branch_names.keys())
 
     exact_name = repo.experiments.get_exact_name(rev_set)
 
     for baseline, baseline_results in result.items():
+        name_set: Set[str] = set()
         for rev, rev_result in baseline_results.items():
             name: Optional[str] = None
             if rev == "baseline":
                 rev = baseline
                 if rev == "workspace":
                     continue
-            name = names.get(rev, None) or exact_name[rev]
-            if name:
+            name = branch_names.get(rev, None) or exact_name[rev]
+            if name and name not in name_set:
+                name_set.add(name)
                 rev_result["data"]["name"] = name
 
 
@@ -336,6 +354,40 @@ def update_new(
             to_dict[baseline][rev] = to_dict[baseline].get(rev, experiment)
 
 
+def move_properties_to_head(result: Dict[str, Dict[str, Dict[str, Any]]]):
+    for _, baseline_results in result.items():
+        checkpoint: bool = False
+        head: Dict[str, Any] = {}
+        for rev, rev_data in baseline_results.items():
+            if (
+                "data" not in rev_data
+                or rev_data["data"].get("checkpoint_tip", None) is None
+            ):
+                checkpoint = False
+                head = {}
+                continue
+
+            rev_result: Dict[str, Any] = rev_data["data"]
+            if (
+                checkpoint is True
+                and rev_result["checkpoint_tip"] == head["checkpoint_tip"]
+            ):
+                if "name" in rev_result and "name" not in head:
+                    head["name"] = rev_result["name"]
+                    del rev_result["name"]
+                if rev_result["executor"]:
+                    if not head["executor"]:
+                        head["executor"] = rev_result["executor"]
+                    rev_result["executor"] = None
+                if rev_result["status"] == ExpStatus.Running.name:
+                    head["status"] = ExpStatus.Running.name
+                    rev_result["status"] = ExpStatus.Success.name
+            else:
+                if rev_result["checkpoint_tip"] == rev:
+                    head = rev_result
+                    checkpoint = True
+
+
 def show(
     repo: "Repo",
     all_branches=False,
@@ -354,8 +406,7 @@ def show(
     if repo.scm.no_commits:
         return {}
 
-    if onerror is None:
-        onerror = _show_onerror_collect
+    onerror = onerror or _show_onerror_collect
 
     res: Dict[str, Dict] = defaultdict(OrderedDict)
 
@@ -368,6 +419,7 @@ def show(
     found_revs.update(
         iter_revs(repo.scm, revs, num, all_branches, all_tags, all_commits)
     )
+    branch_names = get_branch_names(repo.scm, found_revs)
 
     running: Dict[str, Dict] = repo.experiments.get_running_exps(
         fetch_refs=fetch_running
@@ -421,6 +473,8 @@ def show(
     update_new(res, queued_experiment)
 
     if not sha_only:
-        get_names(repo, res)
+        update_names(repo, branch_names, res)
+
+    move_properties_to_head(res)
 
     return res

dvc/repo/experiments/utils.py

@@ -1,6 +1,9 @@
+import os
+import sys
 from collections import defaultdict
 from functools import wraps
 from typing import (
+    TYPE_CHECKING,
     Callable,
     Dict,
     Generator,
@@ -9,6 +12,7 @@ from typing import (
     Mapping,
     Optional,
     Set,
+    Tuple,
     Union,
 )
 
@@ -16,6 +20,7 @@ from scmrepo.git import Git
 
 from dvc.exceptions import InvalidArgumentError
 from dvc.repo.experiments.exceptions import AmbiguousExpRefInfo
+from dvc.rwlock import rwlock
 
 from .refs import (
     EXEC_APPLY,
@@ -28,18 +33,34 @@ from .refs import (
     ExpRefInfo,
 )
 
+if TYPE_CHECKING:
+    from dvc.repo import Repo
 
-def scm_locked(f):
-    # Lock the experiments workspace so that we don't try to perform two
-    # different sequences of git operations at once
-    @wraps(f)
-    def wrapper(exp, *args, **kwargs):
-        from dvc.scm import map_scm_exception
 
-        with map_scm_exception(), exp.scm_lock:
-            return f(exp, *args, **kwargs)
+EXEC_TMP_DIR = "exps"
+EXEC_PID_DIR = "run"
 
-    return wrapper
+
+def get_exp_rwlock(
+    repo: "Repo",
+    reads: Optional[List[str]] = None,
+    writes: Optional[List[str]] = None,
+):
+    reads = reads or []
+    writes = writes or []
+
+    cmd = " ".join(sys.argv)
+    path = os.path.join(repo.tmp_dir, EXEC_TMP_DIR)
+    repo.fs.makedirs(path, exist_ok=True)
+
+    return rwlock(
+        path,
+        repo.fs,
+        cmd,
+        reads,
+        writes,
+        repo.config["core"].get("hardlink_lock", False),
+    )
 
 
 def unlocked_repo(f):
@@ -79,13 +100,13 @@ def exp_refs_by_rev(scm: "Git", rev: str) -> Generator[ExpRefInfo, None, None]:
 
 
 def exp_refs_by_baseline(
-    scm: "Git", revs: Set[str], url: Optional[str] = None
+    scm: "Git", revs: Optional[Set[str]] = None, url: Optional[str] = None
 ) -> Mapping[str, List[ExpRefInfo]]:
     """Iterate over all experiment refs with the specified baseline."""
     all_exp_refs = exp_refs(scm, url)
     result = defaultdict(list)
     for ref in all_exp_refs:
-        if ref.baseline_sha in revs:
+        if revs is None or ref.baseline_sha in revs:
             result[ref.baseline_sha].append(ref)
     return result
 
@@ -108,8 +129,7 @@ def iter_remote_refs(
 def push_refspec(
     scm: "Git",
     url: str,
-    src: Optional[str],
-    dest: str,
+    push_list=List[Tuple[Optional[str], str]],
     force: bool = False,
     on_diverged: Optional[Callable[[str, str], bool]] = None,
     **kwargs,
@@ -119,20 +139,21 @@ def push_refspec(
 
     from ...scm import GitAuthError, SCMError
 
-    if not src:
-        refspecs = [f":{dest}"]
-    elif src.endswith("/"):
-        refspecs = []
-        dest = dest.rstrip("/") + "/"
-        for ref in scm.iter_refs(base=src):
-            refname = ref.split("/")[-1]
-            refspecs.append(f"{ref}:{dest}{refname}")
-    else:
-        if dest.endswith("/"):
-            refname = src.split("/")[-1]
-            refspecs = [f"{src}:{dest}/{refname}"]
+    refspecs = []
+    for src, dest in push_list:
+        if not src:
+            refspecs.append(f":{dest}")
+        elif src.endswith("/"):
+            dest = dest.rstrip("/") + "/"
+            for ref in scm.iter_refs(base=src):
+                refname = ref.split("/")[-1]
+                refspecs.append(f"{ref}:{dest}{refname}")
         else:
-            refspecs = [f"{src}:{dest}"]
+            if dest.endswith("/"):
+                refname = src.split("/")[-1]
+                refspecs.append(f"{src}:{dest}/{refname}")
+            else:
+                refspecs.append(f"{src}:{dest}")
 
     try:
         results = scm.push_refspecs(

dvc/repo/fetch.py

@@ -17,20 +17,6 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-def _fetch_worktree(repo, remote):
-    from dvc_data.index import save
-
-    index = repo.index.data["repo"]
-    for key, entry in index.iteritems():
-        entry.fs = remote.fs
-        entry.path = remote.fs.path.join(
-            remote.path,
-            *key,
-        )
-    save(index)
-    return len(index)
-
-
 @locked
 def fetch(
     self,
@@ -59,6 +45,7 @@ def fetch(
             remote is configured
     """
     from dvc.repo.imports import save_imports
+    from dvc.repo.worktree import fetch_worktree
     from dvc_data.hashfile.transfer import TransferResult
 
     if isinstance(targets, str):
@@ -67,7 +54,7 @@ def fetch(
     with suppress(NoRemoteError):
         _remote = self.cloud.get_remote(name=remote)
         if _remote.worktree:
-            return _fetch_worktree(self, _remote)
+            return fetch_worktree(self, _remote)
 
     failed_count = 0
 

dvc/repo/imp_url.py

@@ -97,6 +97,9 @@ def imp_url(
     else:
         stage.run(jobs=jobs, no_download=no_download)
 
+    if not no_exec and stage.deps[0].fs.version_aware:
+        stage.outs[0].can_push = False
+
     stage.frozen = frozen
 
     dvcfile.dump(stage)

dvc/repo/index.py

@@ -11,6 +11,7 @@ from typing import (
     Optional,
     Set,
     Tuple,
+    Union,
 )
 
 from funcy import cached_property, nullcontext
@@ -67,14 +68,50 @@ class Index:
         if stages is not None:
             self.stages: List["Stage"] = stages
         self._collected_targets: Dict[int, List["StageInfo"]] = {}
+        self._metrics: Dict[str, List[str]] = {}
+        self._plots: Dict[str, Any] = {}
+        self._params: Dict[str, List[str]] = {}
 
     @cached_property
     def stages(self) -> List["Stage"]:  # pylint: disable=method-hidden
         # note that ideally we should be keeping this in a set as it is unique,
         # hashable and has no concept of orderliness on its own. But we depend
         # on this to be somewhat ordered for status/metrics/plots, etc.
+        return self._collect()
+
+    @cached_property
+    def _top_metrics(self):
+        self._collect()
+        return self._metrics
+
+    @cached_property
+    def _top_plots(self):
+        self._collect()
+        return self._plots
+
+    @cached_property
+    def _top_params(self):
+        self._collect()
+        return self._params
+
+    def _collect(self):
+        if "stages" in self.__dict__:
+            return self.stages
+
         onerror = self.repo.stage_collection_error_handler
-        return self.stage_collector.collect_repo(onerror=onerror)
+
+        # pylint: disable=protected-access
+        (
+            stages,
+            metrics,
+            plots,
+            params,
+        ) = self.stage_collector._collect_all_from_repo(onerror=onerror)
+        self.stages = stages
+        self._metrics = metrics
+        self._plots = plots
+        self._params = params
+        return stages
 
     def __repr__(self) -> str:
         from dvc.fs import LocalFileSystem
@@ -152,7 +189,7 @@ class Index:
 
     def targets_view(
         self,
-        targets: "TargetType",
+        targets: Optional["TargetType"],
         stage_filter: Optional[Callable[["Stage"], bool]] = None,
         outs_filter: Optional[Callable[["Output"], bool]] = None,
         **kwargs: Any,
@@ -268,6 +305,7 @@ class Index:
         force: bool = False,
         recursive: bool = False,
         jobs: int = None,
+        push: bool = False,
     ) -> "ObjectContainer":
         from collections import defaultdict
 
@@ -281,6 +319,7 @@ class Index:
                 force=force,
                 jobs=jobs,
                 filter_info=filter_info,
+                push=push,
             ).items():
                 used[odb].update(objs)
         return used
@@ -427,10 +466,10 @@ class IndexView:
         return prefixes
 
     @cached_property
-    def data(self) -> "Dict[str, DataIndexView]":
+    def data(self) -> Dict[str, Union["DataIndex", "DataIndexView"]]:
         from functools import partial
 
-        from dvc_data.index import view
+        from dvc_data.index import DataIndex, view
 
         def key_filter(workspace: str, key: "DataIndexKey"):
             try:
@@ -442,10 +481,15 @@ class IndexView:
             except KeyError:
                 return False
 
-        data = {}
+        data: Dict[str, Union["DataIndex", "DataIndexView"]] = {}
         for workspace, data_index in self._index.data.items():
-            data_index.load()
-            data[workspace] = view(data_index, partial(key_filter, workspace))
+            if self._stages:
+                data_index.load()
+                data[workspace] = view(
+                    data_index, partial(key_filter, workspace)
+                )
+            else:
+                data[workspace] = DataIndex()
         return data
 
 

dvc/repo/metrics/show.py

@@ -9,7 +9,12 @@ from dvc.output import Output
 from dvc.repo import locked
 from dvc.repo.collect import StrPaths, collect
 from dvc.scm import NoSCMError
-from dvc.utils import error_handler, errored_revisions, onerror_collect
+from dvc.utils import (
+    as_posix,
+    error_handler,
+    errored_revisions,
+    onerror_collect,
+)
 from dvc.utils.collections import ensure_list
 from dvc.utils.serialize import load_path
 
@@ -28,6 +33,17 @@ def _to_fs_paths(metrics: List[Output]) -> StrPaths:
     return result
 
 
+def _collect_top_level_metrics(repo):
+    top_metrics = repo.index._top_metrics  # pylint: disable=protected-access
+    for dvcfile, metrics in top_metrics.items():
+        wdir = repo.fs.path.relpath(
+            repo.fs.path.parent(dvcfile), repo.root_dir
+        )
+        for file in metrics:
+            path = repo.fs.path.join(wdir, as_posix(file))
+            yield repo.fs.path.normpath(path)
+
+
 def _collect_metrics(repo, targets, revision, recursive):
     metrics, fs_paths = collect(
         repo,
@@ -94,6 +110,7 @@ def _read_metrics(repo, metrics, rev, onerror=None):
 
 def _gather_metrics(repo, targets, rev, recursive, onerror=None):
     metrics = _collect_metrics(repo, targets, rev, recursive)
+    metrics.extend(_collect_top_level_metrics(repo))
     return _read_metrics(repo, metrics, rev, onerror=onerror)
 
 

dvc/repo/params/show.py

@@ -20,7 +20,12 @@ from dvc.repo.collect import collect
 from dvc.scm import NoSCMError
 from dvc.stage import PipelineStage
 from dvc.ui import ui
-from dvc.utils import error_handler, errored_revisions, onerror_collect
+from dvc.utils import (
+    as_posix,
+    error_handler,
+    errored_revisions,
+    onerror_collect,
+)
 from dvc.utils.collections import ensure_list
 from dvc.utils.serialize import load_path
 
@@ -35,6 +40,17 @@ def _is_params(dep: "Output"):
     return isinstance(dep, ParamsDependency)
 
 
+def _collect_top_level_params(repo):
+    top_params = repo.index._top_params  # pylint: disable=protected-access
+    for dvcfile, params in top_params.items():
+        wdir = repo.fs.path.relpath(
+            repo.fs.path.parent(dvcfile), repo.root_dir
+        )
+        for file in params:
+            path = repo.fs.path.join(wdir, as_posix(file))
+            yield repo.fs.path.normpath(path)
+
+
 def _collect_configs(
     repo: "Repo", rev, targets=None, deps=False, stages=None
 ) -> Tuple[List["Output"], List[str]]:
@@ -176,6 +192,7 @@ def _gather_params(
     param_outs, params_fs_paths = _collect_configs(
         repo, rev, targets=targets, deps=deps, stages=stages
     )
+    params_fs_paths.extend(_collect_top_level_params(repo=repo))
     params = _read_params(
         repo,
         params=param_outs,

dvc/repo/plots/__init__.py

@@ -19,7 +19,7 @@ from typing import (
 
 import dpath.options
 import dpath.util
-from funcy import cached_property, first, project
+from funcy import cached_property, distinct, first, project
 
 from dvc.exceptions import DvcException
 from dvc.utils import error_handler, errored_revisions, onerror_collect
@@ -314,16 +314,18 @@ def _get_data_targets(definitions: Dict):
 
 
 def infer_data_sources(plot_id, config=None):
-    def _deduplicate(lst: List):
-        return list({elem: None for elem in lst}.keys())
-
     y = config.get("y", None)
+
     if isinstance(y, dict):
         sources = list(y.keys())
     else:
         sources = [plot_id]
 
-    return _deduplicate(source for source in sources)
+    x = config.get("x", None)
+    if isinstance(x, dict):
+        sources.append(first(x.keys()))
+
+    return distinct(source for source in sources)
 
 
 def _matches(targets, config_file, plot_id):
@@ -437,36 +439,33 @@ def _resolve_definitions(
 
 
 def _collect_pipeline_files(repo, targets: List[str], props, onerror=None):
-    from dvc.dvcfile import PipelineFile
-
     result: Dict[str, Dict] = {}
-    dvcfiles = {stage.dvcfile for stage in repo.index.stages}
-    for dvcfile in dvcfiles:
-        if isinstance(dvcfile, PipelineFile):
-            dvcfile_path = _relpath(repo.dvcfs, dvcfile.path)
-            dvcfile_defs = dvcfile.load().get("plots", {})
-            dvcfile_defs_dict: Dict[str, Union[Dict, None]] = {}
-            if isinstance(dvcfile_defs, list):
-                for elem in dvcfile_defs:
-                    if isinstance(elem, str):
-                        dvcfile_defs_dict[elem] = None
-                    else:
-                        k, v = list(elem.items())[0]
-                        dvcfile_defs_dict[k] = v
-            else:
-                dvcfile_defs_dict = dvcfile_defs
-            resolved = _resolve_definitions(
-                repo.dvcfs,
-                targets,
-                props,
-                dvcfile_path,
-                dvcfile_defs_dict,
-                onerror=onerror,
-            )
-            dpath.util.merge(
-                result,
-                {dvcfile_path: resolved},
-            )
+    top_plots = repo.index._top_plots  # pylint: disable=protected-access
+    for dvcfile, plots_def in top_plots.items():
+        dvcfile_path = _relpath(repo.dvcfs, dvcfile)
+        dvcfile_defs_dict: Dict[str, Union[Dict, None]] = {}
+        if isinstance(plots_def, list):
+            for elem in plots_def:
+                if isinstance(elem, str):
+                    dvcfile_defs_dict[elem] = None
+                else:
+                    k, v = list(elem.items())[0]
+                    dvcfile_defs_dict[k] = v
+        else:
+            dvcfile_defs_dict = plots_def
+
+        resolved = _resolve_definitions(
+            repo.dvcfs,
+            targets,
+            props,
+            dvcfile_path,
+            dvcfile_defs_dict,
+            onerror=onerror,
+        )
+        dpath.util.merge(
+            result,
+            {dvcfile_path: resolved},
+        )
     return result
 
 

dvc/repo/push.py

@@ -9,48 +9,6 @@ if TYPE_CHECKING:
     from dvc_objects.db.base import ObjectDB
 
 
-def _push_worktree(repo, remote):
-    from dvc_data.index import checkout
-    from dvc_data.index.save import build_tree
-
-    index = repo.index.data["repo"]
-    checkout(index, remote.path, remote.fs)
-
-    for stage in repo.index.stages:
-        for out in stage.outs:
-            if not out.use_cache:
-                continue
-
-            if not out.is_in_repo:
-                continue
-
-            workspace, key = out.index_key
-            index = repo.index.data[workspace]
-            entry = index[key]
-            if out.isdir():
-                old_tree = out.get_obj()
-                entry.hash_info = old_tree.hash_info
-                entry.meta = out.meta
-                for subkey, entry in index.iteritems(key):
-                    if entry.meta.isdir:
-                        continue
-                    fs_path = repo.fs.path.join(repo.root_dir, *subkey)
-                    _, hash_info = old_tree.get(
-                        repo.fs.path.relparts(fs_path, out.fs_path)
-                    )
-                    entry.hash_info = hash_info
-                tree_meta, new_tree = build_tree(index, key)
-                out.obj = new_tree
-                out.hash_info = new_tree.hash_info
-                out.meta = tree_meta
-            else:
-                out.hash_info = entry.hash_info
-                out.meta = entry.meta
-        stage.dvcfile.dump(stage, with_files=True, update_pipeline=False)
-
-    return len(index)
-
-
 @locked
 def push(
     self,
@@ -68,9 +26,11 @@ def push(
     odb: Optional["ObjectDB"] = None,
     include_imports=False,
 ):
+    from dvc.repo.worktree import push_worktree
+
     _remote = self.cloud.get_remote(name=remote)
-    if _remote.worktree:
-        return _push_worktree(self, _remote)
+    if _remote.worktree or _remote.fs.version_aware:
+        return push_worktree(self, _remote)
 
     used_run_cache = (
         self.stage_cache.push(remote, odb=odb) if run_cache else []
@@ -93,6 +53,7 @@ def push(
         recursive=recursive,
         used_run_cache=used_run_cache,
         revs=revs,
+        push=True,
     )
 
     pushed = len(used_run_cache)

dvc/repo/stage.py

@@ -23,6 +23,7 @@ from dvc.exceptions import (
     OutputNotFoundError,
 )
 from dvc.repo import lock_repo
+from dvc.ui import ui
 from dvc.utils import as_posix, parse_target, relpath
 
 logger = logging.getLogger(__name__)
@@ -139,7 +140,14 @@ class StageLoad:
         )
         with self.repo.scm_context:
             stage.dump(update_lock=update_lock)
-            stage.ignore_outs()
+            try:
+                stage.ignore_outs()
+            except FileNotFoundError as exc:
+                ui.warn(
+                    f"Could not create .gitignore entry in {exc.filename}."
+                    " DVC will attempt to create .gitignore entry again when"
+                    " the stage is run."
+                )
 
         return stage
 
@@ -502,3 +510,104 @@ class StageLoad:
 
     def collect_repo(self, onerror: Callable[[str, Exception], None] = None):
         return list(self._collect_repo(onerror))
+
+    def _load_file(self, path):
+        from dvc.dvcfile import Dvcfile
+        from dvc.stage.loader import SingleStageLoader, StageLoader
+
+        path = self._get_filepath(path)
+        dvcfile = Dvcfile(self.repo, path)
+        # `dvcfile.stages` is not cached
+        stages = dvcfile.stages  # type: ignore
+
+        if isinstance(stages, SingleStageLoader):
+            stages_ = [stages[None]]
+        else:
+            assert isinstance(stages, StageLoader)
+            keys = self._get_keys(stages)
+            stages_ = [stages[key] for key in keys]
+
+        return (
+            stages_,
+            stages.metrics_data,
+            stages.plots_data,
+            stages.params_data,
+        )
+
+    def _collect_all_from_repo(
+        self, onerror: Callable[[str, Exception], None] = None
+    ):
+        """Collects all of the stages present in the DVC repo.
+
+        Args:
+            onerror (optional): callable that will be called with two args:
+                the filepath whose collection failed and the exc instance.
+                It can report the error to continue with the collection
+                (and, skip failed ones), or raise the exception to abort
+                the collection.
+        """
+        from dvc.dvcfile import is_valid_filename
+        from dvc.fs import LocalFileSystem
+
+        scm = self.repo.scm
+        sep = self.fs.sep
+        outs: Set[str] = set()
+
+        is_local_fs = isinstance(self.fs, LocalFileSystem)
+
+        def is_ignored(path):
+            # apply only for the local fs
+            return is_local_fs and scm.is_ignored(path)
+
+        def is_dvcfile_and_not_ignored(root, file):
+            return is_valid_filename(file) and not is_ignored(
+                f"{root}{sep}{file}"
+            )
+
+        def is_out_or_ignored(root, directory):
+            dir_path = f"{root}{sep}{directory}"
+            # trailing slash needed to check if a directory is gitignored
+            return dir_path in outs or is_ignored(f"{dir_path}{sep}")
+
+        walk_iter = self.repo.dvcignore.walk(self.fs, self.repo.root_dir)
+        if logger.isEnabledFor(logging.TRACE):  # type: ignore[attr-defined]
+            walk_iter = log_walk(walk_iter)
+
+        stages = []
+        metrics = {}
+        plots = {}
+        params = {}
+
+        for root, dirs, files in walk_iter:
+            dvcfile_filter = partial(is_dvcfile_and_not_ignored, root)
+            for file in filter(dvcfile_filter, files):
+                file_path = self.fs.path.join(root, file)
+                try:
+                    (
+                        new_stages,
+                        new_metrics,
+                        new_plots,
+                        new_params,
+                    ) = self._load_file(file_path)
+                except DvcException as exc:
+                    if onerror:
+                        onerror(relpath(file_path), exc)
+                        continue
+                    raise
+
+                stages.extend(new_stages)
+                if new_metrics:
+                    metrics[file_path] = new_metrics
+                if new_plots:
+                    plots[file_path] = new_plots
+                if new_params:
+                    params[file_path] = new_params
+
+                outs.update(
+                    out.fspath
+                    for stage in new_stages
+                    for out in stage.outs
+                    if out.protocol == "local"
+                )
+            dirs[:] = [d for d in dirs if not is_out_or_ignored(root, d)]
+        return stages, metrics, plots, params

dvc/repo/status.py

@@ -89,6 +89,7 @@ def _cloud_status(
         remote=remote,
         jobs=jobs,
         recursive=recursive,
+        push=True,
     )
 
     ret = {}

dvc/repo/worktree.py

@@ -0,0 +1,161 @@
+import logging
+from typing import TYPE_CHECKING, Any, Dict, Optional
+
+from dvc.fs.callbacks import Callback
+
+if TYPE_CHECKING:
+    from dvc.cloud import Remote
+    from dvc.index import Index, IndexView
+    from dvc.output import Output
+    from dvc.repo import Repo
+    from dvc.stage import Stage
+    from dvc.types import TargetType
+    from dvc_data.hashfile.meta import Meta
+
+logger = logging.getLogger(__name__)
+
+
+def worktree_view(
+    index: "Index",
+    targets: Optional["TargetType"] = None,
+    push: bool = False,
+    latest_only: bool = True,
+    **kwargs,
+) -> "IndexView":
+    """Return view of data that can be stored in worktree remotes.
+
+    Args:
+        targets: Optional targets.
+        push: Whether the view should be restricted to pushable data only.
+
+    Additional kwargs will be passed into target collection.
+    """
+
+    def stage_filter(stage: "Stage") -> bool:
+        if push and stage.is_repo_import:
+            return False
+        return True
+
+    def outs_filter(out: "Output") -> bool:
+        if (
+            not out.is_in_repo
+            or not out.use_cache
+            or (push and not out.can_push)
+        ):
+            return False
+        # If we are not enforcing push to latest version and have a version
+        # for this out, we assume it still exists and can skip pushing it
+        if push and not latest_only and out.meta.version_id is not None:
+            return False
+        return True
+
+    return index.targets_view(
+        targets,
+        stage_filter=stage_filter,
+        outs_filter=outs_filter,
+        **kwargs,
+    )
+
+
+def fetch_worktree(repo: "Repo", remote: "Remote") -> int:
+    from dvc_data.index import save
+
+    view = worktree_view(repo.index)
+    index = view.data["repo"]
+    for key, entry in index.iteritems():
+        entry.fs = remote.fs
+        entry.path = remote.fs.path.join(
+            remote.path,
+            *key,
+        )
+    total = len(index)
+    with Callback.as_tqdm_callback(
+        unit="file", desc="Fetch", disable=total == 0
+    ) as cb:
+        cb.set_size(total)
+        return save(index, callback=cb)
+
+
+def push_worktree(repo: "Repo", remote: "Remote") -> int:
+    from dvc_data.index import build, checkout
+
+    view = worktree_view(repo.index, push=True, latest_only=remote.worktree)
+    new_index = view.data["repo"]
+    if remote.worktree:
+        logger.debug("Indexing latest worktree for '%s'", remote.path)
+        old_index = build(remote.path, remote.fs)
+        logger.debug("Pushing worktree changes to '%s'", remote.path)
+    else:
+        old_index = None
+        logger.debug("Pushing version-aware files to '%s'", remote.path)
+
+    if remote.worktree:
+
+        # for files, if our version's checksum (etag) matches the latest remote
+        # checksum, we do not need to push, even if the version IDs don't match
+        def _checksum(meta: "Meta") -> Any:
+            if not meta or meta.isdir:
+                return meta
+            return getattr(meta, remote.fs.PARAM_CHECKSUM)
+
+        diff_kwargs: Dict[str, Any] = {
+            "meta_only": True,
+            "meta_cmp_key": _checksum,
+        }
+    else:
+        diff_kwargs = {}
+
+    total = len(new_index)
+    with Callback.as_tqdm_callback(
+        unit="file", desc="Push", disable=total == 0
+    ) as cb:
+        cb.set_size(total)
+        pushed = checkout(
+            new_index,
+            remote.path,
+            remote.fs,
+            old=old_index,
+            delete=remote.worktree,
+            callback=cb,
+            latest_only=remote.worktree,
+            **diff_kwargs,
+        )
+    if pushed:
+        _update_pushed_meta(repo, view)
+    return pushed
+
+
+def _update_pushed_meta(repo: "Repo", view: "IndexView"):
+    from dvc_data.index.save import build_tree
+
+    for stage in view.stages:
+        for out in stage.outs:
+            workspace, key = out.index_key
+            index = repo.index.data[workspace]
+            entry = index[key]
+            if out.isdir():
+                old_tree = out.get_obj()
+                entry.hash_info = old_tree.hash_info
+                entry.meta = out.meta
+                for subkey, entry in index.iteritems(key):
+                    if entry.meta.isdir:
+                        continue
+                    fs_path = repo.fs.path.join(repo.root_dir, *subkey)
+                    meta, hash_info = old_tree.get(
+                        repo.fs.path.relparts(fs_path, out.fs_path)
+                    )
+                    entry.hash_info = hash_info
+                    if meta.version_id is not None:
+                        # preserve existing version IDs for unchanged files in
+                        # this dir (entry will have the latest remote version
+                        # ID after checkout)
+                        entry.meta = meta
+                tree_meta, new_tree = build_tree(index, key)
+                out.obj = new_tree
+                out.hash_info = new_tree.hash_info
+                out.meta = tree_meta
+            else:
+                out.hash_info = entry.hash_info
+                if out.meta.version_id is None:
+                    out.meta = entry.meta
+        stage.dvcfile.dump(stage, with_files=True, update_pipeline=False)

dvc/rwlock.py

@@ -4,6 +4,7 @@ import os
 from collections import defaultdict
 from contextlib import contextmanager
 
+import psutil
 from voluptuous import Invalid, Optional, Required, Schema
 
 from .exceptions import DvcException
@@ -72,35 +73,67 @@ def _infos_to_str(infos):
     )
 
 
-def _check_blockers(lock, info, *, mode, waiters):
+def _check_blockers(tmp_dir, lock, info, *, mode, waiters):
     from .lock import LockError
 
-    for waiter_path in waiters:
-        blockers = [
-            blocker
-            for path, infos in lock[mode].items()
-            if localfs.path.overlaps(waiter_path, path)
-            for blocker in (infos if isinstance(infos, list) else [infos])
-            if blocker != info
-        ]
+    non_existing_pid = set()
 
-        if not blockers:
+    blockers = []
+    to_release = defaultdict(list)
+    for path, infos in lock[mode].items():
+        for waiter_path in waiters:
+            if localfs.path.overlaps(waiter_path, path):
+                break
+        else:
             continue
 
+        infos = infos if isinstance(infos, list) else [infos]
+        for blocker in infos:
+            if blocker == info:
+                continue
+
+            pid = int(blocker["pid"])
+
+            if pid in non_existing_pid:
+                pass
+            elif psutil.pid_exists(pid):
+                blockers.append(blocker)
+                continue
+            else:
+                non_existing_pid.add(pid)
+                cmd = blocker["cmd"]
+                logger.warning(
+                    "Process '%s' with (Pid %s), in RWLock-file '%s'"
+                    " had been killed. Auto remove it from the lock file.",
+                    cmd,
+                    pid,
+                    relpath(path),
+                )
+            to_release[json.dumps(blocker, sort_keys=True)].append(path)
+
+    if to_release:
+        for info_json, path_list in to_release.items():
+            info = json.loads(info_json)
+            if mode == "read":
+                _release_read(lock, info, path_list)
+            elif mode == "write":
+                _release_write(lock, info, path_list)
+
+    if blockers:
         raise LockError(
-            "'{path}' is busy, it is being blocked by:\n"
-            "{blockers}\n"
+            f"'{waiter_path}' is busy, it is being blocked by:\n"
+            f"{_infos_to_str(blockers)}\n"
             "\n"
             "If there are no processes with such PIDs, you can manually "
-            "remove '.dvc/tmp/rwlock' and try again.".format(
-                path=waiter_path, blockers=_infos_to_str(blockers)
-            )
+            f"remove '{tmp_dir}/rwlock' and try again."
         )
 
 
 def _acquire_read(lock, info, paths):
     changes = []
 
+    lock["read"] = lock.get("read", defaultdict(list))
+
     for path in paths:
         readers = lock["read"][path]
         if info in readers:
@@ -115,6 +148,8 @@ def _acquire_read(lock, info, paths):
 def _acquire_write(lock, info, paths):
     changes = []
 
+    lock["write"] = lock.get("write", defaultdict(dict))
+
     for path in paths:
         if lock["write"][path] == info:
             continue
@@ -171,8 +206,10 @@ def rwlock(tmp_dir, fs, cmd, read, write, hardlink):
 
     with _edit_rwlock(tmp_dir, fs, hardlink) as lock:
 
-        _check_blockers(lock, info, mode="write", waiters=read + write)
-        _check_blockers(lock, info, mode="read", waiters=write)
+        _check_blockers(
+            tmp_dir, lock, info, mode="write", waiters=read + write
+        )
+        _check_blockers(tmp_dir, lock, info, mode="read", waiters=write)
 
         rchanges = _acquire_read(lock, info, read)
         wchanges = _acquire_write(lock, info, write)

dvc/schema.py

@@ -51,6 +51,7 @@ OUT_PSTAGE_DETAILED_SCHEMA = {
         Output.PARAM_PERSIST: bool,
         Output.PARAM_CHECKPOINT: bool,
         Output.PARAM_REMOTE: str,
+        Output.PARAM_PUSH: bool,
     }
 }
 
@@ -103,7 +104,7 @@ def either_or(primary, fallback, fallback_includes=None):
 
 
 PLOT_DEFINITION = {
-    Output.PARAM_PLOT_X: str,
+    Output.PARAM_PLOT_X: Any(str, {str: str}),
     Output.PARAM_PLOT_Y: Any(str, [str], {str: Any(str, [str])}),
     Output.PARAM_PLOT_X_LABEL: str,
     Output.PARAM_PLOT_Y_LABEL: str,
@@ -122,6 +123,8 @@ MULTI_STAGE_SCHEMA = {
     PLOTS: Any(SINGLE_PLOT_SCHEMA, [Any(str, SINGLE_PLOT_SCHEMA)]),
     STAGES: SINGLE_PIPELINE_STAGE_SCHEMA,
     VARS_KWD: VARS_SCHEMA,
+    StageParams.PARAM_PARAMS: [str],
+    StageParams.PARAM_METRICS: [str],
 }
 
 COMPILED_SINGLE_STAGE_SCHEMA = Schema(SINGLE_STAGE_SCHEMA)

dvc/stage/__init__.py

@@ -113,10 +113,11 @@ def restore_fields(stage):
     stage.meta = old.meta
     stage.desc = old.desc
 
-    old_fields = {out.def_path: (out.annot, out.remote) for out in old.outs}
+    old_outs = {out.def_path: out for out in old.outs}
     for out in stage.outs:
-        if out_fields := old_fields.get(out.def_path, None):
-            out.annot, out.remote = out_fields
+        old_out = old_outs.get(out.def_path, None)
+        if old_out is not None:
+            out.restore_fields(old_out)
 
 
 class Stage(params.StageParams):
@@ -462,10 +463,12 @@ class Stage(params.StageParams):
         logger.debug("Computed %s md5: '%s'", self, m)
         return m
 
-    def save(self, allow_missing=False):
+    def save(self, allow_missing: bool = False, merge_versioned: bool = False):
         self.save_deps(allow_missing=allow_missing)
 
-        self.save_outs(allow_missing=allow_missing)
+        self.save_outs(
+            allow_missing=allow_missing, merge_versioned=merge_versioned
+        )
         self.md5 = self.compute_md5()
 
         self.repo.stage_cache.save(self)
@@ -480,15 +483,40 @@ class Stage(params.StageParams):
                 if not allow_missing:
                     raise
 
-    def save_outs(self, allow_missing=False):
+    def save_outs(
+        self, allow_missing: bool = False, merge_versioned: bool = False
+    ):
         from dvc.output import OutputDoesNotExistError
 
+        from .exceptions import StageFileDoesNotExistError, StageNotFound
+
+        if merge_versioned:
+            try:
+                old = self.reload()
+                old_outs = {out.def_path: out for out in old.outs}
+                merge_versioned = any(
+                    (
+                        out.files is not None
+                        or (
+                            out.meta is not None
+                            and out.meta.version_id is not None
+                        )
+                    )
+                    for out in old_outs.values()
+                )
+            except (StageFileDoesNotExistError, StageNotFound):
+                merge_versioned = False
+
         for out in self.outs:
             try:
                 out.save()
             except OutputDoesNotExistError:
                 if not (allow_missing or out.checkpoint):
                     raise
+            if merge_versioned:
+                old_out = old_outs.get(out.def_path)
+                if old_out is not None:
+                    out.merge_version_meta(old_out)
 
     def ignore_outs(self):
         for out in self.outs:

dvc/stage/cache.py

@@ -180,7 +180,7 @@ class StageCache:
         dump_yaml(tmp, cache)
         self.repo.odb.local.move(tmp, path)
 
-    def restore(self, stage, run_cache=True, pull=False):
+    def restore(self, stage, run_cache=True, pull=False, dry=False):
         from .serialize import to_single_stage_lockfile
 
         if not _can_hash(stage):
@@ -195,14 +195,15 @@ class StageCache:
         else:
             if not run_cache:  # backward compatibility
                 raise RunCacheNotFoundError(stage)
-            stage.save_deps()
+            if not dry:
+                stage.save_deps()
             cache = self._load(stage)
             if not cache:
                 raise RunCacheNotFoundError(stage)
 
         cached_stage = self._create_stage(cache, wdir=stage.wdir)
 
-        if pull:
+        if pull and not dry:
             for objs in cached_stage.get_used_objs().values():
                 self.repo.cloud.pull(objs)
 
@@ -213,7 +214,8 @@ class StageCache:
             "Stage '%s' is cached - skipping run, checking out outputs",
             stage.addressing,
         )
-        cached_stage.checkout()
+        if not dry:
+            cached_stage.checkout()
 
     def transfer(self, from_odb, to_odb):
         from dvc.fs import HTTPFileSystem, LocalFileSystem

dvc/stage/loader.py

@@ -24,6 +24,9 @@ class StageLoader(Mapping):
         self.dvcfile = dvcfile
         self.data = data or {}
         self.stages_data = self.data.get("stages", {})
+        self.metrics_data = self.data.get("metrics", [])
+        self.params_data = self.data.get("params", [])
+        self.plots_data = self.data.get("plots", {})
         self.repo = self.dvcfile.repo
 
         lockfile_data = lockfile_data or {}
@@ -171,6 +174,9 @@ class SingleStageLoader(Mapping):
         self.dvcfile = dvcfile
         self.stage_data = stage_data or {}
         self.stage_text = stage_text
+        self.metrics_data = []
+        self.params_data = []
+        self.plots_data = {}
 
     def __getitem__(self, item):
         if item:

dvc/stage/run.py

@@ -139,14 +139,16 @@ def cmd_run(stage, dry=False, checkpoint_func=None, run_env=None):
 def run_stage(
     stage, dry=False, force=False, checkpoint_func=None, run_env=None, **kwargs
 ):
-    if not (dry or force or checkpoint_func):
+    if not (force or checkpoint_func):
         from .cache import RunCacheNotFoundError
 
         try:
-            stage.repo.stage_cache.restore(stage, **kwargs)
-            return
+            stage.repo.stage_cache.restore(stage, dry=dry, **kwargs)
+            if not dry:
+                return
         except RunCacheNotFoundError:
-            stage.save_deps()
+            if not dry:
+                stage.save_deps()
 
     run = cmd_run if dry else unlocked_repo(cmd_run)
     run(stage, dry=dry, checkpoint_func=checkpoint_func, run_env=run_env)

dvc/stage/serialize.py

@@ -29,6 +29,7 @@ PARAM_PERSIST = Output.PARAM_PERSIST
 PARAM_CHECKPOINT = Output.PARAM_CHECKPOINT
 PARAM_DESC = Annotation.PARAM_DESC
 PARAM_REMOTE = Output.PARAM_REMOTE
+PARAM_PUSH = Output.PARAM_PUSH
 
 DEFAULT_PARAMS_FILE = ParamsDependency.DEFAULT_PARAMS_FILE
 
@@ -54,6 +55,8 @@ def _get_flags(out):
         yield from out.plot.items()
     if out.remote:
         yield PARAM_REMOTE, out.remote
+    if not out.can_push:
+        yield PARAM_PUSH, False
 
 
 def _serialize_out(out):

dvc/testing/api_tests.py

@@ -13,14 +13,22 @@ class TestAPI:
         assert api.get_url("foo") == expected_url
 
     def test_open(self, tmp_dir, dvc, remote):
-        tmp_dir.dvc_gen("foo", "foo-text")
+        tmp_dir.dvc_gen(
+            {
+                "foo": "foo-text",
+                "dir": {"bar": "bar-text"},
+            }
+        )
         dvc.push()
 
         # Remove cache to force download
         remove(dvc.odb.local.path)
 
-        with api.open("foo") as fd:
-            assert fd.read() == "foo-text"
+        with api.open("foo") as fobj:
+            assert fobj.read() == "foo-text"
+
+        with api.open("dir/bar") as fobj:
+            assert fobj.read() == "bar-text"
 
     @pytest.mark.parametrize(
         "clear_cache", [True, False], ids=["cache", "no_cache"]

dvc/ui/__init__.py

@@ -270,7 +270,9 @@ class Console:
 
         path = Path(file).resolve()
         url = (
-            relpath(path) if "Microsoft" in uname().release else path.as_uri()
+            relpath(path)
+            if "microsoft" in uname().release.lower()
+            else path.as_uri()
         )
 
         opened = webbrowser.open(url)

dvc/utils/hydra.py

@@ -54,8 +54,12 @@ def compose_and_dump(
     with initialize_config_dir(config_dir, version_base=None):
         cfg = compose(config_name=config_name, overrides=overrides)
 
-    dumper = DUMPERS[Path(output_file).suffix.lower()]
-    dumper(output_file, OmegaConf.to_object(cfg))
+    suffix = Path(output_file).suffix.lower()
+    if suffix not in [".yml", ".yaml"]:
+        dumper = DUMPERS[suffix]
+        dumper(output_file, OmegaConf.to_object(cfg))
+    else:
+        Path(output_file).write_text(OmegaConf.to_yaml(cfg), encoding="utf-8")
 
 
 def apply_overrides(path: "StrPath", overrides: List[str]) -> None:

dvc/utils/plots.py

@@ -1,2 +1,6 @@
-def get_plot_id(plot_id: str, config_file_path: str = ""):
-    return f"{config_file_path}::{plot_id}" if config_file_path else plot_id
+def get_plot_id(config_plot_id: str, config_file_path: str = ""):
+    return (
+        f"{config_file_path}::{config_plot_id}"
+        if config_file_path
+        else config_plot_id
+    )

pyproject.toml

@@ -50,11 +50,11 @@ dependencies = [
     "rich>=10.13.0",
     "pyparsing>=2.4.7",
     "typing-extensions>=3.7.4",
-    "scmrepo==0.1.3",
-    "dvc-render==0.0.12",
-    "dvc-task==0.1.5",
+    "scmrepo==0.1.4",
+    "dvc-render==0.0.14",
+    "dvc-task==0.1.6",
     "dvclive>=1.0",
-    "dvc-data==0.25.3",
+    "dvc-data==0.28.3",
     "dvc-http==2.27.2",
     "hydra-core>=1.1.0",
     "iterative-telemetry==0.0.6",
@@ -70,10 +70,10 @@ classifiers = [
 ]
 
 [project.optional-dependencies]
-azure = ["dvc-azure==2.20.4"]
+azure = ["dvc-azure==2.20.5"]
 gdrive = ["dvc-gdrive==2.19.1"]
 gs = ["dvc-gs==2.20.0"]
-hdfs = ["dvc-hdfs==2.19.0; python_version < '3.11'"]
+hdfs = ["dvc-hdfs==2.19.0"]
 oss = ["dvc-oss==2.19.0"]
 s3 = ["dvc-s3==2.21.0"]
 ssh = ["dvc-ssh==2.20.0"]
@@ -98,12 +98,10 @@ tests = [
     "flaky==3.7.0",
     "pytest-timeout==2.1.0",
     "filelock==3.8.0",
-    # optional dependencies
-    "dvclive>=1.0",
-    "pywin32>=225; sys_platform == 'win32'",
     "beautifulsoup4==4.11.1",
+    "pywin32>=225; sys_platform == 'win32'",  # optional dependency
     # pylint requirements
-    "pylint==2.15.5",
+    "pylint==2.15.7",
     # we use this to suppress pytest-related false positives in our tests.
     "pylint-pytest==1.1.2",
     # we use this to suppress some messages in tests, eg: foo/bar naming,
@@ -111,9 +109,9 @@ tests = [
     "pylint-plugin-utils==0.7",
     # type-checking
     "mypy==0.990",
-    "types-requests>=2.27.15",
-    "types-tabulate>=0.8.6",
-    "types-toml>=0.10.4",
+    "types-requests",
+    "types-tabulate",
+    "types-toml",
 ]
 all = ["dvc[azure,gdrive,gs,hdfs,oss,s3,ssh,webdav,webhdfs]"]
 dev = ["dvc[azure,gdrive,gs,hdfs,oss,s3,ssh,webdav,webhdfs,tests]"]

scripts/fpm/build.py

@@ -44,6 +44,10 @@ if args.pkg == "osxpkg":
         ]
     )
 else:
+    if args.pkg == "rpm":
+        # https://github.com/jordansissel/fpm/issues/1503
+        flags.extend(["--rpm-rpmbuild-define", "_build_id_links none"])
+
     bash_dir = build / "etc" / "bash_completion.d"
     dirs = ["usr", "etc"]
     flags.extend(["--depends", "git >= 1.7.0"])  # needed for gitpython

scripts/pyinstaller/hooks/hook-asyncssh.py

@@ -0,0 +1 @@
+hiddenimports = ["win32timezone"]

tests/conftest.py

@@ -1,3 +1,4 @@
+import gc
 import json
 import os
 import sys
@@ -259,3 +260,19 @@ def mock_hydra_conf(mocker):
     # NOTE: using sentinel here so that any imports from `hydra.conf`
     # return a mock.
     sys.modules["hydra.conf"] = mocker.sentinel
+
+
+@pytest.fixture(autouse=True)
+def gc_collect_on_dvc_close_on_win_311(mocker):
+    if sys.version_info < (3, 11) and os.name != "nt":
+        return
+
+    from dvc.repo import Repo
+
+    close = Repo.close
+
+    def wrapped(repo):
+        close(repo)
+        gc.collect()
+
+    mocker.patch("dvc.repo.Repo.close", wrapped)

tests/func/experiments/executor/test_ssh.py

@@ -38,7 +38,7 @@ def test_init_from_stash(tmp_dir, scm, dvc, machine_instance, mocker):
 
 @pytest.mark.needs_internet
 @pytest.mark.parametrize("cloud", [pytest.lazy_fixture("git_ssh")])
-def test_init_git(tmp_dir, scm, cloud, mocker):
+def test_init_git(tmp_dir, dvc, scm, cloud, mocker):
     tmp_dir.scm_gen({"foo": "foo", "dir": {"bar": "bar"}}, commit="init")
     baseline_rev = scm.get_rev()
     tmp_dir.gen("foo", "stashed")
@@ -59,7 +59,7 @@ def test_init_git(tmp_dir, scm, cloud, mocker):
         fs_factory=partial(_ssh_factory, cloud),
     )
     infofile = str((root_url / "foo.run").path)
-    executor.init_git(scm, rev, mock, infofile=infofile)
+    executor.init_git(dvc, scm, rev, mock, infofile=infofile)
     assert root_url.path == executor._repo_abspath
 
     fs = cloud._ssh

tests/func/experiments/test_experiments.py

@@ -2,12 +2,14 @@ import itertools
 import logging
 import os
 import stat
+from textwrap import dedent
 
 import pytest
 from funcy import first
 
 from dvc.dvcfile import PIPELINE_FILE
 from dvc.exceptions import ReproductionError
+from dvc.repo.experiments.exceptions import ExperimentExistsError
 from dvc.repo.experiments.queue.base import BaseStashQueue
 from dvc.repo.experiments.utils import exp_refs_by_rev
 from dvc.scm import resolve_rev
@@ -43,8 +45,6 @@ def test_new_simple(tmp_dir, scm, dvc, exp_stage, mocker, name, workspace):
 
 
 def test_experiment_exists(tmp_dir, scm, dvc, exp_stage, mocker, workspace):
-    from dvc.repo.experiments.exceptions import ExperimentExistsError
-
     dvc.experiments.run(
         exp_stage.addressing,
         name="foo",
@@ -428,22 +428,28 @@ def test_list(tmp_dir, scm, dvc, exp_stage):
     ref_info_b = first(exp_refs_by_rev(scm, exp_b))
 
     tmp_dir.scm_gen("new", "new", commit="new")
-    baseline_c = scm.get_rev()
     results = dvc.experiments.run(exp_stage.addressing, params=["foo=4"])
     exp_c = first(results)
     ref_info_c = first(exp_refs_by_rev(scm, exp_c))
 
-    assert dvc.experiments.ls() == {baseline_c: [ref_info_c.name]}
+    assert dvc.experiments.ls() == {"master": [ref_info_c.name]}
 
     exp_list = dvc.experiments.ls(rev=ref_info_a.baseline_sha)
     assert {key: set(val) for key, val in exp_list.items()} == {
-        baseline_a: {ref_info_a.name, ref_info_b.name}
+        baseline_a[:7]: {ref_info_a.name, ref_info_b.name}
     }
 
     exp_list = dvc.experiments.ls(all_commits=True)
     assert {key: set(val) for key, val in exp_list.items()} == {
-        baseline_a: {ref_info_a.name, ref_info_b.name},
-        baseline_c: {ref_info_c.name},
+        baseline_a[:7]: {ref_info_a.name, ref_info_b.name},
+        "master": {ref_info_c.name},
+    }
+
+    scm.checkout("branch", True)
+    exp_list = dvc.experiments.ls(all_commits=True)
+    assert {key: set(val) for key, val in exp_list.items()} == {
+        baseline_a[:7]: {ref_info_a.name, ref_info_b.name},
+        "branch": {ref_info_c.name},
     }
 
 
@@ -676,3 +682,29 @@ def test_experiments_workspace_not_log_exception(caplog, dvc, scm):
             dvc.experiments.run()
 
     assert not caplog.text
+
+
+def test_run_env(tmp_dir, dvc, scm):
+    dump_run_env = dedent(
+        """\
+        import os
+        from dvc.env import DVC_EXP_BASELINE_REV, DVC_EXP_NAME
+        with open("DVC_EXP_BASELINE_REV", "w") as f:
+            f.write(os.environ.get(DVC_EXP_BASELINE_REV, ""))
+        with open("DVC_EXP_NAME", "w") as f:
+            f.write(os.environ.get(DVC_EXP_NAME, ""))
+        """
+    )
+    (tmp_dir / "dump_run_env.py").write_text(dump_run_env)
+    baseline = scm.get_rev()
+    dvc.stage.add(
+        cmd="python dump_run_env.py",
+        name="run_env",
+    )
+    dvc.experiments.run()
+    assert (tmp_dir / "DVC_EXP_BASELINE_REV").read_text().strip() == baseline
+    assert (tmp_dir / "DVC_EXP_NAME").read_text().strip() == ""
+
+    dvc.experiments.run(name="foo")
+    assert (tmp_dir / "DVC_EXP_BASELINE_REV").read_text().strip() == baseline
+    assert (tmp_dir / "DVC_EXP_NAME").read_text().strip() == "foo"

tests/func/experiments/test_remote.py

@@ -151,7 +151,6 @@ def test_list_remote(tmp_dir, scm, dvc, git_downstream, exp_stage, use_url):
     ref_info_b = first(exp_refs_by_rev(scm, exp_b))
 
     tmp_dir.scm_gen("new", "new", commit="new")
-    baseline_c = scm.get_rev()
     results = dvc.experiments.run(exp_stage.addressing, params=["foo=4"])
     exp_c = first(results)
     ref_info_c = first(exp_refs_by_rev(scm, exp_c))
@@ -165,13 +164,13 @@ def test_list_remote(tmp_dir, scm, dvc, git_downstream, exp_stage, use_url):
     git_downstream.tmp_dir.scm.fetch_refspecs(remote, ["master:master"])
     exp_list = downstream_exp.ls(rev=baseline_a, git_remote=remote)
     assert {key: set(val) for key, val in exp_list.items()} == {
-        baseline_a: {ref_info_a.name, ref_info_b.name}
+        baseline_a[:7]: {ref_info_a.name, ref_info_b.name}
     }
 
     exp_list = downstream_exp.ls(all_commits=True, git_remote=remote)
     assert {key: set(val) for key, val in exp_list.items()} == {
-        baseline_a: {ref_info_a.name, ref_info_b.name},
-        baseline_c: {ref_info_c.name},
+        baseline_a[:7]: {ref_info_a.name, ref_info_b.name},
+        "master": {ref_info_c.name},
     }
 
 

tests/func/experiments/test_save.py

@@ -0,0 +1,85 @@
+from contextlib import nullcontext
+
+import pytest
+from funcy import first
+
+from dvc.repo.experiments.exceptions import ExperimentExistsError
+from dvc.repo.experiments.utils import exp_refs_by_rev
+from dvc.scm import resolve_rev
+from dvc.stage.exceptions import StageCommitError
+
+
+@pytest.mark.parametrize("name", (None, "test"))
+def test_exp_save(tmp_dir, dvc, scm, exp_stage, name):
+    baseline = scm.get_rev()
+
+    exp = dvc.experiments.save(name=name)
+    ref_info = first(exp_refs_by_rev(scm, exp))
+    assert ref_info and ref_info.baseline_sha == baseline
+
+    exp_name = name if name else ref_info.name
+    assert dvc.experiments.get_exact_name([exp])[exp] == exp_name
+    assert resolve_rev(scm, exp_name) == exp
+
+
+@pytest.mark.parametrize(
+    ("force", "expected_raises"),
+    (
+        (False, pytest.raises(StageCommitError)),
+        (True, nullcontext()),
+    ),
+)
+def test_exp_save_force(tmp_dir, dvc, scm, exp_stage, force, expected_raises):
+    with open(tmp_dir / "copy.py", "a", encoding="utf-8") as fh:
+        fh.write("\n# dummy change")
+
+    with expected_raises:
+        dvc.experiments.save(force=force)
+
+
+def test_exp_save_overwrite_experiment(tmp_dir, dvc, scm, exp_stage):
+    dvc.experiments.save(name="dummy")
+
+    with open(tmp_dir / "copy.py", "a", encoding="utf-8") as fh:
+        fh.write("\n# dummy change")
+
+    with pytest.raises(ExperimentExistsError):
+        dvc.experiments.save(name="dummy")
+
+    dvc.experiments.save(name="dummy", force=True)
+
+
+def test_exp_save_after_commit(tmp_dir, dvc, scm, exp_stage):
+    baseline = scm.get_rev()
+    dvc.experiments.save(name="exp-1")
+
+    tmp_dir.scm_gen({"new_file": "new_file"}, commit="new baseline")
+    dvc.experiments.save(name="exp-2")
+
+    all_exps = dvc.experiments.ls(all_commits=True)
+    assert all_exps[baseline[:7]] == ["exp-1"]
+    assert all_exps["master"] == ["exp-2"]
+
+
+def test_exp_save_with_staged_changes(tmp_dir, dvc, scm):
+    tmp_dir.gen({"new_file": "new_file"})
+    scm.add("new_file")
+
+    dvc.experiments.save(name="exp")
+
+    _, _, unstaged = scm.status()
+    assert "new_file" in unstaged
+
+
+def test_exp_save_include_untracked(tmp_dir, dvc, scm, exp_stage):
+    new_file = tmp_dir / "new_file"
+    for i in range(2):
+        new_file.write_text(f"exp-{i}")
+        dvc.experiments.save(name=f"exp-{i}", include_untracked=["new_file"])
+
+    _, _, unstaged = scm.status()
+    assert "new_file" in unstaged
+    assert new_file.read_text() == f"exp-{i}"
+
+    dvc.experiments.apply("exp-0")
+    assert new_file.read_text() == "exp-0"

tests/func/experiments/test_show.py

@@ -10,15 +10,17 @@ from scmrepo.exceptions import SCMError
 
 from dvc.cli import main
 from dvc.repo.experiments.executor.base import (
-    EXEC_PID_DIR,
-    EXEC_TMP_DIR,
     BaseExecutor,
     ExecutorInfo,
     TaskStatus,
 )
 from dvc.repo.experiments.queue.base import QueueEntry
 from dvc.repo.experiments.refs import CELERY_STASH, ExpRefInfo
-from dvc.repo.experiments.utils import exp_refs_by_rev
+from dvc.repo.experiments.utils import (
+    EXEC_PID_DIR,
+    EXEC_TMP_DIR,
+    exp_refs_by_rev,
+)
 from dvc.utils import relpath
 from dvc.utils.fs import makedirs
 from dvc.utils.serialize import YAMLFileCorruptedError

tests/func/metrics/test_diff.py

@@ -1,6 +1,10 @@
 import json
+from os.path import join
+
+import pytest
 
 from dvc.cli import main
+from dvc.utils import relpath
 
 
 def test_metrics_diff_simple(tmp_dir, scm, dvc, run_copy_metrics):
@@ -206,3 +210,28 @@ def test_metrics_diff_non_metrics(tmp_dir, scm, dvc):
     assert result == {
         "some_file.yaml": {"foo": {"old": 1, "new": 3, "diff": 2}}
     }
+
+
+@pytest.mark.parametrize(
+    "dvcfile, metrics_file",
+    [
+        ("dvc.yaml", "my_metrics.yaml"),
+        ("dir/dvc.yaml", "my_metrics.yaml"),
+        ("dir/dvc.yaml", join("..", "my_metrics.yaml")),
+    ],
+)
+def test_diff_top_level_metrics(tmp_dir, dvc, scm, dvcfile, metrics_file):
+    directory = (tmp_dir / dvcfile).parent
+    directory.mkdir(exist_ok=True)
+    (tmp_dir / dvcfile).dump({"metrics": [metrics_file]})
+
+    metrics_file = directory / metrics_file
+    metrics_file.dump({"foo": 3})
+    scm.add_commit([metrics_file, tmp_dir / dvcfile], message="add metrics")
+
+    metrics_file.dump({"foo": 5})
+    assert dvc.metrics.diff() == {
+        relpath(directory / metrics_file): {
+            "foo": {"diff": 2, "new": 5, "old": 3}
+        }
+    }

tests/func/params/test_diff.py

@@ -1,3 +1,5 @@
+from os.path import join
+
 import pytest
 
 from dvc.utils import relpath
@@ -241,3 +243,28 @@ def test_diff_without_targets_specified(tmp_dir, dvc, scm, file):
             "y": {"new": "100", "old": None},
         }
     }
+
+
+@pytest.mark.parametrize(
+    "dvcfile, params_file",
+    [
+        ("dvc.yaml", "my_params.yaml"),
+        ("dir/dvc.yaml", "my_params.yaml"),
+        ("dir/dvc.yaml", join("..", "my_params.yaml")),
+    ],
+)
+def test_diff_top_level_params(tmp_dir, dvc, scm, dvcfile, params_file):
+    directory = (tmp_dir / dvcfile).parent
+    directory.mkdir(exist_ok=True)
+    (tmp_dir / dvcfile).dump({"params": [params_file]})
+
+    params_file = directory / params_file
+    params_file.dump({"foo": 3})
+    scm.add_commit([params_file, tmp_dir / dvcfile], message="add params")
+
+    params_file.dump({"foo": 5})
+    assert dvc.params.diff() == {
+        relpath(directory / params_file): {
+            "foo": {"diff": 2, "new": 5, "old": 3}
+        }
+    }

tests/func/parsing/test_interpolated_entry.py

@@ -6,6 +6,7 @@ import pytest
 from dvc.dependency import _merge_params
 from dvc.parsing import DEFAULT_PARAMS_FILE, DataResolver
 from dvc.parsing.context import recurse_not_a_node
+from dvc.parsing.interpolate import escape_str
 
 from . import (
     CONTEXT_DATA,
@@ -272,14 +273,17 @@ def test_cmd_dict(tmp_dir, dvc, bool_config, list_config):
         if list_config:
             conf["parsing"]["list"] = list_config
 
+    string = "spaced string"
+    mixed_quote_string = "quote\"'d"
     data = {
         "dict": {
             "foo": "foo",
             "bar": 2,
-            "string": "spaced string",
+            "string": string,
+            "mixed_quote_string": mixed_quote_string,
             "bool": True,
             "bool-false": False,
-            "list": [1, 2, "foo"],
+            "list": [1, 2, "foo", mixed_quote_string],
             "nested": {"foo": "foo"},
         }
     }
@@ -296,9 +300,10 @@ def test_cmd_dict(tmp_dir, dvc, bool_config, list_config):
         bool_resolved = " --bool --no-bool-false"
 
     if list_config is None or list_config == "nargs":
-        list_resolved = " --list 1 2 'foo'"
+        list_resolved = f" --list 1 2 foo {escape_str(mixed_quote_string)}"
     else:
-        list_resolved = " --list 1 --list 2 --list 'foo'"
+        list_resolved = " --list 1 --list 2 --list foo"
+        list_resolved += f" --list {escape_str(mixed_quote_string)}"
 
     assert_stage_equal(
         resolver.resolve(),
@@ -306,11 +311,12 @@ def test_cmd_dict(tmp_dir, dvc, bool_config, list_config):
             "stages": {
                 "stage1": {
                     "cmd": "python script.py"
-                    " --foo 'foo' --bar 2"
-                    " --string 'spaced string'"
+                    " --foo foo --bar 2"
+                    f" --string {escape_str(string)}"
+                    f" --mixed_quote_string {escape_str(mixed_quote_string)}"
                     f"{bool_resolved}"
                     f"{list_resolved}"
-                    " --nested.foo 'foo'"
+                    " --nested.foo foo"
                 }
             }
         },

tests/func/plots/test_show.py

@@ -336,16 +336,15 @@ def test_collect_non_existing_dir(tmp_dir, dvc, run_copy_metrics):
 @pytest.mark.parametrize(
     "plot_config,expected_datafiles",
     [
-        # TODO - enable providing data files for x
-        # (
-        #     {
-        #         "comparison": {
-        #             "x": {"data1.json": "a"},
-        #             "y": {"sub/dir/data2.json": "b"},
-        #         }
-        #     },
-        #     ["data1.json", os.path.join("sub", "dir", "data2.json")],
-        # ),
+        (
+            {
+                "comparison": {
+                    "x": {"data1.json": "a"},
+                    "y": {"sub/dir/data2.json": "b"},
+                }
+            },
+            ["data1.json", os.path.join("sub", "dir", "data2.json")],
+        ),
         (
             {"data1.json": {"x": "c", "y": "a", "title": "File as key test"}},
             ["data1.json"],

tests/func/test_import.py

@@ -304,7 +304,7 @@ def test_pull_imported_directory_stage(tmp_dir, dvc, erepo_dir):
     dvc.imp(os.fspath(erepo_dir), "dir", "dir_imported")
 
     remove("dir_imported")
-    remove(dvc.odb.local.path)
+    dvc.odb.local.clear()
 
     dvc.pull(["dir_imported.dvc"])
 
@@ -320,7 +320,7 @@ def test_pull_wildcard_imported_directory_stage(tmp_dir, dvc, erepo_dir):
     dvc.imp(os.fspath(erepo_dir), "dir123", "dir_imported123")
 
     remove("dir_imported123")
-    remove(dvc.odb.local.path)
+    dvc.odb.local.clear()
 
     dvc.pull(["dir_imported*.dvc"], glob=True)
 
@@ -546,9 +546,8 @@ def test_pull_imported_stage_from_subrepos(
     dvc.imp(os.fspath(erepo_dir), path, out="out")
 
     # clean everything
-    remove(dvc.odb.local.path)
+    dvc.odb.local.clear()
     remove("out")
-    makedirs(dvc.odb.local.path)
 
     stats = dvc.pull(["out.dvc"])
 
@@ -633,7 +632,7 @@ def test_chained_import(tmp_dir, dvc, make_tmp_dir, erepo_dir, local_cloud):
     assert (dst / "foo").read_text() == "foo"
     assert (dst / "bar").read_text() == "bar"
 
-    remove(dvc.odb.local.path)
+    dvc.odb.local.clear()
     remove("dir_imported")
 
     # pulled objects should come from the original upstream repo's remote,

tests/func/test_run_cache.py

@@ -33,7 +33,7 @@ def test_restore(tmp_dir, dvc, run_copy, mocker):
 
     (stage,) = dvc.reproduce("copy-foo-bar")
 
-    mock_restore.assert_called_once_with(stage)
+    mock_restore.assert_called_once_with(stage, dry=False)
     mock_run.assert_not_called()
     assert (tmp_dir / "bar").exists() and not (tmp_dir / "foo").unlink()
     assert (tmp_dir / PIPELINE_LOCK).exists()
@@ -100,7 +100,7 @@ def test_memory_for_multiple_runs_of_same_stage(
     assert (tmp_dir / PIPELINE_LOCK).exists()
     assert (tmp_dir / "bar").read_text() == "foobar"
     mock_run.assert_not_called()
-    mock_restore.assert_called_once_with(stage)
+    mock_restore.assert_called_once_with(stage, dry=False)
     mock_restore.reset_mock()
 
     (tmp_dir / PIPELINE_LOCK).unlink()
@@ -109,7 +109,7 @@ def test_memory_for_multiple_runs_of_same_stage(
 
     assert (tmp_dir / "bar").read_text() == "foo"
     mock_run.assert_not_called()
-    mock_restore.assert_called_once_with(stage)
+    mock_restore.assert_called_once_with(stage, dry=False)
     assert (tmp_dir / "bar").exists() and not (tmp_dir / "foo").unlink()
     assert (tmp_dir / PIPELINE_LOCK).exists()
 
@@ -138,7 +138,7 @@ def test_memory_runs_of_multiple_stages(tmp_dir, dvc, run_copy, mocker):
     assert (tmp_dir / "foo.bak").read_text() == "foo"
     assert (tmp_dir / PIPELINE_LOCK).exists()
     mock_run.assert_not_called()
-    mock_restore.assert_called_once_with(stage)
+    mock_restore.assert_called_once_with(stage, dry=False)
     mock_restore.reset_mock()
 
     (stage,) = dvc.reproduce("backup-bar")
@@ -146,7 +146,7 @@ def test_memory_runs_of_multiple_stages(tmp_dir, dvc, run_copy, mocker):
     assert (tmp_dir / "bar.bak").read_text() == "bar"
     assert (tmp_dir / PIPELINE_LOCK).exists()
     mock_run.assert_not_called()
-    mock_restore.assert_called_once_with(stage)
+    mock_restore.assert_called_once_with(stage, dry=False)
 
 
 def test_restore_pull(tmp_dir, dvc, run_copy, mocker, local_remote):
@@ -168,7 +168,7 @@ def test_restore_pull(tmp_dir, dvc, run_copy, mocker, local_remote):
 
     (stage,) = dvc.reproduce("copy-foo-bar", pull=True)
 
-    mock_restore.assert_called_once_with(stage, pull=True)
+    mock_restore.assert_called_once_with(stage, pull=True, dry=False)
     mock_run.assert_not_called()
     assert mock_checkout.call_count == 2
     assert (tmp_dir / "bar").exists() and not (tmp_dir / "foo").unlink()

tests/func/test_used_objs.py

@@ -51,3 +51,16 @@ def test_from_gitfs_when_pwd_not_in_root(
 
     with wdir.chdir():
         assert dvc.used_objs([target], revs=[scm.get_rev()])
+
+
+def test_used_objs_push(tmp_dir, scm, dvc):
+    stage = tmp_dir.dvc_gen("foo", "foo")[0]
+    hash_info = stage.outs[0].hash_info
+
+    stage.outs[0].can_push = True
+    assert stage.get_used_objs(push=False) == {None: {hash_info}}
+    assert stage.get_used_objs(push=True) == {None: {hash_info}}
+
+    stage.outs[0].can_push = False
+    assert stage.get_used_objs(push=False) == {None: {hash_info}}
+    assert stage.get_used_objs(push=True) == {}

tests/func/utils/test_hydra.py

@@ -174,12 +174,25 @@ def test_compose_and_dump(tmp_dir, suffix, overrides, expected):
     from dvc.utils.hydra import compose_and_dump
 
     config_name = "config"
-    config_dir = hydra_setup(tmp_dir, "conf", "config")
     output_file = tmp_dir / f"params.{suffix}"
+    config_dir = hydra_setup(tmp_dir, "conf", "config")
     compose_and_dump(output_file, config_dir, config_name, overrides)
     assert output_file.parse() == expected
 
 
+@pytest.mark.skipif(sys.version_info >= (3, 11), reason="unsupported on 3.11")
+def test_compose_and_dump_yaml_handles_string(tmp_dir):
+    """Regression test for 8583"""
+    from dvc.utils.hydra import compose_and_dump
+
+    config = tmp_dir / "conf" / "config.yaml"
+    config.parent.mkdir()
+    config.write_text("foo: 'no'\n")
+    output_file = tmp_dir / "params.yaml"
+    compose_and_dump(output_file, str(config.parent), "config", [])
+    assert output_file.read_text() == "foo: 'no'\n"
+
+
 @pytest.mark.parametrize(
     "overrides, expected",
     [

tests/integration/plots/test_plots.py

@@ -456,9 +456,7 @@ def test_repo_with_config_plots(tmp_dir, capsys, repo_with_config_plots):
             "dvc.yaml::confusion_train_vs_test",
         ],
     )
-    assert html_result["dvc.yaml::linear_train_vs_test"]["data"][
-        "values"
-    ] == _update_datapoints(
+    ble = _update_datapoints(
         plots["data"]["linear_train.json"],
         {
             REVISION_FIELD: "linear_train.json",
@@ -470,4 +468,7 @@ def test_repo_with_config_plots(tmp_dir, capsys, repo_with_config_plots):
         },
     )
 
+    assert (
+        html_result["dvc.yaml::linear_train_vs_test"]["data"]["values"] == ble
+    )
     # TODO check json results once vscode is able to handle flexible plots

tests/unit/command/test_experiments.py

@@ -16,6 +16,7 @@ from dvc.commands.experiments.pull import CmdExperimentsPull
 from dvc.commands.experiments.push import CmdExperimentsPush
 from dvc.commands.experiments.remove import CmdExperimentsRemove
 from dvc.commands.experiments.run import CmdExperimentsRun
+from dvc.commands.experiments.save import CmdExperimentsSave
 from dvc.commands.experiments.show import CmdExperimentsShow, show_experiments
 from dvc.exceptions import InvalidArgumentError
 from dvc.repo import Repo
@@ -934,3 +935,17 @@ def test_show_experiments_pcp(tmp_dir, mocker):
 
     assert kwargs["output_path"] == str(tmp_dir / "dvc_plots" / "index.html")
     assert kwargs["color_by"] == "Experiment"
+
+
+def test_experiments_save(dvc, scm, mocker):
+    cli_args = parse_args(["exp", "save", "--name", "exp-name", "--force"])
+    assert cli_args.func == CmdExperimentsSave
+
+    cmd = cli_args.func(cli_args)
+    m = mocker.patch("dvc.repo.experiments.save.save", return_value="acabb")
+
+    assert cmd.run() == 0
+
+    m.assert_called_once_with(
+        cmd.repo, name="exp-name", force=True, include_untracked=[]
+    )

tests/unit/command/test_plots.py

@@ -178,7 +178,7 @@ def test_plots_diff_open(tmp_dir, dvc, mocker, capsys, plots_data, auto_open):
 def test_plots_diff_open_WSL(tmp_dir, dvc, mocker, plots_data):
     mocked_open = mocker.patch("webbrowser.open", return_value=True)
     mocked_uname_result = mocker.MagicMock()
-    mocked_uname_result.release = "Microsoft"
+    mocked_uname_result.release = "microsoft"
     mocker.patch("platform.uname", return_value=mocked_uname_result)
 
     cli_args = parse_args(

tests/unit/command/test_status.py

@@ -81,7 +81,7 @@ def test_status_empty(dvc, mocker, capsys):
 
     cmd = cli_args.func(cli_args)
 
-    spy = mocker.spy(cmd.repo.stage, "_collect_repo")
+    spy = mocker.spy(cmd.repo.stage, "_collect_all_from_repo")
 
     assert cmd.run() == 0
 
@@ -108,7 +108,10 @@ def test_status_up_to_date(dvc, mocker, capsys, cloud_opts, expected_message):
     mocker.patch.dict(cmd.repo.config, {"core": {"remote": "default"}})
     mocker.patch.object(cmd.repo, "status", autospec=True, return_value={})
     mocker.patch.object(
-        cmd.repo.stage, "_collect_repo", return_value=[object()], autospec=True
+        cmd.repo.stage,
+        "_collect_all_from_repo",
+        return_value=[[object()], {}, {}, {}],
+        autospec=True,
     )
 
     assert cmd.run() == 0

tests/unit/render/test_image_converter.py

@@ -3,10 +3,9 @@ from dvc.render.converter.image import ImageConverter
 
 
 def test_image_converter_no_out():
-    data = b"content"
-    converter = ImageConverter()
-
-    datapoints, _ = converter.convert(data, "r", "image.png")
+    data = {"image.png": b"content"}
+    converter = ImageConverter("image.png", data)
+    datapoints, _ = converter.flat_datapoints("r")
 
     assert datapoints[0] == {
         REVISION_FIELD: "r",
@@ -16,10 +15,10 @@ def test_image_converter_no_out():
 
 
 def test_image_converter_with_out(tmp_dir):
-    data = b"content"
-    converter = ImageConverter({"out": tmp_dir / "foo"})
+    data = {"image.png": b"content"}
+    converter = ImageConverter("image.png", data, {"out": tmp_dir / "foo"})
 
-    datapoints, _ = converter.convert(data, "r", "image.png")
+    datapoints, _ = converter.flat_datapoints("r")
 
     assert datapoints[0] == {
         REVISION_FIELD: "r",
@@ -32,10 +31,10 @@ def test_image_converter_with_out(tmp_dir):
 
 def test_image_converter_with_slash_in_revision(tmp_dir):
     """Regression test for #7934"""
-    data = b"content"
-    converter = ImageConverter({"out": tmp_dir / "foo"})
+    data = {"image.png": b"content"}
+    converter = ImageConverter("image.png", data, {"out": tmp_dir / "foo"})
 
-    datapoints, _ = converter.convert(data, "feature/r", "image.png")
+    datapoints, _ = converter.flat_datapoints("feature/r")
 
     assert datapoints[0] == {
         REVISION_FIELD: "feature/r",

tests/unit/render/test_match.py

@@ -111,6 +111,8 @@ def test_match_renderers(mocker):
         "title": "config_file_1::plot_id_1",
         "x": "x",
         "y": "y",
+        "x_label": "x",
+        "y_label": "y",
     }
 
 

tests/unit/render/test_vega_converter.py

@@ -2,39 +2,9 @@ from collections import OrderedDict
 
 import pytest
 
+from dvc.exceptions import DvcException
 from dvc.render import VERSION_FIELD
-from dvc.render.converter.vega import (
-    FieldsNotFoundError,
-    PlotDataStructureError,
-    VegaConverter,
-    _filter_fields,
-    _find_first_list,
-    _lists,
-)
-
-
-def test_find_first_list_in_dict():
-    m1 = [{"accuracy": 1, "loss": 2}, {"accuracy": 3, "loss": 4}]
-    m2 = [{"x": 1}, {"x": 2}]
-    dmetric = OrderedDict([("t1", m1), ("t2", m2)])
-
-    assert _find_first_list(dmetric, fields=set()) == m1
-    assert _find_first_list(dmetric, fields={"x"}) == m2
-
-    with pytest.raises(PlotDataStructureError):
-        _find_first_list(dmetric, fields={"foo"})
-
-
-def test_filter_fields():
-    m = [{"accuracy": 1, "loss": 2}, {"accuracy": 3, "loss": 4}]
-
-    assert _filter_fields(m, fields=set()) == m
-
-    expected = [{"accuracy": 1}, {"accuracy": 3}]
-    assert _filter_fields(m, fields={"accuracy"}) == expected
-
-    with pytest.raises(FieldsNotFoundError):
-        _filter_fields(m, fields={"bad_field"})
+from dvc.render.converter.vega import VegaConverter, _lists
 
 
 @pytest.mark.parametrize(
@@ -58,7 +28,7 @@ def test_finding_lists(dictionary, expected_result):
     "input_data,properties,expected_datapoints,expected_properties",
     [
         pytest.param(
-            {"metric": [{"v": 1}, {"v": 2}]},
+            {"f": {"metric": [{"v": 1}, {"v": 2}]}},
             {},
             [
                 {
@@ -80,41 +50,11 @@ def test_finding_lists(dictionary, expected_result):
                     },
                 },
             ],
-            {"x": "step", "y": "v"},
+            {"x": "step", "y": "v", "x_label": "step", "y_label": "v"},
             id="default_x_y",
         ),
         pytest.param(
-            {"metric": [{"v": 1, "v2": 0.1}, {"v": 2, "v2": 0.2}]},
-            {"fields": {"v"}},
-            [
-                {
-                    "v": 1,
-                    "step": 0,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v",
-                    },
-                },
-                {
-                    "v": 2,
-                    "step": 1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v",
-                    },
-                },
-            ],
-            {
-                "x": "step",
-                "y": "v",
-                "fields": {"v", "step"},
-            },
-            id="filter_fields",
-        ),
-        pytest.param(
-            {"metric": [{"v": 1, "v2": 0.1}, {"v": 2, "v2": 0.2}]},
+            {"f": {"metric": [{"v": 1, "v2": 0.1}, {"v": 2, "v2": 0.2}]}},
             {"x": "v", "y": "v2"},
             [
                 {
@@ -136,52 +76,24 @@ def test_finding_lists(dictionary, expected_result):
                     },
                 },
             ],
-            {"x": "v", "y": "v2"},
+            {"x": "v", "y": "v2", "x_label": "v", "y_label": "v2"},
             id="choose_x_y",
         ),
         pytest.param(
             {
-                "metric": [
-                    {"v": 1, "v2": 0.1, "v3": 0.01, "v4": 0.001},
-                    {"v": 2, "v2": 0.2, "v3": 0.02, "v4": 0.002},
-                ]
-            },
-            {"x": "v3", "y": "v4", "fields": {"v"}},
-            [
-                {
-                    "v": 1,
-                    "v3": 0.01,
-                    "v4": 0.001,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v4",
+                "f": {
+                    "some": "noise",
+                    "very": {
+                        "nested": {
+                            "metric": [
+                                {"v": 1, "v2": 0.1},
+                                {"v": 2, "v2": 0.2},
+                            ]
+                        }
                     },
-                },
-                {
-                    "v": 2,
-                    "v3": 0.02,
-                    "v4": 0.002,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v4",
-                    },
-                },
-            ],
-            {"x": "v3", "y": "v4", "fields": {"v", "v3", "v4"}},
-            id="append_x_y_to_fields",
-        ),
-        pytest.param(
-            {
-                "some": "noise",
-                "very": {
-                    "nested": {
-                        "metric": [{"v": 1, "v2": 0.1}, {"v": 2, "v2": 0.2}]
-                    }
-                },
+                }
             },
-            {"x": "v", "y": "v2"},
+            {"x": "v", "y": "v2", "x_label": "x", "y_label": "y"},
             [
                 {
                     "v": 1,
@@ -202,24 +114,135 @@ def test_finding_lists(dictionary, expected_result):
                     },
                 },
             ],
-            {"x": "v", "y": "v2"},
+            {"x": "v", "y": "v2", "x_label": "x", "y_label": "y"},
             id="find_in_nested_structure",
         ),
         pytest.param(
-            {"metric": [{"v": 1, "v2": 0.1}, {"v": 2, "v2": 0.2}]},
+            {"f": {"metric": [{"v": 1, "v2": 0.1}, {"v": 2, "v2": 0.2}]}},
             {"y": {"f": ["v", "v2"]}},
             [
                 {
-                    "v": 1,
-                    "v2": 0.1,
                     VERSION_FIELD: {
                         "revision": "r",
                         "filename": "f",
                         "field": "v",
                     },
                     "dvc_inferred_y_value": 1,
+                    "v": 1,
+                    "v2": 0.1,
                     "step": 0,
                 },
+                {
+                    VERSION_FIELD: {
+                        "revision": "r",
+                        "filename": "f",
+                        "field": "v",
+                    },
+                    "dvc_inferred_y_value": 2,
+                    "v": 2,
+                    "v2": 0.2,
+                    "step": 1,
+                },
+                {
+                    VERSION_FIELD: {
+                        "revision": "r",
+                        "filename": "f",
+                        "field": "v2",
+                    },
+                    "dvc_inferred_y_value": 0.1,
+                    "v2": 0.1,
+                    "v": 1,
+                    "step": 0,
+                },
+                {
+                    VERSION_FIELD: {
+                        "revision": "r",
+                        "filename": "f",
+                        "field": "v2",
+                    },
+                    "v": 2,
+                    "v2": 0.2,
+                    "dvc_inferred_y_value": 0.2,
+                    "step": 1,
+                },
+            ],
+            {
+                "x": "step",
+                "y": "dvc_inferred_y_value",
+                "y_label": "y",
+                "x_label": "step",
+            },
+            id="y_def_list",
+        ),
+        pytest.param(
+            {
+                "f": {
+                    "metric": [{"v": 1}, {"v": 2}],
+                    "other_metric": [{"z": 3}, {"z": 4}],
+                }
+            },
+            {"y": {"f": ["v", "z"]}},
+            [
+                {
+                    "dvc_inferred_y_value": 1,
+                    "z": 3,
+                    "v": 1,
+                    "step": 0,
+                    VERSION_FIELD: {
+                        "revision": "r",
+                        "filename": "f",
+                        "field": "v",
+                    },
+                },
+                {
+                    "dvc_inferred_y_value": 2,
+                    "z": 4,
+                    "step": 1,
+                    "v": 2,
+                    VERSION_FIELD: {
+                        "revision": "r",
+                        "filename": "f",
+                        "field": "v",
+                    },
+                },
+                {
+                    "dvc_inferred_y_value": 3,
+                    "v": 1,
+                    "z": 3,
+                    "step": 0,
+                    VERSION_FIELD: {
+                        "revision": "r",
+                        "filename": "f",
+                        "field": "z",
+                    },
+                },
+                {
+                    "dvc_inferred_y_value": 4,
+                    "v": 2,
+                    "z": 4,
+                    "step": 1,
+                    VERSION_FIELD: {
+                        "revision": "r",
+                        "filename": "f",
+                        "field": "z",
+                    },
+                },
+            ],
+            {
+                "x": "step",
+                "y": "dvc_inferred_y_value",
+                "y_label": "y",
+                "x_label": "step",
+            },
+            id="multi_source_json",
+        ),
+        pytest.param(
+            {
+                "f": {"metric": [{"v": 1, "v2": 0.1}, {"v": 2, "v2": 0.2}]},
+                "f2": {"metric": [{"v": 3, "v2": 0.3}]},
+            },
+            {"x": "v", "y": {"f": "v2", "f2": "v2"}},
+            [
                 {
                     "v": 1,
                     "v2": 0.1,
@@ -228,19 +251,6 @@ def test_finding_lists(dictionary, expected_result):
                         "filename": "f",
                         "field": "v2",
                     },
-                    "dvc_inferred_y_value": 0.1,
-                    "step": 0,
-                },
-                {
-                    "v": 2,
-                    "v2": 0.2,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v",
-                    },
-                    "dvc_inferred_y_value": 2,
-                    "step": 1,
                 },
                 {
                     "v": 2,
@@ -250,12 +260,19 @@ def test_finding_lists(dictionary, expected_result):
                         "filename": "f",
                         "field": "v2",
                     },
-                    "dvc_inferred_y_value": 0.2,
-                    "step": 1,
+                },
+                {
+                    "v": 3,
+                    "v2": 0.3,
+                    VERSION_FIELD: {
+                        "revision": "r",
+                        "filename": "f2",
+                        "field": "v2",
+                    },
                 },
             ],
-            {"x": "step", "y": "dvc_inferred_y_value", "y_label": "y"},
-            id="y_def_list",
+            {"x": "v", "y": "v2", "x_label": "v", "y_label": "v2"},
+            id="multi_file_json",
         ),
     ],
 )
@@ -265,32 +282,67 @@ def test_convert(
     expected_datapoints,
     expected_properties,
 ):
-    converter = VegaConverter(properties)
-    datapoints, resolved_properties = converter.convert(
-        data=input_data, revision="r", filename="f"
-    )
+    converter = VegaConverter("f", input_data, properties)
+    datapoints, resolved_properties = converter.flat_datapoints("r")
 
     assert datapoints == expected_datapoints
     assert resolved_properties == expected_properties
 
 
-def test_convert_skip_step():
-    converter = VegaConverter()
-    converter.skip_step("append_index")
-    converter.skip_step("generate_y")
+@pytest.mark.parametrize(
+    "input_data,properties",
+    [
+        pytest.param(
+            {
+                "f": {
+                    "metric": [
+                        {"v": 1},
+                        {"v": 2},
+                    ]
+                },
+                "f2": {"metric": [{"v2": 0.1}]},
+            },
+            {"x": {"f": "v"}, "y": {"f2": "v2"}},
+            id="unequal_datapoints",
+        ),
+        pytest.param(
+            {
+                "f": {
+                    "metric": [
+                        {"v": 1, "v2": 0.1},
+                        {"v": 2, "v2": 0.2},
+                    ]
+                },
+                "f2": {
+                    "metric": [
+                        {"v": 3, "v2": 0.3},
+                    ]
+                },
+            },
+            {"x": {"f": "v", "f2": "v3"}, "y": {"f": "v2"}},
+            id="unequal_x_y",
+        ),
+    ],
+)
+def test_convert_fail(input_data, properties):
+    converter = VegaConverter("f", input_data, properties)
+    with pytest.raises(DvcException):
+        converter.flat_datapoints("r")
 
-    datapoints, resolved_properties = converter.convert(
-        data={"a": "b", "metric": [{"v": 1}, {"v": 2}]},
-        revision="r",
-        filename="f",
-    )
 
-    assert datapoints == [
-        {
-            "v": 1,
-        },
-        {
-            "v": 2,
-        },
-    ]
-    assert resolved_properties == {"x": "step", "y": "v"}
+@pytest.mark.parametrize(
+    "properties,label",
+    [
+        ({"x": {"actual.csv": "actual"}}, "actual"),
+        (
+            {"x": {"train_actual.csv": "actual", "val_actual.csv": "actual"}},
+            "actual",
+        ),
+        (
+            {"x": {"actual.csv": "actual", "predicted.csv": "predicted"}},
+            "x",
+        ),
+    ],
+)
+def test_infer_x_label(properties, label):
+    assert VegaConverter.infer_x_label(properties) == label

tests/unit/repo/experiments/queue/test_celery.py

@@ -2,11 +2,12 @@ import time
 
 import pytest
 from celery import shared_task
-from flaky.flaky_decorator import flaky
+from celery.result import AsyncResult
 
 from dvc.exceptions import DvcException
 from dvc.repo.experiments.exceptions import UnresolvedExpNamesError
 from dvc.repo.experiments.queue.base import QueueDoneResult
+from dvc.repo.experiments.queue.exceptions import CannotKillTasksError
 
 
 def test_shutdown_no_tasks(test_queue, mocker):
@@ -51,16 +52,14 @@ def test_shutdown_with_kill(test_queue, mocker):
     shutdown_spy.assert_called_once()
 
 
-# pytest-celery worker thread may finish the task before we check for PENDING
-@flaky(max_runs=3, min_passes=1)
 def test_post_run_after_kill(test_queue):
 
     from celery import chain
 
     sig_bar = test_queue.proc.run_signature(
-        ["python3", "-c", "import time; time.sleep(5)"], name="bar"
+        ["python3", "-c", "import time; time.sleep(10)"], name="bar"
     )
-    result_bar = sig_bar.freeze()
+    sig_bar.freeze()
     sig_foo = _foo.s()
     result_foo = sig_foo.freeze()
     run_chain = chain(sig_bar, sig_foo)
@@ -69,25 +68,28 @@ def test_post_run_after_kill(test_queue):
     timeout = time.time() + 10
 
     while True:
-        if result_bar.status == "STARTED" or result_bar.ready():
+        try:
+            test_queue.proc.kill("bar")
+            assert result_foo.status == "PENDING"
             break
+        except ProcessLookupError:
+            time.sleep(0.1)
         if time.time() > timeout:
-            raise AssertionError()
-
-    assert result_foo.status == "PENDING"
-    test_queue.proc.kill("bar")
+            raise TimeoutError()
 
     assert result_foo.get(timeout=10) == "foo"
 
 
 def test_celery_queue_kill(test_queue, mocker):
 
-    mock_entry = mocker.Mock(stash_rev=_foo.name)
+    mock_entry_foo = mocker.Mock(stash_rev="foo")
+    mock_entry_bar = mocker.Mock(stash_rev="bar")
+    mock_entry_foobar = mocker.Mock(stash_rev="foobar")
 
     mocker.patch.object(
         test_queue,
         "iter_active",
-        return_value={mock_entry},
+        return_value={mock_entry_foo, mock_entry_bar, mock_entry_foobar},
     )
     mocker.patch.object(
         test_queue,
@@ -100,12 +102,54 @@ def test_celery_queue_kill(test_queue, mocker):
     mocker.patch.object(
         test_queue,
         "match_queue_entry_by_name",
-        return_value={"bar": mock_entry},
+        return_value={
+            "bar": mock_entry_bar,
+            "foo": mock_entry_foo,
+            "foobar": mock_entry_foobar,
+        },
+    )
+    mocker.patch.object(
+        test_queue,
+        "_get_running_task_ids",
+        return_value={"foo", "foobar"},
+    )
+    mocker.patch.object(
+        test_queue,
+        "_iter_processed",
+        return_value=[
+            (mocker.Mock(headers={"id": "foo"}), mock_entry_foo),
+            (mocker.Mock(headers={"id": "bar"}), mock_entry_bar),
+            (mocker.Mock(headers={"id": "foobar"}), mock_entry_foobar),
+        ],
+    )
+    mocker.patch.object(
+        AsyncResult,
+        "ready",
+        return_value=False,
+    )
+    mark_mocker = mocker.patch.object(
+        test_queue.celery.backend,
+        "mark_as_failure",
     )
 
-    spy = mocker.patch.object(test_queue.proc, "kill")
-    test_queue.kill("bar")
-    assert spy.called_once_with(mock_entry.stash_rev)
+    def kill_function(rev):
+        if rev == "foo":
+            return True
+        raise ProcessLookupError
+
+    kill_mock = mocker.patch.object(
+        test_queue.proc,
+        "kill",
+        side_effect=mocker.MagicMock(side_effect=kill_function),
+    )
+    with pytest.raises(
+        CannotKillTasksError, match="Task 'foobar' is initializing,"
+    ):
+        test_queue.kill(["bar", "foo", "foobar"])
+    assert kill_mock.called_once_with(mock_entry_foo.stash_rev)
+    assert kill_mock.called_once_with(mock_entry_bar.stash_rev)
+    assert kill_mock.called_once_with(mock_entry_foobar.stash_rev)
+    mark_mocker.assert_called_once_with("bar", None)
 
 
 @pytest.mark.parametrize("status", ["FAILURE", "SUCCESS"])

tests/unit/repo/experiments/test_show.py

@@ -1,7 +1,13 @@
+import pytest
 from funcy import first
 from scmrepo.git import Git
 
-from dvc.repo.experiments.show import get_names
+from dvc.repo.experiments.show import (
+    ExpStatus,
+    get_branch_names,
+    move_properties_to_head,
+    update_names,
+)
 
 
 def test_get_show_branch(tmp_dir, scm: "Git", dvc, exp_stage):
@@ -21,7 +27,192 @@ def test_get_show_branch(tmp_dir, scm: "Git", dvc, exp_stage):
         baseline: {"baseline": {"data": {}}, exp_a: {"data": {}}},
     }
 
-    get_names(dvc, result)
+    branch_names = get_branch_names(scm, ["workspace", exp_a, baseline])
+    assert branch_names == {exp_a: new_branch, baseline: base_branch}
+
+    update_names(dvc, branch_names, result)
     assert result[exp_a]["baseline"]["data"] == {"name": new_branch}
     assert result[baseline]["baseline"]["data"] == {"name": base_branch}
     assert result[baseline][exp_a]["data"] == {"name": new_branch}
+
+
+@pytest.mark.parametrize(
+    "res_in,res_out",
+    [
+        (
+            {
+                "1": {
+                    "data": {
+                        "checkpoint_tip": "1",
+                        "executor": None,
+                        "status": ExpStatus.Success.name,
+                    }
+                },
+                "2": {
+                    "data": {
+                        "checkpoint_tip": "1",
+                        "executor": "local",
+                        "status": ExpStatus.Running.name,
+                        "name": "a",
+                    }
+                },
+                "3": {
+                    "data": {
+                        "checkpoint_tip": "3",
+                        "executor": None,
+                        "status": ExpStatus.Success.name,
+                    }
+                },
+                "4": {
+                    "data": {
+                        "checkpoint_tip": "3",
+                        "executor": "local",
+                        "status": ExpStatus.Running.name,
+                        "name": "b",
+                    }
+                },
+            },
+            {
+                "1": {
+                    "data": {
+                        "checkpoint_tip": "1",
+                        "executor": "local",
+                        "status": ExpStatus.Running.name,
+                        "name": "a",
+                    }
+                },
+                "2": {
+                    "data": {
+                        "checkpoint_tip": "1",
+                        "executor": None,
+                        "status": ExpStatus.Success.name,
+                    }
+                },
+                "3": {
+                    "data": {
+                        "checkpoint_tip": "3",
+                        "executor": "local",
+                        "status": ExpStatus.Running.name,
+                        "name": "b",
+                    }
+                },
+                "4": {
+                    "data": {
+                        "checkpoint_tip": "3",
+                        "executor": None,
+                        "status": ExpStatus.Success.name,
+                    }
+                },
+            },
+        ),
+        (
+            {
+                "0": {"error": {"msg": "something"}},
+                "1": {
+                    "data": {
+                        "checkpoint_tip": "1",
+                        "executor": None,
+                        "status": ExpStatus.Success.name,
+                    }
+                },
+                "2": {
+                    "data": {
+                        "checkpoint_tip": "1",
+                        "executor": "local",
+                        "status": ExpStatus.Running.name,
+                        "name": "a",
+                    }
+                },
+                "3": {"data": {"name": "b"}},
+            },
+            {
+                "0": {"error": {"msg": "something"}},
+                "1": {
+                    "data": {
+                        "checkpoint_tip": "1",
+                        "executor": "local",
+                        "status": ExpStatus.Running.name,
+                        "name": "a",
+                    }
+                },
+                "2": {
+                    "data": {
+                        "checkpoint_tip": "1",
+                        "executor": None,
+                        "status": ExpStatus.Success.name,
+                    }
+                },
+                "3": {"data": {"name": "b"}},
+            },
+        ),
+        (
+            {
+                "1": {
+                    "data": {
+                        "checkpoint_tip": "1",
+                        "executor": None,
+                        "status": ExpStatus.Success.name,
+                    }
+                },
+                "2": {
+                    "data": {
+                        "checkpoint_tip": "1",
+                        "executor": "local",
+                        "status": ExpStatus.Running.name,
+                        "name": "a",
+                    }
+                },
+                "3": {
+                    "data": {
+                        "checkpoint_tip": "1",
+                        "executor": "local",
+                        "status": ExpStatus.Running.name,
+                        "name": "b",
+                    }
+                },
+                "4": {
+                    "data": {
+                        "checkpoint_tip": "1",
+                        "executor": "local",
+                        "status": ExpStatus.Running.name,
+                    }
+                },
+            },
+            {
+                "1": {
+                    "data": {
+                        "checkpoint_tip": "1",
+                        "executor": "local",
+                        "status": ExpStatus.Running.name,
+                        "name": "a",
+                    }
+                },
+                "2": {
+                    "data": {
+                        "checkpoint_tip": "1",
+                        "executor": None,
+                        "status": ExpStatus.Success.name,
+                    }
+                },
+                "3": {
+                    "data": {
+                        "checkpoint_tip": "1",
+                        "executor": None,
+                        "status": ExpStatus.Success.name,
+                        "name": "b",
+                    }
+                },
+                "4": {
+                    "data": {
+                        "checkpoint_tip": "1",
+                        "executor": None,
+                        "status": ExpStatus.Success.name,
+                    }
+                },
+            },
+        ),
+    ],
+)
+def test_move_properties_to_head(res_in, res_out):
+    move_properties_to_head({"baseline": res_in})
+    assert res_in == res_out

tests/unit/stage/test_run.py

@@ -13,9 +13,9 @@ from dvc.stage.run import run_stage
         (["mycmd1 arg1", "mycmd2 arg2"], ["> mycmd1 arg1", "> mycmd2 arg2"]),
     ],
 )
-def test_run_stage_dry(caplog, cmd, expected):
+def test_run_stage_dry(caplog, dvc, cmd, expected):
     with caplog.at_level(level=logging.INFO, logger="dvc"):
-        stage = Stage(None, "stage.dvc", cmd=cmd)
+        stage = Stage(dvc, "stage.dvc", cmd=cmd)
         run_stage(stage, dry=True)
 
     expected.insert(0, "Running stage 'stage.dvc':")

tests/unit/test_rwlock.py

@@ -1,3 +1,4 @@
+import json
 import os
 
 import pytest
@@ -105,3 +106,37 @@ def test_broken_rwlock(tmp_path):
     with pytest.raises(RWLockFileCorruptedError):
         with _edit_rwlock(dir_path, localfs, False):
             pass
+
+
+@pytest.mark.parametrize("return_value", [True, False])
+def test_corrupted_rwlock(tmp_path, mocker, return_value):
+    dir_path = os.fspath(tmp_path)
+    path = tmp_path / "rwlock"
+
+    foo = "foo"
+    bar = "bar"
+    cmd_foo = "cmd_foo"
+    cmd_bar = "cmd_bar"
+    mocker.patch("psutil.pid_exists", return_value=return_value)
+
+    corrupted_rwlock = {
+        "write": {foo: {"pid": 1234, "cmd": cmd_foo}},
+        "read": {
+            foo: [{"pid": 5555, "cmd": cmd_foo}],
+            bar: [
+                {"pid": 6666, "cmd": cmd_bar},
+                {"pid": 7777, "cmd": cmd_bar},
+            ],
+        },
+    }
+
+    path.write_text(json.dumps(corrupted_rwlock), encoding="utf-8")
+
+    if return_value:
+        with pytest.raises(LockError):
+            with rwlock(dir_path, localfs, "cmd_other", [], [foo, bar], False):
+                pass
+    else:
+        with rwlock(dir_path, localfs, "cmd_other", [], [foo, bar], False):
+            pass
+        assert path.read_text() == """{"read": {}}"""