deps: update dvc-data to >=2.24 (#10158)

* unprotect: add progress bar

* Update pyproject.toml

.github/workflows/benchmarks.yaml

@@ -10,7 +10,7 @@ jobs:
     name: run benchmarks
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
             python-version: "3.11"
       - uses: iterative/dvc-bench@main

.github/workflows/build.yaml

@@ -23,7 +23,7 @@ jobs:
       with:
         fetch-depth: 0
 
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: 3.8
 

.github/workflows/plugin_tests.yaml

@@ -34,7 +34,7 @@ jobs:
         path: ${{ matrix.plugin }}
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.pyv }}
         cache: pip

.github/workflows/tests.yaml

@@ -23,14 +23,14 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-20.04, windows-latest, macos-latest]
-        pyv: ["3.8", "3.9", "3.10", "3.11"]
+        pyv: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.pyv }}
         cache: pip

dvc/cli/parser.py

@@ -19,6 +19,7 @@ from dvc.commands import (
     data_sync,
     destroy,
     diff,
+    du,
     experiments,
     freeze,
     gc,
@@ -95,6 +96,7 @@ COMMANDS = [
     data,
     artifacts,
     studio,
+    du,
 ]
 
 

dvc/commands/du.py

@@ -0,0 +1,80 @@
+import argparse
+import logging
+
+from dvc.cli import completion
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import DictAction, append_doc_link
+from dvc.ui import ui
+
+logger = logging.getLogger(__name__)
+
+
+class CmdDU(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+        from dvc.utils.humanize import naturalsize
+
+        entries = Repo.du(
+            self.args.url,
+            self.args.path,
+            rev=self.args.rev,
+            summarize=self.args.summarize,
+            config=self.args.config,
+            remote=self.args.remote,
+            remote_config=self.args.remote_config,
+        )
+        ui.table([(naturalsize(size), path) for path, size in entries])
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    DU_HELP = "Show disk usage."
+    du_parser = subparsers.add_parser(
+        "du",
+        parents=[parent_parser],
+        description=append_doc_link(DU_HELP, "du"),
+        help=DU_HELP,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    du_parser.add_argument("url", help="Location of DVC repository")
+    du_parser.add_argument(
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
+    )
+    du_parser.add_argument(
+        "-s",
+        "--summarize",
+        action="store_true",
+        help="Show total disk usage.",
+    )
+    du_parser.add_argument(
+        "--config",
+        type=str,
+        help=(
+            "Path to a config file that will be merged with the config "
+            "in the target repository."
+        ),
+    )
+    du_parser.add_argument(
+        "--remote",
+        type=str,
+        help="Remote name to set as a default in the target repository.",
+    )
+    du_parser.add_argument(
+        "--remote-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help=(
+            "Remote config options to merge with a remote's config (default or one "
+            "specified by '--remote') in the target repository."
+        ),
+    )
+    du_parser.add_argument(
+        "path",
+        nargs="?",
+        help="Path to directory within the repository",
+    ).complete = completion.DIR
+    du_parser.set_defaults(func=CmdDU)

dvc/commands/imp_db.py

@@ -13,7 +13,7 @@ logger = logger.getChild(__name__)
 
 class CmdTestDb(CmdBase):
     def run(self):
-        from dvc.database import get_adapter
+        from dvc.database import get_client
         from dvc.database.dbt_utils import DBT_PROJECT_FILE, is_dbt_project
         from dvc.dependency.db import _get_dbt_config
         from dvc.exceptions import DvcException
@@ -47,7 +47,7 @@ class CmdTestDb(CmdBase):
                     "provide arguments or set a configuration"
                 )
 
-        adapter = get_adapter(conn_config, project_dir=project_dir, **dbt_config)
+        adapter = get_client(conn_config, project_dir=project_dir, **dbt_config)
         with adapter as db:
             ui.write(f"Testing with {db}", styled=True)
 

dvc/commands/plots.py

@@ -59,49 +59,6 @@ def _show_json(
     ui.write_json(compact({"errors": all_errors, "data": data}), highlight=False)
 
 
-def _adjust_vega_renderers(renderers):
-    from dvc.render import REVISION_FIELD, VERSION_FIELD
-    from dvc_render import VegaRenderer
-
-    for r in renderers:
-        if isinstance(r, VegaRenderer):
-            if _data_versions_count(r) > 1:
-                summary = _summarize_version_infos(r)
-                for dp in r.datapoints:
-                    vi = dp.pop(VERSION_FIELD, {})
-                    keys = list(vi.keys())
-                    for key in keys:
-                        if not (len(summary.get(key, set())) > 1):
-                            vi.pop(key)
-                    if vi:
-                        dp["rev"] = "::".join(vi.values())
-            else:
-                for dp in r.datapoints:
-                    dp[REVISION_FIELD] = dp[VERSION_FIELD]["revision"]
-                    dp.pop(VERSION_FIELD, {})
-
-
-def _summarize_version_infos(renderer):
-    from collections import defaultdict
-
-    from dvc.render import VERSION_FIELD
-
-    result = defaultdict(set)
-
-    for dp in renderer.datapoints:
-        for key, value in dp.get(VERSION_FIELD, {}).items():
-            result[key].add(value)
-    return dict(result)
-
-
-def _data_versions_count(renderer):
-    from itertools import product
-
-    summary = _summarize_version_infos(renderer)
-    x = product(summary.get("filename", {None}), summary.get("field", {None}))
-    return len(set(x))
-
-
 class CmdPlots(CmdBase):
     def _func(self, *args, **kwargs):
         raise NotImplementedError
@@ -175,11 +132,10 @@ class CmdPlots(CmdBase):
                 return 0
 
             renderers = [r.renderer for r in renderers_with_errors]
-            _adjust_vega_renderers(renderers)
             if self.args.show_vega:
                 renderer = first(filter(lambda r: r.TYPE == "vega", renderers))
                 if renderer:
-                    ui.write_json(renderer.get_filled_template(as_string=False))
+                    ui.write_json(renderer.get_filled_template())
                 return 0
 
             output_file: Path = (Path.cwd() / out).resolve() / "index.html"

dvc/data_cloud.py

@@ -3,7 +3,6 @@
 from typing import TYPE_CHECKING, Iterable, Optional, Set, Tuple
 
 from dvc.config import NoRemoteError, RemoteConfigError
-from dvc.fs.callbacks import Callback
 from dvc.log import logger
 from dvc.utils.objects import cached_property
 from dvc_data.hashfile.db import get_index
@@ -206,11 +205,13 @@ class DataCloud:
         jobs: Optional[int] = None,
         odb: "HashFileDB",
     ) -> "TransferResult":
+        from dvc.fs.callbacks import TqdmCallback
+
         if odb.hash_name == "md5-dos2unix":
             cache = self.repo.cache.legacy
         else:
             cache = self.repo.cache.local
-        with Callback.as_tqdm_callback(
+        with TqdmCallback(
             desc=f"Pushing to {odb.fs.unstrip_protocol(odb.path)}",
             unit="file",
         ) as cb:
@@ -265,11 +266,13 @@ class DataCloud:
         jobs: Optional[int] = None,
         odb: "HashFileDB",
     ) -> "TransferResult":
+        from dvc.fs.callbacks import TqdmCallback
+
         if odb.hash_name == "md5-dos2unix":
             cache = self.repo.cache.legacy
         else:
             cache = self.repo.cache.local
-        with Callback.as_tqdm_callback(
+        with TqdmCallback(
             desc=f"Fetching from {odb.fs.unstrip_protocol(odb.path)}",
             unit="file",
         ) as cb:

dvc/database/__init__.py

@@ -4,19 +4,19 @@ from . import dbt_query, sqla
 from .dbt_models import get_model
 from .serializer import export
 
-Adapter = Union[sqla.SQLAlchemyAdapter, dbt_query.DbtAdapter]
+Client = Union[sqla.SQLAlchemyClient, dbt_query.DbtClient]
 
 
-def get_adapter(
+def get_client(
     config,
     project_dir: Optional[str] = None,
     profile: Optional[str] = None,
     target: Optional[str] = None,
     **kwargs: Any,
-) -> "ContextManager[Adapter]":
+) -> "ContextManager[Client]":
     if config:
-        return sqla.adapter(config, **kwargs)
-    return dbt_query.adapter(project_dir=project_dir, profile=profile, target=target)
+        return sqla.client(config, **kwargs)
+    return dbt_query.client(project_dir=project_dir, profile=profile, target=target)
 
 
-__all__ = ["export", "get_adapter", "get_model", "Adapter"]
+__all__ = ["export", "get_client", "get_model", "Client"]

dvc/database/dbt_query.py

@@ -17,7 +17,7 @@ logger = logger.getChild(__name__)
 
 
 @define
-class DbtAdapter:
+class DbtClient:
     adapter: "BaseAdapter" = field(repr=lambda o: type(o).__qualname__)
     creds: Dict[str, Any] = field(repr=False)
 
@@ -59,12 +59,12 @@ class DbtAdapter:
 
 @contextmanager
 @check_dbt("query")
-def adapter(
+def client(
     project_dir: Optional[str] = None,
     profiles_dir: Optional[str] = None,
     profile: Optional[str] = None,
     target: Optional[str] = None,
-) -> Iterator["DbtAdapter"]:
+) -> Iterator["DbtClient"]:
     from dbt.adapters import factory as adapters_factory
     from dbt.adapters.base.impl import BaseAdapter
 
@@ -85,4 +85,4 @@ def adapter(
         except:  # noqa: E722
             creds = {}
 
-        yield DbtAdapter(adapter, creds)
+        yield DbtClient(adapter, creds)

dvc/database/sqla.py

@@ -26,7 +26,7 @@ def url_from_config(config: Union[str, URL, Dict[str, str]]) -> URL:
 
 
 @dataclass
-class SQLAlchemyAdapter:
+class SQLAlchemyClient:
     engine: Engine
 
     @contextmanager
@@ -61,13 +61,13 @@ def handle_error(url: URL):
 
 
 @contextmanager
-def adapter(
+def client(
     url_or_config: Union[str, URL, Dict[str, str]], **engine_kwargs: Any
-) -> Iterator[SQLAlchemyAdapter]:
+) -> Iterator[SQLAlchemyClient]:
     url = url_from_config(url_or_config)
     with handle_error(url):
         engine = create_engine(url, **engine_kwargs)
     try:
-        yield SQLAlchemyAdapter(engine)
+        yield SQLAlchemyClient(engine)
     finally:
         engine.dispose()

dvc/dependency/db.py

@@ -122,7 +122,7 @@ class DbDependency(AbstractDependency):
         file_format: Optional[str] = None,
         **kwargs: Any,
     ) -> None:
-        from dvc.database import export, get_adapter
+        from dvc.database import export, get_client
 
         db_info = self.info.get(PARAM_DB, {})
         query = db_info.get(self.PARAM_QUERY)
@@ -140,7 +140,7 @@ class DbDependency(AbstractDependency):
             raise DvcException(f"connection {connection} not found in config")
 
         project_dir = self.repo.root_dir
-        with get_adapter(
+        with get_client(
             config, project_dir=project_dir, profile=profile, target=target
         ) as db:
             logger.debug("using adapter: %s", db)

dvc/fs/__init__.py

@@ -27,7 +27,7 @@ from dvc_objects.fs.errors import (  # noqa: F401
 )
 from dvc_objects.fs.path import Path  # noqa: F401
 
-from .callbacks import Callback
+from .callbacks import Callback  # noqa: F401
 from .data import DataFileSystem  # noqa: F401
 from .dvc import DVCFileSystem
 from .git import GitFileSystem  # noqa: F401
@@ -51,7 +51,9 @@ def download(
 ) -> int:
     from dvc.scm import lfs_prefetch
 
-    with Callback.as_tqdm_callback(
+    from .callbacks import TqdmCallback
+
+    with TqdmCallback(
         desc=f"Downloading {fs.path.name(fs_path)}",
         unit="files",
     ) as cb:

dvc/fs/dvc.py

@@ -4,6 +4,7 @@ import ntpath
 import os
 import posixpath
 import threading
+from collections import deque
 from contextlib import ExitStack, suppress
 from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Type, Union
 
@@ -60,6 +61,7 @@ def _merge_info(repo, key, fs_info, dvc_info):
     if fs_info:
         ret["type"] = fs_info["type"]
         ret["size"] = fs_info["size"]
+        ret["fs_info"] = fs_info
         isexec = False
         if fs_info["type"] == "file":
             isexec = utils.is_exec(fs_info["mode"])
@@ -421,6 +423,45 @@ class _DVCFileSystem(AbstractFileSystem):
         dvc_path = _get_dvc_path(dvc_fs, subkey)
         return dvc_fs.get_file(dvc_path, lpath, **kwargs)
 
+    def du(self, path, total=True, maxdepth=None, withdirs=False, **kwargs):
+        if maxdepth is not None:
+            raise NotImplementedError
+
+        sizes = {}
+        dus = {}
+        todo = deque([self.info(path)])
+        while todo:
+            info = todo.popleft()
+            isdir = info["type"] == "directory"
+            size = info["size"] or 0
+            name = info["name"]
+
+            if not isdir:
+                sizes[name] = size
+                continue
+
+            dvc_info = info.get("dvc_info") or {}
+            fs_info = info.get("fs_info")
+            entry = dvc_info.get("entry")
+            if (
+                dvc_info
+                and not fs_info
+                and entry is not None
+                and entry.size is not None
+            ):
+                dus[name] = entry.size
+                continue
+
+            if withdirs:
+                sizes[name] = size
+
+            todo.extend(self.ls(info["name"], detail=True))
+
+        if total:
+            return sum(sizes.values()) + sum(dus.values())
+
+        return sizes
+
     def close(self):
         self._repo_stack.close()
 

dvc/output.py

@@ -786,7 +786,7 @@ class Output:
                     self.hash_name,
                     ignore=self.dvcignore,
                 )
-                with Callback.as_tqdm_callback(
+                with TqdmCallback(
                     desc=f"Committing {self} to cache",
                     unit="file",
                 ) as cb:
@@ -826,7 +826,7 @@ class Output:
         save_obj = obj.filter(prefix)
         assert isinstance(save_obj, Tree)
         checkout_obj = save_obj.get_obj(self.cache, prefix)
-        with Callback.as_tqdm_callback(
+        with TqdmCallback(
             desc=f"Committing {self} to cache",
             unit="file",
         ) as cb:
@@ -1059,7 +1059,7 @@ class Output:
             upload=upload,
             no_progress_bar=no_progress_bar,
         )
-        with Callback.as_tqdm_callback(
+        with TqdmCallback(
             desc=f"Transferring to {odb.fs.unstrip_protocol(odb.path)}",
             unit="file",
         ) as cb:
@@ -1092,7 +1092,10 @@ class Output:
 
     def unprotect(self):
         if self.exists and self.use_cache:
-            self.cache.unprotect(self.fs_path)
+            with TqdmCallback(
+                size=self.meta.nfiles or -1, desc=f"Unprotecting {self}"
+            ) as callback:
+                self.cache.unprotect(self.fs_path, callback=callback)
 
     def get_dir_cache(self, **kwargs) -> Optional["Tree"]:
         if not self.is_dir_checksum:
@@ -1435,7 +1438,7 @@ class Output:
 
         assert staging
         assert obj.hash_info
-        with Callback.as_tqdm_callback(
+        with TqdmCallback(
             desc=f"Adding {self} to cache",
             unit="file",
         ) as cb:

dvc/render/__init__.py

@@ -1,7 +1,8 @@
-INDEX_FIELD = "step"
-REVISION_FIELD = "rev"
-FILENAME_FIELD = "filename"
-VERSION_FIELD = "dvc_data_version_info"
-REVISIONS_KEY = "revisions"
+INDEX = "step"
+REVISION = "rev"
+FILENAME = "filename"
+FIELD = "field"
+REVISIONS = "revisions"
+ANCHOR_DEFINITIONS = "anchor_definitions"
 TYPE_KEY = "type"
-SRC_FIELD = "src"
+SRC = "src"

dvc/render/convert.py

@@ -1,7 +1,6 @@
-from collections import defaultdict
 from typing import Dict, List, Union
 
-from dvc.render import REVISION_FIELD, REVISIONS_KEY, SRC_FIELD, TYPE_KEY, VERSION_FIELD
+from dvc.render import REVISION, REVISIONS, SRC, TYPE_KEY
 from dvc.render.converter.image import ImageConverter
 from dvc.render.converter.vega import VegaConverter
 
@@ -19,39 +18,31 @@ def _get_converter(
     raise ValueError(f"Invalid renderer class {renderer_class}")
 
 
-def _group_by_rev(datapoints):
-    grouped = defaultdict(list)
-    for datapoint in datapoints:
-        rev = datapoint.get(VERSION_FIELD, {}).get("revision")
-        grouped[rev].append(datapoint)
-    return dict(grouped)
-
-
 def to_json(renderer, split: bool = False) -> List[Dict]:
     if renderer.TYPE == "vega":
-        grouped = _group_by_rev(renderer.datapoints)
+        if not renderer.datapoints:
+            return []
+        revs = renderer.get_revs()
         if split:
-            content = renderer.get_filled_template(
-                skip_anchors=["data"], as_string=False
-            )
+            content, split_content = renderer.get_partial_filled_template()
         else:
-            content = renderer.get_filled_template(as_string=False)
-        if grouped:
-            return [
-                {
-                    TYPE_KEY: renderer.TYPE,
-                    REVISIONS_KEY: sorted(grouped.keys()),
-                    "content": content,
-                    "datapoints": grouped,
-                }
-            ]
-        return []
+            content = renderer.get_filled_template()
+            split_content = {}
+
+        return [
+            {
+                TYPE_KEY: renderer.TYPE,
+                REVISIONS: revs,
+                "content": content,
+                **split_content,
+            }
+        ]
     if renderer.TYPE == "image":
         return [
             {
                 TYPE_KEY: renderer.TYPE,
-                REVISIONS_KEY: [datapoint.get(REVISION_FIELD)],
-                "url": datapoint.get(SRC_FIELD),
+                REVISIONS: [datapoint.get(REVISION)],
+                "url": datapoint.get(SRC),
             }
             for datapoint in renderer.datapoints
         ]

dvc/render/converter/image.py

@@ -2,7 +2,7 @@ import base64
 import os
 from typing import TYPE_CHECKING, Any, Dict, List, Tuple
 
-from dvc.render import FILENAME_FIELD, REVISION_FIELD, SRC_FIELD
+from dvc.render import FILENAME, REVISION, SRC
 
 from . import Converter
 
@@ -58,9 +58,9 @@ class ImageConverter(Converter):
             else:
                 src = self._encode_image(image_data)
             datapoint = {
-                REVISION_FIELD: revision,
-                FILENAME_FIELD: filename,
-                SRC_FIELD: src,
+                REVISION: revision,
+                FILENAME: filename,
+                SRC: src,
             }
             datapoints.append(datapoint)
         return datapoints, properties

dvc/render/converter/vega.py

@@ -4,7 +4,7 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
 from funcy import first, last
 
 from dvc.exceptions import DvcException
-from dvc.render import FILENAME_FIELD, INDEX_FIELD, VERSION_FIELD
+from dvc.render import FIELD, FILENAME, INDEX, REVISION
 
 from . import Converter
 
@@ -112,7 +112,6 @@ class VegaConverter(Converter):
     ):
         super().__init__(plot_id, data, properties)
         self.plot_id = plot_id
-        self.inferred_properties: Dict = {}
 
     def _infer_y_from_data(self):
         if self.plot_id in self.data:
@@ -120,34 +119,38 @@ class VegaConverter(Converter):
                 if all(isinstance(item, dict) for item in lst):
                     datapoint = first(lst)
                     field = last(datapoint.keys())
-                    self.inferred_properties["y"] = {self.plot_id: field}
-                    break
+                    return {self.plot_id: field}
+        return None
 
     def _infer_x_y(self):
         x = self.properties.get("x", None)
         y = self.properties.get("y", None)
 
+        inferred_properties: Dict = {}
+
         # Infer x.
         if isinstance(x, str):
-            self.inferred_properties["x"] = {}
+            inferred_properties["x"] = {}
             # If multiple y files, duplicate x for each file.
             if isinstance(y, dict):
                 for file, fields in y.items():
                     # Duplicate x for each y.
                     if isinstance(fields, list):
-                        self.inferred_properties["x"][file] = [x] * len(fields)
+                        inferred_properties["x"][file] = [x] * len(fields)
                     else:
-                        self.inferred_properties["x"][file] = x
+                        inferred_properties["x"][file] = x
             # Otherwise use plot ID as file.
             else:
-                self.inferred_properties["x"][self.plot_id] = x
+                inferred_properties["x"][self.plot_id] = x
 
         # Infer y.
         if y is None:
-            self._infer_y_from_data()
+            inferred_properties["y"] = self._infer_y_from_data()
         # If y files not provided, use plot ID as file.
         elif not isinstance(y, dict):
-            self.inferred_properties["y"] = {self.plot_id: y}
+            inferred_properties["y"] = {self.plot_id: y}
+
+        return inferred_properties
 
     def _find_datapoints(self):
         result = {}
@@ -182,7 +185,7 @@ class VegaConverter(Converter):
 
         x = properties.get("x", None)
         if not isinstance(x, dict):
-            return INDEX_FIELD
+            return INDEX
 
         fields = {field for _, field in _file_field(x)}
         if len(fields) == 1:
@@ -192,7 +195,7 @@ class VegaConverter(Converter):
     def flat_datapoints(self, revision):  # noqa: C901, PLR0912
         file2datapoints, properties = self.convert()
 
-        props_update = {}
+        props_update: Dict[str, Union[str, List[Dict[str, str]]]] = {}
 
         xs = list(_get_xs(properties, file2datapoints))
 
@@ -200,15 +203,17 @@ class VegaConverter(Converter):
         if not xs:
             x_file, x_field = (
                 None,
-                INDEX_FIELD,
+                INDEX,
             )
         else:
             x_file, x_field = xs[0]
-        props_update["x"] = x_field
+
+        num_xs = len(xs)
+        multiple_x_fields = num_xs > 1 and len({x[1] for x in xs}) > 1
+        props_update["x"] = "dvc_inferred_x_value" if multiple_x_fields else x_field
 
         ys = list(_get_ys(properties, file2datapoints))
 
-        num_xs = len(xs)
         num_ys = len(ys)
         if num_xs > 1 and num_xs != num_ys:
             raise DvcException(
@@ -237,6 +242,14 @@ class VegaConverter(Converter):
         else:
             common_prefix_len = 0
 
+        props_update["anchors_y_definitions"] = [
+            {
+                FILENAME: _get_short_y_file(y_file, common_prefix_len),
+                FIELD: y_field,
+            }
+            for y_file, y_field in ys
+        ]
+
         for i, (y_file, y_field) in enumerate(ys):
             if num_xs > 1:
                 x_file, x_field = xs[i]
@@ -249,15 +262,16 @@ class VegaConverter(Converter):
                     source_field=y_field,
                 )
 
-            if x_field == INDEX_FIELD and x_file is None:
-                _update_from_index(datapoints, INDEX_FIELD)
+            if x_field == INDEX and x_file is None:
+                _update_from_index(datapoints, INDEX)
             else:
                 x_datapoints = file2datapoints.get(x_file, [])
                 try:
                     _update_from_field(
                         datapoints,
-                        field=x_field,
+                        field="dvc_inferred_x_value" if multiple_x_fields else x_field,
                         source_datapoints=x_datapoints,
+                        source_field=x_field,
                     )
                 except IndexError:
                     raise DvcException(  # noqa: B904
@@ -266,15 +280,12 @@ class VegaConverter(Converter):
                         "They have to have same length."
                     )
 
-            y_file_short = y_file[common_prefix_len:].strip("/\\")
             _update_all(
                 datapoints,
                 update_dict={
-                    VERSION_FIELD: {
-                        "revision": revision,
-                        FILENAME_FIELD: y_file_short,
-                        "field": y_field,
-                    }
+                    REVISION: revision,
+                    FILENAME: _get_short_y_file(y_file, common_prefix_len),
+                    FIELD: y_field,
                 },
             )
 
@@ -295,10 +306,10 @@ class VegaConverter(Converter):
         generated datapoints and updated properties. `x`, `y` values and labels
         are inferred and always provided.
         """
-        self._infer_x_y()
+        inferred_properties = self._infer_x_y()
 
         datapoints = self._find_datapoints()
-        properties = {**self.properties, **self.inferred_properties}
+        properties = {**self.properties, **inferred_properties}
 
         properties["y_label"] = self.infer_y_label(properties)
         properties["x_label"] = self.infer_x_label(properties)
@@ -306,6 +317,10 @@ class VegaConverter(Converter):
         return datapoints, properties
 
 
+def _get_short_y_file(y_file, common_prefix_len):
+    return y_file[common_prefix_len:].strip("/\\")
+
+
 def _update_from_field(
     target_datapoints: List[Dict],
     field: str,

dvc/render/match.py

@@ -80,7 +80,7 @@ def match_defs_renderers(  # noqa: C901, PLR0912
     for plot_id, group in plots_data.group_definitions().items():
         plot_datapoints: List[Dict] = []
         props = _squash_plots_properties(group)
-        final_props: Dict = {}
+        first_props: Dict = {}
 
         def_errors: Dict[str, Exception] = {}
         src_errors: DefaultDict[str, Dict[str, Exception]] = defaultdict(dict)
@@ -90,6 +90,7 @@ def match_defs_renderers(  # noqa: C901, PLR0912
         if templates_dir is not None:
             props["template_dir"] = templates_dir
 
+        revs = []
         for rev, inner_id, plot_definition in group:
             plot_sources = infer_data_sources(inner_id, plot_definition)
             definitions_data = plots_data.get_definition_data(plot_sources, rev)
@@ -109,19 +110,24 @@ def match_defs_renderers(  # noqa: C901, PLR0912
 
             try:
                 dps, rev_props = converter.flat_datapoints(rev)
+                if dps and rev not in revs:
+                    revs.append(rev)
             except Exception as e:  # noqa: BLE001
                 logger.warning("In %r, %s", rev, str(e).lower())
                 def_errors[rev] = e
                 continue
 
-            if not final_props and rev_props:
-                final_props = rev_props
+            if not first_props and rev_props:
+                first_props = rev_props
             plot_datapoints.extend(dps)
 
-        if "title" not in final_props:
-            final_props["title"] = renderer_id
+        if "title" not in first_props:
+            first_props["title"] = renderer_id
+
+        if revs:
+            first_props["revs_with_datapoints"] = revs
 
         if renderer_cls is not None:
-            renderer = renderer_cls(plot_datapoints, renderer_id, **final_props)
+            renderer = renderer_cls(plot_datapoints, renderer_id, **first_props)
             renderers.append(RendererWithErrors(renderer, dict(src_errors), def_errors))
     return renderers

dvc/repo/__init__.py

@@ -70,6 +70,7 @@ class Repo:
     from dvc.repo.commit import commit  # type: ignore[misc]
     from dvc.repo.destroy import destroy  # type: ignore[misc]
     from dvc.repo.diff import diff  # type: ignore[misc]
+    from dvc.repo.du import du as _du  # type: ignore[misc]
     from dvc.repo.fetch import fetch  # type: ignore[misc]
     from dvc.repo.freeze import freeze, unfreeze  # type: ignore[misc]
     from dvc.repo.gc import gc  # type: ignore[misc]
@@ -93,6 +94,7 @@ class Repo:
     from .cache import check_missing as cache_check_missing  # type: ignore[misc]
     from .data import status as data_status  # type: ignore[misc]
 
+    du = staticmethod(_du)
     ls = staticmethod(_ls)
     ls_url = staticmethod(_ls_url)
     get = staticmethod(_get)
@@ -447,7 +449,10 @@ class Repo:
         return init(root_dir=root_dir, no_scm=no_scm, force=force, subdir=subdir)
 
     def unprotect(self, target):
-        return self.cache.repo.unprotect(target)
+        from dvc.fs.callbacks import TqdmCallback
+
+        with TqdmCallback(desc=f"Unprotecting {target}") as callback:
+            return self.cache.repo.unprotect(target, callback=callback)
 
     def _ignore(self):
         flist = [self.config.files["local"]]

dvc/repo/artifacts.py

@@ -210,7 +210,8 @@ class Artifacts:
     ) -> Tuple[int, str]:
         from dvc_studio_client.model_registry import get_download_uris
 
-        from dvc.fs import Callback, HTTPFileSystem, generic, localfs
+        from dvc.fs import HTTPFileSystem, generic, localfs
+        from dvc.fs.callbacks import TqdmCallback
 
         logger.debug("Trying to download artifact '%s' via studio", name)
         out = out or os.getcwd()
@@ -244,7 +245,7 @@ class Artifacts:
             ) from exc
         fs = HTTPFileSystem()
         jobs = jobs or fs.jobs
-        with Callback.as_tqdm_callback(
+        with TqdmCallback(
             desc=f"Downloading '{name}' from '{repo_url}'",
             unit="files",
         ) as cb:

dvc/repo/brancher.py

@@ -20,6 +20,7 @@ def brancher(
     all_tags=False,
     all_commits=False,
     all_experiments=False,
+    workspace=True,
     commit_date: Optional[str] = None,
     sha_only=False,
     num=1,
@@ -31,6 +32,7 @@ def brancher(
         all_branches (bool): iterate over all available branches.
         all_commits (bool): iterate over all commits.
         all_tags (bool): iterate over all available tags.
+        workspace (bool): include workspace.
         commit_date (str): Keep experiments from the commits after(include)
                             a certain date. Date must match the extended
                             ISO 8601 format (YYYY-MM-DD).
@@ -73,7 +75,8 @@ def brancher(
 
     logger.trace("switching fs to workspace")
     self.fs = LocalFileSystem(url=self.root_dir)
-    yield "workspace"
+    if workspace:
+        yield "workspace"
 
     revs = revs.copy() if revs else []
     if "workspace" in revs:

dvc/repo/du.py

@@ -0,0 +1,42 @@
+from typing import Any, Dict, Optional, Union
+
+
+def du(
+    url: str,
+    path: Optional[str] = None,
+    rev: Optional[str] = None,
+    summarize: bool = False,
+    config: Union[None, Dict[str, Any], str] = None,
+    remote: Optional[str] = None,
+    remote_config: Optional[dict] = None,
+):
+    from dvc.config import Config
+
+    from . import Repo
+
+    if config and not isinstance(config, dict):
+        config_dict = Config.load_file(config)
+    else:
+        config_dict = None
+
+    with Repo.open(
+        url,
+        rev=rev,
+        subrepos=True,
+        uninitialized=True,
+        config=config_dict,
+        remote=remote,
+        remote_config=remote_config,
+    ) as repo:
+        path = path or ""
+
+        fs = repo.dvcfs
+
+        if summarize or not fs.isdir(path):
+            return [(path, fs.du(path, total=True))]
+
+        ret = [
+            (entry_path, fs.du(entry_path, total=True)) for entry_path in fs.ls(path)
+        ]
+        ret.append((path, sum(entry[1] for entry in ret)))
+        return ret

dvc/repo/experiments/pull.py

@@ -112,4 +112,6 @@ def _pull_cache(
         refs = [refs]
     revs = list(exp_commits(repo.scm, refs))
     logger.debug("dvc fetch experiment '%s'", refs)
-    repo.fetch(jobs=jobs, remote=dvc_remote, run_cache=run_cache, revs=revs)
+    repo.fetch(
+        jobs=jobs, remote=dvc_remote, run_cache=run_cache, revs=revs, workspace=False
+    )

dvc/repo/experiments/push.py

@@ -188,4 +188,6 @@ def _push_cache(
     assert isinstance(repo.scm, Git)
     revs = list(exp_commits(repo.scm, refs))
     logger.debug("dvc push experiment '%s'", refs)
-    return repo.push(jobs=jobs, remote=dvc_remote, run_cache=run_cache, revs=revs)
+    return repo.push(
+        jobs=jobs, remote=dvc_remote, run_cache=run_cache, revs=revs, workspace=False
+    )

dvc/repo/fetch.py

@@ -32,6 +32,7 @@ def _collect_indexes(  # noqa: PLR0913
     recursive=False,
     all_commits=False,
     revs=None,
+    workspace=True,
     max_size=None,
     types=None,
     config=None,
@@ -62,6 +63,7 @@ def _collect_indexes(  # noqa: PLR0913
         all_branches=all_branches,
         all_tags=all_tags,
         all_commits=all_commits,
+        workspace=workspace,
     ):
         try:
             repo.config.merge(config)
@@ -79,11 +81,11 @@ def _collect_indexes(  # noqa: PLR0913
             idx.data["repo"].onerror = _make_index_onerror(onerror, rev)
 
             indexes[rev or "workspace"] = idx
-        except Exception as exc:
+        except Exception as exc:  # noqa: BLE001
             if onerror:
                 onerror(rev, None, exc)
             collection_exc = exc
-            logger.exception("failed to collect '%s'", rev or "workspace")
+            logger.warning("failed to collect '%s', skipping", rev or "workspace")
 
     if not indexes and collection_exc:
         raise collection_exc
@@ -104,6 +106,7 @@ def fetch(  # noqa: PLR0913
     all_commits=False,
     run_cache=False,
     revs=None,
+    workspace=True,
     max_size=None,
     types=None,
     config=None,
@@ -148,6 +151,7 @@ def fetch(  # noqa: PLR0913
         recursive=recursive,
         all_commits=all_commits,
         revs=revs,
+        workspace=workspace,
         max_size=max_size,
         types=types,
         config=config,

dvc/repo/imports.py

@@ -100,10 +100,10 @@ def save_imports(
     Returns:
         Objects which were downloaded from source location.
     """
+    from dvc.fs.callbacks import TqdmCallback
     from dvc.stage.exceptions import DataSourceChanged
     from dvc_data.index import md5, save
     from dvc_data.index.checkout import apply, compare
-    from dvc_objects.fs.callbacks import Callback
 
     downloaded: Set["HashInfo"] = set()
 
@@ -122,7 +122,7 @@ def save_imports(
             if not cache.fs.exists(cache.path):
                 os.makedirs(cache.path)
             with TemporaryDirectory(dir=cache.path) as tmpdir:
-                with Callback.as_tqdm_callback(
+                with TqdmCallback(
                     desc="Downloading imports from source",
                     unit="files",
                 ) as cb:

dvc/repo/push.py

@@ -46,6 +46,7 @@ def push(  # noqa: PLR0913
     all_commits=False,
     run_cache=False,
     revs=None,
+    workspace=True,
     glob=False,
 ):
     from fsspec.utils import tokenize
@@ -87,6 +88,7 @@ def push(  # noqa: PLR0913
         recursive=recursive,
         all_commits=all_commits,
         revs=revs,
+        workspace=workspace,
         push=True,
     )
 

dvc/repo/worktree.py

@@ -3,7 +3,6 @@ from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Tuple, Union
 
 from funcy import first
 
-from dvc.fs.callbacks import Callback
 from dvc.log import logger
 from dvc.stage.exceptions import StageUpdateError
 
@@ -247,11 +246,12 @@ def _fetch_out_changes(
     remote_index: Union["DataIndex", "DataIndexView"],
     remote: "Remote",
 ):
+    from dvc.fs.callbacks import TqdmCallback
     from dvc_data.index.checkout import apply, compare
 
     old, new = _get_diff_indexes(out, local_index, remote_index)
 
-    with Callback.as_tqdm_callback(
+    with TqdmCallback(
         unit="entry",
         desc="Comparing indexes",
     ) as cb:
@@ -265,9 +265,7 @@ def _fetch_out_changes(
         )
 
     total = len(new)
-    with Callback.as_tqdm_callback(
-        unit="file", desc=f"Updating '{out}'", disable=total == 0
-    ) as cb:
+    with TqdmCallback(unit="file", desc=f"Updating '{out}'", disable=total == 0) as cb:
         cb.set_size(total)
         apply(
             diff,

dvc/stage/cache.py

@@ -231,7 +231,7 @@ class StageCache:
 
     def transfer(self, from_odb, to_odb):
         from dvc.fs import HTTPFileSystem, LocalFileSystem
-        from dvc.fs.callbacks import Callback
+        from dvc.fs.callbacks import TqdmCallback
 
         from_fs = from_odb.fs
         to_fs = to_odb.fs
@@ -266,7 +266,7 @@ class StageCache:
 
             src_name = from_fs.path.name(src)
             parent_name = from_fs.path.name(from_fs.path.parent(src))
-            with Callback.as_tqdm_callback(
+            with TqdmCallback(
                 desc=src_name,
                 bytes=True,
             ) as cb:

dvc/testing/benchmarks/cli/commands/test_get.py

@@ -1,3 +1,7 @@
+import pytest
+
+
+@pytest.mark.flaky(max_runs=3, min_passes=1)
 def test_get(bench_dvc, tmp_dir, scm, dvc, make_dataset, remote):
     dataset = make_dataset(
         cache=False, files=False, dvcfile=True, commit=True, remote=True

dvc/testing/benchmarks/cli/commands/test_import.py

@@ -1,3 +1,7 @@
+import pytest
+
+
+@pytest.mark.flaky(max_runs=3, min_passes=1)
 def test_import(bench_dvc, tmp_dir, scm, dvc, make_dataset, remote):
     dataset = make_dataset(
         cache=False, files=False, dvcfile=True, commit=True, remote=True

pyproject.toml

@@ -29,14 +29,15 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
 ]
 dynamic = ["version"]
+
 dependencies = [
     "colorama>=0.3.9",
     "configobj>=5.0.6",
     "distro>=1.3",
     "dpath<3,>=2.1.0",
-    "dvc-data>=2.22.1,<2.23.0",
+    "dvc-data>=2.24,<2.25",
     "dvc-http>=2.29.0",
-    "dvc-render>=0.3.1,<1",
+    "dvc-render>=1.0.0,<2",
     "dvc-studio-client>=0.17.1,<1",
     "dvc-task>=0.3.0,<1",
     "flatten_dict<1,>=0.4.1",

tests/func/experiments/test_remote.py

@@ -1,3 +1,5 @@
+import logging
+
 import pytest
 from funcy import first
 
@@ -360,3 +362,17 @@ def test_get(tmp_dir, scm, dvc, exp_stage, erepo_dir, use_ref):
             rev=exp_ref.name if use_ref else exp_rev,
         )
         assert (erepo_dir / "params.yaml").read_text().strip() == "foo: 2"
+
+
+def test_push_pull_invalid_workspace(
+    tmp_dir, scm, dvc, git_upstream, exp_stage, local_remote, caplog
+):
+    dvc.experiments.run()
+
+    with open("dvc.yaml", mode="a") as f:
+        f.write("\ninvalid")
+
+    with caplog.at_level(logging.WARNING, logger="dvc"):
+        dvc.experiments.push(git_upstream.remote, push_cache=True)
+        dvc.experiments.pull(git_upstream.remote, pull_cache=True)
+        assert "failed to collect" not in caplog.text

tests/func/test_du.py

@@ -0,0 +1,49 @@
+import os
+
+
+def test_du(tmp_dir, dvc):
+    tmp_dir.gen(
+        {
+            "file": b"file",
+            "dvcfile": b"dvcfile",
+            "dir": {
+                "dirfile": b"dirfile",
+                "subdir": {
+                    "subdirfile": b"subdirfile",
+                },
+                "dvcsubdir": {
+                    "dvcsubdirfile": b"dvcsubdirfile",
+                },
+            },
+        }
+    )
+
+    dvc.add("dvcfile")
+    dvc.add(os.path.join("dir", "dvcsubdir"))
+
+    assert dvc.du(".", "file") == [("file", 4)]
+    assert dvc.du(".", "dvcfile") == [("dvcfile", 7)]
+    assert set(dvc.du(".", "dir/subdir")) == {
+        ("dir/subdir/subdirfile", 10),
+        ("dir/subdir", 10),
+    }
+    assert dvc.du(".", "dir/subdir", summarize=True) == [("dir/subdir", 10)]
+    assert set(dvc.du(".", "dir/dvcsubdir")) == {
+        ("dir/dvcsubdir/dvcsubdirfile", 13),
+        ("dir/dvcsubdir", 13),
+    }
+    assert dvc.du(".", "dir/dvcsubdir", summarize=True) == [("dir/dvcsubdir", 13)]
+    assert set(dvc.du(".", "dir")) == {
+        ("dir/dvcsubdir", 13),
+        ("dir/subdir", 10),
+        ("dir/dirfile", 7),
+        ("dir", 30),
+    }
+    assert dvc.du(".", "dir", summarize=True) == [("dir", 30)]
+    assert set(dvc.du(".", "/")) == {
+        ("/dvcfile", 7),
+        ("/dir", 30),
+        ("/file", 4),
+        ("/", 41),
+    }
+    assert dvc.du(".", "/", summarize=True) == [("/", 41)]

tests/func/test_import_db.py

@@ -11,15 +11,15 @@ serializers = AgateSerializer(data1), AgateSerializer(data2)
 
 
 @pytest.fixture
-def adapter(mocker):
-    m = mocker.patch("dvc.database.get_adapter")
-    adapter = mocker.MagicMock()
-    adapter.query.return_value.__enter__.side_effect = serializers
-    m.return_value.__enter__.return_value = adapter
+def client(mocker):
+    m = mocker.patch("dvc.database.get_client")
+    client = mocker.MagicMock()
+    client.query.return_value.__enter__.side_effect = serializers
+    m.return_value.__enter__.return_value = client
     return m
 
 
-def test_sql(adapter, tmp_dir, dvc):
+def test_sql(client, tmp_dir, dvc):
     stage = dvc.imp_db(
         sql="select * from model", profile="profile", output_format="json"
     )
@@ -56,7 +56,7 @@ def test_sql(adapter, tmp_dir, dvc):
     }
 
 
-def test_sql_conn_string(adapter, tmp_dir, dvc):
+def test_sql_conn_string(client, tmp_dir, dvc):
     with dvc.config.edit(level="local") as conf:
         conf["db"] = {"conn": {"url": "conn"}}
 

tests/func/test_odb.py

@@ -165,7 +165,7 @@ def test_default_cache_type(dvc):
 @pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.")
 @pytest.mark.parametrize("group", [False, True])
 def test_shared_cache(tmp_dir, dvc, group):
-    from dvc.fs import system
+    from dvc_data.hashfile.db.local import umask
 
     if group:
         with dvc.config.edit() as conf:
@@ -178,7 +178,7 @@ def test_shared_cache(tmp_dir, dvc, group):
     tmp_dir.dvc_gen({"file": "file content", "dir": {"file2": "file 2 content"}})
 
     file_mode = oct(0o444)
-    dir_mode = oct(0o2775 if group else (0o777 & ~system.umask))
+    dir_mode = oct(0o2775 if group else (0o777 & ~umask))
     for root, dnames, fnames in os.walk(cache_dir):
         for dname in dnames:
             path = os.path.join(root, dname)

tests/integration/plots/test_plots.py

@@ -11,7 +11,7 @@ from bs4 import BeautifulSoup
 from funcy import first
 
 from dvc.cli import main
-from dvc.render import REVISION_FIELD, VERSION_FIELD
+from dvc.render import ANCHOR_DEFINITIONS, FILENAME, REVISION
 
 JSON_OUT = "vis_data"
 
@@ -102,6 +102,9 @@ def verify_vega(
     html_result,
     json_result,
     split_json_result,
+    title,
+    x_label,
+    y_label,
 ):
     if isinstance(versions, str):
         versions = [versions]
@@ -111,27 +114,60 @@ def verify_vega(
         assert j[0]["type"] == "vega"
         assert set(j[0]["revisions"]) == set(versions)
 
-    assert json_result[0]["datapoints"] == split_json_result[0]["datapoints"]
-    assert set(versions) == set(json_result[0]["datapoints"].keys())
+    assert (
+        json_result[0]["content"]["data"]["values"]
+        == split_json_result[0][ANCHOR_DEFINITIONS]["<DVC_METRIC_DATA>"]
+    )
+
+    assert set(versions) == set(json_result[0]["revisions"])
 
     assert json_result[0]["content"]["data"]["values"]
     assert html_result["data"]["values"]
-    assert split_json_result[0]["content"]["data"]["values"] == "<DVC_METRIC_DATA>"
 
-    def _assert_templates_equal(html_template, filled_template, split_template):
-        # besides data, json and split json should be equal
-        path = ["data", "values"]
+    content_str = json.dumps(split_json_result[0]["content"])
+    assert "<DVC_METRIC_DATA>" in content_str
+    assert "<DVC_METRIC_X_LABEL>" in content_str
+    assert "<DVC_METRIC_Y_LABEL>" in content_str
+
+    def _assert_templates_equal(
+        html_template, filled_template, split_template, title, x_label, y_label
+    ):
+        # besides split anchors, json and split json should be equal
+        paths = [["data", "values"], ["encoding", "color"]]
         tmp1 = deepcopy(html_template)
         tmp2 = deepcopy(filled_template)
-        tmp3 = deepcopy(split_template)
-        dpath.set(tmp1, path, {})
-        dpath.set(tmp2, path, {})
-        dpath.set(tmp3, path, {})
+        tmp3 = json.loads(
+            json.dumps(split_template)
+            .replace('"<DVC_METRIC_PLOT_HEIGHT>"', "300")
+            .replace('"<DVC_METRIC_PLOT_WIDTH>"', "300")
+            .replace("<DVC_METRIC_TITLE>", title)
+            .replace("<DVC_METRIC_X_LABEL>", x_label)
+            .replace("<DVC_METRIC_Y_LABEL>", y_label)
+            .replace(
+                '"<DVC_METRIC_ZOOM_AND_PAN>"',
+                json.dumps(
+                    {
+                        "name": "grid",
+                        "select": "interval",
+                        "bind": "scales",
+                    }
+                ),
+            )
+        )
+        for path in paths:
+            dpath.set(tmp1, path, {})
+            dpath.set(tmp2, path, {})
+            dpath.set(tmp3, path, {})
 
         assert tmp1 == tmp2 == tmp3
 
     _assert_templates_equal(
-        html_result, json_result[0]["content"], split_json_result[0]["content"]
+        html_result,
+        json_result[0]["content"],
+        split_json_result[0]["content"],
+        title,
+        x_label,
+        y_label,
     )
 
 
@@ -193,17 +229,13 @@ def test_repo_with_plots(tmp_dir, scm, dvc, capsys, run_copy_metrics, repo_with_
     ] == _update_datapoints(
         linear_v1,
         {
-            VERSION_FIELD: {
-                "revision": "workspace",
-                "filename": "linear.json",
-                "field": "y",
-            },
+            REVISION: "workspace",
         },
     )
     assert html_result["linear.json"]["data"]["values"] == _update_datapoints(
         linear_v1,
         {
-            REVISION_FIELD: "workspace",
+            REVISION: "workspace",
         },
     )
     assert json_data["confusion.json"][0]["content"]["data"][
@@ -211,27 +243,29 @@ def test_repo_with_plots(tmp_dir, scm, dvc, capsys, run_copy_metrics, repo_with_
     ] == _update_datapoints(
         confusion_v1,
         {
-            VERSION_FIELD: {
-                "revision": "workspace",
-                "filename": "confusion.json",
-                "field": "actual",
-            },
+            REVISION: "workspace",
         },
     )
     assert html_result["confusion.json"]["data"]["values"] == _update_datapoints(
         confusion_v1,
         {
-            REVISION_FIELD: "workspace",
+            REVISION: "workspace",
         },
     )
     verify_image(tmp_dir, "workspace", "image.png", image_v1, html_path, json_data)
 
-    for plot in ["linear.json", "confusion.json"]:
+    for plot, title, x_label, y_label in [
+        ("linear.json", "linear", "x", "y"),
+        ("confusion.json", "confusion matrix", "predicted", "actual"),
+    ]:
         verify_vega(
             "workspace",
             html_result[plot],
             json_data[plot],
             split_json_data[plot],
+            title,
+            x_label,
+            y_label,
         )
 
     verify_vega_props("confusion.json", json_data, **confusion_props)
@@ -250,12 +284,18 @@ def test_repo_with_plots(tmp_dir, scm, dvc, capsys, run_copy_metrics, repo_with_
     verify_image(tmp_dir, "workspace", "image.png", image_v2, html_path, json_data)
     verify_image(tmp_dir, "HEAD", "image.png", image_v1, html_path, json_data)
 
-    for plot in ["linear.json", "confusion.json"]:
+    for plot, title, x_label, y_label in [
+        ("linear.json", "linear", "x", "y"),
+        ("confusion.json", "confusion matrix", "predicted", "actual"),
+    ]:
         verify_vega(
             ["HEAD", "workspace"],
             html_result[plot],
             json_data[plot],
             split_json_data[plot],
+            title,
+            x_label,
+            y_label,
         )
     verify_vega_props("confusion.json", json_data, **confusion_props)
     path = tmp_dir / "subdir"
@@ -277,31 +317,23 @@ def test_repo_with_plots(tmp_dir, scm, dvc, capsys, run_copy_metrics, repo_with_
         ] == _update_datapoints(
             linear_v2,
             {
-                VERSION_FIELD: {
-                    "revision": "workspace",
-                    "filename": "../linear.json",
-                    "field": "y",
-                },
+                REVISION: "workspace",
             },
         ) + _update_datapoints(
             linear_v1,
             {
-                VERSION_FIELD: {
-                    "revision": "HEAD",
-                    "filename": "../linear.json",
-                    "field": "y",
-                },
+                REVISION: "HEAD",
             },
         )
         assert html_result["../linear.json"]["data"]["values"] == _update_datapoints(
             linear_v2,
             {
-                REVISION_FIELD: "workspace",
+                REVISION: "workspace",
             },
         ) + _update_datapoints(
             linear_v1,
             {
-                REVISION_FIELD: "HEAD",
+                REVISION: "HEAD",
             },
         )
         assert json_data["../confusion.json"][0]["content"]["data"][
@@ -309,43 +341,38 @@ def test_repo_with_plots(tmp_dir, scm, dvc, capsys, run_copy_metrics, repo_with_
         ] == _update_datapoints(
             confusion_v2,
             {
-                VERSION_FIELD: {
-                    "revision": "workspace",
-                    "filename": "../confusion.json",
-                    "field": "actual",
-                },
+                REVISION: "workspace",
             },
         ) + _update_datapoints(
             confusion_v1,
             {
-                VERSION_FIELD: {
-                    "revision": "HEAD",
-                    "filename": "../confusion.json",
-                    "field": "actual",
-                },
+                REVISION: "HEAD",
             },
         )
         assert html_result["../confusion.json"]["data"]["values"] == _update_datapoints(
             confusion_v2,
             {
-                REVISION_FIELD: "workspace",
+                REVISION: "workspace",
             },
         ) + _update_datapoints(
             confusion_v1,
             {
-                REVISION_FIELD: "HEAD",
+                REVISION: "HEAD",
             },
         )
 
-        for plot in [
-            "../linear.json",
-            "../confusion.json",
+        for plot, title, x_label, y_label in [
+            ("../linear.json", "linear", "x", "y"),
+            ("../confusion.json", "confusion matrix", "predicted", "actual"),
         ]:
             verify_vega(
                 ["HEAD", "workspace"],
                 html_result[plot],
                 json_data[plot],
                 split_json_data[plot],
+                title,
+                x_label,
+                y_label,
             )
         verify_image(
             path,
@@ -434,7 +461,7 @@ def test_repo_with_config_plots(tmp_dir, capsys, repo_with_config_plots):
     repo_state = repo_with_config_plots()
     plots = next(repo_state)
 
-    html_path, _, __ = call(capsys)
+    html_path, _, split_json_result = call(capsys)
 
     assert os.path.exists(html_path)
     html_result = extract_vega_specs(
@@ -444,20 +471,28 @@ def test_repo_with_config_plots(tmp_dir, capsys, repo_with_config_plots):
             "confusion_train_vs_test",
         ],
     )
+
     ble = _update_datapoints(
         plots["data"]["linear_train.json"],
         {
-            REVISION_FIELD: "linear_train.json",
+            REVISION: "workspace",
+            FILENAME: "linear_train.json",
         },
     ) + _update_datapoints(
         plots["data"]["linear_test.json"],
         {
-            REVISION_FIELD: "linear_test.json",
+            REVISION: "workspace",
+            FILENAME: "linear_test.json",
         },
     )
 
     assert html_result["linear_train_vs_test"]["data"]["values"] == ble
-    # TODO check json results once vscode is able to handle flexible plots
+    assert (
+        split_json_result["data"]["linear_train_vs_test"][0][ANCHOR_DEFINITIONS][
+            "<DVC_METRIC_DATA>"
+        ]
+        == ble
+    )
 
 
 @pytest.mark.vscode
@@ -466,7 +501,7 @@ def test_repo_with_dvclive_plots(tmp_dir, capsys, repo_with_dvclive_plots):
 
     for s in ("show", "diff"):
         _, json_result, split_json_result = call(capsys, subcommand=s)
-        expected_result = {
+        expected_result: Dict[str, Dict[str, list[str]]] = {
             "data": {
                 "dvclive/plots/metrics/metric.tsv": [],
             },

tests/unit/command/test_du.py

@@ -0,0 +1,21 @@
+from dvc.cli import parse_args
+from dvc.commands.du import CmdDU
+
+
+def test_du(mocker):
+    cli_args = parse_args(["du", "myurl", "mypath", "--summarize", "--rev", "myrev"])
+    assert cli_args.func == CmdDU
+
+    cmd = cli_args.func(cli_args)
+    mock_du = mocker.patch("dvc.repo.Repo.du")
+
+    assert cmd.run() == 0
+    mock_du.assert_called_once_with(
+        "myurl",
+        "mypath",
+        rev="myrev",
+        summarize=True,
+        config=None,
+        remote=None,
+        remote_config=None,
+    )

tests/unit/render/test_convert.py

@@ -1,109 +1,87 @@
-from dvc.render import REVISION_FIELD, REVISIONS_KEY, SRC_FIELD, TYPE_KEY, VERSION_FIELD
+import json
+
+import pytest
+
+from dvc.render import ANCHOR_DEFINITIONS, FILENAME, REVISION, REVISIONS, SRC, TYPE_KEY
 from dvc.render.convert import to_json
 
 
 def test_to_json_vega(mocker):
     vega_renderer = mocker.MagicMock()
     vega_renderer.TYPE = "vega"
+    vega_renderer.get_revs.return_value = ["bar", "foo"]
     vega_renderer.get_filled_template.return_value = {"this": "is vega"}
-    vega_renderer.datapoints = [
-        {
-            "x": 1,
-            "y": 2,
-            VERSION_FIELD: {"revision": "foo"},
-            "filename": "foo.json",
-        },
-        {
-            "x": 2,
-            "y": 1,
-            VERSION_FIELD: {"revision": "bar"},
-            "filename": "foo.json",
-        },
-    ]
     result = to_json(vega_renderer)
     assert result[0] == {
         TYPE_KEY: vega_renderer.TYPE,
-        REVISIONS_KEY: ["bar", "foo"],
+        REVISIONS: ["bar", "foo"],
         "content": {"this": "is vega"},
-        "datapoints": {
-            "foo": [
-                {
-                    "x": 1,
-                    "y": 2,
-                    "filename": "foo.json",
-                    VERSION_FIELD: {"revision": "foo"},
-                },
-            ],
-            "bar": [
-                {
-                    "x": 2,
-                    "y": 1,
-                    "filename": "foo.json",
-                    VERSION_FIELD: {"revision": "bar"},
-                },
-            ],
-        },
     }
     vega_renderer.get_filled_template.assert_called()
 
 
+@pytest.mark.vscode
 def test_to_json_vega_split(mocker):
+    revs = ["bar", "foo"]
+    content = json.dumps(
+        {
+            "this": "is split vega",
+            "encoding": {"color": "<DVC_METRIC_COLOR>"},
+            "data": {"values": "<DVC_METRIC_DATA>"},
+        }
+    )
+    anchor_definitions = {
+        "<DVC_METRIC_COLOR>": {
+            "field": "rev",
+            "scale": {
+                "domain": revs,
+                "range": ["#ff0000", "#00ff00"],
+            },
+        },
+        "<DVC_METRIC_DATA>": [
+            {
+                "x": 1,
+                "y": 2,
+                REVISION: "foo",
+                FILENAME: "foo.json",
+            },
+            {
+                "x": 2,
+                "y": 1,
+                REVISION: "bar",
+                FILENAME: "foo.json",
+            },
+        ],
+    }
+
     vega_renderer = mocker.MagicMock()
     vega_renderer.TYPE = "vega"
-    vega_renderer.get_filled_template.return_value = {"this": "is split vega"}
-    vega_renderer.datapoints = [
-        {
-            "x": 1,
-            "y": 2,
-            VERSION_FIELD: {"revision": "foo"},
-            "filename": "foo.json",
-        },
-        {
-            "x": 2,
-            "y": 1,
-            VERSION_FIELD: {"revision": "bar"},
-            "filename": "foo.json",
-        },
-    ]
+    vega_renderer.get_partial_filled_template.return_value = (
+        content,
+        {ANCHOR_DEFINITIONS: anchor_definitions},
+    )
+    vega_renderer.get_revs.return_value = ["bar", "foo"]
+
     result = to_json(vega_renderer, split=True)
     assert result[0] == {
+        ANCHOR_DEFINITIONS: anchor_definitions,
         TYPE_KEY: vega_renderer.TYPE,
-        REVISIONS_KEY: ["bar", "foo"],
-        "content": {"this": "is split vega"},
-        "datapoints": {
-            "foo": [
-                {
-                    "x": 1,
-                    "y": 2,
-                    "filename": "foo.json",
-                    VERSION_FIELD: {"revision": "foo"},
-                }
-            ],
-            "bar": [
-                {
-                    "x": 2,
-                    "y": 1,
-                    "filename": "foo.json",
-                    VERSION_FIELD: {"revision": "bar"},
-                }
-            ],
-        },
+        REVISIONS: revs,
+        "content": content,
     }
-    vega_renderer.get_filled_template.assert_called_with(
-        as_string=False, skip_anchors=["data"]
-    )
+    vega_renderer.get_partial_filled_template.assert_called_once()
 
 
 def test_to_json_image(mocker):
     image_renderer = mocker.MagicMock()
     image_renderer.TYPE = "image"
     image_renderer.datapoints = [
-        {SRC_FIELD: "contentfoo", REVISION_FIELD: "foo"},
-        {SRC_FIELD: "contentbar", REVISION_FIELD: "bar"},
+        {SRC: "contentfoo", REVISION: "foo"},
+        {SRC: "contentbar", REVISION: "bar"},
     ]
     result = to_json(image_renderer)
     assert result[0] == {
-        "url": image_renderer.datapoints[0].get(SRC_FIELD),
-        REVISIONS_KEY: [image_renderer.datapoints[0].get(REVISION_FIELD)],
+        "url": image_renderer.datapoints[0].get(SRC),
+        REVISIONS: [image_renderer.datapoints[0].get(REVISION)],
         TYPE_KEY: image_renderer.TYPE,
     }

tests/unit/render/test_image_converter.py

@@ -1,4 +1,4 @@
-from dvc.render import REVISION_FIELD, SRC_FIELD
+from dvc.render import FILENAME, REVISION, SRC
 from dvc.render.converter.image import ImageConverter
 
 
@@ -8,9 +8,9 @@ def test_image_converter_no_out():
     datapoints, _ = converter.flat_datapoints("r")
 
     assert datapoints[0] == {
-        REVISION_FIELD: "r",
-        "filename": "image.png",
-        SRC_FIELD: converter._encode_image(b"content"),
+        REVISION: "r",
+        FILENAME: "image.png",
+        SRC: converter._encode_image(b"content"),
     }
 
 
@@ -21,9 +21,9 @@ def test_image_converter_with_out(tmp_dir):
     datapoints, _ = converter.flat_datapoints("r")
 
     assert datapoints[0] == {
-        REVISION_FIELD: "r",
-        "filename": "image.png",
-        SRC_FIELD: str(tmp_dir / "foo" / "r_image.png"),
+        REVISION: "r",
+        FILENAME: "image.png",
+        SRC: str(tmp_dir / "foo" / "r_image.png"),
     }
 
     assert (tmp_dir / "foo" / "r_image.png").read_bytes() == b"content"
@@ -37,9 +37,9 @@ def test_image_converter_with_slash_in_revision(tmp_dir):
     datapoints, _ = converter.flat_datapoints("feature/r")
 
     assert datapoints[0] == {
-        REVISION_FIELD: "feature/r",
-        "filename": "image.png",
-        SRC_FIELD: str(tmp_dir / "foo" / "feature_r_image.png"),
+        REVISION: "feature/r",
+        FILENAME: "image.png",
+        SRC: str(tmp_dir / "foo" / "feature_r_image.png"),
     }
 
     assert (tmp_dir / "foo" / "feature_r_image.png").read_bytes() == b"content"

tests/unit/render/test_match.py

@@ -1,9 +1,13 @@
 import pytest
 from funcy import set_in
 
-from dvc.render import VERSION_FIELD
+from dvc.render import FIELD, FILENAME, REVISION
 from dvc.render.converter.vega import VegaConverter
-from dvc.render.match import PlotsData, _squash_plots_properties, match_defs_renderers
+from dvc.render.match import (
+    PlotsData,
+    _squash_plots_properties,
+    match_defs_renderers,
+)
 
 
 @pytest.mark.parametrize(
@@ -158,25 +162,23 @@ def test_match_renderers(M):
     renderer = renderer_with_errors[0]
     assert renderer.datapoints == [
         {
-            VERSION_FIELD: {
-                "revision": "v1",
-                "filename": "file.json",
-                "field": "y",
-            },
+            REVISION: "v1",
+            FILENAME: "file.json",
+            FIELD: "y",
             "x": 1,
             "y": 1,
         },
         {
-            VERSION_FIELD: {
-                "revision": "v1",
-                "filename": "file.json",
-                "field": "y",
-            },
+            REVISION: "v1",
+            FILENAME: "file.json",
+            FIELD: "y",
             "x": 2,
             "y": 2,
         },
     ]
     assert renderer.properties == {
+        "anchors_y_definitions": [{FILENAME: "file.json", FIELD: "y"}],
+        "revs_with_datapoints": ["v1"],
         "title": "plot_id_1",
         "x": "x",
         "y": "y",

tests/unit/render/test_vega_converter.py

@@ -3,7 +3,7 @@ from collections import OrderedDict
 import pytest
 
 from dvc.exceptions import DvcException
-from dvc.render import VERSION_FIELD
+from dvc.render import FIELD, FILENAME, REVISION
 from dvc.render.converter.vega import FieldNotFoundError, VegaConverter, _lists
 
 
@@ -35,23 +35,25 @@ def test_finding_lists(dictionary, expected_result):
                 {
                     "v": 1,
                     "step": 0,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
                 },
                 {
                     "v": 2,
                     "step": 1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
                 },
             ],
-            {"x": "step", "y": "v", "x_label": "step", "y_label": "v"},
+            {
+                "anchors_y_definitions": [{FILENAME: "f", FIELD: "v"}],
+                "x": "step",
+                "y": "v",
+                "x_label": "step",
+                "y_label": "v",
+            },
             id="default_x_y",
         ),
         pytest.param(
@@ -61,23 +63,25 @@ def test_finding_lists(dictionary, expected_result):
                 {
                     "v": 1,
                     "v2": 0.1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                 },
                 {
                     "v": 2,
                     "v2": 0.2,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                 },
             ],
-            {"x": "v", "y": "v2", "x_label": "v", "y_label": "v2"},
+            {
+                "anchors_y_definitions": [{FILENAME: "f", FIELD: "v2"}],
+                "x": "v",
+                "y": "v2",
+                "x_label": "v",
+                "y_label": "v2",
+            },
             id="choose_x_y",
         ),
         pytest.param(
@@ -99,23 +103,25 @@ def test_finding_lists(dictionary, expected_result):
                 {
                     "v": 1,
                     "v2": 0.1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                 },
                 {
                     "v": 2,
                     "v2": 0.2,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                 },
             ],
-            {"x": "v", "y": "v2", "x_label": "x", "y_label": "y"},
+            {
+                "anchors_y_definitions": [{FILENAME: "f", FIELD: "v2"}],
+                "x": "v",
+                "y": "v2",
+                "x_label": "x",
+                "y_label": "y",
+            },
             id="find_in_nested_structure",
         ),
         pytest.param(
@@ -123,44 +129,36 @@ def test_finding_lists(dictionary, expected_result):
             {"y": {"f": ["v", "v2"]}},
             [
                 {
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
                     "dvc_inferred_y_value": 1,
                     "v": 1,
                     "v2": 0.1,
                     "step": 0,
                 },
                 {
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
                     "dvc_inferred_y_value": 2,
                     "v": 2,
                     "v2": 0.2,
                     "step": 1,
                 },
                 {
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                     "dvc_inferred_y_value": 0.1,
                     "v2": 0.1,
                     "v": 1,
                     "step": 0,
                 },
                 {
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                     "v": 2,
                     "v2": 0.2,
                     "dvc_inferred_y_value": 0.2,
@@ -168,6 +166,10 @@ def test_finding_lists(dictionary, expected_result):
                 },
             ],
             {
+                "anchors_y_definitions": [
+                    {FILENAME: "f", FIELD: "v"},
+                    {FILENAME: "f", FIELD: "v2"},
+                ],
                 "x": "step",
                 "y": "dvc_inferred_y_value",
                 "y_label": "y",
@@ -189,47 +191,43 @@ def test_finding_lists(dictionary, expected_result):
                     "z": 3,
                     "v": 1,
                     "step": 0,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
                 },
                 {
                     "dvc_inferred_y_value": 2,
                     "z": 4,
                     "step": 1,
                     "v": 2,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
                 },
                 {
                     "dvc_inferred_y_value": 3,
                     "v": 1,
                     "z": 3,
                     "step": 0,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "z",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "z",
                 },
                 {
                     "dvc_inferred_y_value": 4,
                     "v": 2,
                     "z": 4,
                     "step": 1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "z",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "z",
                 },
             ],
             {
+                "anchors_y_definitions": [
+                    {FILENAME: "f", FIELD: "v"},
+                    {FILENAME: "f", FIELD: "z"},
+                ],
                 "x": "step",
                 "y": "dvc_inferred_y_value",
                 "y_label": "y",
@@ -247,32 +245,35 @@ def test_finding_lists(dictionary, expected_result):
                 {
                     "v": 1,
                     "v2": 0.1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                 },
                 {
                     "v": 2,
                     "v2": 0.2,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                 },
                 {
                     "v": 3,
                     "v2": 0.3,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f2",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f2",
+                    FIELD: "v2",
                 },
             ],
-            {"x": "v", "y": "v2", "x_label": "v", "y_label": "v2"},
+            {
+                "anchors_y_definitions": [
+                    {FILENAME: "f", FIELD: "v2"},
+                    {FILENAME: "f2", FIELD: "v2"},
+                ],
+                "x": "v",
+                "y": "v2",
+                "x_label": "v",
+                "y_label": "v2",
+            },
             id="multi_file_json",
         ),
         pytest.param(
@@ -284,47 +285,43 @@ def test_finding_lists(dictionary, expected_result):
                     "v": 1,
                     "v2": 0.1,
                     "step": 0,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
                 },
                 {
                     "dvc_inferred_y_value": 2,
                     "v": 2,
                     "v2": 0.2,
                     "step": 1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
                 },
                 {
                     "dvc_inferred_y_value": 0.1,
                     "v": 1,
                     "v2": 0.1,
                     "step": 0,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                 },
                 {
                     "dvc_inferred_y_value": 0.2,
                     "v": 2,
                     "v2": 0.2,
                     "step": 1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                 },
             ],
             {
+                "anchors_y_definitions": [
+                    {FILENAME: "f", FIELD: "v"},
+                    {FILENAME: "f", FIELD: "v2"},
+                ],
                 "x": "step",
                 "y": "dvc_inferred_y_value",
                 "x_label": "step",
@@ -344,35 +341,34 @@ def test_finding_lists(dictionary, expected_result):
                     "v": 1,
                     "v2": 0.1,
                     "v3": 0.01,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                 },
                 {
                     "dvc_inferred_y_value": 0.01,
                     "v": 1,
                     "v2": 0.1,
                     "v3": 0.01,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v3",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v3",
                 },
                 {
                     "dvc_inferred_y_value": 0.1,
                     "v": 1,
                     "v2": 0.1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f2",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f2",
+                    FIELD: "v2",
                 },
             ],
             {
+                "anchors_y_definitions": [
+                    {FILENAME: "f", FIELD: "v2"},
+                    {FILENAME: "f", FIELD: "v3"},
+                    {FILENAME: "f2", FIELD: "v2"},
+                ],
                 "x": "v",
                 "y": "dvc_inferred_y_value",
                 "x_label": "v",
@@ -390,23 +386,23 @@ def test_finding_lists(dictionary, expected_result):
                 {
                     "v": 1,
                     "v2": 0.1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                 },
                 {
                     "v": 1,
                     "v2": 0.1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f2",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f2",
+                    FIELD: "v2",
                 },
             ],
             {
+                "anchors_y_definitions": [
+                    {FILENAME: "f", FIELD: "v2"},
+                    {FILENAME: "f2", FIELD: "v2"},
+                ],
                 "x": "v",
                 "y": "v2",
                 "x_label": "v",
@@ -414,6 +410,107 @@ def test_finding_lists(dictionary, expected_result):
             },
             id="multi_file_y_same_prefix",
         ),
+        pytest.param(
+            {
+                "f": {"metric": [{"x1": 1, "v": 0.1}]},
+                "f2": {"metric": [{"x2": 100, "v": 0.1}]},
+            },
+            {"y": {"f": ["v"], "f2": ["v"]}, "x": {"f": "x1", "f2": "x2"}},
+            [
+                {
+                    "x1": 1,
+                    "v": 0.1,
+                    "dvc_inferred_x_value": 1,
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
+                },
+                {
+                    "x2": 100,
+                    "v": 0.1,
+                    "dvc_inferred_x_value": 100,
+                    REVISION: "r",
+                    FILENAME: "f2",
+                    FIELD: "v",
+                },
+            ],
+            {
+                "anchors_y_definitions": [
+                    {FILENAME: "f", FIELD: "v"},
+                    {FILENAME: "f2", FIELD: "v"},
+                ],
+                "x": "dvc_inferred_x_value",
+                "y": "v",
+                "x_label": "x",
+                "y_label": "v",
+            },
+            id="multiple_x_fields",
+        ),
+        pytest.param(
+            {
+                "f": {
+                    "metric": [
+                        {"v": 1, "v2": 0.1, "x1": 100},
+                        {"v": 2, "v2": 0.2, "x1": 1000},
+                    ]
+                },
+                "f2": {"metric": [{"x2": -2}, {"x2": -4}]},
+            },
+            {"y": ["v", "v2"], "x": {"f": "x1", "f2": "x2"}},
+            [
+                {
+                    "dvc_inferred_x_value": 100,
+                    "dvc_inferred_y_value": 1,
+                    "v": 1,
+                    "v2": 0.1,
+                    "x1": 100,
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
+                },
+                {
+                    "dvc_inferred_x_value": 1000,
+                    "dvc_inferred_y_value": 2,
+                    "v": 2,
+                    "v2": 0.2,
+                    "x1": 1000,
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
+                },
+                {
+                    "dvc_inferred_x_value": -2,
+                    "dvc_inferred_y_value": 0.1,
+                    "v": 1,
+                    "v2": 0.1,
+                    "x1": 100,
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
+                },
+                {
+                    "dvc_inferred_x_value": -4,
+                    "dvc_inferred_y_value": 0.2,
+                    "v": 2,
+                    "v2": 0.2,
+                    "x1": 1000,
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
+                },
+            ],
+            {
+                "anchors_y_definitions": [
+                    {FILENAME: "f", FIELD: "v"},
+                    {FILENAME: "f", FIELD: "v2"},
+                ],
+                "x": "dvc_inferred_x_value",
+                "y": "dvc_inferred_y_value",
+                "x_label": "x",
+                "y_label": "y",
+            },
+            id="y_list_x_dict",
+        ),
     ],
 )
 def test_convert(