[pre-commit.ci] pre-commit autoupdate (#10183)

updates:
- [github.com/astral-sh/ruff-pre-commit: v0.1.7 → v0.1.8](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.7...v0.1.8)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/benchmarks.yaml

@@ -10,7 +10,7 @@ jobs:
     name: run benchmarks
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
             python-version: "3.11"
       - uses: iterative/dvc-bench@main

.github/workflows/build.yaml

@@ -23,7 +23,7 @@ jobs:
       with:
         fetch-depth: 0
 
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: 3.8
 
@@ -39,7 +39,7 @@ jobs:
         SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DVC: ${{ env.version }}
     - run: twine check --strict dist/*
 
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
         name: Packages
         path: dist/
@@ -58,7 +58,7 @@ jobs:
       id-token: write
 
     steps:
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v4
       with:
         name: Packages
         path: dist
@@ -83,7 +83,7 @@ jobs:
       id-token: write
 
     steps:
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v4
       with:
         name: Packages
         path: dist

.github/workflows/codeql.yml

@@ -31,10 +31,10 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         queries: security-extended
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/plugin_tests.yaml

@@ -34,7 +34,7 @@ jobs:
         path: ${{ matrix.plugin }}
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.pyv }}
         cache: pip

.github/workflows/tests.yaml

@@ -23,14 +23,14 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-20.04, windows-latest, macos-latest]
-        pyv: ["3.8", "3.9", "3.10", "3.11"]
+        pyv: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.pyv }}
         cache: pip

.pre-commit-config.yaml

@@ -22,7 +22,7 @@ repos:
       - id: sort-simple-yaml
       - id: trailing-whitespace
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.1.5'
+    rev: 'v0.1.8'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

dvc/_debug.py

@@ -23,10 +23,11 @@ def viztracer_profile(
     tracer = viztracer.VizTracer(max_stack_depth=depth, log_async=log_async)
 
     tracer.start()
-    yield
-    tracer.stop()
-
-    tracer.save(path() if callable(path) else path)
+    try:
+        yield
+    finally:
+        tracer.stop()
+        tracer.save(path() if callable(path) else path)
 
 
 @contextmanager
@@ -45,31 +46,33 @@ def yappi_profile(
     yappi.set_clock_type("wall" if wall_clock else "cpu")
 
     yappi.start()
-    yield
-    yappi.stop()
+    try:
+        yield
+    finally:
+        yappi.stop()
 
-    threads = yappi.get_thread_stats()
-    stats = {}
-    if separate_threads:
-        for thread in threads:
-            ctx_id = thread.id
-            stats[ctx_id] = yappi.get_func_stats(ctx_id=ctx_id)
-    else:
-        stats[None] = yappi.get_func_stats()
-
-    fpath = path() if callable(path) else path
-    for ctx_id, st in stats.items():
-        if fpath:
-            out = f"{fpath}-{ctx_id}" if ctx_id is not None else fpath
-            st.save(out, type="callgrind")
+        threads = yappi.get_thread_stats()
+        stats = {}
+        if separate_threads:
+            for thread in threads:
+                ctx_id = thread.id
+                stats[ctx_id] = yappi.get_func_stats(ctx_id=ctx_id)
         else:
-            if ctx_id is not None:
-                print(f"\nThread {ctx_id}")  # noqa: T201
-            st.print_all()
-            if ctx_id is None:
-                threads.print_all()
+            stats[None] = yappi.get_func_stats()
 
-    yappi.clear_stats()
+        fpath = path() if callable(path) else path
+        for ctx_id, st in stats.items():
+            if fpath:
+                out = f"{fpath}-{ctx_id}" if ctx_id is not None else fpath
+                st.save(out, type="callgrind")
+            else:
+                if ctx_id is not None:
+                    print(f"\nThread {ctx_id}")  # noqa: T201
+                st.print_all()
+                if ctx_id is None:
+                    threads.print_all()
+
+        yappi.clear_stats()
 
 
 @contextmanager
@@ -85,13 +88,15 @@ def instrument(html_output=False):
     profiler = Profiler()
 
     profiler.start()
-    yield
-    profiler.stop()
+    try:
+        yield
+    finally:
+        profiler.stop()
 
-    if html_output:
-        profiler.open_in_browser()
-        return
-    print(profiler.output_text(unicode=True, color=True))  # noqa: T201
+        if html_output:
+            profiler.open_in_browser()
+        else:
+            print(profiler.output_text(unicode=True, color=True))  # noqa: T201
 
 
 @contextmanager
@@ -102,13 +107,14 @@ def profile(dump_path: Optional[str] = None):
     prof = cProfile.Profile()
     prof.enable()
 
-    yield
-
-    prof.disable()
-    if not dump_path:
-        prof.print_stats(sort="cumtime")
-        return
-    prof.dump_stats(dump_path)
+    try:
+        yield
+    finally:
+        prof.disable()
+        if dump_path:
+            prof.dump_stats(dump_path)
+        else:
+            prof.print_stats(sort="cumtime")
 
 
 @contextmanager

dvc/cachemgr.py

@@ -27,7 +27,7 @@ def _get_odb(
     cls, config, fs_path = get_cloud_fs(repo.config, **settings)
     fs = fs or cls(**config)
     if prefix:
-        fs_path = fs.path.join(fs_path, *prefix)
+        fs_path = fs.join(fs_path, *prefix)
     if hash_name:
         config["hash_name"] = hash_name
     return get_odb(fs, fs_path, state=repo.state, **config)
@@ -86,7 +86,7 @@ class CacheManager:
         return FileStorage(
             key=(),
             fs=self.local.fs,
-            path=self.local.fs.path.join(self.default_local_cache_dir, self.FS_DIR),
+            path=self.local.fs.join(self.default_local_cache_dir, self.FS_DIR),
         )
 
     def _init_odb(self, schemes):

dvc/cli/parser.py

@@ -19,6 +19,7 @@ from dvc.commands import (
     data_sync,
     destroy,
     diff,
+    du,
     experiments,
     freeze,
     gc,
@@ -42,6 +43,7 @@ from dvc.commands import (
     repro,
     root,
     stage,
+    studio,
     unprotect,
     update,
     version,
@@ -93,6 +95,8 @@ COMMANDS = [
     check_ignore,
     data,
     artifacts,
+    studio,
+    du,
 ]
 
 

dvc/commands/dag.py

@@ -86,7 +86,7 @@ def _collect_targets(repo, target, outs):
             targets.extend([str(out) for out in stage.outs])
             continue
 
-        for out in outs_trie.itervalues(prefix=repo.fs.path.parts(path)):
+        for out in outs_trie.itervalues(prefix=repo.fs.parts(path)):
             targets.extend(str(out))
 
     return targets

dvc/commands/du.py

@@ -0,0 +1,80 @@
+import argparse
+import logging
+
+from dvc.cli import completion
+from dvc.cli.command import CmdBaseNoRepo
+from dvc.cli.utils import DictAction, append_doc_link
+from dvc.ui import ui
+
+logger = logging.getLogger(__name__)
+
+
+class CmdDU(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+        from dvc.utils.humanize import naturalsize
+
+        entries = Repo.du(
+            self.args.url,
+            self.args.path,
+            rev=self.args.rev,
+            summarize=self.args.summarize,
+            config=self.args.config,
+            remote=self.args.remote,
+            remote_config=self.args.remote_config,
+        )
+        ui.table([(naturalsize(size), path) for path, size in entries])
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    DU_HELP = "Show disk usage."
+    du_parser = subparsers.add_parser(
+        "du",
+        parents=[parent_parser],
+        description=append_doc_link(DU_HELP, "du"),
+        help=DU_HELP,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    du_parser.add_argument("url", help="Location of DVC repository")
+    du_parser.add_argument(
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
+    )
+    du_parser.add_argument(
+        "-s",
+        "--summarize",
+        action="store_true",
+        help="Show total disk usage.",
+    )
+    du_parser.add_argument(
+        "--config",
+        type=str,
+        help=(
+            "Path to a config file that will be merged with the config "
+            "in the target repository."
+        ),
+    )
+    du_parser.add_argument(
+        "--remote",
+        type=str,
+        help="Remote name to set as a default in the target repository.",
+    )
+    du_parser.add_argument(
+        "--remote-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help=(
+            "Remote config options to merge with a remote's config (default or one "
+            "specified by '--remote') in the target repository."
+        ),
+    )
+    du_parser.add_argument(
+        "path",
+        nargs="?",
+        help="Path to directory within the repository",
+    ).complete = completion.DIR
+    du_parser.set_defaults(func=CmdDU)

dvc/commands/imp_db.py

@@ -13,7 +13,7 @@ logger = logger.getChild(__name__)
 
 class CmdTestDb(CmdBase):
     def run(self):
-        from dvc.database import get_adapter
+        from dvc.database import get_client
         from dvc.database.dbt_utils import DBT_PROJECT_FILE, is_dbt_project
         from dvc.dependency.db import _get_dbt_config
         from dvc.exceptions import DvcException
@@ -47,7 +47,7 @@ class CmdTestDb(CmdBase):
                     "provide arguments or set a configuration"
                 )
 
-        adapter = get_adapter(conn_config, project_dir=project_dir, **dbt_config)
+        adapter = get_client(conn_config, project_dir=project_dir, **dbt_config)
         with adapter as db:
             ui.write(f"Testing with {db}", styled=True)
 

dvc/commands/plots.py

@@ -59,49 +59,6 @@ def _show_json(
     ui.write_json(compact({"errors": all_errors, "data": data}), highlight=False)
 
 
-def _adjust_vega_renderers(renderers):
-    from dvc.render import REVISION_FIELD, VERSION_FIELD
-    from dvc_render import VegaRenderer
-
-    for r in renderers:
-        if isinstance(r, VegaRenderer):
-            if _data_versions_count(r) > 1:
-                summary = _summarize_version_infos(r)
-                for dp in r.datapoints:
-                    vi = dp.pop(VERSION_FIELD, {})
-                    keys = list(vi.keys())
-                    for key in keys:
-                        if not (len(summary.get(key, set())) > 1):
-                            vi.pop(key)
-                    if vi:
-                        dp["rev"] = "::".join(vi.values())
-            else:
-                for dp in r.datapoints:
-                    dp[REVISION_FIELD] = dp[VERSION_FIELD]["revision"]
-                    dp.pop(VERSION_FIELD, {})
-
-
-def _summarize_version_infos(renderer):
-    from collections import defaultdict
-
-    from dvc.render import VERSION_FIELD
-
-    result = defaultdict(set)
-
-    for dp in renderer.datapoints:
-        for key, value in dp.get(VERSION_FIELD, {}).items():
-            result[key].add(value)
-    return dict(result)
-
-
-def _data_versions_count(renderer):
-    from itertools import product
-
-    summary = _summarize_version_infos(renderer)
-    x = product(summary.get("filename", {None}), summary.get("field", {None}))
-    return len(set(x))
-
-
 class CmdPlots(CmdBase):
     def _func(self, *args, **kwargs):
         raise NotImplementedError
@@ -175,11 +132,10 @@ class CmdPlots(CmdBase):
                 return 0
 
             renderers = [r.renderer for r in renderers_with_errors]
-            _adjust_vega_renderers(renderers)
             if self.args.show_vega:
                 renderer = first(filter(lambda r: r.TYPE == "vega", renderers))
                 if renderer:
-                    ui.write_json(renderer.get_filled_template(as_string=False))
+                    ui.write_json(renderer.get_filled_template())
                 return 0
 
             output_file: Path = (Path.cwd() / out).resolve() / "index.html"

dvc/commands/studio.py

@@ -0,0 +1,175 @@
+import argparse
+import os
+
+from funcy import get_in
+
+from dvc.cli.utils import append_doc_link, fix_subparsers
+from dvc.commands.config import CmdConfig
+from dvc.log import logger
+
+logger = logger.getChild(__name__)
+
+
+class CmdStudioLogin(CmdConfig):
+    def run(self):
+        from dvc_studio_client.auth import StudioAuthError, get_access_token
+
+        from dvc.env import DVC_STUDIO_URL
+        from dvc.ui import ui
+        from dvc.utils.studio import STUDIO_URL
+
+        name = self.args.name
+        hostname = self.args.hostname or os.environ.get(DVC_STUDIO_URL) or STUDIO_URL
+        scopes = self.args.scopes
+
+        try:
+            token_name, access_token = get_access_token(
+                token_name=name,
+                hostname=hostname,
+                scopes=scopes,
+                use_device_code=self.args.no_open,
+                client_name="DVC",
+            )
+        except StudioAuthError as e:
+            ui.error_write(str(e))
+            return 1
+
+        self.save_config(hostname, access_token)
+        ui.write(
+            "Authentication has been successfully completed."
+            "The generated token will now be accessible as"
+            f" {token_name} in the user's Studio profile."
+        )
+        return 0
+
+    def save_config(self, hostname, token):
+        with self.config.edit("global") as conf:
+            conf["studio"]["token"] = token
+            conf["studio"]["url"] = hostname
+
+
+class CmdStudioLogout(CmdConfig):
+    def run(self):
+        from dvc.ui import ui
+
+        with self.config.edit("global") as conf:
+            if not get_in(conf, ["studio", "token"]):
+                ui.error_write("Not logged in to Studio.")
+                return 1
+
+            del conf["studio"]["token"]
+
+        ui.write("Logged out from Studio")
+        return 0
+
+
+class CmdStudioToken(CmdConfig):
+    def run(self):
+        from dvc.ui import ui
+
+        conf = self.config.read("global")
+        token = get_in(conf, ["studio", "token"])
+        if not token:
+            ui.error_write("Not logged in to Studio.")
+            return 1
+
+        ui.write(token)
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    STUDIO_HELP = "Commands to authenticate DVC with Iterative Studio"
+    STUDIO_DESCRIPTION = (
+        "Authenticate DVC with Studio and set the token."
+        " Once this token has been properly configured,\n"
+        " DVC will utilize it for seamlessly sharing live experiments\n"
+        " and sending notifications to Studio regarding any experiments"
+        " that have been pushed."
+    )
+
+    studio_parser = subparsers.add_parser(
+        "studio",
+        parents=[parent_parser],
+        description=append_doc_link(STUDIO_DESCRIPTION, "studio"),
+        help=STUDIO_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    studio_subparser = studio_parser.add_subparsers(
+        dest="cmd",
+        help="Use `DVC studio CMD --help` to display command-specific help.",
+    )
+    fix_subparsers(studio_subparser)
+
+    STUDIO_LOGIN_HELP = "Authenticate DVC with Studio host"
+    STUDIO_LOGIN_DESCRIPTION = (
+        "By default, this command authenticates the DVC with Studio\n"
+        " using default scopes and assigns a random name as the token name."
+    )
+    login_parser = studio_subparser.add_parser(
+        "login",
+        parents=[parent_parser],
+        description=append_doc_link(STUDIO_LOGIN_DESCRIPTION, "studio/login"),
+        help=STUDIO_LOGIN_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    login_parser.add_argument(
+        "-H",
+        "--hostname",
+        action="store",
+        default=None,
+        help="The hostname of the Studio instance to authenticate with.",
+    )
+    login_parser.add_argument(
+        "-s",
+        "--scopes",
+        action="store",
+        default=None,
+        help="The scopes for the authentication token. ",
+    )
+
+    login_parser.add_argument(
+        "-n",
+        "--name",
+        action="store",
+        default=None,
+        help="The name of the authentication token. It will be used to\n"
+        "identify token shown in Studio profile.",
+    )
+
+    login_parser.add_argument(
+        "--no-open",
+        action="store_true",
+        default=False,
+        help="Use authentication flow based on user code.\n"
+        "You will be presented with user code to enter in browser.\n"
+        "DVC will also use this if it cannot launch browser on your behalf.",
+    )
+    login_parser.set_defaults(func=CmdStudioLogin)
+
+    STUDIO_LOGOUT_HELP = "Logout user from Studio"
+    STUDIO_LOGOUT_DESCRIPTION = (
+        "This removes the studio token from your global config.\n"
+    )
+
+    logout_parser = studio_subparser.add_parser(
+        "logout",
+        parents=[parent_parser],
+        description=append_doc_link(STUDIO_LOGOUT_DESCRIPTION, "studio/logout"),
+        help=STUDIO_LOGOUT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    logout_parser.set_defaults(func=CmdStudioLogout)
+
+    STUDIO_TOKEN_HELP = "View the token dvc uses to contact Studio"  # noqa: S105 # nosec B105
+
+    logout_parser = studio_subparser.add_parser(
+        "token",
+        parents=[parent_parser],
+        description=append_doc_link(STUDIO_TOKEN_HELP, "studio/token"),
+        help=STUDIO_TOKEN_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    logout_parser.set_defaults(func=CmdStudioToken)

dvc/config.py

@@ -14,7 +14,7 @@ from .utils.objects import cached_property
 
 if TYPE_CHECKING:
     from dvc.fs import FileSystem
-    from dvc.types import DictStrAny, StrPath
+    from dvc.types import DictStrAny
 
 logger = logger.getChild(__name__)
 
@@ -89,8 +89,8 @@ class Config(dict):
 
     def __init__(
         self,
-        dvc_dir: Optional["StrPath"] = None,
-        local_dvc_dir: Optional["StrPath"] = None,
+        dvc_dir: Optional[str] = None,
+        local_dvc_dir: Optional[str] = None,
         validate: bool = True,
         fs: Optional["FileSystem"] = None,
         config: Optional["DictStrAny"] = None,
@@ -105,7 +105,7 @@ class Config(dict):
         self.fs = fs or self.wfs
 
         if dvc_dir:
-            self.dvc_dir = self.fs.path.abspath(dvc_dir)
+            self.dvc_dir = self.fs.abspath(dvc_dir)
 
         self.local_dvc_dir = local_dvc_dir
         if not fs and not local_dvc_dir:
@@ -145,10 +145,10 @@ class Config(dict):
         }
 
         if self.dvc_dir is not None:
-            files["repo"] = self.fs.path.join(self.dvc_dir, self.CONFIG)
+            files["repo"] = self.fs.join(self.dvc_dir, self.CONFIG)
 
         if self.local_dvc_dir is not None:
-            files["local"] = self.wfs.path.join(self.local_dvc_dir, self.CONFIG_LOCAL)
+            files["local"] = self.wfs.join(self.local_dvc_dir, self.CONFIG_LOCAL)
 
         return files
 
@@ -302,11 +302,11 @@ class Config(dict):
             return path.def_path
 
         if os.path.expanduser(path) != path:
-            return localfs.path.as_posix(path)
+            return localfs.as_posix(path)
 
         if isinstance(path, RelPath) or not os.path.isabs(path):
             path = relpath(path, conf_dir)
-            return localfs.path.as_posix(path)
+            return localfs.as_posix(path)
 
         return path
 

dvc/data_cloud.py

@@ -3,7 +3,6 @@
 from typing import TYPE_CHECKING, Iterable, Optional, Set, Tuple
 
 from dvc.config import NoRemoteError, RemoteConfigError
-from dvc.fs.callbacks import Callback
 from dvc.log import logger
 from dvc.utils.objects import cached_property
 from dvc_data.hashfile.db import get_index
@@ -36,11 +35,9 @@ class Remote:
 
         path = self.path
         if self.worktree:
-            path = self.fs.path.join(
-                path, ".dvc", CacheManager.FILES_DIR, DEFAULT_ALGORITHM
-            )
+            path = self.fs.join(path, ".dvc", CacheManager.FILES_DIR, DEFAULT_ALGORITHM)
         else:
-            path = self.fs.path.join(path, CacheManager.FILES_DIR, DEFAULT_ALGORITHM)
+            path = self.fs.join(path, CacheManager.FILES_DIR, DEFAULT_ALGORITHM)
         return get_odb(self.fs, path, hash_name=DEFAULT_ALGORITHM, **self.config)
 
     @cached_property
@@ -206,11 +203,13 @@ class DataCloud:
         jobs: Optional[int] = None,
         odb: "HashFileDB",
     ) -> "TransferResult":
+        from dvc.fs.callbacks import TqdmCallback
+
         if odb.hash_name == "md5-dos2unix":
             cache = self.repo.cache.legacy
         else:
             cache = self.repo.cache.local
-        with Callback.as_tqdm_callback(
+        with TqdmCallback(
             desc=f"Pushing to {odb.fs.unstrip_protocol(odb.path)}",
             unit="file",
         ) as cb:
@@ -265,11 +264,13 @@ class DataCloud:
         jobs: Optional[int] = None,
         odb: "HashFileDB",
     ) -> "TransferResult":
+        from dvc.fs.callbacks import TqdmCallback
+
         if odb.hash_name == "md5-dos2unix":
             cache = self.repo.cache.legacy
         else:
             cache = self.repo.cache.local
-        with Callback.as_tqdm_callback(
+        with TqdmCallback(
             desc=f"Fetching from {odb.fs.unstrip_protocol(odb.path)}",
             unit="file",
         ) as cb:

dvc/database/__init__.py

@@ -4,19 +4,19 @@ from . import dbt_query, sqla
 from .dbt_models import get_model
 from .serializer import export
 
-Adapter = Union[sqla.SQLAlchemyAdapter, dbt_query.DbtAdapter]
+Client = Union[sqla.SQLAlchemyClient, dbt_query.DbtClient]
 
 
-def get_adapter(
+def get_client(
     config,
     project_dir: Optional[str] = None,
     profile: Optional[str] = None,
     target: Optional[str] = None,
     **kwargs: Any,
-) -> "ContextManager[Adapter]":
+) -> "ContextManager[Client]":
     if config:
-        return sqla.adapter(config, **kwargs)
-    return dbt_query.adapter(project_dir=project_dir, profile=profile, target=target)
+        return sqla.client(config, **kwargs)
+    return dbt_query.client(project_dir=project_dir, profile=profile, target=target)
 
 
-__all__ = ["export", "get_adapter", "get_model", "Adapter"]
+__all__ = ["export", "get_client", "get_model", "Client"]

dvc/database/dbt_query.py

@@ -17,7 +17,7 @@ logger = logger.getChild(__name__)
 
 
 @define
-class DbtAdapter:
+class DbtClient:
     adapter: "BaseAdapter" = field(repr=lambda o: type(o).__qualname__)
     creds: Dict[str, Any] = field(repr=False)
 
@@ -59,12 +59,12 @@ class DbtAdapter:
 
 @contextmanager
 @check_dbt("query")
-def adapter(
+def client(
     project_dir: Optional[str] = None,
     profiles_dir: Optional[str] = None,
     profile: Optional[str] = None,
     target: Optional[str] = None,
-) -> Iterator["DbtAdapter"]:
+) -> Iterator["DbtClient"]:
     from dbt.adapters import factory as adapters_factory
     from dbt.adapters.base.impl import BaseAdapter
 
@@ -85,4 +85,4 @@ def adapter(
         except:  # noqa: E722
             creds = {}
 
-        yield DbtAdapter(adapter, creds)
+        yield DbtClient(adapter, creds)

dvc/database/sqla.py

@@ -26,7 +26,7 @@ def url_from_config(config: Union[str, URL, Dict[str, str]]) -> URL:
 
 
 @dataclass
-class SQLAlchemyAdapter:
+class SQLAlchemyClient:
     engine: Engine
 
     @contextmanager
@@ -61,13 +61,13 @@ def handle_error(url: URL):
 
 
 @contextmanager
-def adapter(
+def client(
     url_or_config: Union[str, URL, Dict[str, str]], **engine_kwargs: Any
-) -> Iterator[SQLAlchemyAdapter]:
+) -> Iterator[SQLAlchemyClient]:
     url = url_from_config(url_or_config)
     with handle_error(url):
         engine = create_engine(url, **engine_kwargs)
     try:
-        yield SQLAlchemyAdapter(engine)
+        yield SQLAlchemyClient(engine)
     finally:
         engine.dispose()

dvc/dependency/base.py

@@ -33,7 +33,7 @@ class Dependency(Output):
         if self.fs.version_aware:
             old_fs_path = self.fs_path
             try:
-                self.fs_path = self.fs.path.version_path(self.fs_path, None)
+                self.fs_path = self.fs.version_path(self.fs_path, None)
                 if self.changed_meta():
                     return {str(self): "update available"}
             finally:
@@ -42,9 +42,9 @@ class Dependency(Output):
 
     def update(self, rev=None):
         if self.fs.version_aware:
-            self.fs_path = self.fs.path.version_path(self.fs_path, rev)
+            self.fs_path = self.fs.version_path(self.fs_path, rev)
             self.meta = self.get_meta()
-            self.fs_path = self.fs.path.version_path(self.fs_path, self.meta.version_id)
+            self.fs_path = self.fs.version_path(self.fs_path, self.meta.version_id)
 
     def download(self, to, jobs=None):
         fs_download(self.fs, self.fs_path, to.fs_path, jobs=jobs)
@@ -52,7 +52,7 @@ class Dependency(Output):
     def save(self):
         super().save()
         if self.fs.version_aware:
-            self.fs_path = self.fs.path.version_path(self.fs_path, self.meta.version_id)
+            self.fs_path = self.fs.version_path(self.fs_path, self.meta.version_id)
 
     def dumpd(self, **kwargs):
         if self.fs.version_aware:

dvc/dependency/db.py

@@ -122,7 +122,7 @@ class DbDependency(AbstractDependency):
         file_format: Optional[str] = None,
         **kwargs: Any,
     ) -> None:
-        from dvc.database import export, get_adapter
+        from dvc.database import export, get_client
 
         db_info = self.info.get(PARAM_DB, {})
         query = db_info.get(self.PARAM_QUERY)
@@ -140,7 +140,7 @@ class DbDependency(AbstractDependency):
             raise DvcException(f"connection {connection} not found in config")
 
         project_dir = self.repo.root_dir
-        with get_adapter(
+        with get_client(
             config, project_dir=project_dir, profile=profile, target=target
         ) as db:
             logger.debug("using adapter: %s", db)

dvc/dependency/param.py

@@ -54,12 +54,15 @@ def read_param_file(
                 continue
         return ret
 
-    from dpath import merge
+    from copy import deepcopy
 
-    for key_path in key_paths:
+    from dpath import merge
+    from funcy import distinct
+
+    for key_path in distinct(key_paths):
         merge(
             ret,
-            dpath.search(config, key_path, separator="."),
+            deepcopy(dpath.search(config, key_path, separator=".")),
             separator=".",
         )
     return ret

dvc/dvcfile.py

@@ -294,7 +294,7 @@ class ProjectFile(FileMixin):
     def resolver(self) -> "DataResolver":
         from .parsing import DataResolver
 
-        wdir = self.repo.fs.path.parent(self.path)
+        wdir = self.repo.fs.parent(self.path)
         return DataResolver(self.repo, wdir, self.contents)
 
     @cached_property

dvc/fs/__init__.py

@@ -25,9 +25,8 @@ from dvc_objects.fs.errors import (  # noqa: F401
     ConfigError,
     RemoteMissingDepsError,
 )
-from dvc_objects.fs.path import Path  # noqa: F401
 
-from .callbacks import Callback
+from .callbacks import Callback  # noqa: F401
 from .data import DataFileSystem  # noqa: F401
 from .dvc import DVCFileSystem
 from .git import GitFileSystem  # noqa: F401
@@ -51,24 +50,23 @@ def download(
 ) -> int:
     from dvc.scm import lfs_prefetch
 
-    with Callback.as_tqdm_callback(
-        desc=f"Downloading {fs.path.name(fs_path)}",
+    from .callbacks import TqdmCallback
+
+    with TqdmCallback(
+        desc=f"Downloading {fs.name(fs_path)}",
         unit="files",
     ) as cb:
         # NOTE: We use dvc-objects generic.copy over fs.get since it makes file
         # download atomic and avoids fsspec glob/regex path expansion.
         if fs.isdir(fs_path):
             from_infos = [
-                path
-                for path in fs.find(fs_path)
-                if not path.endswith(fs.path.flavour.sep)
+                path for path in fs.find(fs_path) if not path.endswith(fs.flavour.sep)
             ]
             if not from_infos:
                 localfs.makedirs(to, exist_ok=True)
                 return 0
             to_infos = [
-                localfs.path.join(to, *fs.path.relparts(info, fs_path))
-                for info in from_infos
+                localfs.join(to, *fs.relparts(info, fs_path)) for info in from_infos
             ]
         else:
             from_infos = [fs_path]

dvc/fs/data.py

@@ -29,6 +29,9 @@ class DataFileSystem(FileSystem):
 
         return _DataFileSystem(**self.fs_args)
 
+    def getcwd(self):
+        return self.fs.getcwd()
+
     def isdvc(self, path, **kwargs):
         return self.fs.isdvc(path, **kwargs)
 

dvc/fs/dvc.py

@@ -4,6 +4,7 @@ import ntpath
 import os
 import posixpath
 import threading
+from collections import deque
 from contextlib import ExitStack, suppress
 from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Type, Union
 
@@ -12,7 +13,6 @@ from funcy import wrap_with
 
 from dvc.log import logger
 from dvc_objects.fs.base import FileSystem
-from dvc_objects.fs.path import Path
 
 from .data import DataFileSystem
 
@@ -60,6 +60,7 @@ def _merge_info(repo, key, fs_info, dvc_info):
     if fs_info:
         ret["type"] = fs_info["type"]
         ret["size"] = fs_info["size"]
+        ret["fs_info"] = fs_info
         isexec = False
         if fs_info["type"] == "file":
             isexec = utils.is_exec(fs_info["mode"])
@@ -69,7 +70,7 @@ def _merge_info(repo, key, fs_info, dvc_info):
 
 
 def _get_dvc_path(dvc_fs, subkey):
-    return dvc_fs.path.join(*subkey) if subkey else ""
+    return dvc_fs.join(*subkey) if subkey else ""
 
 
 class _DVCFileSystem(AbstractFileSystem):
@@ -129,55 +130,88 @@ class _DVCFileSystem(AbstractFileSystem):
             ...    rev="main",
             ... )
         """
-        from pygtrie import Trie
-
         super().__init__()
+        self._repo = repo
+        self._repo_factory = repo_factory
+        self._traverse_subrepos = subrepos
         self._repo_stack = ExitStack()
-        if repo is None:
-            url = url if url is not None else fo
-            repo = self._make_repo(
-                url=url,
-                rev=rev,
-                subrepos=subrepos,
-                config=config,
-                remote=remote,
-                remote_config=remote_config,
-            )
-            assert repo is not None
+        self._repo_kwargs = {
+            "url": url if url is not None else fo,
+            "rev": rev,
+            "subrepos": subrepos,
+            "config": config,
+            "remote": remote,
+            "remote_config": remote_config,
+        }
 
-            repo_factory = repo._fs_conf["repo_factory"]
-            self._repo_stack.enter_context(repo)
+    def getcwd(self):
+        relparts: Tuple[str, ...] = ()
+        assert self.repo is not None
+        if self.repo.fs.isin(self.repo.fs.getcwd(), self.repo.root_dir):
+            relparts = self.repo.fs.relparts(self.repo.fs.getcwd(), self.repo.root_dir)
+        return self.root_marker + self.sep.join(relparts)
 
-        if not repo_factory:
+    @classmethod
+    def join(cls, *parts: str) -> str:
+        return posixpath.join(*parts)
+
+    @classmethod
+    def parts(cls, path: str) -> Tuple[str, ...]:
+        ret = []
+        while True:
+            path, part = posixpath.split(path)
+
+            if part:
+                ret.append(part)
+                continue
+
+            if path:
+                ret.append(path)
+
+            break
+
+        ret.reverse()
+
+        return tuple(ret)
+
+    def normpath(self, path: str) -> str:
+        return posixpath.normpath(path)
+
+    def abspath(self, path: str) -> str:
+        if not posixpath.isabs(path):
+            path = self.join(self.getcwd(), path)
+        return self.normpath(path)
+
+    def relpath(self, path: str, start: Optional[str] = None) -> str:
+        if start is None:
+            start = "."
+        return posixpath.relpath(self.abspath(path), start=self.abspath(start))
+
+    def relparts(self, path: str, start: Optional[str] = None) -> Tuple[str, ...]:
+        return self.parts(self.relpath(path, start=start))
+
+    @functools.cached_property
+    def repo(self):
+        if self._repo:
+            return self._repo
+
+        repo = self._make_repo(**self._repo_kwargs)
+
+        self._repo_stack.enter_context(repo)
+        self._repo = repo
+        return repo
+
+    @functools.cached_property
+    def repo_factory(self):
+        if self._repo_factory:
+            return self._repo_factory
+
+        if self._repo:
             from dvc.repo import Repo
 
-            self.repo_factory: RepoFactory = Repo
-        else:
-            self.repo_factory = repo_factory
+            return Repo
 
-        def _getcwd():
-            relparts: Tuple[str, ...] = ()
-            assert repo is not None
-            if repo.fs.path.isin(repo.fs.path.getcwd(), repo.root_dir):
-                relparts = repo.fs.path.relparts(repo.fs.path.getcwd(), repo.root_dir)
-            return self.root_marker + self.sep.join(relparts)
-
-        self.path = Path(self.sep, getcwd=_getcwd)
-        self.repo = repo
-        self.hash_jobs = repo.fs.hash_jobs
-        self._traverse_subrepos = subrepos
-
-        self._subrepos_trie = Trie()
-        """Keeps track of each and every path with the corresponding repo."""
-
-        key = self._get_key(self.repo.root_dir)
-        self._subrepos_trie[key] = repo
-
-        self._datafss = {}
-        """Keep a datafs instance of each repo."""
-
-        if hasattr(repo, "dvc_dir"):
-            self._datafss[key] = DataFileSystem(index=repo.index.data["repo"])
+        return self.repo._fs_conf["repo_factory"]
 
     @functools.cached_property
     def fsid(self) -> str:
@@ -192,20 +226,43 @@ class _DVCFileSystem(AbstractFileSystem):
 
     def _get_key(self, path: "StrPath") -> Key:
         path = os.fspath(path)
-        parts = self.repo.fs.path.relparts(path, self.repo.root_dir)
+        parts = self.repo.fs.relparts(path, self.repo.root_dir)
         if parts == (os.curdir,):
             return ()
         return parts
 
+    @functools.cached_property
+    def _subrepos_trie(self):
+        """Keeps track of each and every path with the corresponding repo."""
+
+        from pygtrie import Trie
+
+        trie = Trie()
+        key = self._get_key(self.repo.root_dir)
+        trie[key] = self.repo
+        return trie
+
     def _get_key_from_relative(self, path) -> Key:
         path = self._strip_protocol(path)
-        parts = self.path.relparts(path, self.root_marker)
+        parts = self.relparts(path, self.root_marker)
         if parts and parts[0] == os.curdir:
             return parts[1:]
         return parts
 
     def _from_key(self, parts: Key) -> str:
-        return self.repo.fs.path.join(self.repo.root_dir, *parts)
+        return self.repo.fs.join(self.repo.root_dir, *parts)
+
+    @functools.cached_property
+    def _datafss(self):
+        """Keep a datafs instance of each repo."""
+
+        datafss = {}
+
+        if hasattr(self.repo, "dvc_dir"):
+            key = self._get_key(self.repo.root_dir)
+            datafss[key] = DataFileSystem(index=self.repo.index.data["repo"])
+
+        return datafss
 
     @property
     def repo_url(self):
@@ -259,7 +316,7 @@ class _DVCFileSystem(AbstractFileSystem):
 
         from dvc.repo import Repo
 
-        repo_path = self.repo.fs.path.join(dir_path, Repo.DVC_DIR)
+        repo_path = self.repo.fs.join(dir_path, Repo.DVC_DIR)
         return self.repo.fs.isdir(repo_path)
 
     def _get_subrepo_info(
@@ -313,7 +370,7 @@ class _DVCFileSystem(AbstractFileSystem):
             dvc_path = _get_dvc_path(dvc_fs, subkey)
             with suppress(FileNotFoundError, NotADirectoryError):
                 for info in dvc_fs.ls(dvc_path, detail=True):
-                    dvc_infos[dvc_fs.path.name(info["name"])] = info
+                    dvc_infos[dvc_fs.name(info["name"])] = info
                 dvc_exists = True
 
         fs_exists = False
@@ -326,7 +383,7 @@ class _DVCFileSystem(AbstractFileSystem):
                 for info in repo.dvcignore.ls(
                     fs, fs_path, detail=True, ignore_subrepos=ignore_subrepos
                 ):
-                    fs_infos[fs.path.name(info["name"])] = info
+                    fs_infos[fs.name(info["name"])] = info
                 fs_exists = True
             except (FileNotFoundError, NotADirectoryError):
                 pass
@@ -345,7 +402,7 @@ class _DVCFileSystem(AbstractFileSystem):
             if not dvcfiles and _is_dvc_file(name):
                 continue
 
-            entry_path = self.path.join(path, name)
+            entry_path = self.join(path, name)
             info = _merge_info(
                 repo, (*subkey, name), fs_infos.get(name), dvc_infos.get(name)
             )
@@ -393,7 +450,7 @@ class _DVCFileSystem(AbstractFileSystem):
         # NOTE: if some parent in fs_path turns out to be a file, it means
         # that the whole repofs branch doesn't exist.
         if dvc_info and not fs_info:
-            for parent in fs.path.parents(fs_path):
+            for parent in fs.parents(fs_path):
                 try:
                     if fs.info(parent)["type"] != "directory":
                         dvc_info = None
@@ -421,6 +478,45 @@ class _DVCFileSystem(AbstractFileSystem):
         dvc_path = _get_dvc_path(dvc_fs, subkey)
         return dvc_fs.get_file(dvc_path, lpath, **kwargs)
 
+    def du(self, path, total=True, maxdepth=None, withdirs=False, **kwargs):
+        if maxdepth is not None:
+            raise NotImplementedError
+
+        sizes = {}
+        dus = {}
+        todo = deque([self.info(path)])
+        while todo:
+            info = todo.popleft()
+            isdir = info["type"] == "directory"
+            size = info["size"] or 0
+            name = info["name"]
+
+            if not isdir:
+                sizes[name] = size
+                continue
+
+            dvc_info = info.get("dvc_info") or {}
+            fs_info = info.get("fs_info")
+            entry = dvc_info.get("entry")
+            if (
+                dvc_info
+                and not fs_info
+                and entry is not None
+                and entry.size is not None
+            ):
+                dus[name] = entry.size
+                continue
+
+            if withdirs:
+                sizes[name] = size
+
+            todo.extend(self.ls(info["name"], detail=True))
+
+        if total:
+            return sum(sizes.values()) + sum(dus.values())
+
+        return sizes
+
     def close(self):
         self._repo_stack.close()
 
@@ -436,6 +532,9 @@ class DVCFileSystem(FileSystem):
     def fs(self) -> "_DVCFileSystem":
         return _DVCFileSystem(**self.fs_args)
 
+    def getcwd(self):
+        return self.fs.getcwd()
+
     @property
     def fsid(self) -> str:
         return self.fs.fsid
@@ -443,10 +542,6 @@ class DVCFileSystem(FileSystem):
     def isdvc(self, path, **kwargs) -> bool:
         return self.fs.isdvc(path, **kwargs)
 
-    @property
-    def path(self) -> Path:
-        return self.fs.path
-
     @property
     def repo(self) -> "Repo":
         return self.fs.repo

dvc/fs/git.py

@@ -46,9 +46,11 @@ class GitFileSystem(FileSystem):
 
         return FsspecGitFileSystem(**self.fs_args)
 
-    @functools.cached_property
-    def path(self):
-        return self.fs.path
+    def getcwd(self):
+        return self.fs.getcwd()
+
+    def chdir(self, path):
+        self.fs.chdir(path)
 
     @property
     def rev(self) -> str:

dvc/ignore.py

@@ -51,8 +51,8 @@ class DvcIgnorePatterns(DvcIgnore):
 
     @classmethod
     def from_file(cls, path, fs, name):
-        assert fs.path.isabs(path)
-        dirname = fs.path.normpath(fs.path.dirname(path))
+        assert fs.isabs(path)
+        dirname = fs.normpath(fs.dirname(path))
         with fs.open(path, encoding="utf-8") as fobj:
             path_spec_lines = [
                 PatternInfo(line, f"{name}:{line_no + 1}:{line}")
@@ -194,7 +194,7 @@ class DvcIgnoreFilter:
         )
 
     def _get_key(self, path):
-        parts = self.fs.path.relparts(path, self.root_dir)
+        parts = self.fs.relparts(path, self.root_dir)
         if parts == (os.curdir,):
             return ()
         return parts
@@ -204,13 +204,13 @@ class DvcIgnoreFilter:
         old_pattern = trie.longest_prefix(key).value
         matches = old_pattern.matches(dirname, DvcIgnore.DVCIGNORE_FILE, False)
 
-        path = self.fs.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
+        path = self.fs.join(dirname, DvcIgnore.DVCIGNORE_FILE)
         if not matches and self.fs.exists(path):
-            name = self.fs.path.relpath(path, self.root_dir)
+            name = self.fs.relpath(path, self.root_dir)
             new_pattern = DvcIgnorePatterns.from_file(path, self.fs, name)
             if old_pattern:
                 plist, prefix = merge_patterns(
-                    self.fs.path.flavour,
+                    self.fs.flavour,
                     old_pattern.pattern_list,
                     old_pattern.dirname,
                     new_pattern.pattern_list,
@@ -239,7 +239,7 @@ class DvcIgnoreFilter:
                     dnames = []
 
             for dname in dnames:
-                self._update_sub_repo(self.fs.path.join(dirname, dname), ignore_trie)
+                self._update_sub_repo(self.fs.join(dirname, dname), ignore_trie)
 
     def _update_sub_repo(self, path, ignore_trie: Trie):
         from dvc.repo import Repo
@@ -247,18 +247,18 @@ class DvcIgnoreFilter:
         if path == self.root_dir:
             return
 
-        dvc_dir = self.fs.path.join(path, Repo.DVC_DIR)
+        dvc_dir = self.fs.join(path, Repo.DVC_DIR)
         if not self.fs.exists(dvc_dir):
             return
 
-        root, dname = self.fs.path.split(path)
+        root, dname = self.fs.split(path)
         key = self._get_key(root)
         pattern_info = PatternInfo(f"/{dname}/", f"in sub_repo:{dname}")
         new_pattern = DvcIgnorePatterns([pattern_info], root, self.fs.sep)
         old_pattern = ignore_trie.longest_prefix(key).value
         if old_pattern:
             plist, prefix = merge_patterns(
-                self.fs.path.flavour,
+                self.fs.flavour,
                 old_pattern.pattern_list,
                 old_pattern.dirname,
                 new_pattern.pattern_list,
@@ -269,7 +269,7 @@ class DvcIgnoreFilter:
             ignore_trie[key] = new_pattern
 
     def __call__(self, root, dirs, files, ignore_subrepos=True):
-        abs_root = self.fs.path.abspath(root)
+        abs_root = self.fs.abspath(root)
         ignore_pattern = self._get_trie_pattern(
             abs_root, dnames=dirs, ignore_subrepos=ignore_subrepos
         )
@@ -283,7 +283,7 @@ class DvcIgnoreFilter:
         nondirs = []
 
         for entry in fs.ls(path, detail=True, **kwargs):
-            name = fs.path.name(entry["name"])
+            name = fs.name(entry["name"])
             fs_dict[name] = entry
             if entry["type"] == "directory":
                 dirs.append(name)
@@ -338,7 +338,7 @@ class DvcIgnoreFilter:
         else:
             ignores_trie = self._ignores_trie_subrepos
 
-        if not self.fs.path.isin_or_eq(dirname, self.root_dir):
+        if not self.fs.isin_or_eq(dirname, self.root_dir):
             # outside of the repo
             return None
 
@@ -349,12 +349,12 @@ class DvcIgnoreFilter:
             return ignore_pattern
 
         prefix_key = ignores_trie.longest_prefix(key).key or ()
-        prefix = self.fs.path.join(self.root_dir, *prefix_key)
+        prefix = self.fs.join(self.root_dir, *prefix_key)
 
         dirs = list(
             takewhile(
                 lambda path: path != prefix,
-                (parent for parent in localfs.path.parents(dirname)),
+                (parent for parent in localfs.parents(dirname)),
             )
         )
         dirs.reverse()
@@ -370,7 +370,7 @@ class DvcIgnoreFilter:
     ):
         if self._outside_repo(path):
             return False
-        dirname, basename = self.fs.path.split(self.fs.path.normpath(path))
+        dirname, basename = self.fs.split(self.fs.normpath(path))
         ignore_pattern = self._get_trie_pattern(dirname, None, ignore_subrepos)
         if ignore_pattern:
             return ignore_pattern.matches(dirname, basename, is_dir)
@@ -378,7 +378,7 @@ class DvcIgnoreFilter:
 
     def is_ignored_dir(self, path: str, ignore_subrepos: bool = True) -> bool:
         # only used in LocalFileSystem
-        path = self.fs.path.abspath(path)
+        path = self.fs.abspath(path)
         if path == self.root_dir:
             return False
 
@@ -386,18 +386,18 @@ class DvcIgnoreFilter:
 
     def is_ignored_file(self, path: str, ignore_subrepos: bool = True) -> bool:
         # only used in LocalFileSystem
-        path = self.fs.path.abspath(path)
+        path = self.fs.abspath(path)
         return self._is_ignored(path, False, ignore_subrepos=ignore_subrepos)
 
     def _outside_repo(self, path):
-        return not self.fs.path.isin_or_eq(path, self.root_dir)
+        return not self.fs.isin_or_eq(path, self.root_dir)
 
     def check_ignore(self, target):
         # NOTE: can only be used in `dvc check-ignore`, see
         # https://github.com/iterative/dvc/issues/5046
-        full_target = self.fs.path.abspath(target)
+        full_target = self.fs.abspath(target)
         if not self._outside_repo(full_target):
-            dirname, basename = self.fs.path.split(self.fs.path.normpath(full_target))
+            dirname, basename = self.fs.split(self.fs.normpath(full_target))
             pattern = self._get_trie_pattern(dirname)
             if pattern:
                 matches = pattern.matches(

dvc/output.py

@@ -378,7 +378,7 @@ class Output:
         if (
             self.repo
             and self.fs.protocol == "local"
-            and not self.fs.path.isabs(self.def_path)
+            and not self.fs.isabs(self.def_path)
         ):
             self.fs = self.repo.fs
 
@@ -411,7 +411,7 @@ class Output:
         self.remote = remote
 
         if self.fs.version_aware:
-            _, version_id = self.fs.path.coalesce_version(
+            _, version_id = self.fs.coalesce_version(
                 self.def_path, self.meta.version_id
             )
             self.meta.version_id = version_id
@@ -465,7 +465,7 @@ class Output:
             parsed.scheme != "remote"
             and self.stage
             and self.stage.repo.fs == fs
-            and not fs.path.isabs(fs_path)
+            and not fs.isabs(fs_path)
         ):
             # NOTE: we can path either from command line or .dvc file,
             # so we should expect both posix and windows style paths.
@@ -473,9 +473,9 @@ class Output:
             #
             # FIXME: if we have Windows path containing / or posix one with \
             # then we have #2059 bug and can't really handle that.
-            fs_path = fs.path.join(self.stage.wdir, fs_path)
+            fs_path = fs.join(self.stage.wdir, fs_path)
 
-        return fs.path.abspath(fs.path.normpath(fs_path))
+        return fs.abspath(fs.normpath(fs_path))
 
     def __repr__(self):
         return f"{type(self).__name__}: {self.def_path!r}"
@@ -491,14 +491,14 @@ class Output:
         ):
             return str(self.def_path)
 
-        if not self.fs.path.isin(self.fs_path, self.repo.root_dir):
+        if not self.fs.isin(self.fs_path, self.repo.root_dir):
             return self.fs_path
 
-        cur_dir = self.fs.path.getcwd()
-        if self.fs.path.isin(cur_dir, self.repo.root_dir):
-            return self.fs.path.relpath(self.fs_path, cur_dir)
+        cur_dir = self.fs.getcwd()
+        if self.fs.isin(cur_dir, self.repo.root_dir):
+            return self.fs.relpath(self.fs_path, cur_dir)
 
-        return self.fs.path.relpath(self.fs_path, self.repo.root_dir)
+        return self.fs.relpath(self.fs_path, self.repo.root_dir)
 
     def clear(self):
         self.hash_info = HashInfo.from_dict({})
@@ -515,10 +515,10 @@ class Output:
         if urlparse(self.def_path).scheme == "remote":
             return False
 
-        if self.fs.path.isabs(self.def_path):
+        if self.fs.isabs(self.def_path):
             return False
 
-        return self.repo and self.fs.path.isin(
+        return self.repo and self.fs.isin(
             self.fs_path,
             self.repo.root_dir,
         )
@@ -607,11 +607,11 @@ class Output:
     def index_key(self) -> Tuple[str, "DataIndexKey"]:
         if self.is_in_repo:
             workspace = "repo"
-            key = self.repo.fs.path.relparts(self.fs_path, self.repo.root_dir)
+            key = self.repo.fs.relparts(self.fs_path, self.repo.root_dir)
         else:
             workspace = self.fs.protocol
-            no_drive = self.fs.path.flavour.splitdrive(self.fs_path)[1]
-            key = self.fs.path.parts(no_drive)[1:]
+            no_drive = self.fs.flavour.splitdrive(self.fs_path)[1]
+            key = self.fs.parts(no_drive)[1:]
         return workspace, key
 
     def changed_checksum(self):
@@ -786,7 +786,7 @@ class Output:
                     self.hash_name,
                     ignore=self.dvcignore,
                 )
-                with Callback.as_tqdm_callback(
+                with TqdmCallback(
                     desc=f"Committing {self} to cache",
                     unit="file",
                 ) as cb:
@@ -799,7 +799,7 @@ class Output:
                         callback=cb,
                     )
             if relink:
-                rel = self.fs.path.relpath(filter_info or self.fs_path)
+                rel = self.fs.relpath(filter_info or self.fs_path)
                 with CheckoutCallback(desc=f"Checking out {rel}", unit="files") as cb:
                     self._checkout(
                         filter_info or self.fs_path,
@@ -814,7 +814,7 @@ class Output:
                 self.set_exec()
 
     def _commit_granular_dir(self, filter_info, hardlink) -> Optional["HashFile"]:
-        prefix = self.fs.path.parts(self.fs.path.relpath(filter_info, self.fs_path))
+        prefix = self.fs.parts(self.fs.relpath(filter_info, self.fs_path))
         staging, _, obj = self._build(
             self.cache,
             self.fs_path,
@@ -826,7 +826,7 @@ class Output:
         save_obj = obj.filter(prefix)
         assert isinstance(save_obj, Tree)
         checkout_obj = save_obj.get_obj(self.cache, prefix)
-        with Callback.as_tqdm_callback(
+        with TqdmCallback(
             desc=f"Committing {self} to cache",
             unit="file",
         ) as cb:
@@ -862,7 +862,7 @@ class Output:
             ret.update(split_file_meta_from_cloud(meta_d))
 
         if self.is_in_repo:
-            path = self.fs.path.as_posix(relpath(self.fs_path, self.stage.wdir))
+            path = self.fs.as_posix(relpath(self.fs_path, self.stage.wdir))
         else:
             path = self.def_path
 
@@ -949,7 +949,7 @@ class Output:
             return None
 
         assert obj
-        fs_path = self.fs.path
+        fs_path = self.fs
         if filter_info and filter_info != self.fs_path:
             prefix = fs_path.relparts(filter_info, self.fs_path)
             assert isinstance(obj, Tree)
@@ -1059,7 +1059,7 @@ class Output:
             upload=upload,
             no_progress_bar=no_progress_bar,
         )
-        with Callback.as_tqdm_callback(
+        with TqdmCallback(
             desc=f"Transferring to {odb.fs.unstrip_protocol(odb.path)}",
             unit="file",
         ) as cb:
@@ -1092,7 +1092,10 @@ class Output:
 
     def unprotect(self):
         if self.exists and self.use_cache:
-            self.cache.unprotect(self.fs_path)
+            with TqdmCallback(
+                size=self.meta.nfiles or -1, desc=f"Unprotecting {self}"
+            ) as callback:
+                self.cache.unprotect(self.fs_path, callback=callback)
 
     def get_dir_cache(self, **kwargs) -> Optional["Tree"]:
         if not self.is_dir_checksum:
@@ -1151,7 +1154,7 @@ class Output:
         assert obj is None or isinstance(obj, Tree)
         if filter_info and filter_info != self.fs_path:
             assert obj
-            prefix = self.fs.path.parts(self.fs.path.relpath(filter_info, self.fs_path))
+            prefix = self.fs.parts(self.fs.relpath(filter_info, self.fs_path))
             return obj.filter(prefix)
         return obj
 
@@ -1294,10 +1297,7 @@ class Output:
     def unstage(self, path: str) -> Tuple["Meta", "Tree"]:
         from pygtrie import Trie
 
-        from dvc_objects.fs.path import Path
-
-        assert isinstance(self.fs.path, Path)
-        rel_key = tuple(self.fs.path.parts(self.fs.path.relpath(path, self.fs_path)))
+        rel_key = tuple(self.fs.parts(self.fs.relpath(path, self.fs_path)))
 
         if self.hash_info:
             tree = self.get_dir_cache()
@@ -1330,11 +1330,8 @@ class Output:
     ) -> Tuple["Meta", "Tree"]:
         from pygtrie import Trie
 
-        from dvc_objects.fs.path import Path
-
-        assert isinstance(self.fs.path, Path)
         append_only = True
-        rel_key = tuple(self.fs.path.parts(self.fs.path.relpath(path, self.fs_path)))
+        rel_key = tuple(self.fs.parts(self.fs.relpath(path, self.fs_path)))
 
         if self.hash_info:
             tree = self.get_dir_cache()
@@ -1435,7 +1432,7 @@ class Output:
 
         assert staging
         assert obj.hash_info
-        with Callback.as_tqdm_callback(
+        with TqdmCallback(
             desc=f"Adding {self} to cache",
             unit="file",
         ) as cb:

dvc/parsing/__init__.py

@@ -143,11 +143,11 @@ class DataResolver:
         self.parsing_config = repo.config.get("parsing", {})
 
         if os.path.isabs(wdir):
-            wdir = fs.path.relpath(wdir)
+            wdir = fs.relpath(wdir)
             wdir = "" if wdir == os.curdir else wdir
 
         self.wdir = wdir
-        self.relpath = fs.path.normpath(fs.path.join(self.wdir, "dvc.yaml"))
+        self.relpath = fs.normpath(fs.join(self.wdir, "dvc.yaml"))
 
         vars_ = d.get(VARS_KWD, [])
         check_interpolations(vars_, VARS_KWD, self.relpath)
@@ -248,7 +248,7 @@ class EntryDefinition:
             wdir = to_str(context.resolve_str(wdir))
         except (ContextError, ParseError) as exc:
             format_and_raise(exc, f"'{self.where}.{name}.wdir'", self.relpath)
-        return self.resolver.fs.path.join(self.wdir, wdir)
+        return self.resolver.fs.join(self.wdir, wdir)
 
     def resolve(self, **kwargs):
         try:

dvc/parsing/context.py

@@ -384,7 +384,7 @@ class Context(CtxDict):
 
     def merge_from(self, fs, item: str, wdir: str, overwrite=False):
         path, _, keys_str = item.partition(":")
-        path = fs.path.normpath(fs.path.join(wdir, path))
+        path = fs.normpath(fs.join(wdir, path))
 
         select_keys = lfilter(bool, keys_str.split(",")) if keys_str else None
         if path in self.imports:
@@ -429,7 +429,7 @@ class Context(CtxDict):
         default: Optional[str] = None,
     ):
         if default:
-            to_import = fs.path.join(wdir, default)
+            to_import = fs.join(wdir, default)
             if fs.exists(to_import):
                 self.merge_from(fs, default, wdir)
             else:

dvc/render/__init__.py

@@ -1,7 +1,8 @@
-INDEX_FIELD = "step"
-REVISION_FIELD = "rev"
-FILENAME_FIELD = "filename"
-VERSION_FIELD = "dvc_data_version_info"
-REVISIONS_KEY = "revisions"
+INDEX = "step"
+REVISION = "rev"
+FILENAME = "filename"
+FIELD = "field"
+REVISIONS = "revisions"
+ANCHOR_DEFINITIONS = "anchor_definitions"
 TYPE_KEY = "type"
-SRC_FIELD = "src"
+SRC = "src"

dvc/render/convert.py

@@ -1,7 +1,6 @@
-from collections import defaultdict
 from typing import Dict, List, Union
 
-from dvc.render import REVISION_FIELD, REVISIONS_KEY, SRC_FIELD, TYPE_KEY, VERSION_FIELD
+from dvc.render import REVISION, REVISIONS, SRC, TYPE_KEY
 from dvc.render.converter.image import ImageConverter
 from dvc.render.converter.vega import VegaConverter
 
@@ -19,39 +18,31 @@ def _get_converter(
     raise ValueError(f"Invalid renderer class {renderer_class}")
 
 
-def _group_by_rev(datapoints):
-    grouped = defaultdict(list)
-    for datapoint in datapoints:
-        rev = datapoint.get(VERSION_FIELD, {}).get("revision")
-        grouped[rev].append(datapoint)
-    return dict(grouped)
-
-
 def to_json(renderer, split: bool = False) -> List[Dict]:
     if renderer.TYPE == "vega":
-        grouped = _group_by_rev(renderer.datapoints)
+        if not renderer.datapoints:
+            return []
+        revs = renderer.get_revs()
         if split:
-            content = renderer.get_filled_template(
-                skip_anchors=["data"], as_string=False
-            )
+            content, split_content = renderer.get_partial_filled_template()
         else:
-            content = renderer.get_filled_template(as_string=False)
-        if grouped:
-            return [
-                {
-                    TYPE_KEY: renderer.TYPE,
-                    REVISIONS_KEY: sorted(grouped.keys()),
-                    "content": content,
-                    "datapoints": grouped,
-                }
-            ]
-        return []
+            content = renderer.get_filled_template()
+            split_content = {}
+
+        return [
+            {
+                TYPE_KEY: renderer.TYPE,
+                REVISIONS: revs,
+                "content": content,
+                **split_content,
+            }
+        ]
     if renderer.TYPE == "image":
         return [
             {
                 TYPE_KEY: renderer.TYPE,
-                REVISIONS_KEY: [datapoint.get(REVISION_FIELD)],
-                "url": datapoint.get(SRC_FIELD),
+                REVISIONS: [datapoint.get(REVISION)],
+                "url": datapoint.get(SRC),
             }
             for datapoint in renderer.datapoints
         ]

dvc/render/converter/image.py

@@ -2,7 +2,7 @@ import base64
 import os
 from typing import TYPE_CHECKING, Any, Dict, List, Tuple
 
-from dvc.render import FILENAME_FIELD, REVISION_FIELD, SRC_FIELD
+from dvc.render import FILENAME, REVISION, SRC
 
 from . import Converter
 
@@ -58,9 +58,9 @@ class ImageConverter(Converter):
             else:
                 src = self._encode_image(image_data)
             datapoint = {
-                REVISION_FIELD: revision,
-                FILENAME_FIELD: filename,
-                SRC_FIELD: src,
+                REVISION: revision,
+                FILENAME: filename,
+                SRC: src,
             }
             datapoints.append(datapoint)
         return datapoints, properties

dvc/render/converter/vega.py

@@ -4,7 +4,7 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
 from funcy import first, last
 
 from dvc.exceptions import DvcException
-from dvc.render import FILENAME_FIELD, INDEX_FIELD, VERSION_FIELD
+from dvc.render import FIELD, FILENAME, INDEX, REVISION
 
 from . import Converter
 
@@ -112,7 +112,6 @@ class VegaConverter(Converter):
     ):
         super().__init__(plot_id, data, properties)
         self.plot_id = plot_id
-        self.inferred_properties: Dict = {}
 
     def _infer_y_from_data(self):
         if self.plot_id in self.data:
@@ -120,34 +119,38 @@ class VegaConverter(Converter):
                 if all(isinstance(item, dict) for item in lst):
                     datapoint = first(lst)
                     field = last(datapoint.keys())
-                    self.inferred_properties["y"] = {self.plot_id: field}
-                    break
+                    return {self.plot_id: field}
+        return None
 
     def _infer_x_y(self):
         x = self.properties.get("x", None)
         y = self.properties.get("y", None)
 
+        inferred_properties: Dict = {}
+
         # Infer x.
         if isinstance(x, str):
-            self.inferred_properties["x"] = {}
+            inferred_properties["x"] = {}
             # If multiple y files, duplicate x for each file.
             if isinstance(y, dict):
                 for file, fields in y.items():
                     # Duplicate x for each y.
                     if isinstance(fields, list):
-                        self.inferred_properties["x"][file] = [x] * len(fields)
+                        inferred_properties["x"][file] = [x] * len(fields)
                     else:
-                        self.inferred_properties["x"][file] = x
+                        inferred_properties["x"][file] = x
             # Otherwise use plot ID as file.
             else:
-                self.inferred_properties["x"][self.plot_id] = x
+                inferred_properties["x"][self.plot_id] = x
 
         # Infer y.
         if y is None:
-            self._infer_y_from_data()
+            inferred_properties["y"] = self._infer_y_from_data()
         # If y files not provided, use plot ID as file.
         elif not isinstance(y, dict):
-            self.inferred_properties["y"] = {self.plot_id: y}
+            inferred_properties["y"] = {self.plot_id: y}
+
+        return inferred_properties
 
     def _find_datapoints(self):
         result = {}
@@ -182,7 +185,7 @@ class VegaConverter(Converter):
 
         x = properties.get("x", None)
         if not isinstance(x, dict):
-            return INDEX_FIELD
+            return INDEX
 
         fields = {field for _, field in _file_field(x)}
         if len(fields) == 1:
@@ -192,7 +195,7 @@ class VegaConverter(Converter):
     def flat_datapoints(self, revision):  # noqa: C901, PLR0912
         file2datapoints, properties = self.convert()
 
-        props_update = {}
+        props_update: Dict[str, Union[str, List[Dict[str, str]]]] = {}
 
         xs = list(_get_xs(properties, file2datapoints))
 
@@ -200,15 +203,17 @@ class VegaConverter(Converter):
         if not xs:
             x_file, x_field = (
                 None,
-                INDEX_FIELD,
+                INDEX,
             )
         else:
             x_file, x_field = xs[0]
-        props_update["x"] = x_field
+
+        num_xs = len(xs)
+        multiple_x_fields = num_xs > 1 and len({x[1] for x in xs}) > 1
+        props_update["x"] = "dvc_inferred_x_value" if multiple_x_fields else x_field
 
         ys = list(_get_ys(properties, file2datapoints))
 
-        num_xs = len(xs)
         num_ys = len(ys)
         if num_xs > 1 and num_xs != num_ys:
             raise DvcException(
@@ -237,6 +242,14 @@ class VegaConverter(Converter):
         else:
             common_prefix_len = 0
 
+        props_update["anchors_y_definitions"] = [
+            {
+                FILENAME: _get_short_y_file(y_file, common_prefix_len),
+                FIELD: y_field,
+            }
+            for y_file, y_field in ys
+        ]
+
         for i, (y_file, y_field) in enumerate(ys):
             if num_xs > 1:
                 x_file, x_field = xs[i]
@@ -249,15 +262,16 @@ class VegaConverter(Converter):
                     source_field=y_field,
                 )
 
-            if x_field == INDEX_FIELD and x_file is None:
-                _update_from_index(datapoints, INDEX_FIELD)
+            if x_field == INDEX and x_file is None:
+                _update_from_index(datapoints, INDEX)
             else:
                 x_datapoints = file2datapoints.get(x_file, [])
                 try:
                     _update_from_field(
                         datapoints,
-                        field=x_field,
+                        field="dvc_inferred_x_value" if multiple_x_fields else x_field,
                         source_datapoints=x_datapoints,
+                        source_field=x_field,
                     )
                 except IndexError:
                     raise DvcException(  # noqa: B904
@@ -266,15 +280,12 @@ class VegaConverter(Converter):
                         "They have to have same length."
                     )
 
-            y_file_short = y_file[common_prefix_len:].strip("/\\")
             _update_all(
                 datapoints,
                 update_dict={
-                    VERSION_FIELD: {
-                        "revision": revision,
-                        FILENAME_FIELD: y_file_short,
-                        "field": y_field,
-                    }
+                    REVISION: revision,
+                    FILENAME: _get_short_y_file(y_file, common_prefix_len),
+                    FIELD: y_field,
                 },
             )
 
@@ -295,10 +306,10 @@ class VegaConverter(Converter):
         generated datapoints and updated properties. `x`, `y` values and labels
         are inferred and always provided.
         """
-        self._infer_x_y()
+        inferred_properties = self._infer_x_y()
 
         datapoints = self._find_datapoints()
-        properties = {**self.properties, **self.inferred_properties}
+        properties = {**self.properties, **inferred_properties}
 
         properties["y_label"] = self.infer_y_label(properties)
         properties["x_label"] = self.infer_x_label(properties)
@@ -306,6 +317,10 @@ class VegaConverter(Converter):
         return datapoints, properties
 
 
+def _get_short_y_file(y_file, common_prefix_len):
+    return y_file[common_prefix_len:].strip("/\\")
+
+
 def _update_from_field(
     target_datapoints: List[Dict],
     field: str,

dvc/render/match.py

@@ -80,7 +80,7 @@ def match_defs_renderers(  # noqa: C901, PLR0912
     for plot_id, group in plots_data.group_definitions().items():
         plot_datapoints: List[Dict] = []
         props = _squash_plots_properties(group)
-        final_props: Dict = {}
+        first_props: Dict = {}
 
         def_errors: Dict[str, Exception] = {}
         src_errors: DefaultDict[str, Dict[str, Exception]] = defaultdict(dict)
@@ -90,6 +90,7 @@ def match_defs_renderers(  # noqa: C901, PLR0912
         if templates_dir is not None:
             props["template_dir"] = templates_dir
 
+        revs = []
         for rev, inner_id, plot_definition in group:
             plot_sources = infer_data_sources(inner_id, plot_definition)
             definitions_data = plots_data.get_definition_data(plot_sources, rev)
@@ -109,19 +110,24 @@ def match_defs_renderers(  # noqa: C901, PLR0912
 
             try:
                 dps, rev_props = converter.flat_datapoints(rev)
+                if dps and rev not in revs:
+                    revs.append(rev)
             except Exception as e:  # noqa: BLE001
                 logger.warning("In %r, %s", rev, str(e).lower())
                 def_errors[rev] = e
                 continue
 
-            if not final_props and rev_props:
-                final_props = rev_props
+            if not first_props and rev_props:
+                first_props = rev_props
             plot_datapoints.extend(dps)
 
-        if "title" not in final_props:
-            final_props["title"] = renderer_id
+        if "title" not in first_props:
+            first_props["title"] = renderer_id
+
+        if revs:
+            first_props["revs_with_datapoints"] = revs
 
         if renderer_cls is not None:
-            renderer = renderer_cls(plot_datapoints, renderer_id, **final_props)
+            renderer = renderer_cls(plot_datapoints, renderer_id, **first_props)
             renderers.append(RendererWithErrors(renderer, dict(src_errors), def_errors))
     return renderers

dvc/repo/__init__.py

@@ -70,6 +70,7 @@ class Repo:
     from dvc.repo.commit import commit  # type: ignore[misc]
     from dvc.repo.destroy import destroy  # type: ignore[misc]
     from dvc.repo.diff import diff  # type: ignore[misc]
+    from dvc.repo.du import du as _du  # type: ignore[misc]
     from dvc.repo.fetch import fetch  # type: ignore[misc]
     from dvc.repo.freeze import freeze, unfreeze  # type: ignore[misc]
     from dvc.repo.gc import gc  # type: ignore[misc]
@@ -93,6 +94,7 @@ class Repo:
     from .cache import check_missing as cache_check_missing  # type: ignore[misc]
     from .data import status as data_status  # type: ignore[misc]
 
+    du = staticmethod(_du)
     ls = staticmethod(_ls)
     ls_url = staticmethod(_ls_url)
     get = staticmethod(_get)
@@ -112,7 +114,7 @@ class Repo:
         try:
             root_dir = self.find_root(root_dir, fs)
             fs = fs or localfs
-            dvc_dir = fs.path.join(root_dir, self.DVC_DIR)
+            dvc_dir = fs.join(root_dir, self.DVC_DIR)
         except NotDvcRepoError:
             if not uninitialized:
                 raise
@@ -205,7 +207,7 @@ class Repo:
                 self.fs.makedirs(self.tmp_dir, exist_ok=True)
 
                 self.lock = make_lock(
-                    self.fs.path.join(self.tmp_dir, "lock"),
+                    self.fs.join(self.tmp_dir, "lock"),
                     tmp_dir=self.tmp_dir,
                     hardlink_lock=self.config["core"].get("hardlink_lock", False),
                     friendly=True,
@@ -249,7 +251,7 @@ class Repo:
         )
 
     @cached_property
-    def local_dvc_dir(self):
+    def local_dvc_dir(self) -> Optional[str]:
         from dvc.fs import GitFileSystem, LocalFileSystem
 
         if not self.dvc_dir:
@@ -261,10 +263,10 @@ class Repo:
         if not isinstance(self.fs, GitFileSystem):
             return None
 
-        relparts = ()
+        relparts: Tuple[str, ...] = ()
         if self.root_dir != "/":
             # subrepo
-            relparts = self.fs.path.relparts(self.root_dir, "/")
+            relparts = self.fs.relparts(self.root_dir, "/")
 
         dvc_dir = os.path.join(
             self.scm.root_dir,
@@ -393,7 +395,7 @@ class Repo:
             index = subrepo.index.data[workspace]
         else:
             index = self.index.data[workspace]
-            key = self.fs.path.relparts(path, self.root_dir)
+            key = self.fs.relparts(path, self.root_dir)
 
         try:
             return index, index[key]
@@ -409,18 +411,18 @@ class Repo:
 
         fs = fs or localfs
         root = root or os.curdir
-        root_dir = fs.path.abspath(root)
+        root_dir = fs.abspath(root)
 
         if not fs.isdir(root_dir):
             raise NotDvcRepoError(f"directory '{root}' does not exist")
 
         while True:
-            dvc_dir = fs.path.join(root_dir, cls.DVC_DIR)
+            dvc_dir = fs.join(root_dir, cls.DVC_DIR)
             if fs.isdir(dvc_dir):
                 return root_dir
             if isinstance(fs, LocalFileSystem) and os.path.ismount(root_dir):
                 break
-            parent = fs.path.parent(root_dir)
+            parent = fs.parent(root_dir)
             if parent == root_dir:
                 break
             root_dir = parent
@@ -438,7 +440,7 @@ class Repo:
 
         fs = fs or localfs
         root_dir = cls.find_root(root, fs=fs)
-        return fs.path.join(root_dir, cls.DVC_DIR)
+        return fs.join(root_dir, cls.DVC_DIR)
 
     @staticmethod
     def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False) -> "Repo":
@@ -447,7 +449,10 @@ class Repo:
         return init(root_dir=root_dir, no_scm=no_scm, force=force, subdir=subdir)
 
     def unprotect(self, target):
-        return self.cache.repo.unprotect(target)
+        from dvc.fs.callbacks import TqdmCallback
+
+        with TqdmCallback(desc=f"Unprotecting {target}") as callback:
+            return self.cache.repo.unprotect(target, callback=callback)
 
     def _ignore(self):
         flist = [self.config.files["local"]]
@@ -537,19 +542,19 @@ class Repo:
         # using `outs_graph` to ensure graph checks are run
         outs = outs or self.index.outs_graph
 
-        abs_path = self.fs.path.abspath(path)
+        abs_path = self.fs.abspath(path)
         fs_path = abs_path
 
         def func(out):
             def eq(one, two):
                 return one == two
 
-            match = eq if strict else out.fs.path.isin_or_eq
+            match = eq if strict else out.fs.isin_or_eq
 
             if out.protocol == "local" and match(fs_path, out.fs_path):
                 return True
 
-            if recursive and out.fs.path.isin(out.fs_path, fs_path):
+            if recursive and out.fs.isin(out.fs_path, fs_path):
                 return True
 
             return False
@@ -561,7 +566,7 @@ class Repo:
         return matched
 
     def is_dvc_internal(self, path):
-        path_parts = self.fs.path.normpath(path).split(self.fs.sep)
+        path_parts = self.fs.normpath(path).split(self.fs.sep)
         return self.DVC_DIR in path_parts
 
     @cached_property
@@ -608,10 +613,10 @@ class Repo:
         cache_dir = self.config["core"].get("site_cache_dir") or site_cache_dir()
 
         if isinstance(self.fs, GitFileSystem):
-            relparts = ()
+            relparts: Tuple[str, ...] = ()
             if self.root_dir != "/":
                 # subrepo
-                relparts = self.fs.path.relparts(self.root_dir, "/")
+                relparts = self.fs.relparts(self.root_dir, "/")
             root_dir = os.path.join(self.scm.root_dir, *relparts)
         else:
             root_dir = self.root_dir

dvc/repo/add.py

@@ -181,7 +181,7 @@ def _add_transfer(
 
 def _add(stage: "Stage", source: Optional[str] = None, no_commit: bool = False) -> None:
     out = stage.outs[0]
-    path = out.fs.path.abspath(source) if source else None
+    path = out.fs.abspath(source) if source else None
     try:
         stage.add_outs(path, no_commit=no_commit)
     except CacheLinkError:

dvc/repo/artifacts.py

@@ -210,7 +210,8 @@ class Artifacts:
     ) -> Tuple[int, str]:
         from dvc_studio_client.model_registry import get_download_uris
 
-        from dvc.fs import Callback, HTTPFileSystem, generic, localfs
+        from dvc.fs import HTTPFileSystem, generic, localfs
+        from dvc.fs.callbacks import TqdmCallback
 
         logger.debug("Trying to download artifact '%s' via studio", name)
         out = out or os.getcwd()
@@ -228,7 +229,7 @@ class Artifacts:
                 dvc_studio_config=dvc_studio_config,
                 **kwargs,
             ).items():
-                to_info = localfs.path.join(out, path)
+                to_info = localfs.join(out, path)
                 if localfs.exists(to_info) and not force:
                     hint = "\nTo override it, re-run with '--force'."
                     raise FileExistsLocallyError(  # noqa: TRY301
@@ -244,7 +245,7 @@ class Artifacts:
             ) from exc
         fs = HTTPFileSystem()
         jobs = jobs or fs.jobs
-        with Callback.as_tqdm_callback(
+        with TqdmCallback(
             desc=f"Downloading '{name}' from '{repo_url}'",
             unit="files",
         ) as cb:
@@ -253,7 +254,7 @@ class Artifacts:
                 fs, from_infos, localfs, to_infos, callback=cb, batch_size=jobs
             )
 
-        return len(to_infos), relpath(localfs.path.commonpath(to_infos))
+        return len(to_infos), relpath(localfs.commonpath(to_infos))
 
     @classmethod
     def get(  # noqa: PLR0913

dvc/repo/brancher.py

@@ -20,6 +20,7 @@ def brancher(
     all_tags=False,
     all_commits=False,
     all_experiments=False,
+    workspace=True,
     commit_date: Optional[str] = None,
     sha_only=False,
     num=1,
@@ -31,6 +32,7 @@ def brancher(
         all_branches (bool): iterate over all available branches.
         all_commits (bool): iterate over all commits.
         all_tags (bool): iterate over all available tags.
+        workspace (bool): include workspace.
         commit_date (str): Keep experiments from the commits after(include)
                             a certain date. Date must match the extended
                             ISO 8601 format (YYYY-MM-DD).
@@ -58,12 +60,12 @@ def brancher(
     from dvc.fs import LocalFileSystem
 
     repo_root_parts: Tuple[str, ...] = ()
-    if self.fs.path.isin(self.root_dir, self.scm.root_dir):
-        repo_root_parts = self.fs.path.relparts(self.root_dir, self.scm.root_dir)
+    if self.fs.isin(self.root_dir, self.scm.root_dir):
+        repo_root_parts = self.fs.relparts(self.root_dir, self.scm.root_dir)
 
     cwd_parts: Tuple[str, ...] = ()
-    if self.fs.path.isin(self.fs.path.getcwd(), self.scm.root_dir):
-        cwd_parts = self.fs.path.relparts(self.fs.path.getcwd(), self.scm.root_dir)
+    if self.fs.isin(self.fs.getcwd(), self.scm.root_dir):
+        cwd_parts = self.fs.relparts(self.fs.getcwd(), self.scm.root_dir)
 
     saved_fs = self.fs
     saved_root = self.root_dir
@@ -73,7 +75,8 @@ def brancher(
 
     logger.trace("switching fs to workspace")
     self.fs = LocalFileSystem(url=self.root_dir)
-    yield "workspace"
+    if workspace:
+        yield "workspace"
 
     revs = revs.copy() if revs else []
     if "workspace" in revs:
@@ -122,18 +125,18 @@ def _switch_fs(
     logger.trace("switching fs to revision %s", rev[:7])
     assert isinstance(repo.scm, Git)
     fs = GitFileSystem(scm=repo.scm, rev=rev)
-    root_dir = repo.fs.path.join("/", *repo_root_parts)
+    root_dir = repo.fs.join("/", *repo_root_parts)
     if not fs.exists(root_dir):
         raise NotDvcRepoError(f"Commit '{rev[:7]}' does not contain a DVC repo")
 
     repo.fs = fs
     repo.root_dir = root_dir
-    repo.dvc_dir = fs.path.join(root_dir, repo.DVC_DIR)
+    repo.dvc_dir = fs.join(root_dir, repo.DVC_DIR)
     repo._reset()
 
     if cwd_parts:
-        cwd = repo.fs.path.join("/", *cwd_parts)
-        repo.fs.path.chdir(cwd)
+        cwd = repo.fs.join("/", *cwd_parts)
+        repo.fs.chdir(cwd)
 
 
 @contextmanager
@@ -145,12 +148,12 @@ def switch(repo: "Repo", rev: str) -> Iterator[str]:
         rev = resolve_rev(repo.scm, rev)
 
     repo_root_parts: Tuple[str, ...] = ()
-    if repo.fs.path.isin(repo.root_dir, repo.scm.root_dir):
-        repo_root_parts = repo.fs.path.relparts(repo.root_dir, repo.scm.root_dir)
+    if repo.fs.isin(repo.root_dir, repo.scm.root_dir):
+        repo_root_parts = repo.fs.relparts(repo.root_dir, repo.scm.root_dir)
 
     cwd_parts: Tuple[str, ...] = ()
-    if repo.fs.path.isin(repo.fs.path.getcwd(), repo.scm.root_dir):
-        cwd_parts = repo.fs.path.relparts(repo.fs.path.getcwd(), repo.scm.root_dir)
+    if repo.fs.isin(repo.fs.getcwd(), repo.scm.root_dir):
+        cwd_parts = repo.fs.relparts(repo.fs.getcwd(), repo.scm.root_dir)
 
     saved_fs = repo.fs
     saved_root = repo.root_dir

dvc/repo/checkout.py

@@ -81,7 +81,7 @@ def _check_can_delete(
         if cache_fs.exists(cache_path):
             continue
 
-        entry_paths.append(fs.path.join(path, *(entry.key or ())))
+        entry_paths.append(fs.join(path, *(entry.key or ())))
 
     if not entry_paths:
         return
@@ -172,7 +172,7 @@ def checkout(  # noqa: C901
         )
 
         for out_path in out_paths:
-            if self.fs.path.isin_or_eq(dest_path, out_path):
+            if self.fs.isin_or_eq(dest_path, out_path):
                 failed.add(out_path)
 
     with ui.progress(
@@ -195,7 +195,7 @@ def checkout(  # noqa: C901
 
     typ_map = {ADD: "added", DELETE: "deleted", MODIFY: "modified"}
     for key, typ in out_changes.items():
-        out_path = self.fs.path.join(self.root_dir, *key)
+        out_path = self.fs.join(self.root_dir, *key)
 
         if out_path in failed:
             self.fs.remove(out_path, recursive=True)

dvc/repo/commit.py

@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Set
 
 from dvc import prompt
 
@@ -7,6 +7,7 @@ from .scm_context import scm_context
 
 if TYPE_CHECKING:
     from . import Repo
+    from .index import IndexView
 
 
 def _prepare_message(stage, changes):
@@ -88,14 +89,7 @@ def commit_2_to_3(repo: "Repo", dry: bool = False):
         outs_filter=lambda o: o.hash_name == "md5-dos2unix",
         recursive=True,
     )
-    migrated = set()
-    for out in view.outs:
-        dvcfile = out.stage.dvcfile.relpath
-        if isinstance(out.stage.dvcfile, ProjectFile):
-            lockfile = out.stage.dvcfile._lockfile.relpath
-            migrated.add(f"{dvcfile} ({lockfile})")
-        else:
-            migrated.add(dvcfile)
+    migrated = _migrateable_dvcfiles(view)
     if not migrated:
         ui.write("No DVC files in the repo to migrate to the 3.0 format.")
         return
@@ -110,11 +104,41 @@ def commit_2_to_3(repo: "Repo", dry: bool = False):
             for out in stage.filter_outs(filter_info)
             if outs_filter is not None and outs_filter(out)
         }
+        modified = False
         if outs:
             for out in outs:
                 out.update_legacy_hash_name(force=True)
+            modified = True
+        deps = {dep for dep in stage.deps if not stage.is_import and dep.is_in_repo}
+        if deps:
+            for dep in deps:
+                dep.update_legacy_hash_name(force=True)
+            modified = True
+        if modified:
             stage.save(allow_missing=True)
             stage.commit(allow_missing=True, relink=True)
             if not isinstance(stage.dvcfile, ProjectFile):
                 ui.write(f"Updating DVC file '{stage.dvcfile.relpath}'")
             stage.dump(update_pipeline=False)
+
+
+def _migrateable_dvcfiles(view: "IndexView") -> Set[str]:
+    from dvc.dvcfile import ProjectFile
+
+    migrated = set()
+    for stage, filter_info in view._stage_infos:
+        outs_filter = view._outs_filter
+        dvcfile = stage.dvcfile.relpath
+        assert outs_filter
+        if any(outs_filter(out) for out in stage.filter_outs(filter_info)) or (
+            not stage.is_import
+            and any(
+                dep.is_in_repo and dep.hash_name == "md5-dos2unix" for dep in stage.deps
+            )
+        ):
+            if isinstance(stage.dvcfile, ProjectFile):
+                lockfile = stage.dvcfile._lockfile.relpath
+                migrated.add(f"{dvcfile} ({lockfile})")
+            else:
+                migrated.add(dvcfile)
+    return migrated

dvc/repo/data.py

@@ -205,7 +205,7 @@ class Status(TypedDict):
 
 def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> List[str]:
     """Transform files rel. to Git root to DVC root, and drop outside files."""
-    rel = repo.fs.path.relpath(repo.root_dir, repo.scm.root_dir).rstrip("/")
+    rel = repo.fs.relpath(repo.root_dir, repo.scm.root_dir).rstrip("/")
 
     # if we have repo root in a different location than scm's root,
     # i.e. subdir repo, all git_paths need to be transformed rel. to the DVC
@@ -215,11 +215,11 @@ def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> List[str]
         length = len(prefix)
         files = (file[length:] for file in files if file.startswith(prefix))
 
-    start = repo.fs.path.relpath(repo.fs.path.getcwd(), repo.root_dir)
+    start = repo.fs.relpath(repo.fs.getcwd(), repo.root_dir)
     if start in (os.curdir, ""):
         return list(files)
     # we need to convert repo relative paths to curdir relative.
-    return [repo.fs.path.relpath(file, start) for file in files]
+    return [repo.fs.relpath(file, start) for file in files]
 
 
 def status(

dvc/repo/du.py

@@ -0,0 +1,42 @@
+from typing import Any, Dict, Optional, Union
+
+
+def du(
+    url: str,
+    path: Optional[str] = None,
+    rev: Optional[str] = None,
+    summarize: bool = False,
+    config: Union[None, Dict[str, Any], str] = None,
+    remote: Optional[str] = None,
+    remote_config: Optional[dict] = None,
+):
+    from dvc.config import Config
+
+    from . import Repo
+
+    if config and not isinstance(config, dict):
+        config_dict = Config.load_file(config)
+    else:
+        config_dict = None
+
+    with Repo.open(
+        url,
+        rev=rev,
+        subrepos=True,
+        uninitialized=True,
+        config=config_dict,
+        remote=remote,
+        remote_config=remote_config,
+    ) as repo:
+        path = path or ""
+
+        fs = repo.dvcfs
+
+        if summarize or not fs.isdir(path):
+            return [(path, fs.du(path, total=True))]
+
+        ret = [
+            (entry_path, fs.du(entry_path, total=True)) for entry_path in fs.ls(path)
+        ]
+        ret.append((path, sum(entry[1] for entry in ret)))
+        return ret

dvc/repo/experiments/pull.py

@@ -112,4 +112,6 @@ def _pull_cache(
         refs = [refs]
     revs = list(exp_commits(repo.scm, refs))
     logger.debug("dvc fetch experiment '%s'", refs)
-    repo.fetch(jobs=jobs, remote=dvc_remote, run_cache=run_cache, revs=revs)
+    repo.fetch(
+        jobs=jobs, remote=dvc_remote, run_cache=run_cache, revs=revs, workspace=False
+    )

dvc/repo/experiments/push.py

@@ -188,4 +188,6 @@ def _push_cache(
     assert isinstance(repo.scm, Git)
     revs = list(exp_commits(repo.scm, refs))
     logger.debug("dvc push experiment '%s'", refs)
-    return repo.push(jobs=jobs, remote=dvc_remote, run_cache=run_cache, revs=revs)
+    return repo.push(
+        jobs=jobs, remote=dvc_remote, run_cache=run_cache, revs=revs, workspace=False
+    )

dvc/repo/experiments/stash.py

@@ -171,7 +171,7 @@ class ApplyStash(Stash):
         right_fs = self.scm.get_fs(right_rev)
         paths = [path for path in left_fs.find("/") if not right_fs.exists(path)]
         dest_paths = [
-            localfs.path.join(self.scm.root_dir, left_fs.path.relpath(path, "/"))
+            localfs.join(self.scm.root_dir, left_fs.relpath(path, "/"))
             for path in paths
         ]
         for src, dest in zip(paths, dest_paths):

dvc/repo/fetch.py

@@ -32,6 +32,7 @@ def _collect_indexes(  # noqa: PLR0913
     recursive=False,
     all_commits=False,
     revs=None,
+    workspace=True,
     max_size=None,
     types=None,
     config=None,
@@ -62,6 +63,7 @@ def _collect_indexes(  # noqa: PLR0913
         all_branches=all_branches,
         all_tags=all_tags,
         all_commits=all_commits,
+        workspace=workspace,
     ):
         try:
             repo.config.merge(config)
@@ -79,11 +81,11 @@ def _collect_indexes(  # noqa: PLR0913
             idx.data["repo"].onerror = _make_index_onerror(onerror, rev)
 
             indexes[rev or "workspace"] = idx
-        except Exception as exc:
+        except Exception as exc:  # noqa: BLE001
             if onerror:
                 onerror(rev, None, exc)
             collection_exc = exc
-            logger.exception("failed to collect '%s'", rev or "workspace")
+            logger.warning("failed to collect '%s', skipping", rev or "workspace")
 
     if not indexes and collection_exc:
         raise collection_exc
@@ -104,6 +106,7 @@ def fetch(  # noqa: PLR0913
     all_commits=False,
     run_cache=False,
     revs=None,
+    workspace=True,
     max_size=None,
     types=None,
     config=None,
@@ -148,6 +151,7 @@ def fetch(  # noqa: PLR0913
         recursive=recursive,
         all_commits=all_commits,
         revs=revs,
+        workspace=workspace,
         max_size=max_size,
         types=types,
         config=config,
@@ -215,7 +219,7 @@ def _log_unversioned(data: List["DataIndex"]) -> Tuple[List["DataIndex"], int]:
         index.storage_map = fs_index.storage_map
         for key, entry in fs_index.iteritems():
             if entry.meta and not entry.meta.isdir and entry.meta.version_id is None:
-                unversioned.append(fs.unstrip_protocol(fs.path.join(remote.path, *key)))
+                unversioned.append(fs.unstrip_protocol(fs.join(remote.path, *key)))
             else:
                 index[key] = entry
         fs_index.close()

dvc/repo/graph.py

@@ -126,7 +126,7 @@ def build_graph(stages, outs_trie=None):
     outs_trie = outs_trie or build_outs_trie(stages)
 
     for stage in stages:
-        out = outs_trie.shortest_prefix(localfs.path.parts(stage.path)).value
+        out = outs_trie.shortest_prefix(localfs.parts(stage.path)).value
         if out:
             raise StagePathAsOutputError(stage, str(out))
 
@@ -139,7 +139,7 @@ def build_graph(stages, outs_trie=None):
             continue
 
         for dep in stage.deps:
-            dep_key = dep.fs.path.parts(dep.fs_path)
+            dep_key = dep.fs.parts(dep.fs_path)
             overlapping = [n.value for n in outs_trie.prefixes(dep_key)]
             if outs_trie.has_subtrie(dep_key):
                 overlapping.extend(outs_trie.values(prefix=dep_key))
@@ -165,7 +165,7 @@ def build_outs_graph(graph, outs_trie):
         if stage.is_db_import:
             continue
         for dep in stage.deps:
-            dep_key = dep.fs.path.parts(dep.fs_path)
+            dep_key = dep.fs.parts(dep.fs_path)
             overlapping = [n.value for n in outs_trie.prefixes(dep_key)]
             if outs_trie.has_subtrie(dep_key):
                 overlapping.extend(outs_trie.values(prefix=dep_key))

dvc/repo/imports.py

@@ -100,10 +100,10 @@ def save_imports(
     Returns:
         Objects which were downloaded from source location.
     """
+    from dvc.fs.callbacks import TqdmCallback
     from dvc.stage.exceptions import DataSourceChanged
     from dvc_data.index import md5, save
     from dvc_data.index.checkout import apply, compare
-    from dvc_objects.fs.callbacks import Callback
 
     downloaded: Set["HashInfo"] = set()
 
@@ -122,7 +122,7 @@ def save_imports(
             if not cache.fs.exists(cache.path):
                 os.makedirs(cache.path)
             with TemporaryDirectory(dir=cache.path) as tmpdir:
-                with Callback.as_tqdm_callback(
+                with TqdmCallback(
                     desc="Downloading imports from source",
                     unit="files",
                 ) as cb:

dvc/repo/index.py

@@ -97,7 +97,7 @@ def collect_files(
     for root, dirs, files in walk_iter:
         dvcfile_filter = partial(is_dvcfile_and_not_ignored, root)
         for file in filter(dvcfile_filter, files):
-            file_path = fs.path.join(root, file)
+            file_path = fs.join(root, file)
             try:
                 index = Index.from_file(repo, file_path)
             except DvcException as exc:
@@ -201,7 +201,7 @@ def _load_storage_from_out(storage_map, key, out):
                 FileStorage(
                     key,
                     fs_cache.fs,
-                    fs_cache.fs.path.join(
+                    fs_cache.fs.join(
                         fs_cache.path, dep.fs.protocol, tokenize(dep.fs_path)
                     ),
                 )
@@ -414,7 +414,7 @@ class Index:
             by_workspace[workspace].add(key)
 
         for path in _collect_top_level_metrics(self.repo):
-            key = self.repo.fs.path.relparts(path, self.repo.root_dir)
+            key = self.repo.fs.relparts(path, self.repo.root_dir)
             by_workspace["repo"].add(key)
 
         return dict(by_workspace)
@@ -432,7 +432,7 @@ class Index:
             param_paths = chain(param_paths, [default_file])
 
         for path in param_paths:
-            key = self.repo.fs.path.relparts(path, self.repo.root_dir)
+            key = self.repo.fs.relparts(path, self.repo.root_dir)
             by_workspace["repo"].add(key)
 
         return dict(by_workspace)
@@ -451,7 +451,7 @@ class Index:
             by_workspace[workspace].add(key)
 
         for path in self._plot_sources:
-            key = self.repo.fs.path.parts(path)
+            key = self.repo.fs.parts(path)
             by_workspace["repo"].add(key)
 
         return dict(by_workspace)
@@ -701,8 +701,8 @@ class IndexView:
             if not out.use_cache:
                 continue
             workspace, key = out.index_key
-            if filter_info and out.fs.path.isin(filter_info, out.fs_path):
-                key = key + out.fs.path.relparts(filter_info, out.fs_path)
+            if filter_info and out.fs.isin(filter_info, out.fs_path):
+                key = key + out.fs.relparts(filter_info, out.fs_path)
             entry = self._index.data[workspace].get(key)
             if entry and entry.meta and entry.meta.isdir:
                 prefixes[workspace].recursive.add(key)
@@ -718,8 +718,8 @@ class IndexView:
                 continue
 
             workspace, key = out.index_key
-            if filter_info and out.fs.path.isin(filter_info, out.fs_path):
-                key = key + out.fs.path.relparts(filter_info, out.fs_path)
+            if filter_info and out.fs.isin(filter_info, out.fs_path):
+                key = key + out.fs.relparts(filter_info, out.fs_path)
             ret[workspace].add(key)
 
         return dict(ret)
@@ -782,7 +782,7 @@ def build_data_index(  # noqa: C901, PLR0912
     data = DataIndex()
     parents = set()
     for key in index.data_keys.get(workspace, set()):
-        out_path = fs.path.join(path, *key)
+        out_path = fs.join(path, *key)
 
         for key_len in range(1, len(key)):
             parents.add(key[:key_len])
@@ -827,7 +827,7 @@ def build_data_index(  # noqa: C901, PLR0912
             callback.relative_update(1)
 
     for key in parents:
-        parent_path = fs.path.join(path, *key)
+        parent_path = fs.join(path, *key)
         if not fs.exists(parent_path):
             continue
         direntry = DataIndexEntry(key=key, meta=Meta(isdir=True), loaded=True)

dvc/repo/ls.py

@@ -89,7 +89,7 @@ def _ls(
         if not recursive:
             files.update(dirs)
 
-        parts = fs.path.relparts(root, fs_path)
+        parts = fs.relparts(root, fs_path)
         if parts == (".",):
             parts = ()
 

dvc/repo/ls_url.py

@@ -18,7 +18,7 @@ def ls_url(url, *, fs_config=None, recursive=False, config=None):
 
         for info in files.values():
             ls_info = {
-                "path": fs.path.relpath(info["name"], fs_path),
+                "path": fs.relpath(info["name"], fs_path),
                 "isdir": info["type"] == "directory",
                 "size": info.get("size"),
             }

dvc/repo/metrics/show.py

@@ -35,10 +35,10 @@ logger = logger.getChild(__name__)
 def _collect_top_level_metrics(repo: "Repo") -> Iterator[str]:
     top_metrics = repo.index._metrics
     for dvcfile, metrics in top_metrics.items():
-        wdir = repo.fs.path.relpath(repo.fs.path.parent(dvcfile), repo.root_dir)
+        wdir = repo.fs.relpath(repo.fs.parent(dvcfile), repo.root_dir)
         for file in metrics:
-            path = repo.fs.path.join(wdir, as_posix(file))
-            yield repo.fs.path.normpath(path)
+            path = repo.fs.join(wdir, as_posix(file))
+            yield repo.fs.normpath(path)
 
 
 def _extract_metrics(metrics, path: str):
@@ -101,7 +101,7 @@ def _collect_metrics(
 
     if not targets or outs_only:
         outs = metrics_from_target(repo, stages) if stages else repo.index.metrics
-        relpath = repo.fs.path.relpath
+        relpath = repo.fs.relpath
         metrics.extend(relpath(out.fs_path, repo.root_dir) for out in outs)
 
     if not targets and not outs_only and not stages:
@@ -128,8 +128,8 @@ class Result(TypedDict, total=False):
 
 
 def to_relpath(fs: "FileSystem", root_dir: str, d: Result) -> Result:
-    relpath = fs.path.relpath
-    cwd = fs.path.getcwd()
+    relpath = fs.relpath
+    cwd = fs.getcwd()
 
     start = relpath(cwd, root_dir)
     data = d.get("data")
@@ -156,7 +156,7 @@ def _gather_metrics(
     fs = repo.dvcfs
     for fs_path, result in _read_metrics(fs, files, cache=True):
         repo_path = fs_path.lstrip(fs.root_marker)
-        repo_os_path = os.sep.join(fs.path.parts(repo_path))
+        repo_os_path = os.sep.join(fs.parts(repo_path))
         if not isinstance(result, Exception):
             data.update({repo_os_path: FileResult(data=result)})
             continue

dvc/repo/open_repo.py

@@ -71,7 +71,7 @@ def erepo_factory(url, root_dir, cache_config):
         _config = cache_config.copy()
         if os.path.isdir(url):
             fs = fs or localfs
-            repo_path = os.path.join(url, *fs.path.relparts(path, root_dir))
+            repo_path = os.path.join(url, *fs.relparts(path, root_dir))
             _config.update(_get_remote_config(repo_path))
         return Repo(path, fs=fs, config=_config, **_kwargs)
 

dvc/repo/params/show.py

@@ -21,10 +21,10 @@ logger = logger.getChild(__name__)
 def _collect_top_level_params(repo: "Repo") -> Iterator[str]:
     top_params = repo.index._params
     for dvcfile, params in top_params.items():
-        wdir = repo.fs.path.relpath(repo.fs.path.parent(dvcfile), repo.root_dir)
+        wdir = repo.fs.relpath(repo.fs.parent(dvcfile), repo.root_dir)
         for file in params:
-            path = repo.fs.path.join(wdir, as_posix(file))
-            yield repo.fs.path.normpath(path)
+            path = repo.fs.join(wdir, as_posix(file))
+            yield repo.fs.normpath(path)
 
 
 def params_from_target(
@@ -55,7 +55,7 @@ def _collect_params(
 
     if not targets or stages:
         deps = params_from_target(repo, stages) if stages else repo.index.params
-        relpath = repo.fs.path.relpath
+        relpath = repo.fs.relpath
         params.extend(
             {relpath(dep.fs_path, repo.root_dir): list(dep.params)} for dep in deps
         )
@@ -93,7 +93,7 @@ def _collect_vars(repo, params, stages=None) -> Dict:
                 # to reduce noise and duplication, they are skipped
 
                 # `file` is relative
-                abspath = repo.fs.path.abspath(file)
+                abspath = repo.fs.abspath(file)
                 repo_path = repo.dvcfs.from_os_path(abspath)
                 if repo_path in params:
                     continue
@@ -138,7 +138,7 @@ def _gather_params(
     fs = repo.dvcfs
     for fs_path, result in _read_params(fs, files_keypaths, cache=True):
         repo_path = fs_path.lstrip(fs.root_marker)
-        repo_os_path = os.sep.join(fs.path.parts(repo_path))
+        repo_os_path = os.sep.join(fs.parts(repo_path))
         if not isinstance(result, Exception):
             data.update({repo_os_path: FileResult(data=result)})
             continue

dvc/repo/plots/__init__.py

@@ -386,7 +386,7 @@ def _matches(targets, config_file, plot_id):
 
 
 def _normpath(path):
-    # TODO dvcfs.path.normopath normalizes to windows path on Windows
+    # TODO dvcfs.normopath normalizes to windows path on Windows
     # even though other methods work as expected
     import posixpath
 
@@ -399,7 +399,7 @@ def _relpath(fs, path):
     # and invoking from some subdir `dvcfile.relpath` returns strange long
     # relative paths
     # ("../../../../../../dvc.yaml") - investigate
-    return fs.path.relpath(fs.path.join("/", fs.from_os_path(path)), fs.path.getcwd())
+    return fs.relpath(fs.join("/", fs.from_os_path(path)), fs.getcwd())
 
 
 def _collect_output_plots(repo, targets, props, onerror: Optional[Callable] = None):
@@ -413,7 +413,7 @@ def _collect_output_plots(repo, targets, props, onerror: Optional[Callable] = No
         if _matches(targets, config_path, str(plot)):
             unpacked = unpack_if_dir(
                 fs,
-                _normpath(fs.path.join(wdir_relpath, plot.def_path)),
+                _normpath(fs.join(wdir_relpath, plot.def_path)),
                 props={**plot_props, **props},
                 onerror=onerror,
             )
@@ -438,7 +438,7 @@ def _adjust_sources(fs, plot_props, config_dir):
     old_y = new_plot_props.pop("y", {})
     new_y = {}
     for filepath, val in old_y.items():
-        new_y[_normpath(fs.path.join(config_dir, filepath))] = val
+        new_y[_normpath(fs.join(config_dir, filepath))] = val
     new_plot_props["y"] = new_y
     return new_plot_props
 
@@ -452,13 +452,13 @@ def _resolve_definitions(
     onerror: Optional[Callable[[Any], Any]] = None,
 ):
     config_path = os.fspath(config_path)
-    config_dir = fs.path.dirname(config_path)
+    config_dir = fs.dirname(config_path)
     result: Dict[str, Dict] = {}
     for plot_id, plot_props in definitions.items():
         if plot_props is None:
             plot_props = {}
         if _id_is_path(plot_props):
-            data_path = _normpath(fs.path.join(config_dir, plot_id))
+            data_path = _normpath(fs.join(config_dir, plot_id))
             if _matches(targets, config_path, plot_id):
                 unpacked = unpack_if_dir(
                     fs,

dvc/repo/push.py

@@ -46,6 +46,7 @@ def push(  # noqa: PLR0913
     all_commits=False,
     run_cache=False,
     revs=None,
+    workspace=True,
     glob=False,
 ):
     from fsspec.utils import tokenize
@@ -87,6 +88,7 @@ def push(  # noqa: PLR0913
         recursive=recursive,
         all_commits=all_commits,
         revs=revs,
+        workspace=workspace,
         push=True,
     )
 

dvc/repo/stage.py

@@ -219,7 +219,7 @@ class StageLoad:
         self, path: Optional[str] = None, name: Optional[str] = None
     ) -> str:
         if path:
-            return self.repo.fs.path.abspath(path)
+            return self.repo.fs.abspath(path)
 
         path = PROJECT_FILE
         logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
@@ -348,7 +348,7 @@ class StageLoad:
         if recursive and self.fs.isdir(target):
             from dvc.repo.graph import collect_inside_path
 
-            path = self.fs.path.abspath(target)
+            path = self.fs.abspath(target)
             return collect_inside_path(path, graph or self.repo.index.graph)
 
         stages = self.from_target(target, glob=glob)
@@ -393,7 +393,7 @@ class StageLoad:
             if not (recursive and self.fs.isdir(target)):
                 try:
                     (out,) = self.repo.find_outs_by_path(target, strict=False)
-                    return [StageInfo(out.stage, self.fs.path.abspath(target))]
+                    return [StageInfo(out.stage, self.fs.abspath(target))]
                 except OutputNotFoundError:
                     pass
 

dvc/repo/trie.py

@@ -9,7 +9,7 @@ def build_outs_trie(stages):
 
     for stage in stages:
         for out in stage.outs:
-            out_key = out.fs.path.parts(out.fs_path)
+            out_key = out.fs.parts(out.fs_path)
 
             # Check for dup outs
             if out_key in outs:

dvc/repo/worktree.py

@@ -3,7 +3,6 @@ from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Tuple, Union
 
 from funcy import first
 
-from dvc.fs.callbacks import Callback
 from dvc.log import logger
 from dvc.stage.exceptions import StageUpdateError
 
@@ -132,10 +131,11 @@ def _merge_push_meta(  # noqa: C901
             entries.append(entry)
             if entry.meta is not None and entry.meta.isdir:
                 continue
-            fs_path = repo.fs.path.join(repo.root_dir, *subkey)
-            meta, hash_info = old_tree.get(
-                repo.fs.path.relparts(fs_path, out.fs_path)
-            ) or (None, None)
+            fs_path = repo.fs.join(repo.root_dir, *subkey)
+            meta, hash_info = old_tree.get(repo.fs.relparts(fs_path, out.fs_path)) or (
+                None,
+                None,
+            )
             entry.hash_info = hash_info
             if entry.meta:
                 entry.meta.remote = remote
@@ -247,11 +247,12 @@ def _fetch_out_changes(
     remote_index: Union["DataIndex", "DataIndexView"],
     remote: "Remote",
 ):
+    from dvc.fs.callbacks import TqdmCallback
     from dvc_data.index.checkout import apply, compare
 
     old, new = _get_diff_indexes(out, local_index, remote_index)
 
-    with Callback.as_tqdm_callback(
+    with TqdmCallback(
         unit="entry",
         desc="Comparing indexes",
     ) as cb:
@@ -265,9 +266,7 @@ def _fetch_out_changes(
         )
 
     total = len(new)
-    with Callback.as_tqdm_callback(
-        unit="file", desc=f"Updating '{out}'", disable=total == 0
-    ) as cb:
+    with TqdmCallback(unit="file", desc=f"Updating '{out}'", disable=total == 0) as cb:
         cb.set_size(total)
         apply(
             diff,
@@ -356,11 +355,11 @@ def _get_update_diff_index(
             # downloading
             if out.isdir():
                 if not entry.meta.isdir:
-                    fs_path = repo.fs.path.join(repo.root_dir, *entry.key)
+                    fs_path = repo.fs.join(repo.root_dir, *entry.key)
                     tree = out.obj
                     assert isinstance(tree, Tree)
                     _, entry.hash_info = tree.get(  # type: ignore[misc]
-                        repo.fs.path.relparts(fs_path, out.fs_path)
+                        repo.fs.relparts(fs_path, out.fs_path)
                     )
             else:
                 entry.hash_info = out.hash_info

dvc/rwlock.py

@@ -43,10 +43,10 @@ class RWLockFileFormatError(DvcException):
 
 @contextmanager
 def _edit_rwlock(lock_dir, fs, hardlink):
-    path = fs.path.join(lock_dir, RWLOCK_FILE)
+    path = fs.join(lock_dir, RWLOCK_FILE)
 
     rwlock_guard = make_lock(
-        fs.path.join(lock_dir, RWLOCK_LOCK),
+        fs.join(lock_dir, RWLOCK_LOCK),
         tmp_dir=lock_dir,
         hardlink_lock=hardlink,
     )
@@ -82,7 +82,7 @@ def _check_blockers(tmp_dir, lock, info, *, mode, waiters):  # noqa: C901, PLR09
     to_release = defaultdict(list)
     for path, infos in lock[mode].items():
         for waiter_path in waiters:
-            if localfs.path.overlaps(waiter_path, path):
+            if localfs.overlaps(waiter_path, path):
                 break
         else:
             continue

dvc/stage/__init__.py

@@ -656,7 +656,7 @@ class Stage(params.StageParams):
 
     def filter_outs(self, fs_path) -> Iterable["Output"]:
         def _func(o):
-            return o.fs.path.isin_or_eq(fs_path, o.fs_path)
+            return o.fs.isin_or_eq(fs_path, o.fs_path)
 
         return filter(_func, self.outs) if fs_path else self.outs
 

dvc/stage/cache.py

@@ -181,9 +181,9 @@ class StageCache:
 
         path = self._get_cache_path(cache_key, cache_value)
         local_fs = self.repo.cache.legacy.fs
-        parent = local_fs.path.parent(path)
+        parent = local_fs.parent(path)
         self.repo.cache.legacy.makedirs(parent)
-        tmp = local_fs.path.join(parent, fs.utils.tmp_fname())
+        tmp = local_fs.join(parent, fs.utils.tmp_fname())
         assert os.path.exists(parent)
         assert os.path.isdir(parent)
         dump_yaml(tmp, cache)
@@ -231,12 +231,12 @@ class StageCache:
 
     def transfer(self, from_odb, to_odb):
         from dvc.fs import HTTPFileSystem, LocalFileSystem
-        from dvc.fs.callbacks import Callback
+        from dvc.fs.callbacks import TqdmCallback
 
         from_fs = from_odb.fs
         to_fs = to_odb.fs
         func = fs.generic.log_exceptions(fs.generic.copy)
-        runs = from_fs.path.join(from_odb.path, "runs")
+        runs = from_fs.join(from_odb.path, "runs")
 
         http_odb = next(
             (odb for odb in (from_odb, to_odb) if isinstance(odb.fs, HTTPFileSystem)),
@@ -252,21 +252,21 @@ class StageCache:
             return ret
 
         for src in from_fs.find(runs):
-            rel = from_fs.path.relpath(src, from_odb.path)
+            rel = from_fs.relpath(src, from_odb.path)
             if not isinstance(to_fs, LocalFileSystem):
-                rel = from_fs.path.as_posix(rel)
+                rel = from_fs.as_posix(rel)
 
-            dst = to_fs.path.join(to_odb.path, rel)
-            key = to_fs.path.parent(dst)
+            dst = to_fs.join(to_odb.path, rel)
+            key = to_fs.parent(dst)
             # check if any build cache already exists for this key
             # TODO: check if MaxKeys=1 or something like that applies
             # or otherwise this will take a lot of time!
             if to_fs.exists(key) and first(to_fs.find(key)):
                 continue
 
-            src_name = from_fs.path.name(src)
-            parent_name = from_fs.path.name(from_fs.path.parent(src))
-            with Callback.as_tqdm_callback(
+            src_name = from_fs.name(src)
+            parent_name = from_fs.name(from_fs.parent(src))
+            with TqdmCallback(
                 desc=src_name,
                 bytes=True,
             ) as cb:

dvc/stage/utils.py

@@ -184,9 +184,9 @@ def resolve_wdir(wdir, path):
 
 
 def resolve_paths(fs, path, wdir=None):
-    path = fs.path.abspath(path)
+    path = fs.abspath(path)
     wdir = wdir or os.curdir
-    wdir = fs.path.abspath(fs.path.join(fs.path.dirname(path), wdir))
+    wdir = fs.abspath(fs.join(fs.dirname(path), wdir))
     return path, wdir
 
 

dvc/testing/benchmarks/cli/commands/test_get.py

@@ -1,3 +1,7 @@
+import pytest
+
+
+@pytest.mark.flaky(max_runs=3, min_passes=1)
 def test_get(bench_dvc, tmp_dir, scm, dvc, make_dataset, remote):
     dataset = make_dataset(
         cache=False, files=False, dvcfile=True, commit=True, remote=True

dvc/testing/benchmarks/cli/commands/test_import.py

@@ -1,3 +1,7 @@
+import pytest
+
+
+@pytest.mark.flaky(max_runs=3, min_passes=1)
 def test_import(bench_dvc, tmp_dir, scm, dvc, make_dataset, remote):
     dataset = make_dataset(
         cache=False, files=False, dvcfile=True, commit=True, remote=True

dvc/testing/workspace_tests.py

@@ -163,8 +163,8 @@ class TestImportURLVersionAware:
 
 
 def match_files(fs, entries, expected):
-    entries_content = {(fs.path.normpath(d["path"]), d["isdir"]) for d in entries}
-    expected_content = {(fs.path.normpath(d["path"]), d["isdir"]) for d in expected}
+    entries_content = {(fs.normpath(d["path"]), d["isdir"]) for d in entries}
+    expected_content = {(fs.normpath(d["path"]), d["isdir"]) for d in expected}
     assert entries_content == expected_content
 
 
@@ -177,7 +177,7 @@ class TestLsUrl:
         match_files(
             fs,
             result,
-            [{"path": fs.path.join(fs_path, fname), "isdir": False}],
+            [{"path": fs.join(fs_path, fname), "isdir": False}],
         )
 
     def test_dir(self, cloud):

dvc/utils/__init__.py

@@ -287,7 +287,7 @@ def resolve_paths(repo, out, always_local=False):
         # urlparse interprets windows drive letters as URL scheme
         scheme = ""
 
-    if scheme or not localfs.path.isin_or_eq(abspath, repo.root_dir):
+    if scheme or not localfs.isin_or_eq(abspath, repo.root_dir):
         wdir = os.getcwd()
     elif contains_symlink_up_to(dirname, repo.root_dir) or (
         os.path.isdir(abspath) and localfs.is_symlink(abspath)

dvc/utils/plots.py

@@ -9,7 +9,7 @@ def get_plot_id(config_plot_id: str, config_file_path: str = ""):
 
 
 def group_definitions_by_id(
-    definitions: Dict[str, Dict]
+    definitions: Dict[str, Dict],
 ) -> Dict[str, Tuple[str, Dict]]:
     """
     Format ID and extracts plot_definition for each plot.

dvc/utils/serialize/__init__.py

@@ -21,7 +21,7 @@ PARSERS.update(
 
 
 def load_path(fs_path, fs, **kwargs):
-    suffix = fs.path.suffix(fs_path).lower()
+    suffix = fs.suffix(fs_path).lower()
     loader = LOADERS[suffix]
     return loader(fs_path, fs=fs, **kwargs)
 

pyproject.toml

@@ -29,22 +29,23 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
 ]
 dynamic = ["version"]
+
 dependencies = [
     "colorama>=0.3.9",
     "configobj>=5.0.6",
     "distro>=1.3",
     "dpath<3,>=2.1.0",
-    "dvc-data>=2.22.1,<2.23.0",
+    "dvc-data>=3,<3.1",
     "dvc-http>=2.29.0",
-    "dvc-render>=0.3.1,<1",
-    "dvc-studio-client>=0.13.0,<1",
+    "dvc-render>=1.0.0,<2",
+    "dvc-studio-client>=0.17.1,<1",
     "dvc-task>=0.3.0,<1",
     "flatten_dict<1,>=0.4.1",
     # https://github.com/iterative/dvc/issues/9654
     "flufl.lock>=5,<8",
     "funcy>=1.14",
     "grandalf<1,>=0.7",
-    "gto>=1.4.0,<2",
+    "gto>=1.6.0,<2",
     "hydra-core>=1.1",
     "iterative-telemetry>=0.0.7",
     "networkx>=2.5",
@@ -58,7 +59,7 @@ dependencies = [
     "requests>=2.22",
     "rich>=12",
     "ruamel.yaml>=0.17.11",
-    "scmrepo>=1.5.0,<2",
+    "scmrepo>=2.0.2,<3",
     "shortuuid>=0.5",
     "shtab<2,>=1.3.4",
     "tabulate>=0.8.7",
@@ -70,13 +71,13 @@ dependencies = [
 
 [project.optional-dependencies]
 all = ["dvc[azure,gdrive,gs,hdfs,oss,s3,ssh,webdav,webhdfs]"]
-azure = ["dvc-azure>=2.23.0"]
+azure = ["dvc-azure>=3,<4"]
 dev = ["dvc[azure,gdrive,gs,hdfs,lint,oss,s3,ssh,tests,webdav,webhdfs]"]
-gdrive = ["dvc-gdrive==2.20"]
-gs = ["dvc-gs==2.22.1"]
-hdfs = ["dvc-hdfs==2.19"]
+gdrive = ["dvc-gdrive>=3,<4"]
+gs = ["dvc-gs>=3,<4"]
+hdfs = ["dvc-hdfs>=3,<4"]
 lint = [
-    "mypy==1.7.0",
+    "mypy==1.7.1",
     "pandas-stubs",
     "types-colorama",
     "types-psutil",
@@ -86,10 +87,10 @@ lint = [
     "types-toml",
     "types-tqdm",
 ]
-oss = ["dvc-oss>=2.20.0"]
-s3 = ["dvc-s3==2.23.0"]
-ssh = ["dvc-ssh>=2.22.1,<3"]
-ssh_gssapi = ["dvc-ssh[gssapi]>=2.22.1,<3"]
+oss = ["dvc-oss>=3,<4"]
+s3 = ["dvc-s3>=3,<4"]
+ssh = ["dvc-ssh>=3,<4"]
+ssh_gssapi = ["dvc-ssh[gssapi]>=3,<4"]
 testing = [
   "pytest-benchmark[histogram]",
   "pytest-test-utils",
@@ -115,9 +116,9 @@ tests = [
     "sqlalchemy>=1,<3", # optional dependency for `import-db`
     "pandas>=1",
 ]
-webdav = ["dvc-webdav==2.19.1"]
-webhdfs = ["dvc-webhdfs==2.19"]
-webhdfs_kerberos = ["dvc-webhdfs[kerberos]==2.19"]
+webdav = ["dvc-webdav>=3,<4"]
+webhdfs = ["dvc-webhdfs>=3,<4"]
+webhdfs_kerberos = ["dvc-webhdfs[kerberos]>=3,<4"]
 
 [project.urls]
 Documentation = "https://dvc.org/doc"

tests/func/experiments/test_remote.py

@@ -1,3 +1,5 @@
+import logging
+
 import pytest
 from funcy import first
 
@@ -360,3 +362,17 @@ def test_get(tmp_dir, scm, dvc, exp_stage, erepo_dir, use_ref):
             rev=exp_ref.name if use_ref else exp_rev,
         )
         assert (erepo_dir / "params.yaml").read_text().strip() == "foo: 2"
+
+
+def test_push_pull_invalid_workspace(
+    tmp_dir, scm, dvc, git_upstream, exp_stage, local_remote, caplog
+):
+    dvc.experiments.run()
+
+    with open("dvc.yaml", mode="a") as f:
+        f.write("\ninvalid")
+
+    with caplog.at_level(logging.WARNING, logger="dvc"):
+        dvc.experiments.push(git_upstream.remote, push_cache=True)
+        dvc.experiments.pull(git_upstream.remote, pull_cache=True)
+        assert "failed to collect" not in caplog.text

tests/func/repro/test_repro.py

@@ -1331,10 +1331,10 @@ def test_repro_keep_going(mocker, tmp_dir, dvc, copy_script):
         stage1, upstream=[bar_stage], force=False, interactive=False
     )
     foo_call = mocker.call(foo_stage, upstream=[], force=False, interactive=False)
-    assert spy.call_args_list in (
-        [bar_call, stage1_call, foo_call],
-        [foo_call, bar_call, stage1_call],
-    )
+    assert len(spy.call_args_list) == 3
+    assert foo_call in spy.call_args_list
+    assert bar_call in spy.call_args_list
+    assert stage1_call in spy.call_args_list
 
 
 def test_repro_ignore_errors(mocker, tmp_dir, dvc, copy_script):
@@ -1362,10 +1362,11 @@ def test_repro_ignore_errors(mocker, tmp_dir, dvc, copy_script):
         force=False,
         interactive=False,
     )
-    assert spy.call_args_list in (
-        [bar_call, stage1_call, foo_call, stage2_call],
-        [foo_call, bar_call, stage1_call, stage2_call],
-    )
+    assert len(spy.call_args_list) == 4
+    assert foo_call in spy.call_args_list
+    assert bar_call in spy.call_args_list
+    assert stage1_call in spy.call_args_list
+    assert stage2_call in spy.call_args_list
 
 
 @pytest.mark.parametrize("persist", [True, False])

tests/func/test_du.py

@@ -0,0 +1,49 @@
+import os
+
+
+def test_du(tmp_dir, dvc):
+    tmp_dir.gen(
+        {
+            "file": b"file",
+            "dvcfile": b"dvcfile",
+            "dir": {
+                "dirfile": b"dirfile",
+                "subdir": {
+                    "subdirfile": b"subdirfile",
+                },
+                "dvcsubdir": {
+                    "dvcsubdirfile": b"dvcsubdirfile",
+                },
+            },
+        }
+    )
+
+    dvc.add("dvcfile")
+    dvc.add(os.path.join("dir", "dvcsubdir"))
+
+    assert dvc.du(".", "file") == [("file", 4)]
+    assert dvc.du(".", "dvcfile") == [("dvcfile", 7)]
+    assert set(dvc.du(".", "dir/subdir")) == {
+        ("dir/subdir/subdirfile", 10),
+        ("dir/subdir", 10),
+    }
+    assert dvc.du(".", "dir/subdir", summarize=True) == [("dir/subdir", 10)]
+    assert set(dvc.du(".", "dir/dvcsubdir")) == {
+        ("dir/dvcsubdir/dvcsubdirfile", 13),
+        ("dir/dvcsubdir", 13),
+    }
+    assert dvc.du(".", "dir/dvcsubdir", summarize=True) == [("dir/dvcsubdir", 13)]
+    assert set(dvc.du(".", "dir")) == {
+        ("dir/dvcsubdir", 13),
+        ("dir/subdir", 10),
+        ("dir/dirfile", 7),
+        ("dir", 30),
+    }
+    assert dvc.du(".", "dir", summarize=True) == [("dir", 30)]
+    assert set(dvc.du(".", "/")) == {
+        ("/dvcfile", 7),
+        ("/dir", 30),
+        ("/file", 4),
+        ("/", 41),
+    }
+    assert dvc.du(".", "/", summarize=True) == [("/", 41)]

tests/func/test_fs.py

@@ -13,15 +13,15 @@ def test_cleanfs_subrepo(tmp_dir, dvc, scm, monkeypatch):
 
     path = subrepo_dir.fs_path
 
-    assert dvc.fs.exists(dvc.fs.path.join(path, "foo"))
-    assert dvc.fs.isfile(dvc.fs.path.join(path, "foo"))
-    assert dvc.fs.exists(dvc.fs.path.join(path, "dir"))
-    assert dvc.fs.isdir(dvc.fs.path.join(path, "dir"))
+    assert dvc.fs.exists(dvc.fs.join(path, "foo"))
+    assert dvc.fs.isfile(dvc.fs.join(path, "foo"))
+    assert dvc.fs.exists(dvc.fs.join(path, "dir"))
+    assert dvc.fs.isdir(dvc.fs.join(path, "dir"))
 
-    assert subrepo.fs.exists(subrepo.fs.path.join(path, "foo"))
-    assert subrepo.fs.isfile(subrepo.fs.path.join(path, "foo"))
-    assert subrepo.fs.exists(subrepo.fs.path.join(path, "dir"))
-    assert subrepo.fs.isdir(subrepo.fs.path.join(path, "dir"))
+    assert subrepo.fs.exists(subrepo.fs.join(path, "foo"))
+    assert subrepo.fs.isfile(subrepo.fs.join(path, "foo"))
+    assert subrepo.fs.exists(subrepo.fs.join(path, "dir"))
+    assert subrepo.fs.isdir(subrepo.fs.join(path, "dir"))
 
 
 def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc):

tests/func/test_import_db.py

@@ -11,15 +11,15 @@ serializers = AgateSerializer(data1), AgateSerializer(data2)
 
 
 @pytest.fixture
-def adapter(mocker):
-    m = mocker.patch("dvc.database.get_adapter")
-    adapter = mocker.MagicMock()
-    adapter.query.return_value.__enter__.side_effect = serializers
-    m.return_value.__enter__.return_value = adapter
+def client(mocker):
+    m = mocker.patch("dvc.database.get_client")
+    client = mocker.MagicMock()
+    client.query.return_value.__enter__.side_effect = serializers
+    m.return_value.__enter__.return_value = client
     return m
 
 
-def test_sql(adapter, tmp_dir, dvc):
+def test_sql(client, tmp_dir, dvc):
     stage = dvc.imp_db(
         sql="select * from model", profile="profile", output_format="json"
     )
@@ -56,7 +56,7 @@ def test_sql(adapter, tmp_dir, dvc):
     }
 
 
-def test_sql_conn_string(adapter, tmp_dir, dvc):
+def test_sql_conn_string(client, tmp_dir, dvc):
     with dvc.config.edit(level="local") as conf:
         conf["db"] = {"conn": {"url": "conn"}}
 

tests/func/test_odb.py

@@ -165,7 +165,7 @@ def test_default_cache_type(dvc):
 @pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.")
 @pytest.mark.parametrize("group", [False, True])
 def test_shared_cache(tmp_dir, dvc, group):
-    from dvc.fs import system
+    from dvc_data.hashfile.db.local import umask
 
     if group:
         with dvc.config.edit() as conf:
@@ -178,7 +178,7 @@ def test_shared_cache(tmp_dir, dvc, group):
     tmp_dir.dvc_gen({"file": "file content", "dir": {"file2": "file 2 content"}})
 
     file_mode = oct(0o444)
-    dir_mode = oct(0o2775 if group else (0o777 & ~system.umask))
+    dir_mode = oct(0o2775 if group else (0o777 & ~umask))
     for root, dnames, fnames in os.walk(cache_dir):
         for dname in dnames:
             path = os.path.join(root, dname)

tests/integration/plots/test_plots.py

@@ -11,7 +11,7 @@ from bs4 import BeautifulSoup
 from funcy import first
 
 from dvc.cli import main
-from dvc.render import REVISION_FIELD, VERSION_FIELD
+from dvc.render import ANCHOR_DEFINITIONS, FILENAME, REVISION
 
 JSON_OUT = "vis_data"
 
@@ -102,6 +102,9 @@ def verify_vega(
     html_result,
     json_result,
     split_json_result,
+    title,
+    x_label,
+    y_label,
 ):
     if isinstance(versions, str):
         versions = [versions]
@@ -111,27 +114,60 @@ def verify_vega(
         assert j[0]["type"] == "vega"
         assert set(j[0]["revisions"]) == set(versions)
 
-    assert json_result[0]["datapoints"] == split_json_result[0]["datapoints"]
-    assert set(versions) == set(json_result[0]["datapoints"].keys())
+    assert (
+        json_result[0]["content"]["data"]["values"]
+        == split_json_result[0][ANCHOR_DEFINITIONS]["<DVC_METRIC_DATA>"]
+    )
+
+    assert set(versions) == set(json_result[0]["revisions"])
 
     assert json_result[0]["content"]["data"]["values"]
     assert html_result["data"]["values"]
-    assert split_json_result[0]["content"]["data"]["values"] == "<DVC_METRIC_DATA>"
 
-    def _assert_templates_equal(html_template, filled_template, split_template):
-        # besides data, json and split json should be equal
-        path = ["data", "values"]
+    content_str = json.dumps(split_json_result[0]["content"])
+    assert "<DVC_METRIC_DATA>" in content_str
+    assert "<DVC_METRIC_X_LABEL>" in content_str
+    assert "<DVC_METRIC_Y_LABEL>" in content_str
+
+    def _assert_templates_equal(
+        html_template, filled_template, split_template, title, x_label, y_label
+    ):
+        # besides split anchors, json and split json should be equal
+        paths = [["data", "values"], ["encoding", "color"]]
         tmp1 = deepcopy(html_template)
         tmp2 = deepcopy(filled_template)
-        tmp3 = deepcopy(split_template)
-        dpath.set(tmp1, path, {})
-        dpath.set(tmp2, path, {})
-        dpath.set(tmp3, path, {})
+        tmp3 = json.loads(
+            json.dumps(split_template)
+            .replace('"<DVC_METRIC_PLOT_HEIGHT>"', "300")
+            .replace('"<DVC_METRIC_PLOT_WIDTH>"', "300")
+            .replace("<DVC_METRIC_TITLE>", title)
+            .replace("<DVC_METRIC_X_LABEL>", x_label)
+            .replace("<DVC_METRIC_Y_LABEL>", y_label)
+            .replace(
+                '"<DVC_METRIC_ZOOM_AND_PAN>"',
+                json.dumps(
+                    {
+                        "name": "grid",
+                        "select": "interval",
+                        "bind": "scales",
+                    }
+                ),
+            )
+        )
+        for path in paths:
+            dpath.set(tmp1, path, {})
+            dpath.set(tmp2, path, {})
+            dpath.set(tmp3, path, {})
 
         assert tmp1 == tmp2 == tmp3
 
     _assert_templates_equal(
-        html_result, json_result[0]["content"], split_json_result[0]["content"]
+        html_result,
+        json_result[0]["content"],
+        split_json_result[0]["content"],
+        title,
+        x_label,
+        y_label,
     )
 
 
@@ -193,17 +229,13 @@ def test_repo_with_plots(tmp_dir, scm, dvc, capsys, run_copy_metrics, repo_with_
     ] == _update_datapoints(
         linear_v1,
         {
-            VERSION_FIELD: {
-                "revision": "workspace",
-                "filename": "linear.json",
-                "field": "y",
-            },
+            REVISION: "workspace",
         },
     )
     assert html_result["linear.json"]["data"]["values"] == _update_datapoints(
         linear_v1,
         {
-            REVISION_FIELD: "workspace",
+            REVISION: "workspace",
         },
     )
     assert json_data["confusion.json"][0]["content"]["data"][
@@ -211,27 +243,29 @@ def test_repo_with_plots(tmp_dir, scm, dvc, capsys, run_copy_metrics, repo_with_
     ] == _update_datapoints(
         confusion_v1,
         {
-            VERSION_FIELD: {
-                "revision": "workspace",
-                "filename": "confusion.json",
-                "field": "actual",
-            },
+            REVISION: "workspace",
         },
     )
     assert html_result["confusion.json"]["data"]["values"] == _update_datapoints(
         confusion_v1,
         {
-            REVISION_FIELD: "workspace",
+            REVISION: "workspace",
         },
     )
     verify_image(tmp_dir, "workspace", "image.png", image_v1, html_path, json_data)
 
-    for plot in ["linear.json", "confusion.json"]:
+    for plot, title, x_label, y_label in [
+        ("linear.json", "linear", "x", "y"),
+        ("confusion.json", "confusion matrix", "predicted", "actual"),
+    ]:
         verify_vega(
             "workspace",
             html_result[plot],
             json_data[plot],
             split_json_data[plot],
+            title,
+            x_label,
+            y_label,
         )
 
     verify_vega_props("confusion.json", json_data, **confusion_props)
@@ -250,12 +284,18 @@ def test_repo_with_plots(tmp_dir, scm, dvc, capsys, run_copy_metrics, repo_with_
     verify_image(tmp_dir, "workspace", "image.png", image_v2, html_path, json_data)
     verify_image(tmp_dir, "HEAD", "image.png", image_v1, html_path, json_data)
 
-    for plot in ["linear.json", "confusion.json"]:
+    for plot, title, x_label, y_label in [
+        ("linear.json", "linear", "x", "y"),
+        ("confusion.json", "confusion matrix", "predicted", "actual"),
+    ]:
         verify_vega(
             ["HEAD", "workspace"],
             html_result[plot],
             json_data[plot],
             split_json_data[plot],
+            title,
+            x_label,
+            y_label,
         )
     verify_vega_props("confusion.json", json_data, **confusion_props)
     path = tmp_dir / "subdir"
@@ -277,31 +317,23 @@ def test_repo_with_plots(tmp_dir, scm, dvc, capsys, run_copy_metrics, repo_with_
         ] == _update_datapoints(
             linear_v2,
             {
-                VERSION_FIELD: {
-                    "revision": "workspace",
-                    "filename": "../linear.json",
-                    "field": "y",
-                },
+                REVISION: "workspace",
             },
         ) + _update_datapoints(
             linear_v1,
             {
-                VERSION_FIELD: {
-                    "revision": "HEAD",
-                    "filename": "../linear.json",
-                    "field": "y",
-                },
+                REVISION: "HEAD",
             },
         )
         assert html_result["../linear.json"]["data"]["values"] == _update_datapoints(
             linear_v2,
             {
-                REVISION_FIELD: "workspace",
+                REVISION: "workspace",
             },
         ) + _update_datapoints(
             linear_v1,
             {
-                REVISION_FIELD: "HEAD",
+                REVISION: "HEAD",
             },
         )
         assert json_data["../confusion.json"][0]["content"]["data"][
@@ -309,43 +341,38 @@ def test_repo_with_plots(tmp_dir, scm, dvc, capsys, run_copy_metrics, repo_with_
         ] == _update_datapoints(
             confusion_v2,
             {
-                VERSION_FIELD: {
-                    "revision": "workspace",
-                    "filename": "../confusion.json",
-                    "field": "actual",
-                },
+                REVISION: "workspace",
             },
         ) + _update_datapoints(
             confusion_v1,
             {
-                VERSION_FIELD: {
-                    "revision": "HEAD",
-                    "filename": "../confusion.json",
-                    "field": "actual",
-                },
+                REVISION: "HEAD",
             },
         )
         assert html_result["../confusion.json"]["data"]["values"] == _update_datapoints(
             confusion_v2,
             {
-                REVISION_FIELD: "workspace",
+                REVISION: "workspace",
             },
         ) + _update_datapoints(
             confusion_v1,
             {
-                REVISION_FIELD: "HEAD",
+                REVISION: "HEAD",
             },
         )
 
-        for plot in [
-            "../linear.json",
-            "../confusion.json",
+        for plot, title, x_label, y_label in [
+            ("../linear.json", "linear", "x", "y"),
+            ("../confusion.json", "confusion matrix", "predicted", "actual"),
         ]:
             verify_vega(
                 ["HEAD", "workspace"],
                 html_result[plot],
                 json_data[plot],
                 split_json_data[plot],
+                title,
+                x_label,
+                y_label,
             )
         verify_image(
             path,
@@ -434,7 +461,7 @@ def test_repo_with_config_plots(tmp_dir, capsys, repo_with_config_plots):
     repo_state = repo_with_config_plots()
     plots = next(repo_state)
 
-    html_path, _, __ = call(capsys)
+    html_path, _, split_json_result = call(capsys)
 
     assert os.path.exists(html_path)
     html_result = extract_vega_specs(
@@ -444,20 +471,28 @@ def test_repo_with_config_plots(tmp_dir, capsys, repo_with_config_plots):
             "confusion_train_vs_test",
         ],
     )
+
     ble = _update_datapoints(
         plots["data"]["linear_train.json"],
         {
-            REVISION_FIELD: "linear_train.json",
+            REVISION: "workspace",
+            FILENAME: "linear_train.json",
         },
     ) + _update_datapoints(
         plots["data"]["linear_test.json"],
         {
-            REVISION_FIELD: "linear_test.json",
+            REVISION: "workspace",
+            FILENAME: "linear_test.json",
         },
     )
 
     assert html_result["linear_train_vs_test"]["data"]["values"] == ble
-    # TODO check json results once vscode is able to handle flexible plots
+    assert (
+        split_json_result["data"]["linear_train_vs_test"][0][ANCHOR_DEFINITIONS][
+            "<DVC_METRIC_DATA>"
+        ]
+        == ble
+    )
 
 
 @pytest.mark.vscode
@@ -466,7 +501,7 @@ def test_repo_with_dvclive_plots(tmp_dir, capsys, repo_with_dvclive_plots):
 
     for s in ("show", "diff"):
         _, json_result, split_json_result = call(capsys, subcommand=s)
-        expected_result = {
+        expected_result: Dict[str, Dict[str, list[str]]] = {
             "data": {
                 "dvclive/plots/metrics/metric.tsv": [],
             },

tests/unit/command/test_du.py

@@ -0,0 +1,21 @@
+from dvc.cli import parse_args
+from dvc.commands.du import CmdDU
+
+
+def test_du(mocker):
+    cli_args = parse_args(["du", "myurl", "mypath", "--summarize", "--rev", "myrev"])
+    assert cli_args.func == CmdDU
+
+    cmd = cli_args.func(cli_args)
+    mock_du = mocker.patch("dvc.repo.Repo.du")
+
+    assert cmd.run() == 0
+    mock_du.assert_called_once_with(
+        "myurl",
+        "mypath",
+        rev="myrev",
+        summarize=True,
+        config=None,
+        remote=None,
+        remote_config=None,
+    )

tests/unit/command/test_studio.py

@@ -0,0 +1,81 @@
+from dvc_studio_client.auth import AuthenticationExpired
+
+from dvc.cli import main
+from dvc.utils.studio import STUDIO_URL
+
+
+def test_studio_login_token_check_failed(mocker):
+    mocker.patch(
+        "dvc_studio_client.auth.get_access_token", side_effect=AuthenticationExpired
+    )
+
+    assert main(["studio", "login"]) == 1
+
+
+def test_studio_login_success(mocker, dvc):
+    mocker.patch(
+        "dvc_studio_client.auth.get_access_token",
+        return_value=("token_name", "isat_access_token"),
+    )
+
+    assert main(["studio", "login"]) == 0
+
+    config = dvc.config.load_one("global")
+    assert config["studio"]["token"] == "isat_access_token"
+    assert config["studio"]["url"] == STUDIO_URL
+
+
+def test_studio_login_arguments(mocker):
+    mock = mocker.patch(
+        "dvc_studio_client.auth.get_access_token",
+        return_value=("token_name", "isat_access_token"),
+    )
+
+    assert (
+        main(
+            [
+                "studio",
+                "login",
+                "--name",
+                "token_name",
+                "--hostname",
+                "https://example.com",
+                "--scopes",
+                "experiments",
+                "--no-open",
+            ]
+        )
+        == 0
+    )
+
+    mock.assert_called_with(
+        token_name="token_name",
+        hostname="https://example.com",
+        scopes="experiments",
+        use_device_code=True,
+        client_name="DVC",
+    )
+
+
+def test_studio_logout(dvc):
+    with dvc.config.edit("global") as conf:
+        conf["studio"]["token"] = "isat_access_token"
+
+    assert main(["studio", "logout"]) == 0
+    config = dvc.config.load_one("global")
+    assert "token" not in config["studio"]
+
+    assert main(["studio", "logout"]) == 1
+
+
+def test_studio_token(dvc, capsys):
+    with dvc.config.edit("global") as conf:
+        conf["studio"]["token"] = "isat_access_token"
+
+    assert main(["studio", "token"]) == 0
+    assert capsys.readouterr().out == "isat_access_token\n"
+
+    with dvc.config.edit("global") as conf:
+        del conf["studio"]["token"]
+
+    assert main(["studio", "token"]) == 1

tests/unit/fs/test_path.py

@@ -1,29 +0,0 @@
-import pytest
-
-from dvc.fs import Path
-
-
-@pytest.mark.parametrize("prefix", ["", "/"])
-@pytest.mark.parametrize("postfix", ["", "/"])
-@pytest.mark.parametrize(
-    "path,expected",
-    [
-        ("path", ("path",)),
-        ("some/path", ("some", "path")),
-    ],
-)
-def test_parts_posix(prefix, postfix, path, expected):
-    assert Path("/").parts(prefix + path + postfix) == tuple(prefix) + expected
-
-
-@pytest.mark.parametrize("postfix", ["", "\\"])
-@pytest.mark.parametrize(
-    "path,expected",
-    [
-        ("path", ("path",)),
-        ("c:\\path", ("c:", "\\", "path")),
-        ("some\\path", ("some", "path")),
-    ],
-)
-def test_parts_nt(postfix, path, expected):
-    assert Path("\\").parts(path + postfix) == expected

tests/unit/render/test_convert.py

@@ -1,109 +1,87 @@
-from dvc.render import REVISION_FIELD, REVISIONS_KEY, SRC_FIELD, TYPE_KEY, VERSION_FIELD
+import json
+
+import pytest
+
+from dvc.render import ANCHOR_DEFINITIONS, FILENAME, REVISION, REVISIONS, SRC, TYPE_KEY
 from dvc.render.convert import to_json
 
 
 def test_to_json_vega(mocker):
     vega_renderer = mocker.MagicMock()
     vega_renderer.TYPE = "vega"
+    vega_renderer.get_revs.return_value = ["bar", "foo"]
     vega_renderer.get_filled_template.return_value = {"this": "is vega"}
-    vega_renderer.datapoints = [
-        {
-            "x": 1,
-            "y": 2,
-            VERSION_FIELD: {"revision": "foo"},
-            "filename": "foo.json",
-        },
-        {
-            "x": 2,
-            "y": 1,
-            VERSION_FIELD: {"revision": "bar"},
-            "filename": "foo.json",
-        },
-    ]
     result = to_json(vega_renderer)
     assert result[0] == {
         TYPE_KEY: vega_renderer.TYPE,
-        REVISIONS_KEY: ["bar", "foo"],
+        REVISIONS: ["bar", "foo"],
         "content": {"this": "is vega"},
-        "datapoints": {
-            "foo": [
-                {
-                    "x": 1,
-                    "y": 2,
-                    "filename": "foo.json",
-                    VERSION_FIELD: {"revision": "foo"},
-                },
-            ],
-            "bar": [
-                {
-                    "x": 2,
-                    "y": 1,
-                    "filename": "foo.json",
-                    VERSION_FIELD: {"revision": "bar"},
-                },
-            ],
-        },
     }
     vega_renderer.get_filled_template.assert_called()
 
 
+@pytest.mark.vscode
 def test_to_json_vega_split(mocker):
+    revs = ["bar", "foo"]
+    content = json.dumps(
+        {
+            "this": "is split vega",
+            "encoding": {"color": "<DVC_METRIC_COLOR>"},
+            "data": {"values": "<DVC_METRIC_DATA>"},
+        }
+    )
+    anchor_definitions = {
+        "<DVC_METRIC_COLOR>": {
+            "field": "rev",
+            "scale": {
+                "domain": revs,
+                "range": ["#ff0000", "#00ff00"],
+            },
+        },
+        "<DVC_METRIC_DATA>": [
+            {
+                "x": 1,
+                "y": 2,
+                REVISION: "foo",
+                FILENAME: "foo.json",
+            },
+            {
+                "x": 2,
+                "y": 1,
+                REVISION: "bar",
+                FILENAME: "foo.json",
+            },
+        ],
+    }
+
     vega_renderer = mocker.MagicMock()
     vega_renderer.TYPE = "vega"
-    vega_renderer.get_filled_template.return_value = {"this": "is split vega"}
-    vega_renderer.datapoints = [
-        {
-            "x": 1,
-            "y": 2,
-            VERSION_FIELD: {"revision": "foo"},
-            "filename": "foo.json",
-        },
-        {
-            "x": 2,
-            "y": 1,
-            VERSION_FIELD: {"revision": "bar"},
-            "filename": "foo.json",
-        },
-    ]
+    vega_renderer.get_partial_filled_template.return_value = (
+        content,
+        {ANCHOR_DEFINITIONS: anchor_definitions},
+    )
+    vega_renderer.get_revs.return_value = ["bar", "foo"]
+
     result = to_json(vega_renderer, split=True)
     assert result[0] == {
+        ANCHOR_DEFINITIONS: anchor_definitions,
         TYPE_KEY: vega_renderer.TYPE,
-        REVISIONS_KEY: ["bar", "foo"],
-        "content": {"this": "is split vega"},
-        "datapoints": {
-            "foo": [
-                {
-                    "x": 1,
-                    "y": 2,
-                    "filename": "foo.json",
-                    VERSION_FIELD: {"revision": "foo"},
-                }
-            ],
-            "bar": [
-                {
-                    "x": 2,
-                    "y": 1,
-                    "filename": "foo.json",
-                    VERSION_FIELD: {"revision": "bar"},
-                }
-            ],
-        },
+        REVISIONS: revs,
+        "content": content,
     }
-    vega_renderer.get_filled_template.assert_called_with(
-        as_string=False, skip_anchors=["data"]
-    )
+    vega_renderer.get_partial_filled_template.assert_called_once()
 
 
 def test_to_json_image(mocker):
     image_renderer = mocker.MagicMock()
     image_renderer.TYPE = "image"
     image_renderer.datapoints = [
-        {SRC_FIELD: "contentfoo", REVISION_FIELD: "foo"},
-        {SRC_FIELD: "contentbar", REVISION_FIELD: "bar"},
+        {SRC: "contentfoo", REVISION: "foo"},
+        {SRC: "contentbar", REVISION: "bar"},
     ]
     result = to_json(image_renderer)
     assert result[0] == {
-        "url": image_renderer.datapoints[0].get(SRC_FIELD),
-        REVISIONS_KEY: [image_renderer.datapoints[0].get(REVISION_FIELD)],
+        "url": image_renderer.datapoints[0].get(SRC),
+        REVISIONS: [image_renderer.datapoints[0].get(REVISION)],
         TYPE_KEY: image_renderer.TYPE,
     }

tests/unit/render/test_image_converter.py

@@ -1,4 +1,4 @@
-from dvc.render import REVISION_FIELD, SRC_FIELD
+from dvc.render import FILENAME, REVISION, SRC
 from dvc.render.converter.image import ImageConverter
 
 
@@ -8,9 +8,9 @@ def test_image_converter_no_out():
     datapoints, _ = converter.flat_datapoints("r")
 
     assert datapoints[0] == {
-        REVISION_FIELD: "r",
-        "filename": "image.png",
-        SRC_FIELD: converter._encode_image(b"content"),
+        REVISION: "r",
+        FILENAME: "image.png",
+        SRC: converter._encode_image(b"content"),
     }
 
 
@@ -21,9 +21,9 @@ def test_image_converter_with_out(tmp_dir):
     datapoints, _ = converter.flat_datapoints("r")
 
     assert datapoints[0] == {
-        REVISION_FIELD: "r",
-        "filename": "image.png",
-        SRC_FIELD: str(tmp_dir / "foo" / "r_image.png"),
+        REVISION: "r",
+        FILENAME: "image.png",
+        SRC: str(tmp_dir / "foo" / "r_image.png"),
     }
 
     assert (tmp_dir / "foo" / "r_image.png").read_bytes() == b"content"
@@ -37,9 +37,9 @@ def test_image_converter_with_slash_in_revision(tmp_dir):
     datapoints, _ = converter.flat_datapoints("feature/r")
 
     assert datapoints[0] == {
-        REVISION_FIELD: "feature/r",
-        "filename": "image.png",
-        SRC_FIELD: str(tmp_dir / "foo" / "feature_r_image.png"),
+        REVISION: "feature/r",
+        FILENAME: "image.png",
+        SRC: str(tmp_dir / "foo" / "feature_r_image.png"),
     }
 
     assert (tmp_dir / "foo" / "feature_r_image.png").read_bytes() == b"content"

tests/unit/render/test_match.py

@@ -1,9 +1,13 @@
 import pytest
 from funcy import set_in
 
-from dvc.render import VERSION_FIELD
+from dvc.render import FIELD, FILENAME, REVISION
 from dvc.render.converter.vega import VegaConverter
-from dvc.render.match import PlotsData, _squash_plots_properties, match_defs_renderers
+from dvc.render.match import (
+    PlotsData,
+    _squash_plots_properties,
+    match_defs_renderers,
+)
 
 
 @pytest.mark.parametrize(
@@ -158,25 +162,23 @@ def test_match_renderers(M):
     renderer = renderer_with_errors[0]
     assert renderer.datapoints == [
         {
-            VERSION_FIELD: {
-                "revision": "v1",
-                "filename": "file.json",
-                "field": "y",
-            },
+            REVISION: "v1",
+            FILENAME: "file.json",
+            FIELD: "y",
             "x": 1,
             "y": 1,
         },
         {
-            VERSION_FIELD: {
-                "revision": "v1",
-                "filename": "file.json",
-                "field": "y",
-            },
+            REVISION: "v1",
+            FILENAME: "file.json",
+            FIELD: "y",
             "x": 2,
             "y": 2,
         },
     ]
     assert renderer.properties == {
+        "anchors_y_definitions": [{FILENAME: "file.json", FIELD: "y"}],
+        "revs_with_datapoints": ["v1"],
         "title": "plot_id_1",
         "x": "x",
         "y": "y",

tests/unit/render/test_vega_converter.py

@@ -3,7 +3,7 @@ from collections import OrderedDict
 import pytest
 
 from dvc.exceptions import DvcException
-from dvc.render import VERSION_FIELD
+from dvc.render import FIELD, FILENAME, REVISION
 from dvc.render.converter.vega import FieldNotFoundError, VegaConverter, _lists
 
 
@@ -35,23 +35,25 @@ def test_finding_lists(dictionary, expected_result):
                 {
                     "v": 1,
                     "step": 0,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
                 },
                 {
                     "v": 2,
                     "step": 1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
                 },
             ],
-            {"x": "step", "y": "v", "x_label": "step", "y_label": "v"},
+            {
+                "anchors_y_definitions": [{FILENAME: "f", FIELD: "v"}],
+                "x": "step",
+                "y": "v",
+                "x_label": "step",
+                "y_label": "v",
+            },
             id="default_x_y",
         ),
         pytest.param(
@@ -61,23 +63,25 @@ def test_finding_lists(dictionary, expected_result):
                 {
                     "v": 1,
                     "v2": 0.1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                 },
                 {
                     "v": 2,
                     "v2": 0.2,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                 },
             ],
-            {"x": "v", "y": "v2", "x_label": "v", "y_label": "v2"},
+            {
+                "anchors_y_definitions": [{FILENAME: "f", FIELD: "v2"}],
+                "x": "v",
+                "y": "v2",
+                "x_label": "v",
+                "y_label": "v2",
+            },
             id="choose_x_y",
         ),
         pytest.param(
@@ -99,23 +103,25 @@ def test_finding_lists(dictionary, expected_result):
                 {
                     "v": 1,
                     "v2": 0.1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                 },
                 {
                     "v": 2,
                     "v2": 0.2,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                 },
             ],
-            {"x": "v", "y": "v2", "x_label": "x", "y_label": "y"},
+            {
+                "anchors_y_definitions": [{FILENAME: "f", FIELD: "v2"}],
+                "x": "v",
+                "y": "v2",
+                "x_label": "x",
+                "y_label": "y",
+            },
             id="find_in_nested_structure",
         ),
         pytest.param(
@@ -123,44 +129,36 @@ def test_finding_lists(dictionary, expected_result):
             {"y": {"f": ["v", "v2"]}},
             [
                 {
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
                     "dvc_inferred_y_value": 1,
                     "v": 1,
                     "v2": 0.1,
                     "step": 0,
                 },
                 {
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
                     "dvc_inferred_y_value": 2,
                     "v": 2,
                     "v2": 0.2,
                     "step": 1,
                 },
                 {
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                     "dvc_inferred_y_value": 0.1,
                     "v2": 0.1,
                     "v": 1,
                     "step": 0,
                 },
                 {
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                     "v": 2,
                     "v2": 0.2,
                     "dvc_inferred_y_value": 0.2,
@@ -168,6 +166,10 @@ def test_finding_lists(dictionary, expected_result):
                 },
             ],
             {
+                "anchors_y_definitions": [
+                    {FILENAME: "f", FIELD: "v"},
+                    {FILENAME: "f", FIELD: "v2"},
+                ],
                 "x": "step",
                 "y": "dvc_inferred_y_value",
                 "y_label": "y",
@@ -189,47 +191,43 @@ def test_finding_lists(dictionary, expected_result):
                     "z": 3,
                     "v": 1,
                     "step": 0,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
                 },
                 {
                     "dvc_inferred_y_value": 2,
                     "z": 4,
                     "step": 1,
                     "v": 2,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
                 },
                 {
                     "dvc_inferred_y_value": 3,
                     "v": 1,
                     "z": 3,
                     "step": 0,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "z",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "z",
                 },
                 {
                     "dvc_inferred_y_value": 4,
                     "v": 2,
                     "z": 4,
                     "step": 1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "z",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "z",
                 },
             ],
             {
+                "anchors_y_definitions": [
+                    {FILENAME: "f", FIELD: "v"},
+                    {FILENAME: "f", FIELD: "z"},
+                ],
                 "x": "step",
                 "y": "dvc_inferred_y_value",
                 "y_label": "y",
@@ -247,32 +245,35 @@ def test_finding_lists(dictionary, expected_result):
                 {
                     "v": 1,
                     "v2": 0.1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                 },
                 {
                     "v": 2,
                     "v2": 0.2,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                 },
                 {
                     "v": 3,
                     "v2": 0.3,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f2",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f2",
+                    FIELD: "v2",
                 },
             ],
-            {"x": "v", "y": "v2", "x_label": "v", "y_label": "v2"},
+            {
+                "anchors_y_definitions": [
+                    {FILENAME: "f", FIELD: "v2"},
+                    {FILENAME: "f2", FIELD: "v2"},
+                ],
+                "x": "v",
+                "y": "v2",
+                "x_label": "v",
+                "y_label": "v2",
+            },
             id="multi_file_json",
         ),
         pytest.param(
@@ -284,47 +285,43 @@ def test_finding_lists(dictionary, expected_result):
                     "v": 1,
                     "v2": 0.1,
                     "step": 0,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
                 },
                 {
                     "dvc_inferred_y_value": 2,
                     "v": 2,
                     "v2": 0.2,
                     "step": 1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
                 },
                 {
                     "dvc_inferred_y_value": 0.1,
                     "v": 1,
                     "v2": 0.1,
                     "step": 0,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                 },
                 {
                     "dvc_inferred_y_value": 0.2,
                     "v": 2,
                     "v2": 0.2,
                     "step": 1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                 },
             ],
             {
+                "anchors_y_definitions": [
+                    {FILENAME: "f", FIELD: "v"},
+                    {FILENAME: "f", FIELD: "v2"},
+                ],
                 "x": "step",
                 "y": "dvc_inferred_y_value",
                 "x_label": "step",
@@ -344,35 +341,34 @@ def test_finding_lists(dictionary, expected_result):
                     "v": 1,
                     "v2": 0.1,
                     "v3": 0.01,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                 },
                 {
                     "dvc_inferred_y_value": 0.01,
                     "v": 1,
                     "v2": 0.1,
                     "v3": 0.01,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v3",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v3",
                 },
                 {
                     "dvc_inferred_y_value": 0.1,
                     "v": 1,
                     "v2": 0.1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f2",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f2",
+                    FIELD: "v2",
                 },
             ],
             {
+                "anchors_y_definitions": [
+                    {FILENAME: "f", FIELD: "v2"},
+                    {FILENAME: "f", FIELD: "v3"},
+                    {FILENAME: "f2", FIELD: "v2"},
+                ],
                 "x": "v",
                 "y": "dvc_inferred_y_value",
                 "x_label": "v",
@@ -390,23 +386,23 @@ def test_finding_lists(dictionary, expected_result):
                 {
                     "v": 1,
                     "v2": 0.1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
                 },
                 {
                     "v": 1,
                     "v2": 0.1,
-                    VERSION_FIELD: {
-                        "revision": "r",
-                        "filename": "f2",
-                        "field": "v2",
-                    },
+                    REVISION: "r",
+                    FILENAME: "f2",
+                    FIELD: "v2",
                 },
             ],
             {
+                "anchors_y_definitions": [
+                    {FILENAME: "f", FIELD: "v2"},
+                    {FILENAME: "f2", FIELD: "v2"},
+                ],
                 "x": "v",
                 "y": "v2",
                 "x_label": "v",
@@ -414,6 +410,107 @@ def test_finding_lists(dictionary, expected_result):
             },
             id="multi_file_y_same_prefix",
         ),
+        pytest.param(
+            {
+                "f": {"metric": [{"x1": 1, "v": 0.1}]},
+                "f2": {"metric": [{"x2": 100, "v": 0.1}]},
+            },
+            {"y": {"f": ["v"], "f2": ["v"]}, "x": {"f": "x1", "f2": "x2"}},
+            [
+                {
+                    "x1": 1,
+                    "v": 0.1,
+                    "dvc_inferred_x_value": 1,
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
+                },
+                {
+                    "x2": 100,
+                    "v": 0.1,
+                    "dvc_inferred_x_value": 100,
+                    REVISION: "r",
+                    FILENAME: "f2",
+                    FIELD: "v",
+                },
+            ],
+            {
+                "anchors_y_definitions": [
+                    {FILENAME: "f", FIELD: "v"},
+                    {FILENAME: "f2", FIELD: "v"},
+                ],
+                "x": "dvc_inferred_x_value",
+                "y": "v",
+                "x_label": "x",
+                "y_label": "v",
+            },
+            id="multiple_x_fields",
+        ),
+        pytest.param(
+            {
+                "f": {
+                    "metric": [
+                        {"v": 1, "v2": 0.1, "x1": 100},
+                        {"v": 2, "v2": 0.2, "x1": 1000},
+                    ]
+                },
+                "f2": {"metric": [{"x2": -2}, {"x2": -4}]},
+            },
+            {"y": ["v", "v2"], "x": {"f": "x1", "f2": "x2"}},
+            [
+                {
+                    "dvc_inferred_x_value": 100,
+                    "dvc_inferred_y_value": 1,
+                    "v": 1,
+                    "v2": 0.1,
+                    "x1": 100,
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
+                },
+                {
+                    "dvc_inferred_x_value": 1000,
+                    "dvc_inferred_y_value": 2,
+                    "v": 2,
+                    "v2": 0.2,
+                    "x1": 1000,
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v",
+                },
+                {
+                    "dvc_inferred_x_value": -2,
+                    "dvc_inferred_y_value": 0.1,
+                    "v": 1,
+                    "v2": 0.1,
+                    "x1": 100,
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
+                },
+                {
+                    "dvc_inferred_x_value": -4,
+                    "dvc_inferred_y_value": 0.2,
+                    "v": 2,
+                    "v2": 0.2,
+                    "x1": 1000,
+                    REVISION: "r",
+                    FILENAME: "f",
+                    FIELD: "v2",
+                },
+            ],
+            {
+                "anchors_y_definitions": [
+                    {FILENAME: "f", FIELD: "v"},
+                    {FILENAME: "f", FIELD: "v2"},
+                ],
+                "x": "dvc_inferred_x_value",
+                "y": "dvc_inferred_y_value",
+                "x_label": "x",
+                "y_label": "y",
+            },
+            id="y_list_x_dict",
+        ),
     ],
 )
 def test_convert(

tests/unit/test_ignore.py

@@ -173,11 +173,10 @@ def test_match_ignore_from_file(
     )
     dvcignore_dirname = os.path.dirname(dvcignore_path)
 
-    fs = mocker.MagicMock()
-    fs.path = localfs.path
-    fs.sep = localfs.sep
-    mocker.patch.object(fs, "open", mocker.mock_open(read_data="\n".join(patterns)))
-    ignore_file = DvcIgnorePatterns.from_file(dvcignore_path, fs, "mocked")
+    mocker.patch.object(
+        localfs, "open", mocker.mock_open(read_data="\n".join(patterns))
+    )
+    ignore_file = DvcIgnorePatterns.from_file(dvcignore_path, localfs, "mocked")
 
     assert (
         ignore_file.matches(dvcignore_dirname, file_to_ignore_relpath) == expected_match