artifacts: add support for monorepo (#10386)

updates:
- [github.com/astral-sh/ruff-pre-commit: v0.3.5 → v0.3.7](https://github.com/astral-sh/ruff-pre-commit/compare/v0.3.5...v0.3.7)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/mergify.yml

@@ -1,10 +0,0 @@
-pull_request_rules:
-  - name: backport patches to 2.x
-    conditions:
-      - base=main
-      - label=backport
-    actions:
-      backport:
-        branches:
-          - "2.x"
-        title: ({{ destination_branch }}) {{ title }}

.github/workflows/codeql.yml

@@ -2,7 +2,7 @@ name: "CodeQL"
 
 on:
   push:
-    branches: ["main", "2.x"]
+    branches: ["main"]
   pull_request:
     branches: ["main"]
   schedule:

.github/workflows/tests.yaml

@@ -2,7 +2,7 @@ name: Tests
 
 on:
   push:
-    branches: [main, 2.x]
+    branches: [main]
   pull_request:
   schedule:
     - cron: '5 1 * * *'  # every day at 01:05
@@ -66,7 +66,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Slack Notification
-      uses: rtCamp/action-slack-notify@v2.2.1
+      uses: rtCamp/action-slack-notify@v2.3.0
       env:
         SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         SLACK_COLOR: ${{ job.status }}

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ ci:
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -22,7 +22,7 @@ repos:
       - id: sort-simple-yaml
       - id: trailing-whitespace
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.2.1'
+    rev: 'v0.3.7'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

CITATION.cff

@@ -0,0 +1,25 @@
+cff-version: 1.2.0
+title: 'DVC: Data Version Control - Git for Data & Models"'
+message: >-
+  If you use this software as part of a publication and wish
+  to cite it, please use the metadata from this file.
+type: software
+authors:
+  - name: The DVC team and contributors
+    website: 'https://github.com/iterative'
+repository-code: 'https://github.com/iterative/dvc'
+url: 'https://dvc.org'
+abstract: >-
+  DVC is a tool for data science that takes advantage of existing software engineering toolset.
+  It helps machine learning teams manage large datasets, make projects reproducible, and collaborate better.
+keywords:
+  - ai
+  - collaboration
+  - data-science
+  - data-version-control
+  - developer-tools
+  - git
+  - machine-learning
+  - python
+  - reproducibility
+license: Apache-2.0

dvc/__init__.py

@@ -3,6 +3,7 @@ DVC
 ----
 Make your data science projects reproducible and shareable.
 """
+
 import dvc.logger
 from dvc.build import PKG  # noqa: F401
 from dvc.version import __version__, version_tuple  # noqa: F401

dvc/__main__.py

@@ -1,4 +1,5 @@
 """Main entry point for DVC command line tool."""
+
 import sys
 
 from dvc.cli import main

dvc/_debug.py

@@ -162,7 +162,7 @@ def _get_path_func(tool: str, ext: str):
     fmt = f"{tool}.dvc-{{now:%Y%m%d}}_{{now:%H%M%S}}.{ext}"
 
     def func(now: Optional["datetime"] = None) -> str:
-        return fmt.format(now=now or datetime.now())
+        return fmt.format(now=now or datetime.now())  # noqa: DTZ005
 
     return func
 

dvc/api/__init__.py

@@ -2,6 +2,7 @@ from dvc.fs.dvc import _DVCFileSystem as DVCFileSystem
 
 from .artifacts import artifacts_show
 from .data import get_url, open, read
+from .dataset import get as get_dataset
 from .experiments import exp_save, exp_show
 from .scm import all_branches, all_commits, all_tags
 from .show import metrics_show, params_show
@@ -14,6 +15,7 @@ __all__ = [
     "artifacts_show",
     "exp_save",
     "exp_show",
+    "get_dataset",
     "get_url",
     "metrics_show",
     "open",

dvc/api/artifacts.py

@@ -1,3 +1,4 @@
+import os
 from typing import Any, Optional
 
 from dvc.repo import Repo
@@ -36,12 +37,22 @@ def artifacts_show(
     if version and stage:
         raise ValueError("Artifact version and stage are mutually exclusive.")
 
+    from dvc.repo.artifacts import Artifacts
+    from dvc.utils import as_posix
+
     repo_kwargs: dict[str, Any] = {
         "subrepos": True,
         "uninitialized": True,
     }
+
+    dirname, _ = Artifacts.parse_path(name)
     with Repo.open(repo, **repo_kwargs) as _repo:
         rev = _repo.artifacts.get_rev(name, version=version, stage=stage)
         with _repo.switch(rev):
-            path = _repo.artifacts.get_path(name)
-        return {"rev": rev, "path": path}
+            root = _repo.fs.root_marker
+            _dirname = _repo.fs.join(root, dirname) if dirname else root
+            with Repo(_dirname, fs=_repo.fs, scm=_repo.scm) as r:
+                path = r.artifacts.get_path(name)
+                path = _repo.fs.join(_repo.fs.root_marker, as_posix(path))
+                parts = _repo.fs.relparts(path, _repo.root_dir)
+                return {"rev": rev, "path": os.path.join(*parts)}

dvc/api/data.py

@@ -25,7 +25,14 @@ def _wrap_exceptions(repo, url):
         raise PathMissingError(exc.path, url) from exc
 
 
-def get_url(path, repo=None, rev=None, remote=None):
+def get_url(
+    path: str,
+    repo: Optional[str] = None,
+    rev: Optional[str] = None,
+    remote: Optional[str] = None,
+    config: Optional[dict[str, Any]] = None,
+    remote_config: Optional[dict[str, Any]] = None,
+):
     """
     Returns the URL to the storage location of a data file or directory tracked
     in a DVC repo. For Git repos, HEAD is used unless a rev argument is
@@ -35,15 +42,44 @@ def get_url(path, repo=None, rev=None, remote=None):
 
     NOTE: This function does not check for the actual existence of the file or
     directory in the remote storage.
+
+    Args:
+        path (str): location and file name of the target, relative to the root
+            of `repo`.
+        repo (str, optional): location of the DVC project or Git Repo.
+            Defaults to the current DVC project (found by walking up from the
+            current working directory tree).
+            It can be a URL or a file system path.
+            Both HTTP and SSH protocols are supported for online Git repos
+            (e.g. [user@]server:project.git).
+        rev (str, optional): Any `Git revision`_ such as a branch or tag name,
+            a commit hash or a dvc experiment name.
+            Defaults to HEAD.
+            If `repo` is not a Git repo, this option is ignored.
+        remote (str, optional): Name of the `DVC remote`_ used to form the
+            returned URL string.
+            Defaults to the `default remote`_ of `repo`.
+            For local projects, the cache is tried before the default remote.
+        config(dict, optional): config to be passed to the DVC repository.
+            Defaults to None.
+        remote_config(dict, optional): remote config to be passed to the DVC
+            repository.
+            Defaults to None.
+
+    Returns:
+        str: URL to the file or directory.
     """
     from dvc.config import NoRemoteError
     from dvc_data.index import StorageKeyError
 
-    repo_kwargs: dict[str, Any] = {}
-    if remote:
-        repo_kwargs["config"] = {"core": {"remote": remote}}
     with Repo.open(
-        repo, rev=rev, subrepos=True, uninitialized=True, **repo_kwargs
+        repo,
+        rev=rev,
+        subrepos=True,
+        uninitialized=True,
+        remote=remote,
+        config=config,
+        remote_config=remote_config,
     ) as _repo:
         index, entry = _repo.get_data_index_entry(path)
         with reraise(

dvc/api/dataset.py

@@ -0,0 +1,66 @@
+from typing import Literal, TypedDict, Union
+
+
+class DVCXDataset(TypedDict):
+    type: Literal["dvcx"]
+    name: str
+    version: int
+
+
+class DVCDataset(TypedDict):
+    type: Literal["dvc"]
+    url: str
+    path: str
+    sha: str
+
+
+class URLDataset(TypedDict):
+    type: Literal["url"]
+    files: list[str]
+    path: str
+
+
+def get(name: str) -> Union[DVCXDataset, DVCDataset, URLDataset]:
+    from difflib import get_close_matches
+
+    from dvc.fs import get_cloud_fs
+    from dvc.repo import Repo, datasets
+
+    repo = Repo()
+    try:
+        dataset = repo.datasets[name]
+    except datasets.DatasetNotFoundError as e:
+        add_note = getattr(e, "add_note", lambda _: None)
+        if matches := get_close_matches(name, repo.datasets):
+            add_note(f"Did you mean: {matches[0]!r}?")
+        raise
+
+    if dataset._invalidated:
+        raise ValueError(f"dataset not in sync. Sync with 'dvc ds update {name}'.")
+    if not dataset.lock:
+        raise ValueError("missing lock information")
+    if dataset.type == "dvc":
+        return DVCDataset(
+            type="dvc",
+            url=dataset.lock.url,
+            path=dataset.lock.path,
+            sha=dataset.lock.rev_lock,
+        )
+    if dataset.type == "dvcx":
+        return DVCXDataset(
+            type="dvcx", name=dataset.name_version[0], version=dataset.lock.version
+        )
+    if dataset.type == "url":
+        fs_cls, _, path = get_cloud_fs(repo.config, url=dataset.lock.url)
+        assert fs_cls
+        join_version = getattr(fs_cls, "join_version", lambda path, _: path)
+        protocol = fs_cls.protocol
+        versioned_path = join_version(path, dataset.lock.meta.version_id)
+        versioned_path = f"{protocol}://{versioned_path}"
+        files = [
+            join_version(
+                fs_cls.join(versioned_path, file.relpath), file.meta.version_id
+            )
+            for file in dataset.lock.files
+        ]
+        return URLDataset(type="url", files=files, path=versioned_path)

dvc/cli/parser.py

@@ -1,4 +1,5 @@
 """Main parser for the dvc cli."""
+
 import argparse
 import os
 from functools import lru_cache
@@ -17,6 +18,7 @@ from dvc.commands import (
     dag,
     data,
     data_sync,
+    dataset,
     destroy,
     diff,
     du,
@@ -67,6 +69,7 @@ COMMANDS = [
     dag,
     data,
     data_sync,
+    dataset,
     destroy,
     diff,
     du,

dvc/commands/add.py

@@ -51,6 +51,7 @@ class CmdAdd(CmdBase):
                 to_remote=self.args.to_remote,
                 remote_jobs=self.args.remote_jobs,
                 force=self.args.force,
+                relink=self.args.relink,
             )
         except FileNotFoundError:
             logger.exception("")
@@ -119,6 +120,13 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Override local file or folder if exists.",
     )
+    parser.add_argument(
+        "--no-relink",
+        dest="relink",
+        action="store_false",
+        help="Don't recreate links from cache to workspace.",
+    )
+    parser.set_defaults(relink=True)
     parser.add_argument(
         "targets", nargs="+", help="Input files/directories to add."
     ).complete = completion.FILE

dvc/commands/commit.py

@@ -20,6 +20,7 @@ class CmdCommit(CmdBase):
                     with_deps=self.args.with_deps,
                     recursive=self.args.recursive,
                     force=self.args.force,
+                    relink=self.args.relink,
                 )
             except DvcException:
                 logger.exception("failed to commit%s", (" " + target) if target else "")
@@ -64,6 +65,13 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Commit cache for subdirectories of the specified directory.",
     )
+    commit_parser.add_argument(
+        "--no-relink",
+        dest="relink",
+        action="store_false",
+        help="Don't recreate links from cache to workspace.",
+    )
+    commit_parser.set_defaults(relink=True)
     commit_parser.add_argument(
         "targets",
         nargs="*",

dvc/commands/dataset.py

@@ -0,0 +1,225 @@
+from typing import TYPE_CHECKING, Optional
+
+from dvc.cli import formatter
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.exceptions import DvcException
+from dvc.log import logger
+
+if TYPE_CHECKING:
+    from rich.text import Text
+
+    from dvc.repo.datasets import Dataset, FileInfo
+
+logger = logger.getChild(__name__)
+
+
+def diff_files(old: list["FileInfo"], new: list["FileInfo"]) -> dict[str, list[str]]:
+    old_files = {d.relpath: d for d in old}
+    new_files = {d.relpath: d for d in new}
+    rest = old_files.keys() & new_files.keys()
+    return {
+        "added": list(new_files.keys() - old_files.keys()),
+        "deleted": list(old_files.keys() - new_files.keys()),
+        "modified": [p for p in rest if new_files[p] != old_files[p]],
+    }
+
+
+class CmdDatasetAdd(CmdBase):
+    @classmethod
+    def display(cls, name: str, dataset: "Dataset", action: str = "Adding"):
+        from dvc.ui import ui
+
+        assert dataset.lock
+
+        url = dataset.spec.url
+        ver: str = ""
+        if dataset.type == "dvcx":
+            ver = f"v{dataset.lock.version}"
+        if dataset.type == "dvc":
+            if dataset.lock.path:
+                url = f"{url}:/{dataset.lock.path.lstrip('/')}"
+            if rev := dataset.lock.rev:
+                ver = rev
+
+        ver_part: Optional["Text"] = None
+        if ver:
+            ver_part = ui.rich_text.assemble(" @ ", (ver, "repr.number"))
+        text = ui.rich_text.assemble("(", (url, "repr.url"), ver_part or "", ")")
+        ui.write(action, ui.rich_text(name, "cyan"), text, styled=True)
+
+    def run(self):
+        if not self.args.dvc and self.args.rev:
+            raise DvcException("--rev can't be used without --dvc")
+        if not self.args.dvc and self.args.path:
+            raise DvcException("--path can't be used without --dvc")
+
+        d = vars(self.args)
+        for key in ["dvc", "dvcx", "url"]:
+            if url := d.pop(key, None):
+                d.update({"type": key, "url": url})
+                break
+
+        existing = self.repo.datasets.get(self.args.name)
+        with self.repo.scm_context:
+            if not self.args.force and existing:
+                path = self.repo.fs.relpath(existing.manifest_path)
+                raise DvcException(
+                    f"{self.args.name} already exists in {path}, "
+                    "use the --force to overwrite"
+                )
+            dataset = self.repo.datasets.add(**d)
+            self.display(self.args.name, dataset)
+            return 0
+
+
+class CmdDatasetUpdate(CmdBase):
+    def display(self, name: str, dataset: "Dataset", new: "Dataset"):
+        from dvc.commands.checkout import log_changes
+        from dvc.ui import ui
+
+        action = "Updating"
+        if not dataset.lock:
+            return CmdDatasetAdd.display(name, new, action)
+        if dataset == new:
+            ui.write("[yellow]Nothing to update[/]", styled=True)
+            return
+
+        assert new.lock
+
+        v: Optional[tuple[str, str]] = None
+        if dataset.type == "dvcx":
+            assert new.type == "dvcx"
+            if new.lock.version < dataset.lock.version:
+                action = "Downgrading"
+
+            v = (f"v{dataset.lock.version}", f"v{new.lock.version}")
+        if dataset.type == "dvc":
+            assert new.type == "dvc"
+            v = (f"{dataset.lock.rev_lock[:9]}", f"{new.lock.rev_lock[:9]}")
+
+        if v:
+            part = ui.rich_text.assemble(
+                (v[0], "repr.number"),
+                " -> ",
+                (v[1], "repr.number"),
+            )
+        else:
+            part = ui.rich_text(dataset.spec.url, "repr.url")
+        changes = ui.rich_text.assemble("(", part, ")")
+        ui.write(action, ui.rich_text(name, "cyan"), changes, styled=True)
+        if dataset.type == "url":
+            assert new.type == "url"
+            stats = diff_files(dataset.lock.files, new.lock.files)
+            log_changes(stats)
+
+    def run(self):
+        from difflib import get_close_matches
+
+        from dvc.repo.datasets import DatasetNotFoundError
+        from dvc.ui import ui
+
+        version = None
+        if self.args.rev:
+            try:
+                version = int(self.args.rev.lstrip("v"))
+            except ValueError:
+                version = self.args.rev
+
+        d = vars(self.args) | {"version": version}
+        with self.repo.scm_context:
+            try:
+                dataset, new = self.repo.datasets.update(**d)
+            except DatasetNotFoundError:
+                logger.exception("")
+                if matches := get_close_matches(self.args.name, self.repo.datasets):
+                    ui.write(
+                        "did you mean?",
+                        ui.rich_text(matches[0], "cyan"),
+                        stderr=True,
+                        styled=True,
+                    )
+                return 1
+            self.display(self.args.name, dataset, new)
+            return 0
+
+
+def add_parser(subparsers, parent_parser):
+    ds_parser = subparsers.add_parser(
+        "dataset",
+        aliases=["ds"],
+        parents=[parent_parser],
+        formatter_class=formatter.RawDescriptionHelpFormatter,
+    )
+    ds_subparsers = ds_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc dataset CMD --help` to display command-specific help.",
+        required=True,
+    )
+
+    dataset_add_help = "Add a dataset."
+    ds_add_parser = ds_subparsers.add_parser(
+        "add",
+        parents=[parent_parser],
+        description=append_doc_link(dataset_add_help, "dataset/add"),
+        formatter_class=formatter.RawTextHelpFormatter,
+        help=dataset_add_help,
+    )
+
+    url_exclusive_group = ds_add_parser.add_mutually_exclusive_group(required=True)
+    url_exclusive_group.add_argument(
+        "--dvcx", metavar="name", help="Name of the dvcx dataset to track"
+    )
+    url_exclusive_group.add_argument(
+        "--dvc",
+        help="Path or URL to a Git/DVC repository to track",
+        metavar="url",
+    )
+    url_exclusive_group.add_argument(
+        "--url",
+        help="""\
+URL of a cloud-versioned remote to track. Supported URLs:
+
+s3://bucket/key/path
+gs://bucket/path/to/file/or/dir
+azure://mycontainer/path
+remote://remote_name/path/to/file/or/dir (see `dvc remote`)
+""",
+    )
+    ds_add_parser.add_argument("name", help="Name of the dataset to add")
+    ds_add_parser.add_argument(
+        "--rev",
+        help="Git revision, e.g. SHA, branch, tag (only applicable with --dvc)",
+        metavar="<commit>",
+    )
+    ds_add_parser.add_argument(
+        "--path",
+        help="Path to a file or a directory within a git repository "
+        "(only applicable with --dvc)",
+    )
+    ds_add_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Overwrite existing dataset",
+    )
+    ds_add_parser.set_defaults(func=CmdDatasetAdd)
+
+    dataset_update_help = "Update a dataset."
+    ds_update_parser = ds_subparsers.add_parser(
+        "update",
+        parents=[parent_parser],
+        description=append_doc_link(dataset_update_help, "dataset/update"),
+        formatter_class=formatter.RawDescriptionHelpFormatter,
+        help=dataset_update_help,
+    )
+    ds_update_parser.add_argument("name", help="Name of the dataset to update")
+    ds_update_parser.add_argument(
+        "--rev",
+        "--version",
+        nargs="?",
+        help="DVCX dataset version or Git revision (e.g. SHA, branch, tag)",
+        metavar="<version>",
+    )
+    ds_update_parser.set_defaults(func=CmdDatasetUpdate)

dvc/commands/get.py

@@ -12,7 +12,13 @@ class CmdGet(CmdBaseNoRepo):
         from dvc.api import get_url
         from dvc.ui import ui
 
-        url = get_url(self.args.path, repo=self.args.url, rev=self.args.rev)
+        url = get_url(
+            self.args.path,
+            repo=self.args.url,
+            rev=self.args.rev,
+            remote=self.args.remote,
+            remote_config=self.args.remote_config,
+        )
         ui.write(url, force=True)
 
         return 0

dvc/commands/studio.py

@@ -13,11 +13,10 @@ logger = logger.getChild(__name__)
 
 class CmdStudioLogin(CmdConfig):
     def run(self):
-        from dvc_studio_client.auth import StudioAuthError, get_access_token
-
         from dvc.env import DVC_STUDIO_URL
         from dvc.ui import ui
         from dvc.utils.studio import STUDIO_URL
+        from dvc_studio_client.auth import StudioAuthError, get_access_token
 
         name = self.args.name
         hostname = self.args.hostname or os.environ.get(DVC_STUDIO_URL) or STUDIO_URL
@@ -46,6 +45,15 @@ class CmdStudioLogin(CmdConfig):
 
         self.save_config(hostname, access_token)
 
+        if not self.config["exp"].get("auto_push", True):
+            from dvc.ui import ui
+
+            ui.warn(
+                "exp.auto_push is disabled. \n"
+                "Enable with 'dvc config exp.auto_push true' "
+                "to automatically push experiments to Studio."
+            )
+
         config_path = self.config.files["global"]
         ui.write(f"Authentication complete. Saved token to {config_path}.")
         return 0
@@ -54,6 +62,8 @@ class CmdStudioLogin(CmdConfig):
         with self.config.edit("global") as conf:
             conf["studio"]["token"] = token
             conf["studio"]["url"] = hostname
+            if "auto_push" not in conf["exp"]:
+                conf["exp"]["auto_push"] = True
 
 
 class CmdStudioLogout(CmdConfig):

dvc/compare.py

@@ -88,12 +88,10 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
         return list(it)
 
     @overload
-    def __setitem__(self, item: int, value: Sequence["CellT"]) -> None:
-        ...
+    def __setitem__(self, item: int, value: Sequence["CellT"]) -> None: ...
 
     @overload
-    def __setitem__(self, item: slice, value: Iterable[Sequence["CellT"]]) -> None:
-        ...
+    def __setitem__(self, item: slice, value: Iterable[Sequence["CellT"]]) -> None: ...
 
     def __setitem__(self, item, value) -> None:
         it = value

dvc/config.py

@@ -1,4 +1,5 @@
 """DVC config objects."""
+
 import os
 import re
 from contextlib import contextmanager

dvc/config_schema.py

@@ -336,6 +336,8 @@ SCHEMA = {
         "params": str,
         "plots": str,
         "live": str,
+        "auto_push": Bool,
+        "git_remote": str,
     },
     "parsing": {
         "bool": All(Lower, Choices("store_true", "boolean_optional")),

dvc/database.py

@@ -5,9 +5,9 @@ from dataclasses import dataclass
 from tempfile import NamedTemporaryFile
 from typing import TYPE_CHECKING, Any, Callable, Optional, Union
 
-from sqlalchemy import create_engine
-from sqlalchemy.engine import make_url as _make_url
-from sqlalchemy.exc import NoSuchModuleError
+from sqlalchemy import create_engine  # type: ignore[import]
+from sqlalchemy.engine import make_url as _make_url  # type: ignore[import]
+from sqlalchemy.exc import NoSuchModuleError  # type: ignore[import]
 
 from dvc import env
 from dvc.exceptions import DvcException
@@ -17,7 +17,7 @@ from dvc.utils import env2bool
 
 if TYPE_CHECKING:
     from sqlalchemy.engine import URL, Connectable, Engine
-    from sqlalchemy.sql.expression import Selectable
+    from sqlalchemy.sql.expression import Selectable  # type: ignore[import]
 
 
 logger = logger.getChild(__name__)

dvc/dependency/dataset.py

@@ -3,6 +3,7 @@ from urllib.parse import urlparse
 
 from funcy import compact, merge
 
+from dvc.exceptions import DvcException
 from dvc_data.hashfile.hash_info import HashInfo
 
 from .db import AbstractDependency
@@ -47,9 +48,15 @@ class DatasetDependency(AbstractDependency):
         )
 
     def workspace_status(self):
-        registered = self.repo.index.datasets.get(self.name, {})
+        ds = self.repo.datasets[self.name]
+        if not ds.lock:
+            return {str(self): "not in sync"}
+
         info: dict[str, Any] = self.hash_info.value if self.hash_info else {}  # type: ignore[assignment]
-        if info != registered:
+        lock = self.repo.datasets._lock_from_info(info)
+        if not lock:
+            return {str(self): "new"}
+        if lock != ds.lock:
             return {str(self): "modified"}
         return {}
 
@@ -57,7 +64,15 @@ class DatasetDependency(AbstractDependency):
         return self.workspace_status()
 
     def get_hash(self):
-        return HashInfo(self.PARAM_DATASET, self.repo.index.datasets.get(self.name, {}))
+        ds = self.repo.datasets[self.name]
+        if not ds.lock:
+            if ds._invalidated:
+                raise DvcException(
+                    "Dataset information is not in sync. "
+                    f"Run 'dvc ds update {self.name}' to sync."
+                )
+            raise DvcException("Dataset information missing from dvc.lock file")
+        return HashInfo(self.PARAM_DATASET, ds.lock.to_dict())  # type: ignore[arg-type]
 
     def save(self):
         self.hash_info = self.get_hash()

dvc/dependency/db.py

@@ -11,6 +11,7 @@ from .base import Dependency
 
 if TYPE_CHECKING:
     from dvc.output import Output
+    from dvc.repo import Repo
     from dvc.stage import Stage
 
 logger = logger.getChild(__name__)
@@ -21,9 +22,12 @@ def download_progress(to: "Output") -> Iterator[Callable[[int], Any]]:
     from dvc.ui import ui
     from dvc.ui._rich_progress import DbDownloadProgress
 
-    with log_durations(logger.debug, f"Saving to {to}"), DbDownloadProgress(
-        console=ui.error_console,
-    ) as progress:
+    with (
+        log_durations(logger.debug, f"Saving to {to}"),
+        DbDownloadProgress(
+            console=ui.error_console,
+        ) as progress,
+    ):
         task = progress.add_task("Saving", total=None, output=to)
         yield lambda n: progress.advance(task, advance=n)
         progress.update(task, description="Saved", total=0)
@@ -33,7 +37,7 @@ class AbstractDependency(Dependency):
     """Dependency without workspace/fs/fs_path"""
 
     def __init__(self, stage: "Stage", info: dict[str, Any], *args, **kwargs):
-        self.repo = stage.repo
+        self.repo: "Repo" = stage.repo
         self.stage = stage
         self.fs = None
         self.fs_path = None

dvc/dvcfile.py

@@ -68,10 +68,9 @@ def is_git_ignored(repo, path):
 def check_dvcfile_path(repo, path):
     if not is_valid_filename(path):
         raise StageFileBadNameError(
-            "bad DVC file name '{}'. DVC files should be named "
-            "'{}' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
-                relpath(path), PROJECT_FILE, os.path.basename(path)
-            )
+            f"bad DVC file name '{relpath(path)}'. DVC files should be named "
+            f"'{PROJECT_FILE}' or have a '.dvc' suffix "
+            f"(e.g. '{os.path.basename(path)}.dvc')."
         )
 
     if is_git_ignored(repo, path):
@@ -172,6 +171,7 @@ class SingleStageFile(FileMixin):
     from dvc.stage.loader import SingleStageLoader as LOADER  # noqa: N814
 
     datasets: ClassVar[list[dict[str, Any]]] = []
+    datasets_lock: ClassVar[list[dict[str, Any]]] = []
     metrics: ClassVar[list[str]] = []
     plots: ClassVar[Any] = {}
     params: ClassVar[list[str]] = []
@@ -240,6 +240,26 @@ class ProjectFile(FileMixin):
         if update_lock:
             self._dump_lockfile(stage, **kwargs)
 
+    def dump_dataset(self, dataset):
+        with modify_yaml(self.path, fs=self.repo.fs) as data:
+            parsed = self.datasets if data else []
+            raw = data.setdefault("datasets", [])
+            loc = next(
+                (i for i, ds in enumerate(parsed) if ds["name"] == dataset["name"]),
+                None,
+            )
+            if loc is not None:
+                if raw[loc] != parsed[loc]:
+                    raise ParametrizedDumpError(
+                        "cannot update a parametrized dataset entry"
+                    )
+
+                apply_diff(dataset, raw[loc])
+                raw[loc] = dataset
+            else:
+                raw.append(dataset)
+        self.repo.scm_context.track_file(self.relpath)
+
     def _dump_lockfile(self, stage, **kwargs):
         self._lockfile.dump(stage, **kwargs)
 
@@ -293,24 +313,28 @@ class ProjectFile(FileMixin):
         return self.LOADER(self, self.contents, self.lockfile_contents)
 
     @property
-    def metrics(self) -> list[str]:
-        return self.contents.get("metrics", [])
+    def artifacts(self) -> dict[str, Optional[dict[str, Any]]]:
+        return self.resolver.resolve_artifacts()
 
     @property
-    def plots(self) -> Any:
-        return self.contents.get("plots", {})
+    def metrics(self) -> list[str]:
+        return self.resolver.resolve_metrics()
 
     @property
     def params(self) -> list[str]:
-        return self.contents.get("params", [])
+        return self.resolver.resolve_params()
+
+    @property
+    def plots(self) -> list[Any]:
+        return self.resolver.resolve_plots()
 
     @property
     def datasets(self) -> list[dict[str, Any]]:
-        return self.contents.get("datasets", [])
+        return self.resolver.resolve_datasets()
 
     @property
-    def artifacts(self) -> dict[str, Optional[dict[str, Any]]]:
-        return self.contents.get("artifacts", {})
+    def datasets_lock(self) -> list[dict[str, Any]]:
+        return self.lockfile_contents.get("datasets", [])
 
     def remove(self, force=False):
         if not force:
@@ -357,6 +381,24 @@ class Lockfile(FileMixin):
             self._check_gitignored()
             return {}, ""
 
+    def dump_dataset(self, dataset: dict):
+        with modify_yaml(self.path, fs=self.repo.fs) as data:
+            data.update({"schema": "2.0"})
+            if not data:
+                logger.info("Generating lock file '%s'", self.relpath)
+
+            datasets: list[dict] = data.setdefault("datasets", [])
+            loc = next(
+                (i for i, ds in enumerate(datasets) if ds["name"] == dataset["name"]),
+                None,
+            )
+            if loc is not None:
+                datasets[loc] = dataset
+            else:
+                datasets.append(dataset)
+            data.setdefault("stages", {})
+        self.repo.scm_context.track_file(self.relpath)
+
     def dump(self, stage, **kwargs):
         stage_data = serialize.to_lockfile(stage, **kwargs)
 

dvc/env.py

@@ -19,3 +19,4 @@ DVC_STUDIO_URL = "DVC_STUDIO_URL"
 DVC_SQLALCHEMY_ECHO = "DVC_SQLALCHEMY_ECHO"
 DVC_SYSTEM_CONFIG_DIR = "DVC_SYSTEM_CONFIG_DIR"
 DVC_UPDATER_ENDPOINT = "DVC_UPDATER_ENDPOINT"
+DVC_STAGE = "DVC_STAGE"

dvc/exceptions.py

@@ -1,4 +1,5 @@
 """Exceptions raised by the dvc."""
+
 import errno
 from typing import TYPE_CHECKING, Optional
 
@@ -132,11 +133,11 @@ class MoveNotDataSourceError(DvcException):
     def __init__(self, path):
         msg = (
             "move is not permitted for stages that are not data sources. "
-            "You need to either move '{path}' to a new location and edit "
-            "it by hand, or remove '{path}' and create a new one at the "
+            f"You need to either move {path!r} to a new location and edit "
+            f"it by hand, or remove {path!r} and create a new one at the "
             "desired location."
         )
-        super().__init__(msg.format(path=path))
+        super().__init__(msg)
 
 
 class NotDvcRepoError(DvcException):
@@ -270,9 +271,8 @@ class NoOutputInExternalRepoError(DvcException):
         from dvc.utils import relpath
 
         super().__init__(
-            "Output '{}' not found in target repository '{}'".format(
-                relpath(path, external_repo_path), external_repo_url
-            )
+            f"Output {relpath(path, external_repo_path)!r} "
+            f"not found in target repository '{external_repo_url}'"
         )
 
 

dvc/fs/__init__.py

@@ -1,10 +1,9 @@
 from typing import Optional
 from urllib.parse import urlparse
 
-from dvc_http import HTTPFileSystem, HTTPSFileSystem  # noqa: F401
-
 from dvc.config import ConfigError as RepoConfigError
 from dvc.config_schema import SCHEMA, Invalid
+from dvc_http import HTTPFileSystem, HTTPSFileSystem  # noqa: F401
 
 # pylint: disable=unused-import
 from dvc_objects.fs import (  # noqa: F401

dvc/fs/dvc_path.py

@@ -0,0 +1,54 @@
+"""UPath implementation for DVCFileSystem.
+
+This provides a `pathlib.Path` like interface to
+work with DVCFileSystem.
+
+Examples
+--------
+
+>>> from upath import UPath
+
+>>> local = UPath("dvc://path/to/local/repo")
+>>> https = UPath("dvc+https://github.com/iterative/example-get-started", rev="main")
+>>> ssh = UPath("dvc+ssh://[email protected]:iterative/example-get-started.git")
+"""
+
+from urllib.parse import urlsplit
+
+from upath import UPath
+
+
+class DVCPath(UPath):
+    @classmethod
+    def _transform_init_args(cls, args, protocol, storage_options):
+        if not args:
+            args = ("/",)
+        elif (
+            args
+            and "url" not in storage_options
+            and protocol in {"dvc+http", "dvc+https", "dvc+ssh"}
+        ):
+            url, *rest = args
+            url = urlsplit(str(url))
+            proto = protocol.split("+")[1]
+            if proto == "ssh":
+                base_url = url.netloc + url.path
+            else:
+                base_url = url._replace(scheme=proto).geturl()
+            storage_options["url"] = base_url
+            # Assume the given path is a root url
+            args = ("/", *rest)
+        return super()._transform_init_args(args, "dvc", storage_options)
+
+    def __str__(self):
+        s = super().__str__()
+        if url := self.storage_options.get("url"):
+            return s.replace("dvc://", f"dvc+{url}", 1)
+        return s
+
+    def with_segments(self, *pathsegments):
+        obj = super().with_segments(*pathsegments)
+        # cache filesystem, as dvcfs does not cache filesystem
+        # caveat: any joinpath operation will instantiate filesystem
+        obj._fs_cached = self.fs
+        return obj

dvc/output.py

@@ -814,7 +814,7 @@ class Output:
 
         ret: dict[str, Any] = {}
         with_files = (
-            (not self.IS_DEPENDENCY or self.stage.is_import)
+            (not self.IS_DEPENDENCY or kwargs.get("datasets") or self.stage.is_import)
             and self.hash_info.isdir
             and (kwargs.get("with_files") or self.files is not None)
         )
@@ -1365,7 +1365,7 @@ class Output:
                 dry_run=not self.use_cache,
             )
         except FileNotFoundError as exc:
-            if self.fs_path == path:
+            if not self.exists:
                 raise self.DoesNotExistError(self) from exc
             if not self.is_dir_checksum:
                 raise

dvc/parsing/__init__.py

@@ -21,6 +21,7 @@ from .context import (
     VarsAlreadyLoaded,
 )
 from .interpolate import (
+    check_expression,
     check_recursive_parse_errors,
     is_interpolated_string,
     recurse,
@@ -38,10 +39,16 @@ if TYPE_CHECKING:
 
 logger = logger.getChild(__name__)
 
-STAGES_KWD = "stages"
 VARS_KWD = "vars"
 WDIR_KWD = "wdir"
+
+ARTIFACTS_KWD = "artifacts"
+DATASETS_KWD = "datasets"
+METRICS_KWD = "metrics"
 PARAMS_KWD = "params"
+PLOTS_KWD = "plots"
+STAGES_KWD = "stages"
+
 FOREACH_KWD = "foreach"
 MATRIX_KWD = "matrix"
 DO_KWD = "do"
@@ -163,6 +170,27 @@ class DataResolver:
             for name, definition in stages_data.items()
         }
 
+        self.artifacts = [
+            ArtifactDefinition(self, self.context, name, definition, ARTIFACTS_KWD)
+            for name, definition in d.get(ARTIFACTS_KWD, {}).items()
+        ]
+        self.datasets = [
+            TopDefinition(self, self.context, str(i), definition, DATASETS_KWD)
+            for i, definition in enumerate(d.get(DATASETS_KWD, []))
+        ]
+        self.metrics = [
+            TopDefinition(self, self.context, str(i), definition, METRICS_KWD)
+            for i, definition in enumerate(d.get(METRICS_KWD, []))
+        ]
+        self.params = [
+            TopDefinition(self, self.context, str(i), definition, PARAMS_KWD)
+            for i, definition in enumerate(d.get(PARAMS_KWD, []))
+        ]
+        self.plots = [
+            TopDefinition(self, self.context, str(i), definition, PLOTS_KWD)
+            for i, definition in enumerate(d.get(PLOTS_KWD, []))
+        ]
+
     def resolve_one(self, name: str):
         group, key = split_group_name(name)
 
@@ -186,6 +214,27 @@ class DataResolver:
         logger.trace("Resolved dvc.yaml:\n%s", data)
         return {STAGES_KWD: data}
 
+    # Top-level sections are eagerly evaluated, whereas stages are lazily evaluated,
+    # one-by-one.
+
+    def resolve_artifacts(self) -> dict[str, Optional[dict[str, Any]]]:
+        d: dict[str, Optional[dict[str, Any]]] = {}
+        for item in self.artifacts:
+            d.update(item.resolve())
+        return d
+
+    def resolve_datasets(self) -> list[dict[str, Any]]:
+        return [item.resolve() for item in self.datasets]
+
+    def resolve_metrics(self) -> list[str]:
+        return [item.resolve() for item in self.metrics]
+
+    def resolve_params(self) -> list[str]:
+        return [item.resolve() for item in self.params]
+
+    def resolve_plots(self) -> list[Any]:
+        return [item.resolve() for item in self.plots]
+
     def has_key(self, key: str):
         return self._has_group_and_key(*split_group_name(key))
 
@@ -565,3 +614,43 @@ class MatrixDefinition:
                 return entry.resolve_stage(skip_checks=True)
             except ContextError as exc:
                 format_and_raise(exc, f"stage '{generated}'", self.relpath)
+
+
+class TopDefinition:
+    def __init__(
+        self,
+        resolver: DataResolver,
+        context: Context,
+        name: str,
+        definition: "Any",
+        where: str,
+    ):
+        self.resolver = resolver
+        self.context = context
+        self.name = name
+        self.definition = definition
+        self.where = where
+        self.relpath = self.resolver.relpath
+
+    def resolve(self):
+        try:
+            check_recursive_parse_errors(self.definition)
+            return self.context.resolve(self.definition)
+        except (ParseError, ContextError) as exc:
+            format_and_raise(exc, f"'{self.where}.{self.name}'", self.relpath)
+
+
+class ArtifactDefinition(TopDefinition):
+    def resolve(self) -> dict[str, Optional[dict[str, Any]]]:
+        try:
+            check_expression(self.name)
+            name = self.context.resolve(self.name)
+            if not isinstance(name, str):
+                typ = type(name).__name__
+                raise ResolveError(
+                    f"failed to resolve '{self.where}.{self.name}'"
+                    f" in '{self.relpath}': expected str, got " + typ
+                )
+        except (ParseError, ContextError) as exc:
+            format_and_raise(exc, f"'{self.where}.{self.name}'", self.relpath)
+        return {name: super().resolve()}

dvc/progress.py

@@ -1,4 +1,5 @@
 """Manages progress bars for DVC repo."""
+
 import logging
 import sys
 from threading import RLock

dvc/repo/__init__.py

@@ -13,7 +13,6 @@ from dvc.exceptions import (
 )
 from dvc.ignore import DvcIgnoreFilter
 from dvc.log import logger
-from dvc.utils import as_posix
 from dvc.utils.objects import cached_property
 
 if TYPE_CHECKING:
@@ -151,6 +150,7 @@ class Repo:
         from dvc.fs import GitFileSystem, LocalFileSystem, localfs
         from dvc.lock import LockNoop, make_lock
         from dvc.repo.artifacts import Artifacts
+        from dvc.repo.datasets import Datasets
         from dvc.repo.metrics import Metrics
         from dvc.repo.params import Params
         from dvc.repo.plots import Plots
@@ -221,6 +221,7 @@ class Repo:
         self.plots: Plots = Plots(self)
         self.params: Params = Params(self)
         self.artifacts: Artifacts = Artifacts(self)
+        self.datasets: Datasets = Datasets(self)
 
         self.stage_collection_error_handler: Optional[
             Callable[[str, Exception], None]
@@ -351,16 +352,6 @@ class Repo:
         # fs.
         self._reset()
 
-    @property
-    def subrepo_relpath(self) -> str:
-        from dvc.fs import GitFileSystem
-
-        scm_root_dir = "/" if isinstance(self.fs, GitFileSystem) else self.scm.root_dir
-
-        relpath = as_posix(self.fs.relpath(self.root_dir, scm_root_dir))
-
-        return "" if relpath == "." else relpath
-
     @property
     def data_index(self) -> "DataIndex":
         from dvc_data.index import DataIndex
@@ -560,11 +551,7 @@ class Repo:
 
             if out.protocol == "local" and match(fs_path, out.fs_path):
                 return True
-
-            if recursive and out.fs.isin(out.fs_path, fs_path):
-                return True
-
-            return False
+            return recursive and out.fs.isin(out.fs_path, fs_path)
 
         matched = list(filter(func, outs))
         if not matched:
@@ -664,6 +651,7 @@ class Repo:
 
     def _reset(self):
         self.scm._reset()
+        self.datasets._reset()
         self.state.close()
         if "dvcfs" in self.__dict__:
             self.dvcfs.close()

dvc/repo/add.py

@@ -171,11 +171,16 @@ def _add_transfer(
     stage.dump()
 
 
-def _add(stage: "Stage", source: Optional[str] = None, no_commit: bool = False) -> None:
+def _add(
+    stage: "Stage",
+    source: Optional[str] = None,
+    no_commit: bool = False,
+    relink: bool = True,
+) -> None:
     out = stage.outs[0]
     path = out.fs.abspath(source) if source else None
     try:
-        stage.add_outs(path, no_commit=no_commit)
+        stage.add_outs(path, no_commit=no_commit, relink=relink)
     except CacheLinkError:
         stage.dump()
         raise
@@ -194,6 +199,7 @@ def add(
     to_remote: bool = False,
     remote_jobs: Optional[int] = None,
     force: bool = False,
+    relink: bool = True,
 ) -> list["Stage"]:
     add_targets = find_targets(targets, glob=glob)
     if not add_targets:
@@ -224,7 +230,12 @@ def add(
     with warn_link_failures() as link_failures:
         for source, (stage, output_exists) in progress_iter(stages_with_targets):
             try:
-                _add(stage, source if output_exists else None, no_commit=no_commit)
+                _add(
+                    stage,
+                    source if output_exists else None,
+                    no_commit=no_commit,
+                    relink=relink,
+                )
             except CacheLinkError:
                 link_failures.append(stage.relpath)
     return stages

dvc/repo/artifacts.py

@@ -12,7 +12,7 @@ from dvc.exceptions import (
     InvalidArgumentError,
 )
 from dvc.log import logger
-from dvc.utils import relpath, resolve_output
+from dvc.utils import as_posix, relpath, resolve_output
 from dvc.utils.objects import cached_property
 from dvc.utils.serialize import modify_yaml
 
@@ -99,7 +99,7 @@ class Artifacts:
         """Read artifacts from dvc.yaml."""
         artifacts: dict[str, dict[str, Artifact]] = {}
         for dvcfile, dvcfile_artifacts in self.repo.index._artifacts.items():
-            dvcyaml = relpath(dvcfile, self.repo.root_dir)
+            dvcyaml = self.repo.fs.relpath(dvcfile, self.repo.root_dir)
             artifacts[dvcyaml] = {}
             for name, value in dvcfile_artifacts.items():
                 try:
@@ -147,8 +147,8 @@ class Artifacts:
         gto_tags: list["GTOTag"] = sort_versions(parse_tag(tag) for tag in tags)
         return gto_tags[0].tag.target
 
-    def get_path(self, name: str):
-        """Return repo fspath for the given artifact."""
+    @classmethod
+    def parse_path(cls, name: str) -> tuple[Optional[str], str]:
         from gto.constants import SEPARATOR_IN_NAME, fullname_re
 
         name = _reformat_name(name)
@@ -158,13 +158,37 @@ class Artifacts:
         dirname = m.group("dirname")
         if dirname:
             dirname = dirname.rstrip(SEPARATOR_IN_NAME)
-        dvcyaml = os.path.join(dirname, PROJECT_FILE) if dirname else PROJECT_FILE
-        artifact_name = m.group("name")
+
+        return dirname, m.group("name")
+
+    def get_path(self, name: str):
+        """Return fspath for the given artifact relative to the git root."""
+        from dvc.fs import GitFileSystem
+
+        dirname, artifact_name = self.parse_path(name)
+        # `name`/`dirname` are expected to be a git root relative.
+        # We convert it to dvc-root relative path so that we can read artifacts
+        # from dvc.yaml file.
+        # But we return dirname intact, as we want to return a git-root relative path.
+        # This is useful when reading from `dvcfs` from remote.
+        fs = self.repo.fs
+        assert self.scm
+        if isinstance(fs, GitFileSystem):
+            scm_root = fs.root_marker
+        else:
+            scm_root = self.scm.root_dir
+
+        dirparts = posixpath.normpath(dirname).split(posixpath.sep) if dirname else ()
+        abspath = fs.join(scm_root, *dirparts, PROJECT_FILE)
+        rela = fs.relpath(abspath, self.repo.root_dir)
         try:
-            artifact = self.read()[dvcyaml][artifact_name]
+            artifact = self.read()[rela][artifact_name]
         except KeyError as exc:
             raise ArtifactNotFoundError(name) from exc
-        return os.path.join(dirname, artifact.path) if dirname else artifact.path
+
+        path = posixpath.join(dirname or "", artifact.path)
+        parts = posixpath.normpath(path).split(posixpath.sep)
+        return os.path.join(*parts)
 
     def download(
         self,
@@ -177,15 +201,28 @@ class Artifacts:
     ) -> tuple[int, str]:
         """Download the specified artifact."""
         from dvc.fs import download as fs_download
+        from dvc.repo import Repo
 
         logger.debug("Trying to download artifact '%s' via DVC", name)
         rev = self.get_rev(name, version=version, stage=stage)
+
+        dirname, _ = self.parse_path(name)
         with self.repo.switch(rev):
-            path = self.get_path(name)
+            root = self.repo.fs.root_marker
+            _dirname = self.repo.fs.join(root, dirname) if dirname else root
+            with Repo(_dirname, fs=self.repo.fs, scm=self.repo.scm) as r:
+                path = r.artifacts.get_path(name)
+                path = self.repo.fs.join(root, as_posix(path))
+                path = self.repo.fs.relpath(path, self.repo.root_dir)
+                # when the `repo` is a subrepo, the path `/subrepo/myart.pkl` for dvcfs
+                # should be translated as `/myart.pkl`,
+                # i.e. relative to the root of the subrepo
+                path = self.repo.fs.join(root, path)
+                path = self.repo.fs.normpath(path)
+
             out = resolve_output(path, out, force=force)
             fs = self.repo.dvcfs
-            fs_path = fs.from_os_path(path)
-            count = fs_download(fs, fs_path, os.path.abspath(out), jobs=jobs)
+            count = fs_download(fs, path, os.path.abspath(out), jobs=jobs)
         return count, out
 
     @staticmethod
@@ -200,10 +237,9 @@ class Artifacts:
         dvc_studio_config: Optional[dict[str, Any]] = None,
         **kwargs,
     ) -> tuple[int, str]:
-        from dvc_studio_client.model_registry import get_download_uris
-
         from dvc.fs import HTTPFileSystem, generic, localfs
         from dvc.fs.callbacks import TqdmCallback
+        from dvc_studio_client.model_registry import get_download_uris
 
         logger.debug("Trying to download artifact '%s' via studio", name)
         out = out or os.getcwd()

dvc/repo/datasets.py

@@ -0,0 +1,422 @@
+import os
+from collections.abc import Iterator, Mapping
+from datetime import datetime
+from functools import cached_property
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, Union, cast
+from urllib.parse import urlparse
+
+from attrs import Attribute, AttrsInstance, asdict, evolve, field, fields, frozen
+from attrs.converters import default_if_none
+
+from dvc.dvcfile import Lockfile, ProjectFile
+from dvc.exceptions import DvcException
+from dvc.log import logger
+from dvc.types import StrPath
+from dvc_data.hashfile.meta import Meta
+
+if TYPE_CHECKING:
+    from dvcx.dataset import DatasetRecord, DatasetVersion  # type: ignore[import]
+    from typing_extensions import Self
+
+    from dvc.repo import Repo
+
+
+logger = logger.getChild(__name__)
+
+
+def _get_dataset_record(name: str) -> "DatasetRecord":
+    from dvc.exceptions import DvcException
+
+    try:
+        from dvcx.catalog import get_catalog  # type: ignore[import]
+
+    except ImportError as exc:
+        raise DvcException("dvcx is not installed") from exc
+
+    catalog = get_catalog()
+    return catalog.get_remote_dataset(name)
+
+
+def _get_dataset_info(
+    name: str, record: Optional["DatasetRecord"] = None, version: Optional[int] = None
+) -> "DatasetVersion":
+    record = record or _get_dataset_record(name)
+    assert record
+    v = record.latest_version if version is None else version
+    assert v is not None
+    return record.get_version(v)
+
+
+def default_str(v) -> str:
+    return default_if_none("")(v)
+
+
+def to_datetime(d: Union[str, datetime]) -> datetime:
+    return datetime.fromisoformat(d) if isinstance(d, str) else d
+
+
+def ensure(cls):
+    def inner(v):
+        return cls.from_dict(v) if isinstance(v, dict) else v
+
+    return inner
+
+
+class SerDe:
+    def to_dict(self: AttrsInstance) -> dict[str, Any]:
+        def filter_defaults(attr: Attribute, v: Any):
+            if attr.metadata.get("exclude_falsy", False) and not v:
+                return False
+            return attr.default != v
+
+        def value_serializer(_inst, _field, v):
+            return v.isoformat() if isinstance(v, datetime) else v
+
+        return asdict(self, filter=filter_defaults, value_serializer=value_serializer)
+
+    @classmethod
+    def from_dict(cls: type["Self"], d: dict[str, Any]) -> "Self":
+        _fields = fields(cast("type[AttrsInstance]", cls))
+        kwargs = {f.name: d[f.name] for f in _fields if f.name in d}
+        return cls(**kwargs)
+
+
+@frozen(kw_only=True)
+class DatasetSpec(SerDe):
+    name: str
+    url: str
+    type: Literal["dvc", "dvcx", "url"]
+
+
+@frozen(kw_only=True)
+class DVCDatasetSpec(DatasetSpec):
+    type: Literal["dvc"]
+    path: str = field(default="", converter=default_str)
+    rev: Optional[str] = None
+
+
+@frozen(kw_only=True, order=True)
+class FileInfo(SerDe):
+    relpath: str
+    meta: Meta = field(order=False, converter=ensure(Meta))  # type: ignore[misc]
+
+
+@frozen(kw_only=True)
+class DVCDatasetLock(DVCDatasetSpec):
+    rev_lock: str
+
+
+@frozen(kw_only=True)
+class DVCXDatasetLock(DatasetSpec):
+    version: int
+    created_at: datetime = field(converter=to_datetime)
+
+
+@frozen(kw_only=True)
+class URLDatasetLock(DatasetSpec):
+    meta: Meta = field(converter=ensure(Meta))  # type: ignore[misc]
+    files: list[FileInfo] = field(
+        factory=list,
+        converter=lambda f: sorted(map(ensure(FileInfo), f)),
+        metadata={"exclude_falsy": True},
+    )
+
+
+def to_spec(lock: "Lock") -> "Spec":
+    cls = DVCDatasetSpec if lock.type == "dvc" else DatasetSpec
+    return cls(**{f.name: getattr(lock, f.name) for f in fields(cls)})
+
+
+@frozen(kw_only=True)
+class DVCDataset:
+    manifest_path: str
+    spec: DVCDatasetSpec
+    lock: Optional[DVCDatasetLock] = None
+    _invalidated: bool = field(default=False, eq=False, repr=False)
+
+    type: ClassVar[Literal["dvc"]] = "dvc"
+
+    def update(self, repo, rev: Optional[str] = None, **kwargs) -> "Self":
+        from dvc.dependency import RepoDependency
+
+        spec = self.spec
+        if rev:
+            spec = evolve(self.spec, rev=rev)
+
+        def_repo = {
+            RepoDependency.PARAM_REV: spec.rev,
+            RepoDependency.PARAM_URL: spec.url,
+        }
+        dep = RepoDependency(def_repo, None, spec.path, repo=repo)  # type: ignore[arg-type]
+        dep.save()
+        d = dep.dumpd()
+
+        repo_info = d[RepoDependency.PARAM_REPO]
+        assert isinstance(repo_info, dict)
+        rev_lock = repo_info[RepoDependency.PARAM_REV_LOCK]
+        lock = DVCDatasetLock(**spec.to_dict(), rev_lock=rev_lock)
+        return evolve(self, spec=spec, lock=lock)
+
+
+@frozen(kw_only=True)
+class DVCXDataset:
+    manifest_path: str
+    spec: "DatasetSpec"
+    lock: "Optional[DVCXDatasetLock]" = field(default=None)
+    _invalidated: bool = field(default=False, eq=False, repr=False)
+
+    type: ClassVar[Literal["dvcx"]] = "dvcx"
+
+    @property
+    def pinned(self) -> bool:
+        return self.name_version[1] is not None
+
+    @property
+    def name_version(self) -> tuple[str, Optional[int]]:
+        url = urlparse(self.spec.url)
+        path = url.netloc + url.path
+        parts = path.split("@v")
+        assert parts
+
+        name = parts[0]
+        version = int(parts[1]) if len(parts) > 1 else None
+        return name, version
+
+    def update(
+        self,
+        repo,  # noqa: ARG002
+        record: Optional["DatasetRecord"] = None,
+        version: Optional[int] = None,
+        **kwargs,
+    ) -> "Self":
+        name, _version = self.name_version
+        version = version if version is not None else _version
+        version_info = _get_dataset_info(name, record=record, version=version)
+        lock = DVCXDatasetLock(
+            **self.spec.to_dict(),
+            version=version_info.version,
+            created_at=version_info.created_at,
+        )
+        return evolve(self, lock=lock)
+
+
+@frozen(kw_only=True)
+class URLDataset:
+    manifest_path: str
+    spec: "DatasetSpec"
+    lock: "Optional[URLDatasetLock]" = None
+    _invalidated: bool = field(default=False, eq=False, repr=False)
+
+    type: ClassVar[Literal["url"]] = "url"
+
+    def update(self, repo, **kwargs):
+        from dvc.dependency import Dependency
+
+        dep = Dependency(
+            None, self.spec.url, repo=repo, fs_config={"version_aware": True}
+        )
+        dep.save()
+        d = dep.dumpd(datasets=True)
+        files = [
+            FileInfo(relpath=info["relpath"], meta=Meta.from_dict(info))
+            for info in d.get("files", [])
+        ]
+        lock = URLDatasetLock(**self.spec.to_dict(), meta=dep.meta, files=files)
+        return evolve(self, lock=lock)
+
+
+Lock = Union[DVCDatasetLock, DVCXDatasetLock, URLDatasetLock]
+Spec = Union[DatasetSpec, DVCDatasetSpec]
+Dataset = Union[DVCDataset, DVCXDataset, URLDataset]
+
+
+class DatasetNotFoundError(DvcException, KeyError):
+    def __init__(self, name, *args):
+        self.name = name
+        super().__init__("dataset not found", *args)
+
+    def __str__(self) -> str:
+        return self.msg
+
+
+class Datasets(Mapping[str, Dataset]):
+    def __init__(self, repo: "Repo") -> None:
+        self.repo: "Repo" = repo
+
+    def __repr__(self):
+        return repr(dict(self))
+
+    def __rich_repr__(self):
+        yield dict(self)
+
+    def __getitem__(self, name: str) -> Dataset:
+        try:
+            return self._datasets[name]
+        except KeyError as exc:
+            raise DatasetNotFoundError(name) from exc
+
+    def __setitem__(self, name: str, dataset: Dataset) -> None:
+        self._datasets[name] = dataset
+
+    def __contains__(self, name: object) -> bool:
+        return name in self._datasets
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._datasets)
+
+    def __len__(self) -> int:
+        return len(self._datasets)
+
+    @cached_property
+    def _spec(self) -> dict[str, tuple[str, dict[str, Any]]]:
+        return {
+            dataset["name"]: (path, dataset)
+            for path, datasets in self.repo.index._datasets.items()
+            for dataset in datasets
+        }
+
+    @cached_property
+    def _lock(self) -> dict[str, Optional[dict[str, Any]]]:
+        datasets_lock = self.repo.index._datasets_lock
+
+        def find(path, name) -> Optional[dict[str, Any]]:
+            # only look for `name` in the lock file next to the
+            # corresponding `dvc.yaml` file
+            lock = datasets_lock.get(path, [])
+            return next((dataset for dataset in lock if dataset["name"] == name), None)
+
+        return {ds["name"]: find(path, name) for name, (path, ds) in self._spec.items()}
+
+    @cached_property
+    def _datasets(self) -> dict[str, Dataset]:
+        return {
+            name: self._build_dataset(path, spec, self._lock[name])
+            for name, (path, spec) in self._spec.items()
+        }
+
+    def _reset(self) -> None:
+        self.__dict__.pop("_spec", None)
+        self.__dict__.pop("_lock", None)
+        self.__dict__.pop("_datasets", None)
+
+    @staticmethod
+    def _spec_from_info(spec: dict[str, Any]) -> Spec:
+        typ = spec.get("type")
+        if not typ:
+            raise ValueError("type should be present in spec")
+        if typ == "dvc":
+            return DVCDatasetSpec.from_dict(spec)
+        if typ in {"dvcx", "url"}:
+            return DatasetSpec.from_dict(spec)
+        raise ValueError(f"unknown dataset type: {spec.get('type', '')}")
+
+    @staticmethod
+    def _lock_from_info(lock: Optional[dict[str, Any]]) -> Optional[Lock]:
+        kl = {"dvc": DVCDatasetLock, "dvcx": DVCXDatasetLock, "url": URLDatasetLock}
+        if lock and (cls := kl.get(lock.get("type", ""))):  # type: ignore[assignment]
+            return cls.from_dict(lock)  # type: ignore[attr-defined]
+        return None
+
+    @classmethod
+    def _build_dataset(
+        cls,
+        manifest_path: str,
+        spec_data: dict[str, Any],
+        lock_data: Optional[dict[str, Any]] = None,
+    ) -> Dataset:
+        _invalidated = False
+        spec = cls._spec_from_info(spec_data)
+        lock = cls._lock_from_info(lock_data)
+        # if dvc.lock and dvc.yaml file are not in sync, we invalidate the lock.
+        if lock is not None and to_spec(lock) != spec:
+            logger.debug(
+                "invalidated lock data for %s in %s",
+                spec.name,
+                manifest_path,
+            )
+            _invalidated = True  # signal is used during `dvc repro`/`dvc status`.
+            lock = None
+
+        assert isinstance(spec, DatasetSpec)
+        if spec.type == "dvc":
+            assert lock is None or isinstance(lock, DVCDatasetLock)
+            assert isinstance(spec, DVCDatasetSpec)
+            return DVCDataset(
+                manifest_path=manifest_path,
+                spec=spec,
+                lock=lock,
+                invalidated=_invalidated,
+            )
+        if spec.type == "url":
+            assert lock is None or isinstance(lock, URLDatasetLock)
+            return URLDataset(
+                manifest_path=manifest_path,
+                spec=spec,
+                lock=lock,
+                invalidated=_invalidated,
+            )
+        if spec.type == "dvcx":
+            assert lock is None or isinstance(lock, DVCXDatasetLock)
+            return DVCXDataset(
+                manifest_path=manifest_path,
+                spec=spec,
+                lock=lock,
+                invalidated=_invalidated,
+            )
+        raise ValueError(f"unknown dataset type: {spec.type!r}")
+
+    def add(
+        self,
+        name: str,
+        url: str,
+        type: str,  # noqa: A002
+        manifest_path: StrPath = "dvc.yaml",
+        **kwargs: Any,
+    ) -> Dataset:
+        assert type in {"dvc", "dvcx", "url"}
+        kwargs.update({"name": name, "url": url, "type": type})
+        dataset = self._build_dataset(os.path.abspath(manifest_path), kwargs)
+        dataset = dataset.update(self.repo)
+
+        self.dump(dataset)
+        self[name] = dataset
+        return dataset
+
+    def update(self, name, **kwargs) -> tuple[Dataset, Dataset]:
+        dataset = self[name]
+        version = kwargs.get("version")
+
+        if dataset.type == "url" and (version or kwargs.get("rev")):
+            raise ValueError("cannot update version/revision for a url")
+        if dataset.type == "dvcx" and version is not None:
+            if not isinstance(version, int):
+                raise TypeError(
+                    f"dvcx version has to be an integer, got {type(version).__name__!r}"
+                )
+            if version < 1:
+                raise ValueError(f"dvcx version should be >=1, got {version}")
+
+        new = dataset.update(self.repo, **kwargs)
+
+        self.dump(new, old=dataset)
+        self[name] = new
+        return dataset, new
+
+    def _dump_spec(self, manifest_path: StrPath, spec: Spec) -> None:
+        spec_data = spec.to_dict()
+        assert spec_data.keys() & {"type", "name", "url"}
+        project_file = ProjectFile(self.repo, manifest_path)
+        project_file.dump_dataset(spec_data)
+
+    def _dump_lock(self, manifest_path: StrPath, lock: Lock) -> None:
+        lock_data = lock.to_dict()
+        assert lock_data.keys() & {"type", "name", "url"}
+        lockfile = Lockfile(self.repo, Path(manifest_path).with_suffix(".lock"))
+        lockfile.dump_dataset(lock_data)
+
+    def dump(self, dataset: Dataset, old: Optional[Dataset] = None) -> None:
+        if not old or old.spec != dataset.spec:
+            self._dump_spec(dataset.manifest_path, dataset.spec)
+        if dataset.lock and (not old or old.lock != dataset.lock):
+            self._dump_lock(dataset.manifest_path, dataset.lock)

dvc/repo/experiments/collect.py

@@ -78,7 +78,7 @@ def _collect_rev(
             timestamp: Optional[datetime] = None
         else:
             commit = repo.scm.resolve_commit(rev)
-            timestamp = datetime.fromtimestamp(commit.commit_time)
+            timestamp = datetime.fromtimestamp(commit.commit_time)  # noqa: DTZ006
 
         return SerializableExp.from_repo(
             repo,
@@ -338,6 +338,6 @@ def _sorted_ranges(exp_ranges: Iterable["ExpRange"]) -> list["ExpRange"]:
         if head_exp and head_exp.data and head_exp.data.timestamp:
             return head_exp.data.timestamp, head_exp.rev
 
-        return datetime.fromtimestamp(0), ""
+        return datetime.fromtimestamp(0), ""  # noqa: DTZ006
 
     return sorted(exp_ranges, key=_head_timestamp, reverse=True)

dvc/repo/experiments/executor/base.py

@@ -23,7 +23,11 @@ from dvc.repo.params.show import _collect_top_level_params
 from dvc.stage.serialize import to_lockfile
 from dvc.utils import dict_sha256, env2bool, relpath
 from dvc.utils.fs import remove
-from dvc.utils.studio import env_to_config
+from dvc.utils.studio import (
+    env_to_config,
+    get_repo_url,
+    get_subrepo_relpath,
+)
 
 if TYPE_CHECKING:
     from queue import Queue
@@ -294,13 +298,16 @@ class BaseExecutor(ABC):
             exp_hash = cls.hash_exp(stages)
             if include_untracked:
                 dvc.scm.add(include_untracked, force=True)  # type: ignore[call-arg]
-            cls.commit(
-                dvc.scm,  # type: ignore[arg-type]
-                exp_hash,
-                exp_name=info.name,
-                force=force,
-                message=message,
-            )
+
+            with cls.auto_push(dvc):
+                cls.commit(
+                    dvc.scm,  # type: ignore[arg-type]
+                    exp_hash,
+                    exp_name=info.name,
+                    force=force,
+                    message=message,
+                )
+
             ref: Optional[str] = dvc.scm.get_ref(EXEC_BRANCH, follow=False)
             exp_ref = ExpRefInfo.from_ref(ref) if ref else None
             untracked = dvc.scm.untracked_files()
@@ -457,9 +464,6 @@ class BaseExecutor(ABC):
         from dvc.repo.checkout import checkout as dvc_checkout
         from dvc.ui import ui
 
-        auto_push = env2bool(DVC_EXP_AUTO_PUSH)
-        git_remote = os.getenv(DVC_EXP_GIT_REMOTE, None)
-
         if queue is not None:
             queue.put((rev, os.getpid()))
         if log_errors and log_level is not None:
@@ -480,9 +484,6 @@ class BaseExecutor(ABC):
             message=message,
             **kwargs,
         ) as dvc:
-            if auto_push:
-                cls._validate_remotes(dvc, git_remote)
-
             args, kwargs = cls._repro_args(dvc)
             if args:
                 targets: Optional[Union[list, str]] = args[0]
@@ -516,8 +517,6 @@ class BaseExecutor(ABC):
                     dvc,
                     info,
                     exp_hash,
-                    auto_push,
-                    git_remote,
                     repro_force,
                     message=message,
                 )
@@ -547,20 +546,18 @@ class BaseExecutor(ABC):
         dvc,
         info,
         exp_hash,
-        auto_push,
-        git_remote,
         repro_force,
         message: Optional[str] = None,
     ) -> tuple[Optional[str], Optional["ExpRefInfo"], bool]:
-        cls.commit(
-            dvc.scm,
-            exp_hash,
-            exp_name=info.name,
-            force=repro_force,
-            message=message,
-        )
-        if auto_push:
-            cls._auto_push(dvc, dvc.scm, git_remote)
+        with cls.auto_push(dvc):
+            cls.commit(
+                dvc.scm,
+                exp_hash,
+                exp_name=info.name,
+                force=repro_force,
+                message=message,
+            )
+
         ref: Optional[str] = dvc.scm.get_ref(EXEC_BRANCH, follow=False)
         exp_ref: Optional["ExpRefInfo"] = ExpRefInfo.from_ref(ref) if ref else None
         if cls.WARN_UNTRACKED:
@@ -588,9 +585,8 @@ class BaseExecutor(ABC):
         message: Optional[str] = None,
         **kwargs,
     ) -> Iterator["Repo"]:
-        from dvc_studio_client.post_live_metrics import post_live_metrics
-
         from dvc.repo import Repo
+        from dvc_studio_client.post_live_metrics import post_live_metrics
 
         with Repo(os.path.join(info.root_dir, info.dvc_dir)) as dvc:
             info.status = TaskStatus.RUNNING
@@ -615,6 +611,8 @@ class BaseExecutor(ABC):
             # inferring repo url will fail if not set here
             run_env_config = env_to_config(kwargs.get("run_env", {}))
             dvc_studio_config = run_env_config | dvc_studio_config
+            # override studio repo url if exp git remote set
+            repo_url = get_repo_url(dvc)
             try:
                 post_live_metrics(
                     "start",
@@ -624,6 +622,8 @@ class BaseExecutor(ABC):
                     params=to_studio_params(dvc.params.show()),
                     dvc_studio_config=dvc_studio_config,
                     message=message,
+                    subdir=get_subrepo_relpath(dvc),
+                    studio_repo_url=repo_url,
                 )
                 logger.debug("Running repro in '%s'", os.getcwd())
                 yield dvc
@@ -649,6 +649,7 @@ class BaseExecutor(ABC):
                     experiment_rev=dvc.experiments.scm.get_ref(EXEC_BRANCH),
                     metrics=_gather_metrics(dvc, on_error="return"),
                     dvc_studio_config=dvc_studio_config,
+                    studio_repo_url=repo_url,
                 )
 
                 if infofile is not None:
@@ -668,15 +669,46 @@ class BaseExecutor(ABC):
             kwargs = {}
         return args, kwargs
 
+    @classmethod
+    @contextmanager
+    def auto_push(cls, dvc: "Repo") -> Iterator[None]:
+        exp_config = dvc.config.get("exp", {})
+        auto_push = env2bool(DVC_EXP_AUTO_PUSH, exp_config.get("auto_push", False))
+        if not auto_push:
+            yield
+            return
+
+        git_remote = os.getenv(
+            DVC_EXP_GIT_REMOTE, exp_config.get("git_remote", "origin")
+        )
+        try:
+            cls._validate_remotes(dvc, git_remote)
+        except DvcException as exc:
+            logger.warning("Failed to validate remotes. Disabling auto push: %s", exc)
+
+            yield
+            return
+        yield
+        cls._auto_push(dvc, git_remote)
+
     @staticmethod
     def _auto_push(
         dvc: "Repo",
-        scm: "Git",
         git_remote: Optional[str],
         push_cache=True,
         run_cache=True,
     ):
-        branch = scm.get_ref(EXEC_BRANCH, follow=False)
+        from dvc.ui import ui
+        from dvc.utils import format_link
+
+        branch = dvc.scm.get_ref(EXEC_BRANCH, follow=False)
+        link = format_link(
+            "https://dvc.org/doc/user-guide/experiment-management/sharing-experiments"
+        )
+        ui.write(
+            f"Pushing experiment to '{git_remote}'. Cancel with CTRL+C. "
+            f"See {link} for more info."
+        )
         try:
             dvc.experiments.push(
                 git_remote,
@@ -684,7 +716,7 @@ class BaseExecutor(ABC):
                 push_cache=push_cache,
                 run_cache=run_cache,
             )
-        except BaseException as exc:  # noqa: BLE001
+        except DvcException as exc:
             logger.warning(
                 (
                     "Something went wrong while auto pushing experiment "
@@ -704,6 +736,7 @@ class BaseExecutor(ABC):
         message: Optional[str] = None,
     ):
         """Commit stages as an experiment and return the commit SHA."""
+
         rev = scm.get_rev()
         if not scm.is_dirty(untracked_files=False):
             logger.debug("No changes to commit")

dvc/repo/experiments/executor/local.py

@@ -96,9 +96,10 @@ class TempDirExecutor(BaseLocalExecutor):
             temp_merge: stash_rev,
             temp_baseline: entry.baseline_rev,
         }
-        with get_exp_rwlock(
-            repo, writes=[temp_head, temp_merge, temp_baseline]
-        ), self.set_temp_refs(scm, temp_ref_dict):
+        with (
+            get_exp_rwlock(repo, writes=[temp_head, temp_merge, temp_baseline]),
+            self.set_temp_refs(scm, temp_ref_dict),
+        ):
             # Executor will be initialized with an empty git repo that
             # we populate by pushing:
             #   EXEC_HEAD - the base commit for this experiment
@@ -130,6 +131,8 @@ class TempDirExecutor(BaseLocalExecutor):
 
         self.scm.stash.apply(merge_rev)
         self._update_config(repo.config.read("local"))
+        local_git_config = os.path.join(repo.scm.root_dir, ".git", "config")
+        self._update_git_config(ConfigObj(local_git_config, list_values=False))
 
     def _update_config(self, update):
         local_config = os.path.join(self.root_dir, self.dvc_dir, "config.local")
@@ -143,6 +146,18 @@ class TempDirExecutor(BaseLocalExecutor):
             with open(local_config, "wb") as fobj:
                 conf_obj.write(fobj)
 
+    def _update_git_config(self, update):
+        local_config = os.path.join(self.scm.root_dir, ".git", "config")
+        logger.debug("Writing experiments local Git config '%s'", local_config)
+        if os.path.exists(local_config):
+            conf_obj = ConfigObj(local_config, list_values=False)
+            conf_obj.merge(update)
+        else:
+            conf_obj = ConfigObj(update, list_values=False)
+        if conf_obj:
+            with open(local_config, "wb") as fobj:
+                conf_obj.write(fobj)
+
     def init_cache(
         self,
         repo: "Repo",

dvc/repo/experiments/queue/base.py

@@ -4,7 +4,6 @@ from collections.abc import Collection, Generator, Iterable, Mapping
 from dataclasses import asdict, dataclass
 from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Union
 
-from dvc_studio_client.post_live_metrics import get_studio_config
 from funcy import retry
 
 from dvc.dependency import ParamsDependency
@@ -24,6 +23,7 @@ from dvc.repo.experiments.utils import (
 )
 from dvc.utils.objects import cached_property
 from dvc.utils.studio import config_to_env
+from dvc_studio_client.post_live_metrics import get_studio_config
 
 from .utils import get_remote_executor_refs
 
@@ -180,7 +180,7 @@ class BaseStashQueue(ABC):
 
         def _get_timestamp(rev: str) -> datetime:
             commit = self.scm.resolve_commit(rev)
-            return datetime.fromtimestamp(commit.commit_time)
+            return datetime.fromtimestamp(commit.commit_time)  # noqa: DTZ006
 
         def _format_entry(
             entry: QueueEntry,

dvc/repo/experiments/queue/celery.py

@@ -474,10 +474,10 @@ class LocalCeleryQueue(BaseStashQueue):
             self.scm, exp_names, git_remote
         )
         if not git_remote:
-            queue_entry_match: dict[
-                str, Optional["QueueEntry"]
-            ] = self.match_queue_entry_by_name(
-                exp_names, self.iter_queued(), self.iter_done()
+            queue_entry_match: dict[str, Optional["QueueEntry"]] = (
+                self.match_queue_entry_by_name(
+                    exp_names, self.iter_queued(), self.iter_done()
+                )
             )
 
         for exp_name in exp_names:

dvc/repo/experiments/remove.py

@@ -44,9 +44,9 @@ def remove(  # noqa: C901, PLR0912
     exp_ref_list: list["ExpRefInfo"] = []
     queue_entry_list: list["QueueEntry"] = []
     if exp_names:
-        results: dict[
-            str, "ExpRefAndQueueEntry"
-        ] = celery_queue.get_ref_and_entry_by_names(exp_names, git_remote)
+        results: dict[str, "ExpRefAndQueueEntry"] = (
+            celery_queue.get_ref_and_entry_by_names(exp_names, git_remote)
+        )
         remained: list[str] = []
         for name, result in results.items():
             if not result.exp_ref_info and not result.queue_entry:

dvc/repo/experiments/show.py

@@ -306,7 +306,7 @@ def format_time(
         return fill_value
     if iso:
         return timestamp.isoformat()
-    if timestamp.date() == date.today():
+    if timestamp.date() == date.today():  # noqa: DTZ011
         fmt = "%I:%M %p"
     else:
         fmt = "%b %d, %Y"

dvc/repo/experiments/utils.py

@@ -287,7 +287,7 @@ def gen_random_name():
     # fmt: on
     # Use custom random generator to make sure that names are random even if
     # global random seed is set (common for ML pipelines).
-    random_generator = random.Random()
+    random_generator = random.Random()  # noqa: S311
     adjective = random_generator.choice(ADJECTIVES)
     noun = random_generator.choice(NOUNS)
     return f"{adjective}-{noun}"

dvc/repo/fetch.py

@@ -49,14 +49,12 @@ def _collect_indexes(  # noqa: PLR0913
         config["core"] = core
 
     def stage_filter(stage: "Stage") -> bool:
-        if push and stage.is_repo_import:
-            return False
-        return True
+        return not (push and stage.is_repo_import)
 
     def outs_filter(out: "Output") -> bool:
         if push and not out.can_push:
             return False
-        return True
+        return not (remote and out.remote and remote != out.remote)
 
     for rev in repo.brancher(
         revs=revs,

dvc/repo/imports.py

@@ -1,145 +0,0 @@
-import os
-from functools import partial
-from tempfile import TemporaryDirectory
-from typing import TYPE_CHECKING, Union
-
-from dvc.log import logger
-
-if TYPE_CHECKING:
-    from dvc.dependency.base import Dependency
-    from dvc.repo import Repo
-    from dvc.repo.index import Index, IndexView
-    from dvc.stage import Stage
-    from dvc.types import TargetType
-    from dvc_data.hashfile.hash_info import HashInfo
-
-
-logger = logger.getChild(__name__)
-
-
-def unfetched_view(
-    index: "Index", targets: "TargetType", unpartial: bool = False, **kwargs
-) -> tuple["IndexView", "IndexView", list["Dependency"]]:
-    """Return index view of imports which have not been fetched.
-
-    Returns:
-        Tuple in the form (legacy_view, view, changed_deps) where changed_imports is a
-        list of import dependencies that cannot be fetched due to changed data source.
-    """
-    from dvc.cachemgr import LEGACY_HASH_NAMES
-
-    changed_deps: list["Dependency"] = []
-
-    def need_fetch(stage: "Stage", legacy: bool = False) -> bool:
-        if not stage.is_import or (stage.is_partial_import and not unpartial):
-            return False
-
-        out = stage.outs[0]
-        if not out.changed_cache():
-            return False
-
-        dep = stage.deps[0]
-        if dep.changed_checksum():
-            changed_deps.append(dep)
-            return False
-
-        if out.hash_name in LEGACY_HASH_NAMES and legacy:
-            return True
-        if out.hash_name not in LEGACY_HASH_NAMES and not legacy:
-            return True
-        return False
-
-    legacy_unfetched = index.targets_view(
-        targets,
-        stage_filter=partial(need_fetch, legacy=True),
-        **kwargs,
-    )
-    unfetched = index.targets_view(targets, stage_filter=need_fetch, **kwargs)
-    return legacy_unfetched, unfetched, changed_deps
-
-
-def partial_view(index: "Index", targets: "TargetType", **kwargs) -> "IndexView":
-    return index.targets_view(
-        targets,
-        stage_filter=lambda s: s.is_partial_import,
-        **kwargs,
-    )
-
-
-def unpartial_imports(index: Union["Index", "IndexView"]) -> int:
-    """Update any outs in the index which are no longer partial imports.
-
-    Returns:
-        Total number of files which were unpartialed.
-    """
-    from dvc_data.hashfile.hash_info import HashInfo
-    from dvc_data.hashfile.meta import Meta
-
-    updated = 0
-    for out in index.outs:
-        # we need to use view[key] here and since the out fields have not been
-        # updated yet (out.get_entry() would return the partial-import state)
-        workspace, key = out.index_key
-        entry = index.data[workspace][key]
-        if out.stage.is_partial_import:
-            out.hash_info = entry.hash_info or HashInfo()
-            out.meta = entry.meta or Meta()
-            out.stage.md5 = out.stage.compute_md5()
-            out.stage.dump()
-            updated += out.meta.nfiles if out.meta.nfiles is not None else 1
-    return updated
-
-
-def save_imports(
-    repo: "Repo", targets: "TargetType", unpartial: bool = False, **kwargs
-) -> set["HashInfo"]:
-    """Save (download) imports from their original source location.
-
-    Imports which are already cached will not be downloaded.
-
-    Returns:
-        Objects which were downloaded from source location.
-    """
-    from dvc.fs.callbacks import TqdmCallback
-    from dvc.stage.exceptions import DataSourceChanged
-    from dvc_data.index import md5, save
-    from dvc_data.index.checkout import apply, compare
-
-    downloaded: set["HashInfo"] = set()
-
-    legacy_unfetched, unfetched, changed = unfetched_view(
-        repo.index, targets, unpartial=unpartial, **kwargs
-    )
-    for dep in changed:
-        logger.warning(str(DataSourceChanged(f"{dep.stage} ({dep})")))
-
-    for view, cache in [
-        (legacy_unfetched, repo.cache.legacy),
-        (unfetched, repo.cache.local),
-    ]:
-        data_view = view.data["repo"]
-        if len(data_view):
-            if not cache.fs.exists(cache.path):
-                os.makedirs(cache.path)
-            with TemporaryDirectory(dir=cache.path) as tmpdir:
-                with TqdmCallback(
-                    desc="Downloading imports from source",
-                    unit="files",
-                ) as cb:
-                    diff = compare(None, data_view)
-                    apply(diff, tmpdir, cache.fs, callback=cb, storage="remote")
-                md5(data_view, name=cache.hash_name)
-                save(data_view, odb=cache, hardlink=True)
-
-            downloaded.update(
-                entry.hash_info
-                for _, entry in data_view.iteritems()
-                if entry.meta is not None
-                and not entry.meta.isdir
-                and entry.hash_info is not None
-            )
-
-    if unpartial:
-        unpartial_imports(partial_view(repo.index, targets, **kwargs))
-
-    return downloaded

dvc/repo/index.py

@@ -104,7 +104,7 @@ def collect_files(
         dirs[:] = [d for d in dirs if not is_out_or_ignored(root, d)]
 
 
-def _load_data_from_tree(index, prefix, ws, key, tree):
+def _load_data_from_tree(index, prefix, ws, key, tree, hash_name):
     from dvc_data.index import DataIndexEntry, Meta
 
     parents = set()
@@ -117,7 +117,7 @@ def _load_data_from_tree(index, prefix, ws, key, tree):
         index[(*prefix, ws, *fkey)] = DataIndexEntry(
             key=fkey,
             meta=ometa,
-            hash_info=ohi,
+            hash_info=ohi if (ohi and ohi.name == hash_name) else None,
         )
 
     for parent in parents:
@@ -151,7 +151,7 @@ def _load_data_from_outs(index, prefix, outs):
             tree = out.get_obj()
 
         if tree is not None:
-            _load_data_from_tree(index, prefix, ws, key, tree)
+            _load_data_from_tree(index, prefix, ws, key, tree, out.hash_name)
 
         entry = DataIndexEntry(
             key=key,
@@ -193,16 +193,22 @@ def _load_storage_from_import(storage_map, key, out):
         return
 
     dep = out.stage.deps[0]
-    if not out.hash_info and (
-        not dep.hash_info or dep.hash_info.name != storage_map[key].cache.odb.hash_name
-    ):
-        # partial import
+    if not out.hash_info or dep.fs.version_aware:
+        if dep.meta and dep.meta.isdir:
+            meta_token = dep.hash_info.value
+        else:
+            meta_token = tokenize(dep.meta.to_dict())
+
         fs_cache = out.repo.cache.fs_cache
         storage_map.add_cache(
             FileStorage(
                 key,
                 fs_cache.fs,
-                fs_cache.fs.join(fs_cache.path, dep.fs.protocol, tokenize(dep.fs_path)),
+                fs_cache.fs.join(
+                    fs_cache.path,
+                    dep.fs.protocol,
+                    tokenize(dep.fs_path, meta_token),
+                ),
             )
         )
 
@@ -284,6 +290,7 @@ class Index:
         params: Optional[dict[str, Any]] = None,
         artifacts: Optional[dict[str, Any]] = None,
         datasets: Optional[dict[str, list[dict[str, Any]]]] = None,
+        datasets_lock: Optional[dict[str, list[dict[str, Any]]]] = None,
     ) -> None:
         self.repo = repo
         self.stages = stages or []
@@ -292,6 +299,7 @@ class Index:
         self._params = params or {}
         self._artifacts = artifacts or {}
         self._datasets: dict[str, list[dict[str, Any]]] = datasets or {}
+        self._datasets_lock: dict[str, list[dict[str, Any]]] = datasets_lock or {}
         self._collected_targets: dict[int, list["StageInfo"]] = {}
 
     @cached_property
@@ -316,6 +324,7 @@ class Index:
         params = {}
         artifacts = {}
         datasets = {}
+        datasets_lock = {}
 
         onerror = onerror or repo.stage_collection_error_handler
         for _, idx in collect_files(repo, onerror=onerror):
@@ -325,6 +334,7 @@ class Index:
             params.update(idx._params)
             artifacts.update(idx._artifacts)
             datasets.update(idx._datasets)
+            datasets_lock.update(idx._datasets_lock)
         return cls(
             repo,
             stages=stages,
@@ -333,6 +343,7 @@ class Index:
             params=params,
             artifacts=artifacts,
             datasets=datasets,
+            datasets_lock=datasets_lock,
         )
 
     @classmethod
@@ -348,6 +359,9 @@ class Index:
             params={path: dvcfile.params} if dvcfile.params else {},
             artifacts={path: dvcfile.artifacts} if dvcfile.artifacts else {},
             datasets={path: dvcfile.datasets} if dvcfile.datasets else {},
+            datasets_lock={path: dvcfile.datasets_lock}
+            if dvcfile.datasets_lock
+            else {},
         )
 
     def update(self, stages: Iterable["Stage"]) -> "Index":
@@ -400,11 +414,6 @@ class Index:
         for stage in self.stages:
             yield from stage.outs
 
-    @cached_property
-    def datasets(self) -> dict[str, dict[str, Any]]:
-        datasets = chain.from_iterable(self._datasets.values())
-        return {dataset["name"]: dataset for dataset in datasets}
-
     @cached_property
     def out_data_keys(self) -> dict[str, set["DataIndexKey"]]:
         by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)

dvc/repo/metrics/show.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from collections.abc import Iterable, Iterator
 from itertools import chain
@@ -8,7 +9,7 @@ from scmrepo.exceptions import SCMError
 
 from dvc.log import logger
 from dvc.scm import NoSCMError
-from dvc.utils import as_posix, expand_paths
+from dvc.utils import as_posix
 from dvc.utils.collections import ensure_list
 from dvc.utils.serialize import load_path
 
@@ -103,7 +104,7 @@ def _collect_metrics(
     paths = (fs.from_os_path(metric) for metric in metrics)
     # make paths absolute for DVCFileSystem
     repo_paths = (f"{fs.root_marker}{path}" for path in paths)
-    return ldistinct(expand_paths(fs, repo_paths))
+    return ldistinct(try_expand_paths(fs, repo_paths))
 
 
 class FileResult(TypedDict, total=False):
@@ -116,6 +117,22 @@ class Result(TypedDict, total=False):
     error: Exception
 
 
+def try_expand_paths(fs: "FileSystem", paths: Iterable[str]) -> Iterator[str]:
+    for path in paths:
+        try:
+            if fs.isdir(path):
+                yield from fs.find(path)
+                continue
+        except Exception as e:  # noqa: BLE001
+            logger.debug(
+                "failed to expand %r: %s",
+                path,
+                e,
+                exc_info=logger.isEnabledFor(logging.TRACE),  # type: ignore[attr-defined]
+            )
+        yield path
+
+
 def to_relpath(fs: "FileSystem", root_dir: str, d: Result) -> Result:
     relpath = fs.relpath
     cwd = fs.getcwd()

dvc/repo/params/show.py

@@ -6,9 +6,9 @@ from typing import TYPE_CHECKING, Any, Optional, Union
 
 from dvc.dependency.param import ParamsDependency, read_param_file
 from dvc.log import logger
-from dvc.repo.metrics.show import FileResult, Result
+from dvc.repo.metrics.show import FileResult, Result, try_expand_paths
 from dvc.stage import PipelineStage
-from dvc.utils import as_posix, expand_paths
+from dvc.utils import as_posix
 from dvc.utils.collections import ensure_list
 
 if TYPE_CHECKING:
@@ -77,7 +77,7 @@ def _collect_params(
         path = fs.from_os_path(param)
         # make paths absolute for DVCFileSystem
         repo_path = f"{fs.root_marker}{path}"
-        ret.update(dict.fromkeys(expand_paths(fs, [repo_path]), _params))
+        ret.update(dict.fromkeys(try_expand_paths(fs, [repo_path]), _params))
     return ret
 
 

dvc/repo/plots/__init__.py

@@ -369,11 +369,9 @@ def _matches(targets, config_file, plot_id):
         return True
 
     full_id = get_plot_id(plot_id, config_file)
-    if any(
+    return any(
         (re.match(target, plot_id) or re.match(target, full_id)) for target in targets
-    ):
-        return True
-    return False
+    )
 
 
 def _normpath(path):
@@ -423,11 +421,17 @@ def _id_is_path(plot_props=None):
 
 def _adjust_sources(fs, plot_props, config_dir):
     new_plot_props = deepcopy(plot_props)
-    old_y = new_plot_props.pop("y", {})
-    new_y = {}
-    for filepath, val in old_y.items():
-        new_y[_normpath(fs.join(config_dir, filepath))] = val
-    new_plot_props["y"] = new_y
+    for axis in ["x", "y"]:
+        x_is_inferred = axis == "x" and (
+            axis not in new_plot_props or isinstance(new_plot_props[axis], str)
+        )
+        if x_is_inferred:
+            continue
+        old = new_plot_props.pop(axis, {})
+        new = {}
+        for filepath, val in old.items():
+            new[_normpath(fs.join(config_dir, filepath))] = val
+        new_plot_props[axis] = new
     return new_plot_props
 
 
@@ -442,6 +446,10 @@ def _resolve_definitions(
     config_path = os.fspath(config_path)
     config_dir = fs.dirname(config_path)
     result: dict[str, dict] = {}
+
+    plot_ids_parents = [
+        _normpath(fs.join(config_dir, plot_id)) for plot_id in definitions
+    ]
     for plot_id, plot_props in definitions.items():
         if plot_props is None:
             plot_props = {}
@@ -451,6 +459,13 @@ def _resolve_definitions(
                 unpacked = unpack_if_dir(
                     fs, data_path, props=plot_props | props, onerror=onerror
                 )
+                # use config for parent directory with most specific definition
+                if unpacked.get("data"):
+                    unpacked["data"] = {
+                        k: v
+                        for k, v in unpacked["data"].items()
+                        if _closest_parent(fs, k, plot_ids_parents) == data_path
+                    }
                 dpath.merge(result, unpacked)
         elif _matches(targets, config_path, plot_id):
             adjusted_props = _adjust_sources(fs, plot_props, config_dir)
@@ -459,6 +474,15 @@ def _resolve_definitions(
     return result
 
 
+def _closest_parent(fs, path, parents):
+    best_result = ""
+    for parent in parents:
+        common_path = fs.commonpath([path, parent])
+        if len(common_path) > len(best_result):
+            best_result = common_path
+    return best_result
+
+
 def _collect_pipeline_files(repo, targets: list[str], props, onerror=None):
     result: dict[str, dict] = {}
     top_plots = repo.index._plots
@@ -513,7 +537,7 @@ def unpack_if_dir(fs, path, props: dict[str, str], onerror: Optional[Callable] =
 
     if "data" in unpacked:
         for subpath in unpacked["data"]:
-            result["data"].update({subpath: props})
+            result["data"].update({subpath: props.copy()})
     else:
         result.update(unpacked)
 

dvc/repo/trie.py

@@ -25,15 +25,11 @@ def build_outs_trie(stages):
                 overlapping = out
             if parent and overlapping:
                 msg = (
-                    "The output paths:\n'{}'('{}')\n'{}'('{}')\n"
+                    f"The output paths:\n'{parent!s}'('{parent.stage.addressing}')\n"
+                    f"'{overlapping!s}'('{overlapping.stage.addressing}')\n"
                     "overlap and are thus in the same tracked directory.\n"
                     "To keep reproducibility, outputs should be in separate "
                     "tracked directories or tracked individually."
-                ).format(
-                    str(parent),
-                    parent.stage.addressing,
-                    str(overlapping),
-                    overlapping.stage.addressing,
                 )
                 raise OverlappingOutputPathsError(parent, overlapping, msg)
 

dvc/repo/worktree.py

@@ -76,14 +76,10 @@ def worktree_view(
     """
 
     def stage_filter(stage: "Stage") -> bool:
-        if push and stage.is_repo_import:
-            return False
-        return True
+        return not (push and stage.is_repo_import)
 
     def outs_filter(out: "Output") -> bool:
-        if not out.is_in_repo or not out.use_cache or (push and not out.can_push):
-            return False
-        return True
+        return out.is_in_repo and out.use_cache and (not push or out.can_push)
 
     return index.targets_view(
         targets,

dvc/schema.py

@@ -47,6 +47,7 @@ LOCK_FILE_STAGE_SCHEMA = {
 LOCKFILE_STAGES_SCHEMA = {str: LOCK_FILE_STAGE_SCHEMA}
 LOCKFILE_SCHEMA = {
     vol.Required("schema"): vol.Equal("2.0", "invalid schema version"),
+    "datasets": object,
     STAGES: LOCKFILE_STAGES_SCHEMA,
 }
 
@@ -128,7 +129,7 @@ DATASET_SCHEMA = vol.Schema(
     {vol.Required("type"): str, vol.Required("name"): str}, extra=vol.ALLOW_EXTRA
 )
 MULTI_STAGE_SCHEMA = {
-    "datasets": [DATASET_SCHEMA],
+    "datasets": object,
     PLOTS: [vol.Any(str, SINGLE_PLOT_SCHEMA)],
     STAGES: SINGLE_PIPELINE_STAGE_SCHEMA,
     VARS_KWD: VARS_SCHEMA,

dvc/scm.py

@@ -1,4 +1,5 @@
 """Manages source control systems (e.g. Git)."""
+
 import os
 from collections.abc import Iterator, Mapping
 from contextlib import contextmanager
@@ -70,8 +71,7 @@ def SCM(
     *,
     search_parent_directories: bool = ...,
     no_scm: Literal[False] = ...,
-) -> "Git":
-    ...
+) -> "Git": ...
 
 
 @overload
@@ -80,8 +80,7 @@ def SCM(
     *,
     search_parent_directories: bool = ...,
     no_scm: Literal[True],
-) -> "NoSCM":
-    ...
+) -> "NoSCM": ...
 
 
 @overload
@@ -90,8 +89,7 @@ def SCM(
     *,
     search_parent_directories: bool = ...,
     no_scm: bool = ...,
-) -> Union["Git", "NoSCM"]:
-    ...
+) -> Union["Git", "NoSCM"]: ...
 
 
 def SCM(root_dir, *, search_parent_directories=True, no_scm=False):
@@ -241,7 +239,9 @@ def iter_revs(
         if commit_date:
             from datetime import datetime
 
-            commit_datestamp = datetime.strptime(commit_date, "%Y-%m-%d").timestamp()
+            commit_datestamp = (
+                datetime.strptime(commit_date, "%Y-%m-%d").timestamp()  # noqa: DTZ007
+            )
 
             def _time_filter(rev):
                 try:

dvc/stage/run.py

@@ -39,7 +39,7 @@ def _enforce_cmd_list(cmd):
 
 
 def prepare_kwargs(stage, run_env=None):
-    from dvc.env import DVC_ROOT
+    from dvc.env import DVC_ROOT, DVC_STAGE
 
     kwargs = {"cwd": stage.wdir, "env": fix_env(None), "close_fds": True}
 
@@ -48,6 +48,9 @@ def prepare_kwargs(stage, run_env=None):
     if DVC_ROOT not in kwargs["env"]:
         kwargs["env"][DVC_ROOT] = stage.repo.root_dir
 
+    # Create DVC_STAGE env variable for every command
+    kwargs["env"][DVC_STAGE] = stage.addressing
+
     # NOTE: when you specify `shell=True`, `Popen` [1] will default to
     # `/bin/sh` on *nix and will add ["/bin/sh", "-c"] to your command.
     # But we actually want to run the same shell that we are running

dvc/stage/utils.py

@@ -92,9 +92,7 @@ def check_no_externals(stage):
     from dvc.utils import format_link
 
     def _is_cached_external(out):
-        if out.is_in_repo or not out.use_cache:
-            return False
-        return True
+        return not out.is_in_repo and out.use_cache
 
     outs = [str(out) for out in stage.outs if _is_cached_external(out)]
     if not outs:

dvc/testing/api_tests.py

@@ -33,9 +33,9 @@ class TestAPI:
         [
             {},
             {"url": "{path}"},
-            {"url": "{path}", "rev": "master"},
+            {"url": "{path}", "rev": "{default_branch}"},
             {"url": "file://{posixpath}"},
-            {"url": "file://{posixpath}", "rev": "master"},
+            {"url": "file://{posixpath}", "rev": "{default_branch}"},
         ],
         ids=["current", "local", "local_rev", "git", "git_rev"],
     )
@@ -61,6 +61,8 @@ class TestAPI:
 
         if url := fs_kwargs.get("url"):
             fs_kwargs["url"] = url.format(path=tmp_dir, posixpath=tmp_dir.as_posix())
+        if rev := fs_kwargs.get("rev"):
+            fs_kwargs["rev"] = rev.format(default_branch=scm.active_branch())
 
         fs = DVCFileSystem(**fs_kwargs)
 

dvc/testing/benchmarks/cli/commands/test_get.py

@@ -1,7 +1,7 @@
 import pytest
 
 
-@pytest.mark.flaky(max_runs=3, min_passes=1)
+@pytest.mark.flaky(reruns=3)
 def test_get(bench_dvc, tmp_dir, scm, dvc, make_dataset, remote):
     dataset = make_dataset(
         cache=False, files=False, dvcfile=True, commit=True, remote=True

dvc/testing/benchmarks/cli/commands/test_import.py

@@ -1,7 +1,7 @@
 import pytest
 
 
-@pytest.mark.flaky(max_runs=3, min_passes=1)
+@pytest.mark.flaky(reruns=3)
 def test_import(bench_dvc, tmp_dir, scm, dvc, make_dataset, remote):
     dataset = make_dataset(
         cache=False, files=False, dvcfile=True, commit=True, remote=True

dvc/testing/benchmarks/cli/stories/test_modify_data.py

@@ -1,6 +1,7 @@
 """
 Tests for modifications to an existing dataset.
 """
+
 import glob
 import os
 import random

dvc/testing/fixtures.py

@@ -221,15 +221,8 @@ def docker_compose_project_name():
 
 
 @pytest.fixture(scope="session")
-def docker_services(
-    tmp_path_factory,
-    docker_compose_command,
-    docker_compose_file,
-    docker_compose_project_name,
-    docker_setup,
-):
+def docker_services(tmp_path_factory, request):
     from filelock import FileLock
-    from pytest_docker.plugin import DockerComposeExecutor, Services
 
     if os.environ.get("CI") and os.name == "nt":
         pytest.skip("disabled for Windows on CI")
@@ -251,18 +244,12 @@ def docker_services(
         out = (err.output or b"").decode("utf-8")
         pytest.skip(f"docker-compose is not installed: {out}")
 
-    executor = DockerComposeExecutor(
-        docker_compose_command, docker_compose_file, docker_compose_project_name
-    )
-
     # making sure we don't accidentally launch docker-compose in parallel,
     # as it might result in network conflicts. Inspired by:
     # https://github.com/pytest-dev/pytest-xdist#making-session-scoped-fixtures-execute-only-once
     lockfile = tmp_path_factory.getbasetemp().parent / "docker-compose.lock"
     with FileLock(os.fspath(lockfile)):
-        executor.execute(docker_setup)
-        # note: we are not tearing down the containers here
-        return Services(executor)
+        return request.getfixturevalue("docker_services")
 
 
 @pytest.fixture

dvc/testing/path_info.py

@@ -1,6 +1,7 @@
 import os
 import pathlib
 import posixpath
+import sys
 from typing import Callable, ClassVar
 from urllib.parse import urlparse
 
@@ -27,12 +28,13 @@ class PathInfo(pathlib.PurePath, _BasePath):
     __slots__ = ()
     scheme = "local"
 
-    def __new__(cls, *args):
-        # Construct a proper subclass depending on current os
-        if cls is PathInfo:
-            cls = WindowsPathInfo if os.name == "nt" else PosixPathInfo
+    if sys.version_info < (3, 12):
 
-        return cls._from_parts(args)  # type: ignore[attr-defined]
+        def __new__(cls, *args):
+            if cls is PathInfo:
+                cls = WindowsPathInfo if os.name == "nt" else PosixPathInfo
+
+            return cls._from_parts(args)  # type: ignore[attr-defined]
 
     def as_posix(self):
         f = self._flavour  # type: ignore[attr-defined]
@@ -67,7 +69,7 @@ class PathInfo(pathlib.PurePath, _BasePath):
 
     def isin(self, other):
         if isinstance(other, (str, bytes)):
-            other = self.__class__(other)
+            other = self.__class__(other)  # type: ignore[arg-type]
         elif self.__class__ != other.__class__:
             return False
         # Use cached casefolded parts to compare paths

dvc/testing/tmp_dir.py

@@ -67,13 +67,12 @@ class TmpDir(pathlib.Path):
     def config(self):
         return {"url": self.url}
 
-    def __new__(cls, *args, **kwargs):
-        if cls is TmpDir:
-            cls = WindowsTmpDir if os.name == "nt" else PosixTmpDir
+    if sys.version_info < (3, 12):
+
+        def __new__(cls, *args, **kwargs):
+            if cls is TmpDir:
+                cls = WindowsTmpDir if os.name == "nt" else PosixTmpDir
 
-        if sys.version_info >= (3, 12):
-            return object.__new__(cls)
-        else:  # noqa: RET505
             # init parameter and `_init` method has been removed in Python 3.10.
             kw = {"init": False} if sys.version_info < (3, 10) else {}
             self = cls._from_parts(args, **kw)  # type: ignore[attr-defined]

dvc/testing/workspace_tests.py

@@ -106,6 +106,7 @@ class TestImportURLVersionAware:
         (remote_version_aware / "file").write_text("modified")
         assert dvc.status().get("file.dvc") == [
             {"changed deps": {"remote://upstream/file": "update available"}},
+            {"changed outs": {"file": "not in cache"}},
         ]
         dvc.update(str(tmp_dir / "file.dvc"))
         assert (tmp_dir / "file").read_text() == "modified"
@@ -137,6 +138,7 @@ class TestImportURLVersionAware:
         (remote_version_aware / "data_dir" / "new_file").write_text("new")
         assert dvc.status().get("data_dir.dvc") == [
             {"changed deps": {"remote://upstream/data_dir": "modified"}},
+            {"changed outs": {"data_dir": "not in cache"}},
         ]
         dvc.update(str(tmp_dir / "data_dir.dvc"))
         assert (tmp_dir / "data_dir" / "subdir" / "file").read_text() == "modified"

dvc/utils/__init__.py

@@ -5,7 +5,6 @@ import json
 import os
 import re
 import sys
-from collections.abc import Iterable, Iterator
 from typing import TYPE_CHECKING, Optional
 
 import colorama
@@ -13,8 +12,6 @@ import colorama
 if TYPE_CHECKING:
     from typing import TextIO
 
-    from dvc.fs import FileSystem
-
 
 LARGE_DIR_SIZE = 100
 TARGET_REGEX = re.compile(r"(?P<path>.*?)(:(?P<name>[^\\/:]*))??$")
@@ -410,14 +407,6 @@ def errored_revisions(rev_data: dict) -> list:
     return result
 
 
-def expand_paths(fs: "FileSystem", paths: Iterable[str]) -> Iterator[str]:
-    for path in paths:
-        if fs.isdir(path):
-            yield from fs.find(path)
-        else:
-            yield path
-
-
 def isatty(stream: "Optional[TextIO]") -> bool:
     if stream is None:
         return False

dvc/utils/collections.py

@@ -39,9 +39,7 @@ def apply_diff(src, dest):  # noqa: C901
                     dest[i] = value
     else:
         raise AssertionError(  # noqa: TRY004
-            "Can't apply diff from {} to {}".format(
-                src.__class__.__name__, dest.__class__.__name__
-            )
+            f"Can't apply diff from {type(src).__name__} to {type(dest).__name__}"
         )
 
 

dvc/utils/serialize/_common.py

@@ -1,4 +1,5 @@
 """Common utilities for serialize."""
+
 import os
 from contextlib import AbstractContextManager, contextmanager
 from typing import TYPE_CHECKING, Any, Callable, Optional, Protocol, TextIO, Union
@@ -15,25 +16,21 @@ if TYPE_CHECKING:
 class DumperFn(Protocol):
     def __call__(
         self, path: "StrPath", data: Any, fs: Optional["FileSystem"] = None
-    ) -> Any:
-        ...
+    ) -> Any: ...
 
 
 class DumpersFn(Protocol):
-    def __call__(self, data: Any, stream: TextIO) -> Any:
-        ...
+    def __call__(self, data: Any, stream: TextIO) -> Any: ...
 
 
 class ModifierFn(Protocol):
     def __call__(
         self, path: "StrPath", fs: Optional["FileSystem"] = None
-    ) -> AbstractContextManager[dict]:
-        ...
+    ) -> AbstractContextManager[dict]: ...
 
 
 class LoaderFn(Protocol):
-    def __call__(self, path: "StrPath", fs: Optional["FileSystem"] = None) -> Any:
-        ...
+    def __call__(self, path: "StrPath", fs: Optional["FileSystem"] = None) -> Any: ...
 
 
 ReadType = Union[bytes, None, str]

dvc/utils/strictyaml.py

@@ -6,6 +6,7 @@ Used for parsing dvc.yaml, dvc.lock and .dvc files.
 
 Not to be confused with strictyaml, a python library with similar motivations.
 """
+
 import re
 import typing
 from contextlib import suppress

dvc/utils/studio.py

@@ -1,3 +1,4 @@
+import os
 from typing import TYPE_CHECKING, Any, Optional
 from urllib.parse import urljoin
 
@@ -12,10 +13,14 @@ from dvc.env import (
     DVC_STUDIO_URL,
 )
 from dvc.log import logger
+from dvc.utils import as_posix
 
 if TYPE_CHECKING:
     from requests import Response
 
+    from dvc.repo import Repo
+
+
 logger = logger.getChild(__name__)
 
 STUDIO_URL = "https://studio.iterative.ai"
@@ -111,3 +116,29 @@ def env_to_config(env: dict[str, Any]) -> dict[str, Any]:
     if DVC_STUDIO_URL in env:
         config["url"] = env[DVC_STUDIO_URL]
     return config
+
+
+def get_subrepo_relpath(repo: "Repo") -> str:
+    from dvc.fs import GitFileSystem
+
+    scm_root_dir = "/" if isinstance(repo.fs, GitFileSystem) else repo.scm.root_dir
+
+    relpath = as_posix(repo.fs.relpath(repo.root_dir, scm_root_dir))
+
+    return "" if relpath == "." else relpath
+
+
+def get_repo_url(repo: "Repo") -> str:
+    from dulwich.porcelain import get_remote_repo
+
+    from dvc.env import DVC_EXP_GIT_REMOTE
+
+    repo_url = os.getenv(
+        DVC_EXP_GIT_REMOTE, repo.config.get("exp", {}).get("git_remote")
+    )
+    if repo_url:
+        try:
+            _, repo_url = get_remote_repo(repo.scm.dulwich.repo, repo_url)
+        except IndexError:
+            pass
+    return repo_url

pyproject.toml

@@ -31,24 +31,30 @@ classifiers = [
 dynamic = ["version"]
 
 dependencies = [
+    "attrs>=22.2.0",
+    "celery",
     "colorama>=0.3.9",
     "configobj>=5.0.6",
     "distro>=1.3",
     "dpath<3,>=2.1.0",
-    "dvc-data>=3.13,<3.14",
+    "dulwich",
+    "dvc-data>=3.15,<3.16",
     "dvc-http>=2.29.0",
+    "dvc-objects",
     "dvc-render>=1.0.1,<2",
-    "dvc-studio-client>=0.19,<1",
+    "dvc-studio-client>=0.20,<1",
     "dvc-task>=0.3.0,<1",
     "flatten_dict<1,>=0.4.1",
-    # https://github.com/iterative/dvc/issues/9654
-    "flufl.lock>=5,<8",
+    "flufl.lock>=5,<8", # https://github.com/iterative/dvc/issues/9654
+    "fsspec",
     "funcy>=1.14",
     "grandalf<1,>=0.7",
     "gto>=1.6.0,<2",
     "hydra-core>=1.1",
     "iterative-telemetry>=0.0.7",
+    "kombu",
     "networkx>=2.5",
+    "omegaconf",
     "packaging>=19",
     "pathspec>=0.10.3",
     "platformdirs<4,>=3.1.1",
@@ -77,7 +83,7 @@ gdrive = ["dvc-gdrive>=3,<4"]
 gs = ["dvc-gs>=3,<4"]
 hdfs = ["dvc-hdfs>=3,<4"]
 lint = [
-    "mypy==1.8.0",
+    "mypy==1.9.0",
     "pandas-stubs",
     "types-colorama",
     "types-psutil",
@@ -102,8 +108,8 @@ tests = [
     "beautifulsoup4>=4.4",
     "dvc-ssh",
     "filelock",
-    "flaky",
     "pytest>=7,<9",
+    "pytest-rerunfailures",
     "pytest-cov>=4.1.0",
     "pytest-docker>=1,<4",
     "pytest-mock",
@@ -130,6 +136,14 @@ dvc = "dvc.cli:main"
 [project.entry-points."fsspec.specs"]
 dvc = "dvc.api:DVCFileSystem"
 
+[project.entry-points."universal_pathlib.implementations"]
+dvc = "dvc.fs.dvc_path:DVCPath"
+# universal_pathlib does not support fsspec url chaining yet.
+# see https://github.com/fsspec/universal_pathlib/issues/28.
+"dvc+http" = "dvc.fs.dvc_path:DVCPath"
+"dvc+https" = "dvc.fs.dvc_path:DVCPath"
+"dvc+ssh" = "dvc.fs.dvc_path:DVCPath"
+
 [project.entry-points."pyinstaller40"]
 hook-dirs = "dvc.__pyinstaller:get_hook_dirs"
 tests = "dvc.__pyinstaller:get_PyInstaller_tests"
@@ -232,6 +246,7 @@ module = [
     "ruamel.yaml.*",
     "shortuuid",
     "shtab",
+    "upath",
     "virtualenv",
     "viztracer",
     "voluptuous",
@@ -252,10 +267,10 @@ ignore = [
     "TRY003", "TRY300", "PLR2004", "PLW2901", "LOG007",
 ]
 select = [
-    "F", "E", "W", "C90", "I", "N", "UP", "YTT", "ASYNC", "S", "BLE", "B", "A", "C4", "T10",
+    "F", "E", "W", "C90", "I", "N", "UP", "YTT", "ASYNC", "S", "BLE", "B", "A", "C4", "DTZ", "T10",
     "EXE", "ISC", "ICN", "G", "INP", "PIE", "T20", "PYI", "PT", "Q", "RSE", "RET",
     "SLOT", "SIM", "TID", "TCH", "ARG", "PGH", "PLC", "PLE", "PLR", "PLW", "TRY",
-    "FLY", "PERF101", "LOG", "RUF", "RUF022", "RUF023", "RUF024", "RUF025", "RUF026",
+    "FLY", "PERF101", "LOG", "RUF", "RUF022", "RUF023", "RUF024", "RUF025", "RUF026", "RUF027"
 ]
 preview = true
 explicit-preview-rules = true
@@ -277,7 +292,7 @@ strict = true
 ignore-variadic-names = true
 
 [tool.ruff.lint.isort]
-known-first-party = ["dvc", "dvc_data", "dvc_objects", "dvc_render", "dvc_task", "tests"]
+known-first-party = ["dvc", "dvc_*", "tests"]
 
 [tool.ruff.lint.pep8-naming]
 extend-ignore-names = ["M", "SCM"]

tests/conftest.py

@@ -123,7 +123,7 @@ def pytest_runtest_setup(item):
     if "CI" in os.environ and item.get_closest_marker("needs_internet") is not None:
         # remotes that need internet connection might be flaky,
         # so we rerun them in case it fails.
-        item.add_marker(pytest.mark.flaky(max_runs=5, min_passes=1))
+        item.add_marker(pytest.mark.flaky(reruns=5))
 
 
 @pytest.fixture(scope="session")

tests/dir_helpers.py

@@ -43,7 +43,6 @@ from global repo template to creating everything inplace, which:
     - does not create unnecessary files
 """
 
-
 import os
 
 import pytest

tests/func/api/test_artifacts.py

@@ -0,0 +1,74 @@
+from os.path import join, normpath
+
+import pytest
+
+from dvc.api import artifacts_show
+from dvc.testing.tmp_dir import make_subrepo
+from dvc.utils import as_posix
+from tests.func.artifacts.test_artifacts import get_tag_and_name, make_artifact
+
+
+@pytest.mark.parametrize("sub", ["sub", ""])
+def test_artifacts_show(tmp_dir, dvc, scm, sub):
+    subdir = tmp_dir / sub
+
+    dirname = str(subdir.relative_to(tmp_dir))
+    tag, name = get_tag_and_name(as_posix(dirname), "myart", "v2.0.0")
+    make_artifact(tmp_dir, "myart", tag, subdir / "myart.pkl")
+
+    assert artifacts_show(name) == {
+        "path": normpath(join(dirname, "myart.pkl")),
+        "rev": scm.get_rev(),
+    }
+    assert artifacts_show(name, repo=tmp_dir.fs_path) == {
+        "path": normpath(join(dirname, "myart.pkl")),
+        "rev": scm.get_rev(),
+    }
+    assert artifacts_show(name, repo=f"file://{tmp_dir.as_posix()}") == {
+        "path": normpath(join(dirname, "myart.pkl")),
+        "rev": scm.get_rev(),
+    }
+
+    assert artifacts_show(name, repo=subdir.fs_path) == {
+        "path": normpath(join(dirname, "myart.pkl")),
+        "rev": scm.get_rev(),
+    }
+    with subdir.chdir():
+        assert artifacts_show(name) == {
+            "path": normpath(join(dirname, "myart.pkl")),
+            "rev": scm.get_rev(),
+        }
+
+
+@pytest.mark.parametrize("sub", ["sub", ""])
+def test_artifacts_show_subrepo(tmp_dir, scm, sub):
+    subrepo = tmp_dir / "subrepo"
+    make_subrepo(subrepo, scm)
+    subdir = subrepo / sub
+
+    dirname = str(subdir.relative_to(tmp_dir))
+    tag, name = get_tag_and_name(as_posix(dirname), "myart", "v2.0.0")
+    make_artifact(subrepo, "myart", tag, subdir / "myart.pkl")
+
+    assert artifacts_show(name) == {
+        "path": join(dirname, "myart.pkl"),
+        "rev": scm.get_rev(),
+    }
+    assert artifacts_show(name, repo=tmp_dir.fs_path) == {
+        "path": join(dirname, "myart.pkl"),
+        "rev": scm.get_rev(),
+    }
+    assert artifacts_show(name, repo=f"file://{tmp_dir.as_posix()}") == {
+        "path": join(dirname, "myart.pkl"),
+        "rev": scm.get_rev(),
+    }
+
+    assert artifacts_show(name, repo=subdir.fs_path) == {
+        "path": str((subdir / "myart.pkl").relative_to(subrepo)),
+        "rev": scm.get_rev(),
+    }
+    with subdir.chdir():
+        assert artifacts_show(name) == {
+            "path": str((subdir / "myart.pkl").relative_to(subrepo)),
+            "rev": scm.get_rev(),
+        }

tests/func/api/test_data.py

@@ -31,6 +31,40 @@ def test_get_url_requires_dvc(tmp_dir, scm):
         api.get_url("foo", repo=f"file://{tmp_dir.as_posix()}")
 
 
+def test_get_url_from_remote(tmp_dir, erepo_dir, cloud, local_cloud):
+    erepo_dir.add_remote(config=cloud.config, name="other")
+    erepo_dir.add_remote(config=local_cloud.config, default=True)
+    with erepo_dir.chdir():
+        erepo_dir.dvc_gen("foo", "foo", commit="add foo")
+
+    # Using file url to force clone to tmp repo
+    repo_url = f"file://{erepo_dir.as_posix()}"
+    expected_rel_path = os.path.join(
+        "files", "md5", "ac/bd18db4cc2f85cedef654fccc4a4d8"
+    )
+
+    # Test default remote
+    assert api.get_url("foo", repo=repo_url) == (local_cloud / expected_rel_path).url
+
+    # Test remote arg
+    assert (
+        api.get_url("foo", repo=repo_url, remote="other")
+        == (cloud / expected_rel_path).url
+    )
+
+    # Test config arg
+    assert (
+        api.get_url("foo", repo=repo_url, config={"core": {"remote": "other"}})
+        == (cloud / expected_rel_path).url
+    )
+
+    # Test remote_config arg
+    assert (
+        api.get_url("foo", repo=repo_url, remote_config={"url": cloud.url})
+        == (cloud / expected_rel_path).url
+    )
+
+
 def test_open_external(tmp_dir, erepo_dir, cloud):
     erepo_dir.add_remote(config=cloud.config)
 
@@ -253,3 +287,12 @@ def test_read_from_remote(tmp_dir, erepo_dir, cloud, local_cloud):
         )
         == "foo content"
     )
+
+    assert (
+        api.read(
+            os.path.join("dir", "foo"),
+            repo=f"file://{erepo_dir.as_posix()}",
+            remote_config={"url": cloud.url},
+        )
+        == "foo content"
+    )

tests/func/artifacts/test_artifacts.py

@@ -6,7 +6,9 @@ import pytest
 
 from dvc.annotations import Artifact
 from dvc.exceptions import ArtifactNotFoundError, InvalidArgumentError
-from dvc.repo.artifacts import check_name_format
+from dvc.repo.artifacts import Artifacts, check_name_format
+from dvc.testing.tmp_dir import make_subrepo
+from dvc.utils import as_posix
 from dvc.utils.strictyaml import YAMLSyntaxError, YAMLValidationError
 
 dvcyaml = {
@@ -185,7 +187,7 @@ def test_get_rev(tmp_dir, dvc, scm):
         dvc.artifacts.get_rev("myart", stage="prod")
 
 
-def test_get_path(tmp_dir, dvc):
+def test_get_path(tmp_dir, dvc, scm):
     (tmp_dir / "dvc.yaml").dump(dvcyaml)
     subdir = tmp_dir / "subdir"
     subdir.mkdir()
@@ -196,3 +198,86 @@ def test_get_path(tmp_dir, dvc):
     assert dvc.artifacts.get_path("subdir/dvc.yaml:myart") == os.path.join(
         "subdir", "myart.pkl"
     )
+
+
+def test_parametrized(tmp_dir, dvc):
+    (tmp_dir / "params.yaml").dump({"path": "myart.pkl"})
+    (tmp_dir / "dvc.yaml").dump(
+        {"artifacts": {"myart": {"type": "model", "path": "${path}"}}}
+    )
+    assert tmp_dir.dvc.artifacts.read() == {
+        "dvc.yaml": {"myart": Artifact(path="myart.pkl", type="model")}
+    }
+
+
+def test_get_path_subrepo(tmp_dir, scm, dvc):
+    subrepo = tmp_dir / "subrepo"
+    make_subrepo(subrepo, scm)
+    (subrepo / "dvc.yaml").dump(dvcyaml)
+
+    assert dvc.artifacts.get_path("subrepo:myart") == os.path.join(
+        "subrepo", "myart.pkl"
+    )
+    assert dvc.artifacts.get_path("subrepo/dvc.yaml:myart") == os.path.join(
+        "subrepo", "myart.pkl"
+    )
+
+    assert subrepo.dvc.artifacts.get_path("subrepo:myart") == os.path.join(
+        "subrepo", "myart.pkl"
+    )
+    assert subrepo.dvc.artifacts.get_path("subrepo/dvc.yaml:myart") == os.path.join(
+        "subrepo", "myart.pkl"
+    )
+
+
+def get_tag_and_name(dirname, name, version):
+    tagname = f"{name}@{version}"
+    if dirname in (os.curdir, ""):
+        return tagname, name
+    return f"{dirname}={tagname}", f"{dirname}:{name}"
+
+
+def make_artifact(tmp_dir, name, tag, path) -> Artifact:
+    artifact = Artifact(path=path.name, type="model")
+    dvcfile = path.with_name("dvc.yaml")
+
+    tmp_dir.scm_gen(path, "hello_world", commit="add myart.pkl")
+    tmp_dir.dvc.artifacts.add(name, artifact, dvcfile=os.fspath(dvcfile))
+    tmp_dir.scm.add_commit([dvcfile], message="add dvc.yaml")
+    tmp_dir.scm.tag(tag, annotated=True, message="foo")
+    return artifact
+
+
+@pytest.mark.parametrize("sub", ["sub", ""])
+def test_artifacts_download(tmp_dir, dvc, scm, sub):
+    subdir = tmp_dir / sub
+    dirname = str(subdir.relative_to(tmp_dir))
+    tag, name = get_tag_and_name(as_posix(dirname), "myart", "v2.0.0")
+    make_artifact(tmp_dir, "myart", tag, subdir / "myart.pkl")
+
+    result = (1, "myart.pkl")
+    assert Artifacts.get(".", name, force=True) == result
+    assert Artifacts.get(tmp_dir.fs_path, name, force=True) == result
+    assert Artifacts.get(f"file://{tmp_dir.as_posix()}", name, force=True) == result
+    assert Artifacts.get(subdir.fs_path, name, force=True) == result
+    with subdir.chdir():
+        assert Artifacts.get(".", name, force=True) == result
+
+
+@pytest.mark.parametrize("sub", ["sub", ""])
+def test_artifacts_download_subrepo(tmp_dir, scm, sub):
+    subrepo = tmp_dir / "subrepo"
+    make_subrepo(subrepo, scm)
+    subdir = subrepo / sub
+
+    dirname = str(subdir.relative_to(tmp_dir))
+    tag, name = get_tag_and_name(as_posix(dirname), "myart", "v2.0.0")
+    make_artifact(subrepo, "myart", tag, subdir / "myart.pkl")
+
+    result = (1, "myart.pkl")
+    assert Artifacts.get(".", name) == result
+    assert Artifacts.get(tmp_dir.fs_path, name, force=True) == result
+    assert Artifacts.get(f"file://{tmp_dir.as_posix()}", name, force=True) == result
+    assert Artifacts.get(subdir.fs_path, name, force=True) == result
+    with subdir.chdir():
+        assert Artifacts.get(".", name, force=True) == result

tests/func/experiments/test_remote.py

@@ -372,3 +372,68 @@ def test_push_pull_invalid_workspace(
         dvc.experiments.push(git_upstream.remote, push_cache=True)
         dvc.experiments.pull(git_upstream.remote, pull_cache=True)
         assert "failed to collect" not in caplog.text
+
+
+@pytest.mark.parametrize(
+    "auto_push, expected_key", [(True, "up_to_date"), (False, "success")]
+)
+def test_auto_push_on_run(
+    tmp_dir, scm, dvc, git_upstream, local_remote, exp_stage, auto_push, expected_key
+):
+    remote = git_upstream.remote
+
+    with dvc.config.edit() as conf:
+        conf["exp"]["auto_push"] = auto_push
+        conf["exp"]["git_remote"] = remote
+
+    exp_name = "foo"
+    dvc.experiments.run(exp_stage.addressing, params=["foo=2"], name=exp_name)
+
+    assert first(dvc.experiments.push(name=exp_name, git_remote=remote)) == expected_key
+
+
+@pytest.mark.parametrize(
+    "auto_push, expected_key", [(True, "up_to_date"), (False, "success")]
+)
+def test_auto_push_on_save(
+    tmp_dir, scm, dvc, git_upstream, local_remote, exp_stage, auto_push, expected_key
+):
+    remote = git_upstream.remote
+    exp_name = "foo"
+    dvc.experiments.run(exp_stage.addressing, params=["foo=2"], name=exp_name)
+
+    with dvc.config.edit() as conf:
+        conf["exp"]["auto_push"] = auto_push
+        conf["exp"]["git_remote"] = remote
+
+    dvc.experiments.save(name=exp_name, force=True)
+
+    assert first(dvc.experiments.push(name=exp_name, git_remote=remote)) == expected_key
+
+
+def test_auto_push_misconfigured(
+    tmp_dir, scm, dvc, git_upstream, local_remote, exp_stage, caplog
+):
+    with dvc.config.edit() as conf:
+        conf["exp"]["auto_push"] = True
+        conf["exp"]["git_remote"] = "notfound"
+
+    exp_name = "foo"
+    with caplog.at_level(logging.WARNING, logger="dvc"):
+        dvc.experiments.run(exp_stage.addressing, params=["foo=2"], name=exp_name)
+        assert "Failed to validate remotes" in caplog.text
+
+
+def test_auto_push_tmp_dir(tmp_dir, scm, dvc, git_upstream, local_remote, exp_stage):
+    remote = git_upstream.remote
+
+    with dvc.config.edit() as conf:
+        conf["exp"]["auto_push"] = True
+        conf["exp"]["git_remote"] = remote
+
+    exp_name = "foo"
+    dvc.experiments.run(
+        exp_stage.addressing, params=["foo=2"], name=exp_name, tmp_dir=True
+    )
+
+    assert first(dvc.experiments.push(name=exp_name, git_remote=remote)) == "up_to_date"

tests/func/experiments/test_show.py

@@ -101,7 +101,7 @@ def test_show_simple(tmp_dir, scm, dvc, exp_stage):
 @pytest.mark.parametrize("workspace", [True, False])
 def test_show_experiment(tmp_dir, scm, dvc, exp_stage, workspace):
     baseline_rev = scm.get_rev()
-    timestamp = datetime.fromtimestamp(
+    timestamp = datetime.fromtimestamp(  # noqa: DTZ006
         scm.gitpython.repo.rev_parse(baseline_rev).committed_date
     )
 
@@ -423,7 +423,7 @@ def test_show_csv(tmp_dir, scm, dvc, exp_stage, capsys):
     baseline_rev = scm.get_rev()
 
     def _get_rev_isotimestamp(rev):
-        return datetime.fromtimestamp(
+        return datetime.fromtimestamp(  # noqa: DTZ006
             scm.gitpython.repo.rev_parse(rev).committed_date
         ).isoformat()
 
@@ -453,15 +453,11 @@ def test_show_csv(tmp_dir, scm, dvc, exp_stage, capsys):
         in cap.out
     )
     assert (
-        "{},{},branch_base,{},,2,2,{}".format(
-            ref_info1.name, rev1[:7], _get_rev_isotimestamp(rev1), data_hash
-        )
+        f"{ref_info1.name},{rev1[:7]},branch_base,{_get_rev_isotimestamp(rev1)},,2,2,{data_hash}"
         in cap.out
     )
     assert (
-        "{},{},branch_commit,{},,3,3,{}".format(
-            ref_info2.name, rev2[:7], _get_rev_isotimestamp(rev2), data_hash
-        )
+        f"{ref_info2.name},{rev2[:7]},branch_commit,{_get_rev_isotimestamp(rev2)},,3,3,{data_hash}"
         in cap.out
     )
 
@@ -656,7 +652,7 @@ def test_metrics_renaming(tmp_dir, dvc, scm, capsys, copy_script):
     cap = capsys.readouterr()
 
     def _get_rev_isotimestamp(rev):
-        return datetime.fromtimestamp(
+        return datetime.fromtimestamp(  # noqa: DTZ006
             scm.gitpython.repo.rev_parse(rev).committed_date
         ).isoformat()
 

tests/func/metrics/test_show.py

@@ -1,5 +1,6 @@
 import json
 import os
+import shutil
 from os.path import join
 
 import pytest
@@ -9,8 +10,10 @@ from dvc.cli import main
 from dvc.dvcfile import PROJECT_FILE
 from dvc.exceptions import OverlappingOutputPathsError
 from dvc.repo import Repo
+from dvc.repo.metrics.show import FileResult, Result
 from dvc.utils.fs import remove
 from dvc.utils.serialize import JSONFileCorruptedError
+from dvc_data.index import DataIndexDirError
 
 
 def test_show_simple(tmp_dir, dvc, run_copy_metrics):
@@ -342,3 +345,27 @@ def test_cached_metrics(tmp_dir, dvc, scm, remote):
             }
         }
     }
+
+
+def test_top_level_parametrized(tmp_dir, dvc):
+    tmp_dir.dvc_gen("metrics.yaml", "foo: 3\nbar: 10")
+    (tmp_dir / "params.yaml").dump({"metric_file": "metrics.yaml"})
+    (tmp_dir / "dvc.yaml").dump({"metrics": ["${metric_file}"]})
+    assert dvc.metrics.show() == {
+        "": {"data": {"metrics.yaml": {"data": {"foo": 3, "bar": 10}}}}
+    }
+
+
+def test_metric_in_a_tracked_directory_with_missing_dir_file(M, tmp_dir, dvc):
+    tmp_dir.dvc_gen({"dir": {"file": "2"}})
+    (tmp_dir / "dvc.yaml").dump({"metrics": [join("dir", "file")]})
+    shutil.rmtree(tmp_dir / "dir")  # remove from workspace
+    dvc.cache.local.clear()  # remove .dir file
+
+    assert dvc.metrics.show() == {
+        "": Result(
+            data={
+                join("dir", "file"): FileResult(error=M.instance_of(DataIndexDirError)),
+            }
+        )
+    }

tests/func/params/test_show.py

@@ -1,9 +1,12 @@
+import shutil
 from os.path import join
 
 import pytest
 
+from dvc.dvcfile import PROJECT_FILE
 from dvc.repo import Repo
-from dvc.repo.stage import PROJECT_FILE
+from dvc.repo.metrics.show import FileResult, Result
+from dvc_data.index import DataIndexDirError
 
 
 def test_show_empty(dvc):
@@ -178,3 +181,32 @@ def test_cached_params(tmp_dir, dvc, scm, remote):
             }
         }
     }
+
+
+def test_top_level_parametrized(tmp_dir, dvc):
+    (tmp_dir / "param.json").dump({"foo": 3, "bar": 10})
+    (tmp_dir / "params.yaml").dump({"param_file": "param.json"})
+    (tmp_dir / "dvc.yaml").dump({"params": ["${param_file}"]})
+    assert dvc.params.show() == {
+        "": {
+            "data": {
+                "param.json": {"data": {"foo": 3, "bar": 10}},
+                "params.yaml": {"data": {"param_file": "param.json"}},
+            }
+        }
+    }
+
+
+def test_param_in_a_tracked_directory_with_missing_dir_file(M, tmp_dir, dvc):
+    tmp_dir.dvc_gen({"dir": {"file": "2"}})
+    (tmp_dir / "dvc.yaml").dump({"params": [join("dir", "file")]})
+    shutil.rmtree(tmp_dir / "dir")  # remove from workspace
+    dvc.cache.local.clear()  # remove .dir file
+
+    assert dvc.params.show() == {
+        "": Result(
+            data={
+                join("dir", "file"): FileResult(error=M.instance_of(DataIndexDirError)),
+            }
+        )
+    }

tests/func/parsing/test_errors.py

@@ -1,6 +1,5 @@
 """Negative tests for the parametrization."""
 
-
 import logging
 import re
 

tests/func/parsing/test_foreach.py

@@ -1,4 +1,5 @@
 """Testing happy paths for the foreach."""
+
 import os
 
 import pytest

tests/func/parsing/test_top_level.py

@@ -0,0 +1,168 @@
+from dvc.parsing import DataResolver
+
+
+def test_params(tmp_dir, dvc):
+    (tmp_dir / "params.yaml").dump(
+        {"params": {"param1": "params.json", "param2": "params.toml"}}
+    )
+
+    template = {"params": ["${params.param1}", "param11", "${params.param2}"]}
+    resolver = DataResolver(dvc, tmp_dir, template)
+    assert resolver.resolve_params() == ["params.json", "param11", "params.toml"]
+
+
+def test_metrics(tmp_dir, dvc):
+    (tmp_dir / "params.yaml").dump(
+        {"metrics": {"metric1": "metrics.json", "metric2": "metrics.toml"}}
+    )
+
+    template = {"metrics": ["${metrics.metric1}", "metric11", "${metrics.metric2}"]}
+    resolver = DataResolver(dvc, tmp_dir, template)
+    assert resolver.resolve_metrics() == ["metrics.json", "metric11", "metrics.toml"]
+
+
+def test_plots(tmp_dir, dvc):
+    template = {
+        "plots": [
+            {
+                "${plots.plot1_name}": {
+                    "x": "${plots.x_cls}",
+                    "y": {
+                        "train_classes.csv": "${plots.y_train_cls}",
+                        "test_classes.csv": [
+                            "${plots.y_train_cls}",
+                            "${plots.y_test_cls}",
+                        ],
+                    },
+                    "title": "Compare test vs train confusion matrix",
+                    "template": "confusion",
+                    "x_label": "Actual class",
+                    "y_label": "Predicted class",
+                }
+            },
+            {"eval/importance2.png": None},
+            {"${plots.plot3_name}": None},
+            "eval/importance4.png",
+            "${plots.plot5_name}",
+        ],
+    }
+
+    (tmp_dir / "params.yaml").dump(
+        {
+            "plots": {
+                "x_cls": "actual_class",
+                "y_train_cls": "predicted_class",
+                "y_test_cls": "predicted_class2",
+                "plot1_name": "eval/importance1.png",
+                "plot3_name": "eval/importance3.png",
+                "plot5_name": "eval/importance5.png",
+            }
+        }
+    )
+    resolver = DataResolver(dvc, tmp_dir, template)
+    assert resolver.resolve_plots() == [
+        {
+            "eval/importance1.png": {
+                "x": "actual_class",
+                "y": {
+                    "train_classes.csv": "predicted_class",
+                    "test_classes.csv": ["predicted_class", "predicted_class2"],
+                },
+                "title": "Compare test vs train confusion matrix",
+                "template": "confusion",
+                "x_label": "Actual class",
+                "y_label": "Predicted class",
+            }
+        },
+        {"eval/importance2.png": None},
+        {"eval/importance3.png": None},
+        "eval/importance4.png",
+        "eval/importance5.png",
+    ]
+
+
+def test_artifacts(tmp_dir, dvc):
+    template = {
+        "artifacts": {
+            "${artifacts.name}": {
+                "path": "${artifacts.path}",
+                "type": "model",
+                "desc": "CV classification model, ResNet50",
+                "labels": ["${artifacts.label1}", "${artifacts.label2}"],
+                "meta": {"framework": "${artifacts.framework}"},
+            }
+        }
+    }
+
+    (tmp_dir / "params.yaml").dump(
+        {
+            "artifacts": {
+                "name": "cv-classification",
+                "path": "models/resnet.pt",
+                "label1": "resnet50",
+                "label2": "classification",
+                "framework": "pytorch",
+            }
+        }
+    )
+
+    resolver = DataResolver(dvc, tmp_dir, template)
+    assert resolver.resolve_artifacts() == {
+        "cv-classification": {
+            "path": "models/resnet.pt",
+            "type": "model",
+            "desc": "CV classification model, ResNet50",
+            "labels": ["resnet50", "classification"],
+            "meta": {"framework": "pytorch"},
+        }
+    }
+
+
+def test_datasets(tmp_dir, dvc):
+    template = {
+        "datasets": [
+            {"name": "${ds1.name}", "url": "${ds1.url}", "type": "dvcx"},
+            {
+                "name": "${ds2.name}",
+                "url": "${ds2.url}",
+                "type": "dvc",
+                "path": "${ds2.path}",
+            },
+            {
+                "name": "${ds3.name}",
+                "url": "${ds3.url}",
+                "type": "url",
+            },
+        ]
+    }
+
+    (tmp_dir / "params.yaml").dump(
+        {
+            "ds1": {"name": "dogs", "url": "dvcx://dogs"},
+            "ds2": {
+                "name": "example-get-started",
+                "url": "[email protected]:iterative/example-get-started.git",
+                "path": "path",
+            },
+            "ds3": {
+                "name": "cloud-versioning-demo",
+                "url": "s3://cloud-versioning-demo",
+            },
+        }
+    )
+
+    resolver = DataResolver(dvc, tmp_dir, template)
+    assert resolver.resolve_datasets() == [
+        {"name": "dogs", "url": "dvcx://dogs", "type": "dvcx"},
+        {
+            "name": "example-get-started",
+            "url": "[email protected]:iterative/example-get-started.git",
+            "type": "dvc",
+            "path": "path",
+        },
+        {
+            "name": "cloud-versioning-demo",
+            "url": "s3://cloud-versioning-demo",
+            "type": "url",
+        },
+    ]

tests/func/plots/test_collect.py

@@ -0,0 +1,63 @@
+import dpath
+
+from dvc.repo.plots import Plots
+
+
+def test_subdir_config_not_overwritten_by_parents(tmp_dir, scm, dvc):
+    plot_data = [
+        {"x": 1, "y": 0.1},
+        {"x": 2, "y": 0.2},
+        {"x": 3, "y": 0.3},
+    ]
+    subdir_plot_data = [
+        {"x": 1, "y": 0.2, "z": 0.1},
+        {"x": 2, "y": 0.3, "z": 0.2},
+        {"x": 3, "y": 0.4, "z": 0.3},
+    ]
+
+    (tmp_dir / "plots").mkdir()
+    (tmp_dir / "plots" / "subdir").mkdir()
+
+    (tmp_dir / "plots" / "plot.json").dump_json(plot_data)
+    (tmp_dir / "plots" / "subdir" / "plot.json").dump_json(subdir_plot_data)
+
+    plots_config = [
+        {
+            "plots/subdir/": {
+                "x": "z",
+                "y": "x",
+            }
+        },
+        {"plots": {"x": "x", "y": "y"}},
+        {
+            "subdir axis defined by filename": {
+                "x": {"plots/subdir/plot.json": "x"},
+                "y": {"plots/subdir/plot.json": "y"},
+            }
+        },
+    ]
+
+    from dvc.utils.serialize import modify_yaml
+
+    with modify_yaml("dvc.yaml") as dvcfile_content:
+        dvcfile_content["plots"] = plots_config
+
+    scm.add(
+        [
+            "plots/plot.json",
+            "plots/subdir/plot.json",
+            "dvc.yaml",
+        ]
+    )
+    scm.commit("add data sources")
+
+    plots = next(Plots(dvc).collect())
+
+    assert dpath.get(plots, "workspace/definitions/data/dvc.yaml/data") == {
+        "plots/plot.json": {"x": "x", "y": "y"},
+        "plots/subdir/plot.json": {"x": "z", "y": "x"},
+        "subdir axis defined by filename": {
+            "x": {"plots/subdir/plot.json": "x"},
+            "y": {"plots/subdir/plot.json": "y"},
+        },
+    }

tests/func/plots/test_show.py

@@ -172,6 +172,87 @@ def test_plots_show_overlap(tmp_dir, dvc, run_copy_metrics, clear_before_run):
     )
 
 
+def test_plots_show_nested_x_dict(tmp_dir, dvc, scm):
+    rel_pipeline_dir = "pipelines/data-increment"
+
+    pipeline_rel_dvclive_metrics_dir = "dvclive/plots/metrics"
+    dvc_rel_dvclive_metrics_dir = (
+        f"{rel_pipeline_dir}/{pipeline_rel_dvclive_metrics_dir}"
+    )
+
+    pipeline_dir = tmp_dir / rel_pipeline_dir
+    dvclive_metrics_dir = pipeline_dir / pipeline_rel_dvclive_metrics_dir
+    dvclive_metrics_dir.mkdir(parents=True)
+
+    def _get_plot_defn(rel_dir: str) -> dict:
+        return {
+            "template": "simple",
+            "x": {f"{rel_dir}/Max_Leaf_Nodes.tsv": "Max_Leaf_Nodes"},
+            "y": {f"{rel_dir}/Error.tsv": "Error"},
+        }
+
+    (pipeline_dir / "dvc.yaml").dump(
+        {
+            "plots": [
+                {
+                    "Error vs max_leaf_nodes": _get_plot_defn(
+                        pipeline_rel_dvclive_metrics_dir
+                    )
+                },
+            ]
+        },
+    )
+
+    dvclive_metrics_dir.gen(
+        {
+            "Error.tsv": "step\tError\n" "0\t0.11\n" "1\t0.22\n" "2\t0.44\n",
+            "Max_Leaf_Nodes.tsv": "step\tMax_Leaf_Nodes\n"
+            "0\t5\n"
+            "1\t50\n"
+            "2\t500\n",
+        }
+    )
+
+    scm.commit("add dvc.yaml and dvclive metrics")
+
+    result = dvc.plots.show()
+    assert result == {
+        "workspace": {
+            "definitions": {
+                "data": {
+                    f"{rel_pipeline_dir}/dvc.yaml": {
+                        "data": {
+                            "Error vs max_leaf_nodes": _get_plot_defn(
+                                dvc_rel_dvclive_metrics_dir
+                            )
+                        },
+                    }
+                }
+            },
+            "sources": {
+                "data": {
+                    f"{dvc_rel_dvclive_metrics_dir}/Error.tsv": {
+                        "data": [
+                            {"Error": "0.11", "step": "0"},
+                            {"Error": "0.22", "step": "1"},
+                            {"Error": "0.44", "step": "2"},
+                        ],
+                        "props": {},
+                    },
+                    f"{dvc_rel_dvclive_metrics_dir}/Max_Leaf_Nodes.tsv": {
+                        "data": [
+                            {"Max_Leaf_Nodes": "5", "step": "0"},
+                            {"Max_Leaf_Nodes": "50", "step": "1"},
+                            {"Max_Leaf_Nodes": "500", "step": "2"},
+                        ],
+                        "props": {},
+                    },
+                }
+            },
+        }
+    }
+
+
 def test_dir_plots(tmp_dir, dvc, run_copy_metrics):
     subdir = tmp_dir / "subdir"
     subdir.mkdir()
@@ -419,3 +500,73 @@ def test_show_plots_defined_with_native_os_path(tmp_dir, dvc, scm, capsys):
     json_data = json_out["data"]
     assert json_data[f"{top_level_plot}"]
     assert json_data[stage_plot]
+
+
+@pytest.mark.parametrize(
+    "plot_config,expanded_config,expected_datafiles",
+    [
+        (
+            {
+                "comparison": {
+                    "x": {"${data1}": "${a}"},
+                    "y": {"sub/dir/data2.json": "${b}"},
+                }
+            },
+            {
+                "comparison": {
+                    "x": {"data1.json": "a"},
+                    "y": {"sub/dir/data2.json": "b"},
+                }
+            },
+            ["data1.json", os.path.join("sub", "dir", "data2.json")],
+        ),
+        (
+            {"${data1}": None},
+            {"data1.json": {}},
+            ["data1.json"],
+        ),
+        (
+            "${data1}",
+            {"data1.json": {}},
+            ["data1.json"],
+        ),
+    ],
+)
+def test_top_level_parametrized(
+    tmp_dir, dvc, plot_config, expanded_config, expected_datafiles
+):
+    (tmp_dir / "params.yaml").dump(
+        {"data1": "data1.json", "a": "a", "b": "b", "c": "c"}
+    )
+    data = {
+        "data1.json": [
+            {"a": 1, "b": 0.1, "c": 0.01},
+            {"a": 2, "b": 0.2, "c": 0.02},
+        ],
+        os.path.join("sub", "dir", "data.json"): [
+            {"a": 6, "b": 0.6, "c": 0.06},
+            {"a": 7, "b": 0.7, "c": 0.07},
+        ],
+    }
+
+    for filename, content in data.items():
+        dirname = os.path.dirname(filename)
+        if dirname:
+            os.makedirs(dirname)
+        (tmp_dir / filename).dump_json(content, sort_keys=True)
+
+    config_file = "dvc.yaml"
+    with modify_yaml(config_file) as dvcfile_content:
+        dvcfile_content["plots"] = [plot_config]
+
+    result = dvc.plots.show()
+
+    assert expanded_config == get_plot(
+        result, "workspace", typ="definitions", file=config_file
+    )
+
+    for filename, content in data.items():
+        if filename in expected_datafiles:
+            assert content == get_plot(result, "workspace", file=filename)
+        else:
+            assert filename not in get_plot(result, "workspace")

tests/func/test_data_cloud.py

@@ -1,14 +1,15 @@
 import logging
 import os
 import shutil
+from os.path import join
 
 import pytest
-from flaky.flaky_decorator import flaky
 
 import dvc_data
 from dvc.cli import main
 from dvc.exceptions import CheckoutError
 from dvc.repo.open_repo import clean_repos
+from dvc.scm import CloneError
 from dvc.stage.exceptions import StageNotFound
 from dvc.testing.remote_tests import TestRemote  # noqa: F401
 from dvc.utils.fs import remove
@@ -207,7 +208,7 @@ def test_verify_hashes(tmp_dir, scm, dvc, mocker, tmp_path_factory, local_remote
     assert hash_spy.call_count == 10
 
 
-@flaky(max_runs=3, min_passes=1)
+@pytest.mark.flaky(reruns=3)
 @pytest.mark.parametrize("erepo_type", ["git_dir", "erepo_dir"])
 def test_pull_git_imports(request, tmp_dir, dvc, scm, erepo_type):
     erepo = request.getfixturevalue(erepo_type)
@@ -581,6 +582,58 @@ def test_target_remote(tmp_dir, dvc, make_remote):
     }
 
 
+def test_output_target_remote(tmp_dir, dvc, make_remote):
+    make_remote("default", default=True)
+    make_remote("for_foo", default=False)
+    make_remote("for_bar", default=False)
+
+    tmp_dir.dvc_gen("foo", "foo")
+    tmp_dir.dvc_gen("bar", "bar")
+    tmp_dir.dvc_gen("data", {"one": "one", "two": "two"})
+
+    with (tmp_dir / "foo.dvc").modify() as d:
+        d["outs"][0]["remote"] = "for_foo"
+
+    with (tmp_dir / "bar.dvc").modify() as d:
+        d["outs"][0]["remote"] = "for_bar"
+
+    # push foo and data to for_foo remote
+    dvc.push(remote="for_foo")
+
+    default = dvc.cloud.get_remote_odb("default")
+    for_foo = dvc.cloud.get_remote_odb("for_foo")
+    for_bar = dvc.cloud.get_remote_odb("for_bar")
+
+    # hashes for foo and data, but not bar
+    expected = {
+        "acbd18db4cc2f85cedef654fccc4a4d8",
+        "f97c5d29941bfb1b2fdab0874906ab82",
+        "6b18131dc289fd37006705affe961ef8.dir",
+        "b8a9f715dbb64fd5c56e7783c6820a61",
+    }
+
+    assert set(default.all()) == set()
+    assert set(for_foo.all()) == expected
+    assert set(for_bar.all()) == set()
+
+    # push everything without specifying remote
+    dvc.push()
+    assert set(default.all()) == {
+        "f97c5d29941bfb1b2fdab0874906ab82",
+        "6b18131dc289fd37006705affe961ef8.dir",
+        "b8a9f715dbb64fd5c56e7783c6820a61",
+    }
+    assert set(for_foo.all()) == expected
+    assert set(for_bar.all()) == {"37b51d194a7513e45b56f6524f2d51f2"}
+
+    clean(["foo", "bar", "data"], dvc)
+
+    # pull foo and data from for_foo remote
+    dvc.pull(remote="for_foo", allow_missing=True)
+
+    assert set(dvc.cache.local.all()) == expected
+
+
 def test_pull_allow_missing(tmp_dir, dvc, local_remote):
     dvc.stage.add(name="bar", outs=["bar"], cmd="echo bar > bar")
 
@@ -593,3 +646,35 @@ def test_pull_allow_missing(tmp_dir, dvc, local_remote):
 
     stats = dvc.pull(allow_missing=True)
     assert stats["fetched"] == 1
+
+
+def test_pull_granular_excluding_import_that_cannot_be_pulled(
+    tmp_dir, dvc, local_remote, mocker
+):
+    """Regression test for https://github.com/iterative/dvc/issues/10309."""
+
+    mocker.patch("dvc.fs.dvc._DVCFileSystem", side_effect=CloneError("SCM error"))
+    (stage,) = tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
+    imp_stage = dvc.imp(
+        "https://user:[email protected]/iterative/dvc.git",
+        "dir",
+        out="new_dir",
+        rev="HEAD",
+        no_exec=True,
+    )
+    dvc.push()
+
+    shutil.rmtree("dir")
+    dvc.cache.local.clear()
+
+    assert dvc.pull(stage.addressing) == {
+        "added": [join("dir", "")],
+        "deleted": [],
+        "modified": [],
+        "fetched": 3,
+    }
+
+    with pytest.raises(CloneError, match="SCM error"):
+        dvc.pull()
+    with pytest.raises(CloneError, match="SCM error"):
+        dvc.pull(imp_stage.addressing)

tests/func/test_dataset.py

@@ -0,0 +1,474 @@
+import os
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING
+
+import pytest
+from attrs import define, evolve, field, has
+
+from dvc.dependency.base import Dependency
+from dvc.exceptions import ReproductionError
+from dvc.repo.datasets import (
+    DatasetNotFoundError,
+    DatasetSpec,
+    DVCDataset,
+    DVCDatasetLock,
+    DVCDatasetSpec,
+    DVCXDataset,
+    DVCXDatasetLock,
+    FileInfo,
+    URLDataset,
+    URLDatasetLock,
+)
+from dvc_data.hashfile.meta import Meta
+from dvc_data.index import HashInfo, Tree
+
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+
+
+@define
+class MockedDVCXVersionInfo:
+    version: int
+    created_at: datetime = field(factory=lambda: datetime.now(timezone.utc))
+
+
+def evolve_recursive(inst, **changes):
+    """Recursive attr.evolve() method, where any attr-based attributes
+    will be evolved too.
+    """
+    for key, value in changes.items():
+        v = getattr(inst, key)
+        if has(type(v)) and isinstance(value, dict):
+            value = evolve_recursive(v, **value)
+        changes[key] = value
+    return evolve(inst, **changes)
+
+
+def test_dvc(tmp_dir, scm, dvc: "Repo"):
+    datasets = dvc.datasets
+
+    tmp_dir.scm_gen("file", "file", commit="add file")
+    dataset = datasets.add("mydataset", tmp_dir.fs_path, "dvc", path="file")
+    expected = DVCDataset(
+        manifest_path=(tmp_dir / "dvc.yaml").fs_path,
+        spec=DVCDatasetSpec(
+            name="mydataset", url=tmp_dir.fs_path, type="dvc", path="file"
+        ),
+        lock=DVCDatasetLock(
+            name="mydataset",
+            url=tmp_dir.fs_path,
+            type="dvc",
+            path="file",
+            rev_lock=scm.get_rev(),
+        ),
+    )
+    assert "mydataset" in datasets
+    assert dataset == datasets["mydataset"] == expected
+    tmp_dir.scm_gen("file", "file", commit="update file")
+
+    old, new = datasets.update("mydataset")
+    assert old == dataset
+    assert old != new
+    expected = evolve_recursive(expected, lock={"rev_lock": scm.get_rev()})
+    assert new == datasets["mydataset"] == expected
+
+    # noop
+    old, new = datasets.update("mydataset")
+    assert old == new
+
+
+def test_dvcx(tmp_dir, dvc, mocker):
+    datasets = dvc.datasets
+
+    version_info = [MockedDVCXVersionInfo(1), MockedDVCXVersionInfo(2)]
+    version_info.append(version_info[1])
+    mocker.patch("dvc.repo.datasets._get_dataset_info", side_effect=version_info)
+
+    dataset = datasets.add("mydataset", "dataset", "dvcx")
+    expected = DVCXDataset(
+        manifest_path=(tmp_dir / "dvc.yaml").fs_path,
+        spec=DatasetSpec(name="mydataset", url="dataset", type="dvcx"),
+        lock=DVCXDatasetLock(
+            name="mydataset",
+            url="dataset",
+            type="dvcx",
+            version=1,
+            created_at=version_info[0].created_at,
+        ),
+    )
+    assert "mydataset" in datasets
+    assert dataset == datasets["mydataset"] == expected
+
+    old, new = datasets.update("mydataset")
+    assert old == dataset
+    assert old != new
+    expected = evolve_recursive(
+        expected, lock={"version": 2, "created_at": version_info[1].created_at}
+    )
+    assert new == datasets["mydataset"] == expected
+
+    # noop
+    old, new = datasets.update("mydataset")
+    assert old == new
+
+
+def test_url(tmp_dir, dvc, mocker):
+    datasets = dvc.datasets
+
+    tree = Tree()
+    tree.add(("foo",), Meta(version_id="1"), None)
+    tree.digest(with_meta=True)
+    tree_meta = Meta(isdir=True)
+
+    def mocked_save(d):
+        d.meta, d.obj, d.hash_info = tree_meta, tree, HashInfo("md5", "value.dir")
+
+    mocker.patch.object(Dependency, "save", mocked_save)
+
+    dataset = datasets.add("mydataset", "s3://dataset", "url")
+    expected = URLDataset(
+        manifest_path=(tmp_dir / "dvc.yaml").fs_path,
+        spec=DatasetSpec(name="mydataset", url="s3://dataset", type="url"),
+        lock=URLDatasetLock(
+            name="mydataset",
+            url="s3://dataset",
+            type="url",
+            meta=Meta(isdir=True),
+            files=[FileInfo(relpath="foo", meta=Meta(version_id="1"))],
+        ),
+    )
+    assert "mydataset" in datasets
+    assert dataset == datasets["mydataset"] == expected
+
+    tree.add(("bar",), Meta(version_id="2"), None)
+    old, new = datasets.update("mydataset")
+    assert old == dataset
+    assert old != new
+
+    assert expected.lock
+    new_files = [
+        *expected.lock.files,
+        FileInfo(relpath="bar", meta=Meta(version_id="2")),
+    ]
+
+    expected = evolve_recursive(expected, lock={"files": new_files})
+    assert new == datasets["mydataset"] == expected
+
+    # noop
+    old, new = datasets.update("mydataset")
+    assert old == new
+
+
+def test_dvc_dump(tmp_dir, dvc):
+    manifest_path = os.path.join(tmp_dir, "dvc.yaml")
+    spec = DVCDatasetSpec(
+        name="mydataset", url=tmp_dir.fs_path, type="dvc", path="path", rev="main"
+    )
+    lock = DVCDatasetLock(rev_lock="0" * 40, **spec.to_dict())
+    dataset = DVCDataset(manifest_path=manifest_path, spec=spec, lock=lock)
+
+    dvc.datasets.dump(dataset)
+
+    spec_d = {
+        "name": "mydataset",
+        "type": "dvc",
+        "url": tmp_dir.fs_path,
+        "path": "path",
+        "rev": "main",
+    }
+    assert (tmp_dir / "dvc.yaml").parse() == {"datasets": [spec_d]}
+    assert (tmp_dir / "dvc.lock").parse() == {
+        "schema": "2.0",
+        "stages": {},
+        "datasets": [{**spec_d, "rev_lock": "0" * 40}],
+    }
+
+    dvc._reset()
+    assert "_datasets" not in vars(dvc.datasets)
+    # test that we can read them back
+    assert dvc.datasets["mydataset"] == dataset
+
+
+def test_dvcx_dump(tmp_dir, dvc):
+    manifest_path = os.path.join(tmp_dir, "dvc.yaml")
+    spec = DatasetSpec(name="mydataset", url="dataset", type="dvcx")
+    dt = datetime.now(tz=timezone.utc)
+    lock = DVCXDatasetLock(version=1, created_at=dt, **spec.to_dict())
+    dataset = DVCXDataset(manifest_path=manifest_path, spec=spec, lock=lock)
+
+    dvc.datasets.dump(dataset)
+
+    spec_d = {"name": "mydataset", "type": "dvcx", "url": "dataset"}
+    assert (tmp_dir / "dvc.yaml").parse() == {"datasets": [spec_d]}
+    assert (tmp_dir / "dvc.lock").parse() == {
+        "schema": "2.0",
+        "stages": {},
+        "datasets": [{**spec_d, "version": 1, "created_at": dt.isoformat()}],
+    }
+
+    dvc._reset()
+    assert "_datasets" not in vars(dvc.datasets)
+    # test that we can read them back
+    assert dvc.datasets["mydataset"] == dataset
+
+
+def test_url_dump(tmp_dir, dvc):
+    manifest_path = os.path.join(tmp_dir, "dvc.yaml")
+    spec = DatasetSpec(name="mydataset", url="s3://dataset", type="url")
+    files = [FileInfo(relpath="foo", meta=Meta(version_id="1"))]
+    lock = URLDatasetLock(meta=Meta(isdir=True), files=files, **spec.to_dict())
+    dataset = URLDataset(manifest_path=manifest_path, spec=spec, lock=lock)
+
+    dvc.datasets.dump(dataset)
+
+    spec_d = {"name": "mydataset", "url": "s3://dataset", "type": "url"}
+    assert (tmp_dir / "dvc.yaml").parse() == {"datasets": [spec_d]}
+    assert (tmp_dir / "dvc.lock").parse() == {
+        "schema": "2.0",
+        "datasets": [
+            {
+                **spec_d,
+                "meta": {"isdir": True},
+                "files": [{"relpath": "foo", "meta": {"version_id": "1"}}],
+            }
+        ],
+        "stages": {},
+    }
+
+    dvc._reset()
+    assert "_datasets" not in vars(dvc.datasets)
+    # test that we can read them back
+    assert dvc.datasets["mydataset"] == dataset
+
+
+def test_invalidation(tmp_dir, dvc):
+    manifest_path = os.path.join(tmp_dir, "dvc.yaml")
+    spec = DatasetSpec(name="mydataset", url="url1", type="url")
+    lock = DVCXDatasetLock(
+        name="mydataset",
+        url="dataset",
+        type="dvcx",
+        version=1,
+        created_at=datetime.now(tz=timezone.utc),
+    )
+    dvc.datasets._dump_spec(manifest_path, spec)
+    dvc.datasets._dump_lock(manifest_path, lock)
+
+    assert dvc.datasets["mydataset"] == URLDataset(
+        manifest_path=manifest_path,
+        spec=spec,
+        lock=None,  # lock should be discarded
+    )
+
+
+def test_dvc_dataset_pipeline(tmp_dir, dvc, scm):
+    dvc.datasets.add("mydataset", tmp_dir.fs_path, "dvc")
+
+    stage = dvc.stage.add(cmd="echo", name="train", deps=["ds://mydataset"])
+    assert (tmp_dir / "dvc.yaml").parse() == {
+        "datasets": [{"name": "mydataset", "url": tmp_dir.fs_path, "type": "dvc"}],
+        "stages": {"train": {"cmd": "echo", "deps": ["ds://mydataset"]}},
+    }
+
+    assert dvc.status() == {"train": [{"changed deps": {"ds://mydataset": "new"}}]}
+    assert dvc.reproduce() == [stage]
+
+    d = (tmp_dir / "dvc.lock").parse()
+    assert d["stages"]["train"]["deps"][0] == {
+        "path": "ds://mydataset",
+        "dataset": d["datasets"][0],
+    }
+
+    assert dvc.status() == {}
+    assert dvc.reproduce() == []
+
+    tmp_dir.scm_gen("foo", "foo", commit="add foo")
+    dvc.datasets.update("mydataset")
+
+    assert dvc.status() == {"train": [{"changed deps": {"ds://mydataset": "modified"}}]}
+    assert dvc.reproduce() == [stage]
+
+
+def test_dvcx_dataset_pipeline(mocker, tmp_dir, dvc):
+    version_info = [MockedDVCXVersionInfo(1), MockedDVCXVersionInfo(2)]
+    mocker.patch("dvc.repo.datasets._get_dataset_info", side_effect=version_info)
+
+    dvc.datasets.add("mydataset", "dataset", "dvcx")
+
+    stage = dvc.stage.add(cmd="echo", name="train", deps=["ds://mydataset"])
+    assert (tmp_dir / "dvc.yaml").parse() == {
+        "datasets": [{"name": "mydataset", "url": "dataset", "type": "dvcx"}],
+        "stages": {"train": {"cmd": "echo", "deps": ["ds://mydataset"]}},
+    }
+
+    assert dvc.status() == {"train": [{"changed deps": {"ds://mydataset": "new"}}]}
+    assert dvc.reproduce() == [stage]
+
+    d = (tmp_dir / "dvc.lock").parse()
+    assert d["stages"]["train"]["deps"][0] == {
+        "path": "ds://mydataset",
+        "dataset": d["datasets"][0],
+    }
+
+    assert dvc.status() == {}
+    assert dvc.reproduce() == []
+
+    dvc.datasets.update("mydataset")
+
+    assert dvc.status() == {"train": [{"changed deps": {"ds://mydataset": "modified"}}]}
+    assert dvc.reproduce() == [stage]
+
+
+def test_url_dataset_pipeline(mocker, tmp_dir, dvc):
+    tree = Tree()
+    tree.add(("foo",), Meta(version_id="1"), None)
+    tree.digest(with_meta=True)
+    tree_meta = Meta(isdir=True)
+
+    def mocked_save(d):
+        d.meta, d.obj, d.hash_info = tree_meta, tree, HashInfo("md5", "value.dir")
+
+    mocker.patch.object(Dependency, "save", mocked_save)
+
+    dvc.datasets.add("mydataset", "s3://mydataset", "url")
+
+    stage = dvc.stage.add(cmd="echo", name="train", deps=["ds://mydataset"])
+    assert (tmp_dir / "dvc.yaml").parse() == {
+        "datasets": [{"name": "mydataset", "url": "s3://mydataset", "type": "url"}],
+        "stages": {"train": {"cmd": "echo", "deps": ["ds://mydataset"]}},
+    }
+
+    assert dvc.status() == {"train": [{"changed deps": {"ds://mydataset": "new"}}]}
+    assert dvc.reproduce() == [stage]
+
+    d = (tmp_dir / "dvc.lock").parse()
+    assert d["stages"]["train"]["deps"][0] == {
+        "path": "ds://mydataset",
+        "dataset": d["datasets"][0],
+    }
+
+    assert dvc.status() == {}
+    assert dvc.reproduce() == []
+
+    tree.add(("bar",), Meta(version_id="2"), None)
+    dvc.datasets.update("mydataset")
+
+    assert dvc.status() == {"train": [{"changed deps": {"ds://mydataset": "modified"}}]}
+    assert dvc.reproduce() == [stage]
+
+
+def test_pipeline_when_not_in_sync(tmp_dir, dvc):
+    manifest_path = os.path.join(tmp_dir, "dvc.yaml")
+    spec = DatasetSpec(name="mydataset", url="url1", type="url")
+    lock = DVCXDatasetLock(
+        name="mydataset",
+        url="dataset",
+        type="dvcx",
+        version=1,
+        created_at=datetime.now(tz=timezone.utc),
+    )
+    dvc.datasets._dump_spec(manifest_path, spec)
+    dvc.datasets._dump_lock(manifest_path, lock)
+
+    dvc.stage.add(name="train", cmd="echo", deps=["ds://mydataset"])
+    assert dvc.status() == {
+        "train": [{"changed deps": {"ds://mydataset": "not in sync"}}]
+    }
+    with pytest.raises(ReproductionError) as exc:
+        dvc.reproduce()
+    assert "not in sync" in str(exc.value.__cause__)
+
+
+def test_collect(tmp_dir, dvc):
+    manifest_path1 = os.path.join(tmp_dir, "dvc.yaml")
+    dt = datetime.now(tz=timezone.utc)
+    spec = DatasetSpec(name="mydataset1", url="url1", type="dvcx")
+    lock = DVCXDatasetLock(version=1, created_at=dt, **spec.to_dict())
+    mydataset1 = DVCXDataset(manifest_path=manifest_path1, spec=spec, lock=lock)
+    dvc.datasets.dump(mydataset1)
+
+    (tmp_dir / "sub").mkdir()
+    manifest_path2 = os.path.join(tmp_dir, "sub", "dvc.yaml")
+    spec = DVCDatasetSpec(
+        name="mydataset2", url=tmp_dir.fs_path, type="dvc", path="path"
+    )
+    lock = DVCDatasetLock(rev_lock="0" * 40, **spec.to_dict())
+    mydataset2 = DVCDataset(manifest_path=manifest_path2, spec=spec, lock=lock)
+    dvc.datasets.dump(mydataset2)
+
+    dvc._reset()
+    assert "_datasets" not in vars(dvc.datasets)
+
+    assert len(dvc.datasets) == 2
+    assert "mydataset1" in dvc.datasets
+    assert "mydataset2" in dvc.datasets
+    assert list(iter(dvc.datasets)) == ["mydataset1", "mydataset2"]
+    assert dvc.datasets["mydataset1"] == mydataset1
+    assert dvc.datasets["mydataset2"] == mydataset2
+    assert dict(dvc.datasets.items()) == {
+        "mydataset1": mydataset1,
+        "mydataset2": mydataset2,
+    }
+
+    with pytest.raises(DatasetNotFoundError, match=r"^dataset not found$"):
+        dvc.datasets["not-existing"]
+
+
+def test_parametrized(tmp_dir, dvc):
+    (tmp_dir / "dvc.yaml").dump(
+        {
+            "datasets": [
+                {"name": "${ds1.name}", "url": "${ds1.url}", "type": "dvcx"},
+                {
+                    "name": "${ds2.name}",
+                    "url": "${ds2.url}",
+                    "type": "dvc",
+                    "path": "${ds2.path}",
+                },
+                {
+                    "name": "${ds3.name}",
+                    "url": "${ds3.url}",
+                    "type": "url",
+                },
+            ]
+        }
+    )
+    (tmp_dir / "params.yaml").dump(
+        {
+            "ds1": {"name": "dogs", "url": "dvcx://dogs"},
+            "ds2": {
+                "name": "example-get-started",
+                "url": "[email protected]:iterative/example-get-started.git",
+                "path": "path",
+            },
+            "ds3": {
+                "name": "cloud-versioning-demo",
+                "url": "s3://cloud-versioning-demo",
+            },
+        }
+    )
+
+    path = (tmp_dir / "dvc.yaml").fs_path
+    assert dict(dvc.datasets.items()) == {
+        "dogs": DVCXDataset(
+            manifest_path=path,
+            spec=DatasetSpec(name="dogs", url="dvcx://dogs", type="dvcx"),
+        ),
+        "example-get-started": DVCDataset(
+            manifest_path=path,
+            spec=DVCDatasetSpec(
+                name="example-get-started",
+                url="[email protected]:iterative/example-get-started.git",
+                path="path",
+                type="dvc",
+            ),
+        ),
+        "cloud-versioning-demo": URLDataset(
+            manifest_path=path,
+            spec=DatasetSpec(
+                name="cloud-versioning-demo",
+                url="s3://cloud-versioning-demo",
+                type="url",
+            ),
+        ),
+    }

tests/func/test_gc.py

@@ -297,7 +297,7 @@ def test_gc_rev_num(tmp_dir, scm, dvc):
 def test_date(tmp_dir, scm, dvc):
     tmp_dir.dvc_gen("testfile", "content", commit="add testfile")
 
-    now = datetime.datetime.now()
+    now = datetime.datetime.now(tz=datetime.timezone.utc)
     datestamp = (now.date() + datetime.timedelta(days=1)).isoformat()
 
     tmp_dir.dvc_gen("testfile", "modified", commit="modified")

tests/integration/plots/conftest.py

@@ -86,15 +86,27 @@ def repo_with_plots(tmp_dir, scm, dvc, run_copy_metrics):
 @pytest.fixture
 def repo_with_config_plots(tmp_dir, scm, dvc, run_copy_metrics):
     def make():
+        # test subdir functionality
+        linear_subdir_a_v1 = [
+            {"x": 1, "y": 0.2},
+            {"x": 2, "y": 0.3},
+            {"x": 3, "y": 0.4},
+        ]
+        # test subdir default values for x = step
+        linear_subdir_b_v1 = [
+            {"step": 1, "y": 0.2},
+            {"step": 2, "y": 0.3},
+            {"step": 3, "y": 0.4},
+        ]
         linear_train_v1 = [
             {"x": 1, "y": 0.1},
             {"x": 2, "y": 0.2},
             {"x": 3, "y": 0.3},
         ]
         linear_test_v1 = [
-            {"x": 1, "y": 0.2},
-            {"x": 2, "y": 0.3},
-            {"x": 3, "y": 0.4},
+            {"x": 1, "y": 0.3},
+            {"x": 2, "y": 0.4},
+            {"x": 3, "y": 0.5},
         ]
 
         confusion_train_v1 = [
@@ -110,6 +122,8 @@ def repo_with_config_plots(tmp_dir, scm, dvc, run_copy_metrics):
             {"actual": 1, "predicted": 0},
         ]
 
+        (tmp_dir / "linear_subdir_a_src.json").dump_json(linear_subdir_a_v1)
+        (tmp_dir / "linear_subdir_b_src.json").dump_json(linear_subdir_b_v1)
         (tmp_dir / "linear_train_src.json").dump_json(linear_train_v1)
         (tmp_dir / "linear_test_src.json").dump_json(linear_test_v1)
         (tmp_dir / "confusion_train_src.json").dump_json(confusion_train_v1)
@@ -117,6 +131,8 @@ def repo_with_config_plots(tmp_dir, scm, dvc, run_copy_metrics):
 
         scm.add(
             [
+                "linear_subdir_a_src.json",
+                "linear_subdir_b_src.json",
                 "linear_train_src.json",
                 "linear_test_src.json",
                 "confusion_train_src.json",
@@ -125,6 +141,23 @@ def repo_with_config_plots(tmp_dir, scm, dvc, run_copy_metrics):
         )
         scm.commit("add data sources")
 
+        (tmp_dir / "subdirA").mkdir()
+        (tmp_dir / "subdirB").mkdir()
+
+        run_copy_metrics(
+            "linear_subdir_a_src.json",
+            "subdirA/linear_subdir.json",
+            name="linear_subdir",
+            outs=["subdirA/linear_subdir.json"],
+            commit="linear_subdir",
+        )
+        run_copy_metrics(
+            "linear_subdir_b_src.json",
+            "subdirB/linear_subdir.json",
+            name="linear_subdir",
+            outs=["subdirB/linear_subdir.json"],
+            commit="linear_subdir",
+        )
         run_copy_metrics(
             "linear_train_src.json",
             "linear_train.json",
@@ -153,7 +186,16 @@ def repo_with_config_plots(tmp_dir, scm, dvc, run_copy_metrics):
             outs=["confusion_test.json"],
             commit="confusion_test",
         )
-        plots_config = [
+
+        subdir_a_config = {
+            "x": "x",
+            "y": "y",
+            "title": "subdir plots with x and y defined",
+        }
+
+        subdir_b_config = {"title": "subdir plots with default x and y"}
+
+        other_plots_config = [
             {
                 "linear_train_vs_test": {
                     "x": "x",
@@ -176,18 +218,35 @@ def repo_with_config_plots(tmp_dir, scm, dvc, run_copy_metrics):
         from dvc.utils.serialize import modify_yaml
 
         with modify_yaml("dvc.yaml") as dvcfile_content:
-            dvcfile_content["plots"] = plots_config
+            dvcfile_content["plots"] = [
+                {"subdirA": subdir_a_config},
+                {"subdirB": subdir_b_config},
+                *other_plots_config,
+            ]
 
         scm.add(["dvc.yaml", "dvc.lock"])
         scm.commit("commit dvc files")
+
+        # remove generate .gitignore that are considered as plot otherwise
+        (tmp_dir / "subdirA" / ".gitignore").unlink()
+        (tmp_dir / "subdirB" / ".gitignore").unlink()
+
         yield {
             "data": {
+                "subdirA/linear_subdir.json": linear_subdir_a_v1,
+                "subdirB/linear_subdir.json": linear_subdir_b_v1,
                 "linear_train.json": linear_train_v1,
                 "linear_test.json": linear_test_v1,
                 "confusion_train.json": confusion_train_v1,
                 "confusion_test.json": confusion_test_v1,
             },
-            "configs": {"dvc.yaml": plots_config},
+            "configs": {
+                "dvc.yaml": [
+                    {"subdirA/linear_subdir.json": subdir_a_config},
+                    {"subdirB/linear_subdir.json": subdir_b_config},
+                    *other_plots_config,
+                ]
+            },
         }
 
     return make

tests/integration/plots/test_plots.py

@@ -163,7 +163,7 @@ def verify_vega_props(plot_id, json_result, title, x, y, **kwargs):
     assert len(data) == 1
     data = first(data)
 
-    assert dpath.get(data, ["content", "title"]) == title
+    assert dpath.get(data, ["content", "title", "text"]) == title
 
     try:
         # TODO confusion_matrix_plot - need to find better way of asserting
@@ -440,6 +440,8 @@ def test_repo_with_config_plots(tmp_dir, capsys, repo_with_config_plots):
     html_result = extract_vega_specs(
         html_path,
         [
+            "subdirA/linear_subdir.json",
+            "subdirB/linear_subdir.json",
             "linear_train_vs_test",
             "confusion_train_vs_test",
         ],
@@ -466,6 +468,22 @@ def test_repo_with_config_plots(tmp_dir, capsys, repo_with_config_plots):
         ]
         == ble
     )
+    assert (
+        html_result["subdirA/linear_subdir.json"]["layer"][1]["encoding"]["x"]["field"]
+        == "x"
+    )
+    assert (
+        html_result["subdirA/linear_subdir.json"]["layer"][1]["encoding"]["y"]["field"]
+        == "y"
+    )
+    assert (
+        html_result["subdirB/linear_subdir.json"]["layer"][1]["encoding"]["x"]["field"]
+        == "step"
+    )
+    assert (
+        html_result["subdirB/linear_subdir.json"]["layer"][1]["encoding"]["y"]["field"]
+        == "y"
+    )
 
 
 @pytest.mark.vscode
@@ -481,3 +499,150 @@ def test_repo_with_dvclive_plots(tmp_dir, capsys, repo_with_dvclive_plots):
         }
         assert json_result == expected_result
         assert split_json_result == expected_result
+
+
+@pytest.mark.vscode
+def test_nested_x_defn_collection(tmp_dir, dvc, scm, capsys):
+    rel_pipeline_dir = "pipelines/data-increment"
+    pipeline_rel_dvclive_metrics_dir = "dvclive/plots/metrics"
+    pipeline_rel_other_logger_dir = "other/logger"
+
+    dvc_rel_dvclive_metrics_dir = (
+        f"{rel_pipeline_dir}/{pipeline_rel_dvclive_metrics_dir}"
+    )
+    dvc_rel_other_logger_dir = f"{rel_pipeline_dir}/{pipeline_rel_other_logger_dir}"
+
+    pipeline_dir = tmp_dir / rel_pipeline_dir
+    dvclive_metrics_dir = pipeline_dir / pipeline_rel_dvclive_metrics_dir
+    dvclive_metrics_dir.mkdir(parents=True)
+    other_logger_dir = pipeline_dir / pipeline_rel_other_logger_dir
+    other_logger_dir.mkdir(parents=True)
+
+    (pipeline_dir / "dvc.yaml").dump(
+        {
+            "plots": [
+                {
+                    "Error vs max_leaf_nodes": {
+                        "template": "simple",
+                        "x": {
+                            f"{pipeline_rel_dvclive_metrics_dir}"
+                            "/Max_Leaf_Nodes.tsv": "Max_Leaf_Nodes"
+                        },
+                        "y": {f"{pipeline_rel_dvclive_metrics_dir}/Error.tsv": "Error"},
+                    }
+                },
+                {
+                    f"{pipeline_rel_other_logger_dir}/multiple_metrics.json": {
+                        "x": "x",
+                        "y": ["y1", "y2"],
+                    },
+                },
+                {
+                    f"{pipeline_rel_dvclive_metrics_dir}/Error.tsv": {"y": ["Error"]},
+                },
+                {
+                    "max leaf nodes": {
+                        "y": {
+                            f"{pipeline_rel_dvclive_metrics_dir}"
+                            "/Max_Leaf_Nodes.tsv": "Max_Leaf_Nodes"
+                        }
+                    },
+                },
+            ]
+        },
+    )
+    dvclive_metrics_dir.gen(
+        {
+            "Error.tsv": "step\tError\n" "0\t0.11\n" "1\t0.22\n" "2\t0.44\n",
+            "Max_Leaf_Nodes.tsv": "step\tMax_Leaf_Nodes\n"
+            "0\t5\n"
+            "1\t50\n"
+            "2\t500\n",
+        }
+    )
+    (other_logger_dir / "multiple_metrics.json").dump(
+        [
+            {"x": 0, "y1": 0.1, "y2": 10},
+            {"x": 1, "y1": 0.2, "y2": 22},
+        ]
+    )
+
+    scm.commit("add dvc.yaml and metrics")
+
+    _, _, split_json_result = call(capsys, subcommand="diff")
+    assert len(split_json_result.keys()) == 1
+    assert len(split_json_result["data"].keys()) == 4
+
+    separate_x_file = split_json_result["data"]["Error vs max_leaf_nodes"][0]
+
+    assert separate_x_file["anchor_definitions"]["<DVC_METRIC_DATA>"] == [
+        {"Error": "0.11", "Max_Leaf_Nodes": "5", "step": "0", "rev": "workspace"},
+        {"Error": "0.22", "Max_Leaf_Nodes": "50", "step": "1", "rev": "workspace"},
+        {"Error": "0.44", "Max_Leaf_Nodes": "500", "step": "2", "rev": "workspace"},
+    ]
+
+    same_x_file = split_json_result["data"][
+        f"{dvc_rel_other_logger_dir}/multiple_metrics.json"
+    ][0]
+    assert same_x_file["anchor_definitions"]["<DVC_METRIC_DATA>"] == [
+        {
+            "x": 0,
+            "y1": 0.1,
+            "y2": 10,
+            "dvc_inferred_y_value": 0.1,
+            "field": "y1",
+            "rev": "workspace",
+        },
+        {
+            "x": 1,
+            "y1": 0.2,
+            "y2": 22,
+            "dvc_inferred_y_value": 0.2,
+            "field": "y1",
+            "rev": "workspace",
+        },
+        {
+            "x": 0,
+            "y1": 0.1,
+            "y2": 10,
+            "dvc_inferred_y_value": 10,
+            "field": "y2",
+            "rev": "workspace",
+        },
+        {
+            "x": 1,
+            "y1": 0.2,
+            "y2": 22,
+            "dvc_inferred_y_value": 22,
+            "field": "y2",
+            "rev": "workspace",
+        },
+    ]
+
+    inferred_x_from_str = split_json_result["data"][
+        f"{dvc_rel_dvclive_metrics_dir}/Error.tsv"
+    ][0]
+    assert inferred_x_from_str["anchor_definitions"]["<DVC_METRIC_DATA>"] == [
+        {"step": 0, "Error": "0.11", "rev": "workspace"},
+        {"step": 1, "Error": "0.22", "rev": "workspace"},
+        {"step": 2, "Error": "0.44", "rev": "workspace"},
+    ]
+
+    inferred_x_from_dict = split_json_result["data"]["max leaf nodes"][0]
+    assert inferred_x_from_dict["anchor_definitions"]["<DVC_METRIC_DATA>"] == [
+        {"step": 0, "Max_Leaf_Nodes": "5", "rev": "workspace"},
+        {"step": 1, "Max_Leaf_Nodes": "50", "rev": "workspace"},
+        {"step": 2, "Max_Leaf_Nodes": "500", "rev": "workspace"},
+    ]
+
+
+def test_plots_empty_directory(tmp_dir, dvc, scm, capsys):
+    (tmp_dir / "empty").mkdir()
+    (tmp_dir / "dvc.yaml").dump({"plots": [{"empty": {}}]})
+
+    scm.add(["dvc.yaml"])
+    scm.commit("commit dvc files")
+
+    html_path, _, split_json_result = call(capsys)
+    assert split_json_result == {}
+    assert html_path == ""

tests/integration/test_studio_live_experiments.py

@@ -1,19 +1,25 @@
 import pytest
-from dvc_studio_client import env, post_live_metrics
 from funcy import first
 
 from dvc.env import (
+    DVC_EXP_GIT_REMOTE,
     DVC_STUDIO_OFFLINE,
     DVC_STUDIO_REPO_URL,
     DVC_STUDIO_TOKEN,
     DVC_STUDIO_URL,
 )
+from dvc.repo import Repo
+from dvc.testing.scripts import COPY_SCRIPT
+from dvc.utils.studio import get_subrepo_relpath
+from dvc_studio_client import env, post_live_metrics
 
 
+@pytest.mark.studio
 @pytest.mark.parametrize("tmp", [True, False])
 @pytest.mark.parametrize("offline", [True, False])
+@pytest.mark.parametrize("dvc_exp_git_remote", [None, "DVC_EXP_GIT_REMOTE"])
 def test_post_to_studio(
-    tmp_dir, dvc, scm, exp_stage, mocker, monkeypatch, tmp, offline
+    tmp_dir, dvc, scm, exp_stage, mocker, monkeypatch, tmp, offline, dvc_exp_git_remote
 ):
     valid_response = mocker.MagicMock()
     valid_response.status_code = 200
@@ -24,6 +30,8 @@ def test_post_to_studio(
     monkeypatch.setenv(DVC_STUDIO_TOKEN, "STUDIO_TOKEN")
     monkeypatch.setenv(DVC_STUDIO_URL, "https://0.0.0.0")
     monkeypatch.setenv(DVC_STUDIO_OFFLINE, offline)
+    if dvc_exp_git_remote:
+        monkeypatch.setenv(DVC_EXP_GIT_REMOTE, dvc_exp_git_remote)
 
     baseline_sha = scm.get_rev()
     exp_rev = first(
@@ -48,7 +56,7 @@ def test_post_to_studio(
 
         assert start_call.kwargs["json"] == {
             "type": "start",
-            "repo_url": "STUDIO_REPO_URL",
+            "repo_url": dvc_exp_git_remote or "STUDIO_REPO_URL",
             "baseline_sha": baseline_sha,
             "name": name,
             "params": {"params.yaml": {"foo": 1}},
@@ -57,7 +65,7 @@ def test_post_to_studio(
 
         assert done_call.kwargs["json"] == {
             "type": "done",
-            "repo_url": "STUDIO_REPO_URL",
+            "repo_url": dvc_exp_git_remote or "STUDIO_REPO_URL",
             "baseline_sha": baseline_sha,
             "name": name,
             "client": "dvc",
@@ -66,10 +74,45 @@ def test_post_to_studio(
         }
 
 
+@pytest.mark.studio
 @pytest.mark.parametrize("tmp", [True, False])
-def test_post_to_studio_custom_message(
-    tmp_dir, dvc, scm, exp_stage, mocker, monkeypatch, tmp
-):
+def test_post_to_studio_subdir(tmp_dir, scm, mocker, monkeypatch, tmp):
+    live_exp_subdir = "project_a"
+
+    tmp_dir.scm_gen(
+        {
+            live_exp_subdir: {
+                "params.yaml": "foo: 1",
+                "metrics.yaml": "foo: 1",
+                "copy.py": COPY_SCRIPT.encode("utf-8"),
+            },
+        },
+        commit="git init",
+    )
+
+    project_a_dvc = Repo.init(tmp_dir / live_exp_subdir, subdir=True)
+    with monkeypatch.context() as m:
+        m.chdir(project_a_dvc.root_dir)
+
+        exp_stage = project_a_dvc.run(
+            cmd="python copy.py params.yaml metrics.yaml",
+            metrics_no_cache=["metrics.yaml"],
+            params=["foo"],
+            name="copy-file",
+        )
+
+        scm.add(
+            [
+                ".gitignore",
+                "copy.py",
+                "dvc.lock",
+                "dvc.yaml",
+                "metrics.yaml",
+                "params.yaml",
+            ]
+        )
+        scm.commit("dvc init project_a")
+
     valid_response = mocker.MagicMock()
     valid_response.status_code = 200
     mocked_post = mocker.patch("requests.post", return_value=valid_response)
@@ -79,12 +122,16 @@ def test_post_to_studio_custom_message(
     monkeypatch.setenv(env.STUDIO_TOKEN, "STUDIO_TOKEN")
 
     baseline_sha = scm.get_rev()
-    exp_rev = first(
-        dvc.experiments.run(
-            exp_stage.addressing, params=["foo=1"], tmp_dir=tmp, message="foo"
+    with monkeypatch.context() as m:
+        m.chdir(project_a_dvc.root_dir)
+        exp_rev = first(
+            project_a_dvc.experiments.run(
+                exp_stage.addressing, params=["foo=24"], tmp_dir=tmp
+            )
         )
-    )
-    name = dvc.experiments.get_exact_name([exp_rev])[exp_rev]
+
+    name = project_a_dvc.experiments.get_exact_name([exp_rev])[exp_rev]
+    project_a_dvc.close()
     assert mocked_post.call_count == 2
 
     start_call = mocked_post.call_args_list[0]
@@ -94,7 +141,57 @@ def test_post_to_studio_custom_message(
         "repo_url": "STUDIO_REPO_URL",
         "baseline_sha": baseline_sha,
         "name": name,
-        "params": {"params.yaml": {"foo": 1}},
+        "params": {"params.yaml": {"foo": 24}},
+        "subdir": live_exp_subdir,
         "client": "dvc",
-        "message": "foo",
     }
+
+
+@pytest.mark.studio
+def test_monorepo_relpath(tmp_dir, scm):
+    from dvc.repo.destroy import destroy
+
+    tmp_dir.gen({"project_a": {}, "subdir/project_b": {}})
+
+    non_monorepo = Repo.init(tmp_dir)
+    assert get_subrepo_relpath(non_monorepo) == ""
+
+    destroy(non_monorepo)
+
+    monorepo_project_a = Repo.init(tmp_dir / "project_a", subdir=True)
+
+    assert get_subrepo_relpath(monorepo_project_a) == "project_a"
+
+    monorepo_project_b = Repo.init(tmp_dir / "subdir" / "project_b", subdir=True)
+
+    assert get_subrepo_relpath(monorepo_project_b) == "subdir/project_b"
+
+
+@pytest.mark.studio
+def test_virtual_monorepo_relpath(tmp_dir, scm):
+    from dvc.fs.git import GitFileSystem
+    from dvc.repo.destroy import destroy
+
+    tmp_dir.gen({"project_a": {}, "subdir/project_b": {}})
+    scm.commit("initial commit")
+    gfs = GitFileSystem(scm=scm, rev="master")
+
+    non_monorepo = Repo.init(tmp_dir)
+    non_monorepo.fs = gfs
+    non_monorepo.root_dir = "/"
+
+    assert get_subrepo_relpath(non_monorepo) == ""
+
+    destroy(non_monorepo)
+
+    monorepo_project_a = Repo.init(tmp_dir / "project_a", subdir=True)
+    monorepo_project_a.fs = gfs
+    monorepo_project_a.root_dir = "/project_a"
+
+    assert get_subrepo_relpath(monorepo_project_a) == "project_a"
+
+    monorepo_project_b = Repo.init(tmp_dir / "subdir" / "project_b", subdir=True)
+    monorepo_project_b.fs = gfs
+    monorepo_project_b.root_dir = "/subdir/project_b"
+
+    assert get_subrepo_relpath(monorepo_project_b) == "subdir/project_b"

tests/remotes/git_server.py

@@ -1,4 +1,5 @@
 import pytest
+
 from dvc_ssh.tests.cloud import SSH, TEST_SSH_KEY_PATH, TEST_SSH_USER
 
 

tests/unit/command/test_add.py

@@ -22,6 +22,7 @@ def test_add(mocker, dvc):
         to_remote=False,
         remote_jobs=None,
         force=False,
+        relink=True,
     )
 
 
@@ -45,6 +46,7 @@ def test_add_to_remote(mocker, dvc):
         to_remote=True,
         remote_jobs=None,
         force=False,
+        relink=True,
     )
 
 

tests/unit/command/test_dataset.py

@@ -0,0 +1,137 @@
+from datetime import datetime, timezone
+
+import pytest
+
+from dvc.cli import main
+
+
+@pytest.mark.parametrize(
+    "spec,lock,expected_output",
+    [
+        (
+            {"name": "ds", "url": "url", "type": "dvc", "path": "path", "rev": "main"},
+            {"rev_lock": "0" * 40},
+            "Adding ds (url:/path @ main)\n",
+        ),
+        (
+            {"name": "mydataset", "url": "dvcx://dataset", "type": "dvcx"},
+            {"version": 1, "created_at": datetime.now(tz=timezone.utc)},
+            "Adding mydataset (dvcx://dataset @ v1)\n",
+        ),
+        (
+            {"name": "mydataset", "url": "s3://bucket/path", "type": "url"},
+            {
+                "files": [{"relpath": "foo", "meta": {"version_id": 1}}],
+                "meta": {"isdir": True},
+            },
+            "Adding mydataset (s3://bucket/path)\n",
+        ),
+    ],
+)
+def test_add(dvc, capsys, mocker, spec, lock, expected_output):
+    dataset = dvc.datasets._build_dataset("dvc.yaml", spec, spec | lock)
+
+    m = mocker.patch("dvc.repo.datasets.Datasets.add", return_value=dataset)
+
+    assert main(["dataset", "add", spec["name"], f"--{spec['type']}", spec["url"]]) == 0
+    out, err = capsys.readouterr()
+    assert out == expected_output
+    assert not err
+    m.assert_called_once()
+
+
+def test_add_already_exists(dvc, caplog, mocker):
+    spec = {"name": "ds", "url": "url", "type": "dvc"}
+    dataset = dvc.datasets._build_dataset("dvc.yaml", spec, None)
+    mocker.patch("dvc.repo.datasets.Datasets.get", return_value=dataset)
+
+    assert main(["dataset", "add", "ds", "--dvcx", "dataset"]) == 255
+    assert "ds already exists in dvc.yaml, use the --force to overwrite" in caplog.text
+
+
+@pytest.mark.parametrize("lock", ["missing", "unchanged", "updated"])
+@pytest.mark.parametrize(
+    "spec,old_lock,new_lock,expected_outputs",
+    [
+        (
+            {"name": "ds", "url": "url", "type": "dvc", "path": "path", "rev": "main"},
+            {"rev_lock": "0" * 40},
+            {"rev_lock": "1" * 40},
+            {
+                "missing": "Updating ds (url:/path @ main)\n",
+                "unchanged": "Nothing to update\n",
+                "updated": "Updating ds (000000000 -> 111111111)\n",
+            },
+        ),
+        (
+            {"name": "mydataset", "url": "dvcx://dataset", "type": "dvcx"},
+            {"version": 1, "created_at": datetime.now(tz=timezone.utc)},
+            {"version": 2},
+            {
+                "missing": "Updating mydataset (dvcx://dataset @ v2)\n",
+                "unchanged": "Nothing to update\n",
+                "updated": "Updating mydataset (v1 -> v2)\n",
+            },
+        ),
+        (
+            {"name": "mydataset", "url": "dvcx://dataset", "type": "dvcx"},
+            {"version": 2, "created_at": datetime.now(tz=timezone.utc)},
+            {"version": 1},
+            {
+                "missing": "Updating mydataset (dvcx://dataset @ v1)\n",
+                "unchanged": "Nothing to update\n",
+                "updated": "Downgrading mydataset (v2 -> v1)\n",
+            },
+        ),
+        (
+            {"name": "mydataset", "url": "s3://bucket/path", "type": "url"},
+            {
+                "files": [
+                    {"relpath": "bar", "meta": {"version_id": 2}},
+                    {"relpath": "baz", "meta": {"version_id": 3}},
+                    {"relpath": "foo", "meta": {"version_id": 1}},
+                ],
+                "meta": {"isdir": True},
+            },
+            {
+                "files": [
+                    {"relpath": "bar", "meta": {"version_id": 2}},  # unchanged
+                    {"relpath": "baz", "meta": {"version_id": 4}},  # modified
+                    # `foo` deleted
+                    {"relpath": "foobar", "meta": {"version_id": 5}},  # new
+                ],
+                "meta": {"isdir": True},
+            },
+            {
+                "missing": "Updating mydataset (s3://bucket/path)\n",
+                "unchanged": "Nothing to update\n",
+                "updated": (
+                    "Updating mydataset (s3://bucket/path)\n"
+                    "M\tbaz\n"
+                    "A\tfoobar\n"
+                    "D\tfoo\n"
+                ).expandtabs(8),
+            },
+        ),
+    ],
+)
+def test_update(dvc, capsys, mocker, spec, old_lock, new_lock, expected_outputs, lock):
+    if lock == "missing":
+        new_lock = spec | old_lock | new_lock
+        old_lock = None
+    elif lock == "unchanged":
+        old_lock = new_lock = spec | old_lock
+    else:
+        old_lock = spec | old_lock
+        new_lock = old_lock | new_lock
+
+    old = dvc.datasets._build_dataset("dvc.yaml", spec, old_lock)
+    new = dvc.datasets._build_dataset("dvc.yaml", spec, new_lock)
+
+    m = mocker.patch("dvc.repo.datasets.Datasets.update", return_value=(old, new))
+    assert main(["dataset", "update", spec["name"]]) == 0
+    out, err = capsys.readouterr()
+    assert out == expected_outputs[lock]
+    assert not err
+
+    m.assert_called_once()

tests/unit/command/test_get.py

@@ -44,7 +44,21 @@ def test_get(mocker):
 
 
 def test_get_url(mocker, capsys):
-    cli_args = parse_args(["get", "repo_url", "src", "--rev", "version", "--show-url"])
+    cli_args = parse_args(
+        [
+            "get",
+            "repo_url",
+            "src",
+            "--rev",
+            "version",
+            "--remote",
+            "myremote",
+            "--show-url",
+            "--remote-config",
+            "k1=v1",
+            "k2=v2",
+        ]
+    )
     assert cli_args.func == CmdGet
 
     cmd = cli_args.func(cli_args)
@@ -54,4 +68,10 @@ def test_get_url(mocker, capsys):
     out, _ = capsys.readouterr()
     assert "resource_url" in out
 
-    m.assert_called_once_with("src", repo="repo_url", rev="version")
+    m.assert_called_once_with(
+        "src",
+        repo="repo_url",
+        rev="version",
+        remote="myremote",
+        remote_config={"k1": "v1", "k2": "v2"},
+    )

tests/unit/command/test_help.py

@@ -29,7 +29,7 @@ def command_tuples():
 
     # the no. of commands will usually go up,
     # but if we ever remove commands and drop below, adjust the magic number accordingly
-    assert len(commands) >= 112
+    assert len(commands) >= 116
     return sorted(commands)
 
 

tests/unit/command/test_queue.py

@@ -166,13 +166,13 @@ def test_experiments_status(dvc, scm, mocker, capsys):
         {
             "rev": "c61a525a4ff39007301b4516fb6e54b323a0587b",
             "name": "I40",
-            "timestamp": datetime(2022, 6, 9, 20, 49, 48),
+            "timestamp": datetime(2022, 6, 9, 20, 49, 48),  # noqa: DTZ001
             "status": "Queued",
         },
         {
             "rev": "8da9c339da30636261a3491a90aafdb760a4168f",
             "name": "I60",
-            "timestamp": datetime(2022, 6, 9, 20, 49, 43),
+            "timestamp": datetime(2022, 6, 9, 20, 49, 43),  # noqa: DTZ001
             "status": "Running",
         },
     ]

tests/unit/command/test_studio.py

@@ -1,7 +1,16 @@
-from dvc_studio_client.auth import AuthorizationExpiredError
+import pytest
 
+from dvc import env
 from dvc.cli import main
 from dvc.utils.studio import STUDIO_URL
+from dvc_studio_client.auth import AuthorizationExpiredError
+
+
+@pytest.fixture(autouse=True)
+def global_config_dir(monkeypatch, tmp_dir_factory):
+    monkeypatch.setenv(
+        env.DVC_GLOBAL_CONFIG_DIR, str(tmp_dir_factory.mktemp("studio-login"))
+    )
 
 
 def test_studio_login_token_check_failed(mocker):

tests/unit/fs/test_fs.py

@@ -1,10 +1,10 @@
 import pytest
-from dvc_http import HTTPFileSystem, HTTPSFileSystem
-from dvc_s3 import S3FileSystem
-from dvc_ssh import SSHFileSystem
 
 from dvc.config import RemoteNotFoundError
 from dvc.fs import LocalFileSystem, get_cloud_fs, get_fs_cls, get_fs_config
+from dvc_http import HTTPFileSystem, HTTPSFileSystem
+from dvc_s3 import S3FileSystem
+from dvc_ssh import SSHFileSystem
 
 url_cls_pairs = [
     ("s3://bucket/path", S3FileSystem),

tests/unit/output/test_load.py

@@ -1,10 +1,10 @@
 import pytest
-from dvc_s3 import S3FileSystem
 
 from dvc import output
 from dvc.fs import LocalFileSystem
 from dvc.output import Output
 from dvc.stage import Stage
+from dvc_s3 import S3FileSystem
 
 
 @pytest.mark.parametrize(

tests/unit/remote/test_remote.py

@@ -1,8 +1,8 @@
 import pytest
-from dvc_gs import GSFileSystem
-from dvc_s3 import S3FileSystem
 
 from dvc.fs import get_cloud_fs
+from dvc_gs import GSFileSystem
+from dvc_s3 import S3FileSystem
 
 
 def test_remote_with_hash_jobs(dvc):