[pre-commit.ci] pre-commit autoupdate (#10836)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/ISSUE_TEMPLATE/epic_story.md

@@ -1,60 +0,0 @@
----
-name: Epic/Story Issue Template
-about: Template for "top" level issues - Epics (>2 weeks) / Stories (<2 weeks)
-title: 'Epic: New Feature'
-labels: epic
-assignees:
-
----
-
-## Summary / Background
-What do you want to achieve, why? business context
-...
-
-## Scope
-
-What will be impacted and what won't be?
-What needs implementation and what is invariant?
-e.g.
-- user should be able to run workflow X from UI
-- enable workflow Y, Z from CLI
-
-## Assumptions
-Product / UX assumptions as well as technical assumptions / limitations
-e.g.
-* Support only Python Runtime
-* Focus on DVC experiments only
-* Deployment environments don't change often and can be picked up from shared configuration
-
-## Open Questions
-e.g.
-- How should access control work for shared artifacts (workflow X)
-- Python runtime assumption - is it really valid? in light of <...>
-
-## Blockers / Dependencies
-List issues or other conditions / blockers
-
-## General Approach
-Invocation example:
-```shell
-$ mapper-run task.tar.gz --ray-cluster <ip>:<port>
-```
-
-## Steps
-
-### Must have (p1)
-- [ ] subissue2
-- [ ] step 2
- - info
- - info
-
-### Optional / followup (p2)
-- [ ] ⌛ step 3 wip
-- [ ] step 4
-
-## Timelines
-
-Put your estimations here. Update once certainty changes
-- end of week (Feb 3) for prototype with workflows X, Y
-- Feb 15 - MVP in production
-- Low priority followups can be done later

.github/mergify.yml

@@ -1,10 +0,0 @@
-pull_request_rules:
-  - name: backport patches to 2.x
-    conditions:
-      - base=main
-      - label=backport
-    actions:
-      backport:
-        branches:
-          - "2.x"
-        title: ({{ destination_branch }}) {{ title }}

.github/workflows/benchmarks.yaml

@@ -5,6 +5,10 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+env:
+  FORCE_COLOR: "1"
+  PY_COLORS: "1"
+
 jobs:
   bench:
     name: run benchmarks
@@ -12,7 +16,29 @@ jobs:
     steps:
       - uses: actions/setup-python@v5
         with:
-            python-version: "3.11"
-      - uses: iterative/dvc-bench@main
+            python-version: "3.13"
+
+      - uses: actions/checkout@v4
         with:
-            pytest_options: "-k 'test_init or test_help'"
+          ref: ${{ github.event.pull_request.base.sha }}
+          fetch-depth: 0
+
+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: pyproject.toml
+
+      - run: uv pip install '.[tests]' --system
+      - name: run benchmarks on base branch
+        run: pytest --benchmark-autosave dvc/testing/benchmarks/ -k 'test_init or test_help'
+
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          clean: false
+      - run: uv pip install '.[tests]' --system
+      - name: run benchmarks for PR
+        run: >
+          pytest --benchmark-compare --benchmark-compare-fail=min:5%
+          --benchmark-group-by name
+          dvc/testing/benchmarks/ -k 'test_init or test_help'

.github/workflows/build.yaml

@@ -25,24 +25,27 @@ jobs:
 
     - uses: actions/setup-python@v5
       with:
-        python-version: 3.12
+        python-version: 3.13
 
-    - run: python -m pip install --upgrade pip setuptools_scm build twine
+    - uses: astral-sh/setup-uv@v6
 
-    - name: Force version for Test PyPI upload
-      if: ${{ !startsWith(github.ref, 'refs/tags') }}
-      run: echo version=$(python -m setuptools_scm | awk -F+ '{print $1}' | tail -1) >> $GITHUB_ENV
+    - name: Omit local version for Test PyPI upload
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+      run: echo SETUPTOOLS_SCM_OVERRIDES_FOR_DVC='{local_scheme="no-local-version"}' >> $GITHUB_ENV
 
-    - run: echo 'PKG = "pip"'>dvc/_build.py
-    - run: python -m build
-      env:
-        SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DVC: ${{ env.version }}
-    - run: twine check --strict dist/*
+    - name: Build Python Package
+      run: |
+        echo 'PKG = "pip"'>dvc/_build.py
+        uv build
+
+    - name: Check dist
+      run: uv tool run twine check --strict dist/*
 
     - uses: actions/upload-artifact@v4
       with:
         name: Packages
         path: dist/
+        if-no-files-found: error
 
   test-pypi-publish:
     name: Publish dev package to test.pypi.org
@@ -52,13 +55,13 @@ jobs:
 
     environment:
       name: test-pypi
-      url: https://test.pypi.org/p/dvc
+      url: https://test.pypi.org/p/dvc/${{ github.event.release.tag_name }}
 
     permissions:
       id-token: write
 
     steps:
-    - uses: actions/download-artifact@v4
+    - uses: actions/download-artifact@v5
       with:
         name: Packages
         path: dist
@@ -77,13 +80,13 @@ jobs:
 
     environment:
       name: pypi
-      url: https://pypi.org/p/dvc
+      url: https://pypi.org/p/dvc/${{ github.event.release.tag_name }}
 
     permissions:
       id-token: write
 
     steps:
-    - uses: actions/download-artifact@v4
+    - uses: actions/download-artifact@v5
       with:
         name: Packages
         path: dist

.github/workflows/codeql.yml

@@ -2,7 +2,7 @@ name: "CodeQL"
 
 on:
   push:
-    branches: ["main", "2.x"]
+    branches: ["main"]
   pull_request:
     branches: ["main"]
   schedule:

.github/workflows/plugin_tests.yaml

@@ -18,8 +18,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04]
-        pyv: ["3.10"]
+        os: [ubuntu-latest]
+        pyv: ["3.13"]
         plugin: ["dvc-s3"]
 
     steps:
@@ -37,16 +37,17 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.pyv }}
-        cache: pip
-        cache-dependency-path: |
-          dvc/pyproject.toml
-          ${{ matrix.plugin }}/setup.cfg
+
+    - uses: astral-sh/setup-uv@v6
+      with:
+        enable-cache: true
+        cache-dependency-glob: "**/pyproject.toml"
+        cache-suffix: ${{ matrix.pyv }}
 
     - name: Install plugin + DVC@PR
       run: |
-        pip install --upgrade pip wheel
-        pip install "./dvc[testing]"
-        pip install -e "./${{ matrix.plugin }}[tests]"
+        uv pip install "./dvc[testing]" --system
+        uv pip install -e "./${{ matrix.plugin }}[tests]" --system
 
     - name: Run plugin tests
       timeout-minutes: 15

.github/workflows/tests.yaml

@@ -2,7 +2,7 @@ name: Tests
 
 on:
   push:
-    branches: [main, 2.x]
+    branches: [main]
   pull_request:
   schedule:
     - cron: '5 1 * * *'  # every day at 01:05
@@ -16,15 +16,107 @@ permissions:
   contents: read
 
 jobs:
+  lint:
+    timeout-minutes: 10
+    runs-on: ${{ matrix.os }}
+    env:
+       DVC_NO_ANALYTICS: true
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.13'
+    - uses: astral-sh/setup-uv@v6
+      with:
+        enable-cache: true
+        cache-dependency-glob: pyproject.toml
+    - run: uv pip install -e ".[dev]" --system
+      env:
+        UV_LINK_MODE: ${{ runner.os == 'Windows' && 'symlink' || 'hardlink' }}
+
+    - name: Cache mypy
+      uses: actions/cache@v4
+      with:
+        path: .mypy_cache
+        key: mypy-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}
+    - uses: pre-commit/[email protected]
+
   tests:
-    timeout-minutes: 50
+    timeout-minutes: 20
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, windows-latest, macos-latest]
-        pyv: ["3.9", "3.10", "3.11", "3.12"]
-
+        os: [ubuntu-latest, macos-latest]
+        pyv: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        include:
+        - os: windows-latest
+          pyv: "3.9"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
+        - os: windows-latest
+          pyv: "3.9"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
+        - os: windows-latest
+          pyv: "3.9"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
+        - os: windows-latest
+          pyv: "3.9"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
+        - os: windows-latest
+          pyv: "3.10"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
+        - os: windows-latest
+          pyv: "3.10"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
+        - os: windows-latest
+          pyv: "3.10"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
+        - os: windows-latest
+          pyv: "3.10"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
+        - os: windows-latest
+          pyv: "3.11"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
+        - os: windows-latest
+          pyv: "3.11"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
+        - os: windows-latest
+          pyv: "3.11"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
+        - os: windows-latest
+          pyv: "3.11"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
+        - os: windows-latest
+          pyv: "3.12"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
+        - os: windows-latest
+          pyv: "3.12"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
+        - os: windows-latest
+          pyv: "3.12"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
+        - os: windows-latest
+          pyv: "3.12"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
+        - os: windows-latest
+          pyv: "3.13"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
+        - os: windows-latest
+          pyv: "3.13"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
+        - os: windows-latest
+          pyv: "3.13"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
+        - os: windows-latest
+          pyv: "3.13"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
     steps:
     - uses: actions/checkout@v4
       with:
@@ -33,25 +125,25 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.pyv }}
-        cache: pip
-        cache-dependency-path: pyproject.toml
-    - name: install
-      run: |
-        pip install --upgrade pip wheel
-        pip install -e ".[dev]"
-    - uses: pre-commit/[email protected]
+    - uses: astral-sh/setup-uv@v6
+      with:
+        enable-cache: true
+        cache-dependency-glob: pyproject.toml
+        cache-suffix: ${{ matrix.pyv }}
+    - run: uv pip install -e ".[dev]" pytest-split --system
+      env:
+        UV_LINK_MODE: ${{ runner.os == 'Windows' && 'symlink' || 'hardlink' }}
+
     - name: run tests
       timeout-minutes: 40
       env:
+        COVERAGE_CORE: sysmon
         PYTHONUTF8: 1
       run: >
-        pytest -n=logical --timeout=300 --durations=100
-        --cov --cov-report=xml --cov-report=term
+        pytest ${{ matrix.pytestargs }} -n=logical --dist=worksteal --timeout=300 --durations=0
+        --cov --cov-report=xml --cov-report=term  --durations-path=./.github/.test_durations
     - name: upload coverage report
-      uses: codecov/codecov-action@v3
-      with:
-        file: ./coverage.xml
-        fail_ci_if_error: false
+      uses: codecov/codecov-action@v5
   check:
     if: always()
     needs: [tests]
@@ -66,7 +158,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Slack Notification
-      uses: rtCamp/action-slack-notify@v2.2.1
+      uses: rtCamp/action-slack-notify@v2.3.3
       env:
         SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         SLACK_COLOR: ${{ job.status }}

.gitignore

@@ -43,7 +43,7 @@ scripts/ci/gcp-creds.json
 .gcp-creds.json
 
 *~
-/dvc/_dvc_version.py
+/dvc/_version.py
 
 .mypy_cache/
 .pytest_cache/

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ ci:
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v6.0.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -22,13 +22,13 @@ repos:
       - id: sort-simple-yaml
       - id: trailing-whitespace
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.2.0'
+    rev: 'v0.12.8'
     hooks:
-      - id: ruff
+      - id: ruff-check
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.6
+    rev: v2.4.1
     hooks:
       - id: codespell
         additional_dependencies: ["tomli"]
@@ -50,7 +50,7 @@ repos:
         language: system
         name: DVC pre-commit
         stages:
-          - commit
+          - pre-commit
         verbose: true
         require_serial: true
       - args:
@@ -61,7 +61,7 @@ repos:
         language: system
         name: DVC pre-push
         stages:
-          - push
+          - pre-push
         require_serial: true
       - always_run: true
         args:

.pre-commit-hooks.yaml

@@ -8,7 +8,7 @@
   name: DVC pre-commit
   require_serial: true
   stages:
-  - commit
+  - pre-commit
   verbose: true
 - args:
   - git-hook
@@ -20,7 +20,7 @@
   name: DVC pre-push
   require_serial: true
   stages:
-  - push
+  - pre-push
 - always_run: true
   args:
   - git-hook
@@ -29,7 +29,7 @@
   id: dvc-post-checkout
   language: python
   language_version: python3
-  minimum_pre_commit_version: 2.2.0
+  minimum_pre_commit_version: 3.2.0
   name: DVC post-checkout
   require_serial: true
   stages:

CITATION.cff

@@ -0,0 +1,25 @@
+cff-version: 1.2.0
+title: 'DVC: Data Version Control - Git for Data & Models"'
+message: >-
+  If you use this software as part of a publication and wish
+  to cite it, please use the metadata from this file.
+type: software
+authors:
+  - name: The DVC team and contributors
+    website: 'https://github.com/iterative'
+repository-code: 'https://github.com/iterative/dvc'
+url: 'https://dvc.org'
+abstract: >-
+  DVC is a tool for data science that takes advantage of existing software engineering toolset.
+  It helps machine learning teams manage large datasets, make projects reproducible, and collaborate better.
+keywords:
+  - ai
+  - collaboration
+  - data-science
+  - data-version-control
+  - developer-tools
+  - git
+  - machine-learning
+  - python
+  - reproducibility
+license: Apache-2.0

README.rst

@@ -213,7 +213,7 @@ Community and Support
 * `Forum <https://discuss.dvc.org/>`_
 * `Discord Chat <https://dvc.org/chat>`_
 * `Email <mailto:[email protected]>`_
-* `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
+* `Mailing List <https://dvc.org/community#subscribe>`_
 
 Copyright
 =========
@@ -240,11 +240,11 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
 .. |VS Code Extension Overview| image:: https://raw.githubusercontent.com/iterative/vscode-dvc/main/extension/docs/overview.gif
    :alt: DVC Extension for VS Code
 
-.. |CI| image:: https://github.com/iterative/dvc/workflows/Tests/badge.svg?branch=main
-   :target: https://github.com/iterative/dvc/actions
+.. |CI| image:: https://github.com/iterative/dvc/actions/workflows/tests.yaml/badge.svg
+   :target: https://github.com/iterative/dvc/actions/workflows/tests.yaml
    :alt: GHA Tests
 
-.. |Maintainability| image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
+.. |Maintainability| image:: https://img.shields.io/codeclimate/maintainability/iterative/dvc
    :target: https://codeclimate.com/github/iterative/dvc
    :alt: Code Climate
 
@@ -268,7 +268,7 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
    :target: https://formulae.brew.sh/formula/dvc
    :alt: Homebrew
 
-.. |Conda| image:: https://img.shields.io/conda/v/conda-forge/dvc.svg?label=conda&logo=conda-forge
+.. |Conda| image:: https://anaconda.org/conda-forge/dvc/badges/version.svg
    :target: https://anaconda.org/conda-forge/dvc
    :alt: Conda-forge
 

dvc/__init__.py

@@ -3,6 +3,7 @@ DVC
 ----
 Make your data science projects reproducible and shareable.
 """
+
 import dvc.logger
 from dvc.build import PKG  # noqa: F401
 from dvc.version import __version__, version_tuple  # noqa: F401

dvc/__main__.py

@@ -1,4 +1,5 @@
 """Main entry point for DVC command line tool."""
+
 import sys
 
 from dvc.cli import main

dvc/_debug.py

@@ -162,7 +162,7 @@ def _get_path_func(tool: str, ext: str):
     fmt = f"{tool}.dvc-{{now:%Y%m%d}}_{{now:%H%M%S}}.{ext}"
 
     def func(now: Optional["datetime"] = None) -> str:
-        return fmt.format(now=now or datetime.now())
+        return fmt.format(now=now or datetime.now())  # noqa: DTZ005
 
     return func
 

dvc/analytics.py

@@ -1,10 +1,14 @@
 import json
 import os
+from typing import TYPE_CHECKING, Optional
 
 from dvc.log import logger
 
 from .env import DVC_ANALYTICS_ENDPOINT, DVC_NO_ANALYTICS
 
+if TYPE_CHECKING:
+    from dvc.scm import Base
+
 logger = logger.getChild(__name__)
 
 
@@ -89,20 +93,62 @@ def send(path):
     os.remove(path)
 
 
-def _scm_in_use():
-    from dvc.exceptions import NotDvcRepoError
-    from dvc.repo import Repo
-    from dvc.scm import NoSCM
+def _git_remote_url(scm: Optional["Base"]) -> Optional[str]:
+    from dvc.scm import Git
 
-    from .scm import SCM, SCMError
+    if not isinstance(scm, Git):
+        return None
+
+    from dulwich.porcelain import get_remote_repo
+
+    dulwich_repo = scm.dulwich.repo
+    try:
+        _remote, url = get_remote_repo(dulwich_repo)
+    except IndexError:
+        # IndexError happens when the head is detached
+        _remote, url = get_remote_repo(dulwich_repo, b"origin")
+    # Dulwich returns (None, "origin") if no remote set
+    if (_remote, url) == (None, "origin"):
+        return None
+    return url
+
+
+def _scm_in_use(scm: Optional["Base"]) -> Optional[str]:
+    return type(scm).__name__ if scm else None
+
+
+def _parse_git_remote_path(remote_url: str) -> str:
+    from urllib.parse import urlparse
+
+    from scmrepo.urls import is_scp_style_url
+
+    parsed = urlparse(remote_url)
+    # Windows Path also gets parsed with a drive letter as scheme
+    # https://github.com/python/cpython/issues/86381
+    if parsed.scheme and parsed.scheme in ("http", "https", "git", "ssh"):
+        return parsed.path.strip("/")
+
+    if is_scp_style_url(remote_url):
+        # handle scp-style URL
+        parts = remote_url.split(":", 1)
+        if len(parts) == 2:
+            _, path = parts
+            return path.rstrip("/")
+    return remote_url
+
+
+def _git_remote_path_hash(scm: Optional["Base"]) -> Optional[str]:
+    """Return a hash of the git remote path."""
+    import hashlib
 
     try:
-        scm = SCM(root_dir=Repo.find_root())
-        return type(scm).__name__
-    except SCMError:
-        return NoSCM.__name__
-    except NotDvcRepoError:
-        pass
+        if remote_url := _git_remote_url(scm):
+            path = _parse_git_remote_path(remote_url)
+            h = hashlib.md5(path.encode("utf-8"), usedforsecurity=False)  # for FIPS
+            return h.hexdigest()
+    except Exception:  # noqa: BLE001
+        logger.debug("Failed to get git remote path", exc_info=True)
+    return None
 
 
 def _runtime_info():
@@ -112,6 +158,8 @@ def _runtime_info():
     from iterative_telemetry import _generate_ci_id, find_or_create_user_id
 
     from dvc import __version__
+    from dvc.info import _get_remotes
+    from dvc.repo import Repo
     from dvc.utils import is_binary
 
     ci_id = _generate_ci_id()
@@ -120,13 +168,24 @@ def _runtime_info():
     else:
         group_id, user_id = None, find_or_create_user_id()
 
+    scm = None
+    remotes = None
+    try:
+        repo = Repo()
+        scm = repo.scm
+        remotes = _get_remotes(repo.config)
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("failed to open repo: %s", exc)
+
     return {
         "dvc_version": __version__,
         "is_binary": is_binary(),
-        "scm_class": _scm_in_use(),
+        "scm_class": _scm_in_use(scm),
         "system_info": _system_info(),
         "user_id": user_id,
         "group_id": group_id,
+        "remotes": remotes,
+        "git_remote_hash": _git_remote_path_hash(scm),
     }
 
 

dvc/api/__init__.py

@@ -1,7 +1,8 @@
 from dvc.fs.dvc import _DVCFileSystem as DVCFileSystem
 
 from .artifacts import artifacts_show
-from .data import get_url, open, read
+from .data import get_url, open, read  # noqa: A004
+from .dataset import get as get_dataset
 from .experiments import exp_save, exp_show
 from .scm import all_branches, all_commits, all_tags
 from .show import metrics_show, params_show
@@ -14,6 +15,7 @@ __all__ = [
     "artifacts_show",
     "exp_save",
     "exp_show",
+    "get_dataset",
     "get_url",
     "metrics_show",
     "open",

dvc/api/artifacts.py

@@ -1,3 +1,4 @@
+import os
 from typing import Any, Optional
 
 from dvc.repo import Repo
@@ -36,12 +37,22 @@ def artifacts_show(
     if version and stage:
         raise ValueError("Artifact version and stage are mutually exclusive.")
 
+    from dvc.repo.artifacts import Artifacts
+    from dvc.utils import as_posix
+
     repo_kwargs: dict[str, Any] = {
         "subrepos": True,
         "uninitialized": True,
     }
+
+    dirname, _ = Artifacts.parse_path(name)
     with Repo.open(repo, **repo_kwargs) as _repo:
         rev = _repo.artifacts.get_rev(name, version=version, stage=stage)
         with _repo.switch(rev):
-            path = _repo.artifacts.get_path(name)
-        return {"rev": rev, "path": path}
+            root = _repo.fs.root_marker
+            _dirname = _repo.fs.join(root, dirname) if dirname else root
+            with Repo(_dirname, fs=_repo.fs, scm=_repo.scm) as r:
+                path = r.artifacts.get_path(name)
+                path = _repo.fs.join(_repo.fs.root_marker, as_posix(path))
+                parts = _repo.fs.relparts(path, _repo.root_dir)
+                return {"rev": rev, "path": os.path.join(*parts)}

dvc/api/data.py

@@ -25,7 +25,14 @@ def _wrap_exceptions(repo, url):
         raise PathMissingError(exc.path, url) from exc
 
 
-def get_url(path, repo=None, rev=None, remote=None):
+def get_url(
+    path: str,
+    repo: Optional[str] = None,
+    rev: Optional[str] = None,
+    remote: Optional[str] = None,
+    config: Optional[dict[str, Any]] = None,
+    remote_config: Optional[dict[str, Any]] = None,
+):
     """
     Returns the URL to the storage location of a data file or directory tracked
     in a DVC repo. For Git repos, HEAD is used unless a rev argument is
@@ -35,15 +42,44 @@ def get_url(path, repo=None, rev=None, remote=None):
 
     NOTE: This function does not check for the actual existence of the file or
     directory in the remote storage.
+
+    Args:
+        path (str): location and file name of the target, relative to the root
+            of `repo`.
+        repo (str, optional): location of the DVC project or Git Repo.
+            Defaults to the current DVC project (found by walking up from the
+            current working directory tree).
+            It can be a URL or a file system path.
+            Both HTTP and SSH protocols are supported for online Git repos
+            (e.g. [user@]server:project.git).
+        rev (str, optional): Any `Git revision`_ such as a branch or tag name,
+            a commit hash or a dvc experiment name.
+            Defaults to HEAD.
+            If `repo` is not a Git repo, this option is ignored.
+        remote (str, optional): Name of the `DVC remote`_ used to form the
+            returned URL string.
+            Defaults to the `default remote`_ of `repo`.
+            For local projects, the cache is tried before the default remote.
+        config(dict, optional): config to be passed to the DVC repository.
+            Defaults to None.
+        remote_config(dict, optional): remote config to be passed to the DVC
+            repository.
+            Defaults to None.
+
+    Returns:
+        str: URL to the file or directory.
     """
     from dvc.config import NoRemoteError
     from dvc_data.index import StorageKeyError
 
-    repo_kwargs: dict[str, Any] = {}
-    if remote:
-        repo_kwargs["config"] = {"core": {"remote": remote}}
     with Repo.open(
-        repo, rev=rev, subrepos=True, uninitialized=True, **repo_kwargs
+        repo,
+        rev=rev,
+        subrepos=True,
+        uninitialized=True,
+        remote=remote,
+        config=config,
+        remote_config=remote_config,
     ) as _repo:
         index, entry = _repo.get_data_index_entry(path)
         with reraise(
@@ -249,7 +285,7 @@ def _open(
             if TYPE_CHECKING:
                 from dvc.fs import FileSystem
 
-            fs: Union["FileSystem", DataFileSystem, DVCFileSystem]
+            fs: Union[FileSystem, DataFileSystem, DVCFileSystem]
             if os.path.isabs(path):
                 fs = DataFileSystem(index=_repo.index.data["local"])
                 fs_path = path

dvc/api/dataset.py

@@ -0,0 +1,66 @@
+from typing import Literal, TypedDict, Union
+
+
+class DatachainDataset(TypedDict):
+    type: Literal["dc"]
+    name: str
+    version: int
+
+
+class DVCDataset(TypedDict):
+    type: Literal["dvc"]
+    url: str
+    path: str
+    sha: str
+
+
+class URLDataset(TypedDict):
+    type: Literal["url"]
+    files: list[str]
+    path: str
+
+
+def get(name: str) -> Union[DatachainDataset, DVCDataset, URLDataset]:
+    from difflib import get_close_matches
+
+    from dvc.fs import get_cloud_fs
+    from dvc.repo import Repo, datasets
+
+    repo = Repo()
+    try:
+        dataset = repo.datasets[name]
+    except datasets.DatasetNotFoundError as e:
+        add_note = getattr(e, "add_note", lambda _: None)
+        if matches := get_close_matches(name, repo.datasets):
+            add_note(f"Did you mean: {matches[0]!r}?")
+        raise
+
+    if dataset._invalidated:
+        raise ValueError(f"dataset not in sync. Sync with 'dvc ds update {name}'.")
+    if not dataset.lock:
+        raise ValueError("missing lock information")
+    if dataset.type == "dvc":
+        return DVCDataset(
+            type="dvc",
+            url=dataset.lock.url,
+            path=dataset.lock.path,
+            sha=dataset.lock.rev_lock,
+        )
+    if dataset.type == "dc":
+        return DatachainDataset(
+            type="dc", name=dataset.name_version[0], version=dataset.lock.version
+        )
+    if dataset.type == "url":
+        fs_cls, _, path = get_cloud_fs(repo.config, url=dataset.lock.url)
+        assert fs_cls
+        join_version = getattr(fs_cls, "join_version", lambda path, _: path)
+        protocol = fs_cls.protocol
+        versioned_path = join_version(path, dataset.lock.meta.version_id)
+        versioned_path = f"{protocol}://{versioned_path}"
+        files = [
+            join_version(
+                fs_cls.join(versioned_path, file.relpath), file.meta.version_id
+            )
+            for file in dataset.lock.files
+        ]
+        return URLDataset(type="url", files=files, path=versioned_path)

dvc/api/show.py

@@ -27,8 +27,7 @@ def _postprocess(results):
             }
             processed[rev] = processed[rev] | to_merge
 
-    if "workspace" in processed:
-        del processed["workspace"]
+    processed.pop("workspace", None)
 
     return processed
 

dvc/cachemgr.py

@@ -86,7 +86,7 @@ class CacheManager:
         return FileStorage(
             key=(),
             fs=self.local.fs,
-            path=self.local.fs.join(self.default_local_cache_dir, self.FS_DIR),
+            path=self.local.fs.join(self.local_cache_dir, self.FS_DIR),
         )
 
     def _init_odb(self, schemes):

dvc/cli/__init__.py

@@ -157,6 +157,8 @@ def main(argv=None):  # noqa: C901, PLR0912, PLR0915
     Returns:
         int: command's return code.
     """
+    from contextlib import ExitStack
+
     from dvc._debug import debugtools
     from dvc.config import ConfigError
     from dvc.exceptions import DvcException, NotDvcRepoError
@@ -173,12 +175,11 @@ def main(argv=None):  # noqa: C901, PLR0912, PLR0915
         logging.disable(logging.INFO)
 
     args = None
-
-    outer_log_level = logger.level
-    level = None
+    stack = ExitStack()
     try:
         args = parse_args(argv)
 
+        level = None
         if args.quiet:
             level = logging.CRITICAL
         elif args.verbose == 1:
@@ -187,7 +188,7 @@ def main(argv=None):  # noqa: C901, PLR0912, PLR0915
             level = logging.TRACE  # type: ignore[attr-defined]
 
         if level is not None:
-            set_loggers_level(level)
+            stack.enter_context(set_loggers_level(level))
 
         if level and level <= logging.DEBUG:
             from platform import platform, python_implementation, python_version
@@ -247,7 +248,7 @@ def main(argv=None):  # noqa: C901, PLR0912, PLR0915
 
         return ret
     finally:
-        logger.setLevel(outer_log_level)
+        stack.close()
 
         from dvc.repo.open_repo import clean_repos
 

dvc/cli/command.py

@@ -18,8 +18,11 @@ class CmdBase(ABC):
 
         os.chdir(args.cd)
 
-        self.repo: "Repo" = Repo(uninitialized=self.UNINITIALIZED)
-        self.config: "Config" = self.repo.config
+        self.repo: Repo = Repo(
+            uninitialized=self.UNINITIALIZED,
+            _wait_for_lock=args.wait_for_lock,
+        )
+        self.config: Config = self.repo.config
         self.args = args
 
     def do_run(self):

dvc/cli/parser.py

@@ -1,4 +1,5 @@
 """Main parser for the dvc cli."""
+
 import argparse
 import os
 from functools import lru_cache
@@ -17,6 +18,7 @@ from dvc.commands import (
     dag,
     data,
     data_sync,
+    dataset,
     destroy,
     diff,
     du,
@@ -67,6 +69,7 @@ COMMANDS = [
     dag,
     data,
     data_sync,
+    dataset,
     destroy,
     diff,
     du,
@@ -197,6 +200,14 @@ def get_main_parser():
         type=str,
     )
 
+    parser.add_argument(
+        "--wait-for-lock",
+        action="store_true",
+        default=False,
+        help="Wait for the lock if it is already held by another process, instead of"
+        " failing immediately.",
+    )
+
     # Sub commands
     subparsers = parser.add_subparsers(
         title="Available Commands",

dvc/cli/utils.py

@@ -40,7 +40,7 @@ def hide_subparsers_from_help(subparsers):
     # from the 'positional arguments' choices list
     # see: https://bugs.python.org/issue22848
     # Need to set `add_help=False`, but avoid setting `help`
-    # (not even to `argparse.SUPPPRESS`).
+    # (not even to `argparse.SUPPRESS`).
     # NOTE: The argument is the parent subparser, not the subcommand parser.
     cmds = [cmd for cmd, parser in subparsers.choices.items() if parser.add_help]
     subparsers.metavar = "{{{}}}".format(",".join(cmds))

dvc/commands/add.py

@@ -51,6 +51,7 @@ class CmdAdd(CmdBase):
                 to_remote=self.args.to_remote,
                 remote_jobs=self.args.remote_jobs,
                 force=self.args.force,
+                relink=self.args.relink,
             )
         except FileNotFoundError:
             logger.exception("")
@@ -119,6 +120,13 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Override local file or folder if exists.",
     )
+    parser.add_argument(
+        "--no-relink",
+        dest="relink",
+        action="store_false",
+        help="Don't recreate links from cache to workspace.",
+    )
+    parser.set_defaults(relink=True)
     parser.add_argument(
         "targets", nargs="+", help="Input files/directories to add."
     ).complete = completion.FILE

dvc/commands/artifacts.py

@@ -13,6 +13,9 @@ class CmdArtifactsGet(CmdBaseNoRepo):
         from dvc.scm import CloneError
         from dvc.ui import ui
 
+        if self.args.show_url:
+            return self._show_url()
+
         try:
             count, out = Artifacts.get(
                 self.args.url,
@@ -36,6 +39,28 @@ class CmdArtifactsGet(CmdBaseNoRepo):
             )
             return 1
 
+    def _show_url(self):
+        from dvc.api import artifacts_show, get_url
+        from dvc.ui import ui
+
+        artifact = artifacts_show(
+            self.args.name,
+            version=self.args.rev,
+            stage=self.args.stage,
+            repo=self.args.url,
+        )
+
+        url = get_url(
+            artifact["path"],
+            repo=self.args.url,
+            rev=artifact["rev"],
+            remote=self.args.remote,
+            remote_config=self.args.remote_config,
+        )
+        ui.write(url, force=True)
+
+        return 0
+
 
 def add_parser(subparsers, parent_parser):
     ARTIFACTS_HELP = "DVC model registry artifact commands."
@@ -84,6 +109,14 @@ def add_parser(subparsers, parent_parser):
         help="Destination path to download artifact to",
         metavar="<path>",
     ).complete = completion.DIR
+    get_parser.add_argument(
+        "--show-url",
+        action="store_true",
+        help=(
+            "Print the storage location (URL) the target data would be "
+            "downloaded from, and exit."
+        ),
+    )
     get_parser.add_argument(
         "-j",
         "--jobs",

dvc/commands/commit.py

@@ -20,6 +20,7 @@ class CmdCommit(CmdBase):
                     with_deps=self.args.with_deps,
                     recursive=self.args.recursive,
                     force=self.args.force,
+                    relink=self.args.relink,
                 )
             except DvcException:
                 logger.exception("failed to commit%s", (" " + target) if target else "")
@@ -64,6 +65,13 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Commit cache for subdirectories of the specified directory.",
     )
+    commit_parser.add_argument(
+        "--no-relink",
+        dest="relink",
+        action="store_false",
+        help="Don't recreate links from cache to workspace.",
+    )
+    commit_parser.set_defaults(relink=True)
     commit_parser.add_argument(
         "targets",
         nargs="*",

dvc/commands/dag.py

@@ -17,7 +17,7 @@ def _show_ascii(graph: "DiGraph"):
 
     ret = []
     for pipeline in pipelines:
-        ret.append(draw(pipeline.nodes, pipeline.edges))
+        ret.append(draw(pipeline.nodes, pipeline.edges))  # noqa: PERF401
 
     return "\n".join(ret)
 

dvc/commands/data.py

@@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, ClassVar
 
 from funcy import chunks, compact, log_durations
 
-from dvc.cli import formatter
+from dvc.cli import completion, formatter
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
 from dvc.log import logger
@@ -108,6 +108,7 @@ class CmdDataStatus(CmdBase):
     def run(self) -> int:
         with log_durations(logger.trace, "in data_status"):
             status = self.repo.data_status(
+                targets=self.args.targets,
                 granular=self.args.granular,
                 untracked_files=self.args.untracked_files,
                 not_in_remote=self.args.not_in_remote,
@@ -147,6 +148,14 @@ def add_parser(subparsers, parent_parser):
         formatter_class=formatter.RawDescriptionHelpFormatter,
         help=DATA_STATUS_HELP,
     )
+    data_status_parser.add_argument(
+        "targets",
+        nargs="*",
+        help=(
+            "Limit command scope to these tracked files/directories, "
+            ".dvc files and stage names."
+        ),
+    ).complete = completion.FILE  # type: ignore[attr-defined]
     data_status_parser.add_argument(
         "--json",
         action="store_true",

dvc/commands/data_sync.py

@@ -194,7 +194,7 @@ def add_parser(subparsers, _parent_parser):
     )
     pull_parser.add_argument(
         "--run-cache",
-        action="store_true",
+        action=argparse.BooleanOptionalAction,
         default=False,
         help="Fetch run history for all stages.",
     )
@@ -262,7 +262,7 @@ def add_parser(subparsers, _parent_parser):
     )
     push_parser.add_argument(
         "--run-cache",
-        action="store_true",
+        action=argparse.BooleanOptionalAction,
         default=False,
         help="Push run history for all stages.",
     )
@@ -324,7 +324,7 @@ def add_parser(subparsers, _parent_parser):
     )
     fetch_parser.add_argument(
         "--run-cache",
-        action="store_true",
+        action=argparse.BooleanOptionalAction,
         default=False,
         help="Fetch run history for all stages.",
     )

dvc/commands/dataset.py

@@ -0,0 +1,225 @@
+from typing import TYPE_CHECKING, Optional
+
+from dvc.cli import formatter
+from dvc.cli.command import CmdBase
+from dvc.cli.utils import append_doc_link
+from dvc.exceptions import DvcException
+from dvc.log import logger
+
+if TYPE_CHECKING:
+    from rich.text import Text
+
+    from dvc.repo.datasets import Dataset, FileInfo
+
+logger = logger.getChild(__name__)
+
+
+def diff_files(old: list["FileInfo"], new: list["FileInfo"]) -> dict[str, list[str]]:
+    old_files = {d.relpath: d for d in old}
+    new_files = {d.relpath: d for d in new}
+    rest = old_files.keys() & new_files.keys()
+    return {
+        "added": list(new_files.keys() - old_files.keys()),
+        "deleted": list(old_files.keys() - new_files.keys()),
+        "modified": [p for p in rest if new_files[p] != old_files[p]],
+    }
+
+
+class CmdDatasetAdd(CmdBase):
+    @classmethod
+    def display(cls, name: str, dataset: "Dataset", action: str = "Adding"):
+        from dvc.ui import ui
+
+        assert dataset.lock
+
+        url = dataset.spec.url
+        ver: str = ""
+        if dataset.type == "dc":
+            ver = f"v{dataset.lock.version}"
+        if dataset.type == "dvc":
+            if dataset.lock.path:
+                url = f"{url}:/{dataset.lock.path.lstrip('/')}"
+            if rev := dataset.lock.rev:
+                ver = rev
+
+        ver_part: Optional[Text] = None
+        if ver:
+            ver_part = ui.rich_text.assemble(" @ ", (ver, "repr.number"))
+        text = ui.rich_text.assemble("(", (url, "repr.url"), ver_part or "", ")")
+        ui.write(action, ui.rich_text(name, "cyan"), text, styled=True)
+
+    def run(self):
+        if not self.args.dvc and self.args.rev:
+            raise DvcException("--rev can't be used without --dvc")
+        if not self.args.dvc and self.args.path:
+            raise DvcException("--path can't be used without --dvc")
+
+        d = vars(self.args)
+        for key in ["dvc", "dc", "url"]:
+            if url := d.pop(key, None):
+                d.update({"type": key, "url": url})
+                break
+
+        existing = self.repo.datasets.get(self.args.name)
+        with self.repo.scm_context:
+            if not self.args.force and existing:
+                path = self.repo.fs.relpath(existing.manifest_path)
+                raise DvcException(
+                    f"{self.args.name} already exists in {path}, "
+                    "use the --force to overwrite"
+                )
+            dataset = self.repo.datasets.add(**d)
+            self.display(self.args.name, dataset)
+            return 0
+
+
+class CmdDatasetUpdate(CmdBase):
+    def display(self, name: str, dataset: "Dataset", new: "Dataset"):
+        from dvc.commands.checkout import log_changes
+        from dvc.ui import ui
+
+        action = "Updating"
+        if not dataset.lock:
+            return CmdDatasetAdd.display(name, new, action)
+        if dataset == new:
+            ui.write("[yellow]Nothing to update[/]", styled=True)
+            return
+
+        assert new.lock
+
+        v: Optional[tuple[str, str]] = None
+        if dataset.type == "dc":
+            assert new.type == "dc"
+            if new.lock.version < dataset.lock.version:
+                action = "Downgrading"
+
+            v = (f"v{dataset.lock.version}", f"v{new.lock.version}")
+        if dataset.type == "dvc":
+            assert new.type == "dvc"
+            v = (f"{dataset.lock.rev_lock[:9]}", f"{new.lock.rev_lock[:9]}")
+
+        if v:
+            part = ui.rich_text.assemble(
+                (v[0], "repr.number"),
+                " -> ",
+                (v[1], "repr.number"),
+            )
+        else:
+            part = ui.rich_text(dataset.spec.url, "repr.url")
+        changes = ui.rich_text.assemble("(", part, ")")
+        ui.write(action, ui.rich_text(name, "cyan"), changes, styled=True)
+        if dataset.type == "url":
+            assert new.type == "url"
+            stats = diff_files(dataset.lock.files, new.lock.files)
+            log_changes(stats)
+
+    def run(self):
+        from difflib import get_close_matches
+
+        from dvc.repo.datasets import DatasetNotFoundError
+        from dvc.ui import ui
+
+        version = None
+        if self.args.rev:
+            try:
+                version = int(self.args.rev.lstrip("v"))
+            except ValueError:
+                version = self.args.rev
+
+        d = vars(self.args) | {"version": version}
+        with self.repo.scm_context:
+            try:
+                dataset, new = self.repo.datasets.update(**d)
+            except DatasetNotFoundError:
+                logger.exception("")
+                if matches := get_close_matches(self.args.name, self.repo.datasets):
+                    ui.write(
+                        "did you mean?",
+                        ui.rich_text(matches[0], "cyan"),
+                        stderr=True,
+                        styled=True,
+                    )
+                return 1
+            self.display(self.args.name, dataset, new)
+            return 0
+
+
+def add_parser(subparsers, parent_parser):
+    ds_parser = subparsers.add_parser(
+        "dataset",
+        aliases=["ds"],
+        parents=[parent_parser],
+        formatter_class=formatter.RawDescriptionHelpFormatter,
+    )
+    ds_subparsers = ds_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc dataset CMD --help` to display command-specific help.",
+        required=True,
+    )
+
+    dataset_add_help = "Add a dataset."
+    ds_add_parser = ds_subparsers.add_parser(
+        "add",
+        parents=[parent_parser],
+        description=append_doc_link(dataset_add_help, "dataset/add"),
+        formatter_class=formatter.RawTextHelpFormatter,
+        help=dataset_add_help,
+    )
+
+    url_exclusive_group = ds_add_parser.add_mutually_exclusive_group(required=True)
+    url_exclusive_group.add_argument(
+        "--dc", metavar="name", help="Name of the DataChain dataset to track"
+    )
+    url_exclusive_group.add_argument(
+        "--dvc",
+        help="Path or URL to a Git/DVC repository to track",
+        metavar="url",
+    )
+    url_exclusive_group.add_argument(
+        "--url",
+        help="""\
+URL of a cloud-versioned remote to track. Supported URLs:
+
+s3://bucket/key/path
+gs://bucket/path/to/file/or/dir
+azure://mycontainer/path
+remote://remote_name/path/to/file/or/dir (see `dvc remote`)
+""",
+    )
+    ds_add_parser.add_argument("name", help="Name of the dataset to add")
+    ds_add_parser.add_argument(
+        "--rev",
+        help="Git revision, e.g. SHA, branch, tag (only applicable with --dvc)",
+        metavar="<commit>",
+    )
+    ds_add_parser.add_argument(
+        "--path",
+        help="Path to a file or a directory within a git repository "
+        "(only applicable with --dvc)",
+    )
+    ds_add_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Overwrite existing dataset",
+    )
+    ds_add_parser.set_defaults(func=CmdDatasetAdd)
+
+    dataset_update_help = "Update a dataset."
+    ds_update_parser = ds_subparsers.add_parser(
+        "update",
+        parents=[parent_parser],
+        description=append_doc_link(dataset_update_help, "dataset/update"),
+        formatter_class=formatter.RawDescriptionHelpFormatter,
+        help=dataset_update_help,
+    )
+    ds_update_parser.add_argument("name", help="Name of the dataset to update")
+    ds_update_parser.add_argument(
+        "--rev",
+        "--version",
+        nargs="?",
+        help="DataChain dataset version or Git revision (e.g. SHA, branch, tag)",
+        metavar="<version>",
+    )
+    ds_update_parser.set_defaults(func=CmdDatasetUpdate)

dvc/commands/experiments/__init__.py

@@ -58,6 +58,15 @@ def add_parser(subparsers, parent_parser):
     hide_subparsers_from_help(experiments_subparsers)
 
 
+def add_keep_selection_flag(experiments_subcmd_parser):
+    experiments_subcmd_parser.add_argument(
+        "--keep",
+        action="store_true",
+        default=False,
+        help="Keep the selected experiments instead of removing them.",
+    )
+
+
 def add_rev_selection_flags(
     experiments_subcmd_parser, command: str, default: bool = True
 ):

dvc/commands/experiments/pull.py

@@ -1,3 +1,5 @@
+import argparse
+
 from dvc.cli import formatter
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
@@ -81,7 +83,7 @@ def add_parser(experiments_subparsers, parent_parser):
     )
     experiments_pull_parser.add_argument(
         "--run-cache",
-        action="store_true",
+        action=argparse.BooleanOptionalAction,
         default=False,
         help="Pull run history for all stages.",
     )

dvc/commands/experiments/push.py

@@ -1,3 +1,4 @@
+import argparse
 from typing import Any
 
 from dvc.cli import completion, formatter
@@ -120,7 +121,7 @@ def add_parser(experiments_subparsers, parent_parser):
     )
     experiments_push_parser.add_argument(
         "--run-cache",
-        action="store_true",
+        action=argparse.BooleanOptionalAction,
         default=False,
         help="Push run history for all stages.",
     )

dvc/commands/experiments/remove.py

@@ -34,6 +34,7 @@ class CmdExperimentsRemove(CmdBase):
             num=self.args.num,
             queue=self.args.queue,
             git_remote=self.args.git_remote,
+            keep=self.args.keep,
         )
         if removed:
             ui.write(f"Removed experiments: {humanize.join(map(repr, removed))}")
@@ -44,7 +45,7 @@ class CmdExperimentsRemove(CmdBase):
 
 
 def add_parser(experiments_subparsers, parent_parser):
-    from . import add_rev_selection_flags
+    from . import add_keep_selection_flag, add_rev_selection_flags
 
     EXPERIMENTS_REMOVE_HELP = "Remove experiments."
     experiments_remove_parser = experiments_subparsers.add_parser(
@@ -57,6 +58,7 @@ def add_parser(experiments_subparsers, parent_parser):
     )
     remove_group = experiments_remove_parser.add_mutually_exclusive_group()
     add_rev_selection_flags(experiments_remove_parser, "Remove", False)
+    add_keep_selection_flag(experiments_remove_parser)
     remove_group.add_argument(
         "--queue", action="store_true", help="Remove all queued experiments."
     )

dvc/commands/experiments/show.py

@@ -150,6 +150,7 @@ class CmdExperimentsShow(CmdBase):
                 all_commits=self.args.all_commits,
                 hide_queued=self.args.hide_queued,
                 hide_failed=self.args.hide_failed,
+                hide_workspace=self.args.hide_workspace,
                 revs=self.args.rev,
                 num=self.args.num,
                 sha_only=self.args.sha,
@@ -279,6 +280,12 @@ def add_parser(experiments_subparsers, parent_parser):
         default=False,
         help="Hide queued experiments in the table.",
     )
+    experiments_show_parser.add_argument(
+        "--hide-workspace",
+        action="store_true",
+        default=False,
+        help="Hide workspace row in the table.",
+    )
     experiments_show_parser.add_argument(
         "--json",
         action="store_true",

dvc/commands/gc.py

@@ -54,13 +54,13 @@ class CmdGC(CmdBase):
             msg += " and all experiments"
 
         if self.args.not_in_remote:
-            msg += " that are present in the DVC remote"
+            msg += " that are not present in the DVC remote"
 
         if self.args.repos:
             msg += " of the current and the following repos:"
 
             for repo_path in self.args.repos:
-                msg += "\n  - %s" % os.path.abspath(repo_path)
+                msg += f"\n  - {os.path.abspath(repo_path)}"
         else:
             msg += " of the current repo."
 

dvc/commands/get.py

@@ -12,7 +12,13 @@ class CmdGet(CmdBaseNoRepo):
         from dvc.api import get_url
         from dvc.ui import ui
 
-        url = get_url(self.args.path, repo=self.args.url, rev=self.args.rev)
+        url = get_url(
+            self.args.path,
+            repo=self.args.url,
+            rev=self.args.rev,
+            remote=self.args.remote,
+            remote_config=self.args.remote_config,
+        )
         ui.write(url, force=True)
 
         return 0

dvc/commands/ls/__init__.py

@@ -1,3 +1,5 @@
+from typing import Callable
+
 from dvc.cli import completion, formatter
 from dvc.cli.command import CmdBaseNoRepo
 from dvc.cli.utils import DictAction, append_doc_link
@@ -9,54 +11,178 @@ from dvc.ui import ui
 logger = logger.getChild(__name__)
 
 
-def _format_entry(entry, fmt):
-    from dvc.utils.humanize import naturalsize
+def _get_formatter(with_color: bool = False) -> Callable[[dict], str]:
+    def fmt(entry: dict) -> str:
+        return entry["path"]
 
-    size = entry.get("size")
-    if size is None:
-        size = ""
-    else:
-        size = naturalsize(size)
-    return size, fmt(entry)
-
-
-def show_entries(entries, with_color=False, with_size=False):
     if with_color:
         ls_colors = LsColors()
-        fmt = ls_colors.format
-    else:
+        return ls_colors.format
 
-        def fmt(entry):
-            return entry["path"]
+    return fmt
 
+
+def _format_entry(entry, name, with_size=True, with_hash=False):
+    from dvc.utils.humanize import naturalsize
+
+    ret = []
     if with_size:
-        ui.table([_format_entry(entry, fmt) for entry in entries])
+        size = entry.get("size")
+        if size is None or (size <= 0 and entry.get("isdir")):
+            size = ""
+        else:
+            size = naturalsize(size)
+        ret.append(size)
+    if with_hash:
+        md5 = entry.get("md5", "")
+        ret.append(md5)
+    ret.append(name)
+    return ret
+
+
+def show_entries(entries, with_color=False, with_size=False, with_hash=False):
+    fmt = _get_formatter(with_color)
+    if with_size or with_hash:
+        colalign = ("right",) if with_size else None
+        ui.table(
+            [
+                _format_entry(
+                    entry,
+                    fmt(entry),
+                    with_size=with_size,
+                    with_hash=with_hash,
+                )
+                for entry in entries
+            ],
+            colalign=colalign,
+        )
         return
 
     # NOTE: this is faster than ui.table for very large number of entries
     ui.write("\n".join(fmt(entry) for entry in entries))
 
 
+class TreePart:
+    Edge = "├── "
+    Line = "│   "
+    Corner = "└── "
+    Blank = "    "
+
+
+def _build_tree_structure(
+    entries, with_color=False, with_size=False, with_hash=False, _depth=0, _prefix=""
+):
+    rows = []
+    fmt = _get_formatter(with_color)
+
+    num_entries = len(entries)
+    for i, (name, entry) in enumerate(entries.items()):
+        entry["path"] = name
+        is_last = i >= num_entries - 1
+        tree_part = ""
+        if _depth > 0:
+            tree_part = TreePart.Corner if is_last else TreePart.Edge
+
+        row = _format_entry(
+            entry,
+            _prefix + tree_part + fmt(entry),
+            with_size=with_size,
+            with_hash=with_hash,
+        )
+        rows.append(row)
+
+        if contents := entry.get("contents"):
+            new_prefix = _prefix
+            if _depth > 0:
+                new_prefix += TreePart.Blank if is_last else TreePart.Line
+            new_rows = _build_tree_structure(
+                contents,
+                with_color=with_color,
+                with_size=with_size,
+                with_hash=with_hash,
+                _depth=_depth + 1,
+                _prefix=new_prefix,
+            )
+            rows.extend(new_rows)
+
+    return rows
+
+
+def show_tree(entries, with_color=False, with_size=False, with_hash=False):
+    import tabulate
+
+    rows = _build_tree_structure(
+        entries,
+        with_color=with_color,
+        with_size=with_size,
+        with_hash=with_hash,
+    )
+
+    colalign = ("right",) if with_size else None
+
+    _orig = tabulate.PRESERVE_WHITESPACE
+    tabulate.PRESERVE_WHITESPACE = True
+    try:
+        ui.table(rows, colalign=colalign)
+    finally:
+        tabulate.PRESERVE_WHITESPACE = _orig
+
+
 class CmdList(CmdBaseNoRepo):
-    def run(self):
+    def _show_tree(self):
+        from dvc.repo.ls import ls_tree
+
+        entries = ls_tree(
+            self.args.url,
+            self.args.path,
+            rev=self.args.rev,
+            dvc_only=self.args.dvc_only,
+            config=self.args.config,
+            remote=self.args.remote,
+            remote_config=self.args.remote_config,
+            maxdepth=self.args.level,
+        )
+        show_tree(
+            entries,
+            with_color=True,
+            with_size=self.args.size,
+            with_hash=self.args.show_hash,
+        )
+        return 0
+
+    def _show_list(self):
         from dvc.repo import Repo
 
-        try:
-            entries = Repo.ls(
-                self.args.url,
-                self.args.path,
-                rev=self.args.rev,
-                recursive=self.args.recursive,
-                dvc_only=self.args.dvc_only,
-                config=self.args.config,
-                remote=self.args.remote,
-                remote_config=self.args.remote_config,
+        entries = Repo.ls(
+            self.args.url,
+            self.args.path,
+            rev=self.args.rev,
+            recursive=self.args.recursive,
+            dvc_only=self.args.dvc_only,
+            config=self.args.config,
+            remote=self.args.remote,
+            remote_config=self.args.remote_config,
+            maxdepth=self.args.level,
+        )
+        if self.args.json:
+            ui.write_json(entries)
+        elif entries:
+            show_entries(
+                entries,
+                with_color=True,
+                with_size=self.args.size,
+                with_hash=self.args.show_hash,
             )
-            if self.args.json:
-                ui.write_json(entries)
-            elif entries:
-                show_entries(entries, with_color=True, with_size=self.args.size)
-            return 0
+        return 0
+
+    def run(self):
+        if self.args.tree and self.args.json:
+            raise DvcException("Cannot use --tree and --json options together.")
+
+        try:
+            if self.args.tree:
+                return self._show_tree()
+            return self._show_list()
         except FileNotFoundError:
             logger.exception("")
             return 1
@@ -85,6 +211,19 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         help="Recursively list files.",
     )
+    list_parser.add_argument(
+        "-T",
+        "--tree",
+        action="store_true",
+        help="Recurse into directories as a tree.",
+    )
+    list_parser.add_argument(
+        "-L",
+        "--level",
+        metavar="depth",
+        type=int,
+        help="Limit the depth of recursion.",
+    )
     list_parser.add_argument(
         "--dvc-only", action="store_true", help="Show only DVC outputs."
     )
@@ -123,6 +262,12 @@ def add_parser(subparsers, parent_parser):
         ),
     )
     list_parser.add_argument("--size", action="store_true", help="Show sizes.")
+    list_parser.add_argument(
+        "--show-hash",
+        help="Display hash value for each item.",
+        action="store_true",
+        default=False,
+    )
     list_parser.add_argument(
         "path",
         nargs="?",

dvc/commands/ls/ls_colors.py

@@ -32,7 +32,9 @@ class LsColors:
         if entry.get("isexec", False):
             return self._format(text, code="ex")
 
-        _, ext = os.path.splitext(text)
+        stem, ext = os.path.splitext(text)
+        if not ext and stem.startswith("."):
+            ext = stem
         return self._format(text, ext=ext)
 
     def _format(self, text, code=None, ext=None):

dvc/commands/ls_url.py

@@ -3,26 +3,45 @@ from dvc.cli.command import CmdBaseNoRepo
 from dvc.cli.utils import DictAction, append_doc_link
 from dvc.log import logger
 
-from .ls import show_entries
+from .ls import show_entries, show_tree
 
 logger = logger.getChild(__name__)
 
 
 class CmdListUrl(CmdBaseNoRepo):
-    def run(self):
-        from dvc.config import Config
+    def _show_tree(self, config):
+        from dvc.fs import parse_external_url
+        from dvc.repo.ls import _ls_tree
+
+        fs, fs_path = parse_external_url(
+            self.args.url, fs_config=self.args.fs_config, config=config
+        )
+        entries = _ls_tree(fs, fs_path, maxdepth=self.args.level)
+        show_tree(entries, with_color=True, with_size=self.args.size)
+        return 0
+
+    def _show_list(self, config):
         from dvc.repo import Repo
 
         entries = Repo.ls_url(
             self.args.url,
             recursive=self.args.recursive,
+            maxdepth=self.args.level,
             fs_config=self.args.fs_config,
-            config=Config.from_cwd(),
+            config=config,
         )
         if entries:
             show_entries(entries, with_color=True, with_size=self.args.size)
         return 0
 
+    def run(self):
+        from dvc.config import Config
+
+        config = Config.from_cwd()
+        if self.args.tree:
+            return self._show_tree(config=config)
+        return self._show_list(config=config)
+
 
 def add_parser(subparsers, parent_parser):
     LS_HELP = "List directory contents from URL."
@@ -40,6 +59,19 @@ def add_parser(subparsers, parent_parser):
     lsurl_parser.add_argument(
         "-R", "--recursive", action="store_true", help="Recursively list files."
     )
+    lsurl_parser.add_argument(
+        "-T",
+        "--tree",
+        action="store_true",
+        help="Recurse into directories as a tree.",
+    )
+    lsurl_parser.add_argument(
+        "-L",
+        "--level",
+        metavar="depth",
+        type=int,
+        help="Limit the depth of recursion.",
+    )
     lsurl_parser.add_argument("--size", action="store_true", help="Show sizes.")
     lsurl_parser.add_argument(
         "--fs-config",

dvc/commands/plots.py

@@ -72,7 +72,7 @@ class CmdPlots(CmdBase):
                 html_template_path = os.path.join(self.repo.dvc_dir, html_template_path)
         return html_template_path
 
-    def run(self) -> int:  # noqa: C901, PLR0911, PLR0912
+    def run(self) -> int:  # noqa: C901, PLR0911
         from pathlib import Path
 
         from dvc.render.match import match_defs_renderers

dvc/commands/remote.py

@@ -110,8 +110,16 @@ class CmdRemoteDefault(CmdRemote):
 class CmdRemoteList(CmdRemote):
     def run(self):
         conf = self.config.read(self.args.level)
+        default_remote = conf["core"].get("remote")
+
         for name, remote_conf in conf["remote"].items():
-            ui.write(name, remote_conf["url"], sep="\t")
+            if name == default_remote:
+                text = f"{name}\t{remote_conf['url']}\t(default)"
+                color = "green"
+            else:
+                text = f"{name}\t{remote_conf['url']}"
+                color = ""
+            ui.write(ui.rich_text(text, style=color), styled=True)
         return 0
 
 

dvc/commands/repro.py

@@ -117,10 +117,7 @@ and then the stage name name.
         "--pull",
         action="store_true",
         default=False,
-        help=(
-            "Try automatically pulling missing cache for outputs restored "
-            "from the run-cache."
-        ),
+        help="Try automatically pulling missing data.",
     )
     repro_parser.add_argument(
         "--allow-missing",

dvc/commands/stage.py

@@ -68,7 +68,7 @@ def prepare_stages_data(
 class CmdStageList(CmdBase):
     def _get_stages(self) -> Iterable["Stage"]:
         if self.args.all:
-            stages: list["Stage"] = self.repo.index.stages
+            stages: list[Stage] = self.repo.index.stages
             logger.trace("%d no. of stages found", len(stages))
             return stages
 

dvc/commands/status.py

@@ -71,7 +71,7 @@ class CmdDataStatus(CmdDataBase):
                 return 0
 
             if self.args.quiet:
-                return bool(st)
+                return int(bool(st))
 
             if st:
                 self._show(st, indent)

dvc/commands/studio.py

@@ -5,6 +5,7 @@ from funcy import get_in
 from dvc.cli import formatter
 from dvc.cli.utils import append_doc_link
 from dvc.commands.config import CmdConfig
+from dvc.exceptions import DvcException
 from dvc.log import logger
 
 logger = logger.getChild(__name__)
@@ -12,22 +13,35 @@ logger = logger.getChild(__name__)
 
 class CmdStudioLogin(CmdConfig):
     def run(self):
-        from dvc_studio_client.auth import StudioAuthError, get_access_token
-
         from dvc.env import DVC_STUDIO_URL
         from dvc.ui import ui
         from dvc.utils.studio import STUDIO_URL
+        from dvc_studio_client.auth import StudioAuthError, get_access_token
 
+        studio = self.config.get("studio", {})
         name = self.args.name
-        hostname = self.args.hostname or os.environ.get(DVC_STUDIO_URL) or STUDIO_URL
+        hostname = (
+            self.args.hostname
+            or os.environ.get(DVC_STUDIO_URL)
+            or studio.get("url")
+            or STUDIO_URL
+        )
         scopes = self.args.scopes
 
+        if studio.get("url", hostname) == hostname and "token" in studio:
+            raise DvcException(
+                "Token already exists. "
+                "To login with a different token, "
+                "logout using 'dvc studio logout'."
+            )
+
+        open_browser = not self.args.no_open
         try:
-            token_name, access_token = get_access_token(
+            _, access_token = get_access_token(
                 token_name=name,
                 hostname=hostname,
                 scopes=scopes,
-                use_device_code=self.args.no_open,
+                open_browser=open_browser,
                 client_name="DVC",
             )
         except StudioAuthError as e:
@@ -35,17 +49,26 @@ class CmdStudioLogin(CmdConfig):
             return 1
 
         self.save_config(hostname, access_token)
-        ui.write(
-            "Authentication has been successfully completed."
-            "The generated token will now be accessible as"
-            f" {token_name} in the user's Studio profile."
-        )
+
+        if not self.config["exp"].get("auto_push", True):
+            from dvc.ui import ui
+
+            ui.warn(
+                "exp.auto_push is disabled. \n"
+                "Enable with 'dvc config exp.auto_push true' "
+                "to automatically push experiments to Studio."
+            )
+
+        config_path = self.config.files["global"]
+        ui.write(f"Authentication complete. Saved token to {config_path}.")
         return 0
 
     def save_config(self, hostname, token):
         with self.config.edit("global") as conf:
             conf["studio"]["token"] = token
             conf["studio"]["url"] = hostname
+            if "auto_push" not in conf["exp"]:
+                conf["exp"]["auto_push"] = True
 
 
 class CmdStudioLogout(CmdConfig):
@@ -54,12 +77,14 @@ class CmdStudioLogout(CmdConfig):
 
         with self.config.edit("global") as conf:
             if not get_in(conf, ["studio", "token"]):
-                ui.error_write("Not logged in to Studio.")
+                ui.error_write(
+                    "Not logged in to Studio. Log in with 'dvc studio login'."
+                )
                 return 1
 
             del conf["studio"]["token"]
 
-        ui.write("Logged out from Studio")
+        ui.write("Logged out from Studio (you can log back in with 'dvc studio login')")
         return 0
 
 

dvc/compare.py

@@ -88,12 +88,10 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
         return list(it)
 
     @overload
-    def __setitem__(self, item: int, value: Sequence["CellT"]) -> None:
-        ...
+    def __setitem__(self, item: int, value: Sequence["CellT"]) -> None: ...
 
     @overload
-    def __setitem__(self, item: slice, value: Iterable[Sequence["CellT"]]) -> None:
-        ...
+    def __setitem__(self, item: slice, value: Iterable[Sequence["CellT"]]) -> None: ...
 
     def __setitem__(self, item, value) -> None:
         it = value
@@ -162,7 +160,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
             if key not in keys:
                 self.add_column(key)
 
-        row: list["CellT"] = [
+        row: list[CellT] = [
             with_value(d.get(key), self._fill_value) for key in self.keys()
         ]
         self.append(row)

dvc/config.py

@@ -1,5 +1,8 @@
 """DVC config objects."""
+
+import ntpath
 import os
+import posixpath
 import re
 from contextlib import contextmanager
 from functools import partial
@@ -267,12 +270,17 @@ class Config(dict):
         if re.match(r"\w+://", path):
             return path
 
+        if os.name == "nt" and posixpath.isabs(path) and ntpath.sep not in path:
+            return path
+
         if os.path.isabs(path):
             return path
 
         # on windows convert slashes to backslashes
         # to have path compatible with abs_conf_dir
         if os.path.sep == "\\" and "/" in path:
+            if path.startswith("/"):
+                path = path.replace("/", "\\\\", 1)
             path = path.replace("/", "\\")
 
         expanded = os.path.expanduser(path)
@@ -304,6 +312,9 @@ class Config(dict):
         if os.path.expanduser(path) != path:
             return localfs.as_posix(path)
 
+        if os.name == "nt" and posixpath.isabs(path) and ntpath.sep not in path:
+            return path
+
         if isinstance(path, RelPath) or not os.path.isabs(path):
             path = relpath(path, conf_dir)
             return localfs.as_posix(path)
@@ -329,6 +340,7 @@ class Config(dict):
                     "gdrive_user_credentials_file": func,
                     "gdrive_service_account_json_file_path": func,
                     "credentialpath": func,
+                    "configpath": func,
                     "keyfile": func,
                     "cert_path": func,
                     "key_path": func,

dvc/config_schema.py

@@ -193,6 +193,7 @@ SCHEMA = {
                     "session_token": str,
                     Optional("listobjects", default=False): Bool,  # obsoleted
                     Optional("use_ssl", default=True): Bool,
+                    Optional("allow_anonymous_login", default=False): Bool,
                     "ssl_verify": Any(Bool, str),
                     "sse": str,
                     "sse_kms_key_id": str,
@@ -214,6 +215,7 @@ SCHEMA = {
                     "credentialpath": str,
                     "endpointurl": str,
                     Optional("verify", default=False): Bool,
+                    Optional("allow_anonymous_login", default=False): Bool,
                     **REMOTE_COMMON,
                 },
                 "ssh": {
@@ -232,7 +234,12 @@ SCHEMA = {
                     Optional("verify", default=False): Bool,
                     **REMOTE_COMMON,
                 },
-                "hdfs": {"user": str, "kerb_ticket": str, **REMOTE_COMMON},
+                "hdfs": {
+                    "user": str,
+                    "kerb_ticket": str,
+                    "replication": int,
+                    **REMOTE_COMMON,
+                },
                 "webhdfs": {
                     "kerberos": Bool,
                     "kerberos_principal": str,
@@ -336,6 +343,8 @@ SCHEMA = {
         "params": str,
         "plots": str,
         "live": str,
+        "auto_push": Bool,
+        "git_remote": str,
     },
     "parsing": {
         "bool": All(Lower, Choices("store_true", "boolean_optional")),

dvc/daemon.py

@@ -44,10 +44,10 @@ def _win_detached_subprocess(args: Sequence[str], **kwargs) -> int:
 
     startupinfo = STARTUPINFO()
     startupinfo.dwFlags |= STARTF_USESHOWWINDOW
-    popen = subprocess.Popen(
+    popen = subprocess.Popen(  # noqa: S603
         args,
         close_fds=True,
-        shell=False,  # noqa: S603
+        shell=False,
         startupinfo=startupinfo,
         creationflags=creationflags,
         **kwargs,
@@ -149,7 +149,7 @@ def _spawn(
     env: Optional[Mapping[str, str]] = None,
     output_file: Optional[str] = None,
 ) -> int:
-    file: "AbstractContextManager[Any]" = nullcontext()
+    file: AbstractContextManager[Any] = nullcontext()
     kwargs = {}
     if output_file:
         file = open(output_file, "ab")  # noqa: SIM115
@@ -164,7 +164,7 @@ def _spawn(
         return _detached_subprocess(executable + args, env=env, **kwargs)
 
 
-def daemonize(args: list[str], executable: Union[None, str, list[str]] = None) -> None:
+def daemonize(args: list[str], executable: Union[str, list[str], None] = None) -> None:
     if os.name not in ("posix", "nt"):
         return
 

dvc/dagascii.py

@@ -104,21 +104,21 @@ class AsciiCanvas:
                 if dx == 0:
                     y = y0
                 else:
-                    y = y0 + int(round((x - x0) * dy / float(dx)))
+                    y = y0 + round((x - x0) * dy / float(dx))
                 self.point(x, y, char)
         elif y0 < y1:
             for y in range(y0, y1 + 1):
                 if dy == 0:
                     x = x0
                 else:
-                    x = x0 + int(round((y - y0) * dx / float(dy)))
+                    x = x0 + round((y - y0) * dx / float(dy))
                 self.point(x, y, char)
         else:
             for y in range(y1, y0 + 1):
                 if dy == 0:
                     x = x0
                 else:
-                    x = x1 + int(round((y - y1) * dx / float(dy)))
+                    x = x1 + round((y - y1) * dx / float(dy))
                 self.point(x, y, char)
 
     def text(self, x, y, text):
@@ -258,8 +258,8 @@ def draw(vertices, edges):
     maxx = max(Xs)
     maxy = max(Ys)
 
-    canvas_cols = int(math.ceil(math.ceil(maxx) - math.floor(minx))) + 1
-    canvas_lines = int(round(maxy - miny))
+    canvas_cols = math.ceil(math.ceil(maxx) - math.floor(minx)) + 1
+    canvas_lines = round(maxy - miny)
 
     canvas = AsciiCanvas(canvas_cols, canvas_lines)
 
@@ -270,10 +270,10 @@ def draw(vertices, edges):
             start = edge.view._pts[index - 1]
             end = edge.view._pts[index]
 
-            start_x = int(round(start[0] - minx))
-            start_y = int(round(start[1] - miny))
-            end_x = int(round(end[0] - minx))
-            end_y = int(round(end[1] - miny))
+            start_x = round(start[0] - minx)
+            start_y = round(start[1] - miny)
+            end_x = round(end[0] - minx)
+            end_y = round(end[1] - miny)
 
             assert start_x >= 0
             assert start_y >= 0
@@ -288,12 +288,12 @@ def draw(vertices, edges):
         y = vertex.view.xy[1]
 
         canvas.box(
-            int(round(x - minx)),
-            int(round(y - miny)),
+            round(x - minx),
+            round(y - miny),
             vertex.view.w,
             vertex.view.h,
         )
 
-        canvas.text(int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data)
+        canvas.text(round(x - minx) + 1, round(y - miny) + 1, vertex.data)
 
     return canvas.draw()

dvc/database.py

@@ -5,9 +5,9 @@ from dataclasses import dataclass
 from tempfile import NamedTemporaryFile
 from typing import TYPE_CHECKING, Any, Callable, Optional, Union
 
-from sqlalchemy import create_engine
-from sqlalchemy.engine import make_url as _make_url
-from sqlalchemy.exc import NoSuchModuleError
+from sqlalchemy import create_engine  # type: ignore[import]
+from sqlalchemy.engine import make_url as _make_url  # type: ignore[import]
+from sqlalchemy.exc import NoSuchModuleError  # type: ignore[import]
 
 from dvc import env
 from dvc.exceptions import DvcException
@@ -17,7 +17,7 @@ from dvc.utils import env2bool
 
 if TYPE_CHECKING:
     from sqlalchemy.engine import URL, Connectable, Engine
-    from sqlalchemy.sql.expression import Selectable
+    from sqlalchemy.sql.expression import Selectable  # type: ignore[import]
 
 
 logger = logger.getChild(__name__)

dvc/dependency/base.py

@@ -45,7 +45,7 @@ class Dependency(Output):
             self.fs_path = self.fs.version_path(self.fs_path, self.meta.version_id)
 
     def download(self, to, jobs=None):
-        fs_download(self.fs, self.fs_path, to.fs_path, jobs=jobs)
+        return fs_download(self.fs, self.fs_path, to.fs_path, jobs=jobs)
 
     def save(self):
         super().save()

dvc/dependency/dataset.py

@@ -3,6 +3,7 @@ from urllib.parse import urlparse
 
 from funcy import compact, merge
 
+from dvc.exceptions import DvcException
 from dvc_data.hashfile.hash_info import HashInfo
 
 from .db import AbstractDependency
@@ -47,9 +48,15 @@ class DatasetDependency(AbstractDependency):
         )
 
     def workspace_status(self):
-        registered = self.repo.index.datasets.get(self.name, {})
+        ds = self.repo.datasets[self.name]
+        if not ds.lock:
+            return {str(self): "not in sync"}
+
         info: dict[str, Any] = self.hash_info.value if self.hash_info else {}  # type: ignore[assignment]
-        if info != registered:
+        lock = self.repo.datasets._lock_from_info(info)
+        if not lock:
+            return {str(self): "new"}
+        if lock != ds.lock:
             return {str(self): "modified"}
         return {}
 
@@ -57,7 +64,15 @@ class DatasetDependency(AbstractDependency):
         return self.workspace_status()
 
     def get_hash(self):
-        return HashInfo(self.PARAM_DATASET, self.repo.index.datasets.get(self.name, {}))
+        ds = self.repo.datasets[self.name]
+        if not ds.lock:
+            if ds._invalidated:
+                raise DvcException(
+                    "Dataset information is not in sync. "
+                    f"Run 'dvc ds update {self.name}' to sync."
+                )
+            raise DvcException("Dataset information missing from dvc.lock file")
+        return HashInfo(self.PARAM_DATASET, ds.lock.to_dict())  # type: ignore[arg-type]
 
     def save(self):
         self.hash_info = self.get_hash()

dvc/dependency/db.py

@@ -11,6 +11,7 @@ from .base import Dependency
 
 if TYPE_CHECKING:
     from dvc.output import Output
+    from dvc.repo import Repo
     from dvc.stage import Stage
 
 logger = logger.getChild(__name__)
@@ -21,9 +22,12 @@ def download_progress(to: "Output") -> Iterator[Callable[[int], Any]]:
     from dvc.ui import ui
     from dvc.ui._rich_progress import DbDownloadProgress
 
-    with log_durations(logger.debug, f"Saving to {to}"), DbDownloadProgress(
-        console=ui.error_console,
-    ) as progress:
+    with (
+        log_durations(logger.debug, f"Saving to {to}"),
+        DbDownloadProgress(
+            console=ui.error_console,
+        ) as progress,
+    ):
         task = progress.add_task("Saving", total=None, output=to)
         yield lambda n: progress.advance(task, advance=n)
         progress.update(task, description="Saved", total=0)
@@ -33,13 +37,17 @@ class AbstractDependency(Dependency):
     """Dependency without workspace/fs/fs_path"""
 
     def __init__(self, stage: "Stage", info: dict[str, Any], *args, **kwargs):
-        self.repo = stage.repo
+        self.repo: Repo = stage.repo
         self.stage = stage
         self.fs = None
         self.fs_path = None
         self.def_path = None  # type: ignore[assignment]
         self.info = info or {}
 
+    @property
+    def is_in_repo(self):
+        return False
+
 
 class DbDependency(AbstractDependency):
     PARAM_CONNECTION = "connection"
@@ -66,8 +74,8 @@ class DbDependency(AbstractDependency):
         return self.db_info.get(self.PARAM_QUERY) or self.db_info.get(self.PARAM_TABLE)
 
     def __repr__(self):
-        return "{}:{}".format(
-            self.__class__.__name__, "".join(f"{k}=={v}" for k, v in self.info.items())
+        return "{}: {}".format(
+            self.__class__.__name__, "".join(f"{k}={v}" for k, v in self.info.items())
         )
 
     def __str__(self):

dvc/dependency/repo.py

@@ -9,7 +9,9 @@ from .base import Dependency
 
 if TYPE_CHECKING:
     from dvc.fs import DVCFileSystem
+    from dvc.output import Output
     from dvc.stage import Stage
+    from dvc_data.hashfile.hash_info import HashInfo
 
 
 class RepoDependency(Dependency):
@@ -35,7 +37,7 @@ class RepoDependency(Dependency):
         super().__init__(stage, *args, **kwargs)
 
         self.fs = self._make_fs()
-        self.fs_path = as_posix(self.def_path)
+        self.fs_path = as_posix(self.fs.normpath(self.def_path))
 
     def _parse_path(self, fs, fs_path):  # noqa: ARG002
         return None
@@ -91,6 +93,27 @@ class RepoDependency(Dependency):
             self.PARAM_REPO: self._dump_def_repo(self.def_repo),
         }
 
+    def download(self, to: "Output", jobs: Optional[int] = None):
+        from dvc.fs import LocalFileSystem
+
+        files = super().download(to=to, jobs=jobs)
+        if not isinstance(to.fs, LocalFileSystem):
+            return
+
+        hashes: list[tuple[str, HashInfo, dict[str, Any]]] = []
+        for src_path, dest_path, maybe_info in files:
+            try:
+                info = maybe_info or self.fs.info(src_path)
+                hash_info = info["dvc_info"]["entry"].hash_info
+                dest_info = to.fs.info(dest_path)
+            except (KeyError, AttributeError):
+                # If no hash info found, just keep going and output will be hashed later
+                continue
+            if hash_info:
+                hashes.append((dest_path, hash_info, dest_info))
+        cache = to.cache if to.use_cache else to.local_cache
+        cache.state.save_many(hashes, to.fs)
+
     def update(self, rev: Optional[str] = None):
         if rev:
             self.def_repo[self.PARAM_REV] = rev
@@ -123,11 +146,15 @@ class RepoDependency(Dependency):
         else:
             config = Config.load_file(conf)
 
+        # Setup config to the new DVCFileSystem to use the remote repo, but rely on the
+        # local cache instead of the remote's cache. This avoids re-streaming of data,
+        # but messes up the call to `_get_remote_config()` downstream, which will need
+        # to ignore cache parameters.
         config["cache"] = self.repo.config["cache"]
         config["cache"]["dir"] = self.repo.cache.local_cache_dir
 
         return DVCFileSystem(
-            url=self.def_repo[self.PARAM_URL],
+            repo=self.def_repo[self.PARAM_URL],
             rev=rev or self._get_rev(locked=locked),
             subrepos=True,
             config=config,

dvc/dirs.py

@@ -1,4 +1,5 @@
 import os
+from typing import Optional
 
 import platformdirs
 
@@ -20,7 +21,24 @@ def global_config_dir():
     )
 
 
-def site_cache_dir():
-    return os.getenv(env.DVC_SITE_CACHE_DIR) or platformdirs.site_cache_dir(
-        APPNAME, APPAUTHOR, opinion=True
-    )
+def site_cache_dir(config_site_cache_dir: Optional[str] = None):
+    from platformdirs import PlatformDirs
+    from platformdirs.unix import Unix
+
+    if dvc_site_cache_dir := os.getenv(env.DVC_SITE_CACHE_DIR):
+        return dvc_site_cache_dir
+
+    if config_site_cache_dir is not None:
+        return config_site_cache_dir
+
+    if issubclass(Unix, PlatformDirs):
+        # Return the cache directory shared by users, e.g. `/var/tmp/$appname`
+        # NOTE: platformdirs>=5 changed `site_cache_dir` to return /var/cache/$appname.
+        # as the following path is considered insecure.
+        # For details, see: https://github.com/tox-dev/platformdirs/pull/239
+
+        # FIXME: keeping the old behavior temporarily to avoid dependency conflict.
+        #        In the future, consider migrating to a more secure directory.
+        return f"/var/tmp/{APPNAME}"  # noqa: S108
+
+    return platformdirs.site_cache_dir(APPNAME, APPAUTHOR, opinion=True)

dvc/dvcfile.py

@@ -68,10 +68,9 @@ def is_git_ignored(repo, path):
 def check_dvcfile_path(repo, path):
     if not is_valid_filename(path):
         raise StageFileBadNameError(
-            "bad DVC file name '{}'. DVC files should be named "
-            "'{}' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
-                relpath(path), PROJECT_FILE, os.path.basename(path)
-            )
+            f"bad DVC file name '{relpath(path)}'. DVC files should be named "
+            f"'{PROJECT_FILE}' or have a '.dvc' suffix "
+            f"(e.g. '{os.path.basename(path)}.dvc')."
         )
 
     if is_git_ignored(repo, path):
@@ -172,6 +171,7 @@ class SingleStageFile(FileMixin):
     from dvc.stage.loader import SingleStageLoader as LOADER  # noqa: N814
 
     datasets: ClassVar[list[dict[str, Any]]] = []
+    datasets_lock: ClassVar[list[dict[str, Any]]] = []
     metrics: ClassVar[list[str]] = []
     plots: ClassVar[Any] = {}
     params: ClassVar[list[str]] = []
@@ -240,6 +240,26 @@ class ProjectFile(FileMixin):
         if update_lock:
             self._dump_lockfile(stage, **kwargs)
 
+    def dump_dataset(self, dataset):
+        with modify_yaml(self.path, fs=self.repo.fs) as data:
+            parsed = self.datasets if data else []
+            raw = data.setdefault("datasets", [])
+            loc = next(
+                (i for i, ds in enumerate(parsed) if ds["name"] == dataset["name"]),
+                None,
+            )
+            if loc is not None:
+                if raw[loc] != parsed[loc]:
+                    raise ParametrizedDumpError(
+                        "cannot update a parametrized dataset entry"
+                    )
+
+                apply_diff(dataset, raw[loc])
+                raw[loc] = dataset
+            else:
+                raw.append(dataset)
+        self.repo.scm_context.track_file(self.relpath)
+
     def _dump_lockfile(self, stage, **kwargs):
         self._lockfile.dump(stage, **kwargs)
 
@@ -293,24 +313,28 @@ class ProjectFile(FileMixin):
         return self.LOADER(self, self.contents, self.lockfile_contents)
 
     @property
-    def metrics(self) -> list[str]:
-        return self.contents.get("metrics", [])
+    def artifacts(self) -> dict[str, Optional[dict[str, Any]]]:
+        return self.resolver.resolve_artifacts()
 
     @property
-    def plots(self) -> Any:
-        return self.contents.get("plots", {})
+    def metrics(self) -> list[str]:
+        return self.resolver.resolve_metrics()
 
     @property
     def params(self) -> list[str]:
-        return self.contents.get("params", [])
+        return self.resolver.resolve_params()
+
+    @property
+    def plots(self) -> list[Any]:
+        return self.resolver.resolve_plots()
 
     @property
     def datasets(self) -> list[dict[str, Any]]:
-        return self.contents.get("datasets", [])
+        return self.resolver.resolve_datasets()
 
     @property
-    def artifacts(self) -> dict[str, Optional[dict[str, Any]]]:
-        return self.contents.get("artifacts", {})
+    def datasets_lock(self) -> list[dict[str, Any]]:
+        return self.lockfile_contents.get("datasets", [])
 
     def remove(self, force=False):
         if not force:
@@ -357,6 +381,24 @@ class Lockfile(FileMixin):
             self._check_gitignored()
             return {}, ""
 
+    def dump_dataset(self, dataset: dict):
+        with modify_yaml(self.path, fs=self.repo.fs) as data:
+            data.update({"schema": "2.0"})
+            if not data:
+                logger.info("Generating lock file '%s'", self.relpath)
+
+            datasets: list[dict] = data.setdefault("datasets", [])
+            loc = next(
+                (i for i, ds in enumerate(datasets) if ds["name"] == dataset["name"]),
+                None,
+            )
+            if loc is not None:
+                datasets[loc] = dataset
+            else:
+                datasets.append(dataset)
+            data.setdefault("stages", {})
+        self.repo.scm_context.track_file(self.relpath)
+
     def dump(self, stage, **kwargs):
         stage_data = serialize.to_lockfile(stage, **kwargs)
 

dvc/env.py

@@ -19,3 +19,4 @@ DVC_STUDIO_URL = "DVC_STUDIO_URL"
 DVC_SQLALCHEMY_ECHO = "DVC_SQLALCHEMY_ECHO"
 DVC_SYSTEM_CONFIG_DIR = "DVC_SYSTEM_CONFIG_DIR"
 DVC_UPDATER_ENDPOINT = "DVC_UPDATER_ENDPOINT"
+DVC_STAGE = "DVC_STAGE"

dvc/exceptions.py

@@ -1,4 +1,5 @@
 """Exceptions raised by the dvc."""
+
 import errno
 from typing import TYPE_CHECKING, Optional
 
@@ -132,11 +133,11 @@ class MoveNotDataSourceError(DvcException):
     def __init__(self, path):
         msg = (
             "move is not permitted for stages that are not data sources. "
-            "You need to either move '{path}' to a new location and edit "
-            "it by hand, or remove '{path}' and create a new one at the "
+            f"You need to either move {path!r} to a new location and edit "
+            f"it by hand, or remove {path!r} and create a new one at the "
             "desired location."
         )
-        super().__init__(msg.format(path=path))
+        super().__init__(msg)
 
 
 class NotDvcRepoError(DvcException):
@@ -270,9 +271,8 @@ class NoOutputInExternalRepoError(DvcException):
         from dvc.utils import relpath
 
         super().__init__(
-            "Output '{}' not found in target repository '{}'".format(
-                relpath(path, external_repo_path), external_repo_url
-            )
+            f"Output {relpath(path, external_repo_path)!r} "
+            f"not found in target repository '{external_repo_url}'"
         )
 
 

dvc/fs/__init__.py

@@ -1,10 +1,11 @@
+import glob
+from itertools import repeat
 from typing import Optional
 from urllib.parse import urlparse
 
-from dvc_http import HTTPFileSystem, HTTPSFileSystem  # noqa: F401
-
 from dvc.config import ConfigError as RepoConfigError
 from dvc.config_schema import SCHEMA, Invalid
+from dvc_http import HTTPFileSystem, HTTPSFileSystem  # noqa: F401
 
 # pylint: disable=unused-import
 from dvc_objects.fs import (  # noqa: F401
@@ -19,7 +20,7 @@ from dvc_objects.fs import (  # noqa: F401
     system,
     utils,
 )
-from dvc_objects.fs.base import AnyFSPath, FileSystem  # noqa: F401, TCH001
+from dvc_objects.fs.base import AnyFSPath, FileSystem  # noqa: F401, TC001
 from dvc_objects.fs.errors import (  # noqa: F401
     AuthError,
     ConfigError,
@@ -47,12 +48,24 @@ known_implementations.update(
 
 def download(
     fs: "FileSystem", fs_path: str, to: str, jobs: Optional[int] = None
-) -> int:
+) -> list[tuple[str, str, Optional[dict]]]:
     from dvc.scm import lfs_prefetch
 
     from .callbacks import TqdmCallback
 
     with TqdmCallback(desc=f"Downloading {fs.name(fs_path)}", unit="files") as cb:
+        if isinstance(fs, DVCFileSystem):
+            lfs_prefetch(
+                fs,
+                [
+                    f"{fs.normpath(glob.escape(fs_path))}/**"
+                    if fs.isdir(fs_path)
+                    else glob.escape(fs_path)
+                ],
+            )
+            if not glob.has_magic(fs_path):
+                return fs._get(fs_path, to, batch_size=jobs, callback=cb)
+
         # NOTE: We use dvc-objects generic.copy over fs.get since it makes file
         # download atomic and avoids fsspec glob/regex path expansion.
         if fs.isdir(fs_path):
@@ -61,7 +74,7 @@ def download(
             ]
             if not from_infos:
                 localfs.makedirs(to, exist_ok=True)
-                return 0
+                return []
             to_infos = [
                 localfs.join(to, *fs.relparts(info, fs_path)) for info in from_infos
             ]
@@ -69,12 +82,10 @@ def download(
             from_infos = [fs_path]
             to_infos = [to]
 
-        if isinstance(fs, DVCFileSystem):
-            lfs_prefetch(fs, from_infos)
         cb.set_size(len(from_infos))
         jobs = jobs or fs.jobs
         generic.copy(fs, from_infos, localfs, to_infos, callback=cb, batch_size=jobs)
-        return len(to_infos)
+        return list(zip(from_infos, to_infos, repeat(None)))
 
 
 def parse_external_url(url, fs_config=None, config=None):

dvc/fs/dvc.py

@@ -4,22 +4,28 @@ import ntpath
 import os
 import posixpath
 import threading
-from collections import deque
-from contextlib import ExitStack, suppress
+from collections import defaultdict, deque
+from contextlib import ExitStack, nullcontext, suppress
+from glob import has_magic
 from typing import TYPE_CHECKING, Any, Callable, Optional, Union
 
-from fsspec.spec import AbstractFileSystem
+from fsspec.spec import DEFAULT_CALLBACK, AbstractFileSystem
 from funcy import wrap_with
 
 from dvc.log import logger
-from dvc_objects.fs.base import FileSystem
+from dvc.utils.threadpool import ThreadPoolExecutor
+from dvc_objects.fs.base import AnyFSPath, FileSystem
 
 from .data import DataFileSystem
 
 if TYPE_CHECKING:
+    from contextlib import AbstractContextManager
+
     from dvc.repo import Repo
     from dvc.types import DictStrAny, StrPath
 
+    from .callbacks import Callback
+
 logger = logger.getChild(__name__)
 
 RepoFactory = Union[Callable[..., "Repo"], type["Repo"]]
@@ -77,11 +83,10 @@ class _DVCFileSystem(AbstractFileSystem):
     cachable = False
     root_marker = "/"
 
-    def __init__(  # noqa: PLR0913
+    def __init__(
         self,
-        url: Optional[str] = None,
+        repo: Union["Repo", os.PathLike[str], str, None] = None,
         rev: Optional[str] = None,
-        repo: Optional["Repo"] = None,
         subrepos: bool = False,
         repo_factory: Optional[RepoFactory] = None,
         fo: Optional[str] = None,
@@ -95,7 +100,8 @@ class _DVCFileSystem(AbstractFileSystem):
         """DVC + git-tracked files fs.
 
         Args:
-            path (str, optional): URL or path to a DVC/Git repository.
+            repo (str | os.PathLike[str] | Repo, optional): A url or a path to a DVC/Git
+                repository, or a `Repo` instance.
                 Defaults to a DVC repository in the current working directory.
                 Both HTTP and SSH protocols are supported for remote Git repos
                 (e.g. [user@]server:project.git).
@@ -105,7 +111,6 @@ class _DVCFileSystem(AbstractFileSystem):
                 In case of a local repository, if rev is unspecified, it will
                 default to the working directory.
                 If the repo is not a Git repo, this option is ignored.
-            repo (:obj:`Repo`, optional): `Repo` instance.
             subrepos (bool): traverse to subrepos.
                 By default, it ignores subrepos.
             repo_factory (callable): A function to initialize subrepo with.
@@ -130,13 +135,23 @@ class _DVCFileSystem(AbstractFileSystem):
             ...    rev="main",
             ... )
         """
+        from dvc.repo import Repo
+
+        # kwargs.get("url") is for maintaining backward compatibility
+        repo = repo or fo or kwargs.get("url")
+        if isinstance(repo, Repo):
+            self._repo: Optional[Repo] = repo
+            url = None
+        else:
+            self._repo = None
+            url = os.fspath(repo) if repo else None
+
         super().__init__()
-        self._repo = repo
         self._repo_factory = repo_factory
         self._traverse_subrepos = subrepos
         self._repo_stack = ExitStack()
         self._repo_kwargs = {
-            "url": url if url is not None else fo,
+            "url": url,
             "rev": rev,
             "subrepos": subrepos,
             "config": config,
@@ -360,7 +375,7 @@ class _DVCFileSystem(AbstractFileSystem):
         except FileNotFoundError:
             return False
 
-    def ls(self, path, detail=True, dvc_only=False, **kwargs):  # noqa: C901
+    def ls(self, path, detail=True, dvc_only=False, **kwargs):  # noqa: C901, PLR0912
         key = self._get_key_from_relative(path)
         repo, dvc_fs, subkey = self._get_subrepo_info(key)
 
@@ -429,7 +444,7 @@ class _DVCFileSystem(AbstractFileSystem):
         ignore_subrepos = kwargs.get("ignore_subrepos", True)
         return self._info(key, path, ignore_subrepos=ignore_subrepos)
 
-    def _info(  # noqa: C901, PLR0912
+    def _info(  # noqa: C901
         self, key, path, ignore_subrepos=True, check_ignored=True
     ):
         repo, dvc_fs, subkey = self._get_subrepo_info(key)
@@ -474,9 +489,128 @@ class _DVCFileSystem(AbstractFileSystem):
         info["name"] = path
         return info
 
+    def get(
+        self,
+        rpath,
+        lpath,
+        recursive=False,
+        callback=DEFAULT_CALLBACK,
+        maxdepth=None,
+        batch_size=None,
+        **kwargs,
+    ):
+        self._get(
+            rpath,
+            lpath,
+            recursive=recursive,
+            callback=callback,
+            maxdepth=maxdepth,
+            batch_size=batch_size,
+            **kwargs,
+        )
+
+    def _get(  # noqa: C901, PLR0912, PLR0915
+        self,
+        rpath,
+        lpath,
+        recursive=False,
+        callback=DEFAULT_CALLBACK,
+        maxdepth=None,
+        batch_size=None,
+        **kwargs,
+    ) -> list[tuple[str, str, Optional[dict]]]:
+        if (
+            isinstance(rpath, list)
+            or isinstance(lpath, list)
+            or has_magic(rpath)
+            or not self.exists(rpath)
+            or not recursive
+        ):
+            super().get(
+                rpath,
+                lpath,
+                recursive=recursive,
+                callback=callback,
+                maxdepth=maxdepth,
+                **kwargs,
+            )
+            return []
+
+        if os.path.isdir(lpath) or lpath.endswith(os.path.sep):
+            lpath = self.join(lpath, os.path.basename(rpath))
+
+        if self.isfile(rpath):
+            with callback.branched(rpath, lpath) as child:
+                self.get_file(rpath, lpath, callback=child, **kwargs)
+                return [(rpath, lpath, None)]
+
+        result: list[tuple[str, str, Optional[dict]]] = []
+        _dirs: list[str] = []
+        _files: dict[FileSystem, list[tuple[str, str, Optional[dict]]]]
+        _files = defaultdict(list)
+
+        for root, dirs, files in self.walk(rpath, maxdepth=maxdepth, detail=True):
+            if files:
+                callback.set_size((callback.size or 0) + len(files))
+
+            parts = self.relparts(root, rpath)
+            if parts in ((os.curdir,), ("",)):
+                parts = ()
+            dest_root = os.path.join(lpath, *parts)
+            if not maxdepth or len(parts) < maxdepth - 1:
+                _dirs.extend(f"{dest_root}{os.path.sep}{d}" for d in dirs)
+
+            key = self._get_key_from_relative(root)
+            _, dvc_fs, _ = self._get_subrepo_info(key)
+
+            for name, info in files.items():
+                dvc_info = info.get("dvc_info")
+                fs_info = info.get("fs_info")
+                if dvc_fs and dvc_info and not fs_info:
+                    fs = dvc_fs
+                    fs_path = dvc_info["name"]
+                else:
+                    fs = self.repo.fs
+                    fs_path = fs_info["name"]
+
+                src_path = f"{root}{self.sep}{name}"
+                dest_path = f"{dest_root}{os.path.sep}{name}"
+                _files[fs].append((fs_path, dest_path, dvc_info))
+                result.append((src_path, dest_path, info))
+
+        os.makedirs(lpath, exist_ok=True)
+        for d in _dirs:
+            os.makedirs(d, exist_ok=True)
+
+        def get_file(arg: tuple[FileSystem, tuple[str, str, Optional[dict]]]):
+            fs, (src, dest, info) = arg
+            kw = kwargs
+            if isinstance(fs, DataFileSystem):
+                kw = kw | {"info": info}
+            with callback.branched(src, dest) as child:
+                fs.get_file(src, dest, callback=child, **kw)
+
+        if batch_size == 1:
+            ctx: AbstractContextManager = nullcontext()
+            map_fn: Callable = map
+        else:
+            ctx = ThreadPoolExecutor(max_workers=batch_size)
+            map_fn = ctx.imap_unordered
+
+        with ctx:
+            it = ((fs, f) for fs, files in _files.items() for f in files)
+            deque(callback.wrap(map_fn(get_file, it)), maxlen=0)
+        return result
+
     def get_file(self, rpath, lpath, **kwargs):
+        dvc_info = kwargs.pop("info", {}).pop("dvc_info", None)
         key = self._get_key_from_relative(rpath)
         fs_path = self._from_key(key)
+        dirpath = os.path.dirname(lpath)
+        if dirpath:
+            # makedirs raises error if the string is empty
+            os.makedirs(dirpath, exist_ok=True)
+
         try:
             return self.repo.fs.get_file(fs_path, lpath, **kwargs)
         except FileNotFoundError:
@@ -485,7 +619,7 @@ class _DVCFileSystem(AbstractFileSystem):
                 raise
 
         dvc_path = _get_dvc_path(dvc_fs, subkey)
-        return dvc_fs.get_file(dvc_path, lpath, **kwargs)
+        return dvc_fs.get_file(dvc_path, lpath, info=dvc_info, **kwargs)
 
     def du(self, path, total=True, maxdepth=None, withdirs=False, **kwargs):
         if maxdepth is not None:
@@ -553,6 +687,45 @@ class DVCFileSystem(FileSystem):
     def getcwd(self):
         return self.fs.getcwd()
 
+    def _get(
+        self,
+        from_info: Union[AnyFSPath, list[AnyFSPath]],
+        to_info: Union[AnyFSPath, list[AnyFSPath]],
+        callback: "Callback" = DEFAULT_CALLBACK,
+        recursive: bool = False,
+        batch_size: Optional[int] = None,
+        **kwargs,
+    ) -> list[tuple[str, str, Optional[dict]]]:
+        # FileSystem.get is non-recursive by default if arguments are lists
+        # otherwise, it's recursive.
+        recursive = not (isinstance(from_info, list) and isinstance(to_info, list))
+        return self.fs._get(
+            from_info,
+            to_info,
+            callback=callback,
+            recursive=recursive,
+            batch_size=batch_size,
+            **kwargs,
+        )
+
+    def get(
+        self,
+        from_info: Union[AnyFSPath, list[AnyFSPath]],
+        to_info: Union[AnyFSPath, list[AnyFSPath]],
+        callback: "Callback" = DEFAULT_CALLBACK,
+        recursive: bool = False,
+        batch_size: Optional[int] = None,
+        **kwargs,
+    ) -> None:
+        self._get(
+            from_info,
+            to_info,
+            callback=callback,
+            batch_size=batch_size,
+            recursive=recursive,
+            **kwargs,
+        )
+
     @property
     def fsid(self) -> str:
         return self.fs.fsid
@@ -569,9 +742,10 @@ class DVCFileSystem(FileSystem):
         return self.fs.repo_url
 
     def from_os_path(self, path: str) -> str:
-        if os.path.isabs(path):
+        if os.path.isabs(path) or (
+            os.name == "nt" and posixpath.isabs(path) and ntpath.sep not in path
+        ):
             path = os.path.relpath(path, self.repo.root_dir)
-
         return as_posix(path)
 
     def close(self):

dvc/fs/dvc_path.py

@@ -0,0 +1,54 @@
+"""UPath implementation for DVCFileSystem.
+
+This provides a `pathlib.Path` like interface to
+work with DVCFileSystem.
+
+Examples
+--------
+
+>>> from upath import UPath
+
+>>> local = UPath("dvc://path/to/local/repo")
+>>> https = UPath("dvc+https://github.com/iterative/example-get-started", rev="main")
+>>> ssh = UPath("dvc+ssh://[email protected]:iterative/example-get-started.git")
+"""
+
+from urllib.parse import urlsplit
+
+from upath import UPath
+
+
+class DVCPath(UPath):
+    @classmethod
+    def _transform_init_args(cls, args, protocol, storage_options):
+        if not args:
+            args = ("/",)
+        elif (
+            args
+            and "url" not in storage_options
+            and protocol in {"dvc+http", "dvc+https", "dvc+ssh"}
+        ):
+            url, *rest = args
+            url = urlsplit(str(url))
+            proto = protocol.split("+")[1]
+            if proto == "ssh":
+                base_url = url.netloc + url.path
+            else:
+                base_url = url._replace(scheme=proto).geturl()
+            storage_options["url"] = base_url
+            # Assume the given path is a root url
+            args = ("/", *rest)
+        return super()._transform_init_args(args, "dvc", storage_options)
+
+    def __str__(self):
+        s = super().__str__()
+        if url := self.storage_options.get("url"):
+            return s.replace("dvc://", f"dvc+{url}", 1)
+        return s
+
+    def with_segments(self, *pathsegments):
+        obj = super().with_segments(*pathsegments)
+        # cache filesystem, as dvcfs does not cache filesystem
+        # caveat: any joinpath operation will instantiate filesystem
+        obj._fs_cached = self.fs
+        return obj

dvc/info.py

@@ -48,7 +48,10 @@ def get_dvc_info():
                 info.append("Cache types: " + error_link("no-dvc-cache"))
 
             info.append(f"Caches: {_get_caches(repo.cache)}")
-            info.append(f"Remotes: {_get_remotes(repo.config)}")
+
+            configured_remotes = _get_remotes(repo.config)
+            remotes = ", ".join(configured_remotes) if configured_remotes else None
+            info.append(f"Remotes: {remotes}")
 
             root_directory = repo.root_dir
             fs_root = _get_fs_type(os.path.abspath(root_directory))
@@ -75,12 +78,10 @@ def _get_caches(cache):
 
 
 def _get_remotes(config):
-    schemes = (
+    return [
         get_fs_cls(get_fs_config(config, name=remote)).protocol
         for remote in config["remote"]
-    )
-
-    return ", ".join(schemes) or "None"
+    ]
 
 
 def _get_linktype_support_info(repo):
@@ -123,7 +124,7 @@ def _get_supported_remotes():
         if not fs_cls.get_missing_deps():
             dependencies = []
             for requirement in fs_cls.REQUIRES:
-                dependencies.append(
+                dependencies.append(  # noqa: PERF401
                     f"{requirement} = {importlib_metadata.version(requirement)}"
                 )
 

dvc/lock.py

@@ -2,12 +2,13 @@
 
 import hashlib
 import os
+import time
 from abc import ABC, abstractmethod
 from datetime import timedelta
+from typing import Optional, Union
 
 import flufl.lock
 import zc.lockfile
-from funcy import retry
 
 from dvc.exceptions import DvcException
 from dvc.progress import Tqdm
@@ -87,9 +88,10 @@ class Lock(LockBase):
     Uses zc.lockfile as backend.
     """
 
-    def __init__(self, lockfile, friendly=False, **kwargs):
+    def __init__(self, lockfile, friendly=False, wait=False, **kwargs):
         super().__init__(lockfile)
         self._friendly = friendly
+        self._wait = wait
         self._lock = None
         self._lock_failed = False
 
@@ -100,23 +102,38 @@ class Lock(LockBase):
     def _do_lock(self):
         try:
             self._lock_failed = False
-            with Tqdm(
-                bar_format="{desc}",
-                disable=not self._friendly,
-                desc="If DVC froze, see `hardlink_lock` in {}".format(
-                    format_link("https://man.dvc.org/config#core")
-                ),
-            ):
-                self._lock = zc.lockfile.LockFile(self._lockfile)
+            self._lock = zc.lockfile.LockFile(self._lockfile)
         except zc.lockfile.LockError:
             self._lock_failed = True
             raise LockError(FAILED_TO_LOCK_MESSAGE)  # noqa: B904
 
     def lock(self):
-        retries = 6
-        delay = DEFAULT_TIMEOUT / retries
-        lock_retry = retry(retries, LockError, timeout=delay)(self._do_lock)
-        lock_retry()
+        """Acquire the lock, either waiting forever, or after default_retries."""
+        default_retries = 6
+        delay = DEFAULT_TIMEOUT / default_retries
+        attempts = 0
+
+        max_retries = float("inf") if self._wait else default_retries
+
+        with Tqdm(
+            bar_format="{desc}",
+            disable=not self._friendly,
+            desc="Waiting to acquire lock. "
+            "If DVC froze, see `hardlink_lock` in {}.".format(
+                format_link("https://man.dvc.org/config#core")
+            ),
+        ) as pbar:
+            while True:
+                try:
+                    self._do_lock()
+                    return
+                except LockError:
+                    attempts += 1
+                    if attempts > max_retries:
+                        raise
+                    time.sleep(delay)
+                finally:
+                    pbar.update()
 
     def unlock(self):
         if self._lock_failed:
@@ -149,7 +166,7 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
         tmp_dir (str): a directory to store claim files.
     """
 
-    def __init__(self, lockfile, tmp_dir=None, **kwargs):
+    def __init__(self, lockfile, tmp_dir=None, wait=False, **kwargs):
         import socket
 
         self._tmp_dir = tmp_dir
@@ -172,11 +189,21 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
         self._separator = flufl.lock.SEP
         self._set_claimfile()
         self._owned = True
+        self._wait = wait
         self._retry_errnos = []
+        self._friendly = kwargs.get("friendly", False)
 
-    def lock(self):
+    def lock(self, timeout: Optional[Union[timedelta, int]] = None):
         try:
-            super().lock(timedelta(seconds=DEFAULT_TIMEOUT))
+            if not self._wait:
+                timeout = timeout or timedelta(seconds=DEFAULT_TIMEOUT)
+
+            with Tqdm(
+                bar_format="{desc}",
+                disable=not (self._wait and self._friendly),
+                desc="Waiting to acquire lock",
+            ):
+                super().lock(timeout)
         except flufl.lock.TimeOutError:
             raise LockError(FAILED_TO_LOCK_MESSAGE)  # noqa: B904
 
@@ -185,11 +212,11 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
 
         if self._tmp_dir is not None:
             # Under Windows file path length is limited so we hash it
-            hasher = hashlib.md5(self._claimfile.encode())  # noqa: S324
+            hasher = hashlib.md5(self._claimfile.encode(), usedforsecurity=False)
             filename = hasher.hexdigest()
             self._claimfile = os.path.join(self._tmp_dir, filename + ".lock")
 
 
-def make_lock(lockfile, tmp_dir=None, friendly=False, hardlink_lock=False):
+def make_lock(lockfile, tmp_dir=None, friendly=False, hardlink_lock=False, wait=False):
     cls = HardlinkLock if hardlink_lock else Lock
-    return cls(lockfile, tmp_dir=tmp_dir, friendly=friendly)
+    return cls(lockfile, tmp_dir=tmp_dir, friendly=friendly, wait=wait)

dvc/logger.py

@@ -5,6 +5,8 @@ import logging.config
 import logging.handlers
 import os
 import sys
+from collections.abc import Iterator
+from contextlib import contextmanager
 from typing import ClassVar
 
 import colorama
@@ -36,7 +38,7 @@ def add_logging_level(level_name, level_num, method_name=None):
             self._log(level_num, message, args, **kwargs)
 
     def log_to_root(message, *args, **kwargs):
-        logging.log(level_num, message, *args, **kwargs)
+        logging.log(level_num, message, *args, **kwargs)  # noqa: LOG015
 
     # getLevelName resolves the numeric log level if already defined,
     # otherwise returns a string
@@ -174,9 +176,19 @@ def _iter_causes(exc):
         exc = exc.__cause__
 
 
-def set_loggers_level(level: int = logging.INFO) -> None:
+@contextmanager
+def set_loggers_level(level: int = logging.INFO) -> Iterator[None]:
+    ret: dict[logging.Logger, int] = {}
     for name in ["dvc", "dvc_objects", "dvc_data"]:
-        logging.getLogger(name).setLevel(level)
+        _logger = logging.getLogger(name)
+        ret[_logger] = _logger.getEffectiveLevel()
+        _logger.setLevel(level)
+
+    try:
+        yield
+    finally:
+        for _logger, old_lvl in ret.items():
+            _logger.setLevel(old_lvl)
 
 
 def setup(level: int = logging.INFO, log_colors: bool = True) -> None:

dvc/output.py

@@ -390,7 +390,7 @@ class Output:
         self.can_push = push
 
         self.fs_path = self._parse_path(self.fs, fs_path)
-        self.obj: Optional["HashFile"] = None
+        self.obj: Optional[HashFile] = None
 
         self.remote = remote
 
@@ -431,7 +431,7 @@ class Output:
             self.meta.nfiles = len(self.files)
             self.meta.size = sum(filter(None, (f.get("size") for f in self.files)))
             self.meta.remote = first(f.get("remote") for f in self.files)
-        elif self.meta.nfiles or self.hash_info and self.hash_info.isdir:
+        elif self.meta.nfiles or (self.hash_info and self.hash_info.isdir):
             self.meta.isdir = True
             if not self.hash_info and self.hash_name not in ("md5", "md5-dos2unix"):
                 md5 = getattr(self.meta, "md5", None)
@@ -542,7 +542,9 @@ class Output:
             desc=f"Collecting files and computing hashes in {self}",
             disable=no_progress_bar,
         ) as pb:
-            return build(*args, callback=pb.as_callback(), **kwargs)
+            kwargs["callback"] = pb.as_callback()
+            kwargs.setdefault("checksum_jobs", self.fs.hash_jobs)
+            return build(*args, **kwargs)
 
     def _get_hash_meta(self):
         if self.use_cache:
@@ -730,6 +732,7 @@ class Output:
         from dvc_data.hashfile.checkout import LinkError, PromptError
 
         kwargs.setdefault("ignore", self.dvcignore)
+        kwargs.setdefault("checksum_jobs", self.fs.hash_jobs)
         try:
             return checkout(*args, **kwargs)
         except PromptError as exc:
@@ -749,11 +752,9 @@ class Output:
             granular = (
                 self.is_dir_checksum and filter_info and filter_info != self.fs_path
             )
-            # NOTE: trying to use hardlink during transfer only if we will be
-            # relinking later
-            hardlink = relink
+            hardlink = relink and next(iter(self.cache.cache_types), None) == "hardlink"
             if granular:
-                obj = self._commit_granular_dir(filter_info, hardlink)
+                obj = self._commit_granular_dir(filter_info, hardlink=hardlink)
             else:
                 staging, _, obj = self._build(
                     self.cache,
@@ -786,6 +787,7 @@ class Output:
                         state=self.repo.state,
                         prompt=prompt.confirm,
                         progress_callback=cb,
+                        old=obj,
                     )
                 self.set_exec()
 
@@ -814,7 +816,7 @@ class Output:
 
         ret: dict[str, Any] = {}
         with_files = (
-            (not self.IS_DEPENDENCY or self.stage.is_import)
+            (not self.IS_DEPENDENCY or kwargs.get("datasets") or self.stage.is_import)
             and self.hash_info.isdir
             and (kwargs.get("with_files") or self.files is not None)
         )
@@ -902,7 +904,7 @@ class Output:
     def get_obj(
         self, filter_info: Optional[str] = None, **kwargs
     ) -> Optional["HashFile"]:
-        obj: Optional["HashFile"] = None
+        obj: Optional[HashFile] = None
         if self.obj:
             obj = self.obj
         elif self.files:
@@ -1158,7 +1160,7 @@ class Output:
             logger.warning(msg)
             return {}
 
-        obj: Optional["HashFile"]
+        obj: Optional[HashFile]
         if self.is_dir_checksum:
             obj = self._collect_used_dir_cache(**kwargs)
         else:
@@ -1353,7 +1355,7 @@ class Output:
         cache = self.cache if self.use_cache else self.local_cache
         assert isinstance(cache, HashFileDB)
 
-        new: "HashFile"
+        new: HashFile
         try:
             assert self.hash_name
             staging, meta, obj = self._build(
@@ -1365,7 +1367,7 @@ class Output:
                 dry_run=not self.use_cache,
             )
         except FileNotFoundError as exc:
-            if self.fs_path == path:
+            if not self.exists:
                 raise self.DoesNotExistError(self) from exc
             if not self.is_dir_checksum:
                 raise
@@ -1398,12 +1400,14 @@ class Output:
 
         assert staging
         assert obj.hash_info
+
+        hardlink = relink and next(iter(self.cache.cache_types), None) == "hardlink"
         with TqdmCallback(desc=f"Adding {self} to cache", unit="file") as cb:
             otransfer(
                 staging,
                 self.cache,
                 {obj.hash_info},
-                hardlink=relink,
+                hardlink=hardlink,
                 shallow=False,
                 callback=cb,
             )
@@ -1421,6 +1425,7 @@ class Output:
                     state=self.repo.state,
                     prompt=prompt.confirm,
                     progress_callback=callback,
+                    old=obj,
                 )
             self.set_exec()
         return obj

dvc/parsing/__init__.py

@@ -21,6 +21,7 @@ from .context import (
     VarsAlreadyLoaded,
 )
 from .interpolate import (
+    check_expression,
     check_recursive_parse_errors,
     is_interpolated_string,
     recurse,
@@ -38,10 +39,16 @@ if TYPE_CHECKING:
 
 logger = logger.getChild(__name__)
 
-STAGES_KWD = "stages"
 VARS_KWD = "vars"
 WDIR_KWD = "wdir"
+
+ARTIFACTS_KWD = "artifacts"
+DATASETS_KWD = "datasets"
+METRICS_KWD = "metrics"
 PARAMS_KWD = "params"
+PLOTS_KWD = "plots"
+STAGES_KWD = "stages"
+
 FOREACH_KWD = "foreach"
 MATRIX_KWD = "matrix"
 DO_KWD = "do"
@@ -163,6 +170,27 @@ class DataResolver:
             for name, definition in stages_data.items()
         }
 
+        self.artifacts = [
+            ArtifactDefinition(self, self.context, name, definition, ARTIFACTS_KWD)
+            for name, definition in d.get(ARTIFACTS_KWD, {}).items()
+        ]
+        self.datasets = [
+            TopDefinition(self, self.context, str(i), definition, DATASETS_KWD)
+            for i, definition in enumerate(d.get(DATASETS_KWD, []))
+        ]
+        self.metrics = [
+            TopDefinition(self, self.context, str(i), definition, METRICS_KWD)
+            for i, definition in enumerate(d.get(METRICS_KWD, []))
+        ]
+        self.params = [
+            TopDefinition(self, self.context, str(i), definition, PARAMS_KWD)
+            for i, definition in enumerate(d.get(PARAMS_KWD, []))
+        ]
+        self.plots = [
+            TopDefinition(self, self.context, str(i), definition, PLOTS_KWD)
+            for i, definition in enumerate(d.get(PLOTS_KWD, []))
+        ]
+
     def resolve_one(self, name: str):
         group, key = split_group_name(name)
 
@@ -186,6 +214,27 @@ class DataResolver:
         logger.trace("Resolved dvc.yaml:\n%s", data)
         return {STAGES_KWD: data}
 
+    # Top-level sections are eagerly evaluated, whereas stages are lazily evaluated,
+    # one-by-one.
+
+    def resolve_artifacts(self) -> dict[str, Optional[dict[str, Any]]]:
+        d: dict[str, Optional[dict[str, Any]]] = {}
+        for item in self.artifacts:
+            d.update(item.resolve())
+        return d
+
+    def resolve_datasets(self) -> list[dict[str, Any]]:
+        return [item.resolve() for item in self.datasets]
+
+    def resolve_metrics(self) -> list[str]:
+        return [item.resolve() for item in self.metrics]
+
+    def resolve_params(self) -> list[str]:
+        return [item.resolve() for item in self.params]
+
+    def resolve_plots(self) -> list[Any]:
+        return [item.resolve() for item in self.plots]
+
     def has_key(self, key: str):
         return self._has_group_and_key(*split_group_name(key))
 
@@ -514,10 +563,10 @@ class MatrixDefinition:
         iterable = self.resolved_iterable
         assert isinstance(iterable, Mapping)
 
-        ret: dict[str, "DictStrAny"] = {}
+        ret: dict[str, DictStrAny] = {}
         matrix = {key: enumerate(v) for key, v in iterable.items()}
         for combination in product(*matrix.values()):
-            d: "DictStrAny" = {}
+            d: DictStrAny = {}
             fragments: list[str] = []
             for k, (i, v) in zip(matrix.keys(), combination):
                 d[k] = v
@@ -565,3 +614,43 @@ class MatrixDefinition:
                 return entry.resolve_stage(skip_checks=True)
             except ContextError as exc:
                 format_and_raise(exc, f"stage '{generated}'", self.relpath)
+
+
+class TopDefinition:
+    def __init__(
+        self,
+        resolver: DataResolver,
+        context: Context,
+        name: str,
+        definition: "Any",
+        where: str,
+    ):
+        self.resolver = resolver
+        self.context = context
+        self.name = name
+        self.definition = definition
+        self.where = where
+        self.relpath = self.resolver.relpath
+
+    def resolve(self):
+        try:
+            check_recursive_parse_errors(self.definition)
+            return self.context.resolve(self.definition)
+        except (ParseError, ContextError) as exc:
+            format_and_raise(exc, f"'{self.where}.{self.name}'", self.relpath)
+
+
+class ArtifactDefinition(TopDefinition):
+    def resolve(self) -> dict[str, Optional[dict[str, Any]]]:
+        try:
+            check_expression(self.name)
+            name = self.context.resolve(self.name)
+            if not isinstance(name, str):
+                typ = type(name).__name__
+                raise ResolveError(
+                    f"failed to resolve '{self.where}.{self.name}'"
+                    f" in '{self.relpath}': expected str, got " + typ
+                )
+        except (ParseError, ContextError) as exc:
+            format_and_raise(exc, f"'{self.where}.{self.name}'", self.relpath)
+        return {name: super().resolve()}

dvc/parsing/context.py

@@ -105,7 +105,7 @@ class Meta:
 
     @staticmethod
     def update_path(meta: "Meta", path: Union[str, int]):
-        dpaths = meta.dpaths[:] + [str(path)]
+        dpaths = [*meta.dpaths, str(path)]
         return replace(meta, dpaths=dpaths)
 
     def __str__(self):
@@ -155,7 +155,7 @@ class Value(Node):
 PRIMITIVES = (int, float, str, bytes, bool)
 
 
-class Container(Node, ABC):
+class Container(Node, ABC):  # noqa: PLW1641
     meta: Meta
     data: Union[list, dict]
     _key_transform = staticmethod(identity)
@@ -176,8 +176,9 @@ class Container(Node, ABC):
             return value
         if isinstance(value, (list, dict)):
             assert meta
-            container = CtxDict if isinstance(value, dict) else CtxList
-            return container(value, meta=meta)
+            if isinstance(value, dict):
+                return CtxDict(value, meta=meta)
+            return CtxList(value, meta=meta)
         msg = f"Unsupported value of type '{type(value).__name__}' in '{meta}'"
         raise TypeError(msg)
 

dvc/pathspec_math.py

@@ -27,11 +27,9 @@ def _remove_slash(rule):
 
 def _match_all_level(rule):
     if rule[:-1].find("/") >= 0 and not rule.startswith("**/"):
-        if rule.startswith("/"):
-            rule = rule[1:]
+        rule = rule.removeprefix("/")
         return False, rule
-    if rule.startswith("**/"):
-        rule = rule[3:]
+    rule = rule.removeprefix("**/")
     return True, rule
 
 

dvc/progress.py

@@ -1,4 +1,5 @@
 """Manages progress bars for DVC repo."""
+
 import logging
 import sys
 from threading import RLock
@@ -40,7 +41,7 @@ class Tqdm(tqdm):
         "miniters": 1,
     }
 
-    def __init__(  # noqa: PLR0913
+    def __init__(
         self,
         iterable=None,
         disable=None,

dvc/render/converter/vega.py

@@ -23,7 +23,7 @@ def _lists(blob: Union[dict, list]) -> Iterable[list]:
     if isinstance(blob, list):
         yield blob
     else:
-        for _, value in blob.items():
+        for value in blob.values():
             if isinstance(value, dict):
                 yield from _lists(value)
             elif isinstance(value, list):

dvc/repo/__init__.py

@@ -13,7 +13,6 @@ from dvc.exceptions import (
 )
 from dvc.ignore import DvcIgnoreFilter
 from dvc.log import logger
-from dvc.utils import as_posix
 from dvc.utils.objects import cached_property
 
 if TYPE_CHECKING:
@@ -145,12 +144,14 @@ class Repo:
         scm: Optional[Union["Git", "NoSCM"]] = None,
         remote: Optional[str] = None,
         remote_config: Optional["DictStrAny"] = None,
+        _wait_for_lock: bool = False,
     ):
         from dvc.cachemgr import CacheManager
         from dvc.data_cloud import DataCloud
-        from dvc.fs import GitFileSystem, LocalFileSystem, localfs
+        from dvc.fs import GitFileSystem, LocalFileSystem
         from dvc.lock import LockNoop, make_lock
         from dvc.repo.artifacts import Artifacts
+        from dvc.repo.datasets import Datasets
         from dvc.repo.metrics import Metrics
         from dvc.repo.params import Params
         from dvc.repo.plots import Plots
@@ -161,12 +162,13 @@ class Repo:
 
         self.url = url
         self._fs_conf = {"repo_factory": repo_factory}
-        self._fs = fs or localfs
+        self._fs = fs or LocalFileSystem()
         self._scm = scm
         self._config = config
         self._remote = remote
         self._remote_config = remote_config
-        self._data_index = None
+        self._data_index: Optional[DataIndex] = None
+        self._wait_for_lock = _wait_for_lock
 
         if rev and not fs:
             self._scm = scm = SCM(root_dir or os.curdir)
@@ -184,12 +186,12 @@ class Repo:
         # used by DVCFileSystem to determine if it should traverse subrepos
         self.subrepos = subrepos
 
-        self.cloud: "DataCloud" = DataCloud(self)
-        self.stage: "StageLoad" = StageLoad(self)
+        self.cloud: DataCloud = DataCloud(self)
+        self.stage: StageLoad = StageLoad(self)
 
-        self.lock: "LockBase"
+        self.lock: LockBase
         self.cache: CacheManager
-        self.state: "StateBase"
+        self.state: StateBase
         if isinstance(self.fs, GitFileSystem) or not self.dvc_dir:
             self.lock = LockNoop()
             self.state = StateNoop()
@@ -204,8 +206,14 @@ class Repo:
                     tmp_dir=self.tmp_dir,
                     hardlink_lock=self.config["core"].get("hardlink_lock", False),
                     friendly=True,
+                    wait=self._wait_for_lock,
                 )
                 os.makedirs(self.site_cache_dir, exist_ok=True)
+                if not fs and (
+                    checksum_jobs := self.config["core"].get("checksum_jobs")
+                ):
+                    self.fs.hash_jobs = checksum_jobs
+
                 self.state = State(self.root_dir, self.site_cache_dir, self.dvcignore)
             else:
                 self.lock = LockNoop()
@@ -221,6 +229,7 @@ class Repo:
         self.plots: Plots = Plots(self)
         self.params: Params = Params(self)
         self.artifacts: Artifacts = Artifacts(self)
+        self.datasets: Datasets = Datasets(self)
 
         self.stage_collection_error_handler: Optional[
             Callable[[str, Exception], None]
@@ -351,16 +360,6 @@ class Repo:
         # fs.
         self._reset()
 
-    @property
-    def subrepo_relpath(self) -> str:
-        from dvc.fs import GitFileSystem
-
-        scm_root_dir = "/" if isinstance(self.fs, GitFileSystem) else self.scm.root_dir
-
-        relpath = as_posix(self.fs.relpath(self.root_dir, scm_root_dir))
-
-        return "" if relpath == "." else relpath
-
     @property
     def data_index(self) -> "DataIndex":
         from dvc_data.index import DataIndex
@@ -560,11 +559,7 @@ class Repo:
 
             if out.protocol == "local" and match(fs_path, out.fs_path):
                 return True
-
-            if recursive and out.fs.isin(out.fs_path, fs_path):
-                return True
-
-            return False
+            return recursive and out.fs.isin(out.fs_path, fs_path)
 
         matched = list(filter(func, outs))
         if not matched:
@@ -617,7 +612,7 @@ class Repo:
         from dvc.fs import GitFileSystem
         from dvc.version import version_tuple
 
-        cache_dir = self.config["core"].get("site_cache_dir") or site_cache_dir()
+        cache_dir = site_cache_dir(self.config["core"].get("site_cache_dir"))
 
         subdir = None
         if isinstance(self.fs, GitFileSystem):
@@ -646,10 +641,11 @@ class Repo:
         # that just happened to be at the same path as old deleted ones.
         btime = self._btime or getattr(os.stat(root_dir), "st_birthtime", None)
 
-        md5 = hashlib.md5(  # noqa: S324
+        md5 = hashlib.md5(
             str(
                 (root_dir, subdir, btime, getpass.getuser(), version_tuple[0], salt)
-            ).encode()
+            ).encode(),
+            usedforsecurity=False,
         )
         repo_token = md5.hexdigest()
         return os.path.join(repos_dir, repo_token)
@@ -664,6 +660,7 @@ class Repo:
 
     def _reset(self):
         self.scm._reset()
+        self.datasets._reset()
         self.state.close()
         if "dvcfs" in self.__dict__:
             self.dvcfs.close()

dvc/repo/add.py

@@ -171,11 +171,16 @@ def _add_transfer(
     stage.dump()
 
 
-def _add(stage: "Stage", source: Optional[str] = None, no_commit: bool = False) -> None:
+def _add(
+    stage: "Stage",
+    source: Optional[str] = None,
+    no_commit: bool = False,
+    relink: bool = True,
+) -> None:
     out = stage.outs[0]
     path = out.fs.abspath(source) if source else None
     try:
-        stage.add_outs(path, no_commit=no_commit)
+        stage.add_outs(path, no_commit=no_commit, relink=relink)
     except CacheLinkError:
         stage.dump()
         raise
@@ -194,6 +199,7 @@ def add(
     to_remote: bool = False,
     remote_jobs: Optional[int] = None,
     force: bool = False,
+    relink: bool = True,
 ) -> list["Stage"]:
     add_targets = find_targets(targets, glob=glob)
     if not add_targets:
@@ -224,7 +230,12 @@ def add(
     with warn_link_failures() as link_failures:
         for source, (stage, output_exists) in progress_iter(stages_with_targets):
             try:
-                _add(stage, source if output_exists else None, no_commit=no_commit)
+                _add(
+                    stage,
+                    source if output_exists else None,
+                    no_commit=no_commit,
+                    relink=relink,
+                )
             except CacheLinkError:
                 link_failures.append(stage.relpath)
     return stages

dvc/repo/artifacts.py

@@ -12,7 +12,7 @@ from dvc.exceptions import (
     InvalidArgumentError,
 )
 from dvc.log import logger
-from dvc.utils import relpath, resolve_output
+from dvc.utils import as_posix, relpath, resolve_output
 from dvc.utils.objects import cached_property
 from dvc.utils.serialize import modify_yaml
 
@@ -99,7 +99,7 @@ class Artifacts:
         """Read artifacts from dvc.yaml."""
         artifacts: dict[str, dict[str, Artifact]] = {}
         for dvcfile, dvcfile_artifacts in self.repo.index._artifacts.items():
-            dvcyaml = relpath(dvcfile, self.repo.root_dir)
+            dvcyaml = self.repo.fs.relpath(dvcfile, self.repo.root_dir)
             artifacts[dvcyaml] = {}
             for name, value in dvcfile_artifacts.items():
                 try:
@@ -137,18 +137,16 @@ class Artifacts:
 
         assert not (version and stage)
         name = _reformat_name(name)
-        tags: list["GitTag"] = find(
-            name=name, version=version, stage=stage, scm=self.scm
-        )
+        tags: list[GitTag] = find(name=name, version=version, stage=stage, scm=self.scm)
         if not tags:
             raise ArtifactNotFoundError(name, version=version, stage=stage)
         if version or stage:
             return tags[-1].target
-        gto_tags: list["GTOTag"] = sort_versions(parse_tag(tag) for tag in tags)
+        gto_tags: list[GTOTag] = sort_versions(parse_tag(tag) for tag in tags)
         return gto_tags[0].tag.target
 
-    def get_path(self, name: str):
-        """Return repo fspath for the given artifact."""
+    @classmethod
+    def parse_path(cls, name: str) -> tuple[Optional[str], str]:
         from gto.constants import SEPARATOR_IN_NAME, fullname_re
 
         name = _reformat_name(name)
@@ -158,13 +156,37 @@ class Artifacts:
         dirname = m.group("dirname")
         if dirname:
             dirname = dirname.rstrip(SEPARATOR_IN_NAME)
-        dvcyaml = os.path.join(dirname, PROJECT_FILE) if dirname else PROJECT_FILE
-        artifact_name = m.group("name")
+
+        return dirname, m.group("name")
+
+    def get_path(self, name: str):
+        """Return fspath for the given artifact relative to the git root."""
+        from dvc.fs import GitFileSystem
+
+        dirname, artifact_name = self.parse_path(name)
+        # `name`/`dirname` are expected to be a git root relative.
+        # We convert it to dvc-root relative path so that we can read artifacts
+        # from dvc.yaml file.
+        # But we return dirname intact, as we want to return a git-root relative path.
+        # This is useful when reading from `dvcfs` from remote.
+        fs = self.repo.fs
+        assert self.scm
+        if isinstance(fs, GitFileSystem):
+            scm_root = fs.root_marker
+        else:
+            scm_root = self.scm.root_dir
+
+        dirparts = posixpath.normpath(dirname).split(posixpath.sep) if dirname else ()
+        abspath = fs.join(scm_root, *dirparts, PROJECT_FILE)
+        rela = fs.relpath(abspath, self.repo.root_dir)
         try:
-            artifact = self.read()[dvcyaml][artifact_name]
+            artifact = self.read()[rela][artifact_name]
         except KeyError as exc:
             raise ArtifactNotFoundError(name) from exc
-        return os.path.join(dirname, artifact.path) if dirname else artifact.path
+
+        path = posixpath.join(dirname or "", artifact.path)
+        parts = posixpath.normpath(path).split(posixpath.sep)
+        return os.path.join(*parts)
 
     def download(
         self,
@@ -177,15 +199,28 @@ class Artifacts:
     ) -> tuple[int, str]:
         """Download the specified artifact."""
         from dvc.fs import download as fs_download
+        from dvc.repo import Repo
 
         logger.debug("Trying to download artifact '%s' via DVC", name)
         rev = self.get_rev(name, version=version, stage=stage)
+
+        dirname, _ = self.parse_path(name)
         with self.repo.switch(rev):
-            path = self.get_path(name)
+            root = self.repo.fs.root_marker
+            _dirname = self.repo.fs.join(root, dirname) if dirname else root
+            with Repo(_dirname, fs=self.repo.fs, scm=self.repo.scm) as r:
+                path = r.artifacts.get_path(name)
+                path = self.repo.fs.join(root, as_posix(path))
+                path = self.repo.fs.relpath(path, self.repo.root_dir)
+                # when the `repo` is a subrepo, the path `/subrepo/myart.pkl` for dvcfs
+                # should be translated as `/myart.pkl`,
+                # i.e. relative to the root of the subrepo
+                path = self.repo.fs.join(root, path)
+                path = self.repo.fs.normpath(path)
+
             out = resolve_output(path, out, force=force)
             fs = self.repo.dvcfs
-            fs_path = fs.from_os_path(path)
-            count = fs_download(fs, fs_path, os.path.abspath(out), jobs=jobs)
+            count = len(fs_download(fs, path, os.path.abspath(out), jobs=jobs))
         return count, out
 
     @staticmethod
@@ -200,10 +235,9 @@ class Artifacts:
         dvc_studio_config: Optional[dict[str, Any]] = None,
         **kwargs,
     ) -> tuple[int, str]:
-        from dvc_studio_client.model_registry import get_download_uris
-
         from dvc.fs import HTTPFileSystem, generic, localfs
         from dvc.fs.callbacks import TqdmCallback
+        from dvc_studio_client.model_registry import get_download_uris
 
         logger.debug("Trying to download artifact '%s' via studio", name)
         out = out or os.getcwd()
@@ -231,7 +265,7 @@ class Artifacts:
                 from_infos.append(url)
         except DvcException:
             raise
-        except Exception as exc:  # noqa: BLE001
+        except Exception as exc:
             raise DvcException(
                 f"Failed to download artifact '{name}' via Studio"
             ) from exc
@@ -249,7 +283,7 @@ class Artifacts:
         return len(to_infos), relpath(localfs.commonpath(to_infos))
 
     @classmethod
-    def get(  # noqa: PLR0913
+    def get(
         cls,
         url: str,
         name: str,
@@ -278,6 +312,13 @@ class Artifacts:
 
         name = _reformat_name(name)
         saved_exc: Optional[Exception] = None
+
+        local_dvc_studio_config = Config().get("studio", {})
+        args_dvc_studio_config = {}
+        if config and not isinstance(config, dict):
+            config = Config.load_file(config)
+            args_dvc_studio_config = config.get("studio", {})
+
         try:
             logger.trace("Trying studio-only config")
             return cls._download_studio(
@@ -288,14 +329,13 @@ class Artifacts:
                 out=out,
                 force=force,
                 jobs=jobs,
+                dvc_studio_config=local_dvc_studio_config | args_dvc_studio_config,
             )
         except FileExistsLocallyError:
             raise
         except Exception as exc:  # noqa: BLE001
             saved_exc = exc
 
-        if config and not isinstance(config, dict):
-            config = Config.load_file(config)
         with Repo.open(
             url=url,
             subrepos=True,
@@ -305,7 +345,7 @@ class Artifacts:
             remote_config=remote_config,
         ) as repo:
             logger.trace("Trying repo [studio] config")
-            dvc_studio_config = dict(repo.config.get("studio"))
+            repo_dvc_studio_config = repo.config.get("studio", {})
             try:
                 return cls._download_studio(
                     url,
@@ -315,7 +355,9 @@ class Artifacts:
                     out=out,
                     force=force,
                     jobs=jobs,
-                    dvc_studio_config=dvc_studio_config,
+                    dvc_studio_config=local_dvc_studio_config
+                    | repo_dvc_studio_config
+                    | args_dvc_studio_config,
                 )
             except FileExistsLocallyError:
                 raise
@@ -333,7 +375,7 @@ class Artifacts:
                 )
             except FileExistsLocallyError:
                 raise
-            except Exception as exc:  # noqa: BLE001
+            except Exception as exc:
                 if saved_exc:
                     logger.exception(str(saved_exc), exc_info=saved_exc.__cause__)
                 raise DvcException(

dvc/repo/checkout.py

@@ -125,7 +125,7 @@ def checkout(  # noqa: C901
             (StageFileDoesNotExistError, StageFileBadNameError, NoOutputOrStageError),
         ):
             raise CheckoutErrorSuggestGit(target) from exc
-        raise
+        raise  # noqa: PLE0704
 
     view = self.index.targets_view(
         targets, recursive=recursive, with_deps=with_deps, onerror=onerror
@@ -149,7 +149,10 @@ def checkout(  # noqa: C901
 
     def checkout_onerror(src_path, dest_path, _exc):
         logger.debug(
-            "failed to create '%s' from '%s'", dest_path, src_path, exc_info=True
+            "failed to create '%s' from '%s'",
+            dest_path,
+            src_path,
+            exc_info=True,  # noqa: LOG014
         )
 
         for out_path in out_paths:

dvc/repo/data.py

@@ -1,38 +1,34 @@
 import os
 import posixpath
-from collections.abc import Iterable
-from typing import TYPE_CHECKING, Any, TypedDict, Union
+from collections import defaultdict
+from collections.abc import Iterable, Iterator, Mapping
+from typing import TYPE_CHECKING, Optional, TypedDict, Union
 
-from dvc.fs.callbacks import DEFAULT_CALLBACK
+from dvc.fs.callbacks import DEFAULT_CALLBACK, Callback, TqdmCallback
+from dvc.log import logger
 from dvc.ui import ui
+from dvc_data.index import DataIndexDirError
 
 if TYPE_CHECKING:
-    from dvc.fs.callbacks import Callback
     from dvc.repo import Repo
     from dvc.scm import Git, NoSCM
-    from dvc_data.index import DataIndex
+    from dvc_data.index import (
+        BaseDataIndex,
+        DataIndex,
+        DataIndexEntry,
+        DataIndexKey,
+        DataIndexView,
+    )
     from dvc_data.index.diff import Change
+    from dvc_objects.fs.base import FileSystem
+
+logger = logger.getChild(__name__)
 
 
 def posixpath_to_os_path(path: str) -> str:
     return path.replace(posixpath.sep, os.path.sep)
 
 
-def _adapt_typ(typ: str) -> str:
-    from dvc_data.index.diff import ADD, DELETE, MODIFY
-
-    if typ == MODIFY:
-        return "modified"
-
-    if typ == ADD:
-        return "added"
-
-    if typ == DELETE:
-        return "deleted"
-
-    return typ
-
-
 def _adapt_path(change: "Change") -> str:
     isdir = False
     if change.new and change.new.meta:
@@ -45,27 +41,65 @@ def _adapt_path(change: "Change") -> str:
     return os.path.sep.join(key)
 
 
+def _get_missing_paths(
+    to_check: Mapping["FileSystem", Mapping[str, Iterable["DataIndexEntry"]]],
+    batch_size: Optional[int] = None,
+    callback: "Callback" = DEFAULT_CALLBACK,
+) -> Iterator[str]:
+    for fs, paths_map in to_check.items():
+        if batch_size == 1 or (batch_size is None and fs.protocol == "local"):
+            results = list(callback.wrap(map(fs.exists, paths_map)))
+        else:
+            results = fs.exists(
+                list(paths_map), batch_size=batch_size, callback=callback
+            )
+
+        for cache_path, exists in zip(paths_map, results):
+            if exists:
+                continue
+
+            for entry in paths_map[cache_path]:
+                key = entry.key
+                assert key
+                if entry.meta and entry.meta.isdir:
+                    key = (*key, "")
+                yield os.path.sep.join(key)
+
+
+class StorageCallback(Callback):
+    def __init__(self, parent_cb: Callback) -> None:
+        super().__init__(size=0, value=0)
+        self.parent_cb = parent_cb
+
+    def set_size(self, size: int) -> None:
+        # This is a no-op to prevent `fs.exists` from trying to set the size
+        pass
+
+    def relative_update(self, value: int = 1) -> None:
+        self.parent_cb.relative_update(value)
+
+    def absolute_update(self, value: int) -> None:
+        self.parent_cb.relative_update(value - self.value)
+
+
 def _diff(
-    old: "DataIndex",
-    new: "DataIndex",
+    old: "BaseDataIndex",
+    new: "BaseDataIndex",
     *,
+    filter_keys: Optional[Iterable["DataIndexKey"]] = None,
     granular: bool = False,
     not_in_cache: bool = False,
-    not_in_remote: bool = False,
-    remote_refresh: bool = False,
+    batch_size: Optional[int] = None,
     callback: "Callback" = DEFAULT_CALLBACK,
 ) -> dict[str, list[str]]:
-    from dvc_data.index import StorageError
-    from dvc_data.index.diff import UNCHANGED, UNKNOWN, diff
+    from dvc_data.index.diff import ADD, DELETE, MODIFY, UNCHANGED, UNKNOWN, diff
 
-    ret: dict[str, list[str]] = {}
+    ret: dict[str, list[str]] = defaultdict(list)
+    change_types = {MODIFY: "modified", ADD: "added", DELETE: "deleted"}
 
-    def _add_change(typ, change):
-        typ = _adapt_typ(typ)
-        if typ not in ret:
-            ret[typ] = []
-
-        ret[typ].append(_adapt_path(change))
+    to_check: dict[FileSystem, dict[str, list[DataIndexEntry]]] = defaultdict(
+        lambda: defaultdict(list)
+    )
 
     for change in diff(
         old,
@@ -76,6 +110,14 @@ def _diff(
         with_unknown=True,
         callback=callback,
     ):
+        # The index is a trie, so even when we filter by a specific path
+        # like `dir/file`, all parent nodes leading to that path (e.g., `dir/`)
+        # still appear in the view. As a result, keys like `dir/` will be present
+        # even if only `dir/file` matches the filter.
+        # We need to skip such entries to avoid showing root of tracked directories.
+        if filter_keys and not any(change.key[: len(fk)] == fk for fk in filter_keys):
+            continue
+
         if (
             change.typ == UNCHANGED
             and (not change.old or not change.old.hash_info)
@@ -88,31 +130,27 @@ def _diff(
             # NOTE: emulating previous behaviour
             continue
 
-        if (
-            not_in_cache
-            and change.old
-            and change.old.hash_info
-            and not old.storage_map.cache_exists(change.old)
-        ):
-            # NOTE: emulating previous behaviour
-            _add_change("not_in_cache", change)
+        if not_in_cache and change.old and change.old.hash_info:
+            old_entry = change.old
+            cache_fs, cache_path = old.storage_map.get_cache(old_entry)
+            # check later in batches
+            to_check[cache_fs][cache_path].append(old_entry)
 
-        try:
-            if (
-                not_in_remote
-                and change.old
-                and change.old.hash_info
-                and not old.storage_map.remote_exists(
-                    change.old, refresh=remote_refresh
-                )
-            ):
-                _add_change("not_in_remote", change)
-        except StorageError:
-            pass
+        change_typ = change_types.get(change.typ, change.typ)
+        ret[change_typ].append(_adapt_path(change))
 
-        _add_change(change.typ, change)
-
-    return ret
+    total_items = sum(
+        len(entries) for paths in to_check.values() for entries in paths.values()
+    )
+    with TqdmCallback(size=total_items, desc="Checking cache", unit="entry") as cb:
+        missing_items = list(
+            _get_missing_paths(
+                to_check, batch_size=batch_size, callback=StorageCallback(cb)
+            ),
+        )
+        if missing_items:
+            ret["not_in_cache"] = missing_items
+    return dict(ret)
 
 
 class GitInfo(TypedDict, total=False):
@@ -151,7 +189,48 @@ def _git_info(scm: Union["Git", "NoSCM"], untracked_files: str = "all") -> GitIn
     )
 
 
-def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> dict[str, list[str]]:
+def filter_index(
+    index: Union["DataIndex", "DataIndexView"],
+    filter_keys: Optional[Iterable["DataIndexKey"]] = None,
+) -> "BaseDataIndex":
+    from dvc_data.index.view import DataIndexView
+
+    if not filter_keys:
+        return index
+
+    if isinstance(index, DataIndexView):
+        orig_index = index._index
+        parent_filter_fn = index.filter_fn
+    else:
+        orig_index = index
+        parent_filter_fn = None
+
+    def filter_fn(key: "DataIndexKey") -> bool:
+        if parent_filter_fn is not None and not parent_filter_fn(key):
+            return False
+
+        for filter_key in filter_keys:
+            # eg: if key is "dir/file" and filter_key is "dir/", return True
+            if key[: len(filter_key)] == filter_key:
+                return True
+            # eg: if key is `dir/` and filter_key is `dir/file`, also return True.
+            # This ensures we include parent prefixes needed to reach matching leaves.
+            # Intermediate prefixes must be retained to access nested keys.
+            if filter_key[: len(key)] == key:
+                return True
+        return False
+
+    from dvc_data.index import view
+
+    return view(orig_index, filter_fn=filter_fn)
+
+
+def _diff_index_to_wtree(
+    repo: "Repo",
+    filter_keys: Optional[Iterable["DataIndexKey"]] = None,
+    granular: bool = False,
+    batch_size: Optional[int] = None,
+) -> dict[str, list[str]]:
     from .index import build_data_index
 
     with ui.progress(desc="Building workspace index", unit="entry") as pb:
@@ -162,30 +241,53 @@ def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> dict[str, list[str]]:
             compute_hash=True,
             callback=pb.as_callback(),
         )
+        workspace_view = filter_index(workspace, filter_keys=filter_keys)
 
     with ui.progress(
         desc="Calculating diff between index/workspace",
         unit="entry",
     ) as pb:
+        index = repo.index.data["repo"]
+        view = filter_index(index, filter_keys=filter_keys)
         return _diff(
-            repo.index.data["repo"],
-            workspace,
+            view,
+            workspace_view,
+            filter_keys=filter_keys,
+            granular=granular,
             not_in_cache=True,
+            batch_size=batch_size,
             callback=pb.as_callback(),
-            **kwargs,
         )
 
 
 def _diff_head_to_index(
-    repo: "Repo", head: str = "HEAD", **kwargs: Any
+    repo: "Repo",
+    head: str = "HEAD",
+    filter_keys: Optional[Iterable["DataIndexKey"]] = None,
+    granular: bool = False,
 ) -> dict[str, list[str]]:
-    index = repo.index.data["repo"]
+    from dvc.scm import RevError
+    from dvc_data.index import DataIndex
 
-    with repo.switch(head):
-        head_index = repo.index.data["repo"]
+    index = repo.index.data["repo"]
+    index_view = filter_index(index, filter_keys=filter_keys)
+
+    try:
+        with repo.switch(head):
+            head_index = repo.index.data["repo"]
+            head_view = filter_index(head_index, filter_keys=filter_keys)
+    except RevError:
+        logger.debug("failed to switch to '%s'", head)
+        head_view = DataIndex()
 
     with ui.progress(desc="Calculating diff between head/index", unit="entry") as pb:
-        return _diff(head_index, index, callback=pb.as_callback(), **kwargs)
+        return _diff(
+            head_view,
+            index_view,
+            filter_keys=filter_keys,
+            granular=granular,
+            callback=pb.as_callback(),
+        )
 
 
 class Status(TypedDict):
@@ -217,27 +319,140 @@ def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> list[str]
     return [repo.fs.relpath(file, start) for file in files]
 
 
-def status(repo: "Repo", untracked_files: str = "no", **kwargs: Any) -> Status:
+def _get_entries_not_in_remote(
+    repo: "Repo",
+    filter_keys: Optional[Iterable["DataIndexKey"]] = None,
+    granular: bool = False,
+    remote_refresh: bool = False,
+) -> list[str]:
+    """Get entries that are not in remote storage."""
+    from dvc.repo.worktree import worktree_view
+    from dvc_data.index import StorageKeyError
+
+    entries: dict[DataIndexKey, DataIndexEntry] = {}
+
+    def _onerror(entry, exc):
+        if not isinstance(exc, DataIndexDirError):
+            raise exc
+        # We don't have the contents of this dir file, so we will only check this key.
+        entries[entry.key] = entry
+
+    # View into the index, with only pushable entries
+    index = worktree_view(repo.index, push=True)
+    data_index = index.data["repo"]
+
+    orig_data_index_onerror = data_index.onerror
+    data_index.onerror = _onerror
+
+    view = filter_index(data_index, filter_keys=filter_keys)  # type: ignore[arg-type]
+
+    missing_entries = []
+
+    storage_map = view.storage_map
+
+    n = 0
+    with TqdmCallback(size=n, desc="Checking remote", unit="entry") as cb:
+        for key, entry in view.iteritems(shallow=not granular):
+            if not (entry and entry.hash_info):
+                continue
+
+            # The index is a trie, so even when we filter by a specific path
+            # like `dir/file`, all parent nodes leading to that path (e.g., `dir/`)
+            # still appear in the view. As a result, keys like `dir/` will be present
+            # even if only `dir/file` matches the filter.
+            # We need to skip such entries to avoid showing root of tracked directories.
+            if filter_keys and not any(
+                key[: len(filter_key)] == filter_key for filter_key in filter_keys
+            ):
+                continue
+
+            entries[key] = entry
+            n += 1
+            cb.set_size(n)
+
+        for key, entry in entries.items():
+            k = (*key, "") if entry.meta and entry.meta.isdir else key
+            try:
+                if not storage_map.remote_exists(entry, refresh=remote_refresh):
+                    missing_entries.append(os.path.sep.join(k))
+            except StorageKeyError:
+                pass
+            finally:
+                cb.relative_update()
+    data_index.onerror = orig_data_index_onerror
+    return missing_entries
+
+
+def _matches_target(p: str, targets: Iterable[str]) -> bool:
+    sep = os.sep
+    return any(p == t or p.startswith(t + sep) for t in targets)
+
+
+def _prune_keys(filter_keys: Iterable["DataIndexKey"]) -> list["DataIndexKey"]:
+    sorted_keys = sorted(set(filter_keys), key=len)
+    result: list[DataIndexKey] = []
+
+    for key in sorted_keys:
+        if not any(key[: len(prefix)] == prefix for prefix in result):
+            result.append(key)
+    return result
+
+
+def status(
+    repo: "Repo",
+    targets: Optional[Iterable[Union[os.PathLike[str], str]]] = None,
+    *,
+    granular: bool = False,
+    untracked_files: str = "no",
+    not_in_remote: bool = False,
+    remote_refresh: bool = False,
+    batch_size: Optional[int] = None,
+    head: str = "HEAD",
+) -> Status:
     from dvc.scm import NoSCMError, SCMError
 
-    head = kwargs.pop("head", "HEAD")
-    uncommitted_diff = _diff_index_to_wtree(repo, **kwargs)
+    targets = targets or []
+    filter_keys: list[DataIndexKey] = [repo.fs.relparts(os.fspath(t)) for t in targets]
+    # try to remove duplicate and overlapping keys
+    filter_keys = _prune_keys(filter_keys)
+
+    uncommitted_diff = _diff_index_to_wtree(
+        repo, filter_keys=filter_keys, granular=granular, batch_size=batch_size
+    )
     unchanged = set(uncommitted_diff.pop("unchanged", []))
 
+    entries_not_in_remote: list[str] = []
+    if not_in_remote:
+        entries_not_in_remote = _get_entries_not_in_remote(
+            repo,
+            filter_keys=filter_keys,
+            granular=granular,
+            remote_refresh=remote_refresh,
+        )
+
     try:
-        committed_diff = _diff_head_to_index(repo, head=head, **kwargs)
+        committed_diff = _diff_head_to_index(
+            repo, filter_keys=filter_keys, head=head, granular=granular
+        )
     except (SCMError, NoSCMError):
         committed_diff = {}
     else:
         unchanged &= set(committed_diff.pop("unchanged", []))
 
     git_info = _git_info(repo.scm, untracked_files=untracked_files)
-    untracked = git_info.get("untracked", [])
-    untracked = _transform_git_paths_to_dvc(repo, untracked)
+    scm_filter_targets = {
+        os.path.relpath(os.path.abspath(t), repo.scm.root_dir) for t in targets
+    }
+    untracked_it: Iterable[str] = git_info.get("untracked", [])
+    if scm_filter_targets:
+        untracked_it = (
+            f for f in untracked_it if _matches_target(f, scm_filter_targets)
+        )
+    untracked = _transform_git_paths_to_dvc(repo, untracked_it)
     # order matters here
     return Status(
         not_in_cache=uncommitted_diff.pop("not_in_cache", []),
-        not_in_remote=uncommitted_diff.pop("not_in_remote", []),
+        not_in_remote=entries_not_in_remote,
         committed=committed_diff,
         uncommitted=uncommitted_diff,
         untracked=untracked,

dvc/repo/datasets.py

@@ -0,0 +1,425 @@
+import os
+from collections.abc import Iterator, Mapping
+from datetime import datetime
+from functools import cached_property
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, Union, cast
+from urllib.parse import urlparse
+
+from attrs import Attribute, AttrsInstance, asdict, evolve, field, fields, frozen
+from attrs.converters import default_if_none
+
+from dvc.dvcfile import Lockfile, ProjectFile
+from dvc.exceptions import DvcException
+from dvc.log import logger
+from dvc.types import StrPath
+from dvc_data.hashfile.meta import Meta
+
+if TYPE_CHECKING:
+    from datachain.dataset import DatasetRecord, DatasetVersion  # type: ignore[import]
+    from typing_extensions import Self
+
+    from dvc.repo import Repo
+
+
+logger = logger.getChild(__name__)
+
+
+def _get_dataset_record(name: str) -> "DatasetRecord":
+    from dvc.exceptions import DvcException
+
+    try:
+        from datachain.catalog import get_catalog  # type: ignore[import]
+
+    except ImportError as exc:
+        raise DvcException("datachain is not installed") from exc
+
+    catalog = get_catalog()
+    return catalog.get_remote_dataset(name)
+
+
+def _get_dataset_info(
+    name: str, record: Optional["DatasetRecord"] = None, version: Optional[int] = None
+) -> "DatasetVersion":
+    record = record or _get_dataset_record(name)
+    assert record
+    v = record.latest_version if version is None else version
+    assert v is not None
+    return record.get_version(v)
+
+
+def default_str(v) -> str:
+    return default_if_none("")(v)
+
+
+def to_datetime(d: Union[str, datetime]) -> datetime:
+    return datetime.fromisoformat(d) if isinstance(d, str) else d
+
+
+def ensure(cls):
+    def inner(v):
+        return cls.from_dict(v) if isinstance(v, dict) else v
+
+    return inner
+
+
+class SerDe:
+    def to_dict(self: AttrsInstance) -> dict[str, Any]:
+        def filter_defaults(attr: Attribute, v: Any):
+            if attr.metadata.get("exclude_falsy", False) and not v:
+                return False
+            return attr.default != v
+
+        def value_serializer(_inst, _field, v):
+            return v.isoformat() if isinstance(v, datetime) else v
+
+        return asdict(self, filter=filter_defaults, value_serializer=value_serializer)
+
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> "Self":
+        _fields = fields(cast("type[AttrsInstance]", cls))
+        kwargs = {f.name: d[f.name] for f in _fields if f.name in d}
+        return cls(**kwargs)
+
+
+@frozen(kw_only=True)
+class DatasetSpec(SerDe):
+    name: str
+    url: str
+    type: Literal["dvc", "dc", "url"]
+
+
+@frozen(kw_only=True)
+class DVCDatasetSpec(DatasetSpec):
+    type: Literal["dvc"]
+    path: str = field(default="", converter=default_str)
+    rev: Optional[str] = None
+
+
+@frozen(kw_only=True, order=True)
+class FileInfo(SerDe):
+    relpath: str
+    meta: Meta = field(order=False, converter=ensure(Meta))  # type: ignore[misc]
+
+
+@frozen(kw_only=True)
+class DVCDatasetLock(DVCDatasetSpec):
+    rev_lock: str
+
+
+@frozen(kw_only=True)
+class DatachainDatasetLock(DatasetSpec):
+    version: int
+    created_at: datetime = field(converter=to_datetime)
+
+
+@frozen(kw_only=True)
+class URLDatasetLock(DatasetSpec):
+    meta: Meta = field(converter=ensure(Meta))  # type: ignore[misc]
+    files: list[FileInfo] = field(
+        factory=list,
+        converter=lambda f: sorted(map(ensure(FileInfo), f)),
+        metadata={"exclude_falsy": True},
+    )
+
+
+def to_spec(lock: "Lock") -> "Spec":
+    cls = DVCDatasetSpec if lock.type == "dvc" else DatasetSpec
+    return cls(**{f.name: getattr(lock, f.name) for f in fields(cls)})
+
+
+@frozen(kw_only=True)
+class DVCDataset:
+    manifest_path: str
+    spec: DVCDatasetSpec
+    lock: Optional[DVCDatasetLock] = None
+    _invalidated: bool = field(default=False, eq=False, repr=False)
+
+    type: ClassVar[Literal["dvc"]] = "dvc"
+
+    def update(self, repo, rev: Optional[str] = None, **kwargs) -> "Self":
+        from dvc.dependency import RepoDependency
+
+        spec = self.spec
+        if rev:
+            spec = evolve(self.spec, rev=rev)
+
+        def_repo = {
+            RepoDependency.PARAM_REV: spec.rev,
+            RepoDependency.PARAM_URL: spec.url,
+        }
+        dep = RepoDependency(def_repo, None, spec.path, repo=repo)  # type: ignore[arg-type]
+        dep.save()
+        d = dep.dumpd()
+
+        repo_info = d[RepoDependency.PARAM_REPO]
+        assert isinstance(repo_info, dict)
+        rev_lock = repo_info[RepoDependency.PARAM_REV_LOCK]
+        lock = DVCDatasetLock(**spec.to_dict(), rev_lock=rev_lock)
+        return evolve(self, spec=spec, lock=lock)
+
+
+@frozen(kw_only=True)
+class DatachainDataset:
+    manifest_path: str
+    spec: "DatasetSpec"
+    lock: "Optional[DatachainDatasetLock]" = field(default=None)
+    _invalidated: bool = field(default=False, eq=False, repr=False)
+
+    type: ClassVar[Literal["dc"]] = "dc"
+
+    @property
+    def pinned(self) -> bool:
+        return self.name_version[1] is not None
+
+    @property
+    def name_version(self) -> tuple[str, Optional[int]]:
+        url = urlparse(self.spec.url)
+        path = url.netloc + url.path
+        parts = path.split("@v")
+        assert parts
+
+        name = parts[0]
+        version = int(parts[1]) if len(parts) > 1 else None
+        return name, version
+
+    def update(
+        self,
+        repo,  # noqa: ARG002
+        record: Optional["DatasetRecord"] = None,
+        version: Optional[int] = None,
+        **kwargs,
+    ) -> "Self":
+        name, _version = self.name_version
+        version = version if version is not None else _version
+        version_info = _get_dataset_info(name, record=record, version=version)
+        lock = DatachainDatasetLock(
+            **self.spec.to_dict(),
+            version=version_info.version,
+            created_at=version_info.created_at,
+        )
+        return evolve(self, lock=lock)
+
+
+@frozen(kw_only=True)
+class URLDataset:
+    manifest_path: str
+    spec: "DatasetSpec"
+    lock: "Optional[URLDatasetLock]" = None
+    _invalidated: bool = field(default=False, eq=False, repr=False)
+
+    type: ClassVar[Literal["url"]] = "url"
+
+    def update(self, repo, **kwargs):
+        from dvc.dependency import Dependency
+
+        dep = Dependency(
+            None, self.spec.url, repo=repo, fs_config={"version_aware": True}
+        )
+        dep.save()
+        d = dep.dumpd(datasets=True)
+        files = [
+            FileInfo(relpath=info["relpath"], meta=Meta.from_dict(info))
+            for info in d.get("files", [])
+        ]
+        lock = URLDatasetLock(**self.spec.to_dict(), meta=dep.meta, files=files)
+        return evolve(self, lock=lock)
+
+
+Lock = Union[DVCDatasetLock, DatachainDatasetLock, URLDatasetLock]
+Spec = Union[DatasetSpec, DVCDatasetSpec]
+Dataset = Union[DVCDataset, DatachainDataset, URLDataset]
+
+
+class DatasetNotFoundError(DvcException, KeyError):
+    def __init__(self, name, *args):
+        self.name = name
+        super().__init__("dataset not found", *args)
+
+    def __str__(self) -> str:
+        return self.msg
+
+
+class Datasets(Mapping[str, Dataset]):
+    def __init__(self, repo: "Repo") -> None:
+        self.repo: Repo = repo
+
+    def __repr__(self):
+        return repr(dict(self))
+
+    def __rich_repr__(self):
+        yield dict(self)
+
+    def __getitem__(self, name: str) -> Dataset:
+        try:
+            return self._datasets[name]
+        except KeyError as exc:
+            raise DatasetNotFoundError(name) from exc
+
+    def __setitem__(self, name: str, dataset: Dataset) -> None:
+        self._datasets[name] = dataset
+
+    def __contains__(self, name: object) -> bool:
+        return name in self._datasets
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._datasets)
+
+    def __len__(self) -> int:
+        return len(self._datasets)
+
+    @cached_property
+    def _spec(self) -> dict[str, tuple[str, dict[str, Any]]]:
+        return {
+            dataset["name"]: (path, dataset)
+            for path, datasets in self.repo.index._datasets.items()
+            for dataset in datasets
+        }
+
+    @cached_property
+    def _lock(self) -> dict[str, Optional[dict[str, Any]]]:
+        datasets_lock = self.repo.index._datasets_lock
+
+        def find(path, name) -> Optional[dict[str, Any]]:
+            # only look for `name` in the lock file next to the
+            # corresponding `dvc.yaml` file
+            lock = datasets_lock.get(path, [])
+            return next((dataset for dataset in lock if dataset["name"] == name), None)
+
+        return {ds["name"]: find(path, name) for name, (path, ds) in self._spec.items()}
+
+    @cached_property
+    def _datasets(self) -> dict[str, Dataset]:
+        return {
+            name: self._build_dataset(path, spec, self._lock[name])
+            for name, (path, spec) in self._spec.items()
+        }
+
+    def _reset(self) -> None:
+        self.__dict__.pop("_spec", None)
+        self.__dict__.pop("_lock", None)
+        self.__dict__.pop("_datasets", None)
+
+    @staticmethod
+    def _spec_from_info(spec: dict[str, Any]) -> Spec:
+        typ = spec.get("type")
+        if not typ:
+            raise ValueError("type should be present in spec")
+        if typ == "dvc":
+            return DVCDatasetSpec.from_dict(spec)
+        if typ in {"dc", "url"}:
+            return DatasetSpec.from_dict(spec)
+        raise ValueError(f"unknown dataset type: {spec.get('type', '')}")
+
+    @staticmethod
+    def _lock_from_info(lock: Optional[dict[str, Any]]) -> Optional[Lock]:
+        kl = {"dvc": DVCDatasetLock, "dc": DatachainDatasetLock, "url": URLDatasetLock}
+        if lock and (cls := kl.get(lock.get("type", ""))):  # type: ignore[assignment]
+            return cls.from_dict(lock)  # type: ignore[attr-defined]
+        return None
+
+    @classmethod
+    def _build_dataset(
+        cls,
+        manifest_path: str,
+        spec_data: dict[str, Any],
+        lock_data: Optional[dict[str, Any]] = None,
+    ) -> Dataset:
+        _invalidated = False
+        spec = cls._spec_from_info(spec_data)
+        lock = cls._lock_from_info(lock_data)
+        # if dvc.lock and dvc.yaml file are not in sync, we invalidate the lock.
+        if lock is not None and to_spec(lock) != spec:
+            logger.debug(
+                "invalidated lock data for %s in %s",
+                spec.name,
+                manifest_path,
+            )
+            _invalidated = True  # signal is used during `dvc repro`/`dvc status`.
+            lock = None
+
+        assert isinstance(spec, DatasetSpec)
+        if spec.type == "dvc":
+            assert lock is None or isinstance(lock, DVCDatasetLock)
+            assert isinstance(spec, DVCDatasetSpec)
+            return DVCDataset(
+                manifest_path=manifest_path,
+                spec=spec,
+                lock=lock,
+                invalidated=_invalidated,
+            )
+        if spec.type == "url":
+            assert lock is None or isinstance(lock, URLDatasetLock)
+            return URLDataset(
+                manifest_path=manifest_path,
+                spec=spec,
+                lock=lock,
+                invalidated=_invalidated,
+            )
+        if spec.type == "dc":
+            assert lock is None or isinstance(lock, DatachainDatasetLock)
+            return DatachainDataset(
+                manifest_path=manifest_path,
+                spec=spec,
+                lock=lock,
+                invalidated=_invalidated,
+            )
+        raise ValueError(f"unknown dataset type: {spec.type!r}")
+
+    def add(
+        self,
+        name: str,
+        url: str,
+        type: str,  # noqa: A002
+        manifest_path: StrPath = "dvc.yaml",
+        **kwargs: Any,
+    ) -> Dataset:
+        assert type in {"dvc", "dc", "url"}
+        kwargs.update({"name": name, "url": url, "type": type})
+        dataset = self._build_dataset(os.path.abspath(manifest_path), kwargs)
+        dataset = dataset.update(self.repo)
+
+        self.dump(dataset)
+        self[name] = dataset
+        return dataset
+
+    def update(self, name, **kwargs) -> tuple[Dataset, Dataset]:
+        dataset = self[name]
+        version = kwargs.get("version")
+
+        if dataset.type == "url" and (version or kwargs.get("rev")):
+            raise ValueError("cannot update version/revision for a url")
+        if dataset.type == "dc" and version is not None:
+            if not isinstance(version, int):
+                raise TypeError(
+                    "DataChain dataset version has to be an integer, "
+                    f"got {type(version).__name__!r}"
+                )
+            if version < 1:
+                raise ValueError(
+                    f"DataChain dataset version should be >=1, got {version}"
+                )
+
+        new = dataset.update(self.repo, **kwargs)
+
+        self.dump(new, old=dataset)
+        self[name] = new
+        return dataset, new
+
+    def _dump_spec(self, manifest_path: StrPath, spec: Spec) -> None:
+        spec_data = spec.to_dict()
+        assert spec_data.keys() & {"type", "name", "url"}
+        project_file = ProjectFile(self.repo, manifest_path)
+        project_file.dump_dataset(spec_data)
+
+    def _dump_lock(self, manifest_path: StrPath, lock: Lock) -> None:
+        lock_data = lock.to_dict()
+        assert lock_data.keys() & {"type", "name", "url"}
+        lockfile = Lockfile(self.repo, Path(manifest_path).with_suffix(".lock"))
+        lockfile.dump_dataset(lock_data)
+
+    def dump(self, dataset: Dataset, old: Optional[Dataset] = None) -> None:
+        if not old or old.spec != dataset.spec:
+            self._dump_spec(dataset.manifest_path, dataset.spec)
+        if dataset.lock and (not old or old.lock != dataset.lock):
+            self._dump_lock(dataset.manifest_path, dataset.lock)

dvc/repo/diff.py

@@ -26,7 +26,7 @@ def _diff(old, new, data_keys, with_missing=False):
     from dvc_data.index.diff import ADD, DELETE, MODIFY, RENAME
     from dvc_data.index.diff import diff as idiff
 
-    ret: "dict[str, list[dict]]" = {
+    ret: dict[str, list[dict]] = {
         "added": [],
         "deleted": [],
         "modified": [],

dvc/repo/du.py

@@ -6,7 +6,7 @@ def du(
     path: Optional[str] = None,
     rev: Optional[str] = None,
     summarize: bool = False,
-    config: Union[None, dict[str, Any], str] = None,
+    config: Union[dict[str, Any], str, None] = None,
     remote: Optional[str] = None,
     remote_config: Optional[dict] = None,
 ):

dvc/repo/experiments/__init__.py

@@ -119,7 +119,7 @@ class Experiments:
         **kwargs,
     ):
         """Reproduce and checkout a single (standalone) experiment."""
-        exp_queue: "BaseStashQueue" = (
+        exp_queue: BaseStashQueue = (
             self.tempdir_queue if tmp_dir else self.workspace_queue
         )
         self.queue_one(exp_queue, **kwargs)
@@ -205,7 +205,7 @@ class Experiments:
         Experiment will be derived from the current workspace.
         """
 
-        name = kwargs.get("name", None)
+        name = kwargs.get("name")
         baseline_sha = kwargs.get("baseline_rev") or self.repo.scm.get_rev()
 
         if name:
@@ -249,7 +249,7 @@ class Experiments:
         exec_results = queue.reproduce(copy_paths=copy_paths, message=message)
 
         results: dict[str, str] = {}
-        for _, exp_result in exec_results.items():
+        for exp_result in exec_results.values():
             results.update(exp_result)
         return results
 

dvc/repo/experiments/apply.py

@@ -25,7 +25,7 @@ def apply(repo: "Repo", rev: str, **kwargs):
     from dvc.repo.checkout import checkout as dvc_checkout
     from dvc.scm import RevError, resolve_rev
 
-    exps: "Experiments" = repo.experiments
+    exps: Experiments = repo.experiments
 
     is_stash: bool = False
 
@@ -57,7 +57,7 @@ def apply(repo: "Repo", rev: str, **kwargs):
 
 
 def _apply(repo: "Repo", rev: str, name: Optional[str] = None, is_stash: bool = False):
-    exps: "Experiments" = repo.experiments
+    exps: Experiments = repo.experiments
 
     with exps.apply_stash.preserve_workspace(rev, name=name):
         with repo.scm.detach_head(rev, force=True):

dvc/repo/experiments/collect.py

@@ -69,16 +69,16 @@ def collect_rev(
 
 def _collect_rev(
     repo: "Repo",
-    rev: str,
+    revision: str,
     param_deps: bool = False,
     **kwargs,
 ) -> SerializableExp:
-    with repo.switch(rev) as rev:
+    with repo.switch(revision) as rev:
         if rev == "workspace":
             timestamp: Optional[datetime] = None
         else:
             commit = repo.scm.resolve_commit(rev)
-            timestamp = datetime.fromtimestamp(commit.commit_time)
+            timestamp = datetime.fromtimestamp(commit.commit_time)  # noqa: DTZ006
 
         return SerializableExp.from_repo(
             repo,
@@ -174,7 +174,7 @@ def collect_active(
     """
     if not baseline_revs:
         return {}
-    result: dict[str, list["ExpRange"]] = {}
+    result: dict[str, list[ExpRange]] = {}
     exps = repo.experiments
     for queue in (exps.workspace_queue, exps.tempdir_queue, exps.celery_queue):
         for baseline, active_exps in queue.collect_active_data(
@@ -220,7 +220,7 @@ def collect_successful(
     Returns:
         Dict mapping baseline revision to successful experiments.
     """
-    result: dict[str, list["ExpRange"]] = {}
+    result: dict[str, list[ExpRange]] = {}
     for baseline_rev in baseline_revs:
         result[baseline_rev] = list(_collect_baseline(repo, baseline_rev, **kwargs))
     return result
@@ -274,6 +274,7 @@ def collect(
     num: int = 1,
     hide_queued: bool = False,
     hide_failed: bool = False,
+    hide_workspace: bool = False,
     sha_only: bool = False,
     **kwargs,
 ) -> list["ExpState"]:
@@ -303,8 +304,11 @@ def collect(
             repo.scm, baseline_revs, refs=cached_refs, logger=logger
         )
 
-    workspace_data = collect_rev(repo, "workspace", **kwargs)
-    result: list["ExpState"] = [workspace_data]
+    result: list[ExpState] = []
+    if not hide_workspace:
+        workspace_data = collect_rev(repo, "workspace", **kwargs)
+        result.append(workspace_data)
+
     queued = collect_queued(repo, baseline_revs, **kwargs) if not hide_queued else {}
     active = collect_active(repo, baseline_revs, **kwargs)
     failed = collect_failed(repo, baseline_revs, **kwargs) if not hide_failed else {}
@@ -338,6 +342,6 @@ def _sorted_ranges(exp_ranges: Iterable["ExpRange"]) -> list["ExpRange"]:
         if head_exp and head_exp.data and head_exp.data.timestamp:
             return head_exp.data.timestamp, head_exp.rev
 
-        return datetime.fromtimestamp(0), ""
+        return datetime.fromtimestamp(0), ""  # noqa: DTZ006
 
     return sorted(exp_ranges, key=_head_timestamp, reverse=True)

dvc/repo/experiments/executor/base.py

@@ -1,20 +1,21 @@
-import logging
 import os
 import pickle
 import shutil
 from abc import ABC, abstractmethod
 from collections.abc import Iterable, Iterator
-from contextlib import contextmanager
+from contextlib import AbstractContextManager, contextmanager
 from dataclasses import asdict, dataclass
 from enum import IntEnum
 from itertools import chain
 from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, Union
 
+from funcy import nullcontext
 from scmrepo.exceptions import SCMError
 
 from dvc.env import DVC_EXP_AUTO_PUSH, DVC_EXP_GIT_REMOTE
 from dvc.exceptions import DvcException
 from dvc.log import logger
+from dvc.logger import set_loggers_level
 from dvc.repo.experiments.exceptions import ExperimentExistsError
 from dvc.repo.experiments.refs import EXEC_BASELINE, EXEC_BRANCH, ExpRefInfo
 from dvc.repo.experiments.utils import to_studio_params
@@ -23,7 +24,11 @@ from dvc.repo.params.show import _collect_top_level_params
 from dvc.stage.serialize import to_lockfile
 from dvc.utils import dict_sha256, env2bool, relpath
 from dvc.utils.fs import remove
-from dvc.utils.studio import env_to_config
+from dvc.utils.studio import (
+    env_to_config,
+    get_repo_url,
+    get_subrepo_relpath,
+)
 
 if TYPE_CHECKING:
     from queue import Queue
@@ -195,7 +200,7 @@ class BaseExecutor(ABC):
     @classmethod
     def from_info(cls, info: "ExecutorInfo") -> "Self":
         if info.result_hash:
-            result: Optional["ExecutorResult"] = ExecutorResult(
+            result: Optional[ExecutorResult] = ExecutorResult(
                 info.result_hash,
                 (ExpRefInfo.from_ref(info.result_ref) if info.result_ref else None),
                 info.result_force,
@@ -284,7 +289,7 @@ class BaseExecutor(ABC):
             stages = []
             if targets:
                 for target in targets:
-                    stages.append(
+                    stages.append(  # noqa: PERF401
                         dvc.commit(
                             target, recursive=recursive, force=True, relink=False
                         )
@@ -293,14 +298,19 @@ class BaseExecutor(ABC):
                 stages = dvc.commit([], recursive=recursive, force=True, relink=False)
             exp_hash = cls.hash_exp(stages)
             if include_untracked:
-                dvc.scm.add(include_untracked, force=True)  # type: ignore[call-arg]
-            cls.commit(
-                dvc.scm,  # type: ignore[arg-type]
-                exp_hash,
-                exp_name=info.name,
-                force=force,
-                message=message,
-            )
+                from dvc.scm import add_no_submodules
+
+                add_no_submodules(dvc.scm, include_untracked, force=True)  # type: ignore[call-arg]
+
+            with cls.auto_push(dvc):
+                cls.commit(
+                    dvc.scm,  # type: ignore[arg-type]
+                    exp_hash,
+                    exp_name=info.name,
+                    force=force,
+                    message=message,
+                )
+
             ref: Optional[str] = dvc.scm.get_ref(EXEC_BRANCH, follow=False)
             exp_ref = ExpRefInfo.from_ref(ref) if ref else None
             untracked = dvc.scm.untracked_files()
@@ -457,32 +467,29 @@ class BaseExecutor(ABC):
         from dvc.repo.checkout import checkout as dvc_checkout
         from dvc.ui import ui
 
-        auto_push = env2bool(DVC_EXP_AUTO_PUSH)
-        git_remote = os.getenv(DVC_EXP_GIT_REMOTE, None)
-
         if queue is not None:
             queue.put((rev, os.getpid()))
-        if log_errors and log_level is not None:
-            cls._set_log_level(log_level)
+
+        log_ctx = cls._set_log_level(log_level) if log_errors else nullcontext()
 
         exp_hash: Optional[str] = None
-        exp_ref: Optional["ExpRefInfo"] = None
+        exp_ref: Optional[ExpRefInfo] = None
         repro_force: bool = False
 
         if info.name:
             ui.write(f"Reproducing experiment '{info.name}'")
 
-        with cls._repro_dvc(
-            info,
-            infofile,
-            log_errors=log_errors,
-            copy_paths=copy_paths,
-            message=message,
-            **kwargs,
-        ) as dvc:
-            if auto_push:
-                cls._validate_remotes(dvc, git_remote)
-
+        with (
+            log_ctx,
+            cls._repro_dvc(
+                info,
+                infofile,
+                log_errors=log_errors,
+                copy_paths=copy_paths,
+                message=message,
+                **kwargs,
+            ) as dvc,
+        ):
             args, kwargs = cls._repro_args(dvc)
             if args:
                 targets: Optional[Union[list, str]] = args[0]
@@ -508,7 +515,9 @@ class BaseExecutor(ABC):
             stages = dvc.reproduce(*args, **kwargs)
             if paths := cls._get_top_level_paths(dvc):
                 logger.debug("Staging top-level files: %s", paths)
-                dvc.scm_context.add(paths)
+                from dvc.scm import add_no_submodules
+
+                add_no_submodules(dvc.scm, paths)
 
             exp_hash = cls.hash_exp(stages)
             if not repro_dry:
@@ -516,8 +525,6 @@ class BaseExecutor(ABC):
                     dvc,
                     info,
                     exp_hash,
-                    auto_push,
-                    git_remote,
                     repro_force,
                     message=message,
                 )
@@ -547,22 +554,20 @@ class BaseExecutor(ABC):
         dvc,
         info,
         exp_hash,
-        auto_push,
-        git_remote,
         repro_force,
         message: Optional[str] = None,
     ) -> tuple[Optional[str], Optional["ExpRefInfo"], bool]:
-        cls.commit(
-            dvc.scm,
-            exp_hash,
-            exp_name=info.name,
-            force=repro_force,
-            message=message,
-        )
-        if auto_push:
-            cls._auto_push(dvc, dvc.scm, git_remote)
+        with cls.auto_push(dvc):
+            cls.commit(
+                dvc.scm,
+                exp_hash,
+                exp_name=info.name,
+                force=repro_force,
+                message=message,
+            )
+
         ref: Optional[str] = dvc.scm.get_ref(EXEC_BRANCH, follow=False)
-        exp_ref: Optional["ExpRefInfo"] = ExpRefInfo.from_ref(ref) if ref else None
+        exp_ref: Optional[ExpRefInfo] = ExpRefInfo.from_ref(ref) if ref else None
         if cls.WARN_UNTRACKED:
             untracked = dvc.scm.untracked_files()
             if untracked:
@@ -588,9 +593,8 @@ class BaseExecutor(ABC):
         message: Optional[str] = None,
         **kwargs,
     ) -> Iterator["Repo"]:
-        from dvc_studio_client.post_live_metrics import post_live_metrics
-
         from dvc.repo import Repo
+        from dvc_studio_client.post_live_metrics import post_live_metrics
 
         with Repo(os.path.join(info.root_dir, info.dvc_dir)) as dvc:
             info.status = TaskStatus.RUNNING
@@ -615,6 +619,8 @@ class BaseExecutor(ABC):
             # inferring repo url will fail if not set here
             run_env_config = env_to_config(kwargs.get("run_env", {}))
             dvc_studio_config = run_env_config | dvc_studio_config
+            # override studio repo url if exp git remote set
+            repo_url = get_repo_url(dvc)
             try:
                 post_live_metrics(
                     "start",
@@ -624,6 +630,8 @@ class BaseExecutor(ABC):
                     params=to_studio_params(dvc.params.show()),
                     dvc_studio_config=dvc_studio_config,
                     message=message,
+                    subdir=get_subrepo_relpath(dvc),
+                    studio_repo_url=repo_url,
                 )
                 logger.debug("Running repro in '%s'", os.getcwd())
                 yield dvc
@@ -649,6 +657,7 @@ class BaseExecutor(ABC):
                     experiment_rev=dvc.experiments.scm.get_ref(EXEC_BRANCH),
                     metrics=_gather_metrics(dvc, on_error="return"),
                     dvc_studio_config=dvc_studio_config,
+                    studio_repo_url=repo_url,
                 )
 
                 if infofile is not None:
@@ -668,15 +677,46 @@ class BaseExecutor(ABC):
             kwargs = {}
         return args, kwargs
 
+    @classmethod
+    @contextmanager
+    def auto_push(cls, dvc: "Repo") -> Iterator[None]:
+        exp_config = dvc.config.get("exp", {})
+        auto_push = env2bool(DVC_EXP_AUTO_PUSH, exp_config.get("auto_push", False))
+        if not auto_push:
+            yield
+            return
+
+        git_remote = os.getenv(
+            DVC_EXP_GIT_REMOTE, exp_config.get("git_remote", "origin")
+        )
+        try:
+            cls._validate_remotes(dvc, git_remote)
+        except DvcException as exc:
+            logger.warning("Failed to validate remotes. Disabling auto push: %s", exc)
+
+            yield
+            return
+        yield
+        cls._auto_push(dvc, git_remote)
+
     @staticmethod
     def _auto_push(
         dvc: "Repo",
-        scm: "Git",
         git_remote: Optional[str],
         push_cache=True,
         run_cache=True,
     ):
-        branch = scm.get_ref(EXEC_BRANCH, follow=False)
+        from dvc.ui import ui
+        from dvc.utils import format_link
+
+        branch = dvc.scm.get_ref(EXEC_BRANCH, follow=False)
+        link = format_link(
+            "https://dvc.org/doc/user-guide/experiment-management/sharing-experiments"
+        )
+        ui.write(
+            f"Pushing experiment to '{git_remote}'. Cancel with CTRL+C. "
+            f"See {link} for more info."
+        )
         try:
             dvc.experiments.push(
                 git_remote,
@@ -684,7 +724,7 @@ class BaseExecutor(ABC):
                 push_cache=push_cache,
                 run_cache=run_cache,
             )
-        except BaseException as exc:  # noqa: BLE001
+        except DvcException as exc:
             logger.warning(
                 (
                     "Something went wrong while auto pushing experiment "
@@ -704,6 +744,7 @@ class BaseExecutor(ABC):
         message: Optional[str] = None,
     ):
         """Commit stages as an experiment and return the commit SHA."""
+
         rev = scm.get_rev()
         if not scm.is_dirty(untracked_files=False):
             logger.debug("No changes to commit")
@@ -753,13 +794,10 @@ class BaseExecutor(ABC):
         return orig_rev
 
     @staticmethod
-    def _set_log_level(level):
-        # When executor.reproduce is run in a multiprocessing child process,
-        # dvc.cli.main will not be called for that child process so we need to
-        # setup logging ourselves
-        dvc_logger = logging.getLogger("dvc")
+    def _set_log_level(level: Optional[int]) -> AbstractContextManager[None]:
         if level is not None:
-            dvc_logger.setLevel(level)
+            return set_loggers_level(level)
+        return nullcontext()
 
     @staticmethod
     def _copy_path(src, dst):

dvc/repo/experiments/executor/local.py

@@ -96,9 +96,10 @@ class TempDirExecutor(BaseLocalExecutor):
             temp_merge: stash_rev,
             temp_baseline: entry.baseline_rev,
         }
-        with get_exp_rwlock(
-            repo, writes=[temp_head, temp_merge, temp_baseline]
-        ), self.set_temp_refs(scm, temp_ref_dict):
+        with (
+            get_exp_rwlock(repo, writes=[temp_head, temp_merge, temp_baseline]),
+            self.set_temp_refs(scm, temp_ref_dict),
+        ):
             # Executor will be initialized with an empty git repo that
             # we populate by pushing:
             #   EXEC_HEAD - the base commit for this experiment
@@ -130,6 +131,8 @@ class TempDirExecutor(BaseLocalExecutor):
 
         self.scm.stash.apply(merge_rev)
         self._update_config(repo.config.read("local"))
+        local_git_config = os.path.join(repo.scm.root_dir, ".git", "config")
+        self._update_git_config(ConfigObj(local_git_config, list_values=False))
 
     def _update_config(self, update):
         local_config = os.path.join(self.root_dir, self.dvc_dir, "config.local")
@@ -143,6 +146,18 @@ class TempDirExecutor(BaseLocalExecutor):
             with open(local_config, "wb") as fobj:
                 conf_obj.write(fobj)
 
+    def _update_git_config(self, update):
+        local_config = os.path.join(self.scm.root_dir, ".git", "config")
+        logger.debug("Writing experiments local Git config '%s'", local_config)
+        if os.path.exists(local_config):
+            conf_obj = ConfigObj(local_config, list_values=False)
+            conf_obj.merge(update)
+        else:
+            conf_obj = ConfigObj(update, list_values=False)
+        if conf_obj:
+            with open(local_config, "wb") as fobj:
+                conf_obj.write(fobj)
+
     def init_cache(
         self,
         repo: "Repo",
@@ -186,7 +201,7 @@ class WorkspaceExecutor(BaseLocalExecutor):
     @classmethod
     def from_stash_entry(cls, repo: "Repo", entry: "ExpStashEntry", **kwargs):
         root_dir = repo.scm.root_dir
-        executor: "WorkspaceExecutor" = cls._from_stash_entry(
+        executor: WorkspaceExecutor = cls._from_stash_entry(
             repo, entry, root_dir, **kwargs
         )
         logger.debug("Init workspace executor in '%s'", root_dir)

dvc/repo/experiments/pull.py

@@ -30,7 +30,7 @@ def pull(  # noqa: C901
     pull_cache: bool = False,
     **kwargs,
 ) -> Iterable[str]:
-    exp_ref_set: set["ExpRefInfo"] = set()
+    exp_ref_set: set[ExpRefInfo] = set()
     if all_commits:
         exp_ref_set.update(exp_refs(repo.scm, git_remote))
     elif exp_names:
@@ -55,7 +55,7 @@ def pull(  # noqa: C901
         rev_dict = iter_revs(repo.scm, rev, num)
         rev_set = set(rev_dict.keys())
         ref_info_dict = exp_refs_by_baseline(repo.scm, rev_set, git_remote)
-        for _, ref_info_list in ref_info_dict.items():
+        for ref_info_list in ref_info_dict.values():
             exp_ref_set.update(ref_info_list)
 
     pull_result = _pull(repo, git_remote, exp_ref_set, force)
@@ -97,7 +97,7 @@ def _pull(
     def group_result(refspec):
         return results[str(refspec)]
 
-    pull_result: Mapping[SyncStatus, list["ExpRefInfo"]] = group_by(group_result, refs)
+    pull_result: Mapping[SyncStatus, list[ExpRefInfo]] = group_by(group_result, refs)
 
     return pull_result
 

dvc/repo/experiments/push.py

@@ -78,7 +78,7 @@ def exp_refs_from_rev(scm: "Git", rev: list[str], num: int = 1) -> set["ExpRefIn
     rev_dict = iter_revs(scm, rev, num)
     rev_set = set(rev_dict.keys())
     ref_info_dict = exp_refs_by_baseline(scm, rev_set)
-    for _, ref_info_list in ref_info_dict.items():
+    for ref_info_list in ref_info_dict.values():
         exp_ref_set.update(ref_info_list)
     return exp_ref_set
 
@@ -96,7 +96,7 @@ def push(
     push_cache: bool = False,
     **kwargs: Any,
 ) -> dict[str, Any]:
-    exp_ref_set: set["ExpRefInfo"] = set()
+    exp_ref_set: set[ExpRefInfo] = set()
     assert isinstance(repo.scm, Git)
     if all_commits:
         exp_ref_set.update(exp_refs(repo.scm))
@@ -162,7 +162,7 @@ def _push(
     def group_result(refspec):
         return results[str(refspec)]
 
-    pull_result: Mapping[SyncStatus, list["ExpRefInfo"]] = group_by(group_result, refs)
+    pull_result: Mapping[SyncStatus, list[ExpRefInfo]] = group_by(group_result, refs)
 
     return pull_result
 

dvc/repo/experiments/queue/base.py

@@ -4,7 +4,6 @@ from collections.abc import Collection, Generator, Iterable, Mapping
 from dataclasses import asdict, dataclass
 from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Union
 
-from dvc_studio_client.post_live_metrics import get_studio_config
 from funcy import retry
 
 from dvc.dependency import ParamsDependency
@@ -24,6 +23,7 @@ from dvc.repo.experiments.utils import (
 )
 from dvc.utils.objects import cached_property
 from dvc.utils.studio import config_to_env
+from dvc_studio_client.post_live_metrics import get_studio_config
 
 from .utils import get_remote_executor_refs
 
@@ -38,7 +38,7 @@ logger = logger.getChild(__name__)
 
 
 @dataclass(frozen=True)
-class QueueEntry:
+class QueueEntry:  # noqa: PLW1641
     dvc_root: str
     scm_root: str
     stash_ref: str
@@ -180,7 +180,7 @@ class BaseStashQueue(ABC):
 
         def _get_timestamp(rev: str) -> datetime:
             commit = self.scm.resolve_commit(rev)
-            return datetime.fromtimestamp(commit.commit_time)
+            return datetime.fromtimestamp(commit.commit_time)  # noqa: DTZ006
 
         def _format_entry(
             entry: QueueEntry,
@@ -372,7 +372,7 @@ class BaseStashQueue(ABC):
         )
 
     def _stash_commit_deps(self, *args, **kwargs):
-        if len(args):
+        if args:
             targets = args[0]
         else:
             targets = kwargs.get("targets")
@@ -609,9 +609,9 @@ class BaseStashQueue(ABC):
                 result[exp_name] = entry_name_dict[exp_name]
                 continue
             if self.scm.is_sha(exp_name):
-                for rev in entry_rev_dict:
+                for rev, entry in entry_rev_dict.items():
                     if rev.startswith(exp_name.lower()):
-                        result[exp_name] = entry_rev_dict[rev]
+                        result[exp_name] = entry
                         break
 
         return result

dvc/repo/experiments/queue/celery.py

@@ -1,3 +1,4 @@
+import glob
 import hashlib
 import locale
 import logging
@@ -323,7 +324,7 @@ class LocalCeleryQueue(BaseStashQueue):
     def _get_running_task_ids(self) -> set[str]:
         running_task_ids: set[str] = set()
         active_workers = self.worker_status()
-        for _, tasks in active_workers.items():
+        for tasks in active_workers.values():
             task = first(tasks)
             if task:
                 running_task_ids.add(task["id"])
@@ -434,9 +435,13 @@ class LocalCeleryQueue(BaseStashQueue):
         try:
             proc_info = self.proc[queue_entry.stash_rev]
         except KeyError:
-            raise DvcException(  # noqa: B904
-                f"No output logs found for experiment '{rev}'"
-            )
+            message = f"No output logs found for experiment '{rev}'"
+            if self.match_queue_entry_by_name({rev}, self.iter_failed()).get(rev):
+                message += "\nExperiment likely failed during setup."
+                if celery_logs := glob.glob(os.path.join(self.wdir, "*.out")):
+                    message += " Check the celery logs for more details:"
+                    message += f"\n\tcat {' '.join(celery_logs)}".expandtabs(4)
+            raise DvcException(message)  # noqa: B904
         with open(
             proc_info.stdout, encoding=encoding or locale.getpreferredencoding()
         ) as fobj:
@@ -470,14 +475,14 @@ class LocalCeleryQueue(BaseStashQueue):
             exp_names = [exp_names]
         results: dict[str, ExpRefAndQueueEntry] = {}
 
-        exp_ref_match: dict[str, Optional["ExpRefInfo"]] = resolve_name(
+        exp_ref_match: dict[str, Optional[ExpRefInfo]] = resolve_name(
             self.scm, exp_names, git_remote
         )
         if not git_remote:
-            queue_entry_match: dict[
-                str, Optional["QueueEntry"]
-            ] = self.match_queue_entry_by_name(
-                exp_names, self.iter_queued(), self.iter_done()
+            queue_entry_match: dict[str, Optional[QueueEntry]] = (
+                self.match_queue_entry_by_name(
+                    exp_names, self.iter_queued(), self.iter_done()
+                )
             )
 
         for exp_name in exp_names:
@@ -613,7 +618,7 @@ class LocalCeleryQueue(BaseStashQueue):
         """Map exp refs to any available successful executors."""
         from dvc.repo.experiments.serialize import ExpExecutor, LocalExpExecutor
 
-        result: dict[str, "ExpExecutor"] = {}
+        result: dict[str, ExpExecutor] = {}
         for entry, exec_result in self.iter_success():
             if baseline_revs and entry.baseline_rev not in baseline_revs:
                 continue

dvc/repo/experiments/queue/remove.py

@@ -21,11 +21,11 @@ def remove_tasks(  # noqa: C901, PLR0912
     """
     from celery.result import AsyncResult
 
-    stash_revs: dict[str, "ExpStashEntry"] = {}
-    failed_stash_revs: list["ExpStashEntry"] = []
-    done_entry_set: set["QueueEntry"] = set()
+    stash_revs: dict[str, ExpStashEntry] = {}
+    failed_stash_revs: list[ExpStashEntry] = []
+    done_entry_set: set[QueueEntry] = set()
     stash_rev_all = celery_queue.stash.stash_revs
-    failed_rev_all: dict[str, "ExpStashEntry"] = {}
+    failed_rev_all: dict[str, ExpStashEntry] = {}
     if celery_queue.failed_stash:
         failed_rev_all = celery_queue.failed_stash.stash_revs
     for entry in queue_entries:
@@ -90,17 +90,17 @@ def celery_clear(
     """
 
     removed: list[str] = []
-    entry_list: list["QueueEntry"] = []
+    entry_list: list[QueueEntry] = []
     if queued:
-        queue_entries: list["QueueEntry"] = list(self.iter_queued())
+        queue_entries: list[QueueEntry] = list(self.iter_queued())
         entry_list.extend(queue_entries)
         removed.extend(_get_names(queue_entries))
     if failed:
-        failed_tasks: list["QueueDoneResult"] = list(self.iter_failed())
+        failed_tasks: list[QueueDoneResult] = list(self.iter_failed())
         entry_list.extend([result.entry for result in failed_tasks])
         removed.extend(_get_names(failed_tasks))
     if success:
-        success_tasks: list["QueueDoneResult"] = list(self.iter_success())
+        success_tasks: list[QueueDoneResult] = list(self.iter_success())
         entry_list.extend([result.entry for result in success_tasks])
         removed.extend(_get_names(success_tasks))
 
@@ -125,7 +125,7 @@ def celery_remove(self: "LocalCeleryQueue", revs: Collection[str]) -> list[str]:
 
     remained: list[str] = []
     removed: list[str] = []
-    entry_to_remove: list["QueueEntry"] = []
+    entry_to_remove: list[QueueEntry] = []
     for name, entry in match_results.items():
         if entry:
             entry_to_remove.append(entry)

dvc/repo/experiments/queue/tempdir.py

@@ -125,7 +125,7 @@ class TempDirQueue(WorkspaceQueue):
                 )
         except DvcException:
             raise
-        except Exception as exc:  # noqa: BLE001
+        except Exception as exc:
             raise DvcException(f"Failed to reproduce experiment '{rev[:7]}'") from exc
         finally:
             executor.cleanup(infofile)

dvc/repo/experiments/queue/utils.py

@@ -25,7 +25,7 @@ def get_remote_executor_refs(scm: "Git", remote_url: str) -> list[str]:
     refs = []
     for ref in iter_remote_refs(scm, remote_url, base=EXPS_NAMESPACE):
         if not ref.startswith(EXEC_NAMESPACE) and ref != EXPS_STASH:
-            refs.append(ref)
+            refs.append(ref)  # noqa: PERF401
     return refs
 
 

dvc/repo/experiments/queue/workspace.py

@@ -131,7 +131,7 @@ class WorkspaceQueue(BaseStashQueue):
                 )
         except DvcException:
             raise
-        except Exception as exc:  # noqa: BLE001
+        except Exception as exc:
             raise DvcException(f"Failed to reproduce experiment '{rev[:7]}'") from exc
         finally:
             executor.cleanup(infofile)

dvc/repo/experiments/refs.py

@@ -67,3 +67,12 @@ class ExpRefInfo:
         baseline_sha = parts[2] + parts[3]
         name = parts[4] if len(parts) == 5 else None
         return cls(baseline_sha, name)
+
+    def __eq__(self, other):
+        if not isinstance(other, ExpRefInfo):
+            return False
+
+        return self.baseline_sha == other.baseline_sha and self.name == other.name
+
+    def __hash__(self):
+        return hash((self.baseline_sha, self.name))

dvc/repo/experiments/remove.py

@@ -6,7 +6,7 @@ from dvc.repo import locked
 from dvc.repo.scm_context import scm_context
 from dvc.scm import Git, iter_revs
 
-from .exceptions import UnresolvedExpNamesError
+from .exceptions import InvalidArgumentError, UnresolvedExpNamesError
 from .utils import exp_refs, exp_refs_by_baseline, push_refspec
 
 if TYPE_CHECKING:
@@ -24,29 +24,36 @@ logger = logger.getChild(__name__)
 @scm_context
 def remove(  # noqa: C901, PLR0912
     repo: "Repo",
-    exp_names: Union[None, str, list[str]] = None,
+    exp_names: Union[str, list[str], None] = None,
     rev: Optional[Union[list[str], str]] = None,
     all_commits: bool = False,
     num: int = 1,
     queue: bool = False,
     git_remote: Optional[str] = None,
+    keep: bool = False,
 ) -> list[str]:
     removed: list[str] = []
+
+    if all([keep, queue]):
+        raise InvalidArgumentError("Cannot use both `--keep` and `--queue`.")
+
     if not any([exp_names, queue, all_commits, rev]):
         return removed
-    celery_queue: "LocalCeleryQueue" = repo.experiments.celery_queue
+
+    celery_queue: LocalCeleryQueue = repo.experiments.celery_queue
 
     if queue:
         removed.extend(celery_queue.clear(queued=True))
 
     assert isinstance(repo.scm, Git)
 
-    exp_ref_list: list["ExpRefInfo"] = []
-    queue_entry_list: list["QueueEntry"] = []
+    exp_ref_list: list[ExpRefInfo] = []
+    queue_entry_list: list[QueueEntry] = []
+
     if exp_names:
-        results: dict[
-            str, "ExpRefAndQueueEntry"
-        ] = celery_queue.get_ref_and_entry_by_names(exp_names, git_remote)
+        results: dict[str, ExpRefAndQueueEntry] = (
+            celery_queue.get_ref_and_entry_by_names(exp_names, git_remote)
+        )
         remained: list[str] = []
         for name, result in results.items():
             if not result.exp_ref_info and not result.queue_entry:
@@ -68,6 +75,10 @@ def remove(  # noqa: C901, PLR0912
         exp_ref_list.extend(exp_ref_dict.values())
     elif all_commits:
         exp_ref_list.extend(exp_refs(repo.scm, git_remote))
+        removed.extend([ref.name for ref in exp_ref_list])
+
+    if keep:
+        exp_ref_list = list(set(exp_refs(repo.scm, git_remote)) - set(exp_ref_list))
         removed = [ref.name for ref in exp_ref_list]
 
     if exp_ref_list:
@@ -83,6 +94,7 @@ def remove(  # noqa: C901, PLR0912
 
         removed_refs = [str(r) for r in exp_ref_list]
         notify_refs_to_studio(repo, git_remote, removed=removed_refs)
+
     return removed
 
 
@@ -94,11 +106,11 @@ def _resolve_exp_by_baseline(
 ) -> dict[str, "ExpRefInfo"]:
     assert isinstance(repo.scm, Git)
 
-    commit_ref_dict: dict[str, "ExpRefInfo"] = {}
+    commit_ref_dict: dict[str, ExpRefInfo] = {}
     rev_dict = iter_revs(repo.scm, rev, num)
     rev_set = set(rev_dict.keys())
     ref_info_dict = exp_refs_by_baseline(repo.scm, rev_set, git_remote)
-    for _, ref_info_list in ref_info_dict.items():
+    for ref_info_list in ref_info_dict.values():
         for ref_info in ref_info_list:
             commit_ref_dict[ref_info.name] = ref_info
     return commit_ref_dict

dvc/repo/experiments/run.py

@@ -93,7 +93,7 @@ def run(  # noqa: C901, PLR0912
 
     for idx, sweep_overrides in enumerate(sweeps):
         if hydra_sweep and name_prefix is not None:
-            kwargs["name"] = f"{name_prefix}-{idx+1}"
+            kwargs["name"] = f"{name_prefix}-{idx + 1}"
         queue_entry = repo.experiments.queue_one(
             repo.experiments.celery_queue,
             targets=targets,