fix(data status): handle missing DVC repo at Git HEAD (#10862 )

Avoid raising errors in `data_status()` when Git HEAD does not contain a DVC repository. Instead, fall back to an empty index. Tests now cover both unborn and committed Git repos, with and without subdirs.
[pre-commit.ci] pre-commit autoupdate (#10860 )
2026-11-02 07:18:25 +01:00 · 2025-09-02 12:15:20 +05:45 · 2025-09-02 11:12:59 +05:45 · 2025-09-02 10:52:27 +05:45 · 2025-08-31 11:25:31 +05:45 · 2025-08-29 21:13:15 +05:45
181 changed files with 4985 additions and 1190 deletions
@@ -835,9 +835,9 @@
    "tests/func/test_dataset.py::test_dvc": 0.46001330000012786,
    "tests/func/test_dataset.py::test_dvc_dataset_pipeline": 0.7948393000000351,
    "tests/func/test_dataset.py::test_dvc_dump": 0.17996560000005957,
-    "tests/func/test_dataset.py::test_dvcx": 0.22482930000001033,
-    "tests/func/test_dataset.py::test_dvcx_dataset_pipeline": 0.43405649999999696,
-    "tests/func/test_dataset.py::test_dvcx_dump": 0.1858166000000665,
+    "tests/func/test_dataset.py::test_datachain": 0.22482930000001033,
+    "tests/func/test_dataset.py::test_datachain_dataset_pipeline": 0.43405649999999696,
+    "tests/func/test_dataset.py::test_datachain_dump": 0.1858166000000665,
    "tests/func/test_dataset.py::test_invalidation": 0.16291839999996682,
    "tests/func/test_dataset.py::test_parametrized": 0.19277699999997822,
    "tests/func/test_dataset.py::test_pipeline_when_not_in_sync": 0.2063928999999689,
@@ -1627,7 +1627,7 @@
    "tests/unit/command/test_data_sync.py::test_pull": 0.17549940000003517,
    "tests/unit/command/test_data_sync.py::test_push": 0.16576090000000931,
    "tests/unit/command/test_dataset.py::test_add[spec0-lock0-Adding ds (url:/path @ main)\\n]": 0.22054290000005494,
-    "tests/unit/command/test_dataset.py::test_add[spec1-lock1-Adding mydataset (dvcx://dataset @ v1)\\n]": 0.22979979999990974,
+    "tests/unit/command/test_dataset.py::test_add[spec1-lock1-Adding mydataset (dc://dataset @ v1)\\n]": 0.22979979999990974,
    "tests/unit/command/test_dataset.py::test_add[spec2-lock2-Adding mydataset (s3://bucket/path)\\n]": 0.20264819999988504,
    "tests/unit/command/test_dataset.py::test_add_already_exists": 0.2456899000001158,
    "tests/unit/command/test_dataset.py::test_update[spec0-old_lock0-new_lock0-expected_outputs0-missing]": 0.2354315999998562,
@@ -16,19 +16,23 @@ jobs:
    steps:
      - uses: actions/setup-python@v5
        with:
-            python-version: "3.12"
+            python-version: "3.13"

-      - uses: hynek/setup-cached-uv@v2
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
        with:
          ref: ${{ github.event.pull_request.base.sha }}
          fetch-depth: 0

+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: pyproject.toml
+
      - run: uv pip install '.[tests]' --system
      - name: run benchmarks on base branch
        run: pytest --benchmark-autosave dvc/testing/benchmarks/ -k 'test_init or test_help'

-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
        with:
          fetch-depth: 0
          clean: false
@@ -19,31 +19,33 @@ jobs:
    name: Build & Verify package
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
      with:
        fetch-depth: 0

    - uses: actions/setup-python@v5
      with:
-        python-version: 3.12
+        python-version: 3.13

-    - uses: hynek/setup-cached-uv@v2
-    - run: uv pip install --upgrade setuptools_scm build twine --system
+    - uses: astral-sh/setup-uv@v6

-    - name: Force version for Test PyPI upload
-      if: ${{ !startsWith(github.ref, 'refs/tags') }}
-      run: echo version=$(python -m setuptools_scm | awk -F+ '{print $1}' | tail -1) >> $GITHUB_ENV
+    - name: Omit local version for Test PyPI upload
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+      run: echo SETUPTOOLS_SCM_OVERRIDES_FOR_DVC='{local_scheme="no-local-version"}' >> $GITHUB_ENV

-    - run: echo 'PKG = "pip"'>dvc/_build.py
-    - run: python -m build --installer=uv
-      env:
-        SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DVC: ${{ env.version }}
-    - run: twine check --strict dist/*
+    - name: Build Python Package
+      run: |
+        echo 'PKG = "pip"'>dvc/_build.py
+        uv build
+
+    - name: Check dist
+      run: uv tool run twine check --strict dist/*

    - uses: actions/upload-artifact@v4
      with:
        name: Packages
        path: dist/
+        if-no-files-found: error

  test-pypi-publish:
    name: Publish dev package to test.pypi.org
@@ -53,13 +55,13 @@ jobs:

    environment:
      name: test-pypi
-      url: https://test.pypi.org/p/dvc
+      url: https://test.pypi.org/p/dvc/${{ github.event.release.tag_name }}

    permissions:
      id-token: write

    steps:
-    - uses: actions/download-artifact@v4
+    - uses: actions/download-artifact@v5
      with:
        name: Packages
        path: dist
@@ -78,13 +80,13 @@ jobs:

    environment:
      name: pypi
-      url: https://pypi.org/p/dvc
+      url: https://pypi.org/p/dvc/${{ github.event.release.tag_name }}

    permissions:
      id-token: write

    steps:
-    - uses: actions/download-artifact@v4
+    - uses: actions/download-artifact@v5
      with:
        name: Packages
        path: dist
@@ -28,7 +28,7 @@ jobs:

    steps:
    - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v5

    - name: Initialize CodeQL
      uses: github/codeql-action/init@v3
@@ -19,15 +19,15 @@ jobs:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
-        pyv: ["3.12"]
+        pyv: ["3.13"]
        plugin: ["dvc-s3"]

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
      with:
        path: dvc

-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
      with:
        repository: iterative/${{ matrix.plugin }}
        ref: main
@@ -38,11 +38,11 @@ jobs:
      with:
        python-version: ${{ matrix.pyv }}

-    - uses: hynek/setup-cached-uv@v2
+    - uses: astral-sh/setup-uv@v6
      with:
-        cache-dependency-path: |
-          dvc/pyproject.toml
-          ${{ matrix.plugin }}/pyproject.toml
+        enable-cache: true
+        cache-dependency-glob: "**/pyproject.toml"
+        cache-suffix: ${{ matrix.pyv }}

    - name: Install plugin + DVC@PR
      run: |
@@ -19,21 +19,24 @@ jobs:
  lint:
    timeout-minutes: 10
    runs-on: ${{ matrix.os }}
+    env:
+       DVC_NO_ANALYTICS: true
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
      with:
        fetch-depth: 0
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
-        python-version: '3.12'
-    - uses: hynek/setup-cached-uv@v2
+        python-version: '3.13'
+    - uses: astral-sh/setup-uv@v6
      with:
-        cache-dependency-path: pyproject.toml
+        enable-cache: true
+        cache-dependency-glob: pyproject.toml
    - run: uv pip install -e ".[dev]" --system
      env:
        UV_LINK_MODE: ${{ runner.os == 'Windows' && 'symlink' || 'hardlink' }}
@@ -52,7 +55,7 @@ jobs:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest]
-        pyv: ["3.9", "3.10", "3.11", "3.12"]
+        pyv: ["3.9", "3.10", "3.11", "3.12", "3.13"]
        include:
        - os: windows-latest
          pyv: "3.9"
@@ -66,6 +69,30 @@ jobs:
        - os: windows-latest
          pyv: "3.9"
          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
+        - os: windows-latest
+          pyv: "3.10"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
+        - os: windows-latest
+          pyv: "3.10"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
+        - os: windows-latest
+          pyv: "3.10"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
+        - os: windows-latest
+          pyv: "3.10"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
+        - os: windows-latest
+          pyv: "3.11"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
+        - os: windows-latest
+          pyv: "3.11"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
+        - os: windows-latest
+          pyv: "3.11"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
+        - os: windows-latest
+          pyv: "3.11"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
        - os: windows-latest
          pyv: "3.12"
          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
@@ -78,18 +105,31 @@ jobs:
        - os: windows-latest
          pyv: "3.12"
          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
+        - os: windows-latest
+          pyv: "3.13"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
+        - os: windows-latest
+          pyv: "3.13"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
+        - os: windows-latest
+          pyv: "3.13"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
+        - os: windows-latest
+          pyv: "3.13"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
      with:
        fetch-depth: 0
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.pyv }}
-    - uses: hynek/setup-cached-uv@v2
+    - uses: astral-sh/setup-uv@v6
      with:
+        enable-cache: true
+        cache-dependency-glob: pyproject.toml
        cache-suffix: ${{ matrix.pyv }}
-        cache-dependency-path: pyproject.toml
    - run: uv pip install -e ".[dev]" pytest-split --system
      env:
        UV_LINK_MODE: ${{ runner.os == 'Windows' && 'symlink' || 'hardlink' }}
@@ -97,15 +137,13 @@ jobs:
    - name: run tests
      timeout-minutes: 40
      env:
+        COVERAGE_CORE: sysmon
        PYTHONUTF8: 1
      run: >
        pytest ${{ matrix.pytestargs }} -n=logical --dist=worksteal --timeout=300 --durations=0
        --cov --cov-report=xml --cov-report=term  --durations-path=./.github/.test_durations
    - name: upload coverage report
-      uses: codecov/codecov-action@v3
-      with:
-        file: ./coverage.xml
-        fail_ci_if_error: false
+      uses: codecov/codecov-action@v5
  check:
    if: always()
    needs: [tests]
@@ -120,7 +158,7 @@ jobs:
    runs-on: ubuntu-latest
    steps:
    - name: Slack Notification
-      uses: rtCamp/[email protected].0
+      uses: rtCamp/[email protected].3
      env:
        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
        SLACK_COLOR: ${{ job.status }}
@@ -43,7 +43,7 @@ scripts/ci/gcp-creds.json
 .gcp-creds.json

 *~
-/dvc/_dvc_version.py
+/dvc/_version.py

 .mypy_cache/
 .pytest_cache/
@@ -6,7 +6,7 @@ ci:

 repos:
  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v6.0.0
    hooks:
      - id: check-added-large-files
      - id: check-case-conflict
@@ -22,13 +22,13 @@ repos:
      - id: sort-simple-yaml
      - id: trailing-whitespace
  - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.6.2'
+    rev: 'v0.12.11'
    hooks:
-      - id: ruff
+      - id: ruff-check
        args: [--fix, --exit-non-zero-on-fix]
      - id: ruff-format
  - repo: https://github.com/codespell-project/codespell
-    rev: v2.3.0
+    rev: v2.4.1
    hooks:
      - id: codespell
        additional_dependencies: ["tomli"]
@@ -50,7 +50,7 @@ repos:
        language: system
        name: DVC pre-commit
        stages:
-          - commit
+          - pre-commit
        verbose: true
        require_serial: true
      - args:
@@ -61,7 +61,7 @@ repos:
        language: system
        name: DVC pre-push
        stages:
-          - push
+          - pre-push
        require_serial: true
      - always_run: true
        args:
@@ -8,7 +8,7 @@
  name: DVC pre-commit
  require_serial: true
  stages:
-  - commit
+  - pre-commit
  verbose: true
 - args:
  - git-hook
@@ -20,7 +20,7 @@
  name: DVC pre-push
  require_serial: true
  stages:
-  - push
+  - pre-push
 - always_run: true
  args:
  - git-hook
@@ -29,7 +29,7 @@
  id: dvc-post-checkout
  language: python
  language_version: python3
-  minimum_pre_commit_version: 2.2.0
+  minimum_pre_commit_version: 3.2.0
  name: DVC post-checkout
  require_serial: true
  stages:
@@ -1,6 +1,4 @@
-🚀 Check out our new product `DataChain <https://github.com/iterative/datachain>`_ (and give it a ⭐!) if you need to version and process a large number of files. Contact us at [email protected] to discuss commercial solutions and support for AI reproducibility and data management scenarios.
-
--------------------------
+|Banner|

 `Website <https://dvc.org>`_
 • `Docs <https://dvc.org/doc>`_
@@ -215,7 +213,7 @@ Community and Support
 * `Forum <https://discuss.dvc.org/>`_
 * `Discord Chat <https://dvc.org/chat>`_
 * `Email <mailto:[email protected]>`_
-* `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
+* `Mailing List <https://dvc.org/community#subscribe>`_

 Copyright
 =========
@@ -242,11 +240,11 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
 .. |VS Code Extension Overview| image:: https://raw.githubusercontent.com/iterative/vscode-dvc/main/extension/docs/overview.gif
   :alt: DVC Extension for VS Code

-.. |CI| image:: https://github.com/iterative/dvc/workflows/Tests/badge.svg?branch=main
-   :target: https://github.com/iterative/dvc/actions
+.. |CI| image:: https://github.com/iterative/dvc/actions/workflows/tests.yaml/badge.svg
+   :target: https://github.com/iterative/dvc/actions/workflows/tests.yaml
   :alt: GHA Tests

-.. |Maintainability| image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
+.. |Maintainability| image:: https://img.shields.io/codeclimate/maintainability/iterative/dvc
   :target: https://codeclimate.com/github/iterative/dvc
   :alt: Code Climate

@@ -270,7 +268,7 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
   :target: https://formulae.brew.sh/formula/dvc
   :alt: Homebrew

-.. |Conda| image:: https://img.shields.io/conda/v/conda-forge/dvc.svg?label=conda&logo=conda-forge
+.. |Conda| image:: https://anaconda.org/conda-forge/dvc/badges/version.svg
   :target: https://anaconda.org/conda-forge/dvc
   :alt: Conda-forge

@@ -14,7 +14,7 @@ def viztracer_profile(
    log_async: bool = False,
 ):
    try:
-        import viztracer
+        import viztracer  # ty: ignore[unresolved-import]
    except ImportError:
        print("Failed to run profiler, viztracer is not installed")  # noqa: T201
        yield
@@ -37,7 +37,7 @@ def yappi_profile(
    separate_threads: Optional[bool] = False,
 ):
    try:
-        import yappi
+        import yappi  # ty: ignore[unresolved-import]
    except ImportError:
        print("Failed to run profiler, yappi is not installed")  # noqa: T201
        yield
@@ -79,7 +79,7 @@ def yappi_profile(
 def instrument(html_output=False):
    """Run a statistical profiler"""
    try:
-        from pyinstrument import Profiler
+        from pyinstrument import Profiler  # ty: ignore[unresolved-import]
    except ImportError:
        print("Failed to run profiler, pyinstrument is not installed")  # noqa: T201
        yield
@@ -123,7 +123,7 @@ def debug():
        yield
    except Exception:
        try:
-            import ipdb as pdb  # noqa: T100
+            import ipdb as pdb  # noqa: T100  # ty: ignore[unresolved-import]
        except ImportError:
            import pdb  # type: ignore[no-redef]  # noqa: T100
        pdb.post_mortem()
@@ -1,10 +1,14 @@
 import json
 import os
+from typing import TYPE_CHECKING, Optional

 from dvc.log import logger

 from .env import DVC_ANALYTICS_ENDPOINT, DVC_NO_ANALYTICS

+if TYPE_CHECKING:
+    from dvc.scm import Base
+
 logger = logger.getChild(__name__)


@@ -67,6 +71,7 @@ def send(path):
    thus, the need of removing such file afterwards.
    """
    import requests
+    from requests.exceptions import RequestException

    url = os.environ.get(DVC_ANALYTICS_ENDPOINT, "https://analytics.dvc.org")
    headers = {"content-type": "application/json"}
@@ -81,7 +86,7 @@ def send(path):

    try:
        requests.post(url, json=report, headers=headers, timeout=5)
-    except requests.exceptions.RequestException as e:
+    except RequestException as e:
        logger.trace("", exc_info=True)
        logger.debug("failed to send analytics report %s", str(e))

@@ -89,20 +94,62 @@ def send(path):
    os.remove(path)


-def _scm_in_use():
-    from dvc.exceptions import NotDvcRepoError
-    from dvc.repo import Repo
-    from dvc.scm import NoSCM
+def _git_remote_url(scm: Optional["Base"]) -> Optional[str]:
+    from dvc.scm import Git

-    from .scm import SCM, SCMError
+    if not isinstance(scm, Git):
+        return None
+
+    from dulwich.porcelain import get_remote_repo
+
+    dulwich_repo = scm.dulwich.repo
+    try:
+        _remote, url = get_remote_repo(dulwich_repo)
+    except IndexError:
+        # IndexError happens when the head is detached
+        _remote, url = get_remote_repo(dulwich_repo, b"origin")
+    # Dulwich returns (None, "origin") if no remote set
+    if (_remote, url) == (None, "origin"):
+        return None
+    return url
+
+
+def _scm_in_use(scm: Optional["Base"]) -> Optional[str]:
+    return type(scm).__name__ if scm else None
+
+
+def _parse_git_remote_path(remote_url: str) -> str:
+    from urllib.parse import urlparse
+
+    from scmrepo.urls import is_scp_style_url
+
+    parsed = urlparse(remote_url)
+    # Windows Path also gets parsed with a drive letter as scheme
+    # https://github.com/python/cpython/issues/86381
+    if parsed.scheme and parsed.scheme in ("http", "https", "git", "ssh"):
+        return parsed.path.strip("/")
+
+    if is_scp_style_url(remote_url):
+        # handle scp-style URL
+        parts = remote_url.split(":", 1)
+        if len(parts) == 2:
+            _, path = parts
+            return path.rstrip("/")
+    return remote_url
+
+
+def _git_remote_path_hash(scm: Optional["Base"]) -> Optional[str]:
+    """Return a hash of the git remote path."""
+    import hashlib

    try:
-        scm = SCM(root_dir=Repo.find_root())
-        return type(scm).__name__
-    except SCMError:
-        return NoSCM.__name__
-    except NotDvcRepoError:
-        pass
+        if remote_url := _git_remote_url(scm):
+            path = _parse_git_remote_path(remote_url)
+            h = hashlib.md5(path.encode("utf-8"), usedforsecurity=False)  # for FIPS
+            return h.hexdigest()
+    except Exception:  # noqa: BLE001
+        logger.debug("Failed to get git remote path", exc_info=True)
+    return None


 def _runtime_info():
@@ -112,6 +159,8 @@ def _runtime_info():
    from iterative_telemetry import _generate_ci_id, find_or_create_user_id

    from dvc import __version__
+    from dvc.info import _get_remotes
+    from dvc.repo import Repo
    from dvc.utils import is_binary

    ci_id = _generate_ci_id()
@@ -120,13 +169,24 @@ def _runtime_info():
    else:
        group_id, user_id = None, find_or_create_user_id()

+    scm = None
+    remotes = None
+    try:
+        repo = Repo()
+        scm = repo.scm
+        remotes = _get_remotes(repo.config)
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("failed to open repo: %s", exc)
+
    return {
        "dvc_version": __version__,
        "is_binary": is_binary(),
-        "scm_class": _scm_in_use(),
+        "scm_class": _scm_in_use(scm),
        "system_info": _system_info(),
        "user_id": user_id,
        "group_id": group_id,
+        "remotes": remotes,
+        "git_remote_hash": _git_remote_path_hash(scm),
    }


@@ -1,7 +1,7 @@
 from dvc.fs.dvc import _DVCFileSystem as DVCFileSystem

 from .artifacts import artifacts_show
-from .data import get_url, open, read
+from .data import get_url, open, read  # noqa: A004
 from .dataset import get as get_dataset
 from .experiments import exp_save, exp_show
 from .scm import all_branches, all_commits, all_tags
@@ -1,8 +1,8 @@
 from typing import Literal, TypedDict, Union


-class DVCXDataset(TypedDict):
-    type: Literal["dvcx"]
+class DatachainDataset(TypedDict):
+    type: Literal["dc"]
    name: str
    version: int

@@ -20,7 +20,7 @@ class URLDataset(TypedDict):
    path: str


-def get(name: str) -> Union[DVCXDataset, DVCDataset, URLDataset]:
+def get(name: str) -> Union[DatachainDataset, DVCDataset, URLDataset]:
    from difflib import get_close_matches

    from dvc.fs import get_cloud_fs
@@ -46,9 +46,9 @@ def get(name: str) -> Union[DVCXDataset, DVCDataset, URLDataset]:
            path=dataset.lock.path,
            sha=dataset.lock.rev_lock,
        )
-    if dataset.type == "dvcx":
-        return DVCXDataset(
-            type="dvcx", name=dataset.name_version[0], version=dataset.lock.version
+    if dataset.type == "dc":
+        return DatachainDataset(
+            type="dc", name=dataset.name_version[0], version=dataset.lock.version
        )
    if dataset.type == "url":
        fs_cls, _, path = get_cloud_fs(repo.config, url=dataset.lock.url)
@@ -27,8 +27,7 @@ def _postprocess(results):
            }
            processed[rev] = processed[rev] | to_merge

-    if "workspace" in processed:
-        del processed["workspace"]
+    processed.pop("workspace", None)

    return processed

@@ -86,7 +86,7 @@ class CacheManager:
        return FileStorage(
            key=(),
            fs=self.local.fs,
-            path=self.local.fs.join(self.default_local_cache_dir, self.FS_DIR),
+            path=self.local.fs.join(self.local_cache_dir, self.FS_DIR),
        )

    def _init_odb(self, schemes):
@@ -157,6 +157,8 @@ def main(argv=None):  # noqa: C901, PLR0912, PLR0915
    Returns:
        int: command's return code.
    """
+    from contextlib import ExitStack
+
    from dvc._debug import debugtools
    from dvc.config import ConfigError
    from dvc.exceptions import DvcException, NotDvcRepoError
@@ -173,12 +175,11 @@ def main(argv=None):  # noqa: C901, PLR0912, PLR0915
        logging.disable(logging.INFO)

    args = None
-
-    outer_log_level = logger.level
-    level = None
+    stack = ExitStack()
    try:
        args = parse_args(argv)

+        level = None
        if args.quiet:
            level = logging.CRITICAL
        elif args.verbose == 1:
@@ -187,7 +188,7 @@ def main(argv=None):  # noqa: C901, PLR0912, PLR0915
            level = logging.TRACE  # type: ignore[attr-defined]

        if level is not None:
-            set_loggers_level(level)
+            stack.enter_context(set_loggers_level(level))

        if level and level <= logging.DEBUG:
            from platform import platform, python_implementation, python_version
@@ -247,7 +248,7 @@ def main(argv=None):  # noqa: C901, PLR0912, PLR0915

        return ret
    finally:
-        logger.setLevel(outer_log_level)
+        stack.close()

        from dvc.repo.open_repo import clean_repos

@@ -18,7 +18,10 @@ class CmdBase(ABC):

        os.chdir(args.cd)

-        self.repo: Repo = Repo(uninitialized=self.UNINITIALIZED)
+        self.repo: Repo = Repo(
+            uninitialized=self.UNINITIALIZED,
+            _wait_for_lock=args.wait_for_lock,
+        )
        self.config: Config = self.repo.config
        self.args = args

@@ -22,6 +22,11 @@ _dvc_compgen_exps() {
    local _dvc_exps=($(dvc exp list -q --all-commits --names-only))
    compgen -W "${_dvc_exps[*]}" -- $1
 }
+
+_dvc_compgen_remotes() {
+    local _dvc_remotes=($(dvc remote list | cut -d' ' -f1))
+    compgen -W "${_dvc_remotes[*]}" -- $1
+}
 """

 ZSH_PREAMBLE = """
@@ -55,6 +60,10 @@ _dvc_compadd_stages_and_files() {
 _dvc_compadd_exps() {
    _describe 'experiments' "($(dvc exp list -q --all-commits --names-only))"
 }
+
+_dvc_compadd_remotes() {
+    _describe 'remotes' "($(dvc remote list | cut -d' ' -f1))"
+}
 """

 PREAMBLE = {
@@ -71,3 +80,4 @@ DVCFILES_AND_STAGE = {
    "zsh": "_dvc_compadd_stages_and_files",
 }
 EXPERIMENT = {"bash": "_dvc_compgen_exps", "zsh": "_dvc_compadd_exps"}
+REMOTE = {"bash": "_dvc_compgen_remotes", "zsh": "_dvc_compadd_remotes"}
@@ -200,6 +200,14 @@ def get_main_parser():
        type=str,
    )

+    parser.add_argument(
+        "--wait-for-lock",
+        action="store_true",
+        default=False,
+        help="Wait for the lock if it is already held by another process, instead of"
+        " failing immediately.",
+    )
+
    # Sub commands
    subparsers = parser.add_subparsers(
        title="Available Commands",
@@ -40,7 +40,7 @@ def hide_subparsers_from_help(subparsers):
    # from the 'positional arguments' choices list
    # see: https://bugs.python.org/issue22848
    # Need to set `add_help=False`, but avoid setting `help`
-    # (not even to `argparse.SUPPPRESS`).
+    # (not even to `argparse.SUPPRESS`).
    # NOTE: The argument is the parent subparser, not the subcommand parser.
    cmds = [cmd for cmd, parser in subparsers.choices.items() if parser.add_help]
    subparsers.metavar = "{{{}}}".format(",".join(cmds))
@@ -101,7 +101,7 @@ def add_parser(subparsers, parent_parser):
        "--remote",
        help="Remote storage to download to",
        metavar="<name>",
-    )
+    ).complete = completion.REMOTE
    parser.add_argument(
        "--remote-jobs",
        type=int,
@@ -149,7 +149,7 @@ def add_parser(subparsers, parent_parser):
            "Remote name to set as a default in the target repository "
            "(only applicable when downloading from DVC remote)."
        ),
-    )
+    ).complete = completion.REMOTE
    get_parser.add_argument(
        "--remote-config",
        type=str,
@@ -1,20 +1,28 @@
+from typing import TYPE_CHECKING
+
 from dvc.cli import completion, formatter
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
 from dvc.ui import ui

+if TYPE_CHECKING:
+    from dvc.ignore import CheckIgnoreResult
+

 class CmdCheckIgnore(CmdBase):
    def __init__(self, args):
        super().__init__(args)
        self.ignore_filter = self.repo.dvcignore

-    def _show_results(self, result):
-        if not result.match and not self.args.non_matching:
+    def _show_results(self, result: "CheckIgnoreResult"):
+        if not result.match and not self.args.details:
            return

        if self.args.details:
-            patterns = result.patterns
+            pattern_infos = result.pattern_infos
+            patterns = [str(pi) for pi in pattern_infos]
+            if not patterns and self.args.non_matching:
+                patterns = ["::"]
            if not self.args.all:
                patterns = patterns[-1:]

@@ -22,13 +30,11 @@ class CmdCheckIgnore(CmdBase):
                ui.write(pattern, result.file, sep="\t")
        else:
            ui.write(result.file)
+        return bool(result.pattern_infos)

    def _check_one_file(self, target):
        result = self.ignore_filter.check_ignore(target)
-        self._show_results(result)
-        if result.match:
-            return 0
-        return 1
+        return bool(self._show_results(result))

    def _interactive_mode(self):
        ret = 1
@@ -39,14 +45,14 @@ class CmdCheckIgnore(CmdBase):
                break
            if not target:
                break
-            if not self._check_one_file(target):
+            if self._check_one_file(target):
                ret = 0
        return ret

    def _normal_mode(self):
        ret = 1
        for target in self.args.targets:
-            if not self._check_one_file(target):
+            if self._check_one_file(target):
                ret = 0
        return ret

@@ -76,6 +82,8 @@ class CmdCheckIgnore(CmdBase):


 def add_parser(subparsers, parent_parser):
+    import argparse
+
    ADD_HELP = "Check whether files or directories are excluded due to `.dvcignore`."

    parser = subparsers.add_parser(
@@ -93,14 +101,7 @@ def add_parser(subparsers, parent_parser):
        help="Show the exclude patterns along with each target path.",
    )
    parser.add_argument(
-        "-a",
-        "--all",
-        action="store_true",
-        default=False,
-        help=(
-            "Include the target paths which don't match any pattern "
-            "in the `--details` list."
-        ),
+        "-a", "--all", action="store_true", default=False, help=argparse.SUPPRESS
    )
    parser.add_argument(
        "-n",
@@ -1,5 +1,3 @@
-import operator
-
 from dvc.cli import completion, formatter
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
@@ -30,7 +28,7 @@ class CmdCheckout(CmdBase):

        stats, exc = None, None
        try:
-            stats = self.repo.checkout(
+            result = self.repo.checkout(
                targets=self.args.targets,
                with_deps=self.args.with_deps,
                force=self.args.force,
@@ -40,14 +38,17 @@ class CmdCheckout(CmdBase):
            )
        except CheckoutError as _exc:
            exc = _exc
-            stats = exc.stats
+            result = exc.result

        if self.args.summary:
            default_message = "No changes."
-            msg = get_summary(sorted(stats.items(), key=operator.itemgetter(0)))
+            stats = result["stats"]
+            assert isinstance(stats, dict)
+            msg = get_summary(stats.items())
            ui.write(msg or default_message)
        else:
-            log_changes(stats)
+            result.pop("stats", {})
+            log_changes(result)

        if exc:
            raise exc
@@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, ClassVar

 from funcy import chunks, compact, log_durations

-from dvc.cli import formatter
+from dvc.cli import completion, formatter
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
 from dvc.log import logger
@@ -10,6 +10,7 @@ from dvc.ui import ui
 from dvc.utils import colorize

 if TYPE_CHECKING:
+    from dvc.repo.data import GitInfo
    from dvc.repo.data import Status as DataStatus


@@ -55,20 +56,15 @@ class CmdDataStatus(CmdBase):
    def _process_status(status: "DataStatus"):
        """Flatten stage status, and filter empty stage status contents."""
        for stage, stage_status in status.items():
-            items = stage_status
-            if isinstance(stage_status, dict):
-                items = {
-                    file: state
-                    for state, files in stage_status.items()
-                    for file in files
-                }
-            if not items:
+            if not stage_status or (
+                isinstance(stage_status, dict) and not any(stage_status.values())
+            ):
                continue
-            yield stage, items
+            yield stage, stage_status

    @classmethod
    def _show_status(cls, status: "DataStatus") -> int:  # noqa: C901
-        git_info = status.pop("git")  # type: ignore[misc]
+        git_info: GitInfo = status.pop("git")  # type: ignore[misc]
        result = dict(cls._process_status(status))
        if not result:
            no_changes = "No changes"
@@ -90,7 +86,16 @@ class CmdDataStatus(CmdBase):
                    ui.write(f"  ({hint})")

            if isinstance(stage_status, dict):
-                items = [f"{state}: {file}" for file, state in stage_status.items()]
+                items = [
+                    f"{state}: "
+                    + (
+                        " -> ".join(change.values())
+                        if isinstance(change, dict)
+                        else change
+                    )
+                    for state, changes in stage_status.items()
+                    for change in changes
+                ]
            else:
                items = stage_status

@@ -108,8 +113,10 @@ class CmdDataStatus(CmdBase):
    def run(self) -> int:
        with log_durations(logger.trace, "in data_status"):
            status = self.repo.data_status(
+                targets=self.args.targets,
                granular=self.args.granular,
                untracked_files=self.args.untracked_files,
+                remote=self.args.remote,
                not_in_remote=self.args.not_in_remote,
                remote_refresh=self.args.remote_refresh,
            )
@@ -126,9 +133,12 @@ class CmdDataStatus(CmdBase):


 def add_parser(subparsers, parent_parser):
+    data_help = "Commands related to data management."
    data_parser = subparsers.add_parser(
        "data",
        parents=[parent_parser],
+        description=append_doc_link(data_help, "data/status"),
+        help=data_help,
        formatter_class=formatter.RawDescriptionHelpFormatter,
    )
    data_subparsers = data_parser.add_subparsers(
@@ -147,6 +157,14 @@ def add_parser(subparsers, parent_parser):
        formatter_class=formatter.RawDescriptionHelpFormatter,
        help=DATA_STATUS_HELP,
    )
+    data_status_parser.add_argument(
+        "targets",
+        nargs="*",
+        help=(
+            "Limit command scope to these tracked files/directories, "
+            ".dvc files and stage names."
+        ),
+    ).complete = completion.FILE  # type: ignore[attr-defined]
    data_status_parser.add_argument(
        "--json",
        action="store_true",
@@ -173,6 +191,12 @@ def add_parser(subparsers, parent_parser):
        nargs="?",
        help="Show untracked files.",
    )
+    data_status_parser.add_argument(
+        "-r",
+        "--remote",
+        help="Remote storage to check (only applicable with --not-in-remote).",
+        metavar="<name>",
+    ).complete = completion.REMOTE
    data_status_parser.add_argument(
        "--not-in-remote",
        action="store_true",
@@ -22,17 +22,18 @@ class CmdDataBase(CmdBase):


 class CmdDataPull(CmdDataBase):
-    def log_summary(self, stats):
+    def log_summary(self, result):
        from dvc.commands.checkout import log_changes

-        log_changes(stats)
+        stats = result.pop("stats", {})
+        log_changes(result)
        super().log_summary(stats)

    def run(self):
        from dvc.exceptions import CheckoutError, DvcException

        try:
-            stats = self.repo.pull(
+            result = self.repo.pull(
                targets=self.args.targets,
                jobs=self.args.jobs,
                remote=self.args.remote,
@@ -46,10 +47,10 @@ class CmdDataPull(CmdDataBase):
                glob=self.args.glob,
                allow_missing=self.args.allow_missing,
            )
-            self.log_summary(stats)
+            self.log_summary(result)
        except (CheckoutError, DvcException) as exc:
-            if stats := getattr(exc, "stats", {}):
-                self.log_summary(stats)
+            if result := getattr(exc, "result", {}):
+                self.log_summary(result)
            logger.exception("failed to pull data from the cloud")
            return 1

@@ -149,7 +150,7 @@ def add_parser(subparsers, _parent_parser):
    )
    pull_parser.add_argument(
        "-r", "--remote", help="Remote storage to pull from", metavar="<name>"
-    )
+    ).complete = completion.REMOTE
    pull_parser.add_argument(
        "-a",
        "--all-branches",
@@ -224,7 +225,7 @@ def add_parser(subparsers, _parent_parser):
    )
    push_parser.add_argument(
        "-r", "--remote", help="Remote storage to push to", metavar="<name>"
-    )
+    ).complete = completion.REMOTE
    push_parser.add_argument(
        "-a",
        "--all-branches",
@@ -286,7 +287,7 @@ def add_parser(subparsers, _parent_parser):
    )
    fetch_parser.add_argument(
        "-r", "--remote", help="Remote storage to fetch from", metavar="<name>"
-    )
+    ).complete = completion.REMOTE
    fetch_parser.add_argument(
        "-a",
        "--all-branches",
@@ -379,7 +380,7 @@ def add_parser(subparsers, _parent_parser):
        "--remote",
        help="Remote storage to compare local cache to",
        metavar="<name>",
-    )
+    ).complete = completion.REMOTE
    status_parser.add_argument(
        "-a",
        "--all-branches",
@@ -34,7 +34,7 @@ class CmdDatasetAdd(CmdBase):

        url = dataset.spec.url
        ver: str = ""
-        if dataset.type == "dvcx":
+        if dataset.type == "dc":
            ver = f"v{dataset.lock.version}"
        if dataset.type == "dvc":
            if dataset.lock.path:
@@ -55,7 +55,7 @@ class CmdDatasetAdd(CmdBase):
            raise DvcException("--path can't be used without --dvc")

        d = vars(self.args)
-        for key in ["dvc", "dvcx", "url"]:
+        for key in ["dvc", "dc", "url"]:
            if url := d.pop(key, None):
                d.update({"type": key, "url": url})
                break
@@ -88,8 +88,8 @@ class CmdDatasetUpdate(CmdBase):
        assert new.lock

        v: Optional[tuple[str, str]] = None
-        if dataset.type == "dvcx":
-            assert new.type == "dvcx"
+        if dataset.type == "dc":
+            assert new.type == "dc"
            if new.lock.version < dataset.lock.version:
                action = "Downgrading"

@@ -168,7 +168,7 @@ def add_parser(subparsers, parent_parser):

    url_exclusive_group = ds_add_parser.add_mutually_exclusive_group(required=True)
    url_exclusive_group.add_argument(
-        "--dvcx", metavar="name", help="Name of the dvcx dataset to track"
+        "--dc", metavar="name", help="Name of the DataChain dataset to track"
    )
    url_exclusive_group.add_argument(
        "--dvc",
@@ -219,7 +219,7 @@ remote://remote_name/path/to/file/or/dir (see `dvc remote`)
        "--rev",
        "--version",
        nargs="?",
-        help="DVCX dataset version or Git revision (e.g. SHA, branch, tag)",
+        help="DataChain dataset version or Git revision (e.g. SHA, branch, tag)",
        metavar="<version>",
    )
    ds_update_parser.set_defaults(func=CmdDatasetUpdate)
@@ -60,7 +60,7 @@ def add_parser(subparsers, parent_parser):
        "--remote",
        type=str,
        help="Remote name to set as a default in the target repository.",
-    )
+    ).complete = completion.REMOTE
    du_parser.add_argument(
        "--remote-config",
        type=str,
@@ -58,6 +58,15 @@ def add_parser(subparsers, parent_parser):
    hide_subparsers_from_help(experiments_subparsers)


+def add_keep_selection_flag(experiments_subcmd_parser):
+    experiments_subcmd_parser.add_argument(
+        "--keep",
+        action="store_true",
+        default=False,
+        help="Keep the selected experiments instead of removing them.",
+    )
+
+
 def add_rev_selection_flags(
    experiments_subcmd_parser, command: str, default: bool = True
 ):
@@ -1,6 +1,6 @@
 import argparse

-from dvc.cli import formatter
+from dvc.cli import completion, formatter
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
 from dvc.log import logger
@@ -71,7 +71,7 @@ def add_parser(experiments_subparsers, parent_parser):
        dest="dvc_remote",
        metavar="<name>",
        help="Name of the DVC remote to use when pulling cached outputs.",
-    )
+    ).complete = completion.REMOTE
    experiments_pull_parser.add_argument(
        "-j",
        "--jobs",
@@ -111,7 +111,7 @@ def add_parser(experiments_subparsers, parent_parser):
        dest="dvc_remote",
        metavar="<name>",
        help="Name of the DVC remote to use when pushing cached outputs.",
-    )
+    ).complete = completion.REMOTE
    experiments_push_parser.add_argument(
        "-j",
        "--jobs",
@@ -34,6 +34,7 @@ class CmdExperimentsRemove(CmdBase):
            num=self.args.num,
            queue=self.args.queue,
            git_remote=self.args.git_remote,
+            keep=self.args.keep,
        )
        if removed:
            ui.write(f"Removed experiments: {humanize.join(map(repr, removed))}")
@@ -44,7 +45,7 @@ class CmdExperimentsRemove(CmdBase):


 def add_parser(experiments_subparsers, parent_parser):
-    from . import add_rev_selection_flags
+    from . import add_keep_selection_flag, add_rev_selection_flags

    EXPERIMENTS_REMOVE_HELP = "Remove experiments."
    experiments_remove_parser = experiments_subparsers.add_parser(
@@ -57,6 +58,7 @@ def add_parser(experiments_subparsers, parent_parser):
    )
    remove_group = experiments_remove_parser.add_mutually_exclusive_group()
    add_rev_selection_flags(experiments_remove_parser, "Remove", False)
+    add_keep_selection_flag(experiments_remove_parser)
    remove_group.add_argument(
        "--queue", action="store_true", help="Remove all queued experiments."
    )
@@ -1,6 +1,6 @@
 import argparse
 import re
-from collections.abc import Iterable
+from collections.abc import Collection, Iterable
 from datetime import date, datetime
 from typing import TYPE_CHECKING

@@ -59,17 +59,16 @@ def baseline_styler(typ):
 def show_experiments(
    td: "TabularData",
    headers: dict[str, Iterable[str]],
-    keep=None,
-    drop=None,
+    keep: Collection[str] = (),
+    drop: Collection[str] = (),
    pager=True,
    csv=False,
    markdown=False,
    **kwargs,
 ):
    if keep:
-        for col in td.keys():  # noqa: SIM118
-            if re.match(keep, col):
-                td.protect(col)
+        keep_re = re.compile("|".join(keep))
+        td.protect(*(col for col in td.keys() if keep_re.match(col)))  # noqa: SIM118

    for col in ("State", "Executor"):
        if td.is_empty(col):
@@ -110,8 +109,9 @@ def show_experiments(
        td.drop_duplicates("cols", ignore_empty=False)

    cols_to_drop = set()
-    if drop is not None:
-        cols_to_drop = {col for col in td.keys() if re.match(drop, col)}  # noqa: SIM118
+    if drop:
+        drop_re = re.compile("|".join(drop))
+        cols_to_drop = {col for col in td.keys() if drop_re.match(col)}  # noqa: SIM118
    td.drop(*cols_to_drop)

    td.render(
@@ -150,6 +150,7 @@ class CmdExperimentsShow(CmdBase):
                all_commits=self.args.all_commits,
                hide_queued=self.args.hide_queued,
                hide_failed=self.args.hide_failed,
+                hide_workspace=self.args.hide_workspace,
                revs=self.args.rev,
                num=self.args.num,
                sha_only=self.args.sha,
@@ -236,11 +237,15 @@ def add_parser(experiments_subparsers, parent_parser):
    )
    experiments_show_parser.add_argument(
        "--drop",
+        action="append",
+        default=[],
        help="Remove the columns matching the specified regex pattern.",
        metavar="<regex_pattern>",
    )
    experiments_show_parser.add_argument(
        "--keep",
+        action="append",
+        default=[],
        help="Preserve the columns matching the specified regex pattern.",
        metavar="<regex_pattern>",
    )
@@ -279,6 +284,12 @@ def add_parser(experiments_subparsers, parent_parser):
        default=False,
        help="Hide queued experiments in the table.",
    )
+    experiments_show_parser.add_argument(
+        "--hide-workspace",
+        action="store_true",
+        default=False,
+        help="Hide workspace row in the table.",
+    )
    experiments_show_parser.add_argument(
        "--json",
        action="store_true",
@@ -1,6 +1,6 @@
 import os

-from dvc.cli import formatter
+from dvc.cli import completion, formatter
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
 from dvc.log import logger
@@ -54,7 +54,7 @@ class CmdGC(CmdBase):
            msg += " and all experiments"

        if self.args.not_in_remote:
-            msg += " that are present in the DVC remote"
+            msg += " that are not present in the DVC remote"

        if self.args.repos:
            msg += " of the current and the following repos:"
@@ -188,7 +188,7 @@ def add_parser(subparsers, parent_parser):
        "--remote",
        help="Remote storage to collect garbage in",
        metavar="<name>",
-    )
+    ).complete = completion.REMOTE
    gc_parser.add_argument(
        "--skip-failed",
        action="store_true",
@@ -121,7 +121,7 @@ def add_parser(subparsers, parent_parser):
        "--remote",
        type=str,
        help="Remote name to set as a default in the target repository.",
-    )
+    ).complete = completion.REMOTE
    get_parser.add_argument(
        "--remote-config",
        type=str,
@@ -115,7 +115,7 @@ def add_parser(subparsers, parent_parser):
        "--remote",
        type=str,
        help="Remote name to set as a default in the target repository.",
-    )
+    ).complete = completion.REMOTE
    import_parser.add_argument(
        "--remote-config",
        type=str,
@@ -73,7 +73,7 @@ def add_parser(subparsers, parent_parser):
        "--remote",
        help="Remote storage to download to",
        metavar="<name>",
-    )
+    ).complete = completion.REMOTE
    no_download_exec_group = import_parser.add_mutually_exclusive_group()
    no_download_exec_group.add_argument(
        "--no-exec",
@@ -42,7 +42,7 @@ class CmdInit(CmdBaseNoRepo):

        try:
            with Repo.init(
-                ".",
+                self.args.directory,
                no_scm=self.args.no_scm,
                force=self.args.force,
                subdir=self.args.subdir,
@@ -57,10 +57,11 @@ class CmdInit(CmdBaseNoRepo):

 def add_parser(subparsers, parent_parser):
    """Setup parser for `dvc init`."""
-    INIT_HELP = "Initialize DVC in the current directory."
+    INIT_HELP = "Initialize DVC repository."
    INIT_DESCRIPTION = (
-        "Initialize DVC in the current directory. Expects directory\n"
-        "to be a Git repository unless --no-scm option is specified."
+        "Initialize DVC repository in the given directory (defaults to the current "
+        "working directory).\n"
+        "Expects directory to be a Git repository unless --no-scm option is specified."
    )

    init_parser = subparsers.add_parser(
@@ -70,6 +71,14 @@ def add_parser(subparsers, parent_parser):
        help=INIT_HELP,
        formatter_class=formatter.RawDescriptionHelpFormatter,
    )
+    init_parser.add_argument(
+        "directory",
+        nargs="?",
+        default=".",
+        help=(
+            "Directory to initialize DVC in. Defaults to the current working directory."
+        ),
+    )
    init_parser.add_argument(
        "--no-scm",
        action="store_true",
@@ -1,3 +1,5 @@
+from typing import Callable
+
 from dvc.cli import completion, formatter
 from dvc.cli.command import CmdBaseNoRepo
 from dvc.cli.utils import DictAction, append_doc_link
@@ -9,54 +11,178 @@ from dvc.ui import ui
 logger = logger.getChild(__name__)


-def _format_entry(entry, fmt):
-    from dvc.utils.humanize import naturalsize
+def _get_formatter(with_color: bool = False) -> Callable[[dict], str]:
+    def fmt(entry: dict) -> str:
+        return entry["path"]

-    size = entry.get("size")
-    if size is None:
-        size = ""
-    else:
-        size = naturalsize(size)
-    return size, fmt(entry)
-
-
-def show_entries(entries, with_color=False, with_size=False):
    if with_color:
        ls_colors = LsColors()
-        fmt = ls_colors.format
-    else:
+        return ls_colors.format

-        def fmt(entry):
-            return entry["path"]
+    return fmt

+
+def _format_entry(entry, name, with_size=True, with_hash=False):
+    from dvc.utils.humanize import naturalsize
+
+    ret = []
    if with_size:
-        ui.table([_format_entry(entry, fmt) for entry in entries])
+        size = entry.get("size")
+        if size is None or (size <= 0 and entry.get("isdir")):
+            size = ""
+        else:
+            size = naturalsize(size)
+        ret.append(size)
+    if with_hash:
+        md5 = entry.get("md5", "")
+        ret.append(md5)
+    ret.append(name)
+    return ret
+
+
+def show_entries(entries, with_color=False, with_size=False, with_hash=False):
+    fmt = _get_formatter(with_color)
+    if with_size or with_hash:
+        colalign = ("right",) if with_size else None
+        ui.table(
+            [
+                _format_entry(
+                    entry,
+                    fmt(entry),
+                    with_size=with_size,
+                    with_hash=with_hash,
+                )
+                for entry in entries
+            ],
+            colalign=colalign,
+        )
        return

    # NOTE: this is faster than ui.table for very large number of entries
    ui.write("\n".join(fmt(entry) for entry in entries))


+class TreePart:
+    Edge = "├── "
+    Line = "│   "
+    Corner = "└── "
+    Blank = "    "
+
+
+def _build_tree_structure(
+    entries, with_color=False, with_size=False, with_hash=False, _depth=0, _prefix=""
+):
+    rows = []
+    fmt = _get_formatter(with_color)
+
+    num_entries = len(entries)
+    for i, (name, entry) in enumerate(entries.items()):
+        entry["path"] = name
+        is_last = i >= num_entries - 1
+        tree_part = ""
+        if _depth > 0:
+            tree_part = TreePart.Corner if is_last else TreePart.Edge
+
+        row = _format_entry(
+            entry,
+            _prefix + tree_part + fmt(entry),
+            with_size=with_size,
+            with_hash=with_hash,
+        )
+        rows.append(row)
+
+        if contents := entry.get("contents"):
+            new_prefix = _prefix
+            if _depth > 0:
+                new_prefix += TreePart.Blank if is_last else TreePart.Line
+            new_rows = _build_tree_structure(
+                contents,
+                with_color=with_color,
+                with_size=with_size,
+                with_hash=with_hash,
+                _depth=_depth + 1,
+                _prefix=new_prefix,
+            )
+            rows.extend(new_rows)
+
+    return rows
+
+
+def show_tree(entries, with_color=False, with_size=False, with_hash=False):
+    import tabulate
+
+    rows = _build_tree_structure(
+        entries,
+        with_color=with_color,
+        with_size=with_size,
+        with_hash=with_hash,
+    )
+
+    colalign = ("right",) if with_size else None
+
+    _orig = tabulate.PRESERVE_WHITESPACE
+    tabulate.PRESERVE_WHITESPACE = True
+    try:
+        ui.table(rows, colalign=colalign)
+    finally:
+        tabulate.PRESERVE_WHITESPACE = _orig
+
+
 class CmdList(CmdBaseNoRepo):
-    def run(self):
+    def _show_tree(self):
+        from dvc.repo.ls import ls_tree
+
+        entries = ls_tree(
+            self.args.url,
+            self.args.path,
+            rev=self.args.rev,
+            dvc_only=self.args.dvc_only,
+            config=self.args.config,
+            remote=self.args.remote,
+            remote_config=self.args.remote_config,
+            maxdepth=self.args.level,
+        )
+        show_tree(
+            entries,
+            with_color=True,
+            with_size=self.args.size,
+            with_hash=self.args.show_hash,
+        )
+        return 0
+
+    def _show_list(self):
        from dvc.repo import Repo

-        try:
-            entries = Repo.ls(
-                self.args.url,
-                self.args.path,
-                rev=self.args.rev,
-                recursive=self.args.recursive,
-                dvc_only=self.args.dvc_only,
-                config=self.args.config,
-                remote=self.args.remote,
-                remote_config=self.args.remote_config,
+        entries = Repo.ls(
+            self.args.url,
+            self.args.path,
+            rev=self.args.rev,
+            recursive=self.args.recursive,
+            dvc_only=self.args.dvc_only,
+            config=self.args.config,
+            remote=self.args.remote,
+            remote_config=self.args.remote_config,
+            maxdepth=self.args.level,
+        )
+        if self.args.json:
+            ui.write_json(entries)
+        elif entries:
+            show_entries(
+                entries,
+                with_color=True,
+                with_size=self.args.size,
+                with_hash=self.args.show_hash,
            )
-            if self.args.json:
-                ui.write_json(entries)
-            elif entries:
-                show_entries(entries, with_color=True, with_size=self.args.size)
-            return 0
+        return 0
+
+    def run(self):
+        if self.args.tree and self.args.json:
+            raise DvcException("Cannot use --tree and --json options together.")
+
+        try:
+            if self.args.tree:
+                return self._show_tree()
+            return self._show_list()
        except FileNotFoundError:
            logger.exception("")
            return 1
@@ -85,6 +211,19 @@ def add_parser(subparsers, parent_parser):
        action="store_true",
        help="Recursively list files.",
    )
+    list_parser.add_argument(
+        "-T",
+        "--tree",
+        action="store_true",
+        help="Recurse into directories as a tree.",
+    )
+    list_parser.add_argument(
+        "-L",
+        "--level",
+        metavar="depth",
+        type=int,
+        help="Limit the depth of recursion.",
+    )
    list_parser.add_argument(
        "--dvc-only", action="store_true", help="Show only DVC outputs."
    )
@@ -111,7 +250,7 @@ def add_parser(subparsers, parent_parser):
        "--remote",
        type=str,
        help="Remote name to set as a default in the target repository.",
-    )
+    ).complete = completion.REMOTE
    list_parser.add_argument(
        "--remote-config",
        type=str,
@@ -123,6 +262,12 @@ def add_parser(subparsers, parent_parser):
        ),
    )
    list_parser.add_argument("--size", action="store_true", help="Show sizes.")
+    list_parser.add_argument(
+        "--show-hash",
+        help="Display hash value for each item.",
+        action="store_true",
+        default=False,
+    )
    list_parser.add_argument(
        "path",
        nargs="?",
@@ -32,7 +32,9 @@ class LsColors:
        if entry.get("isexec", False):
            return self._format(text, code="ex")

-        _, ext = os.path.splitext(text)
+        stem, ext = os.path.splitext(text)
+        if not ext and stem.startswith("."):
+            ext = stem
        return self._format(text, ext=ext)

    def _format(self, text, code=None, ext=None):
@@ -3,26 +3,45 @@ from dvc.cli.command import CmdBaseNoRepo
 from dvc.cli.utils import DictAction, append_doc_link
 from dvc.log import logger

-from .ls import show_entries
+from .ls import show_entries, show_tree

 logger = logger.getChild(__name__)


 class CmdListUrl(CmdBaseNoRepo):
-    def run(self):
-        from dvc.config import Config
+    def _show_tree(self, config):
+        from dvc.fs import parse_external_url
+        from dvc.repo.ls import _ls_tree
+
+        fs, fs_path = parse_external_url(
+            self.args.url, fs_config=self.args.fs_config, config=config
+        )
+        entries = _ls_tree(fs, fs_path, maxdepth=self.args.level)
+        show_tree(entries, with_color=True, with_size=self.args.size)
+        return 0
+
+    def _show_list(self, config):
        from dvc.repo import Repo

        entries = Repo.ls_url(
            self.args.url,
            recursive=self.args.recursive,
+            maxdepth=self.args.level,
            fs_config=self.args.fs_config,
-            config=Config.from_cwd(),
+            config=config,
        )
        if entries:
            show_entries(entries, with_color=True, with_size=self.args.size)
        return 0

+    def run(self):
+        from dvc.config import Config
+
+        config = Config.from_cwd()
+        if self.args.tree:
+            return self._show_tree(config=config)
+        return self._show_list(config=config)
+

 def add_parser(subparsers, parent_parser):
    LS_HELP = "List directory contents from URL."
@@ -40,6 +59,19 @@ def add_parser(subparsers, parent_parser):
    lsurl_parser.add_argument(
        "-R", "--recursive", action="store_true", help="Recursively list files."
    )
+    lsurl_parser.add_argument(
+        "-T",
+        "--tree",
+        action="store_true",
+        help="Recurse into directories as a tree.",
+    )
+    lsurl_parser.add_argument(
+        "-L",
+        "--level",
+        metavar="depth",
+        type=int,
+        help="Limit the depth of recursion.",
+    )
    lsurl_parser.add_argument("--size", action="store_true", help="Show sizes.")
    lsurl_parser.add_argument(
        "--fs-config",
@@ -1,4 +1,4 @@
-from dvc.cli import formatter
+from dvc.cli import completion, formatter
 from dvc.cli.utils import append_doc_link
 from dvc.commands.config import CmdConfig
 from dvc.ui import ui
@@ -110,8 +110,16 @@ class CmdRemoteDefault(CmdRemote):
 class CmdRemoteList(CmdRemote):
    def run(self):
        conf = self.config.read(self.args.level)
+        default_remote = conf["core"].get("remote")
+
        for name, remote_conf in conf["remote"].items():
-            ui.write(name, remote_conf["url"], sep="\t")
+            if name == default_remote:
+                text = f"{name}\t{remote_conf['url']}\t(default)"
+                color = "green"
+            else:
+                text = f"{name}\t{remote_conf['url']}"
+                color = ""
+            ui.write(ui.rich_text(text, style=color), styled=True)
        return 0


@@ -202,7 +210,9 @@ def add_parser(subparsers, parent_parser):
        help=REMOTE_DEFAULT_HELP,
        formatter_class=formatter.RawDescriptionHelpFormatter,
    )
-    remote_default_parser.add_argument("name", nargs="?", help="Name of the remote")
+    remote_default_parser.add_argument(
+        "name", nargs="?", help="Name of the remote"
+    ).complete = completion.REMOTE
    remote_default_parser.add_argument(
        "-u",
        "--unset",
@@ -220,7 +230,9 @@ def add_parser(subparsers, parent_parser):
        help=REMOTE_MODIFY_HELP,
        formatter_class=formatter.RawDescriptionHelpFormatter,
    )
-    remote_modify_parser.add_argument("name", help="Name of the remote")
+    remote_modify_parser.add_argument(
+        "name", help="Name of the remote"
+    ).complete = completion.REMOTE
    remote_modify_parser.add_argument("option", help="Name of the option to modify.")
    remote_modify_parser.add_argument(
        "value", nargs="?", help="(optional) Value of the option."
@@ -252,7 +264,9 @@ def add_parser(subparsers, parent_parser):
        help=REMOTE_REMOVE_HELP,
        formatter_class=formatter.RawDescriptionHelpFormatter,
    )
-    remote_remove_parser.add_argument("name", help="Name of the remote to remove.")
+    remote_remove_parser.add_argument(
+        "name", help="Name of the remote to remove."
+    ).complete = completion.REMOTE
    remote_remove_parser.set_defaults(func=CmdRemoteRemove)
    REMOTE_RENAME_HELP = "Rename a DVC remote"
    remote_rename_parser = remote_subparsers.add_parser(
@@ -71,7 +71,7 @@ class CmdDataStatus(CmdDataBase):
                return 0

            if self.args.quiet:
-                return bool(st)
+                return int(bool(st))

            if st:
                self._show(st, indent)
@@ -74,7 +74,7 @@ def add_parser(subparsers, parent_parser):
        "--remote",
        help="Remote storage to perform updates to",
        metavar="<name>",
-    )
+    ).complete = completion.REMOTE
    update_parser.add_argument(
        "-j",
        "--jobs",
@@ -1,6 +1,8 @@
 """DVC config objects."""

+import ntpath
 import os
+import posixpath
 import re
 from contextlib import contextmanager
 from functools import partial
@@ -268,12 +270,17 @@ class Config(dict):
        if re.match(r"\w+://", path):
            return path

+        if os.name == "nt" and posixpath.isabs(path) and ntpath.sep not in path:
+            return path
+
        if os.path.isabs(path):
            return path

        # on windows convert slashes to backslashes
        # to have path compatible with abs_conf_dir
        if os.path.sep == "\\" and "/" in path:
+            if path.startswith("/"):
+                path = path.replace("/", "\\\\", 1)
            path = path.replace("/", "\\")

        expanded = os.path.expanduser(path)
@@ -305,6 +312,9 @@ class Config(dict):
        if os.path.expanduser(path) != path:
            return localfs.as_posix(path)

+        if os.name == "nt" and posixpath.isabs(path) and ntpath.sep not in path:
+            return path
+
        if isinstance(path, RelPath) or not os.path.isabs(path):
            path = relpath(path, conf_dir)
            return localfs.as_posix(path)
@@ -330,6 +340,7 @@ class Config(dict):
                    "gdrive_user_credentials_file": func,
                    "gdrive_service_account_json_file_path": func,
                    "credentialpath": func,
+                    "configpath": func,
                    "keyfile": func,
                    "cert_path": func,
                    "key_path": func,
@@ -215,6 +215,7 @@ SCHEMA = {
                    "credentialpath": str,
                    "endpointurl": str,
                    Optional("verify", default=False): Bool,
+                    Optional("allow_anonymous_login", default=False): Bool,
                    **REMOTE_COMMON,
                },
                "ssh": {
@@ -32,10 +32,10 @@ def _win_detached_subprocess(args: Sequence[str], **kwargs) -> int:
    assert os.name == "nt"

    from subprocess import (  # type: ignore[attr-defined]
-        CREATE_NEW_PROCESS_GROUP,
-        CREATE_NO_WINDOW,
-        STARTF_USESHOWWINDOW,
-        STARTUPINFO,
+        CREATE_NEW_PROCESS_GROUP,  # ty: ignore[unresolved-import]
+        CREATE_NO_WINDOW,  # ty: ignore[unresolved-import]
+        STARTF_USESHOWWINDOW,  # ty: ignore[unresolved-import]
+        STARTUPINFO,  # ty: ignore[unresolved-import]
    )

    # https://stackoverflow.com/a/7006424
@@ -164,7 +164,7 @@ def _spawn(
        return _detached_subprocess(executable + args, env=env, **kwargs)


-def daemonize(args: list[str], executable: Union[None, str, list[str]] = None) -> None:
+def daemonize(args: list[str], executable: Union[str, list[str], None] = None) -> None:
    if os.name not in ("posix", "nt"):
        return

@@ -104,21 +104,21 @@ class AsciiCanvas:
                if dx == 0:
                    y = y0
                else:
-                    y = y0 + int(round((x - x0) * dy / float(dx)))
+                    y = y0 + round((x - x0) * dy / float(dx))
                self.point(x, y, char)
        elif y0 < y1:
            for y in range(y0, y1 + 1):
                if dy == 0:
                    x = x0
                else:
-                    x = x0 + int(round((y - y0) * dx / float(dy)))
+                    x = x0 + round((y - y0) * dx / float(dy))
                self.point(x, y, char)
        else:
            for y in range(y1, y0 + 1):
                if dy == 0:
                    x = x0
                else:
-                    x = x1 + int(round((y - y1) * dx / float(dy)))
+                    x = x1 + round((y - y1) * dx / float(dy))
                self.point(x, y, char)

    def text(self, x, y, text):
@@ -195,7 +195,7 @@ def _build_sugiyama_layout(vertices, edges):

    sug.yspace = VertexViewer.HEIGHT
    sug.xspace = minw
-    sug.route_edge = route_with_lines
+    sug.route_edge = route_with_lines  # ty: ignore[unresolved-attribute]

    sug.draw()

@@ -258,8 +258,8 @@ def draw(vertices, edges):
    maxx = max(Xs)
    maxy = max(Ys)

-    canvas_cols = int(math.ceil(math.ceil(maxx) - math.floor(minx))) + 1
-    canvas_lines = int(round(maxy - miny))
+    canvas_cols = math.ceil(math.ceil(maxx) - math.floor(minx)) + 1
+    canvas_lines = round(maxy - miny)

    canvas = AsciiCanvas(canvas_cols, canvas_lines)

@@ -270,10 +270,10 @@ def draw(vertices, edges):
            start = edge.view._pts[index - 1]
            end = edge.view._pts[index]

-            start_x = int(round(start[0] - minx))
-            start_y = int(round(start[1] - miny))
-            end_x = int(round(end[0] - minx))
-            end_y = int(round(end[1] - miny))
+            start_x = round(start[0] - minx)
+            start_y = round(start[1] - miny)
+            end_x = round(end[0] - minx)
+            end_y = round(end[1] - miny)

            assert start_x >= 0
            assert start_y >= 0
@@ -288,12 +288,12 @@ def draw(vertices, edges):
        y = vertex.view.xy[1]

        canvas.box(
-            int(round(x - minx)),
-            int(round(y - miny)),
+            round(x - minx),
+            round(y - miny),
            vertex.view.w,
            vertex.view.h,
        )

-        canvas.text(int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data)
+        canvas.text(round(x - minx) + 1, round(y - miny) + 1, vertex.data)

    return canvas.draw()
@@ -146,11 +146,15 @@ class RepoDependency(Dependency):
        else:
            config = Config.load_file(conf)

+        # Setup config to the new DVCFileSystem to use the remote repo, but rely on the
+        # local cache instead of the remote's cache. This avoids re-streaming of data,
+        # but messes up the call to `_get_remote_config()` downstream, which will need
+        # to ignore cache parameters.
        config["cache"] = self.repo.config["cache"]
        config["cache"]["dir"] = self.repo.cache.local_cache_dir

        return DVCFileSystem(
-            url=self.def_repo[self.PARAM_URL],
+            repo=self.def_repo[self.PARAM_URL],
            rev=rev or self._get_rev(locked=locked),
            subrepos=True,
            config=config,
@@ -1,4 +1,5 @@
 import os
+from typing import Optional

 import platformdirs

@@ -20,7 +21,24 @@ def global_config_dir():
    )


-def site_cache_dir():
-    return os.getenv(env.DVC_SITE_CACHE_DIR) or platformdirs.site_cache_dir(
-        APPNAME, APPAUTHOR, opinion=True
-    )
+def site_cache_dir(config_site_cache_dir: Optional[str] = None):
+    from platformdirs import PlatformDirs
+    from platformdirs.unix import Unix
+
+    if dvc_site_cache_dir := os.getenv(env.DVC_SITE_CACHE_DIR):
+        return dvc_site_cache_dir
+
+    if config_site_cache_dir is not None:
+        return config_site_cache_dir
+
+    if issubclass(Unix, PlatformDirs):
+        # Return the cache directory shared by users, e.g. `/var/tmp/$appname`
+        # NOTE: platformdirs>=5 changed `site_cache_dir` to return /var/cache/$appname.
+        # as the following path is considered insecure.
+        # For details, see: https://github.com/tox-dev/platformdirs/pull/239
+
+        # FIXME: keeping the old behavior temporarily to avoid dependency conflict.
+        #        In the future, consider migrating to a more secure directory.
+        return f"/var/tmp/{APPNAME}"  # noqa: S108
+
+    return platformdirs.site_cache_dir(APPNAME, APPAUTHOR, opinion=True)
@@ -162,6 +162,9 @@ class FileMixin:
    def dump(self, stage, **kwargs):
        raise NotImplementedError

+    def dump_stages(self, stages, **kwargs):
+        raise NotImplementedError
+
    def merge(self, ancestor, other, allowed=None):
        raise NotImplementedError

@@ -198,6 +201,13 @@ class SingleStageFile(FileMixin):
        dump_yaml(self.path, serialize.to_single_stage_file(stage, **kwargs))
        self.repo.scm_context.track_file(self.relpath)

+    def dump_stages(self, stages, **kwargs) -> None:
+        if not stages:
+            return
+
+        assert len(stages) == 1, "SingleStageFile can only dump one stage."
+        return self.dump(stages[0], **kwargs)
+
    def remove_stage(self, stage):  # noqa: ARG002
        self.remove()

@@ -228,17 +238,27 @@ class ProjectFile(FileMixin):

    def dump(self, stage, update_pipeline=True, update_lock=True, **kwargs):
        """Dumps given stage appropriately in the dvcfile."""
+        return self.dump_stages(
+            [stage], update_pipeline=update_pipeline, update_lock=update_lock, **kwargs
+        )
+
+    def dump_stages(self, stages, update_pipeline=True, update_lock=True, **kwargs):
        from dvc.stage import PipelineStage

-        assert isinstance(stage, PipelineStage)
+        if not stages:
+            return
+
+        for stage in stages:
+            assert isinstance(stage, PipelineStage)
+
        if self.verify:
            check_dvcfile_path(self.repo, self.path)

-        if update_pipeline and not stage.is_data_source:
-            self._dump_pipeline_file(stage)
+        if update_pipeline:
+            self._dump_pipeline_file(stages)

        if update_lock:
-            self._dump_lockfile(stage, **kwargs)
+            self._dump_lockfile(stages, **kwargs)

    def dump_dataset(self, dataset):
        with modify_yaml(self.path, fs=self.repo.fs) as data:
@@ -260,32 +280,37 @@ class ProjectFile(FileMixin):
                raw.append(dataset)
        self.repo.scm_context.track_file(self.relpath)

-    def _dump_lockfile(self, stage, **kwargs):
-        self._lockfile.dump(stage, **kwargs)
+    def _dump_lockfile(self, stages, **kwargs):
+        self._lockfile.dump_stages(stages, **kwargs)

    @staticmethod
    def _check_if_parametrized(stage, action: str = "dump") -> None:
        if stage.raw_data.parametrized:
            raise ParametrizedDumpError(f"cannot {action} a parametrized {stage}")

-    def _dump_pipeline_file(self, stage):
-        self._check_if_parametrized(stage)
-        stage_data = serialize.to_pipeline_file(stage)
+    def _dump_pipeline_file(self, stages):
+        stages = stages if isinstance(stages, list) else [stages]
+        if not stages:
+            return
+
+        for stage in stages:
+            self._check_if_parametrized(stage)

        with modify_yaml(self.path, fs=self.repo.fs) as data:
            if not data:
                logger.info("Creating '%s'", self.relpath)

            data["stages"] = data.get("stages", {})
-            existing_entry = stage.name in data["stages"]
-            action = "Modifying" if existing_entry else "Adding"
-            logger.info("%s stage '%s' in '%s'", action, stage.name, self.relpath)
-
-            if existing_entry:
-                orig_stage_data = data["stages"][stage.name]
-                apply_diff(stage_data[stage.name], orig_stage_data)
-            else:
-                data["stages"].update(stage_data)
+            for stage in stages:
+                stage_data = serialize.to_pipeline_file(stage)
+                existing_entry = stage.name in data["stages"]
+                action = "Modifying" if existing_entry else "Adding"
+                logger.info("%s stage '%s' in '%s'", action, stage.name, self.relpath)
+                if existing_entry:
+                    orig_stage_data = data["stages"][stage.name]
+                    apply_diff(stage_data[stage.name], orig_stage_data)
+                else:
+                    data["stages"].update(stage_data)

        self.repo.scm_context.track_file(self.relpath)

@@ -399,9 +424,12 @@ class Lockfile(FileMixin):
            data.setdefault("stages", {})
        self.repo.scm_context.track_file(self.relpath)

-    def dump(self, stage, **kwargs):
-        stage_data = serialize.to_lockfile(stage, **kwargs)
+    def dump_stages(self, stages, **kwargs):
+        if not stages:
+            return

+        is_modified = False
+        log_updated = False
        with modify_yaml(self.path, fs=self.repo.fs) as data:
            if not data:
                data.update({"schema": "2.0"})
@@ -409,17 +437,24 @@ class Lockfile(FileMixin):
                logger.info("Generating lock file '%s'", self.relpath)

            data["stages"] = data.get("stages", {})
-            modified = data["stages"].get(stage.name, {}) != stage_data.get(
-                stage.name, {}
-            )
-            if modified:
-                logger.info("Updating lock file '%s'", self.relpath)
+            for stage in stages:
+                stage_data = serialize.to_lockfile(stage, **kwargs)
+                modified = data["stages"].get(stage.name, {}) != stage_data.get(
+                    stage.name, {}
+                )
+                if modified:
+                    is_modified = True
+                    if not log_updated:
+                        logger.info("Updating lock file '%s'", self.relpath)
+                        log_updated = True
+                data["stages"].update(stage_data)

-            data["stages"].update(stage_data)
-
-        if modified:
+        if is_modified:
            self.repo.scm_context.track_file(self.relpath)

+    def dump(self, stage, **kwargs):
+        self.dump_stages([stage], **kwargs)
+
    def remove_stage(self, stage):
        if not self.exists():
            return
@@ -242,11 +242,11 @@ class UploadError(FileTransferError):


 class CheckoutError(DvcException):
-    def __init__(self, target_infos: list[str], stats: dict[str, list[str]]):
+    def __init__(self, target_infos: list[str], result: dict):
        from dvc.utils import error_link

        self.target_infos = target_infos
-        self.stats = stats
+        self.result = result
        targets = [str(t) for t in target_infos]
        m = (
            "Checkout failed for following targets:\n{}\nIs your "
@@ -20,7 +20,7 @@ from dvc_objects.fs import (  # noqa: F401
    system,
    utils,
 )
-from dvc_objects.fs.base import AnyFSPath, FileSystem  # noqa: F401, TCH001
+from dvc_objects.fs.base import AnyFSPath, FileSystem  # noqa: F401, TC001
 from dvc_objects.fs.errors import (  # noqa: F401
    AuthError,
    ConfigError,
@@ -1,8 +1,8 @@
 from contextlib import ExitStack
 from typing import TYPE_CHECKING, BinaryIO, Optional, Union

-import fsspec
 from fsspec.callbacks import DEFAULT_CALLBACK, Callback  # noqa: F401
+from fsspec.callbacks import TqdmCallback as _TqdmCallback

 from dvc.progress import Tqdm
 from dvc.utils.objects import cached_property
@@ -14,7 +14,7 @@ if TYPE_CHECKING:
    from dvc.ui._rich_progress import RichTransferProgress


-class TqdmCallback(fsspec.callbacks.TqdmCallback):
+class TqdmCallback(_TqdmCallback):
    def __init__(
        self,
        size: Optional[int] = None,
@@ -83,11 +83,10 @@ class _DVCFileSystem(AbstractFileSystem):
    cachable = False
    root_marker = "/"

-    def __init__(  # noqa: PLR0913
+    def __init__(
        self,
-        url: Optional[str] = None,
+        repo: Union["Repo", os.PathLike[str], str, None] = None,
        rev: Optional[str] = None,
-        repo: Optional["Repo"] = None,
        subrepos: bool = False,
        repo_factory: Optional[RepoFactory] = None,
        fo: Optional[str] = None,
@@ -101,7 +100,8 @@ class _DVCFileSystem(AbstractFileSystem):
        """DVC + git-tracked files fs.

        Args:
-            path (str, optional): URL or path to a DVC/Git repository.
+            repo (str | os.PathLike[str] | Repo, optional): A url or a path to a DVC/Git
+                repository, or a `Repo` instance.
                Defaults to a DVC repository in the current working directory.
                Both HTTP and SSH protocols are supported for remote Git repos
                (e.g. [user@]server:project.git).
@@ -111,7 +111,6 @@ class _DVCFileSystem(AbstractFileSystem):
                In case of a local repository, if rev is unspecified, it will
                default to the working directory.
                If the repo is not a Git repo, this option is ignored.
-            repo (:obj:`Repo`, optional): `Repo` instance.
            subrepos (bool): traverse to subrepos.
                By default, it ignores subrepos.
            repo_factory (callable): A function to initialize subrepo with.
@@ -136,13 +135,23 @@ class _DVCFileSystem(AbstractFileSystem):
            ...    rev="main",
            ... )
        """
+        from dvc.repo import Repo
+
+        # kwargs.get("url") is for maintaining backward compatibility
+        repo = repo or fo or kwargs.get("url")
+        if isinstance(repo, Repo):
+            self._repo: Optional[Repo] = repo
+            url = None
+        else:
+            self._repo = None
+            url = os.fspath(repo) if repo else None
+
        super().__init__()
-        self._repo = repo
        self._repo_factory = repo_factory
        self._traverse_subrepos = subrepos
        self._repo_stack = ExitStack()
        self._repo_kwargs = {
-            "url": url if url is not None else fo,
+            "url": url,
            "rev": rev,
            "subrepos": subrepos,
            "config": config,
@@ -500,7 +509,7 @@ class _DVCFileSystem(AbstractFileSystem):
            **kwargs,
        )

-    def _get(  # noqa: C901, PLR0912
+    def _get(  # noqa: C901, PLR0912, PLR0915
        self,
        rpath,
        lpath,
@@ -571,14 +580,15 @@ class _DVCFileSystem(AbstractFileSystem):

        os.makedirs(lpath, exist_ok=True)
        for d in _dirs:
-            os.mkdir(d)
+            os.makedirs(d, exist_ok=True)

        def get_file(arg: tuple[FileSystem, tuple[str, str, Optional[dict]]]):
            fs, (src, dest, info) = arg
            kw = kwargs
            if isinstance(fs, DataFileSystem):
                kw = kw | {"info": info}
-            return fs.get_file(src, dest, callback=callback, **kw)
+            with callback.branched(src, dest) as child:
+                fs.get_file(src, dest, callback=child, **kw)

        if batch_size == 1:
            ctx: AbstractContextManager = nullcontext()
@@ -589,7 +599,7 @@ class _DVCFileSystem(AbstractFileSystem):

        with ctx:
            it = ((fs, f) for fs, files in _files.items() for f in files)
-            deque(map_fn(get_file, it), maxlen=0)
+            deque(callback.wrap(map_fn(get_file, it)), maxlen=0)
        return result

    def get_file(self, rpath, lpath, **kwargs):
@@ -732,9 +742,10 @@ class DVCFileSystem(FileSystem):
        return self.fs.repo_url

    def from_os_path(self, path: str) -> str:
-        if os.path.isabs(path):
+        if os.path.isabs(path) or (
+            os.name == "nt" and posixpath.isabs(path) and ntpath.sep not in path
+        ):
            path = os.path.relpath(path, self.repo.root_dir)
-
        return as_posix(path)

    def close(self):
@@ -15,7 +15,7 @@ Examples

 from urllib.parse import urlsplit

-from upath import UPath
+from upath import UPath  # ty: ignore[unresolved-import]


 class DVCPath(UPath):
@@ -1,8 +1,9 @@
+import functools
 import os
 import re
-from collections import namedtuple
+from collections.abc import Iterable, Iterator
 from itertools import chain, groupby, takewhile
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Optional, Union, overload

 from pathspec.patterns import GitWildMatchPattern
 from pathspec.util import normalize_file
@@ -13,7 +14,9 @@ from dvc.log import logger
 from dvc.pathspec_math import PatternInfo, merge_patterns

 if TYPE_CHECKING:
-    from dvc.fs import AnyFSPath, FileSystem
+    from typing_extensions import Self
+
+    from dvc.fs import FileSystem

 logger = logger.getChild(__name__)

@@ -21,36 +24,79 @@ logger = logger.getChild(__name__)
 class DvcIgnore:
    DVCIGNORE_FILE = ".dvcignore"

-    def __call__(self, root, dirs, files):
+    def __call__(
+        self, root: str, dirs: list[str], files: list[str]
+    ) -> tuple[list[str], list[str]]:
        raise NotImplementedError


 class DvcIgnorePatterns(DvcIgnore):
-    def __init__(self, pattern_list, dirname, sep):
+    def __init__(
+        self, pattern_list: Iterable[Union[PatternInfo, str]], dirname: str, sep: str
+    ) -> None:
        from pathspec.patterns.gitwildmatch import _DIR_MARK

-        if pattern_list and isinstance(pattern_list[0], str):
-            pattern_list = [PatternInfo(pattern, "") for pattern in pattern_list]
-
-        self.sep = sep
-        self.pattern_list = pattern_list
-        self.dirname = dirname
-
-        self.regex_pattern_list = []
-        for count, pattern in enumerate(pattern_list):
-            pattern, group = GitWildMatchPattern.pattern_to_regex(pattern.patterns)
-            if pattern:
-                pattern = pattern.replace(f"<{_DIR_MARK}>", f"<{_DIR_MARK}{count}>")
-                self.regex_pattern_list.append((pattern, group))
-
-        self.ignore_spec = [
-            (ignore, re.compile("|".join(item[0] for item in group)))
-            for ignore, group in groupby(self.regex_pattern_list, lambda x: x[1])
-            if ignore is not None
+        pattern_infos = [
+            pattern if isinstance(pattern, PatternInfo) else PatternInfo(pattern, "")
+            for pattern in pattern_list
        ]

+        self.sep = sep
+        self.pattern_list: list[PatternInfo] = []
+        self.dirname = dirname
+        self.find_matching_pattern = functools.cache(self._find_matching_pattern)
+
+        regex_pattern_list: list[tuple[str, bool, bool, PatternInfo]] = []
+        for count, pattern_info in enumerate(pattern_infos):
+            regex, ignore = GitWildMatchPattern.pattern_to_regex(pattern_info.patterns)
+            if regex is not None and ignore is not None:
+                self.pattern_list.append(pattern_info)
+                regex = regex.replace(f"<{_DIR_MARK}>", f"<{_DIR_MARK}{count}>")
+                regex_pattern_list.append(
+                    (regex, ignore, pattern_info.patterns.endswith("/"), pattern_info)
+                )
+
+        def keyfunc(item: tuple[str, bool, bool, PatternInfo]) -> tuple[bool, bool]:
+            _, ignore, dir_only_pattern, _ = item
+            return ignore, dir_only_pattern
+
+        self.ignore_spec: list[
+            tuple[
+                re.Pattern[str],
+                bool,
+                bool,
+                dict[Optional[str], tuple[str, PatternInfo]],
+            ]
+        ]
+        self.ignore_spec = []
+        for (ignore, dir_only_pattern), group in groupby(
+            regex_pattern_list, key=keyfunc
+        ):
+            if ignore:
+                # For performance, we combine all exclude patterns.
+                # But we still need to figure out which pattern matched which rule,
+                # (eg: to show in `dvc check-ignore`).
+                # So, we use named groups and keep a map of group name to pattern.
+                pattern_map: dict[Optional[str], tuple[str, PatternInfo]] = {
+                    f"rule_{i}": (regex, pi)
+                    for i, (regex, _, _, pi) in enumerate(group)
+                }
+                combined_regex = "|".join(
+                    f"(?P<{name}>{regex})" for name, (regex, _) in pattern_map.items()
+                )
+                self.ignore_spec.append(
+                    (re.compile(combined_regex), ignore, dir_only_pattern, pattern_map)
+                )
+            else:
+                # unignored patterns are not combined with `|`.
+                for regex, _, _, pi in group:
+                    pattern_map = {None: (regex, pi)}
+                    self.ignore_spec.append(
+                        (re.compile(regex), ignore, dir_only_pattern, pattern_map)
+                    )
+
    @classmethod
-    def from_file(cls, path, fs, name):
+    def from_file(cls, path: str, fs: "FileSystem", name: str) -> "Self":
        assert fs.isabs(path)
        dirname = fs.normpath(fs.dirname(path))
        with fs.open(path, encoding="utf-8") as fobj:
@@ -62,13 +108,15 @@ class DvcIgnorePatterns(DvcIgnore):

        return cls(path_spec_lines, dirname, fs.sep)

-    def __call__(self, root: list[str], dirs: list[str], files: list[str]):
+    def __call__(
+        self, root: str, dirs: list[str], files: list[str]
+    ) -> tuple[list[str], list[str]]:
        files = [f for f in files if not self.matches(root, f)]
        dirs = [d for d in dirs if not self.matches(root, d, True)]

        return dirs, files

-    def _get_normalize_path(self, dirname, basename):
+    def _get_normalize_path(self, dirname: str, basename: str) -> Optional[str]:
        # NOTE: `relpath` is too slow, so we have to assume that both
        # `dirname` and `self.dirname` are relative or absolute together.

@@ -81,62 +129,105 @@ class DvcIgnorePatterns(DvcIgnore):
            # NOTE: `os.path.join` is ~x5.5 slower
            path = f"{rel}{self.sep}{basename}"
        else:
-            return False
+            return None

        if os.name == "nt":
            return normalize_file(path)
        return path

-    def matches(self, dirname, basename, is_dir=False, details: bool = False):
+    @overload
+    def matches(
+        self,
+        dirname: str,
+        basename: str,
+        is_dir: bool = False,
+        details: Literal[False] = ...,
+    ) -> bool: ...
+
+    @overload
+    def matches(
+        self,
+        dirname: str,
+        basename: str,
+        is_dir: bool = False,
+        details: Literal[True] = ...,
+    ) -> tuple[bool, list[PatternInfo]]: ...
+
+    @overload
+    def matches(
+        self,
+        dirname: str,
+        basename: str,
+        is_dir: bool = False,
+        details: bool = False,
+    ) -> Union[bool, tuple[bool, list[PatternInfo]]]: ...
+
+    def matches(
+        self,
+        dirname: str,
+        basename: str,
+        is_dir: bool = False,
+        details: bool = False,
+    ) -> Union[bool, tuple[bool, list[PatternInfo]]]:
        path = self._get_normalize_path(dirname, basename)
-        if not path:
-            return False
-
-        if details:
-            return self._ignore_details(path, is_dir)
-        return self.ignore(path, is_dir)
-
-    def ignore(self, path, is_dir):
-        def matches(pattern, path, is_dir) -> bool:
-            matches_ = bool(pattern.match(path))
-
-            if is_dir:
-                matches_ |= bool(pattern.match(f"{path}/"))
-
-            return matches_
-
        result = False
+        _match: list[PatternInfo] = []
+        if path:
+            result, _match = self._ignore(path, is_dir)
+        return (result, _match) if details else result

-        for ignore, pattern in self.ignore_spec[::-1]:
-            if matches(pattern, path, is_dir):
-                result = ignore
-                break
-        return result
+    def _find_matching_pattern(
+        self, path: str, is_dir: bool
+    ) -> tuple[bool, list[PatternInfo]]:
+        paths = [path]
+        if is_dir and not path.endswith("/"):
+            paths.append(f"{path}/")

-    def _ignore_details(self, path, is_dir: bool):
-        result = []
-        for (regex, _), pattern_info in list(
-            zip(self.regex_pattern_list, self.pattern_list)
+        for pattern, ignore, dir_only_pattern, pattern_map in reversed(
+            self.ignore_spec
        ):
-            # skip system pattern
-            if not pattern_info.file_info:
+            if dir_only_pattern and not is_dir:
                continue
+            for p in paths:
+                match = pattern.match(p)
+                if not match:
+                    continue
+                if ignore:
+                    group_name, _match = next(
+                        (
+                            (name, _match)
+                            for name, _match in match.groupdict().items()
+                            if name.startswith("rule_") and _match is not None
+                        )
+                    )
+                else:
+                    # unignored patterns are not combined with `|`,
+                    # so there are no groups.
+                    group_name = None
+                _regex, pattern_info = pattern_map[group_name]
+                return ignore, [pattern_info]
+        return False, []

-            regex = re.compile(regex)
+    def _ignore(self, path: str, is_dir: bool) -> tuple[bool, list[PatternInfo]]:
+        parts = path.split("/")
+        result = False
+        matches: list[PatternInfo] = []
+        for i in range(1, len(parts) + 1):
+            rel_path = "/".join(parts[:i])
+            result, _matches = self.find_matching_pattern(
+                rel_path, is_dir or i < len(parts)
+            )
+            if i < len(parts) and not result:
+                continue
+            matches.extend(_matches)
+            if result:
+                break
+        return result, matches

-            matches = bool(regex.match(path))
-            if is_dir:
-                matches |= bool(regex.match(f"{path}/"))
-
-            if matches:
-                result.append(pattern_info.file_info)
-
-        return result
-
-    def __hash__(self):
+    def __hash__(self) -> int:
        return hash(self.dirname + ":" + str(self.pattern_list))

-    def __eq__(self, other):
+    def __eq__(self, other: object) -> bool:
        if not isinstance(other, DvcIgnorePatterns):
            return NotImplemented
        return (self.dirname == other.dirname) & (
@@ -144,21 +235,18 @@ class DvcIgnorePatterns(DvcIgnore):
            == [pattern.patterns for pattern in other.pattern_list]
        )

-    def __bool__(self):
+    def __bool__(self) -> bool:
        return bool(self.pattern_list)


-CheckIgnoreResult = namedtuple(  # noqa: PYI024
-    "CheckIgnoreResult", ["file", "match", "patterns"]
-)
-
-
-def _no_match(path):
-    return CheckIgnoreResult(path, False, ["::"])
+class CheckIgnoreResult(NamedTuple):
+    file: str
+    match: bool
+    pattern_infos: list[PatternInfo]


 class DvcIgnoreFilter:
-    def __init__(self, fs, root_dir):
+    def __init__(self, fs: "FileSystem", root_dir: str) -> None:
        from dvc.repo import Repo

        default_ignore_patterns = [
@@ -193,7 +281,7 @@ class DvcIgnoreFilter:
            ignore_subrepos=True,
        )

-    def _get_key(self, path):
+    def _get_key(self, path: str) -> tuple[str, ...]:
        parts = self.fs.relparts(path, self.root_dir)
        if parts == (os.curdir,):
            return ()
@@ -241,7 +329,7 @@ class DvcIgnoreFilter:
            for dname in dnames:
                self._update_sub_repo(self.fs.join(dirname, dname), ignore_trie)

-    def _update_sub_repo(self, path, ignore_trie: Trie):
+    def _update_sub_repo(self, path: str, ignore_trie: Trie) -> None:
        from dvc.repo import Repo

        if path == self.root_dir:
@@ -268,7 +356,9 @@ class DvcIgnoreFilter:
        else:
            ignore_trie[key] = new_pattern

-    def __call__(self, root, dirs, files, ignore_subrepos=True):
+    def __call__(
+        self, root: str, dirs: list[str], files: list[str], ignore_subrepos: bool = True
+    ) -> tuple[list[str], list[str]]:
        abs_root = self.fs.abspath(root)
        ignore_pattern = self._get_trie_pattern(
            abs_root, dnames=dirs, ignore_subrepos=ignore_subrepos
@@ -277,7 +367,24 @@ class DvcIgnoreFilter:
            dirs, files = ignore_pattern(abs_root, dirs, files)
        return dirs, files

-    def ls(self, fs, path, detail=True, **kwargs):
+    @overload
+    def ls(
+        self, fs: "FileSystem", path: str, detail: Literal[True], **kwargs: Any
+    ) -> list[dict[str, Any]]: ...
+
+    @overload
+    def ls(
+        self, fs: "FileSystem", path: str, detail: Literal[False], **kwargs
+    ) -> list[str]: ...
+
+    @overload
+    def ls(
+        self, fs: "FileSystem", path: str, detail: bool = True, **kwargs
+    ) -> Union[list[str], list[dict[str, Any]]]: ...
+
+    def ls(
+        self, fs: "FileSystem", path: str, detail: bool = True, **kwargs: Any
+    ) -> Union[list[str], list[dict[str, Any]]]:
        fs_dict = {}
        dirs = []
        nondirs = []
@@ -297,22 +404,29 @@ class DvcIgnoreFilter:

        return [fs_dict[name] for name in chain(dirs, nondirs)]

-    def walk(self, fs: "FileSystem", path: "AnyFSPath", **kwargs):
+    def walk(
+        self, fs: "FileSystem", path: str, **kwargs: Any
+    ) -> Iterator[
+        Union[
+            tuple[str, list[str], list[str]],
+            tuple[str, dict[str, dict], dict[str, dict]],
+        ]
+    ]:
        detail = kwargs.get("detail", False)
        ignore_subrepos = kwargs.pop("ignore_subrepos", True)
        if fs.protocol == Schemes.LOCAL:
            for root, dirs, files in fs.walk(path, **kwargs):
                if detail:
-                    all_dnames = set(dirs.keys())
-                    all_fnames = set(files.keys())
+                    assert isinstance(dirs, dict)
+                    assert isinstance(files, dict)
                    dnames, fnames = self(
                        root,
-                        all_dnames,
-                        all_fnames,
+                        list(dirs),
+                        list(files),
                        ignore_subrepos=ignore_subrepos,
                    )
-                    list(map(dirs.pop, all_dnames - set(dnames)))
-                    list(map(files.pop, all_fnames - set(fnames)))
+                    list(map(dirs.pop, dirs.keys() - set(dnames)))
+                    list(map(files.pop, files.keys() - set(fnames)))
                else:
                    dirs[:], files[:] = self(
                        root, dirs, files, ignore_subrepos=ignore_subrepos
@@ -321,7 +435,7 @@ class DvcIgnoreFilter:
        else:
            yield from fs.walk(path, **kwargs)

-    def find(self, fs: "FileSystem", path: "AnyFSPath", **kwargs):
+    def find(self, fs: "FileSystem", path: str, **kwargs: Any) -> Iterator[str]:
        if fs.protocol == Schemes.LOCAL:
            for root, _, files in self.walk(fs, path, **kwargs):
                for file in files:
@@ -331,7 +445,7 @@ class DvcIgnoreFilter:
            yield from fs.find(path)

    def _get_trie_pattern(
-        self, dirname, dnames: Optional["list"] = None, ignore_subrepos=True
+        self, dirname: str, dnames: Optional[list[str]] = None, ignore_subrepos=True
    ) -> Optional["DvcIgnorePatterns"]:
        if ignore_subrepos:
            ignores_trie = self.ignores_trie_fs
@@ -367,7 +481,7 @@ class DvcIgnoreFilter:

    def _is_ignored(
        self, path: str, is_dir: bool = False, ignore_subrepos: bool = True
-    ):
+    ) -> bool:
        if self._outside_repo(path):
            return False
        dirname, basename = self.fs.split(self.fs.normpath(path))
@@ -389,24 +503,23 @@ class DvcIgnoreFilter:
        path = self.fs.abspath(path)
        return self._is_ignored(path, False, ignore_subrepos=ignore_subrepos)

-    def _outside_repo(self, path):
+    def _outside_repo(self, path: str) -> bool:
        return not self.fs.isin_or_eq(path, self.root_dir)

-    def check_ignore(self, target):
+    def check_ignore(self, target: str) -> CheckIgnoreResult:
        # NOTE: can only be used in `dvc check-ignore`, see
        # https://github.com/iterative/dvc/issues/5046
        full_target = self.fs.abspath(target)
+        matched_patterns: list[PatternInfo] = []
+        ignore = False
        if not self._outside_repo(full_target):
            dirname, basename = self.fs.split(self.fs.normpath(full_target))
            pattern = self._get_trie_pattern(dirname)
            if pattern:
-                matches = pattern.matches(
-                    dirname, basename, self.fs.isdir(full_target), True
+                ignore, matched_patterns = pattern.matches(
+                    dirname, basename, self.fs.isdir(full_target), details=True
                )
-
-                if matches:
-                    return CheckIgnoreResult(target, True, matches)
-        return _no_match(target)
+        return CheckIgnoreResult(target, ignore, matched_patterns)

    def is_ignored(
        self, fs: "FileSystem", path: str, ignore_subrepos: bool = True
@@ -424,7 +537,7 @@ class DvcIgnoreFilter:
        )


-def init(path):
+def init(path: Union[str, os.PathLike[str]]) -> str:
    dvcignore = os.path.join(path, DvcIgnore.DVCIGNORE_FILE)
    if os.path.exists(dvcignore):
        return dvcignore
@@ -439,7 +552,7 @@ def init(path):
    return dvcignore


-def destroy(path):
+def destroy(path: Union[str, os.PathLike[str]]) -> None:
    from dvc.utils.fs import remove

    dvcignore = os.path.join(path, DvcIgnore.DVCIGNORE_FILE)
@@ -48,7 +48,10 @@ def get_dvc_info():
                info.append("Cache types: " + error_link("no-dvc-cache"))

            info.append(f"Caches: {_get_caches(repo.cache)}")
-            info.append(f"Remotes: {_get_remotes(repo.config)}")
+
+            configured_remotes = _get_remotes(repo.config)
+            remotes = ", ".join(configured_remotes) if configured_remotes else None
+            info.append(f"Remotes: {remotes}")

            root_directory = repo.root_dir
            fs_root = _get_fs_type(os.path.abspath(root_directory))
@@ -75,12 +78,10 @@ def _get_caches(cache):


 def _get_remotes(config):
-    schemes = (
+    return [
        get_fs_cls(get_fs_config(config, name=remote)).protocol
        for remote in config["remote"]
-    )
-
-    return ", ".join(schemes) or "None"
+    ]


 def _get_linktype_support_info(repo):
@@ -2,13 +2,13 @@

 import hashlib
 import os
+import time
 from abc import ABC, abstractmethod
 from datetime import timedelta
 from typing import Optional, Union

 import flufl.lock
 import zc.lockfile
-from funcy import retry

 from dvc.exceptions import DvcException
 from dvc.progress import Tqdm
@@ -88,9 +88,10 @@ class Lock(LockBase):
    Uses zc.lockfile as backend.
    """

-    def __init__(self, lockfile, friendly=False, **kwargs):
+    def __init__(self, lockfile, friendly=False, wait=False, **kwargs):
        super().__init__(lockfile)
        self._friendly = friendly
+        self._wait = wait
        self._lock = None
        self._lock_failed = False

@@ -101,23 +102,38 @@ class Lock(LockBase):
    def _do_lock(self):
        try:
            self._lock_failed = False
-            with Tqdm(
-                bar_format="{desc}",
-                disable=not self._friendly,
-                desc="If DVC froze, see `hardlink_lock` in {}".format(
-                    format_link("https://man.dvc.org/config#core")
-                ),
-            ):
-                self._lock = zc.lockfile.LockFile(self._lockfile)
+            self._lock = zc.lockfile.LockFile(self._lockfile)
        except zc.lockfile.LockError:
            self._lock_failed = True
            raise LockError(FAILED_TO_LOCK_MESSAGE)  # noqa: B904

    def lock(self):
-        retries = 6
-        delay = DEFAULT_TIMEOUT / retries
-        lock_retry = retry(retries, LockError, timeout=delay)(self._do_lock)
-        lock_retry()
+        """Acquire the lock, either waiting forever, or after default_retries."""
+        default_retries = 6
+        delay = DEFAULT_TIMEOUT / default_retries
+        attempts = 0
+
+        max_retries = float("inf") if self._wait else default_retries
+
+        with Tqdm(
+            bar_format="{desc}",
+            disable=not self._friendly,
+            desc="Waiting to acquire lock. "
+            "If DVC froze, see `hardlink_lock` in {}.".format(
+                format_link("https://man.dvc.org/config#core")
+            ),
+        ) as pbar:
+            while True:
+                try:
+                    self._do_lock()
+                    return
+                except LockError:
+                    attempts += 1
+                    if attempts > max_retries:
+                        raise
+                    time.sleep(delay)
+                finally:
+                    pbar.update()

    def unlock(self):
        if self._lock_failed:
@@ -150,7 +166,7 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
        tmp_dir (str): a directory to store claim files.
    """

-    def __init__(self, lockfile, tmp_dir=None, **kwargs):
+    def __init__(self, lockfile, tmp_dir=None, wait=False, **kwargs):
        import socket

        self._tmp_dir = tmp_dir
@@ -173,11 +189,21 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
        self._separator = flufl.lock.SEP
        self._set_claimfile()
        self._owned = True
+        self._wait = wait
        self._retry_errnos = []
+        self._friendly = kwargs.get("friendly", False)

    def lock(self, timeout: Optional[Union[timedelta, int]] = None):
        try:
-            super().lock(timeout or timedelta(seconds=DEFAULT_TIMEOUT))
+            if not self._wait:
+                timeout = timeout or timedelta(seconds=DEFAULT_TIMEOUT)
+
+            with Tqdm(
+                bar_format="{desc}",
+                disable=not (self._wait and self._friendly),
+                desc="Waiting to acquire lock",
+            ):
+                super().lock(timeout)
        except flufl.lock.TimeOutError:
            raise LockError(FAILED_TO_LOCK_MESSAGE)  # noqa: B904

@@ -186,11 +212,11 @@ class HardlinkLock(flufl.lock.Lock, LockBase):

        if self._tmp_dir is not None:
            # Under Windows file path length is limited so we hash it
-            hasher = hashlib.md5(self._claimfile.encode())  # noqa: S324
+            hasher = hashlib.md5(self._claimfile.encode(), usedforsecurity=False)
            filename = hasher.hexdigest()
            self._claimfile = os.path.join(self._tmp_dir, filename + ".lock")


-def make_lock(lockfile, tmp_dir=None, friendly=False, hardlink_lock=False):
+def make_lock(lockfile, tmp_dir=None, friendly=False, hardlink_lock=False, wait=False):
    cls = HardlinkLock if hardlink_lock else Lock
-    return cls(lockfile, tmp_dir=tmp_dir, friendly=friendly)
+    return cls(lockfile, tmp_dir=tmp_dir, friendly=friendly, wait=wait)
@@ -5,6 +5,8 @@ import logging.config
 import logging.handlers
 import os
 import sys
+from collections.abc import Iterator
+from contextlib import contextmanager
 from typing import ClassVar

 import colorama
@@ -36,7 +38,7 @@ def add_logging_level(level_name, level_num, method_name=None):
            self._log(level_num, message, args, **kwargs)

    def log_to_root(message, *args, **kwargs):
-        logging.log(level_num, message, *args, **kwargs)
+        logging.log(level_num, message, *args, **kwargs)  # noqa: LOG015

    # getLevelName resolves the numeric log level if already defined,
    # otherwise returns a string
@@ -174,9 +176,19 @@ def _iter_causes(exc):
        exc = exc.__cause__


-def set_loggers_level(level: int = logging.INFO) -> None:
+@contextmanager
+def set_loggers_level(level: int = logging.INFO) -> Iterator[None]:
+    ret: dict[logging.Logger, int] = {}
    for name in ["dvc", "dvc_objects", "dvc_data"]:
-        logging.getLogger(name).setLevel(level)
+        _logger = logging.getLogger(name)
+        ret[_logger] = _logger.getEffectiveLevel()
+        _logger.setLevel(level)
+
+    try:
+        yield
+    finally:
+        for _logger, old_lvl in ret.items():
+            _logger.setLevel(old_lvl)


 def setup(level: int = logging.INFO, log_colors: bool = True) -> None:
@@ -45,7 +45,7 @@ if TYPE_CHECKING:
    from dvc_data.hashfile.obj import HashFile
    from dvc_data.index import DataIndexKey

-    from .ignore import DvcIgnoreFilter
+    from .ignore import CheckIgnoreResult, DvcIgnoreFilter

 logger = logger.getChild(__name__)

@@ -267,9 +267,9 @@ class OutputIsStageFileError(DvcException):


 class OutputIsIgnoredError(DvcException):
-    def __init__(self, match):
-        lines = "\n".join(match.patterns)
-        super().__init__(f"Path '{match.file}' is ignored by\n{lines}")
+    def __init__(self, result: "CheckIgnoreResult"):
+        pi = result.pattern_infos[-1]
+        super().__init__(f"Path {result.file!r} is ignored by {pi!s}")


 class CheckoutCallback(TqdmCallback):
@@ -431,7 +431,7 @@ class Output:
            self.meta.nfiles = len(self.files)
            self.meta.size = sum(filter(None, (f.get("size") for f in self.files)))
            self.meta.remote = first(f.get("remote") for f in self.files)
-        elif self.meta.nfiles or self.hash_info and self.hash_info.isdir:
+        elif self.meta.nfiles or (self.hash_info and self.hash_info.isdir):
            self.meta.isdir = True
            if not self.hash_info and self.hash_name not in ("md5", "md5-dos2unix"):
                md5 = getattr(self.meta, "md5", None)
@@ -752,8 +752,9 @@ class Output:
            granular = (
                self.is_dir_checksum and filter_info and filter_info != self.fs_path
            )
+            hardlink = relink and next(iter(self.cache.cache_types), None) == "hardlink"
            if granular:
-                obj = self._commit_granular_dir(filter_info, hardlink=False)
+                obj = self._commit_granular_dir(filter_info, hardlink=hardlink)
            else:
                staging, _, obj = self._build(
                    self.cache,
@@ -771,7 +772,7 @@ class Output:
                        self.cache,
                        {obj.hash_info},
                        shallow=False,
-                        hardlink=False,
+                        hardlink=hardlink,
                        callback=cb,
                    )
            if relink:
@@ -1202,8 +1203,9 @@ class Output:
        if stage:
            abs_path = os.path.join(stage.wdir, path)
            if self._is_path_dvcignore(abs_path):
-                check = stage.repo.dvcignore.check_ignore(abs_path)
-                raise self.IsIgnoredError(check)
+                check: CheckIgnoreResult = stage.repo.dvcignore.check_ignore(abs_path)
+                if check.match:
+                    raise self.IsIgnoredError(check)

    def _check_can_merge(self, out):
        if self.protocol != out.protocol:
@@ -1399,12 +1401,14 @@ class Output:

        assert staging
        assert obj.hash_info
+
+        hardlink = relink and next(iter(self.cache.cache_types), None) == "hardlink"
        with TqdmCallback(desc=f"Adding {self} to cache", unit="file") as cb:
            otransfer(
                staging,
                self.cache,
                {obj.hash_info},
-                hardlink=False,
+                hardlink=hardlink,
                shallow=False,
                callback=cb,
            )
@@ -1449,25 +1453,43 @@ class Output:
        self.remote = other.remote
        self.can_push = other.can_push

-    def merge_version_meta(self, other: "Output"):
+    def _get_versioned_meta(
+        self,
+    ) -> Optional[
+        tuple["HashInfo", Optional["Meta"], Optional[Union["HashFile", "Tree"]]]
+    ]:
+        if self.files is not None or (
+            self.meta is not None and self.meta.version_id is not None
+        ):
+            old_obj = self.obj if self.obj is not None else self.get_obj()
+            return self.hash_info, self.meta, old_obj
+        return None
+
+    def merge_version_meta(
+        self,
+        old_hi: "HashInfo",
+        old_meta: Optional["Meta"],
+        old_obj: Optional[Union["HashFile", "Tree"]],
+    ):
        """Merge version meta for files which are unchanged from other."""
        if not self.hash_info:
            return
        if self.hash_info.isdir:
-            return self._merge_dir_version_meta(other)
-        if self.hash_info != other.hash_info:
+            return self._merge_dir_version_meta(old_hi, old_obj)
+        if self.hash_info != old_hi:
            return
-        self.meta = other.meta
+        self.meta = old_meta

-    def _merge_dir_version_meta(self, other: "Output"):
+    def _merge_dir_version_meta(
+        self, old_hi: "HashInfo", old_obj: Optional[Union["HashFile", "Tree"]]
+    ):
        from dvc_data.hashfile.tree import update_meta

-        if not self.obj or not other.hash_info.isdir:
+        if not self.obj or not old_hi.isdir:
            return
-        other_obj = other.obj if other.obj is not None else other.get_obj()
        assert isinstance(self.obj, Tree)
-        assert isinstance(other_obj, Tree)
-        updated = update_meta(self.obj, other_obj)
+        assert isinstance(old_obj, Tree)
+        updated = update_meta(self.obj, old_obj)
        assert updated.hash_info == self.obj.hash_info
        self.obj = updated
        self.files = updated.as_list(with_meta=True)
@@ -105,7 +105,7 @@ class Meta:

    @staticmethod
    def update_path(meta: "Meta", path: Union[str, int]):
-        dpaths = meta.dpaths[:] + [str(path)]
+        dpaths = [*meta.dpaths, str(path)]
        return replace(meta, dpaths=dpaths)

    def __str__(self):
@@ -155,7 +155,7 @@ class Value(Node):
 PRIMITIVES = (int, float, str, bytes, bool)


-class Container(Node, ABC):
+class Container(Node, ABC):  # noqa: PLW1641
    meta: Meta
    data: Union[list, dict]
    _key_transform = staticmethod(identity)
@@ -176,8 +176,9 @@ class Container(Node, ABC):
            return value
        if isinstance(value, (list, dict)):
            assert meta
-            container = CtxDict if isinstance(value, dict) else CtxList
-            return container(value, meta=meta)
+            if isinstance(value, dict):
+                return CtxDict(value, meta=meta)
+            return CtxList(value, meta=meta)
        msg = f"Unsupported value of type '{type(value).__name__}' in '{meta}'"
        raise TypeError(msg)

@@ -2,13 +2,19 @@
 # Including changing base dir of path specification patterns and merging
 # of two path specification patterns with different base
 # All the operations follow the documents of `gitignore`
-from collections import namedtuple
+from typing import NamedTuple

 from pathspec.util import normalize_file

 from dvc.utils import relpath

-PatternInfo = namedtuple("PatternInfo", ["patterns", "file_info"])  # noqa: PYI024
+
+class PatternInfo(NamedTuple):
+    patterns: str
+    file_info: str
+
+    def __str__(self) -> str:
+        return self.file_info or f":{self.patterns}"


 def _not_ignore(rule):
@@ -27,11 +33,9 @@ def _remove_slash(rule):

 def _match_all_level(rule):
    if rule[:-1].find("/") >= 0 and not rule.startswith("**/"):
-        if rule.startswith("/"):
-            rule = rule[1:]
+        rule = rule.removeprefix("/")
        return False, rule
-    if rule.startswith("**/"):
-        rule = rule[3:]
+    rule = rule.removeprefix("**/")
    return True, rule


@@ -144,6 +144,7 @@ class Repo:
        scm: Optional[Union["Git", "NoSCM"]] = None,
        remote: Optional[str] = None,
        remote_config: Optional["DictStrAny"] = None,
+        _wait_for_lock: bool = False,
    ):
        from dvc.cachemgr import CacheManager
        from dvc.data_cloud import DataCloud
@@ -166,7 +167,8 @@ class Repo:
        self._config = config
        self._remote = remote
        self._remote_config = remote_config
-        self._data_index = None
+        self._data_index: Optional[DataIndex] = None
+        self._wait_for_lock = _wait_for_lock

        if rev and not fs:
            self._scm = scm = SCM(root_dir or os.curdir)
@@ -204,6 +206,7 @@ class Repo:
                    tmp_dir=self.tmp_dir,
                    hardlink_lock=self.config["core"].get("hardlink_lock", False),
                    friendly=True,
+                    wait=self._wait_for_lock,
                )
                os.makedirs(self.site_cache_dir, exist_ok=True)
                if not fs and (
@@ -609,7 +612,7 @@ class Repo:
        from dvc.fs import GitFileSystem
        from dvc.version import version_tuple

-        cache_dir = self.config["core"].get("site_cache_dir") or site_cache_dir()
+        cache_dir = site_cache_dir(self.config["core"].get("site_cache_dir"))

        subdir = None
        if isinstance(self.fs, GitFileSystem):
@@ -638,10 +641,11 @@ class Repo:
        # that just happened to be at the same path as old deleted ones.
        btime = self._btime or getattr(os.stat(root_dir), "st_birthtime", None)

-        md5 = hashlib.md5(  # noqa: S324
+        md5 = hashlib.md5(
            str(
                (root_dir, subdir, btime, getpass.getuser(), version_tuple[0], salt)
-            ).encode()
+            ).encode(),
+            usedforsecurity=False,
        )
        repo_token = md5.hexdigest()
        return os.path.join(repos_dir, repo_token)
@@ -1,4 +1,5 @@
 import os
+from collections import defaultdict
 from typing import TYPE_CHECKING

 from dvc.exceptions import (
@@ -14,7 +15,9 @@ from dvc.utils import relpath
 from . import locked

 if TYPE_CHECKING:
-    from dvc_data.index import BaseDataIndex, DataIndexEntry
+    from dvc.repo.index import IndexView
+    from dvc_data.index import BaseDataIndex, DataIndexEntry, DataIndexKey
+    from dvc_data.index.diff import Change
    from dvc_objects.fs.base import FileSystem

 logger = logger.getChild(__name__)
@@ -36,10 +39,12 @@ def _remove_unused_links(repo):
    return ret


-def _build_out_changes(index, changes):
+def _build_out_changes(
+    index: "IndexView", changes: dict["DataIndexKey", "Change"]
+) -> dict["DataIndexKey", tuple[str, dict[str, int]]]:
    from dvc_data.index.checkout import MODIFY

-    out_keys = []
+    out_keys: list[DataIndexKey] = []
    for out in index.outs:
        if not out.use_cache:
            continue
@@ -47,19 +52,28 @@ def _build_out_changes(index, changes):
        ws, key = out.index_key
        if ws != "repo":
            continue
-
        out_keys.append(key)

-    out_changes = {}
+    out_stats: dict[DataIndexKey, dict[str, int]]
+    out_stats = defaultdict(lambda: defaultdict(int))
+
+    out_changes: dict[DataIndexKey, tuple[str, dict[str, int]]] = {}
    for key, change in changes.items():
+        typ = change.typ
+        isdir = change.new and change.new.isdir
        for out_key in out_keys:
            if len(out_key) > len(key) or key[: len(out_key)] != out_key:
                continue

+            stats = out_stats[out_key]
+            if not isdir:
+                stats[typ] += 1
+
            if key == out_key:
-                out_changes[out_key] = change.typ
-            elif not out_changes.get(out_key):
-                out_changes[out_key] = MODIFY
+                out_changes[out_key] = typ, stats
+            elif out_key not in out_changes:
+                typ = MODIFY
+                out_changes[out_key] = typ, stats
            break

    return out_changes
@@ -107,14 +121,13 @@ def checkout(  # noqa: C901
    from dvc.stage.exceptions import StageFileBadNameError, StageFileDoesNotExistError
    from dvc_data.index.checkout import ADD, DELETE, MODIFY, apply, compare

-    stats: dict[str, list[str]] = {
-        "added": [],
-        "deleted": [],
-        "modified": [],
-    }
+    stats = {"modified": 0, "added": 0, "deleted": 0}
+    changes: dict[str, list[str]] = {"modified": [], "added": [], "deleted": []}
+
    if not targets:
        targets = [None]
-        stats["deleted"] = _remove_unused_links(self)
+        changes["deleted"] = _remove_unused_links(self)
+        stats["deleted"] = len(changes["deleted"])

    if isinstance(targets, str):
        targets = [targets]
@@ -127,8 +140,10 @@ def checkout(  # noqa: C901
            raise CheckoutErrorSuggestGit(target) from exc
        raise  # noqa: PLE0704

-    view = self.index.targets_view(
-        targets, recursive=recursive, with_deps=with_deps, onerror=onerror
+    from .index import index_from_targets
+
+    view = index_from_targets(
+        self, targets=targets, recursive=recursive, with_deps=with_deps, onerror=onerror
    )

    with ui.progress(unit="entry", desc="Building workspace index", leave=True) as pb:
@@ -149,7 +164,10 @@ def checkout(  # noqa: C901

    def checkout_onerror(src_path, dest_path, _exc):
        logger.debug(
-            "failed to create '%s' from '%s'", dest_path, src_path, exc_info=True
+            "failed to create '%s' from '%s'",
+            dest_path,
+            src_path,
+            exc_info=True,  # noqa: LOG014
        )

        for out_path in out_paths:
@@ -171,16 +189,26 @@ def checkout(  # noqa: C901
    out_changes = _build_out_changes(view, diff.changes)

    typ_map = {ADD: "added", DELETE: "deleted", MODIFY: "modified"}
-    for key, typ in out_changes.items():
+    for key, (typ, _stats) in out_changes.items():
        out_path = self.fs.join(self.root_dir, *key)

        if out_path in failed:
            self.fs.remove(out_path, recursive=True)
-        else:
-            self.state.save_link(out_path, self.fs)
-            stats[typ_map[typ]].append(_fspath_dir(out_path))
+            continue

+        self.state.save_link(out_path, self.fs)
+        for t, count in _stats.items():
+            stats_typ = typ_map[t]
+            stats[stats_typ] += count
+
+        changes[typ_map[typ]].append(_fspath_dir(out_path))
+
+    for changelist in changes.values():
+        # group directories first, then files. But keep them alphabetically sorted
+        changelist.sort(key=lambda p: (not p.endswith(os.sep), p))
+
+    result = changes | {"stats": stats}
    if failed and not allow_missing:
-        raise CheckoutError([relpath(out_path) for out_path in failed], stats)
-
-    return stats
+        result["failed"] = [relpath(out_path) for out_path in failed]
+        raise CheckoutError([relpath(out_path) for out_path in failed], result)
+    return result
@@ -43,13 +43,13 @@ def _collect_paths(


 def _filter_outs(
-    outs: Outputs, fs_paths: StrPaths, duplicates=False
+    repo: "Repo", outs: Outputs, fs_paths: StrPaths, duplicates=False
 ) -> tuple[Outputs, StrPaths]:
    res_outs: Outputs = []
    fs_res_paths = fs_paths

    for out in outs:
-        fs_path = out.repo.dvcfs.from_os_path(out.fs_path)
+        fs_path = repo.dvcfs.from_os_path(out.fs_path)
        if fs_path in fs_paths:
            res_outs.append(out)
            if not duplicates:
@@ -77,4 +77,4 @@ def collect(

    target_paths = _collect_paths(repo, targets, recursive=recursive)

-    return _filter_outs(outs, target_paths, duplicates=duplicates)
+    return _filter_outs(repo, outs, target_paths, duplicates=duplicates)
@@ -1,3 +1,4 @@
+from itertools import groupby
 from typing import TYPE_CHECKING

 from dvc import prompt
@@ -52,29 +53,38 @@ def commit(
    data_only=False,
    relink=True,
 ):
-    stages_info = [
-        info
-        for info in self.stage.collect_granular(
-            target, with_deps=with_deps, recursive=recursive
-        )
-        if not data_only or info.stage.is_data_source
-    ]
-    for stage_info in stages_info:
-        stage = stage_info.stage
-        if force:
-            stage.save(allow_missing=allow_missing)
-        else:
-            changes = stage.changed_entries()
-            if any(changes):
-                prompt_to_commit(stage, changes, force=force)
+    committed_stages = []
+    groups = groupby(
+        [
+            info
+            for info in self.stage.collect_granular(
+                target, with_deps=with_deps, recursive=recursive
+            )
+            if not data_only or info.stage.is_data_source
+        ],
+        key=lambda info: info.stage.dvcfile,
+    )
+
+    for dvcfile, stages_info_group in groups:
+        to_dump = []
+        for stage_info in stages_info_group:
+            stage = stage_info.stage
+            if force:
                stage.save(allow_missing=allow_missing)
-        stage.commit(
-            filter_info=stage_info.filter_info,
-            allow_missing=allow_missing,
-            relink=relink,
-        )
-        stage.dump(update_pipeline=False)
-    return [s.stage for s in stages_info]
+            else:
+                changes = stage.changed_entries()
+                if any(changes):
+                    prompt_to_commit(stage, changes, force=force)
+                    stage.save(allow_missing=allow_missing)
+            stage.commit(
+                filter_info=stage_info.filter_info,
+                allow_missing=allow_missing,
+                relink=relink,
+            )
+            to_dump.append(stage)
+        dvcfile.dump_stages(to_dump, update_pipeline=False)
+        committed_stages.extend(to_dump)
+    return committed_stages


@locked
@@ -1,38 +1,34 @@
 import os
 import posixpath
-from collections.abc import Iterable
-from typing import TYPE_CHECKING, Any, TypedDict, Union
+from collections import defaultdict
+from collections.abc import Iterable, Iterator, Mapping
+from typing import TYPE_CHECKING, Optional, TypedDict, Union

-from dvc.fs.callbacks import DEFAULT_CALLBACK
+from dvc.fs.callbacks import DEFAULT_CALLBACK, Callback, TqdmCallback
+from dvc.log import logger
 from dvc.ui import ui
+from dvc_data.index import DataIndexDirError

 if TYPE_CHECKING:
-    from dvc.fs.callbacks import Callback
    from dvc.repo import Repo
    from dvc.scm import Git, NoSCM
-    from dvc_data.index import DataIndex
+    from dvc_data.index import (
+        BaseDataIndex,
+        DataIndex,
+        DataIndexEntry,
+        DataIndexKey,
+        DataIndexView,
+    )
    from dvc_data.index.diff import Change
+    from dvc_objects.fs.base import FileSystem
+
+logger = logger.getChild(__name__)


 def posixpath_to_os_path(path: str) -> str:
    return path.replace(posixpath.sep, os.path.sep)


-def _adapt_typ(typ: str) -> str:
-    from dvc_data.index.diff import ADD, DELETE, MODIFY
-
-    if typ == MODIFY:
-        return "modified"
-
-    if typ == ADD:
-        return "added"
-
-    if typ == DELETE:
-        return "deleted"
-
-    return typ
-
-
 def _adapt_path(change: "Change") -> str:
    isdir = False
    if change.new and change.new.meta:
@@ -45,27 +41,98 @@ def _adapt_path(change: "Change") -> str:
    return os.path.sep.join(key)


+def _adapt_path_from_entry(entry: "DataIndexEntry") -> str:
+    key = entry.key
+    assert key
+    if entry.meta and entry.meta.isdir:
+        key = (*key, "")
+    return os.sep.join(key)
+
+
+def _get_missing_paths(
+    to_check: Mapping["FileSystem", Mapping[str, Iterable["DataIndexEntry"]]],
+    batch_size: Optional[int] = None,
+    callback: "Callback" = DEFAULT_CALLBACK,
+) -> Iterator[str]:
+    for fs, paths_map in to_check.items():
+        if batch_size == 1 or (batch_size is None and fs.protocol == "local"):
+            results = list(callback.wrap(map(fs.exists, paths_map)))
+        else:
+            results = fs.exists(
+                list(paths_map), batch_size=batch_size, callback=callback
+            )
+
+        for cache_path, exists in zip(paths_map, results):
+            if exists:
+                continue
+
+            for entry in paths_map[cache_path]:
+                yield _adapt_path_from_entry(entry)
+
+
+class StorageCallback(Callback):
+    def __init__(self, parent_cb: Callback) -> None:
+        super().__init__(size=0, value=0)
+        self.parent_cb = parent_cb
+
+    def set_size(self, size: int) -> None:
+        # This is a no-op to prevent `fs.exists` from trying to set the size
+        pass
+
+    def relative_update(self, value: int = 1) -> None:
+        self.parent_cb.relative_update(value)
+
+    def absolute_update(self, value: int) -> None:
+        self.parent_cb.relative_update(value - self.value)
+
+
+class Rename(TypedDict):
+    old: str
+    new: str
+
+
+class DiffResult(TypedDict, total=False):
+    modified: list[str]
+    added: list[str]
+    deleted: list[str]
+    renamed: list[Rename]
+    unchanged: list[str]
+    unknown: list[str]
+    not_in_cache: list[str]
+
+
 def _diff(
-    old: "DataIndex",
-    new: "DataIndex",
+    old: "BaseDataIndex",
+    new: "BaseDataIndex",
    *,
+    filter_keys: Optional[Iterable["DataIndexKey"]] = None,
    granular: bool = False,
    not_in_cache: bool = False,
-    not_in_remote: bool = False,
-    remote_refresh: bool = False,
+    batch_size: Optional[int] = None,
    callback: "Callback" = DEFAULT_CALLBACK,
-) -> dict[str, list[str]]:
-    from dvc_data.index import StorageError
-    from dvc_data.index.diff import UNCHANGED, UNKNOWN, diff
+    with_renames: bool = False,
+) -> DiffResult:
+    from dvc_data.index.diff import (
+        ADD,
+        DELETE,
+        MODIFY,
+        RENAME,
+        UNCHANGED,
+        UNKNOWN,
+        diff,
+    )

-    ret: dict[str, list[str]] = {}
+    ret: DiffResult = defaultdict(list)  # type: ignore[assignment]
+    change_types = {
+        MODIFY: "modified",
+        ADD: "added",
+        DELETE: "deleted",
+        RENAME: "renamed",
+    }

-    def _add_change(typ, change):
-        typ = _adapt_typ(typ)
-        if typ not in ret:
-            ret[typ] = []
-
-        ret[typ].append(_adapt_path(change))
+    to_check: dict[FileSystem, dict[str, list[DataIndexEntry]]] = defaultdict(
+        lambda: defaultdict(list)
+    )

    for change in diff(
        old,
@@ -74,45 +141,72 @@ def _diff(
        shallow=not granular,
        hash_only=True,
        with_unknown=True,
+        with_renames=with_renames,
        callback=callback,
    ):
+        typ = change.typ
+
+        # The index is a trie, so even when we filter by a specific path
+        # like `dir/file`, all parent nodes leading to that path (e.g., `dir/`)
+        # still appear in the view. As a result, keys like `dir/` will be present
+        # even if only `dir/file` matches the filter.
+        # We need to skip such entries to avoid showing root of tracked directories.
+        if filter_keys:
+            # RENAME does not have a `change.key`
+            if typ == RENAME:
+                assert change.new
+                key = change.new.key
+                # match with "new" key only
+                assert key
+            else:
+                key = change.key
+
+            if not any(key[: len(fk)] == fk for fk in filter_keys):
+                continue
+
        if (
-            change.typ == UNCHANGED
+            typ == UNCHANGED
            and (not change.old or not change.old.hash_info)
            and (not change.new or not change.new.hash_info)
        ):
            # NOTE: emulating previous behaviour
            continue

-        if change.typ == UNKNOWN and not change.new:
+        if typ == UNKNOWN and not change.new:
            # NOTE: emulating previous behaviour
            continue

-        if (
-            not_in_cache
-            and change.old
-            and change.old.hash_info
-            and not old.storage_map.cache_exists(change.old)
-        ):
-            # NOTE: emulating previous behaviour
-            _add_change("not_in_cache", change)
+        if not_in_cache and change.old and change.old.hash_info:
+            old_entry = change.old
+            cache_fs, cache_path = old.storage_map.get_cache(old_entry)
+            # check later in batches
+            to_check[cache_fs][cache_path].append(old_entry)

-        try:
-            if (
-                not_in_remote
-                and change.old
-                and change.old.hash_info
-                and not old.storage_map.remote_exists(
-                    change.old, refresh=remote_refresh
-                )
-            ):
-                _add_change("not_in_remote", change)
-        except StorageError:
-            pass
+        change_typ = change_types.get(typ, typ)
+        if typ == RENAME:
+            assert change.old is not None
+            assert change.new is not None
+            ret["renamed"].append(
+                {
+                    "old": _adapt_path_from_entry(change.old),
+                    "new": _adapt_path_from_entry(change.new),
+                }
+            )
+        else:
+            ret[change_typ].append(_adapt_path(change))  # type: ignore[literal-required]

-        _add_change(change.typ, change)
-
-    return ret
+    total_items = sum(
+        len(entries) for paths in to_check.values() for entries in paths.values()
+    )
+    with TqdmCallback(size=total_items, desc="Checking cache", unit="entry") as cb:
+        missing_items = list(
+            _get_missing_paths(
+                to_check, batch_size=batch_size, callback=StorageCallback(cb)
+            ),
+        )
+        if missing_items:
+            ret["not_in_cache"] = missing_items
+    return dict(ret)  # type: ignore[return-value]


 class GitInfo(TypedDict, total=False):
@@ -151,7 +245,49 @@ def _git_info(scm: Union["Git", "NoSCM"], untracked_files: str = "all") -> GitIn
    )


-def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> dict[str, list[str]]:
+def filter_index(
+    index: Union["DataIndex", "DataIndexView"],
+    filter_keys: Optional[Iterable["DataIndexKey"]] = None,
+) -> "BaseDataIndex":
+    from dvc_data.index.view import DataIndexView
+
+    if not filter_keys:
+        return index
+
+    if isinstance(index, DataIndexView):
+        orig_index = index._index
+        parent_filter_fn = index.filter_fn
+    else:
+        orig_index = index
+        parent_filter_fn = None
+
+    def filter_fn(key: "DataIndexKey") -> bool:
+        if parent_filter_fn is not None and not parent_filter_fn(key):
+            return False
+
+        for filter_key in filter_keys:
+            # eg: if key is "dir/file" and filter_key is "dir/", return True
+            if key[: len(filter_key)] == filter_key:
+                return True
+            # eg: if key is `dir/` and filter_key is `dir/file`, also return True.
+            # This ensures we include parent prefixes needed to reach matching leaves.
+            # Intermediate prefixes must be retained to access nested keys.
+            if filter_key[: len(key)] == key:
+                return True
+        return False
+
+    from dvc_data.index import view
+
+    return view(orig_index, filter_fn=filter_fn)
+
+
+def _diff_index_to_wtree(
+    repo: "Repo",
+    filter_keys: Optional[Iterable["DataIndexKey"]] = None,
+    granular: bool = False,
+    batch_size: Optional[int] = None,
+    with_renames: bool = False,
+) -> DiffResult:
    from .index import build_data_index

    with ui.progress(desc="Building workspace index", unit="entry") as pb:
@@ -162,37 +298,69 @@ def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> dict[str, list[str]]:
            compute_hash=True,
            callback=pb.as_callback(),
        )
+        workspace_view = filter_index(workspace, filter_keys=filter_keys)

    with ui.progress(
        desc="Calculating diff between index/workspace",
        unit="entry",
    ) as pb:
+        index = repo.index.data["repo"]
+        view = filter_index(index, filter_keys=filter_keys)
        return _diff(
-            repo.index.data["repo"],
-            workspace,
+            view,
+            workspace_view,
+            filter_keys=filter_keys,
+            granular=granular,
            not_in_cache=True,
+            with_renames=with_renames,
+            batch_size=batch_size,
            callback=pb.as_callback(),
-            **kwargs,
        )


 def _diff_head_to_index(
-    repo: "Repo", head: str = "HEAD", **kwargs: Any
-) -> dict[str, list[str]]:
-    index = repo.index.data["repo"]
+    repo: "Repo",
+    head: str = "HEAD",
+    filter_keys: Optional[Iterable["DataIndexKey"]] = None,
+    granular: bool = False,
+    with_renames: bool = False,
+) -> DiffResult:
+    from dvc.exceptions import NotDvcRepoError
+    from dvc.scm import RevError
+    from dvc_data.index import DataIndex

-    with repo.switch(head):
-        head_index = repo.index.data["repo"]
+    index = repo.index.data["repo"]
+    index_view = filter_index(index, filter_keys=filter_keys)
+
+    try:
+        with repo.switch(head):
+            head_index = repo.index.data["repo"]
+            head_view = filter_index(head_index, filter_keys=filter_keys)
+    except RevError:
+        logger.debug("failed to switch to '%s'", head)
+        head_view = DataIndex()
+    except NotDvcRepoError as exc:
+        # NOTE: this only gets raised on subdir repos at the moment,
+        # which looks like a bug in `repo.switch`.
+        logger.warning(exc)
+        head_view = DataIndex()

    with ui.progress(desc="Calculating diff between head/index", unit="entry") as pb:
-        return _diff(head_index, index, callback=pb.as_callback(), **kwargs)
+        return _diff(
+            head_view,
+            index_view,
+            filter_keys=filter_keys,
+            granular=granular,
+            with_renames=with_renames,
+            callback=pb.as_callback(),
+        )


 class Status(TypedDict):
    not_in_cache: list[str]
    not_in_remote: list[str]
-    committed: dict[str, list[str]]
-    uncommitted: dict[str, list[str]]
+    committed: DiffResult
+    uncommitted: DiffResult
    untracked: list[str]
    unchanged: list[str]
    git: GitInfo
@@ -217,27 +385,158 @@ def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> list[str]
    return [repo.fs.relpath(file, start) for file in files]


-def status(repo: "Repo", untracked_files: str = "no", **kwargs: Any) -> Status:
+def _get_entries_not_in_remote(
+    repo: "Repo",
+    filter_keys: Optional[Iterable["DataIndexKey"]] = None,
+    granular: bool = False,
+    remote_refresh: bool = False,
+) -> list[str]:
+    """Get entries that are not in remote storage."""
+    from dvc.repo.worktree import worktree_view
+    from dvc_data.index import StorageKeyError
+
+    entries: dict[DataIndexKey, DataIndexEntry] = {}
+
+    def _onerror(entry, exc):
+        if not isinstance(exc, DataIndexDirError):
+            raise exc
+        # We don't have the contents of this dir file, so we will only check this key.
+        entries[entry.key] = entry
+
+    # View into the index, with only pushable entries
+    index = worktree_view(repo.index, push=True)
+    data_index = index.data["repo"]
+
+    orig_data_index_onerror = data_index.onerror
+    data_index.onerror = _onerror
+
+    view = filter_index(data_index, filter_keys=filter_keys)  # type: ignore[arg-type]
+
+    missing_entries = []
+
+    storage_map = view.storage_map
+
+    n = 0
+    with TqdmCallback(size=n, desc="Checking remote", unit="entry") as cb:
+        for key, entry in view.iteritems(shallow=not granular):
+            if not (entry and entry.hash_info):
+                continue
+
+            # The index is a trie, so even when we filter by a specific path
+            # like `dir/file`, all parent nodes leading to that path (e.g., `dir/`)
+            # still appear in the view. As a result, keys like `dir/` will be present
+            # even if only `dir/file` matches the filter.
+            # We need to skip such entries to avoid showing root of tracked directories.
+            if filter_keys and not any(
+                key[: len(filter_key)] == filter_key for filter_key in filter_keys
+            ):
+                continue
+
+            entries[key] = entry
+            n += 1
+            cb.set_size(n)
+
+        for key, entry in entries.items():
+            k = (*key, "") if entry.meta and entry.meta.isdir else key
+            try:
+                if not storage_map.remote_exists(entry, refresh=remote_refresh):
+                    missing_entries.append(os.path.sep.join(k))
+            except StorageKeyError:
+                pass
+            finally:
+                cb.relative_update()
+    data_index.onerror = orig_data_index_onerror
+    return missing_entries
+
+
+def _matches_target(p: str, targets: Iterable[str]) -> bool:
+    sep = os.sep
+    return any(p == t or p.startswith(t + sep) for t in targets)
+
+
+def _prune_keys(filter_keys: Iterable["DataIndexKey"]) -> list["DataIndexKey"]:
+    sorted_keys = sorted(set(filter_keys), key=len)
+    result: list[DataIndexKey] = []
+
+    for key in sorted_keys:
+        if not any(key[: len(prefix)] == prefix for prefix in result):
+            result.append(key)
+    return result
+
+
+def status(  # noqa: PLR0913
+    repo: "Repo",
+    targets: Optional[Iterable[Union[os.PathLike[str], str]]] = None,
+    *,
+    granular: bool = False,
+    untracked_files: str = "no",
+    remote: Optional[str] = None,
+    not_in_remote: bool = False,
+    remote_refresh: bool = False,
+    config: Optional[dict] = None,
+    batch_size: Optional[int] = None,
+    head: str = "HEAD",
+    with_renames: bool = True,
+) -> Status:
    from dvc.scm import NoSCMError, SCMError

-    head = kwargs.pop("head", "HEAD")
-    uncommitted_diff = _diff_index_to_wtree(repo, **kwargs)
+    config = config or {}
+    if remote and not_in_remote:
+        logger.debug("Using remote %r", remote)
+        core = config.setdefault("core", {})
+        core["remote"] = remote
+    repo.config.merge(config)
+
+    targets = targets or []
+    filter_keys: list[DataIndexKey] = [repo.fs.relparts(os.fspath(t)) for t in targets]
+    # try to remove duplicate and overlapping keys
+    filter_keys = _prune_keys(filter_keys)
+
+    uncommitted_diff = _diff_index_to_wtree(
+        repo,
+        filter_keys=filter_keys,
+        granular=granular,
+        batch_size=batch_size,
+        with_renames=with_renames,
+    )
    unchanged = set(uncommitted_diff.pop("unchanged", []))

+    entries_not_in_remote: list[str] = []
+    if not_in_remote:
+        entries_not_in_remote = _get_entries_not_in_remote(
+            repo,
+            filter_keys=filter_keys,
+            granular=granular,
+            remote_refresh=remote_refresh,
+        )
+
    try:
-        committed_diff = _diff_head_to_index(repo, head=head, **kwargs)
+        committed_diff = _diff_head_to_index(
+            repo,
+            filter_keys=filter_keys,
+            head=head,
+            granular=granular,
+            with_renames=with_renames,
+        )
    except (SCMError, NoSCMError):
        committed_diff = {}
    else:
        unchanged &= set(committed_diff.pop("unchanged", []))

    git_info = _git_info(repo.scm, untracked_files=untracked_files)
-    untracked = git_info.get("untracked", [])
-    untracked = _transform_git_paths_to_dvc(repo, untracked)
+    scm_filter_targets = {
+        os.path.relpath(os.path.abspath(t), repo.scm.root_dir) for t in targets
+    }
+    untracked_it: Iterable[str] = git_info.get("untracked", [])
+    if scm_filter_targets:
+        untracked_it = (
+            f for f in untracked_it if _matches_target(f, scm_filter_targets)
+        )
+    untracked = _transform_git_paths_to_dvc(repo, untracked_it)
    # order matters here
    return Status(
        not_in_cache=uncommitted_diff.pop("not_in_cache", []),
-        not_in_remote=uncommitted_diff.pop("not_in_remote", []),
+        not_in_remote=entries_not_in_remote,
        committed=committed_diff,
        uncommitted=uncommitted_diff,
        untracked=untracked,
@@ -16,7 +16,7 @@ from dvc.types import StrPath
 from dvc_data.hashfile.meta import Meta

 if TYPE_CHECKING:
-    from dvcx.dataset import DatasetRecord, DatasetVersion  # type: ignore[import]
+    from datachain.dataset import DatasetRecord, DatasetVersion  # type: ignore[import]
    from typing_extensions import Self

    from dvc.repo import Repo
@@ -29,10 +29,10 @@ def _get_dataset_record(name: str) -> "DatasetRecord":
    from dvc.exceptions import DvcException

    try:
-        from dvcx.catalog import get_catalog  # type: ignore[import]
+        from datachain.catalog import get_catalog  # type: ignore[import]

    except ImportError as exc:
-        raise DvcException("dvcx is not installed") from exc
+        raise DvcException("datachain is not installed") from exc

    catalog = get_catalog()
    return catalog.get_remote_dataset(name)
@@ -86,7 +86,7 @@ class SerDe:
 class DatasetSpec(SerDe):
    name: str
    url: str
-    type: Literal["dvc", "dvcx", "url"]
+    type: Literal["dvc", "dc", "url"]


@frozen(kw_only=True)
@@ -108,7 +108,7 @@ class DVCDatasetLock(DVCDatasetSpec):


@frozen(kw_only=True)
-class DVCXDatasetLock(DatasetSpec):
+class DatachainDatasetLock(DatasetSpec):
    version: int
    created_at: datetime = field(converter=to_datetime)

@@ -160,13 +160,13 @@ class DVCDataset:


@frozen(kw_only=True)
-class DVCXDataset:
+class DatachainDataset:
    manifest_path: str
    spec: "DatasetSpec"
-    lock: "Optional[DVCXDatasetLock]" = field(default=None)
+    lock: "Optional[DatachainDatasetLock]" = field(default=None)
    _invalidated: bool = field(default=False, eq=False, repr=False)

-    type: ClassVar[Literal["dvcx"]] = "dvcx"
+    type: ClassVar[Literal["dc"]] = "dc"

    @property
    def pinned(self) -> bool:
@@ -193,7 +193,7 @@ class DVCXDataset:
        name, _version = self.name_version
        version = version if version is not None else _version
        version_info = _get_dataset_info(name, record=record, version=version)
-        lock = DVCXDatasetLock(
+        lock = DatachainDatasetLock(
            **self.spec.to_dict(),
            version=version_info.version,
            created_at=version_info.created_at,
@@ -226,9 +226,9 @@ class URLDataset:
        return evolve(self, lock=lock)


-Lock = Union[DVCDatasetLock, DVCXDatasetLock, URLDatasetLock]
+Lock = Union[DVCDatasetLock, DatachainDatasetLock, URLDatasetLock]
 Spec = Union[DatasetSpec, DVCDatasetSpec]
-Dataset = Union[DVCDataset, DVCXDataset, URLDataset]
+Dataset = Union[DVCDataset, DatachainDataset, URLDataset]


 class DatasetNotFoundError(DvcException, KeyError):
@@ -307,13 +307,13 @@ class Datasets(Mapping[str, Dataset]):
            raise ValueError("type should be present in spec")
        if typ == "dvc":
            return DVCDatasetSpec.from_dict(spec)
-        if typ in {"dvcx", "url"}:
+        if typ in {"dc", "url"}:
            return DatasetSpec.from_dict(spec)
        raise ValueError(f"unknown dataset type: {spec.get('type', '')}")

    @staticmethod
    def _lock_from_info(lock: Optional[dict[str, Any]]) -> Optional[Lock]:
-        kl = {"dvc": DVCDatasetLock, "dvcx": DVCXDatasetLock, "url": URLDatasetLock}
+        kl = {"dvc": DVCDatasetLock, "dc": DatachainDatasetLock, "url": URLDatasetLock}
        if lock and (cls := kl.get(lock.get("type", ""))):  # type: ignore[assignment]
            return cls.from_dict(lock)  # type: ignore[attr-defined]
        return None
@@ -356,9 +356,9 @@ class Datasets(Mapping[str, Dataset]):
                lock=lock,
                invalidated=_invalidated,
            )
-        if spec.type == "dvcx":
-            assert lock is None or isinstance(lock, DVCXDatasetLock)
-            return DVCXDataset(
+        if spec.type == "dc":
+            assert lock is None or isinstance(lock, DatachainDatasetLock)
+            return DatachainDataset(
                manifest_path=manifest_path,
                spec=spec,
                lock=lock,
@@ -374,7 +374,7 @@ class Datasets(Mapping[str, Dataset]):
        manifest_path: StrPath = "dvc.yaml",
        **kwargs: Any,
    ) -> Dataset:
-        assert type in {"dvc", "dvcx", "url"}
+        assert type in {"dvc", "dc", "url"}
        kwargs.update({"name": name, "url": url, "type": type})
        dataset = self._build_dataset(os.path.abspath(manifest_path), kwargs)
        dataset = dataset.update(self.repo)
@@ -389,13 +389,16 @@ class Datasets(Mapping[str, Dataset]):

        if dataset.type == "url" and (version or kwargs.get("rev")):
            raise ValueError("cannot update version/revision for a url")
-        if dataset.type == "dvcx" and version is not None:
+        if dataset.type == "dc" and version is not None:
            if not isinstance(version, int):
                raise TypeError(
-                    f"dvcx version has to be an integer, got {type(version).__name__!r}"
+                    "DataChain dataset version has to be an integer, "
+                    f"got {type(version).__name__!r}"
                )
            if version < 1:
-                raise ValueError(f"dvcx version should be >=1, got {version}")
+                raise ValueError(
+                    f"DataChain dataset version should be >=1, got {version}"
+                )

        new = dataset.update(self.repo, **kwargs)

@@ -45,6 +45,12 @@ def _diff(old, new, data_keys, with_missing=False):
        with_renames=True,
        meta_cmp_key=meta_cmp_key,
        roots=data_keys,
+        # Include unknown entries from missing dir entry, so that they don't
+        # get reported as added/modified/deleted.
+        # Also return unchanged entries so that we can check if they are missing
+        # from cache.
+        with_unknown=True,
+        with_unchanged=with_missing,
    ):
        if (change.old and change.old.isdir and not change.old.hash_info) or (
            change.new and change.new.isdir and not change.new.hash_info
@@ -6,7 +6,7 @@ def du(
    path: Optional[str] = None,
    rev: Optional[str] = None,
    summarize: bool = False,
-    config: Union[None, dict[str, Any], str] = None,
+    config: Union[dict[str, Any], str, None] = None,
    remote: Optional[str] = None,
    remote_config: Optional[dict] = None,
 ):
@@ -205,7 +205,7 @@ class Experiments:
        Experiment will be derived from the current workspace.
        """

-        name = kwargs.get("name", None)
+        name = kwargs.get("name")
        baseline_sha = kwargs.get("baseline_rev") or self.repo.scm.get_rev()

        if name:
@@ -274,6 +274,7 @@ def collect(
    num: int = 1,
    hide_queued: bool = False,
    hide_failed: bool = False,
+    hide_workspace: bool = False,
    sha_only: bool = False,
    **kwargs,
 ) -> list["ExpState"]:
@@ -303,8 +304,11 @@ def collect(
            repo.scm, baseline_revs, refs=cached_refs, logger=logger
        )

-    workspace_data = collect_rev(repo, "workspace", **kwargs)
-    result: list[ExpState] = [workspace_data]
+    result: list[ExpState] = []
+    if not hide_workspace:
+        workspace_data = collect_rev(repo, "workspace", **kwargs)
+        result.append(workspace_data)
+
    queued = collect_queued(repo, baseline_revs, **kwargs) if not hide_queued else {}
    active = collect_active(repo, baseline_revs, **kwargs)
    failed = collect_failed(repo, baseline_revs, **kwargs) if not hide_failed else {}
@@ -1,20 +1,21 @@
-import logging
 import os
 import pickle
 import shutil
 from abc import ABC, abstractmethod
 from collections.abc import Iterable, Iterator
-from contextlib import contextmanager
+from contextlib import AbstractContextManager, contextmanager
 from dataclasses import asdict, dataclass
 from enum import IntEnum
 from itertools import chain
 from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, Union

+from funcy import nullcontext
 from scmrepo.exceptions import SCMError

 from dvc.env import DVC_EXP_AUTO_PUSH, DVC_EXP_GIT_REMOTE
 from dvc.exceptions import DvcException
 from dvc.log import logger
+from dvc.logger import set_loggers_level
 from dvc.repo.experiments.exceptions import ExperimentExistsError
 from dvc.repo.experiments.refs import EXEC_BASELINE, EXEC_BRANCH, ExpRefInfo
 from dvc.repo.experiments.utils import to_studio_params
@@ -297,7 +298,9 @@ class BaseExecutor(ABC):
                stages = dvc.commit([], recursive=recursive, force=True, relink=False)
            exp_hash = cls.hash_exp(stages)
            if include_untracked:
-                dvc.scm.add(include_untracked, force=True)  # type: ignore[call-arg]
+                from dvc.scm import add_no_submodules
+
+                add_no_submodules(dvc.scm, include_untracked, force=True)  # type: ignore[call-arg]

            with cls.auto_push(dvc):
                cls.commit(
@@ -466,8 +469,8 @@ class BaseExecutor(ABC):

        if queue is not None:
            queue.put((rev, os.getpid()))
-        if log_errors and log_level is not None:
-            cls._set_log_level(log_level)
+
+        log_ctx = cls._set_log_level(log_level) if log_errors else nullcontext()

        exp_hash: Optional[str] = None
        exp_ref: Optional[ExpRefInfo] = None
@@ -476,14 +479,17 @@ class BaseExecutor(ABC):
        if info.name:
            ui.write(f"Reproducing experiment '{info.name}'")

-        with cls._repro_dvc(
-            info,
-            infofile,
-            log_errors=log_errors,
-            copy_paths=copy_paths,
-            message=message,
-            **kwargs,
-        ) as dvc:
+        with (
+            log_ctx,
+            cls._repro_dvc(
+                info,
+                infofile,
+                log_errors=log_errors,
+                copy_paths=copy_paths,
+                message=message,
+                **kwargs,
+            ) as dvc,
+        ):
            args, kwargs = cls._repro_args(dvc)
            if args:
                targets: Optional[Union[list, str]] = args[0]
@@ -509,7 +515,9 @@ class BaseExecutor(ABC):
            stages = dvc.reproduce(*args, **kwargs)
            if paths := cls._get_top_level_paths(dvc):
                logger.debug("Staging top-level files: %s", paths)
-                dvc.scm_context.add(paths)
+                from dvc.scm import add_no_submodules
+
+                add_no_submodules(dvc.scm, paths)

            exp_hash = cls.hash_exp(stages)
            if not repro_dry:
@@ -786,13 +794,10 @@ class BaseExecutor(ABC):
        return orig_rev

    @staticmethod
-    def _set_log_level(level):
-        # When executor.reproduce is run in a multiprocessing child process,
-        # dvc.cli.main will not be called for that child process so we need to
-        # setup logging ourselves
-        dvc_logger = logging.getLogger("dvc")
+    def _set_log_level(level: Optional[int]) -> AbstractContextManager[None]:
        if level is not None:
-            dvc_logger.setLevel(level)
+            return set_loggers_level(level)
+        return nullcontext()

    @staticmethod
    def _copy_path(src, dst):
@@ -38,7 +38,7 @@ logger = logger.getChild(__name__)


@dataclass(frozen=True)
-class QueueEntry:
+class QueueEntry:  # noqa: PLW1641
    dvc_root: str
    scm_root: str
    stash_ref: str
@@ -372,7 +372,7 @@ class BaseStashQueue(ABC):
        )

    def _stash_commit_deps(self, *args, **kwargs):
-        if len(args):
+        if args:
            targets = args[0]
        else:
            targets = kwargs.get("targets")
@@ -609,9 +609,9 @@ class BaseStashQueue(ABC):
                result[exp_name] = entry_name_dict[exp_name]
                continue
            if self.scm.is_sha(exp_name):
-                for rev in entry_rev_dict:
+                for rev, entry in entry_rev_dict.items():
                    if rev.startswith(exp_name.lower()):
-                        result[exp_name] = entry_rev_dict[rev]
+                        result[exp_name] = entry
                        break

        return result
@@ -1,3 +1,4 @@
+import glob
 import hashlib
 import locale
 import logging
@@ -434,9 +435,13 @@ class LocalCeleryQueue(BaseStashQueue):
        try:
            proc_info = self.proc[queue_entry.stash_rev]
        except KeyError:
-            raise DvcException(  # noqa: B904
-                f"No output logs found for experiment '{rev}'"
-            )
+            message = f"No output logs found for experiment '{rev}'"
+            if self.match_queue_entry_by_name({rev}, self.iter_failed()).get(rev):
+                message += "\nExperiment likely failed during setup."
+                if celery_logs := glob.glob(os.path.join(self.wdir, "*.out")):
+                    message += " Check the celery logs for more details:"
+                    message += f"\n\tcat {' '.join(celery_logs)}".expandtabs(4)
+            raise DvcException(message)  # noqa: B904
        with open(
            proc_info.stdout, encoding=encoding or locale.getpreferredencoding()
        ) as fobj:
@@ -67,3 +67,12 @@ class ExpRefInfo:
        baseline_sha = parts[2] + parts[3]
        name = parts[4] if len(parts) == 5 else None
        return cls(baseline_sha, name)
+
+    def __eq__(self, other):
+        if not isinstance(other, ExpRefInfo):
+            return False
+
+        return self.baseline_sha == other.baseline_sha and self.name == other.name
+
+    def __hash__(self):
+        return hash((self.baseline_sha, self.name))
@@ -6,7 +6,7 @@ from dvc.repo import locked
 from dvc.repo.scm_context import scm_context
 from dvc.scm import Git, iter_revs

-from .exceptions import UnresolvedExpNamesError
+from .exceptions import InvalidArgumentError, UnresolvedExpNamesError
 from .utils import exp_refs, exp_refs_by_baseline, push_refspec

 if TYPE_CHECKING:
@@ -24,16 +24,22 @@ logger = logger.getChild(__name__)
@scm_context
 def remove(  # noqa: C901, PLR0912
    repo: "Repo",
-    exp_names: Union[None, str, list[str]] = None,
+    exp_names: Union[str, list[str], None] = None,
    rev: Optional[Union[list[str], str]] = None,
    all_commits: bool = False,
    num: int = 1,
    queue: bool = False,
    git_remote: Optional[str] = None,
+    keep: bool = False,
 ) -> list[str]:
    removed: list[str] = []
+
+    if all([keep, queue]):
+        raise InvalidArgumentError("Cannot use both `--keep` and `--queue`.")
+
    if not any([exp_names, queue, all_commits, rev]):
        return removed
+
    celery_queue: LocalCeleryQueue = repo.experiments.celery_queue

    if queue:
@@ -43,6 +49,7 @@ def remove(  # noqa: C901, PLR0912

    exp_ref_list: list[ExpRefInfo] = []
    queue_entry_list: list[QueueEntry] = []
+
    if exp_names:
        results: dict[str, ExpRefAndQueueEntry] = (
            celery_queue.get_ref_and_entry_by_names(exp_names, git_remote)
@@ -68,6 +75,10 @@ def remove(  # noqa: C901, PLR0912
        exp_ref_list.extend(exp_ref_dict.values())
    elif all_commits:
        exp_ref_list.extend(exp_refs(repo.scm, git_remote))
+        removed.extend([ref.name for ref in exp_ref_list])
+
+    if keep:
+        exp_ref_list = list(set(exp_refs(repo.scm, git_remote)) - set(exp_ref_list))
        removed = [ref.name for ref in exp_ref_list]

    if exp_ref_list:
@@ -83,6 +94,7 @@ def remove(  # noqa: C901, PLR0912

        removed_refs = [str(r) for r in exp_ref_list]
        notify_refs_to_studio(repo, git_remote, removed=removed_refs)
+
    return removed


@@ -93,7 +93,7 @@ def run(  # noqa: C901, PLR0912

    for idx, sweep_overrides in enumerate(sweeps):
        if hydra_sweep and name_prefix is not None:
-            kwargs["name"] = f"{name_prefix}-{idx+1}"
+            kwargs["name"] = f"{name_prefix}-{idx + 1}"
        queue_entry = repo.experiments.queue_one(
            repo.experiments.celery_queue,
            targets=targets,
@@ -30,6 +30,7 @@ def show(
    num: int = 1,
    hide_queued: bool = False,
    hide_failed: bool = False,
+    hide_workspace: bool = False,
    sha_only: bool = False,
    **kwargs,
 ) -> list["ExpState"]:
@@ -42,6 +43,7 @@ def show(
        num=num,
        hide_queued=hide_queued,
        hide_failed=hide_failed,
+        hide_workspace=hide_workspace,
        sha_only=sha_only,
        **kwargs,
    )
@@ -10,6 +10,7 @@ from . import locked

 if TYPE_CHECKING:
    from dvc.output import Output
+    from dvc.repo import Repo
    from dvc.stage import Stage

 logger = logger.getChild(__name__)
@@ -24,7 +25,7 @@ def _make_index_onerror(onerror, rev):


 def _collect_indexes(  # noqa: PLR0913
-    repo,
+    repo: "Repo",
    targets=None,
    remote=None,
    all_branches=False,
@@ -40,6 +41,8 @@ def _collect_indexes(  # noqa: PLR0913
    onerror=None,
    push=False,
 ):
+    from .index import index_from_targets
+
    indexes = {}
    collection_exc = None

@@ -67,7 +70,8 @@ def _collect_indexes(  # noqa: PLR0913
        try:
            repo.config.merge(config)

-            idx = repo.index.targets_view(
+            idx = index_from_targets(
+                repo,
                targets,
                with_deps=with_deps,
                recursive=recursive,
@@ -94,7 +98,7 @@ def _collect_indexes(  # noqa: PLR0913

@locked
 def fetch(  # noqa: PLR0913
-    self,
+    self: "Repo",
    targets=None,
    jobs=None,
    remote=None,
@@ -17,6 +17,7 @@ from dvc.utils.objects import cached_property
 if TYPE_CHECKING:
    from networkx import DiGraph
    from pygtrie import Trie
+    from typing_extensions import Self

    from dvc.dependency import Dependency
    from dvc.fs.callbacks import Callback
@@ -66,14 +67,14 @@ def collect_files(

    is_local_fs = isinstance(fs, LocalFileSystem)

-    def is_ignored(path):
+    def is_ignored(path: str) -> bool:
        # apply only for the local fs
        return is_local_fs and scm.is_ignored(path)

-    def is_dvcfile_and_not_ignored(root, file):
+    def is_dvcfile_and_not_ignored(root: str, file: str) -> bool:
        return is_valid_filename(file) and not is_ignored(f"{root}{sep}{file}")

-    def is_out_or_ignored(root, directory):
+    def is_out_or_ignored(root: str, directory: str) -> bool:
        dir_path = f"{root}{sep}{directory}"
        # trailing slash needed to check if a directory is gitignored
        return dir_path in outs or is_ignored(f"{dir_path}{sep}")
@@ -83,6 +84,8 @@ def collect_files(
        walk_iter = log_walk(walk_iter)

    for root, dirs, files in walk_iter:
+        assert isinstance(dirs, list)
+        assert isinstance(files, list)
        dvcfile_filter = partial(is_dvcfile_and_not_ignored, root)
        for file in filter(dvcfile_filter, files):
            file_path = fs.join(root, file)
@@ -318,32 +321,9 @@ class Index:
        repo: "Repo",
        onerror: Optional[Callable[[str, Exception], None]] = None,
    ) -> "Index":
-        stages = []
-        metrics = {}
-        plots = {}
-        params = {}
-        artifacts = {}
-        datasets = {}
-        datasets_lock = {}
-
        onerror = onerror or repo.stage_collection_error_handler
-        for _, idx in collect_files(repo, onerror=onerror):
-            stages.extend(idx.stages)
-            metrics.update(idx._metrics)
-            plots.update(idx._plots)
-            params.update(idx._params)
-            artifacts.update(idx._artifacts)
-            datasets.update(idx._datasets)
-            datasets_lock.update(idx._datasets_lock)
-        return cls(
-            repo,
-            stages=stages,
-            metrics=metrics,
-            plots=plots,
-            params=params,
-            artifacts=artifacts,
-            datasets=datasets,
-            datasets_lock=datasets_lock,
+        return cls.from_indexes(
+            repo, (idx for _, idx in collect_files(repo, onerror=onerror))
        )

    @classmethod
@@ -364,7 +344,7 @@ class Index:
            else {},
        )

-    def update(self, stages: Iterable["Stage"]) -> "Index":
+    def update(self, stages: Iterable["Stage"]) -> "Self":
        stages = set(stages)
        # we remove existing stages with same hashes at first
        # and then re-add the new ones later.
@@ -379,6 +359,36 @@ class Index:
            datasets=self._datasets,
        )

+    @classmethod
+    def from_indexes(cls, repo, idxs: Iterable["Self"]) -> "Self":
+        stages = []
+        metrics = {}
+        plots = {}
+        params = {}
+        artifacts = {}
+        datasets = {}
+        datasets_lock = {}
+
+        for idx in idxs:
+            stages.extend(idx.stages)
+            metrics.update(idx._metrics)
+            plots.update(idx._plots)
+            params.update(idx._params)
+            artifacts.update(idx._artifacts)
+            datasets.update(idx._datasets)
+            datasets_lock.update(idx._datasets_lock)
+
+        return cls(
+            repo,
+            stages=stages,
+            metrics=metrics,
+            plots=plots,
+            params=params,
+            artifacts=artifacts,
+            datasets=datasets,
+            datasets_lock=datasets_lock,
+        )
+
    @cached_property
    def outs_trie(self) -> "Trie":
        from dvc.repo.trie import build_outs_trie
@@ -735,6 +745,10 @@ class IndexView:
        for stage in self.stages:
            yield from stage.deps

+    @property
+    def index(self) -> "Index":
+        return self._index
+
    @property
    def _filtered_outs(self) -> Iterator[tuple["Output", Optional[str]]]:
        for stage, filter_info in self._stage_infos:
@@ -927,3 +941,51 @@ def _get_entry_hash_name(
            return src_entry.hash_info.name

    return DEFAULT_ALGORITHM
+
+
+def index_from_targets(
+    repo: "Repo",
+    targets: Optional["TargetType"] = None,
+    stage_filter: Optional[Callable[["Stage"], bool]] = None,
+    outs_filter: Optional[Callable[["Output"], bool]] = None,
+    max_size: Optional[int] = None,
+    types: Optional[list[str]] = None,
+    with_deps: bool = False,
+    recursive: bool = False,
+    **kwargs: Any,
+) -> "IndexView":
+    from dvc.stage.exceptions import StageFileDoesNotExistError, StageNotFound
+    from dvc.utils import parse_target
+
+    index: Optional[Index] = None
+    if targets and all(targets) and not with_deps and not recursive:
+        indexes: list[Index] = []
+        try:
+            for target in targets:
+                if not target:
+                    continue
+                file, name = parse_target(target)
+                if file and not name:
+                    index = Index.from_file(repo, file)
+                else:
+                    stages = repo.stage.collect(target)
+                    index = Index(repo, stages=list(stages))
+                indexes.append(index)
+        except (StageFileDoesNotExistError, StageNotFound):
+            pass
+        else:
+            index = Index.from_indexes(repo, indexes)
+            targets = None
+
+    if index is None:
+        index = repo.index
+    return index.targets_view(
+        targets,
+        stage_filter=stage_filter,
+        outs_filter=outs_filter,
+        max_size=max_size,
+        types=types,
+        recursive=recursive,
+        with_deps=with_deps,
+        **kwargs,
+    )
@@ -63,7 +63,7 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False):  # noqa:

        remove(dvc_dir)

-    os.mkdir(dvc_dir)
+    os.makedirs(dvc_dir, exist_ok=True)

    config = Config.init(dvc_dir)

@@ -1,5 +1,6 @@
 from typing import TYPE_CHECKING

+from dvc import version_tuple
 from dvc.exceptions import DvcException

 if TYPE_CHECKING:
@@ -16,19 +17,19 @@ def pre_commit_install(scm: "Git") -> None:
    with modify_yaml(config_path) as config:
        entry = {
            "repo": "https://github.com/iterative/dvc",
-            "rev": "main",
+            "rev": ".".join(map(str, version_tuple[:3])),
            "hooks": [
                {
                    "id": "dvc-pre-commit",
                    "additional_dependencies": [".[all]"],
                    "language_version": "python3",
-                    "stages": ["commit"],
+                    "stages": ["pre-commit"],
                },
                {
                    "id": "dvc-pre-push",
                    "additional_dependencies": [".[all]"],
                    "language_version": "python3",
-                    "stages": ["push"],
+                    "stages": ["pre-push"],
                },
                {
                    "id": "dvc-post-checkout",
@@ -4,8 +4,44 @@ from typing import TYPE_CHECKING, Any, Optional, Union
 if TYPE_CHECKING:
    from dvc.fs.dvc import DVCFileSystem

+
+def _open_repo(
+    url: str,
+    rev: Optional[str] = None,
+    config: Union[dict[str, Any], str, None] = None,
+    remote: Optional[str] = None,
+    remote_config: Optional[dict] = None,
+):
+    from dvc.config import Config
+
    from . import Repo

+    if config and not isinstance(config, dict):
+        config_dict = Config.load_file(config)
+    else:
+        config_dict = None
+
+    return Repo.open(
+        url,
+        rev=rev,
+        subrepos=True,
+        uninitialized=True,
+        config=config_dict,
+        remote=remote,
+        remote_config=remote_config,
+    )
+
+
+def _adapt_info(info: dict[str, Any]) -> dict[str, Any]:
+    dvc_info = info.get("dvc_info", {})
+    return {
+        "isout": dvc_info.get("isout", False),
+        "isdir": info["type"] == "directory",
+        "isexec": info.get("isexec", False),
+        "size": info.get("size"),
+        "md5": dvc_info.get("md5") or dvc_info.get("md5-dos2unix"),
+    }
+

 def ls(
    url: str,
@@ -13,9 +49,10 @@ def ls(
    rev: Optional[str] = None,
    recursive: Optional[bool] = None,
    dvc_only: bool = False,
-    config: Union[None, dict[str, Any], str] = None,
+    config: Union[dict[str, Any], str, None] = None,
    remote: Optional[str] = None,
    remote_config: Optional[dict] = None,
+    maxdepth: Optional[int] = None,
 ):
    """Methods for getting files and outputs for the repo.

@@ -41,60 +78,60 @@ def ls(
            "isexec": bool,
        }
    """
-    from dvc.config import Config
-
-    from . import Repo
-
-    if config and not isinstance(config, dict):
-        config_dict = Config.load_file(config)
-    else:
-        config_dict = None
-
-    with Repo.open(
-        url,
-        rev=rev,
-        subrepos=True,
-        uninitialized=True,
-        config=config_dict,
-        remote=remote,
-        remote_config=remote_config,
-    ) as repo:
+    with _open_repo(url, rev, config, remote, remote_config) as repo:
        path = path or ""
+        fs: DVCFileSystem = repo.dvcfs
+        fs_path = fs.from_os_path(path)
+        return _ls(fs, fs_path, recursive, dvc_only, maxdepth)

-        ret = _ls(repo, path, recursive, dvc_only)

-        ret_list = []
-        for path, info in ret.items():
-            info["path"] = path
-            ret_list.append(info)
-        ret_list.sort(key=lambda f: f["path"])
-        return ret_list
+def ls_tree(
+    url: str,
+    path: Optional[str] = None,
+    rev: Optional[str] = None,
+    dvc_only: bool = False,
+    config: Union[dict[str, Any], str, None] = None,
+    remote: Optional[str] = None,
+    remote_config: Optional[dict] = None,
+    maxdepth: Optional[int] = None,
+):
+    with _open_repo(url, rev, config, remote, remote_config) as repo:
+        path = path or ""
+        fs: DVCFileSystem = repo.dvcfs
+        fs_path = fs.from_os_path(path)
+        return _ls_tree(
+            fs, fs_path, maxdepth=maxdepth, dvc_only=dvc_only, dvcfiles=True
+        )


 def _ls(
-    repo: "Repo",
+    fs: "DVCFileSystem",
    path: str,
    recursive: Optional[bool] = None,
    dvc_only: bool = False,
+    maxdepth: Optional[int] = None,
 ):
-    fs: DVCFileSystem = repo.dvcfs
-    fs_path = fs.from_os_path(path)
-
-    fs_path = fs.info(fs_path)["name"]
+    fs_path = fs.info(path)["name"]

    infos = {}
-    if fs.isfile(fs_path):
-        infos[os.path.basename(path)] = fs.info(fs_path)
+
+    # ignore maxdepth only if recursive is not set
+    maxdepth = maxdepth if recursive else None
+    if maxdepth == 0 or fs.isfile(fs_path):
+        infos[os.path.basename(path) or os.curdir] = fs.info(fs_path)
    else:
        for root, dirs, files in fs.walk(
-            fs_path, dvcfiles=True, dvc_only=dvc_only, detail=True
+            fs_path,
+            dvcfiles=True,
+            dvc_only=dvc_only,
+            detail=True,
+            maxdepth=maxdepth,
        ):
-            if not recursive:
-                files.update(dirs)
-
            parts = fs.relparts(root, fs_path)
            if parts == (".",):
                parts = ()
+            if not recursive or (maxdepth and len(parts) >= maxdepth - 1):
+                files.update(dirs)

            for name, entry in files.items():
                infos[os.path.join(*parts, name)] = entry
@@ -102,14 +139,44 @@ def _ls(
            if not recursive:
                break

-    ret = {}
-    for name, info in infos.items():
-        dvc_info = info.get("dvc_info", {})
-        ret[name] = {
-            "isout": dvc_info.get("isout", False),
-            "isdir": info["type"] == "directory",
-            "isexec": info.get("isexec", False),
-            "size": info.get("size"),
-        }
+    ret_list = []
+    for p, info in sorted(infos.items(), key=lambda x: x[0]):
+        _info = _adapt_info(info)
+        _info["path"] = p
+        ret_list.append(_info)
+    return ret_list

+
+def _ls_tree(fs, path, maxdepth=None, _info=None, **fs_kwargs):
+    info = _info or fs.info(path)
+    if _info is None:
+        # preserve the original path name
+        name = path
+        if not name:
+            name = os.curdir if fs.protocol == "local" else fs.root_marker
+        path = info["name"]
+    else:
+        name = path.rsplit(fs.sep, 1)[-1]
+
+    ret = {}
+    ls_info = _adapt_info(info)
+    ls_info["path"] = path
+
+    recurse = maxdepth is None or maxdepth > 0
+    if recurse and info["type"] == "directory":
+        try:
+            infos = fs.ls(path, detail=True, **fs_kwargs)
+        except FileNotFoundError:
+            # broken symlink?
+            infos = []
+
+        infos.sort(key=lambda f: f["name"])
+        maxdepth = maxdepth - 1 if maxdepth is not None else None
+        contents = {}
+        for info in infos:
+            d = _ls_tree(fs, info["name"], maxdepth=maxdepth, _info=info, **fs_kwargs)
+            contents.update(d)
+        ls_info["contents"] = contents
+
+    ret[name] = ls_info
    return ret
@@ -1,19 +1,30 @@
+from fsspec.implementations.local import LocalFileSystem as _LocalFileSystem
+
 from dvc.exceptions import URLMissingError
-from dvc.fs import parse_external_url
+from dvc.fs import LocalFileSystem, parse_external_url


-def ls_url(url, *, fs_config=None, recursive=False, config=None):
+def ls_url(url, *, fs_config=None, recursive=False, maxdepth=None, config=None):
    fs, fs_path = parse_external_url(url, fs_config=fs_config, config=config)
    try:
        info = fs.info(fs_path)
    except FileNotFoundError as exc:
        raise URLMissingError(url) from exc
-    if info["type"] != "directory":
+    if maxdepth == 0 or info["type"] != "directory":
        return [{"path": info["name"], "isdir": False}]

+    if isinstance(fs, LocalFileSystem):
+        # dvc's LocalFileSystem does not support maxdepth yet
+        walk = _LocalFileSystem().walk
+    else:
+        walk = fs.walk
+
    ret = []
-    for _, dirs, files in fs.walk(fs_path, detail=True):
-        if not recursive:
+    for root, dirs, files in walk(fs_path, detail=True, maxdepth=maxdepth):
+        parts = fs.relparts(root, fs_path)
+        if parts == (".",):
+            parts = ()
+        if not recursive or (maxdepth and len(parts) >= maxdepth - 1):
            files.update(dirs)

        for info in files.values():
@@ -99,7 +99,7 @@ def clean_repos():

 def _get_remote_config(url):
    try:
-        repo = Repo(url)
+        repo = Repo(url, uninitialized=True)
    except NotDvcRepoError:
        return {}

@@ -109,10 +109,16 @@ def _get_remote_config(url):
            # Fill the empty upstream entry with a new remote pointing to the
            # original repo's cache location.
            name = "auto-generated-upstream"
-            return {
-                "core": {"remote": name},
-                "remote": {name: {"url": repo.cache.local_cache_dir}},
-            }
+            try:
+                local_cache_dir = repo.cache.local_cache_dir
+            except AttributeError:
+                # if the `.dvc` dir is missing, we get an AttributeError
+                return {}
+            else:
+                return {
+                    "core": {"remote": name},
+                    "remote": {name: {"url": local_cache_dir}},
+                }

        # Use original remote to make sure that we are using correct url,
        # credential paths, etc if they are relative to the config location.
@@ -32,7 +32,7 @@ logger = logger.getChild(__name__)


 def onerror_collect(result: dict, exception: Exception, *args, **kwargs):
-    logger.debug("", exc_info=True)
+    logger.debug("", exc_info=True)  # noqa: LOG014
    result["error"] = exception


@@ -346,14 +346,14 @@ def _get_data_targets(definitions: dict):


 def infer_data_sources(plot_id, config=None):
-    y = config.get("y", None)
+    y = config.get("y", None) if config else None

    if isinstance(y, dict):
        sources = list(y.keys())
    else:
        sources = [plot_id]

-    x = config.get("x", None)
+    x = config.get("x", None) if config else None
    if isinstance(x, dict):
        sources.append(first(x.keys()))

@@ -507,7 +507,7 @@ def _collect_pipeline_files(repo, targets: list[str], props, onerror=None):
@error_handler
 def _collect_definitions(
    repo: "Repo",
-    targets=None,
+    targets: list[str],
    props: Optional[dict] = None,
    onerror: Optional[Callable] = None,
    **kwargs,
@@ -39,7 +39,7 @@ def pull(  # noqa: PLR0913
        run_cache=run_cache,
    )
    try:
-        stats = self.checkout(
+        result = self.checkout(
            targets=expanded_targets,
            with_deps=with_deps,
            force=force,
@@ -47,8 +47,10 @@ def pull(  # noqa: PLR0913
            allow_missing=allow_missing,
        )
    except CheckoutError as exc:
-        exc.stats["fetched"] = processed_files_count
+        # put fetched counts first
+        exc.result["stats"] = {"fetched": processed_files_count} | exc.result["stats"]
        raise
-
-    stats["fetched"] = processed_files_count
-    return stats
+    else:
+        # put fetched counts first
+        result["stats"] = {"fetched": processed_files_count} | result["stats"]
+    return result
@@ -152,10 +152,12 @@ def push(  # noqa: PLR0913
    finally:
        ws_idx = indexes.get("workspace")
        if ws_idx is not None:
+            from dvc.repo.index import IndexView
+
+            _index = ws_idx.index if isinstance(ws_idx, IndexView) else ws_idx
            _update_meta(
-                self.index,
+                _index,
                targets=glob_targets(targets, glob=glob),
-                remote=remote,
                with_deps=with_deps,
                recursive=recursive,
            )
@@ -41,8 +41,10 @@ class SCMContext:
    def add(self, paths: Union[str, Iterable[str]]) -> None:
        from scmrepo.exceptions import UnsupportedIndexFormat

+        from dvc.scm import add_no_submodules
+
        try:
-            return self.scm.add(paths)
+            add_no_submodules(self.scm, paths)
        except UnsupportedIndexFormat:
            link = "https://github.com/iterative/dvc/issues/610"
            add_cmd = self._make_git_add_cmd([relpath(path) for path in paths])
@@ -130,7 +130,7 @@ def status(  # noqa: PLR0913
            remote=remote,
            all_tags=all_tags,
            all_commits=all_commits,
-            recursive=True,
+            recursive=recursive,
        )

    ignored = list(
@@ -152,7 +152,7 @@ def _merge_push_meta(  # noqa: C901
    else:
        if entry.hash_info:
            out.hash_info = entry.hash_info
-        if out.meta.version_id is None:
+        if out.meta is None or out.meta.version_id is None:
            out.meta = entry.meta
    if out.meta:
        out.meta.remote = remote
@@ -181,7 +181,7 @@ def _update_worktree_out(
 ):
    from dvc_data.index import build

-    remote_name = out.remote or out.meta.remote
+    remote_name = out.remote or (out.meta.remote if out.meta is not None else None)
    if not remote_name:
        logger.warning("Could not update '%s', it was never pushed to a remote", out)
        return
@@ -212,11 +212,12 @@ def _update_worktree_out(
        logger.warning("Could not update '%s', directory is empty in the remote", out)
        return

-    _fetch_out_changes(out, local_index, remote_index, remote)
+    _fetch_out_changes(repo, out, local_index, remote_index, remote)
    _update_out_meta(repo, out, local_index, remote_index, remote)


 def _fetch_out_changes(
+    repo: "Repo",
    out: "Output",
    local_index: Union["DataIndex", "DataIndexView"],
    remote_index: Union["DataIndex", "DataIndexView"],
@@ -242,7 +243,7 @@ def _fetch_out_changes(
        cb.set_size(total)
        apply(
            diff,
-            out.repo.root_dir,
+            repo.root_dir,
            out.fs,
            update_meta=False,
            storage="data",
@@ -1,17 +1,18 @@
 """Manages source control systems (e.g. Git)."""

 import os
-from collections.abc import Iterator, Mapping
+from collections.abc import Iterable, Iterator, Mapping
 from contextlib import contextmanager
 from functools import partial
 from typing import TYPE_CHECKING, Literal, Optional, Union, overload

 from funcy import group_by
-from scmrepo.base import Base  # noqa: F401
+from scmrepo.base import Base  # noqa: TC002
 from scmrepo.git import Git
 from scmrepo.noscm import NoSCM

 from dvc.exceptions import DvcException
+from dvc.log import logger
 from dvc.progress import Tqdm

 if TYPE_CHECKING:
@@ -19,6 +20,8 @@ if TYPE_CHECKING:

    from dvc.fs import FileSystem

+logger = logger.getChild(__name__)
+

 class SCMError(DvcException):
    """Base class for source control management errors."""
@@ -283,3 +286,35 @@ def lfs_prefetch(fs: "FileSystem", paths: list[str]):
            include=[(path if path.startswith("/") else f"/{path}") for path in paths],
            progress=pbar.update_git,
        )
+
+
+def add_no_submodules(
+    scm: "Base",
+    paths: Union[str, Iterable[str]],
+    **kwargs,
+) -> None:
+    """Stage paths to Git, excluding those inside submodules."""
+
+    if isinstance(paths, str):
+        paths = [paths]
+
+    submodule_roots = {os.path.join(scm.root_dir, sub) for sub in scm.list_submodules()}
+
+    repo_paths: list[str] = []
+    skipped_paths: list[str] = []
+
+    for p in paths:
+        abs_path = os.path.abspath(p)
+        if any(
+            abs_path == root or abs_path.startswith(root + os.sep)
+            for root in submodule_roots
+        ):
+            skipped_paths.append(p)
+        else:
+            repo_paths.append(p)
+
+    if skipped_paths:
+        msg = "Skipping staging for path(s) inside submodules: %s"
+        logger.debug(msg, ", ".join(skipped_paths))
+
+    scm.add(repo_paths, **kwargs)
@@ -83,7 +83,7 @@ class RawData:
 def create_stage(cls: type[_T], repo, path, **kwargs) -> _T:
    from dvc.dvcfile import check_dvcfile_path

-    wdir = os.path.abspath(kwargs.get("wdir", None) or os.curdir)
+    wdir = os.path.abspath(kwargs.get("wdir") or os.curdir)
    path = os.path.abspath(path)

    check_dvcfile_path(repo, path)
@@ -435,7 +435,7 @@ class Stage(params.StageParams):
        elif not self.changed(True, upstream) and pull:
            try:
                logger.info("Pulling data for %s", self)
-                self.repo.pull(self.addressing, jobs=kwargs.get("jobs", None))
+                self.repo.pull(self.addressing, jobs=kwargs.get("jobs"))
                self.checkout()
                return None
            except CheckoutError:
@@ -509,34 +509,20 @@ class Stage(params.StageParams):
                if not allow_missing:
                    raise

-    def get_versioned_outs(self) -> dict[str, "Output"]:
-        from .exceptions import StageFileDoesNotExistError, StageNotFound
-
-        try:
-            old = self.reload()
-        except (StageFileDoesNotExistError, StageNotFound):
-            return {}
-
-        return {
-            out.def_path: out
-            for out in old.outs
-            if out.files is not None
-            or (out.meta is not None and out.meta.version_id is not None)
-        }
-
    def save_outs(self, allow_missing: bool = False):
        from dvc.output import OutputDoesNotExistError

-        old_versioned_outs = self.get_versioned_outs()
        for out in self.outs:
+            # old state just before saving so to merge them later
+            old_state = out._get_versioned_meta()
            try:
                out.save()
            except OutputDoesNotExistError:
                if not allow_missing:
                    raise

-            if old_out := old_versioned_outs.get(out.def_path):
-                out.merge_version_meta(old_out)
+            if old_state:
+                out.merge_version_meta(*old_state)

    def ignore_outs(self) -> None:
        for out in self.outs:
@@ -579,8 +565,9 @@ class Stage(params.StageParams):
        from dvc.output import OutputDoesNotExistError

        link_failures = []
-        old_versioned_outs = self.get_versioned_outs()
        for out in self.filter_outs(filter_info):
+            # old state just before saving so to merge them later
+            old_state = out._get_versioned_meta()
            try:
                out.add(filter_info, **kwargs)
            except (FileNotFoundError, OutputDoesNotExistError):
@@ -589,8 +576,8 @@ class Stage(params.StageParams):
            except CacheLinkError:
                link_failures.append(filter_info or out.fs_path)

-            if old_out := old_versioned_outs.get(out.def_path):
-                out.merge_version_meta(old_out)
+            if old_state:
+                out.merge_version_meta(*old_state)

        if link_failures:
            raise CacheLinkError(link_failures)
@@ -609,7 +596,7 @@ class Stage(params.StageParams):
            self.remove_outs(ignore_remove=False, force=False)

        if (self.is_import and not self.frozen) or self.is_partial_import:
-            self._sync_import(dry, force, kwargs.get("jobs", None), no_download)
+            self._sync_import(dry, force, kwargs.get("jobs"), no_download)
        elif not self.frozen and self.cmd:
            self._run_stage(dry, force, **kwargs)
        elif not dry:
@@ -168,6 +168,7 @@ def to_single_stage_lockfile(stage: "Stage", **kwargs) -> dict:
                    for f in _serialize_tree_obj_to_files(obj)
                ]
        else:
+            assert item.meta is not None
            meta_d = item.meta.to_dict()
            meta_d.pop("isdir", None)
            ret.update(_serialize_hi_to_dict(item.hash_info))
@@ -32,12 +32,13 @@ class TestAPI:
        "fs_kwargs",
        [
            {},
-            {"url": "{path}"},
-            {"url": "{path}", "rev": "{default_branch}"},
-            {"url": "file://{posixpath}"},
-            {"url": "file://{posixpath}", "rev": "{default_branch}"},
+            {"repo": "{path}"},
+            {"repo": "{path}", "rev": "{default_branch}"},
+            {"repo": "file://{posixpath}"},
+            {"repo": "file://{posixpath}", "rev": "{default_branch}"},
+            {"url": "{path}"},  # test for backward compatibility
        ],
-        ids=["current", "local", "local_rev", "git", "git_rev"],
+        ids=["current", "local", "local_rev", "git", "git_rev", "local_url"],
    )
    def test_filesystem(
        self,
@@ -59,6 +60,8 @@ class TestAPI:
        if clear_cache:
            remove(dvc.cache.repo.path)

+        if repo := fs_kwargs.get("repo"):
+            fs_kwargs["repo"] = repo.format(path=tmp_dir, posixpath=tmp_dir.as_posix())
        if url := fs_kwargs.get("url"):
            fs_kwargs["url"] = url.format(path=tmp_dir, posixpath=tmp_dir.as_posix())
        if rev := fs_kwargs.get("rev"):
@@ -4,12 +4,15 @@ from shutil import rmtree
 def test_data_status(bench_dvc, tmp_dir, scm, dvc, make_dataset):
    args = ("data", "status")
    dataset = make_dataset(cache=True, files=True, dvcfile=True, commit=False)
+    scm.ignore(dataset)
    rmtree(dvc.tmp_dir)

    bench_dvc(*args, name="new")
    bench_dvc(*args, name="noop")

-    tmp_dir.scm_add(dataset.with_suffix(".dvc").name, commit="add dataset")
+    tmp_dir.scm_add(
+        [dataset.with_suffix(".dvc").name, ".gitignore"], commit="add dataset"
+    )

    (dataset / "new").write_text("new")
    bench_dvc(*args, name="changed")
@@ -26,12 +29,15 @@ def test_data_status_all_flags(bench_dvc, tmp_dir, scm, dvc, make_dataset):
        "--json",
    )
    dataset = make_dataset(cache=True, files=True, dvcfile=True, commit=False)
+    scm.ignore(dataset)
    rmtree(dvc.tmp_dir)

    bench_dvc(*args, name="new")
    bench_dvc(*args, name="noop")

-    tmp_dir.scm_add(dataset.with_suffix(".dvc").name, commit="add dataset")
+    tmp_dir.scm_add(
+        [dataset.with_suffix(".dvc").name, ".gitignore"], commit="add dataset"
+    )

    (dataset / "new").write_text("new")
    bench_dvc(*args, name="changed")
--- a/Show More
+++ b/Show More