[pre-commit.ci] pre-commit autoupdate (#10836)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/ISSUE_TEMPLATE/epic_story.md

@@ -1,60 +0,0 @@
----
-name: Epic/Story Issue Template
-about: Template for "top" level issues - Epics (>2 weeks) / Stories (<2 weeks)
-title: 'Epic: New Feature'
-labels: epic
-assignees:
-
----
-
-## Summary / Background
-What do you want to achieve, why? business context
-...
-
-## Scope
-
-What will be impacted and what won't be?
-What needs implementation and what is invariant?
-e.g.
-- user should be able to run workflow X from UI
-- enable workflow Y, Z from CLI
-
-## Assumptions
-Product / UX assumptions as well as technical assumptions / limitations
-e.g.
-* Support only Python Runtime
-* Focus on DVC experiments only
-* Deployment environments don't change often and can be picked up from shared configuration
-
-## Open Questions
-e.g.
-- How should access control work for shared artifacts (workflow X)
-- Python runtime assumption - is it really valid? in light of <...>
-
-## Blockers / Dependencies
-List issues or other conditions / blockers
-
-## General Approach
-Invocation example:
-```shell
-$ mapper-run task.tar.gz --ray-cluster <ip>:<port>
-```
-
-## Steps
-
-### Must have (p1)
-- [ ] subissue2
-- [ ] step 2
- - info
- - info
-
-### Optional / followup (p2)
-- [ ] ⌛ step 3 wip
-- [ ] step 4
-
-## Timelines
-
-Put your estimations here. Update once certainty changes
-- end of week (Feb 3) for prototype with workflows X, Y
-- Feb 15 - MVP in production
-- Low priority followups can be done later

.github/workflows/benchmarks.yaml

@@ -5,6 +5,10 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+env:
+  FORCE_COLOR: "1"
+  PY_COLORS: "1"
+
 jobs:
   bench:
     name: run benchmarks
@@ -12,7 +16,29 @@ jobs:
     steps:
       - uses: actions/setup-python@v5
         with:
-            python-version: "3.11"
-      - uses: iterative/dvc-bench@main
+            python-version: "3.13"
+
+      - uses: actions/checkout@v4
         with:
-            pytest_options: "-k 'test_init or test_help'"
+          ref: ${{ github.event.pull_request.base.sha }}
+          fetch-depth: 0
+
+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: pyproject.toml
+
+      - run: uv pip install '.[tests]' --system
+      - name: run benchmarks on base branch
+        run: pytest --benchmark-autosave dvc/testing/benchmarks/ -k 'test_init or test_help'
+
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          clean: false
+      - run: uv pip install '.[tests]' --system
+      - name: run benchmarks for PR
+        run: >
+          pytest --benchmark-compare --benchmark-compare-fail=min:5%
+          --benchmark-group-by name
+          dvc/testing/benchmarks/ -k 'test_init or test_help'

.github/workflows/build.yaml

@@ -25,24 +25,27 @@ jobs:
 
     - uses: actions/setup-python@v5
       with:
-        python-version: 3.12
+        python-version: 3.13
 
-    - run: python -m pip install --upgrade pip setuptools_scm build twine
+    - uses: astral-sh/setup-uv@v6
 
-    - name: Force version for Test PyPI upload
-      if: ${{ !startsWith(github.ref, 'refs/tags') }}
-      run: echo version=$(python -m setuptools_scm | awk -F+ '{print $1}' | tail -1) >> $GITHUB_ENV
+    - name: Omit local version for Test PyPI upload
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+      run: echo SETUPTOOLS_SCM_OVERRIDES_FOR_DVC='{local_scheme="no-local-version"}' >> $GITHUB_ENV
 
-    - run: echo 'PKG = "pip"'>dvc/_build.py
-    - run: python -m build
-      env:
-        SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DVC: ${{ env.version }}
-    - run: twine check --strict dist/*
+    - name: Build Python Package
+      run: |
+        echo 'PKG = "pip"'>dvc/_build.py
+        uv build
+
+    - name: Check dist
+      run: uv tool run twine check --strict dist/*
 
     - uses: actions/upload-artifact@v4
       with:
         name: Packages
         path: dist/
+        if-no-files-found: error
 
   test-pypi-publish:
     name: Publish dev package to test.pypi.org
@@ -52,13 +55,13 @@ jobs:
 
     environment:
       name: test-pypi
-      url: https://test.pypi.org/p/dvc
+      url: https://test.pypi.org/p/dvc/${{ github.event.release.tag_name }}
 
     permissions:
       id-token: write
 
     steps:
-    - uses: actions/download-artifact@v4
+    - uses: actions/download-artifact@v5
       with:
         name: Packages
         path: dist
@@ -77,13 +80,13 @@ jobs:
 
     environment:
       name: pypi
-      url: https://pypi.org/p/dvc
+      url: https://pypi.org/p/dvc/${{ github.event.release.tag_name }}
 
     permissions:
       id-token: write
 
     steps:
-    - uses: actions/download-artifact@v4
+    - uses: actions/download-artifact@v5
       with:
         name: Packages
         path: dist

.github/workflows/plugin_tests.yaml

@@ -18,8 +18,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04]
-        pyv: ["3.10"]
+        os: [ubuntu-latest]
+        pyv: ["3.13"]
         plugin: ["dvc-s3"]
 
     steps:
@@ -37,16 +37,17 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.pyv }}
-        cache: pip
-        cache-dependency-path: |
-          dvc/pyproject.toml
-          ${{ matrix.plugin }}/setup.cfg
+
+    - uses: astral-sh/setup-uv@v6
+      with:
+        enable-cache: true
+        cache-dependency-glob: "**/pyproject.toml"
+        cache-suffix: ${{ matrix.pyv }}
 
     - name: Install plugin + DVC@PR
       run: |
-        pip install --upgrade pip wheel
-        pip install "./dvc[testing]"
-        pip install -e "./${{ matrix.plugin }}[tests]"
+        uv pip install "./dvc[testing]" --system
+        uv pip install -e "./${{ matrix.plugin }}[tests]" --system
 
     - name: Run plugin tests
       timeout-minutes: 15

.github/workflows/tests.yaml

@@ -16,15 +16,107 @@ permissions:
   contents: read
 
 jobs:
+  lint:
+    timeout-minutes: 10
+    runs-on: ${{ matrix.os }}
+    env:
+       DVC_NO_ANALYTICS: true
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.13'
+    - uses: astral-sh/setup-uv@v6
+      with:
+        enable-cache: true
+        cache-dependency-glob: pyproject.toml
+    - run: uv pip install -e ".[dev]" --system
+      env:
+        UV_LINK_MODE: ${{ runner.os == 'Windows' && 'symlink' || 'hardlink' }}
+
+    - name: Cache mypy
+      uses: actions/cache@v4
+      with:
+        path: .mypy_cache
+        key: mypy-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}
+    - uses: pre-commit/[email protected]
+
   tests:
-    timeout-minutes: 50
+    timeout-minutes: 20
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, windows-latest, macos-latest]
-        pyv: ["3.9", "3.10", "3.11", "3.12"]
-
+        os: [ubuntu-latest, macos-latest]
+        pyv: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        include:
+        - os: windows-latest
+          pyv: "3.9"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
+        - os: windows-latest
+          pyv: "3.9"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
+        - os: windows-latest
+          pyv: "3.9"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
+        - os: windows-latest
+          pyv: "3.9"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
+        - os: windows-latest
+          pyv: "3.10"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
+        - os: windows-latest
+          pyv: "3.10"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
+        - os: windows-latest
+          pyv: "3.10"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
+        - os: windows-latest
+          pyv: "3.10"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
+        - os: windows-latest
+          pyv: "3.11"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
+        - os: windows-latest
+          pyv: "3.11"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
+        - os: windows-latest
+          pyv: "3.11"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
+        - os: windows-latest
+          pyv: "3.11"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
+        - os: windows-latest
+          pyv: "3.12"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
+        - os: windows-latest
+          pyv: "3.12"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
+        - os: windows-latest
+          pyv: "3.12"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
+        - os: windows-latest
+          pyv: "3.12"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
+        - os: windows-latest
+          pyv: "3.13"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
+        - os: windows-latest
+          pyv: "3.13"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
+        - os: windows-latest
+          pyv: "3.13"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
+        - os: windows-latest
+          pyv: "3.13"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
     steps:
     - uses: actions/checkout@v4
       with:
@@ -33,25 +125,25 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.pyv }}
-        cache: pip
-        cache-dependency-path: pyproject.toml
-    - name: install
-      run: |
-        pip install --upgrade pip wheel
-        pip install -e ".[dev]"
-    - uses: pre-commit/[email protected]
+    - uses: astral-sh/setup-uv@v6
+      with:
+        enable-cache: true
+        cache-dependency-glob: pyproject.toml
+        cache-suffix: ${{ matrix.pyv }}
+    - run: uv pip install -e ".[dev]" pytest-split --system
+      env:
+        UV_LINK_MODE: ${{ runner.os == 'Windows' && 'symlink' || 'hardlink' }}
+
     - name: run tests
       timeout-minutes: 40
       env:
+        COVERAGE_CORE: sysmon
         PYTHONUTF8: 1
       run: >
-        pytest -n=logical --timeout=300 --durations=100
-        --cov --cov-report=xml --cov-report=term
+        pytest ${{ matrix.pytestargs }} -n=logical --dist=worksteal --timeout=300 --durations=0
+        --cov --cov-report=xml --cov-report=term  --durations-path=./.github/.test_durations
     - name: upload coverage report
-      uses: codecov/codecov-action@v3
-      with:
-        file: ./coverage.xml
-        fail_ci_if_error: false
+      uses: codecov/codecov-action@v5
   check:
     if: always()
     needs: [tests]
@@ -66,7 +158,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Slack Notification
-      uses: rtCamp/[email protected].0
+      uses: rtCamp/[email protected].3
       env:
         SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         SLACK_COLOR: ${{ job.status }}

.gitignore

@@ -43,7 +43,7 @@ scripts/ci/gcp-creds.json
 .gcp-creds.json
 
 *~
-/dvc/_dvc_version.py
+/dvc/_version.py
 
 .mypy_cache/
 .pytest_cache/

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ ci:
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v6.0.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -22,13 +22,13 @@ repos:
       - id: sort-simple-yaml
       - id: trailing-whitespace
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.5.6'
+    rev: 'v0.12.8'
     hooks:
-      - id: ruff
+      - id: ruff-check
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.3.0
+    rev: v2.4.1
     hooks:
       - id: codespell
         additional_dependencies: ["tomli"]
@@ -50,7 +50,7 @@ repos:
         language: system
         name: DVC pre-commit
         stages:
-          - commit
+          - pre-commit
         verbose: true
         require_serial: true
       - args:
@@ -61,7 +61,7 @@ repos:
         language: system
         name: DVC pre-push
         stages:
-          - push
+          - pre-push
         require_serial: true
       - always_run: true
         args:

.pre-commit-hooks.yaml

@@ -8,7 +8,7 @@
   name: DVC pre-commit
   require_serial: true
   stages:
-  - commit
+  - pre-commit
   verbose: true
 - args:
   - git-hook
@@ -20,7 +20,7 @@
   name: DVC pre-push
   require_serial: true
   stages:
-  - push
+  - pre-push
 - always_run: true
   args:
   - git-hook
@@ -29,7 +29,7 @@
   id: dvc-post-checkout
   language: python
   language_version: python3
-  minimum_pre_commit_version: 2.2.0
+  minimum_pre_commit_version: 3.2.0
   name: DVC post-checkout
   require_serial: true
   stages:

README.rst

@@ -1,6 +1,4 @@
-🚀 Check out our new product `DataChain <https://github.com/iterative/datachain>`_ (and give it a ⭐!) if you need to version and process a large number of files. Contact us at [email protected] to discuss commercial solutions and support for AI reproducibility and data management scenarios.
-
---------------------------
+|Banner|
 
 `Website <https://dvc.org>`_
 • `Docs <https://dvc.org/doc>`_
@@ -215,7 +213,7 @@ Community and Support
 * `Forum <https://discuss.dvc.org/>`_
 * `Discord Chat <https://dvc.org/chat>`_
 * `Email <mailto:[email protected]>`_
-* `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
+* `Mailing List <https://dvc.org/community#subscribe>`_
 
 Copyright
 =========
@@ -242,11 +240,11 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
 .. |VS Code Extension Overview| image:: https://raw.githubusercontent.com/iterative/vscode-dvc/main/extension/docs/overview.gif
    :alt: DVC Extension for VS Code
 
-.. |CI| image:: https://github.com/iterative/dvc/workflows/Tests/badge.svg?branch=main
-   :target: https://github.com/iterative/dvc/actions
+.. |CI| image:: https://github.com/iterative/dvc/actions/workflows/tests.yaml/badge.svg
+   :target: https://github.com/iterative/dvc/actions/workflows/tests.yaml
    :alt: GHA Tests
 
-.. |Maintainability| image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
+.. |Maintainability| image:: https://img.shields.io/codeclimate/maintainability/iterative/dvc
    :target: https://codeclimate.com/github/iterative/dvc
    :alt: Code Climate
 
@@ -270,7 +268,7 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
    :target: https://formulae.brew.sh/formula/dvc
    :alt: Homebrew
 
-.. |Conda| image:: https://img.shields.io/conda/v/conda-forge/dvc.svg?label=conda&logo=conda-forge
+.. |Conda| image:: https://anaconda.org/conda-forge/dvc/badges/version.svg
    :target: https://anaconda.org/conda-forge/dvc
    :alt: Conda-forge
 

dvc/analytics.py

@@ -1,10 +1,14 @@
 import json
 import os
+from typing import TYPE_CHECKING, Optional
 
 from dvc.log import logger
 
 from .env import DVC_ANALYTICS_ENDPOINT, DVC_NO_ANALYTICS
 
+if TYPE_CHECKING:
+    from dvc.scm import Base
+
 logger = logger.getChild(__name__)
 
 
@@ -89,20 +93,62 @@ def send(path):
     os.remove(path)
 
 
-def _scm_in_use():
-    from dvc.exceptions import NotDvcRepoError
-    from dvc.repo import Repo
-    from dvc.scm import NoSCM
+def _git_remote_url(scm: Optional["Base"]) -> Optional[str]:
+    from dvc.scm import Git
 
-    from .scm import SCM, SCMError
+    if not isinstance(scm, Git):
+        return None
+
+    from dulwich.porcelain import get_remote_repo
+
+    dulwich_repo = scm.dulwich.repo
+    try:
+        _remote, url = get_remote_repo(dulwich_repo)
+    except IndexError:
+        # IndexError happens when the head is detached
+        _remote, url = get_remote_repo(dulwich_repo, b"origin")
+    # Dulwich returns (None, "origin") if no remote set
+    if (_remote, url) == (None, "origin"):
+        return None
+    return url
+
+
+def _scm_in_use(scm: Optional["Base"]) -> Optional[str]:
+    return type(scm).__name__ if scm else None
+
+
+def _parse_git_remote_path(remote_url: str) -> str:
+    from urllib.parse import urlparse
+
+    from scmrepo.urls import is_scp_style_url
+
+    parsed = urlparse(remote_url)
+    # Windows Path also gets parsed with a drive letter as scheme
+    # https://github.com/python/cpython/issues/86381
+    if parsed.scheme and parsed.scheme in ("http", "https", "git", "ssh"):
+        return parsed.path.strip("/")
+
+    if is_scp_style_url(remote_url):
+        # handle scp-style URL
+        parts = remote_url.split(":", 1)
+        if len(parts) == 2:
+            _, path = parts
+            return path.rstrip("/")
+    return remote_url
+
+
+def _git_remote_path_hash(scm: Optional["Base"]) -> Optional[str]:
+    """Return a hash of the git remote path."""
+    import hashlib
 
     try:
-        scm = SCM(root_dir=Repo.find_root())
-        return type(scm).__name__
-    except SCMError:
-        return NoSCM.__name__
-    except NotDvcRepoError:
-        pass
+        if remote_url := _git_remote_url(scm):
+            path = _parse_git_remote_path(remote_url)
+            h = hashlib.md5(path.encode("utf-8"), usedforsecurity=False)  # for FIPS
+            return h.hexdigest()
+    except Exception:  # noqa: BLE001
+        logger.debug("Failed to get git remote path", exc_info=True)
+    return None
 
 
 def _runtime_info():
@@ -112,6 +158,8 @@ def _runtime_info():
     from iterative_telemetry import _generate_ci_id, find_or_create_user_id
 
     from dvc import __version__
+    from dvc.info import _get_remotes
+    from dvc.repo import Repo
     from dvc.utils import is_binary
 
     ci_id = _generate_ci_id()
@@ -120,13 +168,24 @@ def _runtime_info():
     else:
         group_id, user_id = None, find_or_create_user_id()
 
+    scm = None
+    remotes = None
+    try:
+        repo = Repo()
+        scm = repo.scm
+        remotes = _get_remotes(repo.config)
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("failed to open repo: %s", exc)
+
     return {
         "dvc_version": __version__,
         "is_binary": is_binary(),
-        "scm_class": _scm_in_use(),
+        "scm_class": _scm_in_use(scm),
         "system_info": _system_info(),
         "user_id": user_id,
         "group_id": group_id,
+        "remotes": remotes,
+        "git_remote_hash": _git_remote_path_hash(scm),
     }
 
 

dvc/api/__init__.py

@@ -1,7 +1,7 @@
 from dvc.fs.dvc import _DVCFileSystem as DVCFileSystem
 
 from .artifacts import artifacts_show
-from .data import get_url, open, read
+from .data import get_url, open, read  # noqa: A004
 from .dataset import get as get_dataset
 from .experiments import exp_save, exp_show
 from .scm import all_branches, all_commits, all_tags

dvc/api/dataset.py

@@ -1,8 +1,8 @@
 from typing import Literal, TypedDict, Union
 
 
-class DVCXDataset(TypedDict):
-    type: Literal["dvcx"]
+class DatachainDataset(TypedDict):
+    type: Literal["dc"]
     name: str
     version: int
 
@@ -20,7 +20,7 @@ class URLDataset(TypedDict):
     path: str
 
 
-def get(name: str) -> Union[DVCXDataset, DVCDataset, URLDataset]:
+def get(name: str) -> Union[DatachainDataset, DVCDataset, URLDataset]:
     from difflib import get_close_matches
 
     from dvc.fs import get_cloud_fs
@@ -46,9 +46,9 @@ def get(name: str) -> Union[DVCXDataset, DVCDataset, URLDataset]:
             path=dataset.lock.path,
             sha=dataset.lock.rev_lock,
         )
-    if dataset.type == "dvcx":
-        return DVCXDataset(
-            type="dvcx", name=dataset.name_version[0], version=dataset.lock.version
+    if dataset.type == "dc":
+        return DatachainDataset(
+            type="dc", name=dataset.name_version[0], version=dataset.lock.version
         )
     if dataset.type == "url":
         fs_cls, _, path = get_cloud_fs(repo.config, url=dataset.lock.url)

dvc/api/show.py

@@ -27,8 +27,7 @@ def _postprocess(results):
             }
             processed[rev] = processed[rev] | to_merge
 
-    if "workspace" in processed:
-        del processed["workspace"]
+    processed.pop("workspace", None)
 
     return processed
 

dvc/cachemgr.py

@@ -86,7 +86,7 @@ class CacheManager:
         return FileStorage(
             key=(),
             fs=self.local.fs,
-            path=self.local.fs.join(self.default_local_cache_dir, self.FS_DIR),
+            path=self.local.fs.join(self.local_cache_dir, self.FS_DIR),
         )
 
     def _init_odb(self, schemes):

dvc/cli/__init__.py

@@ -157,6 +157,8 @@ def main(argv=None):  # noqa: C901, PLR0912, PLR0915
     Returns:
         int: command's return code.
     """
+    from contextlib import ExitStack
+
     from dvc._debug import debugtools
     from dvc.config import ConfigError
     from dvc.exceptions import DvcException, NotDvcRepoError
@@ -173,12 +175,11 @@ def main(argv=None):  # noqa: C901, PLR0912, PLR0915
         logging.disable(logging.INFO)
 
     args = None
-
-    outer_log_level = logger.level
-    level = None
+    stack = ExitStack()
     try:
         args = parse_args(argv)
 
+        level = None
         if args.quiet:
             level = logging.CRITICAL
         elif args.verbose == 1:
@@ -187,7 +188,7 @@ def main(argv=None):  # noqa: C901, PLR0912, PLR0915
             level = logging.TRACE  # type: ignore[attr-defined]
 
         if level is not None:
-            set_loggers_level(level)
+            stack.enter_context(set_loggers_level(level))
 
         if level and level <= logging.DEBUG:
             from platform import platform, python_implementation, python_version
@@ -247,7 +248,7 @@ def main(argv=None):  # noqa: C901, PLR0912, PLR0915
 
         return ret
     finally:
-        logger.setLevel(outer_log_level)
+        stack.close()
 
         from dvc.repo.open_repo import clean_repos
 

dvc/cli/command.py

@@ -18,7 +18,10 @@ class CmdBase(ABC):
 
         os.chdir(args.cd)
 
-        self.repo: Repo = Repo(uninitialized=self.UNINITIALIZED)
+        self.repo: Repo = Repo(
+            uninitialized=self.UNINITIALIZED,
+            _wait_for_lock=args.wait_for_lock,
+        )
         self.config: Config = self.repo.config
         self.args = args
 

dvc/cli/parser.py

@@ -200,6 +200,14 @@ def get_main_parser():
         type=str,
     )
 
+    parser.add_argument(
+        "--wait-for-lock",
+        action="store_true",
+        default=False,
+        help="Wait for the lock if it is already held by another process, instead of"
+        " failing immediately.",
+    )
+
     # Sub commands
     subparsers = parser.add_subparsers(
         title="Available Commands",

dvc/cli/utils.py

@@ -40,7 +40,7 @@ def hide_subparsers_from_help(subparsers):
     # from the 'positional arguments' choices list
     # see: https://bugs.python.org/issue22848
     # Need to set `add_help=False`, but avoid setting `help`
-    # (not even to `argparse.SUPPPRESS`).
+    # (not even to `argparse.SUPPRESS`).
     # NOTE: The argument is the parent subparser, not the subcommand parser.
     cmds = [cmd for cmd, parser in subparsers.choices.items() if parser.add_help]
     subparsers.metavar = "{{{}}}".format(",".join(cmds))

dvc/commands/dag.py

@@ -17,7 +17,7 @@ def _show_ascii(graph: "DiGraph"):
 
     ret = []
     for pipeline in pipelines:
-        ret.append(draw(pipeline.nodes, pipeline.edges))
+        ret.append(draw(pipeline.nodes, pipeline.edges))  # noqa: PERF401
 
     return "\n".join(ret)
 

dvc/commands/data.py

@@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, ClassVar
 
 from funcy import chunks, compact, log_durations
 
-from dvc.cli import formatter
+from dvc.cli import completion, formatter
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
 from dvc.log import logger
@@ -108,6 +108,7 @@ class CmdDataStatus(CmdBase):
     def run(self) -> int:
         with log_durations(logger.trace, "in data_status"):
             status = self.repo.data_status(
+                targets=self.args.targets,
                 granular=self.args.granular,
                 untracked_files=self.args.untracked_files,
                 not_in_remote=self.args.not_in_remote,
@@ -147,6 +148,14 @@ def add_parser(subparsers, parent_parser):
         formatter_class=formatter.RawDescriptionHelpFormatter,
         help=DATA_STATUS_HELP,
     )
+    data_status_parser.add_argument(
+        "targets",
+        nargs="*",
+        help=(
+            "Limit command scope to these tracked files/directories, "
+            ".dvc files and stage names."
+        ),
+    ).complete = completion.FILE  # type: ignore[attr-defined]
     data_status_parser.add_argument(
         "--json",
         action="store_true",

dvc/commands/dataset.py

@@ -34,7 +34,7 @@ class CmdDatasetAdd(CmdBase):
 
         url = dataset.spec.url
         ver: str = ""
-        if dataset.type == "dvcx":
+        if dataset.type == "dc":
             ver = f"v{dataset.lock.version}"
         if dataset.type == "dvc":
             if dataset.lock.path:
@@ -55,7 +55,7 @@ class CmdDatasetAdd(CmdBase):
             raise DvcException("--path can't be used without --dvc")
 
         d = vars(self.args)
-        for key in ["dvc", "dvcx", "url"]:
+        for key in ["dvc", "dc", "url"]:
             if url := d.pop(key, None):
                 d.update({"type": key, "url": url})
                 break
@@ -88,8 +88,8 @@ class CmdDatasetUpdate(CmdBase):
         assert new.lock
 
         v: Optional[tuple[str, str]] = None
-        if dataset.type == "dvcx":
-            assert new.type == "dvcx"
+        if dataset.type == "dc":
+            assert new.type == "dc"
             if new.lock.version < dataset.lock.version:
                 action = "Downgrading"
 
@@ -168,7 +168,7 @@ def add_parser(subparsers, parent_parser):
 
     url_exclusive_group = ds_add_parser.add_mutually_exclusive_group(required=True)
     url_exclusive_group.add_argument(
-        "--dvcx", metavar="name", help="Name of the dvcx dataset to track"
+        "--dc", metavar="name", help="Name of the DataChain dataset to track"
     )
     url_exclusive_group.add_argument(
         "--dvc",
@@ -219,7 +219,7 @@ remote://remote_name/path/to/file/or/dir (see `dvc remote`)
         "--rev",
         "--version",
         nargs="?",
-        help="DVCX dataset version or Git revision (e.g. SHA, branch, tag)",
+        help="DataChain dataset version or Git revision (e.g. SHA, branch, tag)",
         metavar="<version>",
     )
     ds_update_parser.set_defaults(func=CmdDatasetUpdate)

dvc/commands/experiments/__init__.py

@@ -58,6 +58,15 @@ def add_parser(subparsers, parent_parser):
     hide_subparsers_from_help(experiments_subparsers)
 
 
+def add_keep_selection_flag(experiments_subcmd_parser):
+    experiments_subcmd_parser.add_argument(
+        "--keep",
+        action="store_true",
+        default=False,
+        help="Keep the selected experiments instead of removing them.",
+    )
+
+
 def add_rev_selection_flags(
     experiments_subcmd_parser, command: str, default: bool = True
 ):

dvc/commands/experiments/remove.py

@@ -34,6 +34,7 @@ class CmdExperimentsRemove(CmdBase):
             num=self.args.num,
             queue=self.args.queue,
             git_remote=self.args.git_remote,
+            keep=self.args.keep,
         )
         if removed:
             ui.write(f"Removed experiments: {humanize.join(map(repr, removed))}")
@@ -44,7 +45,7 @@ class CmdExperimentsRemove(CmdBase):
 
 
 def add_parser(experiments_subparsers, parent_parser):
-    from . import add_rev_selection_flags
+    from . import add_keep_selection_flag, add_rev_selection_flags
 
     EXPERIMENTS_REMOVE_HELP = "Remove experiments."
     experiments_remove_parser = experiments_subparsers.add_parser(
@@ -57,6 +58,7 @@ def add_parser(experiments_subparsers, parent_parser):
     )
     remove_group = experiments_remove_parser.add_mutually_exclusive_group()
     add_rev_selection_flags(experiments_remove_parser, "Remove", False)
+    add_keep_selection_flag(experiments_remove_parser)
     remove_group.add_argument(
         "--queue", action="store_true", help="Remove all queued experiments."
     )

dvc/commands/experiments/show.py

@@ -150,6 +150,7 @@ class CmdExperimentsShow(CmdBase):
                 all_commits=self.args.all_commits,
                 hide_queued=self.args.hide_queued,
                 hide_failed=self.args.hide_failed,
+                hide_workspace=self.args.hide_workspace,
                 revs=self.args.rev,
                 num=self.args.num,
                 sha_only=self.args.sha,
@@ -279,6 +280,12 @@ def add_parser(experiments_subparsers, parent_parser):
         default=False,
         help="Hide queued experiments in the table.",
     )
+    experiments_show_parser.add_argument(
+        "--hide-workspace",
+        action="store_true",
+        default=False,
+        help="Hide workspace row in the table.",
+    )
     experiments_show_parser.add_argument(
         "--json",
         action="store_true",

dvc/commands/gc.py

@@ -54,7 +54,7 @@ class CmdGC(CmdBase):
             msg += " and all experiments"
 
         if self.args.not_in_remote:
-            msg += " that are present in the DVC remote"
+            msg += " that are not present in the DVC remote"
 
         if self.args.repos:
             msg += " of the current and the following repos:"

dvc/commands/ls/__init__.py

@@ -1,3 +1,5 @@
+from typing import Callable
+
 from dvc.cli import completion, formatter
 from dvc.cli.command import CmdBaseNoRepo
 from dvc.cli.utils import DictAction, append_doc_link
@@ -9,54 +11,178 @@ from dvc.ui import ui
 logger = logger.getChild(__name__)
 
 
-def _format_entry(entry, fmt):
-    from dvc.utils.humanize import naturalsize
+def _get_formatter(with_color: bool = False) -> Callable[[dict], str]:
+    def fmt(entry: dict) -> str:
+        return entry["path"]
 
-    size = entry.get("size")
-    if size is None:
-        size = ""
-    else:
-        size = naturalsize(size)
-    return size, fmt(entry)
-
-
-def show_entries(entries, with_color=False, with_size=False):
     if with_color:
         ls_colors = LsColors()
-        fmt = ls_colors.format
-    else:
+        return ls_colors.format
 
-        def fmt(entry):
-            return entry["path"]
+    return fmt
 
+
+def _format_entry(entry, name, with_size=True, with_hash=False):
+    from dvc.utils.humanize import naturalsize
+
+    ret = []
     if with_size:
-        ui.table([_format_entry(entry, fmt) for entry in entries])
+        size = entry.get("size")
+        if size is None or (size <= 0 and entry.get("isdir")):
+            size = ""
+        else:
+            size = naturalsize(size)
+        ret.append(size)
+    if with_hash:
+        md5 = entry.get("md5", "")
+        ret.append(md5)
+    ret.append(name)
+    return ret
+
+
+def show_entries(entries, with_color=False, with_size=False, with_hash=False):
+    fmt = _get_formatter(with_color)
+    if with_size or with_hash:
+        colalign = ("right",) if with_size else None
+        ui.table(
+            [
+                _format_entry(
+                    entry,
+                    fmt(entry),
+                    with_size=with_size,
+                    with_hash=with_hash,
+                )
+                for entry in entries
+            ],
+            colalign=colalign,
+        )
         return
 
     # NOTE: this is faster than ui.table for very large number of entries
     ui.write("\n".join(fmt(entry) for entry in entries))
 
 
+class TreePart:
+    Edge = "├── "
+    Line = "│   "
+    Corner = "└── "
+    Blank = "    "
+
+
+def _build_tree_structure(
+    entries, with_color=False, with_size=False, with_hash=False, _depth=0, _prefix=""
+):
+    rows = []
+    fmt = _get_formatter(with_color)
+
+    num_entries = len(entries)
+    for i, (name, entry) in enumerate(entries.items()):
+        entry["path"] = name
+        is_last = i >= num_entries - 1
+        tree_part = ""
+        if _depth > 0:
+            tree_part = TreePart.Corner if is_last else TreePart.Edge
+
+        row = _format_entry(
+            entry,
+            _prefix + tree_part + fmt(entry),
+            with_size=with_size,
+            with_hash=with_hash,
+        )
+        rows.append(row)
+
+        if contents := entry.get("contents"):
+            new_prefix = _prefix
+            if _depth > 0:
+                new_prefix += TreePart.Blank if is_last else TreePart.Line
+            new_rows = _build_tree_structure(
+                contents,
+                with_color=with_color,
+                with_size=with_size,
+                with_hash=with_hash,
+                _depth=_depth + 1,
+                _prefix=new_prefix,
+            )
+            rows.extend(new_rows)
+
+    return rows
+
+
+def show_tree(entries, with_color=False, with_size=False, with_hash=False):
+    import tabulate
+
+    rows = _build_tree_structure(
+        entries,
+        with_color=with_color,
+        with_size=with_size,
+        with_hash=with_hash,
+    )
+
+    colalign = ("right",) if with_size else None
+
+    _orig = tabulate.PRESERVE_WHITESPACE
+    tabulate.PRESERVE_WHITESPACE = True
+    try:
+        ui.table(rows, colalign=colalign)
+    finally:
+        tabulate.PRESERVE_WHITESPACE = _orig
+
+
 class CmdList(CmdBaseNoRepo):
-    def run(self):
+    def _show_tree(self):
+        from dvc.repo.ls import ls_tree
+
+        entries = ls_tree(
+            self.args.url,
+            self.args.path,
+            rev=self.args.rev,
+            dvc_only=self.args.dvc_only,
+            config=self.args.config,
+            remote=self.args.remote,
+            remote_config=self.args.remote_config,
+            maxdepth=self.args.level,
+        )
+        show_tree(
+            entries,
+            with_color=True,
+            with_size=self.args.size,
+            with_hash=self.args.show_hash,
+        )
+        return 0
+
+    def _show_list(self):
         from dvc.repo import Repo
 
-        try:
-            entries = Repo.ls(
-                self.args.url,
-                self.args.path,
-                rev=self.args.rev,
-                recursive=self.args.recursive,
-                dvc_only=self.args.dvc_only,
-                config=self.args.config,
-                remote=self.args.remote,
-                remote_config=self.args.remote_config,
+        entries = Repo.ls(
+            self.args.url,
+            self.args.path,
+            rev=self.args.rev,
+            recursive=self.args.recursive,
+            dvc_only=self.args.dvc_only,
+            config=self.args.config,
+            remote=self.args.remote,
+            remote_config=self.args.remote_config,
+            maxdepth=self.args.level,
+        )
+        if self.args.json:
+            ui.write_json(entries)
+        elif entries:
+            show_entries(
+                entries,
+                with_color=True,
+                with_size=self.args.size,
+                with_hash=self.args.show_hash,
             )
-            if self.args.json:
-                ui.write_json(entries)
-            elif entries:
-                show_entries(entries, with_color=True, with_size=self.args.size)
-            return 0
+        return 0
+
+    def run(self):
+        if self.args.tree and self.args.json:
+            raise DvcException("Cannot use --tree and --json options together.")
+
+        try:
+            if self.args.tree:
+                return self._show_tree()
+            return self._show_list()
         except FileNotFoundError:
             logger.exception("")
             return 1
@@ -85,6 +211,19 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         help="Recursively list files.",
     )
+    list_parser.add_argument(
+        "-T",
+        "--tree",
+        action="store_true",
+        help="Recurse into directories as a tree.",
+    )
+    list_parser.add_argument(
+        "-L",
+        "--level",
+        metavar="depth",
+        type=int,
+        help="Limit the depth of recursion.",
+    )
     list_parser.add_argument(
         "--dvc-only", action="store_true", help="Show only DVC outputs."
     )
@@ -123,6 +262,12 @@ def add_parser(subparsers, parent_parser):
         ),
     )
     list_parser.add_argument("--size", action="store_true", help="Show sizes.")
+    list_parser.add_argument(
+        "--show-hash",
+        help="Display hash value for each item.",
+        action="store_true",
+        default=False,
+    )
     list_parser.add_argument(
         "path",
         nargs="?",

dvc/commands/ls/ls_colors.py

@@ -32,7 +32,9 @@ class LsColors:
         if entry.get("isexec", False):
             return self._format(text, code="ex")
 
-        _, ext = os.path.splitext(text)
+        stem, ext = os.path.splitext(text)
+        if not ext and stem.startswith("."):
+            ext = stem
         return self._format(text, ext=ext)
 
     def _format(self, text, code=None, ext=None):

dvc/commands/ls_url.py

@@ -3,26 +3,45 @@ from dvc.cli.command import CmdBaseNoRepo
 from dvc.cli.utils import DictAction, append_doc_link
 from dvc.log import logger
 
-from .ls import show_entries
+from .ls import show_entries, show_tree
 
 logger = logger.getChild(__name__)
 
 
 class CmdListUrl(CmdBaseNoRepo):
-    def run(self):
-        from dvc.config import Config
+    def _show_tree(self, config):
+        from dvc.fs import parse_external_url
+        from dvc.repo.ls import _ls_tree
+
+        fs, fs_path = parse_external_url(
+            self.args.url, fs_config=self.args.fs_config, config=config
+        )
+        entries = _ls_tree(fs, fs_path, maxdepth=self.args.level)
+        show_tree(entries, with_color=True, with_size=self.args.size)
+        return 0
+
+    def _show_list(self, config):
         from dvc.repo import Repo
 
         entries = Repo.ls_url(
             self.args.url,
             recursive=self.args.recursive,
+            maxdepth=self.args.level,
             fs_config=self.args.fs_config,
-            config=Config.from_cwd(),
+            config=config,
         )
         if entries:
             show_entries(entries, with_color=True, with_size=self.args.size)
         return 0
 
+    def run(self):
+        from dvc.config import Config
+
+        config = Config.from_cwd()
+        if self.args.tree:
+            return self._show_tree(config=config)
+        return self._show_list(config=config)
+
 
 def add_parser(subparsers, parent_parser):
     LS_HELP = "List directory contents from URL."
@@ -40,6 +59,19 @@ def add_parser(subparsers, parent_parser):
     lsurl_parser.add_argument(
         "-R", "--recursive", action="store_true", help="Recursively list files."
     )
+    lsurl_parser.add_argument(
+        "-T",
+        "--tree",
+        action="store_true",
+        help="Recurse into directories as a tree.",
+    )
+    lsurl_parser.add_argument(
+        "-L",
+        "--level",
+        metavar="depth",
+        type=int,
+        help="Limit the depth of recursion.",
+    )
     lsurl_parser.add_argument("--size", action="store_true", help="Show sizes.")
     lsurl_parser.add_argument(
         "--fs-config",

dvc/commands/remote.py

@@ -110,8 +110,16 @@ class CmdRemoteDefault(CmdRemote):
 class CmdRemoteList(CmdRemote):
     def run(self):
         conf = self.config.read(self.args.level)
+        default_remote = conf["core"].get("remote")
+
         for name, remote_conf in conf["remote"].items():
-            ui.write(name, remote_conf["url"], sep="\t")
+            if name == default_remote:
+                text = f"{name}\t{remote_conf['url']}\t(default)"
+                color = "green"
+            else:
+                text = f"{name}\t{remote_conf['url']}"
+                color = ""
+            ui.write(ui.rich_text(text, style=color), styled=True)
         return 0
 
 

dvc/commands/status.py

@@ -71,7 +71,7 @@ class CmdDataStatus(CmdDataBase):
                 return 0
 
             if self.args.quiet:
-                return bool(st)
+                return int(bool(st))
 
             if st:
                 self._show(st, indent)

dvc/config.py

@@ -1,6 +1,8 @@
 """DVC config objects."""
 
+import ntpath
 import os
+import posixpath
 import re
 from contextlib import contextmanager
 from functools import partial
@@ -268,12 +270,17 @@ class Config(dict):
         if re.match(r"\w+://", path):
             return path
 
+        if os.name == "nt" and posixpath.isabs(path) and ntpath.sep not in path:
+            return path
+
         if os.path.isabs(path):
             return path
 
         # on windows convert slashes to backslashes
         # to have path compatible with abs_conf_dir
         if os.path.sep == "\\" and "/" in path:
+            if path.startswith("/"):
+                path = path.replace("/", "\\\\", 1)
             path = path.replace("/", "\\")
 
         expanded = os.path.expanduser(path)
@@ -305,6 +312,9 @@ class Config(dict):
         if os.path.expanduser(path) != path:
             return localfs.as_posix(path)
 
+        if os.name == "nt" and posixpath.isabs(path) and ntpath.sep not in path:
+            return path
+
         if isinstance(path, RelPath) or not os.path.isabs(path):
             path = relpath(path, conf_dir)
             return localfs.as_posix(path)
@@ -330,6 +340,7 @@ class Config(dict):
                     "gdrive_user_credentials_file": func,
                     "gdrive_service_account_json_file_path": func,
                     "credentialpath": func,
+                    "configpath": func,
                     "keyfile": func,
                     "cert_path": func,
                     "key_path": func,

dvc/config_schema.py

@@ -215,6 +215,7 @@ SCHEMA = {
                     "credentialpath": str,
                     "endpointurl": str,
                     Optional("verify", default=False): Bool,
+                    Optional("allow_anonymous_login", default=False): Bool,
                     **REMOTE_COMMON,
                 },
                 "ssh": {

dvc/daemon.py

@@ -164,7 +164,7 @@ def _spawn(
         return _detached_subprocess(executable + args, env=env, **kwargs)
 
 
-def daemonize(args: list[str], executable: Union[None, str, list[str]] = None) -> None:
+def daemonize(args: list[str], executable: Union[str, list[str], None] = None) -> None:
     if os.name not in ("posix", "nt"):
         return
 

dvc/dagascii.py

@@ -104,21 +104,21 @@ class AsciiCanvas:
                 if dx == 0:
                     y = y0
                 else:
-                    y = y0 + int(round((x - x0) * dy / float(dx)))
+                    y = y0 + round((x - x0) * dy / float(dx))
                 self.point(x, y, char)
         elif y0 < y1:
             for y in range(y0, y1 + 1):
                 if dy == 0:
                     x = x0
                 else:
-                    x = x0 + int(round((y - y0) * dx / float(dy)))
+                    x = x0 + round((y - y0) * dx / float(dy))
                 self.point(x, y, char)
         else:
             for y in range(y1, y0 + 1):
                 if dy == 0:
                     x = x0
                 else:
-                    x = x1 + int(round((y - y1) * dx / float(dy)))
+                    x = x1 + round((y - y1) * dx / float(dy))
                 self.point(x, y, char)
 
     def text(self, x, y, text):
@@ -258,8 +258,8 @@ def draw(vertices, edges):
     maxx = max(Xs)
     maxy = max(Ys)
 
-    canvas_cols = int(math.ceil(math.ceil(maxx) - math.floor(minx))) + 1
-    canvas_lines = int(round(maxy - miny))
+    canvas_cols = math.ceil(math.ceil(maxx) - math.floor(minx)) + 1
+    canvas_lines = round(maxy - miny)
 
     canvas = AsciiCanvas(canvas_cols, canvas_lines)
 
@@ -270,10 +270,10 @@ def draw(vertices, edges):
             start = edge.view._pts[index - 1]
             end = edge.view._pts[index]
 
-            start_x = int(round(start[0] - minx))
-            start_y = int(round(start[1] - miny))
-            end_x = int(round(end[0] - minx))
-            end_y = int(round(end[1] - miny))
+            start_x = round(start[0] - minx)
+            start_y = round(start[1] - miny)
+            end_x = round(end[0] - minx)
+            end_y = round(end[1] - miny)
 
             assert start_x >= 0
             assert start_y >= 0
@@ -288,12 +288,12 @@ def draw(vertices, edges):
         y = vertex.view.xy[1]
 
         canvas.box(
-            int(round(x - minx)),
-            int(round(y - miny)),
+            round(x - minx),
+            round(y - miny),
             vertex.view.w,
             vertex.view.h,
         )
 
-        canvas.text(int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data)
+        canvas.text(round(x - minx) + 1, round(y - miny) + 1, vertex.data)
 
     return canvas.draw()

dvc/dependency/base.py

@@ -45,7 +45,7 @@ class Dependency(Output):
             self.fs_path = self.fs.version_path(self.fs_path, self.meta.version_id)
 
     def download(self, to, jobs=None):
-        fs_download(self.fs, self.fs_path, to.fs_path, jobs=jobs)
+        return fs_download(self.fs, self.fs_path, to.fs_path, jobs=jobs)
 
     def save(self):
         super().save()

dvc/dependency/repo.py

@@ -3,7 +3,6 @@ from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
 
 import voluptuous as vol
 
-from dvc.prompt import confirm
 from dvc.utils import as_posix
 
 from .base import Dependency
@@ -12,6 +11,7 @@ if TYPE_CHECKING:
     from dvc.fs import DVCFileSystem
     from dvc.output import Output
     from dvc.stage import Stage
+    from dvc_data.hashfile.hash_info import HashInfo
 
 
 class RepoDependency(Dependency):
@@ -94,29 +94,25 @@ class RepoDependency(Dependency):
         }
 
     def download(self, to: "Output", jobs: Optional[int] = None):
-        from dvc_data.hashfile.build import build
-        from dvc_data.hashfile.checkout import CheckoutError, checkout
+        from dvc.fs import LocalFileSystem
 
-        try:
-            repo = self._make_fs(locked=True).repo
+        files = super().download(to=to, jobs=jobs)
+        if not isinstance(to.fs, LocalFileSystem):
+            return
 
-            _, _, obj = build(
-                repo.cache.local,
-                self.fs_path,
-                repo.dvcfs,
-                repo.cache.local.fs.PARAM_CHECKSUM,
-            )
-            checkout(
-                to.fs_path,
-                to.fs,
-                obj,
-                self.repo.cache.local,
-                ignore=None,
-                state=self.repo.state,
-                prompt=confirm,
-            )
-        except (CheckoutError, FileNotFoundError):
-            super().download(to=to, jobs=jobs)
+        hashes: list[tuple[str, HashInfo, dict[str, Any]]] = []
+        for src_path, dest_path, maybe_info in files:
+            try:
+                info = maybe_info or self.fs.info(src_path)
+                hash_info = info["dvc_info"]["entry"].hash_info
+                dest_info = to.fs.info(dest_path)
+            except (KeyError, AttributeError):
+                # If no hash info found, just keep going and output will be hashed later
+                continue
+            if hash_info:
+                hashes.append((dest_path, hash_info, dest_info))
+        cache = to.cache if to.use_cache else to.local_cache
+        cache.state.save_many(hashes, to.fs)
 
     def update(self, rev: Optional[str] = None):
         if rev:
@@ -150,11 +146,15 @@ class RepoDependency(Dependency):
         else:
             config = Config.load_file(conf)
 
+        # Setup config to the new DVCFileSystem to use the remote repo, but rely on the
+        # local cache instead of the remote's cache. This avoids re-streaming of data,
+        # but messes up the call to `_get_remote_config()` downstream, which will need
+        # to ignore cache parameters.
         config["cache"] = self.repo.config["cache"]
         config["cache"]["dir"] = self.repo.cache.local_cache_dir
 
         return DVCFileSystem(
-            url=self.def_repo[self.PARAM_URL],
+            repo=self.def_repo[self.PARAM_URL],
             rev=rev or self._get_rev(locked=locked),
             subrepos=True,
             config=config,

dvc/dirs.py

@@ -1,4 +1,5 @@
 import os
+from typing import Optional
 
 import platformdirs
 
@@ -20,7 +21,24 @@ def global_config_dir():
     )
 
 
-def site_cache_dir():
-    return os.getenv(env.DVC_SITE_CACHE_DIR) or platformdirs.site_cache_dir(
-        APPNAME, APPAUTHOR, opinion=True
-    )
+def site_cache_dir(config_site_cache_dir: Optional[str] = None):
+    from platformdirs import PlatformDirs
+    from platformdirs.unix import Unix
+
+    if dvc_site_cache_dir := os.getenv(env.DVC_SITE_CACHE_DIR):
+        return dvc_site_cache_dir
+
+    if config_site_cache_dir is not None:
+        return config_site_cache_dir
+
+    if issubclass(Unix, PlatformDirs):
+        # Return the cache directory shared by users, e.g. `/var/tmp/$appname`
+        # NOTE: platformdirs>=5 changed `site_cache_dir` to return /var/cache/$appname.
+        # as the following path is considered insecure.
+        # For details, see: https://github.com/tox-dev/platformdirs/pull/239
+
+        # FIXME: keeping the old behavior temporarily to avoid dependency conflict.
+        #        In the future, consider migrating to a more secure directory.
+        return f"/var/tmp/{APPNAME}"  # noqa: S108
+
+    return platformdirs.site_cache_dir(APPNAME, APPAUTHOR, opinion=True)

dvc/fs/__init__.py

@@ -1,4 +1,5 @@
 import glob
+from itertools import repeat
 from typing import Optional
 from urllib.parse import urlparse
 
@@ -19,7 +20,7 @@ from dvc_objects.fs import (  # noqa: F401
     system,
     utils,
 )
-from dvc_objects.fs.base import AnyFSPath, FileSystem  # noqa: F401, TCH001
+from dvc_objects.fs.base import AnyFSPath, FileSystem  # noqa: F401, TC001
 from dvc_objects.fs.errors import (  # noqa: F401
     AuthError,
     ConfigError,
@@ -47,28 +48,12 @@ known_implementations.update(
 
 def download(
     fs: "FileSystem", fs_path: str, to: str, jobs: Optional[int] = None
-) -> int:
+) -> list[tuple[str, str, Optional[dict]]]:
     from dvc.scm import lfs_prefetch
 
     from .callbacks import TqdmCallback
 
     with TqdmCallback(desc=f"Downloading {fs.name(fs_path)}", unit="files") as cb:
-        # NOTE: We use dvc-objects generic.copy over fs.get since it makes file
-        # download atomic and avoids fsspec glob/regex path expansion.
-        if fs.isdir(fs_path):
-            from_infos = [
-                path for path in fs.find(fs_path) if not path.endswith(fs.flavour.sep)
-            ]
-            if not from_infos:
-                localfs.makedirs(to, exist_ok=True)
-                return 0
-            to_infos = [
-                localfs.join(to, *fs.relparts(info, fs_path)) for info in from_infos
-            ]
-        else:
-            from_infos = [fs_path]
-            to_infos = [to]
-
         if isinstance(fs, DVCFileSystem):
             lfs_prefetch(
                 fs,
@@ -78,10 +63,29 @@ def download(
                     else glob.escape(fs_path)
                 ],
             )
+            if not glob.has_magic(fs_path):
+                return fs._get(fs_path, to, batch_size=jobs, callback=cb)
+
+        # NOTE: We use dvc-objects generic.copy over fs.get since it makes file
+        # download atomic and avoids fsspec glob/regex path expansion.
+        if fs.isdir(fs_path):
+            from_infos = [
+                path for path in fs.find(fs_path) if not path.endswith(fs.flavour.sep)
+            ]
+            if not from_infos:
+                localfs.makedirs(to, exist_ok=True)
+                return []
+            to_infos = [
+                localfs.join(to, *fs.relparts(info, fs_path)) for info in from_infos
+            ]
+        else:
+            from_infos = [fs_path]
+            to_infos = [to]
+
         cb.set_size(len(from_infos))
         jobs = jobs or fs.jobs
         generic.copy(fs, from_infos, localfs, to_infos, callback=cb, batch_size=jobs)
-        return len(to_infos)
+        return list(zip(from_infos, to_infos, repeat(None)))
 
 
 def parse_external_url(url, fs_config=None, config=None):

dvc/fs/dvc.py

@@ -4,22 +4,28 @@ import ntpath
 import os
 import posixpath
 import threading
-from collections import deque
-from contextlib import ExitStack, suppress
+from collections import defaultdict, deque
+from contextlib import ExitStack, nullcontext, suppress
+from glob import has_magic
 from typing import TYPE_CHECKING, Any, Callable, Optional, Union
 
-from fsspec.spec import AbstractFileSystem
+from fsspec.spec import DEFAULT_CALLBACK, AbstractFileSystem
 from funcy import wrap_with
 
 from dvc.log import logger
-from dvc_objects.fs.base import FileSystem
+from dvc.utils.threadpool import ThreadPoolExecutor
+from dvc_objects.fs.base import AnyFSPath, FileSystem
 
 from .data import DataFileSystem
 
 if TYPE_CHECKING:
+    from contextlib import AbstractContextManager
+
     from dvc.repo import Repo
     from dvc.types import DictStrAny, StrPath
 
+    from .callbacks import Callback
+
 logger = logger.getChild(__name__)
 
 RepoFactory = Union[Callable[..., "Repo"], type["Repo"]]
@@ -77,11 +83,10 @@ class _DVCFileSystem(AbstractFileSystem):
     cachable = False
     root_marker = "/"
 
-    def __init__(  # noqa: PLR0913
+    def __init__(
         self,
-        url: Optional[str] = None,
+        repo: Union["Repo", os.PathLike[str], str, None] = None,
         rev: Optional[str] = None,
-        repo: Optional["Repo"] = None,
         subrepos: bool = False,
         repo_factory: Optional[RepoFactory] = None,
         fo: Optional[str] = None,
@@ -95,7 +100,8 @@ class _DVCFileSystem(AbstractFileSystem):
         """DVC + git-tracked files fs.
 
         Args:
-            path (str, optional): URL or path to a DVC/Git repository.
+            repo (str | os.PathLike[str] | Repo, optional): A url or a path to a DVC/Git
+                repository, or a `Repo` instance.
                 Defaults to a DVC repository in the current working directory.
                 Both HTTP and SSH protocols are supported for remote Git repos
                 (e.g. [user@]server:project.git).
@@ -105,7 +111,6 @@ class _DVCFileSystem(AbstractFileSystem):
                 In case of a local repository, if rev is unspecified, it will
                 default to the working directory.
                 If the repo is not a Git repo, this option is ignored.
-            repo (:obj:`Repo`, optional): `Repo` instance.
             subrepos (bool): traverse to subrepos.
                 By default, it ignores subrepos.
             repo_factory (callable): A function to initialize subrepo with.
@@ -130,13 +135,23 @@ class _DVCFileSystem(AbstractFileSystem):
             ...    rev="main",
             ... )
         """
+        from dvc.repo import Repo
+
+        # kwargs.get("url") is for maintaining backward compatibility
+        repo = repo or fo or kwargs.get("url")
+        if isinstance(repo, Repo):
+            self._repo: Optional[Repo] = repo
+            url = None
+        else:
+            self._repo = None
+            url = os.fspath(repo) if repo else None
+
         super().__init__()
-        self._repo = repo
         self._repo_factory = repo_factory
         self._traverse_subrepos = subrepos
         self._repo_stack = ExitStack()
         self._repo_kwargs = {
-            "url": url if url is not None else fo,
+            "url": url,
             "rev": rev,
             "subrepos": subrepos,
             "config": config,
@@ -474,9 +489,128 @@ class _DVCFileSystem(AbstractFileSystem):
         info["name"] = path
         return info
 
+    def get(
+        self,
+        rpath,
+        lpath,
+        recursive=False,
+        callback=DEFAULT_CALLBACK,
+        maxdepth=None,
+        batch_size=None,
+        **kwargs,
+    ):
+        self._get(
+            rpath,
+            lpath,
+            recursive=recursive,
+            callback=callback,
+            maxdepth=maxdepth,
+            batch_size=batch_size,
+            **kwargs,
+        )
+
+    def _get(  # noqa: C901, PLR0912, PLR0915
+        self,
+        rpath,
+        lpath,
+        recursive=False,
+        callback=DEFAULT_CALLBACK,
+        maxdepth=None,
+        batch_size=None,
+        **kwargs,
+    ) -> list[tuple[str, str, Optional[dict]]]:
+        if (
+            isinstance(rpath, list)
+            or isinstance(lpath, list)
+            or has_magic(rpath)
+            or not self.exists(rpath)
+            or not recursive
+        ):
+            super().get(
+                rpath,
+                lpath,
+                recursive=recursive,
+                callback=callback,
+                maxdepth=maxdepth,
+                **kwargs,
+            )
+            return []
+
+        if os.path.isdir(lpath) or lpath.endswith(os.path.sep):
+            lpath = self.join(lpath, os.path.basename(rpath))
+
+        if self.isfile(rpath):
+            with callback.branched(rpath, lpath) as child:
+                self.get_file(rpath, lpath, callback=child, **kwargs)
+                return [(rpath, lpath, None)]
+
+        result: list[tuple[str, str, Optional[dict]]] = []
+        _dirs: list[str] = []
+        _files: dict[FileSystem, list[tuple[str, str, Optional[dict]]]]
+        _files = defaultdict(list)
+
+        for root, dirs, files in self.walk(rpath, maxdepth=maxdepth, detail=True):
+            if files:
+                callback.set_size((callback.size or 0) + len(files))
+
+            parts = self.relparts(root, rpath)
+            if parts in ((os.curdir,), ("",)):
+                parts = ()
+            dest_root = os.path.join(lpath, *parts)
+            if not maxdepth or len(parts) < maxdepth - 1:
+                _dirs.extend(f"{dest_root}{os.path.sep}{d}" for d in dirs)
+
+            key = self._get_key_from_relative(root)
+            _, dvc_fs, _ = self._get_subrepo_info(key)
+
+            for name, info in files.items():
+                dvc_info = info.get("dvc_info")
+                fs_info = info.get("fs_info")
+                if dvc_fs and dvc_info and not fs_info:
+                    fs = dvc_fs
+                    fs_path = dvc_info["name"]
+                else:
+                    fs = self.repo.fs
+                    fs_path = fs_info["name"]
+
+                src_path = f"{root}{self.sep}{name}"
+                dest_path = f"{dest_root}{os.path.sep}{name}"
+                _files[fs].append((fs_path, dest_path, dvc_info))
+                result.append((src_path, dest_path, info))
+
+        os.makedirs(lpath, exist_ok=True)
+        for d in _dirs:
+            os.makedirs(d, exist_ok=True)
+
+        def get_file(arg: tuple[FileSystem, tuple[str, str, Optional[dict]]]):
+            fs, (src, dest, info) = arg
+            kw = kwargs
+            if isinstance(fs, DataFileSystem):
+                kw = kw | {"info": info}
+            with callback.branched(src, dest) as child:
+                fs.get_file(src, dest, callback=child, **kw)
+
+        if batch_size == 1:
+            ctx: AbstractContextManager = nullcontext()
+            map_fn: Callable = map
+        else:
+            ctx = ThreadPoolExecutor(max_workers=batch_size)
+            map_fn = ctx.imap_unordered
+
+        with ctx:
+            it = ((fs, f) for fs, files in _files.items() for f in files)
+            deque(callback.wrap(map_fn(get_file, it)), maxlen=0)
+        return result
+
     def get_file(self, rpath, lpath, **kwargs):
+        dvc_info = kwargs.pop("info", {}).pop("dvc_info", None)
         key = self._get_key_from_relative(rpath)
         fs_path = self._from_key(key)
+        dirpath = os.path.dirname(lpath)
+        if dirpath:
+            # makedirs raises error if the string is empty
+            os.makedirs(dirpath, exist_ok=True)
+
         try:
             return self.repo.fs.get_file(fs_path, lpath, **kwargs)
         except FileNotFoundError:
@@ -485,7 +619,7 @@ class _DVCFileSystem(AbstractFileSystem):
                 raise
 
         dvc_path = _get_dvc_path(dvc_fs, subkey)
-        return dvc_fs.get_file(dvc_path, lpath, **kwargs)
+        return dvc_fs.get_file(dvc_path, lpath, info=dvc_info, **kwargs)
 
     def du(self, path, total=True, maxdepth=None, withdirs=False, **kwargs):
         if maxdepth is not None:
@@ -553,6 +687,45 @@ class DVCFileSystem(FileSystem):
     def getcwd(self):
         return self.fs.getcwd()
 
+    def _get(
+        self,
+        from_info: Union[AnyFSPath, list[AnyFSPath]],
+        to_info: Union[AnyFSPath, list[AnyFSPath]],
+        callback: "Callback" = DEFAULT_CALLBACK,
+        recursive: bool = False,
+        batch_size: Optional[int] = None,
+        **kwargs,
+    ) -> list[tuple[str, str, Optional[dict]]]:
+        # FileSystem.get is non-recursive by default if arguments are lists
+        # otherwise, it's recursive.
+        recursive = not (isinstance(from_info, list) and isinstance(to_info, list))
+        return self.fs._get(
+            from_info,
+            to_info,
+            callback=callback,
+            recursive=recursive,
+            batch_size=batch_size,
+            **kwargs,
+        )
+
+    def get(
+        self,
+        from_info: Union[AnyFSPath, list[AnyFSPath]],
+        to_info: Union[AnyFSPath, list[AnyFSPath]],
+        callback: "Callback" = DEFAULT_CALLBACK,
+        recursive: bool = False,
+        batch_size: Optional[int] = None,
+        **kwargs,
+    ) -> None:
+        self._get(
+            from_info,
+            to_info,
+            callback=callback,
+            batch_size=batch_size,
+            recursive=recursive,
+            **kwargs,
+        )
+
     @property
     def fsid(self) -> str:
         return self.fs.fsid
@@ -569,9 +742,10 @@ class DVCFileSystem(FileSystem):
         return self.fs.repo_url
 
     def from_os_path(self, path: str) -> str:
-        if os.path.isabs(path):
+        if os.path.isabs(path) or (
+            os.name == "nt" and posixpath.isabs(path) and ntpath.sep not in path
+        ):
             path = os.path.relpath(path, self.repo.root_dir)
-
         return as_posix(path)
 
     def close(self):

dvc/info.py

@@ -48,7 +48,10 @@ def get_dvc_info():
                 info.append("Cache types: " + error_link("no-dvc-cache"))
 
             info.append(f"Caches: {_get_caches(repo.cache)}")
-            info.append(f"Remotes: {_get_remotes(repo.config)}")
+
+            configured_remotes = _get_remotes(repo.config)
+            remotes = ", ".join(configured_remotes) if configured_remotes else None
+            info.append(f"Remotes: {remotes}")
 
             root_directory = repo.root_dir
             fs_root = _get_fs_type(os.path.abspath(root_directory))
@@ -75,12 +78,10 @@ def _get_caches(cache):
 
 
 def _get_remotes(config):
-    schemes = (
+    return [
         get_fs_cls(get_fs_config(config, name=remote)).protocol
         for remote in config["remote"]
-    )
-
-    return ", ".join(schemes) or "None"
+    ]
 
 
 def _get_linktype_support_info(repo):
@@ -123,7 +124,7 @@ def _get_supported_remotes():
         if not fs_cls.get_missing_deps():
             dependencies = []
             for requirement in fs_cls.REQUIRES:
-                dependencies.append(
+                dependencies.append(  # noqa: PERF401
                     f"{requirement} = {importlib_metadata.version(requirement)}"
                 )
 

dvc/lock.py

@@ -2,12 +2,13 @@
 
 import hashlib
 import os
+import time
 from abc import ABC, abstractmethod
 from datetime import timedelta
+from typing import Optional, Union
 
 import flufl.lock
 import zc.lockfile
-from funcy import retry
 
 from dvc.exceptions import DvcException
 from dvc.progress import Tqdm
@@ -87,9 +88,10 @@ class Lock(LockBase):
     Uses zc.lockfile as backend.
     """
 
-    def __init__(self, lockfile, friendly=False, **kwargs):
+    def __init__(self, lockfile, friendly=False, wait=False, **kwargs):
         super().__init__(lockfile)
         self._friendly = friendly
+        self._wait = wait
         self._lock = None
         self._lock_failed = False
 
@@ -100,23 +102,38 @@ class Lock(LockBase):
     def _do_lock(self):
         try:
             self._lock_failed = False
-            with Tqdm(
-                bar_format="{desc}",
-                disable=not self._friendly,
-                desc="If DVC froze, see `hardlink_lock` in {}".format(
-                    format_link("https://man.dvc.org/config#core")
-                ),
-            ):
-                self._lock = zc.lockfile.LockFile(self._lockfile)
+            self._lock = zc.lockfile.LockFile(self._lockfile)
         except zc.lockfile.LockError:
             self._lock_failed = True
             raise LockError(FAILED_TO_LOCK_MESSAGE)  # noqa: B904
 
     def lock(self):
-        retries = 6
-        delay = DEFAULT_TIMEOUT / retries
-        lock_retry = retry(retries, LockError, timeout=delay)(self._do_lock)
-        lock_retry()
+        """Acquire the lock, either waiting forever, or after default_retries."""
+        default_retries = 6
+        delay = DEFAULT_TIMEOUT / default_retries
+        attempts = 0
+
+        max_retries = float("inf") if self._wait else default_retries
+
+        with Tqdm(
+            bar_format="{desc}",
+            disable=not self._friendly,
+            desc="Waiting to acquire lock. "
+            "If DVC froze, see `hardlink_lock` in {}.".format(
+                format_link("https://man.dvc.org/config#core")
+            ),
+        ) as pbar:
+            while True:
+                try:
+                    self._do_lock()
+                    return
+                except LockError:
+                    attempts += 1
+                    if attempts > max_retries:
+                        raise
+                    time.sleep(delay)
+                finally:
+                    pbar.update()
 
     def unlock(self):
         if self._lock_failed:
@@ -149,7 +166,7 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
         tmp_dir (str): a directory to store claim files.
     """
 
-    def __init__(self, lockfile, tmp_dir=None, **kwargs):
+    def __init__(self, lockfile, tmp_dir=None, wait=False, **kwargs):
         import socket
 
         self._tmp_dir = tmp_dir
@@ -172,11 +189,21 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
         self._separator = flufl.lock.SEP
         self._set_claimfile()
         self._owned = True
+        self._wait = wait
         self._retry_errnos = []
+        self._friendly = kwargs.get("friendly", False)
 
-    def lock(self):
+    def lock(self, timeout: Optional[Union[timedelta, int]] = None):
         try:
-            super().lock(timedelta(seconds=DEFAULT_TIMEOUT))
+            if not self._wait:
+                timeout = timeout or timedelta(seconds=DEFAULT_TIMEOUT)
+
+            with Tqdm(
+                bar_format="{desc}",
+                disable=not (self._wait and self._friendly),
+                desc="Waiting to acquire lock",
+            ):
+                super().lock(timeout)
         except flufl.lock.TimeOutError:
             raise LockError(FAILED_TO_LOCK_MESSAGE)  # noqa: B904
 
@@ -185,11 +212,11 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
 
         if self._tmp_dir is not None:
             # Under Windows file path length is limited so we hash it
-            hasher = hashlib.md5(self._claimfile.encode())  # noqa: S324
+            hasher = hashlib.md5(self._claimfile.encode(), usedforsecurity=False)
             filename = hasher.hexdigest()
             self._claimfile = os.path.join(self._tmp_dir, filename + ".lock")
 
 
-def make_lock(lockfile, tmp_dir=None, friendly=False, hardlink_lock=False):
+def make_lock(lockfile, tmp_dir=None, friendly=False, hardlink_lock=False, wait=False):
     cls = HardlinkLock if hardlink_lock else Lock
-    return cls(lockfile, tmp_dir=tmp_dir, friendly=friendly)
+    return cls(lockfile, tmp_dir=tmp_dir, friendly=friendly, wait=wait)

dvc/logger.py

@@ -5,6 +5,8 @@ import logging.config
 import logging.handlers
 import os
 import sys
+from collections.abc import Iterator
+from contextlib import contextmanager
 from typing import ClassVar
 
 import colorama
@@ -36,7 +38,7 @@ def add_logging_level(level_name, level_num, method_name=None):
             self._log(level_num, message, args, **kwargs)
 
     def log_to_root(message, *args, **kwargs):
-        logging.log(level_num, message, *args, **kwargs)
+        logging.log(level_num, message, *args, **kwargs)  # noqa: LOG015
 
     # getLevelName resolves the numeric log level if already defined,
     # otherwise returns a string
@@ -174,9 +176,19 @@ def _iter_causes(exc):
         exc = exc.__cause__
 
 
-def set_loggers_level(level: int = logging.INFO) -> None:
+@contextmanager
+def set_loggers_level(level: int = logging.INFO) -> Iterator[None]:
+    ret: dict[logging.Logger, int] = {}
     for name in ["dvc", "dvc_objects", "dvc_data"]:
-        logging.getLogger(name).setLevel(level)
+        _logger = logging.getLogger(name)
+        ret[_logger] = _logger.getEffectiveLevel()
+        _logger.setLevel(level)
+
+    try:
+        yield
+    finally:
+        for _logger, old_lvl in ret.items():
+            _logger.setLevel(old_lvl)
 
 
 def setup(level: int = logging.INFO, log_colors: bool = True) -> None:

dvc/output.py

@@ -431,7 +431,7 @@ class Output:
             self.meta.nfiles = len(self.files)
             self.meta.size = sum(filter(None, (f.get("size") for f in self.files)))
             self.meta.remote = first(f.get("remote") for f in self.files)
-        elif self.meta.nfiles or self.hash_info and self.hash_info.isdir:
+        elif self.meta.nfiles or (self.hash_info and self.hash_info.isdir):
             self.meta.isdir = True
             if not self.hash_info and self.hash_name not in ("md5", "md5-dos2unix"):
                 md5 = getattr(self.meta, "md5", None)
@@ -542,7 +542,9 @@ class Output:
             desc=f"Collecting files and computing hashes in {self}",
             disable=no_progress_bar,
         ) as pb:
-            return build(*args, callback=pb.as_callback(), **kwargs)
+            kwargs["callback"] = pb.as_callback()
+            kwargs.setdefault("checksum_jobs", self.fs.hash_jobs)
+            return build(*args, **kwargs)
 
     def _get_hash_meta(self):
         if self.use_cache:
@@ -730,6 +732,7 @@ class Output:
         from dvc_data.hashfile.checkout import LinkError, PromptError
 
         kwargs.setdefault("ignore", self.dvcignore)
+        kwargs.setdefault("checksum_jobs", self.fs.hash_jobs)
         try:
             return checkout(*args, **kwargs)
         except PromptError as exc:
@@ -749,11 +752,9 @@ class Output:
             granular = (
                 self.is_dir_checksum and filter_info and filter_info != self.fs_path
             )
-            # NOTE: trying to use hardlink during transfer only if we will be
-            # relinking later
-            hardlink = relink
+            hardlink = relink and next(iter(self.cache.cache_types), None) == "hardlink"
             if granular:
-                obj = self._commit_granular_dir(filter_info, hardlink)
+                obj = self._commit_granular_dir(filter_info, hardlink=hardlink)
             else:
                 staging, _, obj = self._build(
                     self.cache,
@@ -786,6 +787,7 @@ class Output:
                         state=self.repo.state,
                         prompt=prompt.confirm,
                         progress_callback=cb,
+                        old=obj,
                     )
                 self.set_exec()
 
@@ -1398,12 +1400,14 @@ class Output:
 
         assert staging
         assert obj.hash_info
+
+        hardlink = relink and next(iter(self.cache.cache_types), None) == "hardlink"
         with TqdmCallback(desc=f"Adding {self} to cache", unit="file") as cb:
             otransfer(
                 staging,
                 self.cache,
                 {obj.hash_info},
-                hardlink=relink,
+                hardlink=hardlink,
                 shallow=False,
                 callback=cb,
             )
@@ -1421,6 +1425,7 @@ class Output:
                     state=self.repo.state,
                     prompt=prompt.confirm,
                     progress_callback=callback,
+                    old=obj,
                 )
             self.set_exec()
         return obj

dvc/parsing/context.py

@@ -105,7 +105,7 @@ class Meta:
 
     @staticmethod
     def update_path(meta: "Meta", path: Union[str, int]):
-        dpaths = meta.dpaths[:] + [str(path)]
+        dpaths = [*meta.dpaths, str(path)]
         return replace(meta, dpaths=dpaths)
 
     def __str__(self):
@@ -155,7 +155,7 @@ class Value(Node):
 PRIMITIVES = (int, float, str, bytes, bool)
 
 
-class Container(Node, ABC):
+class Container(Node, ABC):  # noqa: PLW1641
     meta: Meta
     data: Union[list, dict]
     _key_transform = staticmethod(identity)
@@ -176,8 +176,9 @@ class Container(Node, ABC):
             return value
         if isinstance(value, (list, dict)):
             assert meta
-            container = CtxDict if isinstance(value, dict) else CtxList
-            return container(value, meta=meta)
+            if isinstance(value, dict):
+                return CtxDict(value, meta=meta)
+            return CtxList(value, meta=meta)
         msg = f"Unsupported value of type '{type(value).__name__}' in '{meta}'"
         raise TypeError(msg)
 

dvc/pathspec_math.py

@@ -27,11 +27,9 @@ def _remove_slash(rule):
 
 def _match_all_level(rule):
     if rule[:-1].find("/") >= 0 and not rule.startswith("**/"):
-        if rule.startswith("/"):
-            rule = rule[1:]
+        rule = rule.removeprefix("/")
         return False, rule
-    if rule.startswith("**/"):
-        rule = rule[3:]
+    rule = rule.removeprefix("**/")
     return True, rule
 
 

dvc/render/converter/vega.py

@@ -23,7 +23,7 @@ def _lists(blob: Union[dict, list]) -> Iterable[list]:
     if isinstance(blob, list):
         yield blob
     else:
-        for _, value in blob.items():
+        for value in blob.values():
             if isinstance(value, dict):
                 yield from _lists(value)
             elif isinstance(value, list):

dvc/repo/__init__.py

@@ -144,10 +144,11 @@ class Repo:
         scm: Optional[Union["Git", "NoSCM"]] = None,
         remote: Optional[str] = None,
         remote_config: Optional["DictStrAny"] = None,
+        _wait_for_lock: bool = False,
     ):
         from dvc.cachemgr import CacheManager
         from dvc.data_cloud import DataCloud
-        from dvc.fs import GitFileSystem, LocalFileSystem, localfs
+        from dvc.fs import GitFileSystem, LocalFileSystem
         from dvc.lock import LockNoop, make_lock
         from dvc.repo.artifacts import Artifacts
         from dvc.repo.datasets import Datasets
@@ -161,12 +162,13 @@ class Repo:
 
         self.url = url
         self._fs_conf = {"repo_factory": repo_factory}
-        self._fs = fs or localfs
+        self._fs = fs or LocalFileSystem()
         self._scm = scm
         self._config = config
         self._remote = remote
         self._remote_config = remote_config
-        self._data_index = None
+        self._data_index: Optional[DataIndex] = None
+        self._wait_for_lock = _wait_for_lock
 
         if rev and not fs:
             self._scm = scm = SCM(root_dir or os.curdir)
@@ -204,8 +206,14 @@ class Repo:
                     tmp_dir=self.tmp_dir,
                     hardlink_lock=self.config["core"].get("hardlink_lock", False),
                     friendly=True,
+                    wait=self._wait_for_lock,
                 )
                 os.makedirs(self.site_cache_dir, exist_ok=True)
+                if not fs and (
+                    checksum_jobs := self.config["core"].get("checksum_jobs")
+                ):
+                    self.fs.hash_jobs = checksum_jobs
+
                 self.state = State(self.root_dir, self.site_cache_dir, self.dvcignore)
             else:
                 self.lock = LockNoop()
@@ -604,7 +612,7 @@ class Repo:
         from dvc.fs import GitFileSystem
         from dvc.version import version_tuple
 
-        cache_dir = self.config["core"].get("site_cache_dir") or site_cache_dir()
+        cache_dir = site_cache_dir(self.config["core"].get("site_cache_dir"))
 
         subdir = None
         if isinstance(self.fs, GitFileSystem):
@@ -633,10 +641,11 @@ class Repo:
         # that just happened to be at the same path as old deleted ones.
         btime = self._btime or getattr(os.stat(root_dir), "st_birthtime", None)
 
-        md5 = hashlib.md5(  # noqa: S324
+        md5 = hashlib.md5(
             str(
                 (root_dir, subdir, btime, getpass.getuser(), version_tuple[0], salt)
-            ).encode()
+            ).encode(),
+            usedforsecurity=False,
         )
         repo_token = md5.hexdigest()
         return os.path.join(repos_dir, repo_token)

dvc/repo/artifacts.py

@@ -220,7 +220,7 @@ class Artifacts:
 
             out = resolve_output(path, out, force=force)
             fs = self.repo.dvcfs
-            count = fs_download(fs, path, os.path.abspath(out), jobs=jobs)
+            count = len(fs_download(fs, path, os.path.abspath(out), jobs=jobs))
         return count, out
 
     @staticmethod

dvc/repo/checkout.py

@@ -149,7 +149,10 @@ def checkout(  # noqa: C901
 
     def checkout_onerror(src_path, dest_path, _exc):
         logger.debug(
-            "failed to create '%s' from '%s'", dest_path, src_path, exc_info=True
+            "failed to create '%s' from '%s'",
+            dest_path,
+            src_path,
+            exc_info=True,  # noqa: LOG014
         )
 
         for out_path in out_paths:

dvc/repo/data.py

@@ -1,38 +1,34 @@
 import os
 import posixpath
-from collections.abc import Iterable
-from typing import TYPE_CHECKING, Any, TypedDict, Union
+from collections import defaultdict
+from collections.abc import Iterable, Iterator, Mapping
+from typing import TYPE_CHECKING, Optional, TypedDict, Union
 
-from dvc.fs.callbacks import DEFAULT_CALLBACK
+from dvc.fs.callbacks import DEFAULT_CALLBACK, Callback, TqdmCallback
+from dvc.log import logger
 from dvc.ui import ui
+from dvc_data.index import DataIndexDirError
 
 if TYPE_CHECKING:
-    from dvc.fs.callbacks import Callback
     from dvc.repo import Repo
     from dvc.scm import Git, NoSCM
-    from dvc_data.index import DataIndex
+    from dvc_data.index import (
+        BaseDataIndex,
+        DataIndex,
+        DataIndexEntry,
+        DataIndexKey,
+        DataIndexView,
+    )
     from dvc_data.index.diff import Change
+    from dvc_objects.fs.base import FileSystem
+
+logger = logger.getChild(__name__)
 
 
 def posixpath_to_os_path(path: str) -> str:
     return path.replace(posixpath.sep, os.path.sep)
 
 
-def _adapt_typ(typ: str) -> str:
-    from dvc_data.index.diff import ADD, DELETE, MODIFY
-
-    if typ == MODIFY:
-        return "modified"
-
-    if typ == ADD:
-        return "added"
-
-    if typ == DELETE:
-        return "deleted"
-
-    return typ
-
-
 def _adapt_path(change: "Change") -> str:
     isdir = False
     if change.new and change.new.meta:
@@ -45,27 +41,65 @@ def _adapt_path(change: "Change") -> str:
     return os.path.sep.join(key)
 
 
+def _get_missing_paths(
+    to_check: Mapping["FileSystem", Mapping[str, Iterable["DataIndexEntry"]]],
+    batch_size: Optional[int] = None,
+    callback: "Callback" = DEFAULT_CALLBACK,
+) -> Iterator[str]:
+    for fs, paths_map in to_check.items():
+        if batch_size == 1 or (batch_size is None and fs.protocol == "local"):
+            results = list(callback.wrap(map(fs.exists, paths_map)))
+        else:
+            results = fs.exists(
+                list(paths_map), batch_size=batch_size, callback=callback
+            )
+
+        for cache_path, exists in zip(paths_map, results):
+            if exists:
+                continue
+
+            for entry in paths_map[cache_path]:
+                key = entry.key
+                assert key
+                if entry.meta and entry.meta.isdir:
+                    key = (*key, "")
+                yield os.path.sep.join(key)
+
+
+class StorageCallback(Callback):
+    def __init__(self, parent_cb: Callback) -> None:
+        super().__init__(size=0, value=0)
+        self.parent_cb = parent_cb
+
+    def set_size(self, size: int) -> None:
+        # This is a no-op to prevent `fs.exists` from trying to set the size
+        pass
+
+    def relative_update(self, value: int = 1) -> None:
+        self.parent_cb.relative_update(value)
+
+    def absolute_update(self, value: int) -> None:
+        self.parent_cb.relative_update(value - self.value)
+
+
 def _diff(
-    old: "DataIndex",
-    new: "DataIndex",
+    old: "BaseDataIndex",
+    new: "BaseDataIndex",
     *,
+    filter_keys: Optional[Iterable["DataIndexKey"]] = None,
     granular: bool = False,
     not_in_cache: bool = False,
-    not_in_remote: bool = False,
-    remote_refresh: bool = False,
+    batch_size: Optional[int] = None,
     callback: "Callback" = DEFAULT_CALLBACK,
 ) -> dict[str, list[str]]:
-    from dvc_data.index import StorageError
-    from dvc_data.index.diff import UNCHANGED, UNKNOWN, diff
+    from dvc_data.index.diff import ADD, DELETE, MODIFY, UNCHANGED, UNKNOWN, diff
 
-    ret: dict[str, list[str]] = {}
+    ret: dict[str, list[str]] = defaultdict(list)
+    change_types = {MODIFY: "modified", ADD: "added", DELETE: "deleted"}
 
-    def _add_change(typ, change):
-        typ = _adapt_typ(typ)
-        if typ not in ret:
-            ret[typ] = []
-
-        ret[typ].append(_adapt_path(change))
+    to_check: dict[FileSystem, dict[str, list[DataIndexEntry]]] = defaultdict(
+        lambda: defaultdict(list)
+    )
 
     for change in diff(
         old,
@@ -76,6 +110,14 @@ def _diff(
         with_unknown=True,
         callback=callback,
     ):
+        # The index is a trie, so even when we filter by a specific path
+        # like `dir/file`, all parent nodes leading to that path (e.g., `dir/`)
+        # still appear in the view. As a result, keys like `dir/` will be present
+        # even if only `dir/file` matches the filter.
+        # We need to skip such entries to avoid showing root of tracked directories.
+        if filter_keys and not any(change.key[: len(fk)] == fk for fk in filter_keys):
+            continue
+
         if (
             change.typ == UNCHANGED
             and (not change.old or not change.old.hash_info)
@@ -88,31 +130,27 @@ def _diff(
             # NOTE: emulating previous behaviour
             continue
 
-        if (
-            not_in_cache
-            and change.old
-            and change.old.hash_info
-            and not old.storage_map.cache_exists(change.old)
-        ):
-            # NOTE: emulating previous behaviour
-            _add_change("not_in_cache", change)
+        if not_in_cache and change.old and change.old.hash_info:
+            old_entry = change.old
+            cache_fs, cache_path = old.storage_map.get_cache(old_entry)
+            # check later in batches
+            to_check[cache_fs][cache_path].append(old_entry)
 
-        try:
-            if (
-                not_in_remote
-                and change.old
-                and change.old.hash_info
-                and not old.storage_map.remote_exists(
-                    change.old, refresh=remote_refresh
-                )
-            ):
-                _add_change("not_in_remote", change)
-        except StorageError:
-            pass
+        change_typ = change_types.get(change.typ, change.typ)
+        ret[change_typ].append(_adapt_path(change))
 
-        _add_change(change.typ, change)
-
-    return ret
+    total_items = sum(
+        len(entries) for paths in to_check.values() for entries in paths.values()
+    )
+    with TqdmCallback(size=total_items, desc="Checking cache", unit="entry") as cb:
+        missing_items = list(
+            _get_missing_paths(
+                to_check, batch_size=batch_size, callback=StorageCallback(cb)
+            ),
+        )
+        if missing_items:
+            ret["not_in_cache"] = missing_items
+    return dict(ret)
 
 
 class GitInfo(TypedDict, total=False):
@@ -151,7 +189,48 @@ def _git_info(scm: Union["Git", "NoSCM"], untracked_files: str = "all") -> GitIn
     )
 
 
-def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> dict[str, list[str]]:
+def filter_index(
+    index: Union["DataIndex", "DataIndexView"],
+    filter_keys: Optional[Iterable["DataIndexKey"]] = None,
+) -> "BaseDataIndex":
+    from dvc_data.index.view import DataIndexView
+
+    if not filter_keys:
+        return index
+
+    if isinstance(index, DataIndexView):
+        orig_index = index._index
+        parent_filter_fn = index.filter_fn
+    else:
+        orig_index = index
+        parent_filter_fn = None
+
+    def filter_fn(key: "DataIndexKey") -> bool:
+        if parent_filter_fn is not None and not parent_filter_fn(key):
+            return False
+
+        for filter_key in filter_keys:
+            # eg: if key is "dir/file" and filter_key is "dir/", return True
+            if key[: len(filter_key)] == filter_key:
+                return True
+            # eg: if key is `dir/` and filter_key is `dir/file`, also return True.
+            # This ensures we include parent prefixes needed to reach matching leaves.
+            # Intermediate prefixes must be retained to access nested keys.
+            if filter_key[: len(key)] == key:
+                return True
+        return False
+
+    from dvc_data.index import view
+
+    return view(orig_index, filter_fn=filter_fn)
+
+
+def _diff_index_to_wtree(
+    repo: "Repo",
+    filter_keys: Optional[Iterable["DataIndexKey"]] = None,
+    granular: bool = False,
+    batch_size: Optional[int] = None,
+) -> dict[str, list[str]]:
     from .index import build_data_index
 
     with ui.progress(desc="Building workspace index", unit="entry") as pb:
@@ -162,30 +241,53 @@ def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> dict[str, list[str]]:
             compute_hash=True,
             callback=pb.as_callback(),
         )
+        workspace_view = filter_index(workspace, filter_keys=filter_keys)
 
     with ui.progress(
         desc="Calculating diff between index/workspace",
         unit="entry",
     ) as pb:
+        index = repo.index.data["repo"]
+        view = filter_index(index, filter_keys=filter_keys)
         return _diff(
-            repo.index.data["repo"],
-            workspace,
+            view,
+            workspace_view,
+            filter_keys=filter_keys,
+            granular=granular,
             not_in_cache=True,
+            batch_size=batch_size,
             callback=pb.as_callback(),
-            **kwargs,
         )
 
 
 def _diff_head_to_index(
-    repo: "Repo", head: str = "HEAD", **kwargs: Any
+    repo: "Repo",
+    head: str = "HEAD",
+    filter_keys: Optional[Iterable["DataIndexKey"]] = None,
+    granular: bool = False,
 ) -> dict[str, list[str]]:
-    index = repo.index.data["repo"]
+    from dvc.scm import RevError
+    from dvc_data.index import DataIndex
 
-    with repo.switch(head):
-        head_index = repo.index.data["repo"]
+    index = repo.index.data["repo"]
+    index_view = filter_index(index, filter_keys=filter_keys)
+
+    try:
+        with repo.switch(head):
+            head_index = repo.index.data["repo"]
+            head_view = filter_index(head_index, filter_keys=filter_keys)
+    except RevError:
+        logger.debug("failed to switch to '%s'", head)
+        head_view = DataIndex()
 
     with ui.progress(desc="Calculating diff between head/index", unit="entry") as pb:
-        return _diff(head_index, index, callback=pb.as_callback(), **kwargs)
+        return _diff(
+            head_view,
+            index_view,
+            filter_keys=filter_keys,
+            granular=granular,
+            callback=pb.as_callback(),
+        )
 
 
 class Status(TypedDict):
@@ -217,27 +319,140 @@ def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> list[str]
     return [repo.fs.relpath(file, start) for file in files]
 
 
-def status(repo: "Repo", untracked_files: str = "no", **kwargs: Any) -> Status:
+def _get_entries_not_in_remote(
+    repo: "Repo",
+    filter_keys: Optional[Iterable["DataIndexKey"]] = None,
+    granular: bool = False,
+    remote_refresh: bool = False,
+) -> list[str]:
+    """Get entries that are not in remote storage."""
+    from dvc.repo.worktree import worktree_view
+    from dvc_data.index import StorageKeyError
+
+    entries: dict[DataIndexKey, DataIndexEntry] = {}
+
+    def _onerror(entry, exc):
+        if not isinstance(exc, DataIndexDirError):
+            raise exc
+        # We don't have the contents of this dir file, so we will only check this key.
+        entries[entry.key] = entry
+
+    # View into the index, with only pushable entries
+    index = worktree_view(repo.index, push=True)
+    data_index = index.data["repo"]
+
+    orig_data_index_onerror = data_index.onerror
+    data_index.onerror = _onerror
+
+    view = filter_index(data_index, filter_keys=filter_keys)  # type: ignore[arg-type]
+
+    missing_entries = []
+
+    storage_map = view.storage_map
+
+    n = 0
+    with TqdmCallback(size=n, desc="Checking remote", unit="entry") as cb:
+        for key, entry in view.iteritems(shallow=not granular):
+            if not (entry and entry.hash_info):
+                continue
+
+            # The index is a trie, so even when we filter by a specific path
+            # like `dir/file`, all parent nodes leading to that path (e.g., `dir/`)
+            # still appear in the view. As a result, keys like `dir/` will be present
+            # even if only `dir/file` matches the filter.
+            # We need to skip such entries to avoid showing root of tracked directories.
+            if filter_keys and not any(
+                key[: len(filter_key)] == filter_key for filter_key in filter_keys
+            ):
+                continue
+
+            entries[key] = entry
+            n += 1
+            cb.set_size(n)
+
+        for key, entry in entries.items():
+            k = (*key, "") if entry.meta and entry.meta.isdir else key
+            try:
+                if not storage_map.remote_exists(entry, refresh=remote_refresh):
+                    missing_entries.append(os.path.sep.join(k))
+            except StorageKeyError:
+                pass
+            finally:
+                cb.relative_update()
+    data_index.onerror = orig_data_index_onerror
+    return missing_entries
+
+
+def _matches_target(p: str, targets: Iterable[str]) -> bool:
+    sep = os.sep
+    return any(p == t or p.startswith(t + sep) for t in targets)
+
+
+def _prune_keys(filter_keys: Iterable["DataIndexKey"]) -> list["DataIndexKey"]:
+    sorted_keys = sorted(set(filter_keys), key=len)
+    result: list[DataIndexKey] = []
+
+    for key in sorted_keys:
+        if not any(key[: len(prefix)] == prefix for prefix in result):
+            result.append(key)
+    return result
+
+
+def status(
+    repo: "Repo",
+    targets: Optional[Iterable[Union[os.PathLike[str], str]]] = None,
+    *,
+    granular: bool = False,
+    untracked_files: str = "no",
+    not_in_remote: bool = False,
+    remote_refresh: bool = False,
+    batch_size: Optional[int] = None,
+    head: str = "HEAD",
+) -> Status:
     from dvc.scm import NoSCMError, SCMError
 
-    head = kwargs.pop("head", "HEAD")
-    uncommitted_diff = _diff_index_to_wtree(repo, **kwargs)
+    targets = targets or []
+    filter_keys: list[DataIndexKey] = [repo.fs.relparts(os.fspath(t)) for t in targets]
+    # try to remove duplicate and overlapping keys
+    filter_keys = _prune_keys(filter_keys)
+
+    uncommitted_diff = _diff_index_to_wtree(
+        repo, filter_keys=filter_keys, granular=granular, batch_size=batch_size
+    )
     unchanged = set(uncommitted_diff.pop("unchanged", []))
 
+    entries_not_in_remote: list[str] = []
+    if not_in_remote:
+        entries_not_in_remote = _get_entries_not_in_remote(
+            repo,
+            filter_keys=filter_keys,
+            granular=granular,
+            remote_refresh=remote_refresh,
+        )
+
     try:
-        committed_diff = _diff_head_to_index(repo, head=head, **kwargs)
+        committed_diff = _diff_head_to_index(
+            repo, filter_keys=filter_keys, head=head, granular=granular
+        )
     except (SCMError, NoSCMError):
         committed_diff = {}
     else:
         unchanged &= set(committed_diff.pop("unchanged", []))
 
     git_info = _git_info(repo.scm, untracked_files=untracked_files)
-    untracked = git_info.get("untracked", [])
-    untracked = _transform_git_paths_to_dvc(repo, untracked)
+    scm_filter_targets = {
+        os.path.relpath(os.path.abspath(t), repo.scm.root_dir) for t in targets
+    }
+    untracked_it: Iterable[str] = git_info.get("untracked", [])
+    if scm_filter_targets:
+        untracked_it = (
+            f for f in untracked_it if _matches_target(f, scm_filter_targets)
+        )
+    untracked = _transform_git_paths_to_dvc(repo, untracked_it)
     # order matters here
     return Status(
         not_in_cache=uncommitted_diff.pop("not_in_cache", []),
-        not_in_remote=uncommitted_diff.pop("not_in_remote", []),
+        not_in_remote=entries_not_in_remote,
         committed=committed_diff,
         uncommitted=uncommitted_diff,
         untracked=untracked,

dvc/repo/datasets.py

@@ -16,7 +16,7 @@ from dvc.types import StrPath
 from dvc_data.hashfile.meta import Meta
 
 if TYPE_CHECKING:
-    from dvcx.dataset import DatasetRecord, DatasetVersion  # type: ignore[import]
+    from datachain.dataset import DatasetRecord, DatasetVersion  # type: ignore[import]
     from typing_extensions import Self
 
     from dvc.repo import Repo
@@ -29,10 +29,10 @@ def _get_dataset_record(name: str) -> "DatasetRecord":
     from dvc.exceptions import DvcException
 
     try:
-        from dvcx.catalog import get_catalog  # type: ignore[import]
+        from datachain.catalog import get_catalog  # type: ignore[import]
 
     except ImportError as exc:
-        raise DvcException("dvcx is not installed") from exc
+        raise DvcException("datachain is not installed") from exc
 
     catalog = get_catalog()
     return catalog.get_remote_dataset(name)
@@ -76,7 +76,7 @@ class SerDe:
         return asdict(self, filter=filter_defaults, value_serializer=value_serializer)
 
     @classmethod
-    def from_dict(cls: type["Self"], d: dict[str, Any]) -> "Self":
+    def from_dict(cls, d: dict[str, Any]) -> "Self":
         _fields = fields(cast("type[AttrsInstance]", cls))
         kwargs = {f.name: d[f.name] for f in _fields if f.name in d}
         return cls(**kwargs)
@@ -86,7 +86,7 @@ class SerDe:
 class DatasetSpec(SerDe):
     name: str
     url: str
-    type: Literal["dvc", "dvcx", "url"]
+    type: Literal["dvc", "dc", "url"]
 
 
 @frozen(kw_only=True)
@@ -108,7 +108,7 @@ class DVCDatasetLock(DVCDatasetSpec):
 
 
 @frozen(kw_only=True)
-class DVCXDatasetLock(DatasetSpec):
+class DatachainDatasetLock(DatasetSpec):
     version: int
     created_at: datetime = field(converter=to_datetime)
 
@@ -160,13 +160,13 @@ class DVCDataset:
 
 
 @frozen(kw_only=True)
-class DVCXDataset:
+class DatachainDataset:
     manifest_path: str
     spec: "DatasetSpec"
-    lock: "Optional[DVCXDatasetLock]" = field(default=None)
+    lock: "Optional[DatachainDatasetLock]" = field(default=None)
     _invalidated: bool = field(default=False, eq=False, repr=False)
 
-    type: ClassVar[Literal["dvcx"]] = "dvcx"
+    type: ClassVar[Literal["dc"]] = "dc"
 
     @property
     def pinned(self) -> bool:
@@ -193,7 +193,7 @@ class DVCXDataset:
         name, _version = self.name_version
         version = version if version is not None else _version
         version_info = _get_dataset_info(name, record=record, version=version)
-        lock = DVCXDatasetLock(
+        lock = DatachainDatasetLock(
             **self.spec.to_dict(),
             version=version_info.version,
             created_at=version_info.created_at,
@@ -226,9 +226,9 @@ class URLDataset:
         return evolve(self, lock=lock)
 
 
-Lock = Union[DVCDatasetLock, DVCXDatasetLock, URLDatasetLock]
+Lock = Union[DVCDatasetLock, DatachainDatasetLock, URLDatasetLock]
 Spec = Union[DatasetSpec, DVCDatasetSpec]
-Dataset = Union[DVCDataset, DVCXDataset, URLDataset]
+Dataset = Union[DVCDataset, DatachainDataset, URLDataset]
 
 
 class DatasetNotFoundError(DvcException, KeyError):
@@ -307,13 +307,13 @@ class Datasets(Mapping[str, Dataset]):
             raise ValueError("type should be present in spec")
         if typ == "dvc":
             return DVCDatasetSpec.from_dict(spec)
-        if typ in {"dvcx", "url"}:
+        if typ in {"dc", "url"}:
             return DatasetSpec.from_dict(spec)
         raise ValueError(f"unknown dataset type: {spec.get('type', '')}")
 
     @staticmethod
     def _lock_from_info(lock: Optional[dict[str, Any]]) -> Optional[Lock]:
-        kl = {"dvc": DVCDatasetLock, "dvcx": DVCXDatasetLock, "url": URLDatasetLock}
+        kl = {"dvc": DVCDatasetLock, "dc": DatachainDatasetLock, "url": URLDatasetLock}
         if lock and (cls := kl.get(lock.get("type", ""))):  # type: ignore[assignment]
             return cls.from_dict(lock)  # type: ignore[attr-defined]
         return None
@@ -356,9 +356,9 @@ class Datasets(Mapping[str, Dataset]):
                 lock=lock,
                 invalidated=_invalidated,
             )
-        if spec.type == "dvcx":
-            assert lock is None or isinstance(lock, DVCXDatasetLock)
-            return DVCXDataset(
+        if spec.type == "dc":
+            assert lock is None or isinstance(lock, DatachainDatasetLock)
+            return DatachainDataset(
                 manifest_path=manifest_path,
                 spec=spec,
                 lock=lock,
@@ -374,7 +374,7 @@ class Datasets(Mapping[str, Dataset]):
         manifest_path: StrPath = "dvc.yaml",
         **kwargs: Any,
     ) -> Dataset:
-        assert type in {"dvc", "dvcx", "url"}
+        assert type in {"dvc", "dc", "url"}
         kwargs.update({"name": name, "url": url, "type": type})
         dataset = self._build_dataset(os.path.abspath(manifest_path), kwargs)
         dataset = dataset.update(self.repo)
@@ -389,13 +389,16 @@ class Datasets(Mapping[str, Dataset]):
 
         if dataset.type == "url" and (version or kwargs.get("rev")):
             raise ValueError("cannot update version/revision for a url")
-        if dataset.type == "dvcx" and version is not None:
+        if dataset.type == "dc" and version is not None:
             if not isinstance(version, int):
                 raise TypeError(
-                    f"dvcx version has to be an integer, got {type(version).__name__!r}"
+                    "DataChain dataset version has to be an integer, "
+                    f"got {type(version).__name__!r}"
                 )
             if version < 1:
-                raise ValueError(f"dvcx version should be >=1, got {version}")
+                raise ValueError(
+                    f"DataChain dataset version should be >=1, got {version}"
+                )
 
         new = dataset.update(self.repo, **kwargs)
 

dvc/repo/du.py

@@ -6,7 +6,7 @@ def du(
     path: Optional[str] = None,
     rev: Optional[str] = None,
     summarize: bool = False,
-    config: Union[None, dict[str, Any], str] = None,
+    config: Union[dict[str, Any], str, None] = None,
     remote: Optional[str] = None,
     remote_config: Optional[dict] = None,
 ):

dvc/repo/experiments/__init__.py

@@ -205,7 +205,7 @@ class Experiments:
         Experiment will be derived from the current workspace.
         """
 
-        name = kwargs.get("name", None)
+        name = kwargs.get("name")
         baseline_sha = kwargs.get("baseline_rev") or self.repo.scm.get_rev()
 
         if name:
@@ -249,7 +249,7 @@ class Experiments:
         exec_results = queue.reproduce(copy_paths=copy_paths, message=message)
 
         results: dict[str, str] = {}
-        for _, exp_result in exec_results.items():
+        for exp_result in exec_results.values():
             results.update(exp_result)
         return results
 

dvc/repo/experiments/collect.py

@@ -274,6 +274,7 @@ def collect(
     num: int = 1,
     hide_queued: bool = False,
     hide_failed: bool = False,
+    hide_workspace: bool = False,
     sha_only: bool = False,
     **kwargs,
 ) -> list["ExpState"]:
@@ -303,8 +304,11 @@ def collect(
             repo.scm, baseline_revs, refs=cached_refs, logger=logger
         )
 
-    workspace_data = collect_rev(repo, "workspace", **kwargs)
-    result: list[ExpState] = [workspace_data]
+    result: list[ExpState] = []
+    if not hide_workspace:
+        workspace_data = collect_rev(repo, "workspace", **kwargs)
+        result.append(workspace_data)
+
     queued = collect_queued(repo, baseline_revs, **kwargs) if not hide_queued else {}
     active = collect_active(repo, baseline_revs, **kwargs)
     failed = collect_failed(repo, baseline_revs, **kwargs) if not hide_failed else {}

dvc/repo/experiments/executor/base.py

@@ -1,20 +1,21 @@
-import logging
 import os
 import pickle
 import shutil
 from abc import ABC, abstractmethod
 from collections.abc import Iterable, Iterator
-from contextlib import contextmanager
+from contextlib import AbstractContextManager, contextmanager
 from dataclasses import asdict, dataclass
 from enum import IntEnum
 from itertools import chain
 from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, Union
 
+from funcy import nullcontext
 from scmrepo.exceptions import SCMError
 
 from dvc.env import DVC_EXP_AUTO_PUSH, DVC_EXP_GIT_REMOTE
 from dvc.exceptions import DvcException
 from dvc.log import logger
+from dvc.logger import set_loggers_level
 from dvc.repo.experiments.exceptions import ExperimentExistsError
 from dvc.repo.experiments.refs import EXEC_BASELINE, EXEC_BRANCH, ExpRefInfo
 from dvc.repo.experiments.utils import to_studio_params
@@ -288,7 +289,7 @@ class BaseExecutor(ABC):
             stages = []
             if targets:
                 for target in targets:
-                    stages.append(
+                    stages.append(  # noqa: PERF401
                         dvc.commit(
                             target, recursive=recursive, force=True, relink=False
                         )
@@ -297,7 +298,9 @@ class BaseExecutor(ABC):
                 stages = dvc.commit([], recursive=recursive, force=True, relink=False)
             exp_hash = cls.hash_exp(stages)
             if include_untracked:
-                dvc.scm.add(include_untracked, force=True)  # type: ignore[call-arg]
+                from dvc.scm import add_no_submodules
+
+                add_no_submodules(dvc.scm, include_untracked, force=True)  # type: ignore[call-arg]
 
             with cls.auto_push(dvc):
                 cls.commit(
@@ -466,8 +469,8 @@ class BaseExecutor(ABC):
 
         if queue is not None:
             queue.put((rev, os.getpid()))
-        if log_errors and log_level is not None:
-            cls._set_log_level(log_level)
+
+        log_ctx = cls._set_log_level(log_level) if log_errors else nullcontext()
 
         exp_hash: Optional[str] = None
         exp_ref: Optional[ExpRefInfo] = None
@@ -476,14 +479,17 @@ class BaseExecutor(ABC):
         if info.name:
             ui.write(f"Reproducing experiment '{info.name}'")
 
-        with cls._repro_dvc(
-            info,
-            infofile,
-            log_errors=log_errors,
-            copy_paths=copy_paths,
-            message=message,
-            **kwargs,
-        ) as dvc:
+        with (
+            log_ctx,
+            cls._repro_dvc(
+                info,
+                infofile,
+                log_errors=log_errors,
+                copy_paths=copy_paths,
+                message=message,
+                **kwargs,
+            ) as dvc,
+        ):
             args, kwargs = cls._repro_args(dvc)
             if args:
                 targets: Optional[Union[list, str]] = args[0]
@@ -509,7 +515,9 @@ class BaseExecutor(ABC):
             stages = dvc.reproduce(*args, **kwargs)
             if paths := cls._get_top_level_paths(dvc):
                 logger.debug("Staging top-level files: %s", paths)
-                dvc.scm_context.add(paths)
+                from dvc.scm import add_no_submodules
+
+                add_no_submodules(dvc.scm, paths)
 
             exp_hash = cls.hash_exp(stages)
             if not repro_dry:
@@ -786,13 +794,10 @@ class BaseExecutor(ABC):
         return orig_rev
 
     @staticmethod
-    def _set_log_level(level):
-        # When executor.reproduce is run in a multiprocessing child process,
-        # dvc.cli.main will not be called for that child process so we need to
-        # setup logging ourselves
-        dvc_logger = logging.getLogger("dvc")
+    def _set_log_level(level: Optional[int]) -> AbstractContextManager[None]:
         if level is not None:
-            dvc_logger.setLevel(level)
+            return set_loggers_level(level)
+        return nullcontext()
 
     @staticmethod
     def _copy_path(src, dst):

dvc/repo/experiments/pull.py

@@ -55,7 +55,7 @@ def pull(  # noqa: C901
         rev_dict = iter_revs(repo.scm, rev, num)
         rev_set = set(rev_dict.keys())
         ref_info_dict = exp_refs_by_baseline(repo.scm, rev_set, git_remote)
-        for _, ref_info_list in ref_info_dict.items():
+        for ref_info_list in ref_info_dict.values():
             exp_ref_set.update(ref_info_list)
 
     pull_result = _pull(repo, git_remote, exp_ref_set, force)

dvc/repo/experiments/push.py

@@ -78,7 +78,7 @@ def exp_refs_from_rev(scm: "Git", rev: list[str], num: int = 1) -> set["ExpRefIn
     rev_dict = iter_revs(scm, rev, num)
     rev_set = set(rev_dict.keys())
     ref_info_dict = exp_refs_by_baseline(scm, rev_set)
-    for _, ref_info_list in ref_info_dict.items():
+    for ref_info_list in ref_info_dict.values():
         exp_ref_set.update(ref_info_list)
     return exp_ref_set
 

dvc/repo/experiments/queue/base.py

@@ -38,7 +38,7 @@ logger = logger.getChild(__name__)
 
 
 @dataclass(frozen=True)
-class QueueEntry:
+class QueueEntry:  # noqa: PLW1641
     dvc_root: str
     scm_root: str
     stash_ref: str
@@ -372,7 +372,7 @@ class BaseStashQueue(ABC):
         )
 
     def _stash_commit_deps(self, *args, **kwargs):
-        if len(args):
+        if args:
             targets = args[0]
         else:
             targets = kwargs.get("targets")
@@ -609,9 +609,9 @@ class BaseStashQueue(ABC):
                 result[exp_name] = entry_name_dict[exp_name]
                 continue
             if self.scm.is_sha(exp_name):
-                for rev in entry_rev_dict:
+                for rev, entry in entry_rev_dict.items():
                     if rev.startswith(exp_name.lower()):
-                        result[exp_name] = entry_rev_dict[rev]
+                        result[exp_name] = entry
                         break
 
         return result

dvc/repo/experiments/queue/celery.py

@@ -1,3 +1,4 @@
+import glob
 import hashlib
 import locale
 import logging
@@ -323,7 +324,7 @@ class LocalCeleryQueue(BaseStashQueue):
     def _get_running_task_ids(self) -> set[str]:
         running_task_ids: set[str] = set()
         active_workers = self.worker_status()
-        for _, tasks in active_workers.items():
+        for tasks in active_workers.values():
             task = first(tasks)
             if task:
                 running_task_ids.add(task["id"])
@@ -434,9 +435,13 @@ class LocalCeleryQueue(BaseStashQueue):
         try:
             proc_info = self.proc[queue_entry.stash_rev]
         except KeyError:
-            raise DvcException(  # noqa: B904
-                f"No output logs found for experiment '{rev}'"
-            )
+            message = f"No output logs found for experiment '{rev}'"
+            if self.match_queue_entry_by_name({rev}, self.iter_failed()).get(rev):
+                message += "\nExperiment likely failed during setup."
+                if celery_logs := glob.glob(os.path.join(self.wdir, "*.out")):
+                    message += " Check the celery logs for more details:"
+                    message += f"\n\tcat {' '.join(celery_logs)}".expandtabs(4)
+            raise DvcException(message)  # noqa: B904
         with open(
             proc_info.stdout, encoding=encoding or locale.getpreferredencoding()
         ) as fobj:

dvc/repo/experiments/queue/utils.py

@@ -25,7 +25,7 @@ def get_remote_executor_refs(scm: "Git", remote_url: str) -> list[str]:
     refs = []
     for ref in iter_remote_refs(scm, remote_url, base=EXPS_NAMESPACE):
         if not ref.startswith(EXEC_NAMESPACE) and ref != EXPS_STASH:
-            refs.append(ref)
+            refs.append(ref)  # noqa: PERF401
     return refs
 
 

dvc/repo/experiments/refs.py

@@ -67,3 +67,12 @@ class ExpRefInfo:
         baseline_sha = parts[2] + parts[3]
         name = parts[4] if len(parts) == 5 else None
         return cls(baseline_sha, name)
+
+    def __eq__(self, other):
+        if not isinstance(other, ExpRefInfo):
+            return False
+
+        return self.baseline_sha == other.baseline_sha and self.name == other.name
+
+    def __hash__(self):
+        return hash((self.baseline_sha, self.name))

dvc/repo/experiments/remove.py

@@ -6,7 +6,7 @@ from dvc.repo import locked
 from dvc.repo.scm_context import scm_context
 from dvc.scm import Git, iter_revs
 
-from .exceptions import UnresolvedExpNamesError
+from .exceptions import InvalidArgumentError, UnresolvedExpNamesError
 from .utils import exp_refs, exp_refs_by_baseline, push_refspec
 
 if TYPE_CHECKING:
@@ -24,16 +24,22 @@ logger = logger.getChild(__name__)
 @scm_context
 def remove(  # noqa: C901, PLR0912
     repo: "Repo",
-    exp_names: Union[None, str, list[str]] = None,
+    exp_names: Union[str, list[str], None] = None,
     rev: Optional[Union[list[str], str]] = None,
     all_commits: bool = False,
     num: int = 1,
     queue: bool = False,
     git_remote: Optional[str] = None,
+    keep: bool = False,
 ) -> list[str]:
     removed: list[str] = []
+
+    if all([keep, queue]):
+        raise InvalidArgumentError("Cannot use both `--keep` and `--queue`.")
+
     if not any([exp_names, queue, all_commits, rev]):
         return removed
+
     celery_queue: LocalCeleryQueue = repo.experiments.celery_queue
 
     if queue:
@@ -43,6 +49,7 @@ def remove(  # noqa: C901, PLR0912
 
     exp_ref_list: list[ExpRefInfo] = []
     queue_entry_list: list[QueueEntry] = []
+
     if exp_names:
         results: dict[str, ExpRefAndQueueEntry] = (
             celery_queue.get_ref_and_entry_by_names(exp_names, git_remote)
@@ -68,6 +75,10 @@ def remove(  # noqa: C901, PLR0912
         exp_ref_list.extend(exp_ref_dict.values())
     elif all_commits:
         exp_ref_list.extend(exp_refs(repo.scm, git_remote))
+        removed.extend([ref.name for ref in exp_ref_list])
+
+    if keep:
+        exp_ref_list = list(set(exp_refs(repo.scm, git_remote)) - set(exp_ref_list))
         removed = [ref.name for ref in exp_ref_list]
 
     if exp_ref_list:
@@ -83,6 +94,7 @@ def remove(  # noqa: C901, PLR0912
 
         removed_refs = [str(r) for r in exp_ref_list]
         notify_refs_to_studio(repo, git_remote, removed=removed_refs)
+
     return removed
 
 
@@ -98,7 +110,7 @@ def _resolve_exp_by_baseline(
     rev_dict = iter_revs(repo.scm, rev, num)
     rev_set = set(rev_dict.keys())
     ref_info_dict = exp_refs_by_baseline(repo.scm, rev_set, git_remote)
-    for _, ref_info_list in ref_info_dict.items():
+    for ref_info_list in ref_info_dict.values():
         for ref_info in ref_info_list:
             commit_ref_dict[ref_info.name] = ref_info
     return commit_ref_dict

dvc/repo/experiments/run.py

@@ -93,7 +93,7 @@ def run(  # noqa: C901, PLR0912
 
     for idx, sweep_overrides in enumerate(sweeps):
         if hydra_sweep and name_prefix is not None:
-            kwargs["name"] = f"{name_prefix}-{idx+1}"
+            kwargs["name"] = f"{name_prefix}-{idx + 1}"
         queue_entry = repo.experiments.queue_one(
             repo.experiments.celery_queue,
             targets=targets,

dvc/repo/experiments/show.py

@@ -30,6 +30,7 @@ def show(
     num: int = 1,
     hide_queued: bool = False,
     hide_failed: bool = False,
+    hide_workspace: bool = False,
     sha_only: bool = False,
     **kwargs,
 ) -> list["ExpState"]:
@@ -42,6 +43,7 @@ def show(
         num=num,
         hide_queued=hide_queued,
         hide_failed=hide_failed,
+        hide_workspace=hide_workspace,
         sha_only=sha_only,
         **kwargs,
     )

dvc/repo/fetch.py

@@ -10,6 +10,7 @@ from . import locked
 
 if TYPE_CHECKING:
     from dvc.output import Output
+    from dvc.repo import Repo
     from dvc.stage import Stage
 
 logger = logger.getChild(__name__)
@@ -24,7 +25,7 @@ def _make_index_onerror(onerror, rev):
 
 
 def _collect_indexes(  # noqa: PLR0913
-    repo,
+    repo: "Repo",
     targets=None,
     remote=None,
     all_branches=False,
@@ -94,7 +95,7 @@ def _collect_indexes(  # noqa: PLR0913
 
 @locked
 def fetch(  # noqa: PLR0913
-    self,
+    self: "Repo",
     targets=None,
     jobs=None,
     remote=None,

dvc/repo/install.py

@@ -1,5 +1,6 @@
 from typing import TYPE_CHECKING
 
+from dvc import version_tuple
 from dvc.exceptions import DvcException
 
 if TYPE_CHECKING:
@@ -16,19 +17,19 @@ def pre_commit_install(scm: "Git") -> None:
     with modify_yaml(config_path) as config:
         entry = {
             "repo": "https://github.com/iterative/dvc",
-            "rev": "main",
+            "rev": ".".join(map(str, version_tuple[:3])),
             "hooks": [
                 {
                     "id": "dvc-pre-commit",
                     "additional_dependencies": [".[all]"],
                     "language_version": "python3",
-                    "stages": ["commit"],
+                    "stages": ["pre-commit"],
                 },
                 {
                     "id": "dvc-pre-push",
                     "additional_dependencies": [".[all]"],
                     "language_version": "python3",
-                    "stages": ["push"],
+                    "stages": ["pre-push"],
                 },
                 {
                     "id": "dvc-post-checkout",

dvc/repo/ls.py

@@ -4,8 +4,44 @@ from typing import TYPE_CHECKING, Any, Optional, Union
 if TYPE_CHECKING:
     from dvc.fs.dvc import DVCFileSystem
 
+
+def _open_repo(
+    url: str,
+    rev: Optional[str] = None,
+    config: Union[dict[str, Any], str, None] = None,
+    remote: Optional[str] = None,
+    remote_config: Optional[dict] = None,
+):
+    from dvc.config import Config
+
     from . import Repo
 
+    if config and not isinstance(config, dict):
+        config_dict = Config.load_file(config)
+    else:
+        config_dict = None
+
+    return Repo.open(
+        url,
+        rev=rev,
+        subrepos=True,
+        uninitialized=True,
+        config=config_dict,
+        remote=remote,
+        remote_config=remote_config,
+    )
+
+
+def _adapt_info(info: dict[str, Any]) -> dict[str, Any]:
+    dvc_info = info.get("dvc_info", {})
+    return {
+        "isout": dvc_info.get("isout", False),
+        "isdir": info["type"] == "directory",
+        "isexec": info.get("isexec", False),
+        "size": info.get("size"),
+        "md5": dvc_info.get("md5") or dvc_info.get("md5-dos2unix"),
+    }
+
 
 def ls(
     url: str,
@@ -13,9 +49,10 @@ def ls(
     rev: Optional[str] = None,
     recursive: Optional[bool] = None,
     dvc_only: bool = False,
-    config: Union[None, dict[str, Any], str] = None,
+    config: Union[dict[str, Any], str, None] = None,
     remote: Optional[str] = None,
     remote_config: Optional[dict] = None,
+    maxdepth: Optional[int] = None,
 ):
     """Methods for getting files and outputs for the repo.
 
@@ -41,60 +78,60 @@ def ls(
             "isexec": bool,
         }
     """
-    from dvc.config import Config
-
-    from . import Repo
-
-    if config and not isinstance(config, dict):
-        config_dict = Config.load_file(config)
-    else:
-        config_dict = None
-
-    with Repo.open(
-        url,
-        rev=rev,
-        subrepos=True,
-        uninitialized=True,
-        config=config_dict,
-        remote=remote,
-        remote_config=remote_config,
-    ) as repo:
+    with _open_repo(url, rev, config, remote, remote_config) as repo:
         path = path or ""
+        fs: DVCFileSystem = repo.dvcfs
+        fs_path = fs.from_os_path(path)
+        return _ls(fs, fs_path, recursive, dvc_only, maxdepth)
 
-        ret = _ls(repo, path, recursive, dvc_only)
 
-        ret_list = []
-        for path, info in ret.items():
-            info["path"] = path
-            ret_list.append(info)
-        ret_list.sort(key=lambda f: f["path"])
-        return ret_list
+def ls_tree(
+    url: str,
+    path: Optional[str] = None,
+    rev: Optional[str] = None,
+    dvc_only: bool = False,
+    config: Union[dict[str, Any], str, None] = None,
+    remote: Optional[str] = None,
+    remote_config: Optional[dict] = None,
+    maxdepth: Optional[int] = None,
+):
+    with _open_repo(url, rev, config, remote, remote_config) as repo:
+        path = path or ""
+        fs: DVCFileSystem = repo.dvcfs
+        fs_path = fs.from_os_path(path)
+        return _ls_tree(
+            fs, fs_path, maxdepth=maxdepth, dvc_only=dvc_only, dvcfiles=True
+        )
 
 
 def _ls(
-    repo: "Repo",
+    fs: "DVCFileSystem",
     path: str,
     recursive: Optional[bool] = None,
     dvc_only: bool = False,
+    maxdepth: Optional[int] = None,
 ):
-    fs: DVCFileSystem = repo.dvcfs
-    fs_path = fs.from_os_path(path)
-
-    fs_path = fs.info(fs_path)["name"]
+    fs_path = fs.info(path)["name"]
 
     infos = {}
-    if fs.isfile(fs_path):
-        infos[os.path.basename(path)] = fs.info(fs_path)
+
+    # ignore maxdepth only if recursive is not set
+    maxdepth = maxdepth if recursive else None
+    if maxdepth == 0 or fs.isfile(fs_path):
+        infos[os.path.basename(path) or os.curdir] = fs.info(fs_path)
     else:
         for root, dirs, files in fs.walk(
-            fs_path, dvcfiles=True, dvc_only=dvc_only, detail=True
+            fs_path,
+            dvcfiles=True,
+            dvc_only=dvc_only,
+            detail=True,
+            maxdepth=maxdepth,
         ):
-            if not recursive:
-                files.update(dirs)
-
             parts = fs.relparts(root, fs_path)
             if parts == (".",):
                 parts = ()
+            if not recursive or (maxdepth and len(parts) >= maxdepth - 1):
+                files.update(dirs)
 
             for name, entry in files.items():
                 infos[os.path.join(*parts, name)] = entry
@@ -102,14 +139,44 @@ def _ls(
             if not recursive:
                 break
 
-    ret = {}
-    for name, info in infos.items():
-        dvc_info = info.get("dvc_info", {})
-        ret[name] = {
-            "isout": dvc_info.get("isout", False),
-            "isdir": info["type"] == "directory",
-            "isexec": info.get("isexec", False),
-            "size": info.get("size"),
-        }
+    ret_list = []
+    for p, info in sorted(infos.items(), key=lambda x: x[0]):
+        _info = _adapt_info(info)
+        _info["path"] = p
+        ret_list.append(_info)
+    return ret_list
 
+
+def _ls_tree(fs, path, maxdepth=None, _info=None, **fs_kwargs):
+    info = _info or fs.info(path)
+    if _info is None:
+        # preserve the original path name
+        name = path
+        if not name:
+            name = os.curdir if fs.protocol == "local" else fs.root_marker
+        path = info["name"]
+    else:
+        name = path.rsplit(fs.sep, 1)[-1]
+
+    ret = {}
+    ls_info = _adapt_info(info)
+    ls_info["path"] = path
+
+    recurse = maxdepth is None or maxdepth > 0
+    if recurse and info["type"] == "directory":
+        try:
+            infos = fs.ls(path, detail=True, **fs_kwargs)
+        except FileNotFoundError:
+            # broken symlink?
+            infos = []
+
+        infos.sort(key=lambda f: f["name"])
+        maxdepth = maxdepth - 1 if maxdepth is not None else None
+        contents = {}
+        for info in infos:
+            d = _ls_tree(fs, info["name"], maxdepth=maxdepth, _info=info, **fs_kwargs)
+            contents.update(d)
+        ls_info["contents"] = contents
+
+    ret[name] = ls_info
     return ret

dvc/repo/ls_url.py

@@ -1,19 +1,30 @@
+from fsspec.implementations.local import LocalFileSystem as _LocalFileSystem
+
 from dvc.exceptions import URLMissingError
-from dvc.fs import parse_external_url
+from dvc.fs import LocalFileSystem, parse_external_url
 
 
-def ls_url(url, *, fs_config=None, recursive=False, config=None):
+def ls_url(url, *, fs_config=None, recursive=False, maxdepth=None, config=None):
     fs, fs_path = parse_external_url(url, fs_config=fs_config, config=config)
     try:
         info = fs.info(fs_path)
     except FileNotFoundError as exc:
         raise URLMissingError(url) from exc
-    if info["type"] != "directory":
+    if maxdepth == 0 or info["type"] != "directory":
         return [{"path": info["name"], "isdir": False}]
 
+    if isinstance(fs, LocalFileSystem):
+        # dvc's LocalFileSystem does not support maxdepth yet
+        walk = _LocalFileSystem().walk
+    else:
+        walk = fs.walk
+
     ret = []
-    for _, dirs, files in fs.walk(fs_path, detail=True):
-        if not recursive:
+    for root, dirs, files in walk(fs_path, detail=True, maxdepth=maxdepth):
+        parts = fs.relparts(root, fs_path)
+        if parts == (".",):
+            parts = ()
+        if not recursive or (maxdepth and len(parts) >= maxdepth - 1):
             files.update(dirs)
 
         for info in files.values():

dvc/repo/open_repo.py

@@ -99,7 +99,7 @@ def clean_repos():
 
 def _get_remote_config(url):
     try:
-        repo = Repo(url)
+        repo = Repo(url, uninitialized=True)
     except NotDvcRepoError:
         return {}
 
@@ -109,10 +109,16 @@ def _get_remote_config(url):
             # Fill the empty upstream entry with a new remote pointing to the
             # original repo's cache location.
             name = "auto-generated-upstream"
-            return {
-                "core": {"remote": name},
-                "remote": {name: {"url": repo.cache.local_cache_dir}},
-            }
+            try:
+                local_cache_dir = repo.cache.local_cache_dir
+            except AttributeError:
+                # if the `.dvc` dir is missing, we get an AttributeError
+                return {}
+            else:
+                return {
+                    "core": {"remote": name},
+                    "remote": {name: {"url": local_cache_dir}},
+                }
 
         # Use original remote to make sure that we are using correct url,
         # credential paths, etc if they are relative to the config location.

dvc/repo/plots/__init__.py

@@ -32,7 +32,7 @@ logger = logger.getChild(__name__)
 
 
 def onerror_collect(result: dict, exception: Exception, *args, **kwargs):
-    logger.debug("", exc_info=True)
+    logger.debug("", exc_info=True)  # noqa: LOG014
     result["error"] = exception
 
 

dvc/repo/scm_context.py

@@ -41,8 +41,10 @@ class SCMContext:
     def add(self, paths: Union[str, Iterable[str]]) -> None:
         from scmrepo.exceptions import UnsupportedIndexFormat
 
+        from dvc.scm import add_no_submodules
+
         try:
-            return self.scm.add(paths)
+            add_no_submodules(self.scm, paths)
         except UnsupportedIndexFormat:
             link = "https://github.com/iterative/dvc/issues/610"
             add_cmd = self._make_git_add_cmd([relpath(path) for path in paths])

dvc/scm.py

@@ -1,17 +1,18 @@
 """Manages source control systems (e.g. Git)."""
 
 import os
-from collections.abc import Iterator, Mapping
+from collections.abc import Iterable, Iterator, Mapping
 from contextlib import contextmanager
 from functools import partial
 from typing import TYPE_CHECKING, Literal, Optional, Union, overload
 
 from funcy import group_by
-from scmrepo.base import Base  # noqa: F401
+from scmrepo.base import Base  # noqa: TC002
 from scmrepo.git import Git
 from scmrepo.noscm import NoSCM
 
 from dvc.exceptions import DvcException
+from dvc.log import logger
 from dvc.progress import Tqdm
 
 if TYPE_CHECKING:
@@ -19,6 +20,8 @@ if TYPE_CHECKING:
 
     from dvc.fs import FileSystem
 
+logger = logger.getChild(__name__)
+
 
 class SCMError(DvcException):
     """Base class for source control management errors."""
@@ -283,3 +286,35 @@ def lfs_prefetch(fs: "FileSystem", paths: list[str]):
             include=[(path if path.startswith("/") else f"/{path}") for path in paths],
             progress=pbar.update_git,
         )
+
+
+def add_no_submodules(
+    scm: "Base",
+    paths: Union[str, Iterable[str]],
+    **kwargs,
+) -> None:
+    """Stage paths to Git, excluding those inside submodules."""
+
+    if isinstance(paths, str):
+        paths = [paths]
+
+    submodule_roots = {os.path.join(scm.root_dir, sub) for sub in scm.list_submodules()}
+
+    repo_paths: list[str] = []
+    skipped_paths: list[str] = []
+
+    for p in paths:
+        abs_path = os.path.abspath(p)
+        if any(
+            abs_path == root or abs_path.startswith(root + os.sep)
+            for root in submodule_roots
+        ):
+            skipped_paths.append(p)
+        else:
+            repo_paths.append(p)
+
+    if skipped_paths:
+        msg = "Skipping staging for path(s) inside submodules: %s"
+        logger.debug(msg, ", ".join(skipped_paths))
+
+    scm.add(repo_paths, **kwargs)

dvc/stage/__init__.py

@@ -83,7 +83,7 @@ class RawData:
 def create_stage(cls: type[_T], repo, path, **kwargs) -> _T:
     from dvc.dvcfile import check_dvcfile_path
 
-    wdir = os.path.abspath(kwargs.get("wdir", None) or os.curdir)
+    wdir = os.path.abspath(kwargs.get("wdir") or os.curdir)
     path = os.path.abspath(path)
 
     check_dvcfile_path(repo, path)
@@ -435,7 +435,7 @@ class Stage(params.StageParams):
         elif not self.changed(True, upstream) and pull:
             try:
                 logger.info("Pulling data for %s", self)
-                self.repo.pull(self.addressing, jobs=kwargs.get("jobs", None))
+                self.repo.pull(self.addressing, jobs=kwargs.get("jobs"))
                 self.checkout()
                 return None
             except CheckoutError:
@@ -609,7 +609,7 @@ class Stage(params.StageParams):
             self.remove_outs(ignore_remove=False, force=False)
 
         if (self.is_import and not self.frozen) or self.is_partial_import:
-            self._sync_import(dry, force, kwargs.get("jobs", None), no_download)
+            self._sync_import(dry, force, kwargs.get("jobs"), no_download)
         elif not self.frozen and self.cmd:
             self._run_stage(dry, force, **kwargs)
         elif not dry:

dvc/stage/utils.py

@@ -302,9 +302,9 @@ def _get_stage_files(stage: "Stage") -> list[str]:
             and dep.is_in_repo
             and not stage.repo.dvcfs.isdvc(stage.repo.dvcfs.from_os_path(str(dep)))
         ):
-            ret.append(relpath(dep.fs_path))
+            ret.append(relpath(dep.fs_path))  # noqa: PERF401
 
     for out in stage.outs:
         if not out.use_scm_ignore and out.is_in_repo:
-            ret.append(relpath(out.fs_path))
+            ret.append(relpath(out.fs_path))  # noqa: PERF401
     return ret

dvc/testing/api_tests.py

@@ -32,12 +32,13 @@ class TestAPI:
         "fs_kwargs",
         [
             {},
-            {"url": "{path}"},
-            {"url": "{path}", "rev": "{default_branch}"},
-            {"url": "file://{posixpath}"},
-            {"url": "file://{posixpath}", "rev": "{default_branch}"},
+            {"repo": "{path}"},
+            {"repo": "{path}", "rev": "{default_branch}"},
+            {"repo": "file://{posixpath}"},
+            {"repo": "file://{posixpath}", "rev": "{default_branch}"},
+            {"url": "{path}"},  # test for backward compatibility
         ],
-        ids=["current", "local", "local_rev", "git", "git_rev"],
+        ids=["current", "local", "local_rev", "git", "git_rev", "local_url"],
     )
     def test_filesystem(
         self,
@@ -59,6 +60,8 @@ class TestAPI:
         if clear_cache:
             remove(dvc.cache.repo.path)
 
+        if repo := fs_kwargs.get("repo"):
+            fs_kwargs["repo"] = repo.format(path=tmp_dir, posixpath=tmp_dir.as_posix())
         if url := fs_kwargs.get("url"):
             fs_kwargs["url"] = url.format(path=tmp_dir, posixpath=tmp_dir.as_posix())
         if rev := fs_kwargs.get("rev"):

dvc/testing/benchmarks/cli/commands/test_add.py

@@ -1,2 +1,19 @@
-def test_add(bench_dvc, tmp_dir, dvc, dataset):
-    bench_dvc("add", dataset)
+from .test_checkout import _skip_unsupported_link
+
+
+def generate_test(*, link_type="copy"):
+    def _test_add(bench_dvc, tmp_dir, dvc, dataset):
+        _skip_unsupported_link((tmp_dir / ".dvc" / "cache"), tmp_dir, link_type)
+
+        with dvc.config.edit() as conf:
+            conf["cache"]["type"] = link_type
+
+        bench_dvc("add", dataset)
+        bench_dvc("add", dataset, name="noop")
+
+    return _test_add
+
+
+test_add_copy = generate_test(link_type="copy")
+test_add_symlink = generate_test(link_type="symlink")
+test_add_hardlink = generate_test(link_type="hardlink")

dvc/testing/benchmarks/cli/commands/test_data_status.py

@@ -1,19 +1,18 @@
 from shutil import rmtree
 
-import pytest
-
-pytestmark = pytest.mark.requires(minversion=(2, 15, 0), reason="new command")
-
 
 def test_data_status(bench_dvc, tmp_dir, scm, dvc, make_dataset):
     args = ("data", "status")
     dataset = make_dataset(cache=True, files=True, dvcfile=True, commit=False)
+    scm.ignore(dataset)
     rmtree(dvc.tmp_dir)
 
     bench_dvc(*args, name="new")
     bench_dvc(*args, name="noop")
 
-    tmp_dir.scm_add(dataset.with_suffix(".dvc").name, commit="add dataset")
+    tmp_dir.scm_add(
+        [dataset.with_suffix(".dvc").name, ".gitignore"], commit="add dataset"
+    )
 
     (dataset / "new").write_text("new")
     bench_dvc(*args, name="changed")
@@ -30,12 +29,15 @@ def test_data_status_all_flags(bench_dvc, tmp_dir, scm, dvc, make_dataset):
         "--json",
     )
     dataset = make_dataset(cache=True, files=True, dvcfile=True, commit=False)
+    scm.ignore(dataset)
     rmtree(dvc.tmp_dir)
 
     bench_dvc(*args, name="new")
     bench_dvc(*args, name="noop")
 
-    tmp_dir.scm_add(dataset.with_suffix(".dvc").name, commit="add dataset")
+    tmp_dir.scm_add(
+        [dataset.with_suffix(".dvc").name, ".gitignore"], commit="add dataset"
+    )
 
     (dataset / "new").write_text("new")
     bench_dvc(*args, name="changed")

dvc/testing/benchmarks/cli/commands/test_exp_show.py

@@ -1,7 +1,3 @@
-import pytest
-
-
-@pytest.mark.requires(minversion=(2, 28, 0))
 def test_exp_show(make_project, monkeypatch, bench_dvc, dvc_bin):
     url = "https://github.com/iterative/example-get-started"
     rev = "main"

dvc/testing/benchmarks/cli/commands/test_import.py

@@ -2,6 +2,12 @@ import pytest
 
 
 @pytest.mark.flaky(reruns=3)
+@pytest.mark.requires(
+    "!=3.53.*,!=3.54.0",
+    reason="Takes 10 mins to run. Regression in 3.53.0, fixed in 3.54.1",
+)
+# Introduced in https://github.com/iterative/dvc/pull/10388.
+# Fixed in https://github.com/iterative/dvc/pull/10531.
 def test_import(bench_dvc, tmp_dir, scm, dvc, make_dataset, remote):
     dataset = make_dataset(
         cache=False, files=False, dvcfile=True, commit=True, remote=True

dvc/testing/benchmarks/cli/commands/test_plots.py

@@ -1,10 +1,7 @@
-import pytest
-
 from dvc.repo import Repo
 from dvc.testing.benchmarks.fixtures import _pull
 
 
-@pytest.mark.requires(minversion=(2, 34, 0), reason="top-level plots not supported")
 def test_plots(project, bench_dvc):
     with Repo() as dvc:
         _pull(dvc)

dvc/testing/benchmarks/cli/stories/use_cases/test_sharing.py

@@ -1,9 +1,18 @@
 import shutil
 
+from packaging import version
 
-def test_sharing(bench_dvc, tmp_dir, dvc, dataset, remote):
-    bench_dvc("add", dataset)
-    bench_dvc("add", dataset, name="noop")
+
+def test_sharing(bench_dvc, tmp_dir, dvc, make_dataset, remote, dvc_bin):
+    is_gs = dvc.config["remote"]["upstream"]["url"].startswith("gs://")
+    if is_gs and version.Version(dvc_bin.version) < version.Version("3.59.2"):
+        # support for allow_anonymous_login in gs was introduced in dvc==3.60
+        # This should not impact the test, just that it will make it slower if used
+        # with fake-gcs-server.
+        with dvc.config.edit() as d:
+            d["remote"]["upstream"].pop("allow_anonymous_login", None)
+
+    dataset = make_dataset(cache=True, dvcfile=True)
 
     bench_dvc("push")
     bench_dvc("push", name="noop")
@@ -13,6 +22,3 @@ def test_sharing(bench_dvc, tmp_dir, dvc, dataset, remote):
 
     bench_dvc("fetch")
     bench_dvc("fetch", name="noop")
-
-    bench_dvc("checkout")
-    bench_dvc("checkout", name="noop")

dvc/testing/benchmarks/conftest.py

@@ -1 +1,3 @@
 from .fixtures import *  # noqa: F403
+
+pytest_plugins = ["dvc.testing.plugin"]

dvc/testing/benchmarks/fixtures.py

@@ -1,7 +1,9 @@
+import inspect
 import os
 import shutil
+import sys
 from pathlib import Path
-from subprocess import check_output
+from subprocess import check_call, check_output
 from typing import Optional
 
 import pytest
@@ -23,13 +25,14 @@ class VirtualEnv:
         self.bin = self.path / ("Scripts" if os.name == "nt" else "bin")
 
     def create(self) -> None:
-        import virtualenv
+        check_call([sys.executable, "-m", "uv", "venv", self.path])  # noqa:S603
 
-        virtualenv.cli_run([os.fspath(self.path)])
+    def install(self, *packages: str) -> None:
+        check_call([sys.executable, "-m", "uv", "pip", "install", *packages])  # noqa: S603
 
     def run(self, cmd: str, *args: str, env: Optional[dict[str, str]] = None) -> None:
         exe = self.which(cmd)
-        check_output([exe, *args], env=env)  # noqa: S603
+        check_call([exe, *args], env=env)  # noqa: S603
 
     def which(self, cmd: str) -> str:
         assert self.bin.exists()
@@ -58,8 +61,8 @@ def dvc_venvs():
 
 
 @pytest.fixture(scope="session")
-def dvc_git_repo(tmp_path_factory, bench_config):
-    url = bench_config.dvc_git_repo
+def dvc_repo(tmp_path_factory, bench_config):
+    url = bench_config.dvc_repo
 
     if os.path.isdir(url):
         return url
@@ -71,8 +74,13 @@ def dvc_git_repo(tmp_path_factory, bench_config):
 
 
 @pytest.fixture(scope="session")
-def dvc_bench_git_repo(tmp_path_factory, bench_config):
-    url = bench_config.dvc_bench_git_repo
+def dvc_bench_repo(tmp_path_factory, bench_config):
+    url = bench_config.dvc_bench_repo
+    if url is None:
+        pytest.skip(
+            "--dvc-bench-repo is not set, "
+            "clone https://github.com/iterative/dvc-bench repository and set its path"
+        )
 
     if os.path.isdir(url):
         return Path(url)
@@ -88,61 +96,54 @@ def make_dvc_bin(
     dvc_rev,
     dvc_venvs,
     make_dvc_venv,
-    dvc_git_repo,
+    dvc_repo,
     bench_config,
     request,
 ):
     if dvc_rev:
-        venv = dvc_venvs.get(dvc_rev)
+        venv: VirtualEnv = dvc_venvs.get(dvc_rev)
         if not venv:
             venv = make_dvc_venv(dvc_rev)
-            venv.run("pip", "install", "-U", "pip")
             if bench_config.dvc_install_deps:
-                egg = f"dvc[{bench_config.dvc_install_deps}]"
+                pkg = f"dvc[{bench_config.dvc_install_deps}]"
             else:
-                egg = "dvc"
-            venv.run("pip", "install", f"git+file://{dvc_git_repo}@{dvc_rev}#egg={egg}")
-            if dvc_rev in ["2.18.1", "2.11.0", "2.6.3"]:
-                venv.run("pip", "install", "fsspec==2022.11.0")
+                pkg = "dvc"
+            packages = [f"{pkg} @ git+file://{dvc_repo}@{dvc_rev}"]
             try:
                 if version.Version(dvc_rev) < version.Version("3.50.3"):
-                    venv.run("pip", "install", "pygit2==1.14.1")
+                    packages.append("pygit2==1.14.1")
             except version.InvalidVersion:
                 pass
+            venv.install(*packages)
+
             dvc_venvs[dvc_rev] = venv
         dvc_bin = venv.which("dvc")
     else:
         dvc_bin = bench_config.dvc_bin
 
     def _dvc_bin(*args):
-        return check_output([dvc_bin, *args], text=True)  # noqa: S603
+        check_call([dvc_bin, *args])  # noqa: S603
 
-    _dvc_bin.version = parse_tuple(_dvc_bin("--version"))  # type: ignore[attr-defined]
+    _dvc_bin.version = check_output([dvc_bin, "--version"], text=True)  # type: ignore[attr-defined]  # noqa: S603
     return _dvc_bin
 
 
-def parse_tuple(version_string):
-    from packaging.version import Version
-
-    parsed = version.parse(version_string)
-    assert isinstance(parsed, Version)
-    return (parsed.major, parsed.minor, parsed.micro)
-
-
 @pytest.fixture
 def dvc_bin(request, make_dvc_bin):
     if marker := request.node.get_closest_marker("requires"):
-        minversion = marker.kwargs.get("minversion") or first(marker.args)
-        assert minversion, (
-            "'minversion' needs to be specified as"
-            " a positional or a keyword argument"
-        )
-        reason = marker.kwargs.get("reason", "")
-        if isinstance(minversion, str):
-            minversion = parse_tuple(minversion)
-        if make_dvc_bin.version < minversion:
-            version_repr = ".".join(map(str, minversion))
-            pytest.skip(f"requires dvc>={version_repr}: {reason}")
+        from packaging.specifiers import SpecifierSet
+        from packaging.version import Version, parse
+
+        spec = first(marker.args)
+        assert spec is not None
+        spec = SpecifierSet(spec) if isinstance(spec, str) else spec
+        reason = marker.kwargs["reason"]
+        dvc_version = make_dvc_bin.version
+        version = Version(parse(dvc_version).base_version)
+        if version not in spec:
+            pytest.skip(
+                f"Version {dvc_version} does not satisfy requirement {spec!r}: {reason}"
+            )
     return make_dvc_bin
 
 
@@ -152,7 +153,18 @@ def make_bench(request):
         import pytest_benchmark.plugin
 
         # hack from https://github.com/ionelmc/pytest-benchmark/issues/166
-        bench = pytest_benchmark.plugin.benchmark.__pytest_wrapped__.obj(request)
+        fixture_function = pytest_benchmark.plugin.benchmark
+        try:
+            # pytest >= 8.4.0
+            wrapped_func = fixture_function._get_wrapped_function()
+        except AttributeError:
+            wrapped_func = fixture_function.__pytest_wrapped__.obj
+        assert inspect.isgeneratorfunction(wrapped_func)
+
+        generator = wrapped_func(request)
+        bench = next(generator)
+        assert isinstance(bench, pytest_benchmark.plugin.BenchmarkFixture)
+        request.addfinalizer(lambda: next(generator, None))
 
         suffix = f"-{name}"
 
@@ -169,11 +181,19 @@ def make_bench(request):
 
 
 @pytest.fixture
-def bench_dvc(dvc_bin, make_bench):
+def bench_dvc(request, dvc_bin, make_bench):
     def _bench_dvc(*args, **kwargs):
         name = kwargs.pop("name", None)
         name = f"-{name}" if name else ""
         bench = make_bench(args[0] + name)
+        if request.config.getoption("--dvc-benchmark-cprofile-dump") or kwargs.pop(
+            "cprofile", False
+        ):
+            cprofile_results = request.config.invocation_params.dir / "prof"
+            cprofile_results.mkdir(exist_ok=True)
+            stats_file = cprofile_results / f"{bench.name}.prof"
+            args = (*args, "--cprofile-dump", stats_file)
+
         return bench.pedantic(dvc_bin, args=args, **kwargs)
 
     return _bench_dvc
@@ -190,14 +210,14 @@ def _pull(repo, *args):
 
 
 @pytest.fixture
-def make_dataset(request, bench_config, tmp_dir, dvc_bench_git_repo):
+def make_dataset(request, bench_config, tmp_dir, dvc_bench_repo):
     def _make_dataset(
         dvcfile=False, files=True, cache=False, commit=False, remote=False
     ):
         from dvc.repo import Repo
 
         path = tmp_dir / "dataset"
-        root = dvc_bench_git_repo
+        root = dvc_bench_repo
         src = root / "data" / bench_config.dataset / "dataset"
         src_dvc = src.with_suffix(".dvc")
 
@@ -256,7 +276,7 @@ def make_project(tmp_path_factory):
 @pytest.fixture
 def project(bench_config, monkeypatch, make_project):
     rev = bench_config.project_rev
-    url = bench_config.project_git_repo
+    url = bench_config.project_repo
 
     if os.path.isdir(url):
         path = url

dvc/testing/benchmarks/plugin.py

@@ -1,57 +1,84 @@
+import os
+from dataclasses import dataclass, fields
+from pathlib import Path
+from typing import Optional
+
 DEFAULT_DVC_BIN = "dvc"
-DEFAULT_DVC_GIT_REPO = "https://github.com/iterative/dvc"
-DEFAULT_DVC_BENCH_GIT_REPO = "https://github.com/iterative/dvc-bench"
-DEFAULT_PROJECT_GIT_REPO = "https://github.com/iterative/example-get-started"
+DEFAULT_DVC_REPO = os.fspath(Path(__file__).parents[3])
+DEFAULT_PROJECT_REPO = "https://github.com/iterative/example-get-started"
 
 
 def pytest_report_header(config):
     bconf = config.bench_config
-    return f"dvc-bench: (dataset: {bconf.dataset!r}, revs: {bconf.dvc_revs!r})"
+    return f"dvc-bench: {bconf}"
 
 
 def pytest_generate_tests(metafunc):
-    str_revs = metafunc.config.getoption("--dvc-revs")
-    revs = str_revs.split(",") if str_revs else [None]
+    revs = metafunc.config.getoption("--dvc-revs")
+    if not revs:
+        revs = [None]
     if "dvc_rev" in metafunc.fixturenames:
         metafunc.parametrize("dvc_rev", revs, scope="session")
 
 
+@dataclass
 class DVCBenchConfig:
-    def __init__(self):
-        self.dataset = "small"
-        self.dvc_bin = DEFAULT_DVC_BIN
-        self.dvc_revs = None
-        self.dvc_git_repo = DEFAULT_DVC_GIT_REPO
-        self.dvc_install_deps = None
-        self.dvc_bench_git_repo = DEFAULT_DVC_BENCH_GIT_REPO
-        self.project_rev = None
-        self.project_git_repo = DEFAULT_PROJECT_GIT_REPO
+    dataset: str = "tiny"
+    dvc_repo: str = DEFAULT_DVC_REPO
+    dvc_bench_repo: Optional[str] = None
+    project_repo: str = DEFAULT_PROJECT_REPO
+    project_rev: Optional[str] = None
+    dvc_bin: str = DEFAULT_DVC_BIN
+    dvc_revs: Optional[list[str]] = None
+    dvc_install_deps: Optional[str] = None
+
+    def __repr__(self):
+        args = ", ".join(
+            f"{f.name}={val!r}"
+            for f in fields(self)
+            if (val := getattr(self, f.name)) != f.default
+        )
+        return f"{self.__class__.__name__}({args})"
 
 
 def pytest_configure(config):
     config.addinivalue_line(
-        "markers", "requires(minversion): mark a test as requiring minimum DVC version"
+        "markers",
+        "requires(spec): mark a test to run only on versions that satisfy the spec",
+    )
+    config.bench_config = DVCBenchConfig(
+        dataset=config.getoption("--dataset"),
+        dvc_repo=config.getoption("--dvc-repo"),
+        dvc_bench_repo=config.getoption("--dvc-bench-repo"),
+        project_repo=config.getoption("--project-repo"),
+        project_rev=config.getoption("--project-rev"),
+        dvc_bin=config.getoption("--dvc-bin"),
+        dvc_revs=config.getoption("--dvc-revs"),
+        dvc_install_deps=config.getoption("--dvc-install-deps"),
     )
 
-    config.bench_config = DVCBenchConfig()
-    config.bench_config.dataset = config.getoption("--dataset")
-    config.bench_config.dvc_bin = config.getoption("--dvc-bin")
-    config.bench_config.dvc_revs = config.getoption("--dvc-revs")
-    config.bench_config.dvc_git_repo = config.getoption("--dvc-git-repo")
-    config.bench_config.dvc_install_deps = config.getoption("--dvc-install-deps")
-    config.bench_config.dvc_bench_git_repo = config.getoption("--dvc-bench-git-repo")
-    config.bench_config.project_rev = config.getoption("--project-rev")
-    config.bench_config.project_git_repo = config.getoption("--project-git-repo")
+
+def resolve_path(path):
+    if os.path.isdir(path):
+        return os.path.abspath(path)
+    return path
 
 
 def pytest_addoption(parser):
     parser.addoption(
         "--dataset",
         type=str,
-        default="small",
+        default="tiny",
         help="Dataset name to use in tests (e.g. tiny/small/large/mnist/etc)",
     )
 
+    parser.addoption(
+        "--dvc-benchmark-cprofile-dump",
+        action="store_true",
+        default=False,
+        help="Save cprofile results",
+    )
+
     parser.addoption(
         "--dvc-bin",
         type=str,
@@ -61,36 +88,34 @@ def pytest_addoption(parser):
 
     parser.addoption(
         "--dvc-revs",
-        type=str,
+        type=lambda revs: revs.split(","),
         help=("Comma-separated list of DVC revisions to test (overrides `--dvc-bin`)"),
     )
 
     parser.addoption(
-        "--dvc-git-repo",
-        type=str,
-        default=DEFAULT_DVC_GIT_REPO,
+        "--dvc-repo",
+        type=resolve_path,
+        default=DEFAULT_DVC_REPO,
         help="Path or url to dvc git repo",
     )
 
     parser.addoption(
         "--dvc-install-deps",
         type=str,
-        default="",
         help="Comma-separated list of DVC installation packages",
     )
 
     parser.addoption(
-        "--dvc-bench-git-repo",
-        type=str,
-        default=DEFAULT_DVC_BENCH_GIT_REPO,
+        "--dvc-bench-repo",
+        type=resolve_path,
+        default=None,
         help="Path or url to dvc-bench git repo (for loading benchmark datasets)",
     )
 
     parser.addoption("--project-rev", type=str, help="Project revision to test")
-
     parser.addoption(
-        "--project-git-repo",
-        type=str,
-        default=DEFAULT_PROJECT_GIT_REPO,
+        "--project-repo",
+        type=resolve_path,
+        default=DEFAULT_PROJECT_REPO,
         help="Path or url to dvc project",
     )

dvc/testing/path_info.py

@@ -32,7 +32,7 @@ class PathInfo(pathlib.PurePath, _BasePath):
 
         def __new__(cls, *args):
             if cls is PathInfo:
-                cls = WindowsPathInfo if os.name == "nt" else PosixPathInfo
+                cls = WindowsPathInfo if os.name == "nt" else PosixPathInfo  # noqa: PLW0642
 
             return cls._from_parts(args)  # type: ignore[attr-defined]
 
@@ -79,13 +79,13 @@ class PathInfo(pathlib.PurePath, _BasePath):
             and self._cparts[:n] == other._cparts  # type: ignore[attr-defined]
         )
 
-    def relative_to(self, other):
+    def relative_to(self, other, *args, **kwargs):
         # pathlib relative_to raises exception when one path is not a direct
         # descendant of the other when os.path.relpath would return abspath.
         # For DVC PathInfo we only need the relpath behavior.
         # See: https://bugs.python.org/issue40358
         try:
-            path = super().relative_to(other)
+            path = super().relative_to(other, *args, **kwargs)
         except ValueError:
             path = relpath(self, other)
         return self.__class__(path)
@@ -265,7 +265,7 @@ class CloudURLInfo(URLInfo):
 
 
 class HTTPURLInfo(URLInfo):
-    __hash__: Callable[["HTTPURLInfo"], int] = URLInfo.__hash__
+    __hash__: Callable[["HTTPURLInfo"], int] = URLInfo.__hash__  # type: ignore[assignment]
 
     def __init__(self, url):
         p = urlparse(url)

dvc/testing/plugin.py

@@ -18,3 +18,9 @@ def pytest_configure(config):
     from .benchmarks.plugin import pytest_configure as bench_configure
 
     bench_configure(config)
+
+
+def pytest_report_header(config):
+    from .benchmarks.plugin import pytest_report_header as bench_report_header
+
+    return bench_report_header(config)

dvc/testing/tmp_dir.py

@@ -71,7 +71,7 @@ class TmpDir(pathlib.Path):
 
         def __new__(cls, *args, **kwargs):
             if cls is TmpDir:
-                cls = WindowsTmpDir if os.name == "nt" else PosixTmpDir
+                cls = WindowsTmpDir if os.name == "nt" else PosixTmpDir  # noqa: PLW0642
 
             # init parameter and `_init` method has been removed in Python 3.10.
             kw = {"init": False} if sys.version_info < (3, 10) else {}

dvc/testing/workspace_tests.py

@@ -225,6 +225,39 @@ class TestLsUrl:
             ],
         )
 
+        result = ls_url(
+            str(cloud / "dir"), fs_config=cloud.config, recursive=True, maxdepth=0
+        )
+        match_files(
+            fs,
+            result,
+            [{"path": (cloud / "dir").fs_path, "isdir": False}],
+        )
+
+        result = ls_url(
+            str(cloud / "dir"), fs_config=cloud.config, recursive=True, maxdepth=1
+        )
+        match_files(
+            fs,
+            result,
+            [
+                {"path": "foo", "isdir": False},
+                {"path": "subdir", "isdir": True},
+            ],
+        )
+
+        result = ls_url(
+            str(cloud / "dir"), fs_config=cloud.config, recursive=True, maxdepth=2
+        )
+        match_files(
+            fs,
+            result,
+            [
+                {"path": "foo", "isdir": False},
+                {"path": "subdir/bar", "isdir": False},
+            ],
+        )
+
     def test_nonexistent(self, cloud):
         with pytest.raises(URLMissingError):
             ls_url(str(cloud / "dir"), fs_config=cloud.config)

dvc/ui/__init__.py

@@ -300,6 +300,7 @@ class Console:
         header_styles: Optional[Union[dict[str, "Styles"], Sequence["Styles"]]] = None,
         row_styles: Optional[Sequence["Styles"]] = None,
         borders: Union[bool, str] = False,
+        colalign: Optional[tuple[str, ...]] = None,
     ) -> None:
         from dvc.ui import table as t
 
@@ -321,7 +322,13 @@ class Console:
             return
 
         return t.plain_table(
-            self, data, headers, markdown=markdown, pager=pager, force=force
+            self,
+            data,
+            headers,
+            markdown=markdown,
+            pager=pager,
+            force=force,
+            colalign=colalign,
         )
 
     def status(self, status: str, **kwargs: Any) -> "Status":

dvc/ui/table.py

@@ -29,6 +29,7 @@ def plain_table(
     markdown: bool = False,
     pager: bool = False,
     force: bool = True,
+    colalign: Optional[tuple[str, ...]] = None,
 ) -> None:
     from funcy import nullcontext
     from tabulate import tabulate
@@ -40,6 +41,7 @@ def plain_table(
         disable_numparse=True,
         # None will be shown as "" by default, overriding
         missingval="-",
+        colalign=colalign,
     )
     if markdown:
         # NOTE: md table is incomplete without the trailing newline

dvc/utils/__init__.py

@@ -18,7 +18,7 @@ TARGET_REGEX = re.compile(r"(?P<path>.*?)(:(?P<name>[^\\/:]*))??$")
 
 
 def bytes_hash(byts, typ):
-    hasher = getattr(hashlib, typ)()
+    hasher = getattr(hashlib, typ)(usedforsecurity=False)
     hasher.update(byts)
     return hasher.hexdigest()
 
@@ -243,7 +243,7 @@ def env2bool(var, undefined=False):
     var = os.getenv(var, None)
     if var is None:
         return undefined
-    return bool(re.search("1|y|yes|true", var, flags=re.I))
+    return bool(re.search("1|y|yes|true", var, flags=re.IGNORECASE))
 
 
 def resolve_output(inp: str, out: Optional[str], force=False) -> str:
@@ -382,7 +382,7 @@ def glob_targets(targets, glob=True, recursive=True):
 
 def error_handler(func):
     def wrapper(*args, **kwargs):
-        onerror = kwargs.get("onerror", None)
+        onerror = kwargs.get("onerror")
         result = {}
 
         try:

dvc/utils/hydra.py

@@ -115,6 +115,7 @@ def apply_overrides(path: "StrPath", overrides: list[str]) -> None:
         except hydra_errors as e:
             raise InvalidArgumentError("Invalid `--set-param` value") from e
 
+        assert isinstance(new_data, dict)
         merge_dicts(original_data, new_data)
         remove_missing_keys(original_data, new_data)
 

dvc/utils/serialize/_common.py

@@ -33,7 +33,7 @@ class LoaderFn(Protocol):
     def __call__(self, path: "StrPath", fs: Optional["FileSystem"] = None) -> Any: ...
 
 
-ReadType = Union[bytes, None, str]
+ReadType = Union[bytes, str, None]
 ParserFn = Callable[[ReadType, "StrPath"], dict]
 
 
@@ -61,7 +61,7 @@ def _load_data(
 ):
     open_fn = fs.open if fs else open
     encoding = "utf-8"
-    with open_fn(path, encoding=encoding, **kwargs) as fd:  # type: ignore[operator]
+    with open_fn(path, encoding=encoding, **kwargs) as fd:  # type: ignore[arg-type]
         with reraise(UnicodeDecodeError, EncodingError(path, encoding)):
             return parser(fd.read(), path)
 
@@ -74,7 +74,7 @@ def _dump_data(
     **dumper_args,
 ):
     open_fn = fs.open if fs else open
-    with open_fn(path, "w+", encoding="utf-8") as fd:  # type: ignore[operator]
+    with open_fn(path, "w+", encoding="utf-8") as fd:  # type: ignore[call-overload]
         dumper(data, fd, **dumper_args)
 
 

dvc/utils/studio.py

@@ -23,7 +23,7 @@ if TYPE_CHECKING:
 
 logger = logger.getChild(__name__)
 
-STUDIO_URL = "https://studio.dvc.ai"
+STUDIO_URL = "https://studio.datachain.ai"
 
 
 def post(

dvc/version.py

@@ -1,6 +1,6 @@
 try:
-    from ._dvc_version import version as __version__  # type: ignore[import]
-    from ._dvc_version import version_tuple  # type: ignore[import]
+    from ._version import version as __version__  # type: ignore[import]
+    from ._version import version_tuple  # type: ignore[import]
 except ImportError:
     __version__ = "UNKNOWN"
     version_tuple = (0, 0, __version__)  # type: ignore[assignment]

pyproject.toml

@@ -1,6 +1,6 @@
 [build-system]
 build-backend = "setuptools.build_meta"
-requires = ["setuptools>=61", "setuptools_scm[toml]>=7"]
+requires = ["setuptools>=77", "setuptools_scm[toml]>=8"]
 
 [project]
 name = "dvc"
@@ -16,7 +16,8 @@ keywords = [
     "machine-learning",
     "reproducibility",
 ]
-license = { text = "Apache License 2.0" }
+license = "Apache-2.0"
+license-files = ["LICENSE"]
 maintainers = [{ name = "Iterative", email = "[email protected]" }]
 authors = [{ name = "Dmitry Petrov", email = "[email protected]" }]
 requires-python = ">=3.9"
@@ -27,6 +28,7 @@ classifiers = [
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
 ]
 dynamic = ["version"]
 
@@ -34,18 +36,18 @@ dependencies = [
     "attrs>=22.2.0",
     "celery",
     "colorama>=0.3.9",
-    "configobj>=5.0.6",
+    "configobj>=5.0.9",
     "distro>=1.3",
     "dpath<3,>=2.1.0",
     "dulwich",
-    "dvc-data>=3.15,<3.16",
+    "dvc-data>=3.16.2,<3.17",
     "dvc-http>=2.29.0",
     "dvc-objects",
     "dvc-render>=1.0.1,<2",
     "dvc-studio-client>=0.21,<1",
     "dvc-task>=0.3.0,<1",
     "flatten_dict<1,>=0.4.1",
-    "flufl.lock>=8.1.0,<9", # https://github.com/iterative/dvc/issues/9654
+    "flufl.lock>=8.1.0,<9",
     "fsspec>=2024.2.0",
     "funcy>=1.14",
     "grandalf<1,>=0.7",
@@ -57,7 +59,7 @@ dependencies = [
     "omegaconf",
     "packaging>=19",
     "pathspec>=0.10.3",
-    "platformdirs<4,>=3.1.1",
+    "platformdirs<5,>=3.1.1",
     "psutil>=5.8",
     "pydot>=1.2.4",
     "pygtrie>=2.3.2",
@@ -65,7 +67,7 @@ dependencies = [
     "requests>=2.22",
     "rich>=12",
     "ruamel.yaml>=0.17.11",
-    "scmrepo>=3.3.4,<4",
+    "scmrepo>=3.5.2,<4",
     "shortuuid>=0.5",
     "shtab<2,>=1.3.4",
     "tabulate>=0.8.7",
@@ -80,10 +82,10 @@ all = ["dvc[azure,gdrive,gs,hdfs,oss,s3,ssh,webdav,webhdfs]"]
 azure = ["dvc-azure>=3.1.0,<4"]
 dev = ["dvc[azure,gdrive,gs,hdfs,lint,oss,s3,ssh,tests,webdav,webhdfs]"]
 gdrive = ["dvc-gdrive>=3,<4"]
-gs = ["dvc-gs>=3,<4"]
+gs = ["dvc-gs>=3.0.2,<4"]
 hdfs = ["dvc-hdfs>=3,<4"]
 lint = [
-    "mypy==1.10.1",
+    "mypy==1.17.1",
     "pandas-stubs",
     "types-colorama",
     "types-psutil",
@@ -95,13 +97,13 @@ lint = [
     "typing-extensions",
 ]
 oss = ["dvc-oss>=3,<4"]
-s3 = ["dvc-s3>=3.2.0,<4"]
+s3 = ["dvc-s3>=3.2.1,<4"]
 ssh = ["dvc-ssh>=4,<5"]
 ssh_gssapi = ["dvc-ssh[gssapi]>=4,<5"]
 testing = [
-  "pytest-benchmark[histogram]",
+  "pytest-benchmark[histogram]>=5,<6",
   "pytest-test-utils",
-  "virtualenv",
+  "uv",
 ]
 tests = [
     "dvc[testing]",
@@ -148,18 +150,15 @@ dvc = "dvc.fs.dvc_path:DVCPath"
 hook-dirs = "dvc.__pyinstaller:get_hook_dirs"
 tests = "dvc.__pyinstaller:get_PyInstaller_tests"
 
-[tool.setuptools]
-license-files = ["LICENSE"]
-
 [tool.setuptools.packages.find]
 exclude = ["tests", "tests.*"]
 namespaces = false
 
 [tool.setuptools_scm]
-write_to = "dvc/_dvc_version.py"
+write_to = "dvc/_version.py"
 
 [tool.pytest.ini_options]
-addopts = "-ra --cov-config pyproject.toml --dist worksteal --benchmark-skip"
+addopts = "-ra --cov-config pyproject.toml"
 filterwarnings = [
     "error::ResourceWarning",
     "error::pytest.PytestUnraisableExceptionWarning",
@@ -172,8 +171,10 @@ filterwarnings = [
     "ignore:Deprecated call to `pkg_resources.declare_namespace:DeprecationWarning",
     # see https://github.com/networkx/networkx/issues/5723.
     "ignore:nx.nx_pydot.* depends on the pydot package, which has.*known issues and is not actively maintained:DeprecationWarning",
+    # TODO: investigate where we are not closing sqlite3.Connection
+    "ignore:unclosed database.*sqlite3.Connection:ResourceWarning",
+    "ignore:unclosed.*<socket.socket:ResourceWarning",
 ]
-log_level = "debug"
 markers = [
     "needs_internet: Might need network access for the tests",
     "studio: Tests verifying contract between DVC and Studio",
@@ -264,21 +265,19 @@ show-fixes = true
 
 [tool.ruff.lint]
 ignore = [
-    "N818", "S101", "ISC001", "PT004", "PT007", "RET502", "RET503", "SIM105", "SIM108", "SIM117",
-    "TRY003", "TRY300", "PLR2004", "PLW2901", "LOG007",
+    "N818", "S101", "A005", "PT007", "RET502", "RET503", "SIM105", "SIM108", "SIM117",
+    "TRY003", "TRY300", "PERF203", "PLC0415", "PLR2004",  "PLW2901", "LOG007",
 ]
 select = [
     "F", "E", "W", "C90", "I", "N", "UP", "YTT", "ASYNC", "S", "BLE", "B", "A", "C4", "DTZ", "T10",
-    "EXE", "ISC", "ICN", "G", "INP", "PIE", "T20", "PYI", "PT", "Q", "RSE", "RET",
-    "SLOT", "SIM", "TID", "TCH", "ARG", "PGH", "PLC", "PLE", "PLR", "PLW", "TRY",
-    "FLY", "PERF101", "LOG", "RUF", "RUF022", "RUF023", "RUF024", "RUF025", "RUF026", "RUF027"
+    "EXE", "ISC", "ICN", "LOG", "G", "INP", "PIE", "T20", "PYI", "PT", "Q", "RSE", "RET",
+    "SLOT", "SIM", "TID", "TC", "ARG", "PGH", "PLC", "PLE", "PLR", "PLW", "TRY",
+    "FLY", "PERF", "FURB", "RUF", "RUF027", "RUF029", "RUF036",
 ]
 preview = true
 explicit-preview-rules = true
 
 [tool.ruff.lint.flake8-pytest-style]
-fixture-parentheses = false
-mark-parentheses = false
 parametrize-names-type = "csv"
 raises-extend-require-match-for = ["dvc.exceptions.DvcException", "dvc.scm.SCMError", "scmrepo.exceptions.SCMError"]
 
@@ -305,4 +304,4 @@ max-args = 10
 "dvc/commands/**" = ["N806"]
 "dvc/testing/**" = ["ARG002"]
 "dvc/testing/benchmarks/**" = ["ARG001"]
-"tests/**" = ["S", "ARG001", "ARG002", "TRY002", "TRY301"]
+"tests/**" = ["S", "ARG001", "ARG002", "TRY002", "TRY301", "PERF"]

tests/__main__.py

@@ -3,4 +3,4 @@ if __name__ == "__main__":
 
     import pytest
 
-    sys.exit(pytest.main(["-v", "-n=logical", *sys.argv[1:]]))
+    sys.exit(pytest.main(["-v", "-n=logical", "--dist=worksteal", *sys.argv[1:]]))

tests/conftest.py

@@ -1,7 +1,7 @@
 import json
 import os
 import sys
-from contextlib import suppress
+from contextlib import ExitStack, suppress
 
 import pytest
 
@@ -46,10 +46,10 @@ def reset_loglevel(request, caplog):
         ini_opt = request.config.getini("log_level")
 
     level = request.config.getoption("--log-level") or ini_opt
-    if level:
-        with caplog.at_level(level.upper(), logger="dvc"):
-            yield
-    else:
+    with ExitStack() as stack:
+        if level:
+            for name in ["dvc", "dvc_data", "dvc_objects"]:
+                stack.enter_context(caplog.at_level(level.upper(), logger=name))
         yield