build(deps): bump github/codeql-action from 4.37.4 to 4.38.2

Bumps [github/codeql-action](https://github.com/github/codeql-action) from 4.37.4 to 4.38.2. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v4.37.4...v4.38.2) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 4.38.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <[email protected]>
build(deps): bump actions/setup-python from 6 to 7 (#11073 )
2026-11-02 15:28:30 +01:00 · 2026-09-28 02:03:57 +00:00 · 2026-08-06 10:02:40 +05:45 · 2026-08-06 10:02:19 +05:45 · 2026-07-10 13:12:18 +05:45 · 2026-07-04 18:16:33 +05:45
144 changed files with 1538 additions and 630 deletions
@@ -737,8 +737,8 @@
    "tests/func/test_config.py::test_config_gdrive_fields": 0.12703910000016094,
    "tests/func/test_config.py::test_config_get[args0-0-False]": 0.20948869999995168,
    "tests/func/test_config.py::test_config_get[args1-0-myremote]": 0.1829517000001033,
-    "tests/func/test_config.py::test_config_get[args2-0-iterative]": 0.19686669999998685,
-    "tests/func/test_config.py::test_config_get[args3-0-iterative]": 0.1675059000000374,
+    "tests/func/test_config.py::test_config_get[args2-0-treeverse]": 0.19686669999998685,
+    "tests/func/test_config.py::test_config_get[args3-0-treeverse]": 0.1675059000000374,
    "tests/func/test_config.py::test_config_get[args4-251-option 'profile' doesn't exist]": 0.18096200000002227,
    "tests/func/test_config.py::test_config_get[args5-0-gs://bucket/path]": 0.12368970000022728,
    "tests/func/test_config.py::test_config_get[args6-0-gs://bucket/path]": 0.13007389999995667,
@@ -2642,7 +2642,7 @@
    "tests/unit/test_updater.py::test_is_enabled[config0-True]": 0.13130560000013247,
    "tests/unit/test_updater.py::test_is_enabled[config1-True]": 0.14342600000009043,
    "tests/unit/test_updater.py::test_is_enabled[config2-False]": 0.13593219999984285,
-    "tests/unit/test_updater.py::test_notify_message[None-Find the latest release at https://github.com/iterative/dvc/releases/latest.]": 0.014024399999925663,
+    "tests/unit/test_updater.py::test_notify_message[None-Find the latest release at https://github.com/treeverse/dvc/releases/latest.]": 0.014024399999925663,
    "tests/unit/test_updater.py::test_notify_message[binary-To upgrade, uninstall dvc and reinstall from https://dvc.org.]": 0.013217200000326557,
    "tests/unit/test_updater.py::test_notify_message[brew-To upgrade, run 'brew upgrade dvc'.]": 0.013467300000229443,
    "tests/unit/test_updater.py::test_notify_message[choco-To upgrade, run 'choco upgrade dvc'.]": 0.014645900000004985,
@@ -2652,7 +2652,7 @@
    "tests/unit/test_updater.py::test_notify_message[osxpkg-To upgrade, uninstall dvc and reinstall from https://dvc.org.]": 0.013980499999888707,
    "tests/unit/test_updater.py::test_notify_message[pip-To upgrade, run 'pip install --upgrade dvc'.]": 0.015155299999833005,
    "tests/unit/test_updater.py::test_notify_message[rpm-To upgrade, run 'yum update dvc'.]": 0.014036500000202068,
-    "tests/unit/test_updater.py::test_notify_message[unknown-Find the latest release at https://github.com/iterative/dvc/releases/latest.]": 0.01609110000003966,
+    "tests/unit/test_updater.py::test_notify_message[unknown-Find the latest release at https://github.com/treeverse/dvc/releases/latest.]": 0.01609110000003966,
    "tests/unit/ui/test_console.py::test_capsys_works": 0.003642499999841675,
    "tests/unit/ui/test_console.py::test_write": 0.004320600000255581,
    "tests/unit/ui/test_console.py::test_write_json[False-{\"hello\": \"world\", \"date\": \"1970-01-01 00:00:00\"}\\n]": 0.004433300000073359,
@@ -57,7 +57,7 @@ $ dvc doctor
 **Additional Information (if any):**

 <!--
-Please check https://github.com/iterative/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.
+Please check https://github.com/treeverse/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.

 If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.
 If the issue is regarding the performance, please attach the profiling information and the benchmark comparisons.
@@ -1,5 +1,5 @@
 * [ ] ❗ I have followed the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) checklist.

-* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here.
+* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/treeverse/dvc.org) and linked it here.

 Thank you for the contribution - we'll try to review it as soon as possible. 🙏
@@ -14,16 +14,16 @@ jobs:
    name: run benchmarks
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v7
        with:
-            python-version: "3.13"
+            python-version: "3.14"

-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v7
        with:
          ref: ${{ github.event.pull_request.base.sha }}
          fetch-depth: 0

-      - uses: astral-sh/setup-uv@v6
+      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: pyproject.toml
@@ -32,7 +32,7 @@ jobs:
      - name: run benchmarks on base branch
        run: pytest --benchmark-autosave dvc/testing/benchmarks/ -k 'test_init or test_help'

-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v7
        with:
          fetch-depth: 0
          clean: false
@@ -19,15 +19,15 @@ jobs:
    name: Build & Verify package
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v7
      with:
        fetch-depth: 0

-    - uses: actions/setup-python@v5
+    - uses: actions/setup-python@v7
      with:
-        python-version: 3.13
+        python-version: "3.14"

-    - uses: astral-sh/setup-uv@v6
+    - uses: astral-sh/setup-uv@v7

    - name: Omit local version for Test PyPI upload
      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
@@ -41,7 +41,7 @@ jobs:
    - name: Check dist
      run: uv tool run twine check --strict dist/*

-    - uses: actions/upload-artifact@v4
+    - uses: actions/upload-artifact@v7
      with:
        name: Packages
        path: dist/
@@ -51,7 +51,7 @@ jobs:
    name: Publish dev package to test.pypi.org
    runs-on: ubuntu-latest
    needs: build
-    if: ${{ github.event.action == 'published' || (github.repository == 'iterative/dvc' && github.event_name == 'push' && github.ref == 'refs/heads/main') }}
+    if: ${{ github.event.action == 'published' || (github.repository == 'treeverse/dvc' && github.event_name == 'push' && github.ref == 'refs/heads/main') }}

    environment:
      name: test-pypi
@@ -61,7 +61,7 @@ jobs:
      id-token: write

    steps:
-    - uses: actions/download-artifact@v5
+    - uses: actions/download-artifact@v8
      with:
        name: Packages
        path: dist
@@ -86,7 +86,7 @@ jobs:
      id-token: write

    steps:
-    - uses: actions/download-artifact@v5
+    - uses: actions/download-artifact@v8
      with:
        name: Packages
        path: dist
@@ -28,13 +28,13 @@ jobs:

    steps:
    - name: Checkout repository
-      uses: actions/checkout@v5
+      uses: actions/checkout@v7

    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@v4.38.2
      with:
        languages: ${{ matrix.language }}
        queries: security-extended

    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@v4.38.2
@@ -19,26 +19,26 @@ jobs:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
-        pyv: ["3.13"]
+        pyv: ["3.14"]
        plugin: ["dvc-s3"]

    steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v7
      with:
        path: dvc

-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v7
      with:
-        repository: iterative/${{ matrix.plugin }}
+        repository: treeverse/${{ matrix.plugin }}
        ref: main
        path: ${{ matrix.plugin }}

    - name: Set up Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v7
      with:
        python-version: ${{ matrix.pyv }}

-    - uses: astral-sh/setup-uv@v6
+    - uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true
        cache-dependency-glob: "**/pyproject.toml"
@@ -26,14 +26,14 @@ jobs:
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
    steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v7
      with:
        fetch-depth: 0
    - name: Set up Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v7
      with:
-        python-version: '3.13'
-    - uses: astral-sh/setup-uv@v6
+        python-version: '3.14'
+    - uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true
        cache-dependency-glob: pyproject.toml
@@ -42,7 +42,7 @@ jobs:
        UV_LINK_MODE: ${{ runner.os == 'Windows' && 'symlink' || 'hardlink' }}

    - name: Cache mypy
-      uses: actions/cache@v4
+      uses: actions/cache@v6
      with:
        path: .mypy_cache
        key: mypy-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}
@@ -55,7 +55,7 @@ jobs:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest]
-        pyv: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        pyv: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
        include:
        - os: windows-latest
          pyv: "3.9"
@@ -117,15 +117,27 @@ jobs:
        - os: windows-latest
          pyv: "3.13"
          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
+        - os: windows-latest
+          pyv: "3.14"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
+        - os: windows-latest
+          pyv: "3.14"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
+        - os: windows-latest
+          pyv: "3.14"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
+        - os: windows-latest
+          pyv: "3.14"
+          pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
    steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v7
      with:
        fetch-depth: 0
    - name: Set up Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v7
      with:
        python-version: ${{ matrix.pyv }}
-    - uses: astral-sh/setup-uv@v6
+    - uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true
        cache-dependency-glob: pyproject.toml
@@ -143,7 +155,7 @@ jobs:
        pytest ${{ matrix.pytestargs }} -n=logical --dist=worksteal --timeout=300 --durations=0
        --cov --cov-report=xml --cov-report=term  --durations-path=./.github/.test_durations
    - name: upload coverage report
-      uses: codecov/codecov-action@v5
+      uses: codecov/codecov-action@v7
  check:
    if: always()
    needs: [tests]
@@ -158,7 +170,7 @@ jobs:
    runs-on: ubuntu-latest
    steps:
    - name: Slack Notification
-      uses: rtCamp/action-slack-notify@v2.3.3
+      uses: rtCamp/action-slack-notify@v2.4.0
      env:
        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
        SLACK_COLOR: ${{ job.status }}
@@ -22,13 +22,13 @@ repos:
      - id: sort-simple-yaml
      - id: trailing-whitespace
  - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.12.11'
+    rev: 'v0.15.20'
    hooks:
      - id: ruff-check
        args: [--fix, --exit-non-zero-on-fix]
      - id: ruff-format
  - repo: https://github.com/codespell-project/codespell
-    rev: v2.4.1
+    rev: v2.4.2
    hooks:
      - id: codespell
        additional_dependencies: ["tomli"]
@@ -4,5 +4,5 @@
    "data-science", "data-version-control", "machine-learning", "git",
    "developer-tools", "reproducibility", "collaboration", "ai", "python"],
  "contributors": [
-    {"name": "DVC team", "type": "Other", "affiliation": "Iterative"}]
+    {"name": "DVC team", "type": "Other", "affiliation": "Treeverse"}]
 }
@@ -6,8 +6,8 @@ message: >-
 type: software
 authors:
  - name: The DVC team and contributors
-    website: 'https://github.com/iterative'
-repository-code: 'https://github.com/iterative/dvc'
+    website: 'https://github.com/treeverse'
+repository-code: 'https://github.com/treeverse/dvc'
 url: 'https://dvc.org'
 abstract: >-
  DVC is a tool for data science that takes advantage of existing software engineering toolset.
@@ -186,7 +186,7 @@
      same "printed page" as the copyright notice for easier
      identification within third-party archives.

-   Copyright 2017-2021 Iterative, Inc.
+   Copyright 2025 Treeverse.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -167,7 +167,7 @@ To install the development version, run:

 .. code-block:: bash

-   pip install git+git://github.com/iterative/dvc
+   pip install "dvc @ git+https://github.com/treeverse/dvc"

 Package (Platform-specific)
 ---------------------------
@@ -175,7 +175,7 @@ Package (Platform-specific)
 |Packages|

 Self-contained packages for Linux, Windows, and Mac are available.
-The latest version of the packages can be found on the GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
+The latest version of the packages can be found on the GitHub `releases page <https://github.com/treeverse/dvc/releases>`_.

 Ubuntu / Debian (deb)
 ^^^^^^^^^^^^^^^^^^^^^
@@ -198,8 +198,6 @@ Fedora / CentOS (rpm)
 Contributing
 ============

-|Maintainability|
-
 Contributions are welcome!
 Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more details.
 Thanks to all our contributors!
@@ -237,23 +235,19 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
   :target: https://dvc.org
   :alt: DVC logo

-.. |VS Code Extension Overview| image:: https://raw.githubusercontent.com/iterative/vscode-dvc/main/extension/docs/overview.gif
+.. |VS Code Extension Overview| image:: https://raw.githubusercontent.com/treeverse/vscode-dvc/main/extension/docs/overview.gif
   :alt: DVC Extension for VS Code

-.. |CI| image:: https://github.com/iterative/dvc/actions/workflows/tests.yaml/badge.svg
-   :target: https://github.com/iterative/dvc/actions/workflows/tests.yaml
+.. |CI| image:: https://github.com/treeverse/dvc/actions/workflows/tests.yaml/badge.svg
+   :target: https://github.com/treeverse/dvc/actions/workflows/tests.yaml
   :alt: GHA Tests

-.. |Maintainability| image:: https://img.shields.io/codeclimate/maintainability/iterative/dvc
-   :target: https://codeclimate.com/github/iterative/dvc
-   :alt: Code Climate
-
 .. |Python Version| image:: https://img.shields.io/pypi/pyversions/dvc
   :target: https://pypi.org/project/dvc
   :alt: Python Version

-.. |Coverage| image:: https://codecov.io/gh/iterative/dvc/branch/main/graph/badge.svg
-   :target: https://codecov.io/gh/iterative/dvc
+.. |Coverage| image:: https://codecov.io/gh/treeverse/dvc/branch/main/graph/badge.svg
+   :target: https://codecov.io/gh/treeverse/dvc
   :alt: Codecov

 .. |Snap| image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
@@ -292,8 +286,8 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
   :target: https://dvc.org/img/flow.gif
   :alt: how_dvc_works

-.. |Contribs| image:: https://contrib.rocks/image?repo=iterative/dvc
-   :target: https://github.com/iterative/dvc/graphs/contributors
+.. |Contribs| image:: https://contrib.rocks/image?repo=treeverse/dvc
+   :target: https://github.com/treeverse/dvc/graphs/contributors
   :alt: Contributors

 .. |VS Code| image:: https://img.shields.io/visual-studio-marketplace/v/Iterative.dvc?color=blue&label=VSCode&logo=visualstudiocode&logoColor=blue
@@ -8,7 +8,7 @@ from PyInstaller.utils.hooks import collect_submodules, is_module_or_submodule
 _EXCLUDES = ("celery.bin", "celery.contrib")
 hiddenimports = collect_submodules(
    "celery",
-    filter=lambda name: not any(
-        is_module_or_submodule(name, module) for module in _EXCLUDES
+    filter=lambda name: (
+        not any(is_module_or_submodule(name, module) for module in _EXCLUDES)
    ),
 )
@@ -8,7 +8,6 @@ datas += copy_metadata("gcsfs")
 datas += copy_metadata("pyarrow")
 datas += copy_metadata("pydrive2")
 datas += copy_metadata("s3fs", recursive=True)
-datas += copy_metadata("boto3")
 datas += copy_metadata("ossfs")
 datas += copy_metadata("sshfs")
 datas += copy_metadata("webdav4")
@@ -206,7 +206,7 @@ def add_debugging_flags(parser):
    from argparse import SUPPRESS

    # For detailed info see:
-    # https://github.com/iterative/dvc/wiki/Debugging,-Profiling-and-Benchmarking-DVC
+    # https://github.com/treeverse/dvc/wiki/Debugging,-Profiling-and-Benchmarking-DVC
    args, _ = parser.parse_known_args()
    verbose = args.verbose

@@ -147,7 +147,7 @@ def _git_remote_path_hash(scm: Optional["Base"]) -> Optional[str]:
            path = _parse_git_remote_path(remote_url)
            h = hashlib.md5(path.encode("utf-8"), usedforsecurity=False)  # for FIPS
            return h.hexdigest()
-    except Exception:  # noqa: BLE001
+    except Exception:
        logger.debug("Failed to get git remote path", exc_info=True)
    return None

@@ -64,3 +64,4 @@ def get(name: str) -> Union[DatachainDataset, DVCDataset, URLDataset]:
            for file in dataset.lock.files
        ]
        return URLDataset(type="url", files=files, path=versioned_path)
+    raise AssertionError("unreachable")
@@ -96,7 +96,7 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
                f"See {link} for more info."
            )
        else:
-            link = format_link("https://github.com/iterative/dvc/issues")
+            link = format_link("https://github.com/treeverse/dvc/issues")
            hint = f"\nPlease report this bug to {link}. Thank you!"

        logger.exception(
@@ -27,6 +27,10 @@ _dvc_compgen_remotes() {
    local _dvc_remotes=($(dvc remote list | cut -d' ' -f1))
    compgen -W "${_dvc_remotes[*]}" -- $1
 }
+
+_dvc_compgen_config_vars() {
+    compgen -W "${_dvc_config_vars[*]}" -- $1
+}
 """

 ZSH_PREAMBLE = """
@@ -64,6 +68,10 @@ _dvc_compadd_exps() {
 _dvc_compadd_remotes() {
    _describe 'remotes' "($(dvc remote list | cut -d' ' -f1))"
 }
+
+_dvc_compadd_config_vars() {
+    _describe 'config_vars' _dvc_config_vars
+}
 """

 PREAMBLE = {
@@ -81,3 +89,21 @@ DVCFILES_AND_STAGE = {
 }
 EXPERIMENT = {"bash": "_dvc_compgen_exps", "zsh": "_dvc_compadd_exps"}
 REMOTE = {"bash": "_dvc_compgen_remotes", "zsh": "_dvc_compadd_remotes"}
+CONFIG_VARS = {"bash": "_dvc_compgen_config_vars", "zsh": "_dvc_compadd_config_vars"}
+
+
+def get_preamble() -> dict[str, str]:
+    from dvc.config_schema import config_vars_for_completion
+
+    ret: dict[str, str] = {}
+    config_vars = list(config_vars_for_completion())
+
+    nl = "\n\t".expandtabs(4)
+    config_vars_arr = f"""
+_dvc_config_vars=(
+    {nl.join(config_vars)}
+)
+"""
+    for shell, preamble in PREAMBLE.items():
+        ret[shell] = config_vars_arr + preamble
+    return ret
@@ -16,7 +16,7 @@ class CmdCheckIgnore(CmdBase):

    def _show_results(self, result: "CheckIgnoreResult"):
        if not result.match and not self.args.details:
-            return
+            return None

        if self.args.details:
            pattern_infos = result.pattern_infos
@@ -1,6 +1,6 @@
 from dvc.cli import formatter
 from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.completion import PREAMBLE
+from dvc.cli.completion import get_preamble
 from dvc.cli.utils import append_doc_link
 from dvc.log import logger
 from dvc.ui import ui
@@ -17,7 +17,7 @@ class CmdCompletion(CmdBaseNoRepo):

        shell = self.args.shell
        parser = self.args.parser
-        script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
+        script = shtab.complete(parser, shell=shell, preamble=get_preamble())
        ui.write(script, force=True)
        return 0

@@ -3,7 +3,7 @@ import os

 from funcy import set_in

-from dvc.cli import formatter
+from dvc.cli import completion, formatter
 from dvc.cli.command import CmdBaseNoRepo
 from dvc.cli.utils import append_doc_link
 from dvc.log import logger
@@ -224,7 +224,7 @@ def add_parser(subparsers, parent_parser):
        nargs="?",
        type=_name_type,
        help="Option name (section.option or remote.name.option).",
-    )
+    ).complete = completion.CONFIG_VARS
    config_parser.add_argument("value", nargs="?", help="Option value.")
    config_parser.add_argument(
        "-l",
@@ -127,19 +127,80 @@ def _filter(graph, targets, full):
    return new_graph


-def _build(repo, target=None, full=False, outs=False):
+def _is_foreach_matrix_stage(node, join_string):
+    if node.endswith(".dvc"):
+        return False
+    return join_string in node
+
+
+def _collapse_foreach_matrix_get_nodes(graph):
+    from dvc.parsing import JOIN
+
+    new_nodes = set()
+    nodes_to_remove = set()
+    for _node in list(graph.nodes):
+        if not _is_foreach_matrix_stage(_node, JOIN):
+            continue
+        nodes_to_remove.add(_node)
+        new_nodes.add(_node.split(JOIN)[0])
+    return new_nodes, nodes_to_remove
+
+
+def _collapse_foreach_matrix_get_edges(graph):
+    from dvc.parsing import JOIN
+
+    new_edges = set()
+    edges_to_remove = set()
+    for _e1, _e2 in list(graph.edges):
+        _replace = False
+        _new_e1 = _e1
+        _new_e2 = _e2
+        if _is_foreach_matrix_stage(_e1, JOIN):
+            _new_e1 = _e1.split(JOIN)[0]
+            _replace = True
+        if _is_foreach_matrix_stage(_e2, JOIN):
+            _new_e2 = _e2.split(JOIN)[0]
+            _replace = True
+        if _replace:
+            edges_to_remove.add((_e1, _e2))
+            new_edges.add((_new_e1, _new_e2))
+    return new_edges, edges_to_remove
+
+
+def _collapse_foreach_matrix(graph):
+    new_nodes, nodes_to_remove = _collapse_foreach_matrix_get_nodes(graph)
+    new_edges, edges_to_remove = _collapse_foreach_matrix_get_edges(graph)
+    new_graph = graph.copy()
+    new_graph.remove_edges_from(edges_to_remove)
+    new_graph.add_nodes_from(new_nodes)
+    new_graph.add_edges_from(new_edges)
+    new_graph.remove_nodes_from(nodes_to_remove)
+    return new_graph
+
+
+def _build(repo, target=None, full=False, outs=False, collapse_foreach_matrix=False):
    targets = _collect_targets(repo, target, outs)
    graph = _transform(repo.index, outs)
-    return _filter(graph, targets, full)
+    filtered_graph = _filter(graph, targets, full)
+    if collapse_foreach_matrix:
+        filtered_graph = _collapse_foreach_matrix(filtered_graph)
+    return filtered_graph


 class CmdDAG(CmdBase):
    def run(self):
+        from dvc.exceptions import InvalidArgumentError
+
+        if self.args.outs and self.args.collapse_foreach_matrix:
+            raise InvalidArgumentError(
+                "`--outs` and `--collapse-foreach-matrix` are mutually exclusive"
+            )
        graph = _build(
            self.repo,
            target=self.args.target,
            full=self.args.full,
            outs=self.args.outs,
+            collapse_foreach_matrix=self.args.collapse_foreach_matrix,
        )

        if self.args.dot:
@@ -181,6 +242,14 @@ def add_parser(subparsers, parent_parser):
        dest="markdown",
        help="Print DAG with mermaid format wrapped in Markdown block.",
    )
+    dag_parser.add_argument(
+        "--collapse-foreach-matrix",
+        action="store_true",
+        default=False,
+        help=(
+            "Collapse stages from each foreach/matrix definition into a single node."
+        ),
+    )
    dag_parser.add_argument(
        "--full",
        action="store_true",
@@ -83,7 +83,7 @@ class CmdDatasetUpdate(CmdBase):
            return CmdDatasetAdd.display(name, new, action)
        if dataset == new:
            ui.write("[yellow]Nothing to update[/]", styled=True)
-            return
+            return None

        assert new.lock

@@ -112,6 +112,7 @@ class CmdDatasetUpdate(CmdBase):
            assert new.type == "url"
            stats = diff_files(dataset.lock.files, new.lock.files)
            log_changes(stats)
+        return None

    def run(self):
        from difflib import get_close_matches
@@ -20,6 +20,7 @@ class CmdExperimentsRun(CmdRepro):
            tmp_dir=self.args.tmp_dir,
            copy_paths=self.args.copy_paths,
            message=self.args.message,
+            no_hydra=self.args.no_hydra,
            **self._common_kwargs,
        )

@@ -107,4 +108,14 @@ def _add_run_common(parser):
        default=None,
        help="Custom commit message to use when committing the experiment.",
    )
+    parser.add_argument(
+        "--no-hydra",
+        action="store_true",
+        default=False,
+        help=(
+            "Disables automatically updating `params.yaml` with Hydra configuration. "
+            " You can still use `--set-param` to update individual params if needed."
+            " Default is False."
+        ),
+    )
    parser.add_argument("-M", dest="message", help=argparse.SUPPRESS)  # obsolete
@@ -40,9 +40,16 @@ class CmdGC(CmdBase):
            msg += " and all git commits"
        else:
            if self.args.all_branches and self.args.all_tags:
-                msg += " and all git branches and tags"
+                if self.args.num:
+                    msg += f" and last {self.args.num} commits from all git branches"
+                    msg += " and all git tags"
+                else:
+                    msg += " and all git branches and tags"
            elif self.args.all_branches:
-                msg += " and all git branches"
+                if self.args.num:
+                    msg += f" and last {self.args.num} commits from all git branches"
+                else:
+                    msg += " and all git branches"
            elif self.args.all_tags:
                msg += " and all git tags"
            if self.args.commit_date:
@@ -126,8 +133,8 @@ def add_parser(subparsers, parent_parser):
        metavar="<num>",
        help=(
            "Keep data files used in the last `num` commits "
-            "starting from the `--rev` <commit>. "
-            "Only used if `--rev` is also provided. "
+            "starting from each selected revision root. "
+            "Can be used with `--rev` and `--all-branches`. "
            "Defaults to `1`."
        ),
    )
@@ -29,7 +29,7 @@ def _welcome_message():
        "{yellow}------------{nc}\n"
        f"- Check out the documentation: {fmt_link('https://dvc.org/doc')}\n"
        f"- Get help and share ideas: {fmt_link('https://dvc.org/chat')}\n"
-        f"- Star us on GitHub: {fmt_link('https://github.com/iterative/dvc')}"
+        f"- Star us on GitHub: {fmt_link('https://github.com/treeverse/dvc')}"
    ).format(yellow=colorama.Fore.YELLOW, nc=colorama.Fore.RESET)

    ui.write(msg)
@@ -109,8 +109,6 @@ def _build_tree_structure(


 def show_tree(entries, with_color=False, with_size=False, with_hash=False):
-    import tabulate
-
    rows = _build_tree_structure(
        entries,
        with_color=with_color,
@@ -119,13 +117,7 @@ def show_tree(entries, with_color=False, with_size=False, with_hash=False):
    )

    colalign = ("right",) if with_size else None
-
-    _orig = tabulate.PRESERVE_WHITESPACE
-    tabulate.PRESERVE_WHITESPACE = True
-    try:
-        ui.table(rows, colalign=colalign)
-    finally:
-        tabulate.PRESERVE_WHITESPACE = _orig
+    ui.table(rows, colalign=colalign, preserve_whitespace=True)


 class CmdList(CmdBaseNoRepo):
@@ -41,18 +41,6 @@ class RemoteNotFoundError(RemoteConfigError):
    pass


-class MachineConfigError(ConfigError):
-    pass
-
-
-class NoMachineError(MachineConfigError):
-    pass
-
-
-class MachineNotFoundError(MachineConfigError):
-    pass
-
-
@memoize
 def get_compiled_schema():
    from voluptuous import Schema
@@ -139,6 +127,7 @@ class Config(dict):
            return global_config_dir()
        if level == "system":
            return system_config_dir()
+        return None

    @cached_property
    def files(self) -> dict[str, str]:
@@ -1,4 +1,5 @@
 import os
+from typing import TYPE_CHECKING
 from urllib.parse import urlparse

 from funcy import once, walk_values
@@ -10,6 +11,7 @@ from voluptuous import (
    Exclusive,
    Invalid,
    Lower,
+    Marker,
    Optional,
    Range,
    Schema,
@@ -17,6 +19,9 @@ from voluptuous import (

 from dvc.log import logger

+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
 logger = logger.getChild(__name__)

 Bool = All(
@@ -112,17 +117,19 @@ class FeatureSchema(Schema):
        return ret


+DEPRECATED = "==DEPRECATED=="
+
 REMOTE_COMMON = {
    "url": str,
    "checksum_jobs": All(Coerce(int), Range(1)),
    "jobs": All(Coerce(int), Range(1)),
    Optional("worktree"): Bool,
-    Optional("no_traverse"): Bool,  # obsoleted
+    Optional("no_traverse", description=DEPRECATED): Bool,  # obsoleted
    Optional("version_aware"): Bool,
 }
 LOCAL_COMMON = {
    "type": supported_cache_type,
-    Optional("protected", default=False): Bool,  # obsoleted
+    Optional("protected", default=False, description=DEPRECATED): Bool,  # obsoleted
    "shared": All(Lower, Choices("group")),
    Optional("slow_link_warning", default=True): Bool,
    Optional("verify", default=False): Bool,
@@ -144,6 +151,7 @@ WEBDAV_COMMON = {
    "password": str,
    "ask_password": Bool,
    "token": str,
+    "bearer_token_command": str,
    "custom_auth_header": str,
    "cert_path": str,
    "key_path": str,
@@ -152,6 +160,127 @@ WEBDAV_COMMON = {
    Optional("verify", default=False): Bool,
 }

+REMOTE_SCHEMAS = {
+    "": LOCAL_COMMON | REMOTE_COMMON,
+    "s3": {
+        "region": str,
+        "profile": str,
+        "credentialpath": str,
+        "configpath": str,
+        "endpointurl": str,
+        "access_key_id": str,
+        "secret_access_key": str,
+        "session_token": str,
+        Optional(
+            "listobjects", default=False, description=DEPRECATED
+        ): Bool,  # obsoleted
+        Optional("use_ssl", default=True): Bool,
+        Optional("allow_anonymous_login", default=False): Bool,
+        "ssl_verify": Any(Bool, str),
+        "sse": str,
+        "sse_kms_key_id": str,
+        "sse_customer_algorithm": str,
+        "sse_customer_key": str,
+        "acl": str,
+        "grant_read": str,
+        "grant_read_acp": str,
+        "grant_write_acp": str,
+        "grant_full_control": str,
+        "cache_regions": bool,
+        "read_timeout": Coerce(int),
+        "connect_timeout": Coerce(int),
+        Optional("verify", default=False): Bool,
+        **REMOTE_COMMON,
+    },
+    "gs": {
+        "projectname": str,
+        "credentialpath": str,
+        "endpointurl": str,
+        Optional("verify", default=False): Bool,
+        Optional("allow_anonymous_login", default=False): Bool,
+        **REMOTE_COMMON,
+    },
+    "ssh": {
+        "type": supported_cache_type,
+        "port": Coerce(int),
+        "user": str,
+        "password": str,
+        "ask_password": Bool,
+        "passphrase": str,
+        "ask_passphrase": Bool,
+        "keyfile": str,
+        "timeout": Coerce(int),
+        "gss_auth": Bool,
+        "allow_agent": Bool,
+        "max_sessions": Coerce(int),
+        Optional("verify", default=False): Bool,
+        **REMOTE_COMMON,
+    },
+    "hdfs": {
+        "user": str,
+        "kerb_ticket": str,
+        "replication": int,
+        **REMOTE_COMMON,
+    },
+    "webhdfs": {
+        "kerberos": Bool,
+        "kerberos_principal": str,
+        "proxy_to": str,
+        "ssl_verify": Any(Bool, str),
+        "token": str,
+        "use_https": Bool,
+        "user": str,
+        "password": str,
+        "data_proxy_target": str,
+        Optional("verify", default=False): Bool,
+        **REMOTE_COMMON,
+    },
+    "azure": {
+        "connection_string": str,
+        "sas_token": str,
+        "account_name": str,
+        "account_key": str,
+        "tenant_id": str,
+        "client_id": str,
+        "client_secret": str,
+        "allow_anonymous_login": Bool,
+        "exclude_environment_credential": Bool,
+        "exclude_visual_studio_code_credential": Bool,
+        "exclude_shared_token_cache_credential": Bool,
+        "exclude_managed_identity_credential": Bool,
+        Optional("verify", default=False): Bool,
+        "timeout": Coerce(int),
+        "read_timeout": Coerce(int),
+        "connection_timeout": Coerce(int),
+        **REMOTE_COMMON,
+    },
+    "oss": {
+        "oss_key_id": str,
+        "oss_key_secret": str,
+        "oss_endpoint": str,
+        Optional("verify", default=True): Bool,
+        **REMOTE_COMMON,
+    },
+    "gdrive": {
+        "profile": str,
+        "gdrive_use_service_account": Bool,
+        "gdrive_client_id": str,
+        "gdrive_client_secret": str,
+        "gdrive_user_credentials_file": str,
+        "gdrive_service_account_user_email": str,
+        "gdrive_service_account_json_file_path": str,
+        Optional("gdrive_trash_only", default=False): Bool,
+        Optional("gdrive_acknowledge_abuse", default=False): Bool,
+        Optional("verify", default=True): Bool,
+        **REMOTE_COMMON,
+    },
+    "http": HTTP_COMMON | REMOTE_COMMON,
+    "https": HTTP_COMMON | REMOTE_COMMON,
+    "webdav": WEBDAV_COMMON | REMOTE_COMMON,
+    "webdavs": WEBDAV_COMMON | REMOTE_COMMON,
+    "remote": {str: object},  # Any of the above options are valid
+}
+
 SCHEMA = {
    "core": {
        "remote": Lower,
@@ -161,152 +290,37 @@ SCHEMA = {
        Optional("hardlink_lock", default=False): Bool,
        Optional("no_scm", default=False): Bool,
        Optional("autostage", default=False): Bool,
-        Optional("experiments"): Bool,  # obsoleted
+        Optional("experiments", description=DEPRECATED): Bool,  # obsoleted
        Optional("check_update", default=True): Bool,
        "site_cache_dir": str,
        "machine": Lower,
    },
    "cache": {
-        "local": str,  # obsoleted
-        "s3": str,  # obsoleted
-        "gs": str,  # obsoleted
-        "hdfs": str,  # obsoleted
-        "webhdfs": str,  # obsoleted
-        "ssh": str,  # obsoleted
-        "azure": str,  # obsoleted
+        Marker("local", description=DEPRECATED): str,  # obsoleted
+        Marker("s3", description=DEPRECATED): str,  # obsoleted
+        Marker("gs", description=DEPRECATED): str,  # obsoleted
+        Marker("hdfs", description=DEPRECATED): str,  # obsoleted
+        Marker("webhdfs", description=DEPRECATED): str,  # obsoleted
+        Marker("ssh", description=DEPRECATED): str,  # obsoleted
+        Marker("azure", description=DEPRECATED): str,  # obsoleted
        # This is for default local cache
        "dir": str,
        **LOCAL_COMMON,
    },
    "remote": {
-        str: ByUrl(
-            {
-                "": LOCAL_COMMON | REMOTE_COMMON,
-                "s3": {
-                    "region": str,
-                    "profile": str,
-                    "credentialpath": str,
-                    "configpath": str,
-                    "endpointurl": str,
-                    "access_key_id": str,
-                    "secret_access_key": str,
-                    "session_token": str,
-                    Optional("listobjects", default=False): Bool,  # obsoleted
-                    Optional("use_ssl", default=True): Bool,
-                    Optional("allow_anonymous_login", default=False): Bool,
-                    "ssl_verify": Any(Bool, str),
-                    "sse": str,
-                    "sse_kms_key_id": str,
-                    "sse_customer_algorithm": str,
-                    "sse_customer_key": str,
-                    "acl": str,
-                    "grant_read": str,
-                    "grant_read_acp": str,
-                    "grant_write_acp": str,
-                    "grant_full_control": str,
-                    "cache_regions": bool,
-                    "read_timeout": Coerce(int),
-                    "connect_timeout": Coerce(int),
-                    Optional("verify", default=False): Bool,
-                    **REMOTE_COMMON,
-                },
-                "gs": {
-                    "projectname": str,
-                    "credentialpath": str,
-                    "endpointurl": str,
-                    Optional("verify", default=False): Bool,
-                    Optional("allow_anonymous_login", default=False): Bool,
-                    **REMOTE_COMMON,
-                },
-                "ssh": {
-                    "type": supported_cache_type,
-                    "port": Coerce(int),
-                    "user": str,
-                    "password": str,
-                    "ask_password": Bool,
-                    "passphrase": str,
-                    "ask_passphrase": Bool,
-                    "keyfile": str,
-                    "timeout": Coerce(int),
-                    "gss_auth": Bool,
-                    "allow_agent": Bool,
-                    "max_sessions": Coerce(int),
-                    Optional("verify", default=False): Bool,
-                    **REMOTE_COMMON,
-                },
-                "hdfs": {
-                    "user": str,
-                    "kerb_ticket": str,
-                    "replication": int,
-                    **REMOTE_COMMON,
-                },
-                "webhdfs": {
-                    "kerberos": Bool,
-                    "kerberos_principal": str,
-                    "proxy_to": str,
-                    "ssl_verify": Any(Bool, str),
-                    "token": str,
-                    "use_https": Bool,
-                    "user": str,
-                    "password": str,
-                    "data_proxy_target": str,
-                    Optional("verify", default=False): Bool,
-                    **REMOTE_COMMON,
-                },
-                "azure": {
-                    "connection_string": str,
-                    "sas_token": str,
-                    "account_name": str,
-                    "account_key": str,
-                    "tenant_id": str,
-                    "client_id": str,
-                    "client_secret": str,
-                    "allow_anonymous_login": Bool,
-                    "exclude_environment_credential": Bool,
-                    "exclude_visual_studio_code_credential": Bool,
-                    "exclude_shared_token_cache_credential": Bool,
-                    "exclude_managed_identity_credential": Bool,
-                    Optional("verify", default=False): Bool,
-                    "timeout": Coerce(int),
-                    "read_timeout": Coerce(int),
-                    "connection_timeout": Coerce(int),
-                    **REMOTE_COMMON,
-                },
-                "oss": {
-                    "oss_key_id": str,
-                    "oss_key_secret": str,
-                    "oss_endpoint": str,
-                    Optional("verify", default=True): Bool,
-                    **REMOTE_COMMON,
-                },
-                "gdrive": {
-                    "profile": str,
-                    "gdrive_use_service_account": Bool,
-                    "gdrive_client_id": str,
-                    "gdrive_client_secret": str,
-                    "gdrive_user_credentials_file": str,
-                    "gdrive_service_account_user_email": str,
-                    "gdrive_service_account_json_file_path": str,
-                    Optional("gdrive_trash_only", default=False): Bool,
-                    Optional("gdrive_acknowledge_abuse", default=False): Bool,
-                    Optional("verify", default=True): Bool,
-                    **REMOTE_COMMON,
-                },
-                "http": HTTP_COMMON | REMOTE_COMMON,
-                "https": HTTP_COMMON | REMOTE_COMMON,
-                "webdav": WEBDAV_COMMON | REMOTE_COMMON,
-                "webdavs": WEBDAV_COMMON | REMOTE_COMMON,
-                "remote": {str: object},  # Any of the above options are valid
-            }
-        )
+        str: ByUrl(REMOTE_SCHEMAS),
    },
    "state": {
-        "dir": str,  # obsoleted
-        "row_limit": All(Coerce(int), Range(1)),  # obsoleted
-        "row_cleanup_quota": All(Coerce(int), Range(0, 100)),  # obsoleted
+        Marker("dir", description=DEPRECATED): str,  # obsoleted
+        Marker("row_limit", description=DEPRECATED): All(
+            Coerce(int), Range(1)
+        ),  # obsoleted
+        Marker("row_cleanup_quota", description=DEPRECATED): All(
+            Coerce(int), Range(0, 100)
+        ),  # obsoleted
    },
    "index": {
-        "dir": str,  # obsoleted
+        Marker("dir", description=DEPRECATED): str,  # obsoleted
    },
    "machine": {
        str: {
@@ -336,13 +350,13 @@ SCHEMA = {
        "out_dir": str,
    },
    "exp": {
-        "code": str,
-        "data": str,
-        "models": str,
-        "metrics": str,
-        "params": str,
-        "plots": str,
-        "live": str,
+        Marker("code", description=DEPRECATED): str,
+        Marker("data", description=DEPRECATED): str,
+        Marker("models", description=DEPRECATED): str,
+        Marker("metrics", description=DEPRECATED): str,
+        Marker("params", description=DEPRECATED): str,
+        Marker("plots", description=DEPRECATED): str,
+        Marker("live", description=DEPRECATED): str,
        "auto_push": Bool,
        "git_remote": str,
    },
@@ -371,3 +385,21 @@ SCHEMA = {
        },
    },
 }
+
+
+def config_vars_for_completion(d: dict = SCHEMA, path: str = "") -> "Iterator[str]":
+    for k, v in d.items():
+        if k in ("machine", "feature"):
+            continue
+        if isinstance(k, Marker):
+            if k.description == DEPRECATED:
+                continue
+            k = k.schema
+        if not isinstance(k, str):
+            continue
+
+        keypath = path + k
+        if isinstance(v, dict):
+            yield from config_vars_for_completion(v, keypath + ".")
+        else:
+            yield keypath
@@ -84,7 +84,7 @@ class Client:
        except Exception as exc:
            if callable(onerror):
                onerror()
-            logger.exception(
+            logger.exception(  # noqa: LOG007
                "Could not connect to the database. "
                "Check your database credentials and try again.",
                exc_info=False,
@@ -41,7 +41,7 @@ def _get(stage, p, info, **kwargs):
    return Dependency(stage, p, info, **kwargs)


-def loadd_from(stage, d_list):
+def loadd_from(stage, d_list) -> list[Dependency]:
    ret = []
    for d in d_list:
        p = d.pop(Output.PARAM_PATH, None)
@@ -110,6 +110,7 @@ class ParamsDependency(Dependency):
        except MissingParamsFile:
            return {}

+        assert self.repo
        try:
            return read_param_file(
                self.repo.fs,
@@ -150,6 +150,7 @@ class RepoDependency(Dependency):
        # local cache instead of the remote's cache. This avoids re-streaming of data,
        # but messes up the call to `_get_remote_config()` downstream, which will need
        # to ignore cache parameters.
+        assert self.repo
        config["cache"] = self.repo.config["cache"]
        config["cache"]["dir"] = self.repo.cache.local_cache_dir

@@ -203,7 +203,7 @@ class SingleStageFile(FileMixin):

    def dump_stages(self, stages, **kwargs) -> None:
        if not stages:
-            return
+            return None

        assert len(stages) == 1, "SingleStageFile can only dump one stage."
        return self.dump(stages[0], **kwargs)
@@ -9,7 +9,7 @@ if TYPE_CHECKING:
    from dvc.stage import Stage


-class DvcException(Exception):
+class DvcException(Exception):  # noqa: N818
    """Base class for all dvc exceptions."""

    def __init__(self, msg, *args):
@@ -5,7 +5,15 @@ from collections.abc import Iterable, Iterator
 from itertools import chain, groupby, takewhile
 from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Optional, Union, overload

-from pathspec.patterns import GitWildMatchPattern
+try:
+    from pathspec.patterns.gitignore.spec import (  # type: ignore[import-not-found]
+        GitIgnoreSpecPattern,
+    )
+except ImportError:  # pathspec<1
+    from pathspec.patterns import (
+        GitWildMatchPattern as GitIgnoreSpecPattern,
+    )
+
 from pathspec.util import normalize_file
 from pygtrie import Trie

@@ -34,7 +42,14 @@ class DvcIgnorePatterns(DvcIgnore):
    def __init__(
        self, pattern_list: Iterable[Union[PatternInfo, str]], dirname: str, sep: str
    ) -> None:
-        from pathspec.patterns.gitwildmatch import _DIR_MARK
+        try:
+            from pathspec.patterns.gitignore.spec import (  # type: ignore[import-not-found]
+                _DIR_MARK,
+            )
+        except ImportError:  # pathspec<1
+            from pathspec.patterns.gitwildmatch import (  # type: ignore[attr-defined, no-redef]
+                _DIR_MARK,
+            )

        pattern_infos = [
            pattern if isinstance(pattern, PatternInfo) else PatternInfo(pattern, "")
@@ -48,7 +63,7 @@ class DvcIgnorePatterns(DvcIgnore):

        regex_pattern_list: list[tuple[str, bool, bool, PatternInfo]] = []
        for count, pattern_info in enumerate(pattern_infos):
-            regex, ignore = GitWildMatchPattern.pattern_to_regex(pattern_info.patterns)
+            regex, ignore = GitIgnoreSpecPattern.pattern_to_regex(pattern_info.patterns)
            if regex is not None and ignore is not None:
                self.pattern_list.append(pattern_info)
                regex = regex.replace(f"<{_DIR_MARK}>", f"<{_DIR_MARK}{count}>")
@@ -508,7 +523,7 @@ class DvcIgnoreFilter:

    def check_ignore(self, target: str) -> CheckIgnoreResult:
        # NOTE: can only be used in `dvc check-ignore`, see
-        # https://github.com/iterative/dvc/issues/5046
+        # https://github.com/treeverse/dvc/issues/5046
        full_target = self.fs.abspath(target)
        matched_patterns: list[PatternInfo] = []
        ignore = False
@@ -525,7 +540,7 @@ class DvcIgnoreFilter:
        self, fs: "FileSystem", path: str, ignore_subrepos: bool = True
    ) -> bool:
        # NOTE: can't use self.check_ignore(path).match for now, see
-        # https://github.com/iterative/dvc/issues/4555
+        # https://github.com/treeverse/dvc/issues/4555
        if fs.protocol != Schemes.LOCAL:
            return False
        if fs.isfile(path):
@@ -158,7 +158,7 @@ def _get_fs_type(path):
            except PermissionError:
                pass

-    # need to follow the symlink: https://github.com/iterative/dvc/issues/5065
+    # need to follow the symlink: https://github.com/treeverse/dvc/issues/5065
    path = pathlib.Path(path).resolve()

    for parent in itertools.chain([path], path.parents):
@@ -182,7 +182,7 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
        # number to generate the resulting lock file name, which is unique
        # enough for our application.
        #
-        # [1] https://github.com/iterative/dvc/issues/2582
+        # [1] https://github.com/treeverse/dvc/issues/2582
        self._hostname = socket.gethostname()

        self._lifetime = timedelta(days=365)  # Lock for good by default
@@ -1,8 +1,6 @@
 """Manages logging configuration for DVC repo."""

 import logging
-import logging.config
-import logging.handlers
 import os
 import sys
 from collections.abc import Iterator
@@ -55,7 +53,7 @@ def add_logging_level(level_name, level_num, method_name=None):
        setattr(logging, method_name, log_to_root)


-class LoggingException(Exception):
+class LoggingException(Exception):  # noqa: N818
    def __init__(self, record):
        msg = f"failed to log {record!s}"
        super().__init__(msg)
@@ -42,6 +42,7 @@ from .utils import relpath
 from .utils.fs import path_isin

 if TYPE_CHECKING:
+    from dvc.repo import Repo
    from dvc_data.hashfile.obj import HashFile
    from dvc_data.index import DataIndexKey

@@ -83,7 +84,7 @@ def _get(stage, path, **kwargs):
    return Output(stage, path, **kwargs)


-def loadd_from(stage, d_list):
+def loadd_from(stage, d_list) -> list["Output"]:
    ret = []
    for d in d_list:
        p = d.pop(Output.PARAM_PATH)
@@ -329,7 +330,7 @@ class Output:
        self.annot = Annotation(
            desc=desc, type=type, labels=labels or [], meta=meta or {}
        )
-        self.repo = stage.repo if not repo and stage else repo
+        self.repo: Optional[Repo] = stage.repo if not repo and stage else repo
        meta_d = merge_file_meta_from_cloud(info or {})
        meta = Meta.from_dict(meta_d)
        # NOTE: when version_aware is not passed into get_cloud_fs, it will be
@@ -511,6 +512,7 @@ class Output:
        from dvc.cachemgr import LEGACY_HASH_NAMES

        assert self.is_in_repo
+        assert self.repo
        odb_name = "legacy" if self.hash_name in LEGACY_HASH_NAMES else "repo"
        return getattr(self.repo.cache, odb_name)

@@ -518,6 +520,7 @@ class Output:
    def local_cache(self):
        from dvc.cachemgr import LEGACY_HASH_NAMES

+        assert self.repo
        if self.hash_name in LEGACY_HASH_NAMES:
            return self.repo.cache.legacy
        return self.repo.cache.local
@@ -585,6 +588,7 @@ class Output:
    def index_key(self) -> tuple[str, "DataIndexKey"]:
        if self.is_in_repo:
            workspace = "repo"
+            assert self.repo
            key = self.repo.fs.relparts(self.fs_path, self.repo.root_dir)
        else:
            workspace = self.fs.protocol
@@ -640,6 +644,7 @@ class Output:
    @property
    def dvcignore(self) -> Optional["DvcIgnoreFilter"]:
        if self.fs.protocol == "local":
+            assert self.repo
            return self.repo.dvcignore
        return None

@@ -661,6 +666,7 @@ class Output:
        if not self.use_scm_ignore:
            return

+        assert self.repo
        if self.repo.scm.is_tracked(self.fspath):
            raise OutputAlreadyTrackedError(self)

@@ -670,6 +676,7 @@ class Output:
        if not self.use_scm_ignore:
            return

+        assert self.repo
        self.repo.scm_context.ignore_remove(self.fspath)

    def save(self) -> None:
@@ -776,6 +783,7 @@ class Output:
                        callback=cb,
                    )
            if relink:
+                assert self.repo
                rel = self.fs.relpath(filter_info or self.fs_path)
                with CheckoutCallback(desc=f"Checking out {rel}", unit="files") as cb:
                    self._checkout(
@@ -957,6 +965,7 @@ class Output:
        added = not self.exists

        try:
+            assert self.repo
            modified = self._checkout(
                filter_info or self.fs_path,
                self.fs,
@@ -987,18 +996,24 @@ class Output:
        if ignore_remove:
            self.ignore_remove()

-    def move(self, out):
+    def move(self, out: "Output") -> None:
+        src_exists = self.exists
+        if src_exists:
+            self.fs.move(self.fs_path, out.fs_path)
+        else:
+            logger.warning("%r missing", self.fspath)
+
        if self.protocol == "local" and self.use_scm_ignore:
+            assert self.repo
            self.repo.scm_context.ignore_remove(self.fspath)

-        self.fs.move(self.fs_path, out.fs_path)
        self.def_path = out.def_path
        self.fs_path = out.fs_path
-        self.save()
-        self.commit()
-
-        if self.protocol == "local" and self.use_scm_ignore:
-            self.repo.scm_context.ignore(self.fspath)
+        try:
+            self.save()
+            self.commit()
+        except self.DoesNotExistError:
+            self.ignore()

    def transfer(
        self, source, odb=None, jobs=None, update=False, no_progress_bar=False
@@ -1079,6 +1094,7 @@ class Output:
            if self.remote:
                kwargs["remote"] = self.remote
            with suppress(Exception):
+                assert self.repo
                self.repo.cloud.pull([obj.hash_info], **kwargs)

        if self.obj:
@@ -1172,6 +1188,7 @@ class Output:
            return {}

        if self.remote:
+            assert self.repo
            remote_odb = self.repo.cloud.get_remote_odb(
                name=self.remote, hash_name=self.hash_name
            )
@@ -1473,12 +1490,13 @@ class Output:
    ):
        """Merge version meta for files which are unchanged from other."""
        if not self.hash_info:
-            return
+            return None
        if self.hash_info.isdir:
            return self._merge_dir_version_meta(old_hi, old_obj)
        if self.hash_info != old_hi:
-            return
+            return None
        self.meta = old_meta
+        return None

    def _merge_dir_version_meta(
        self, old_hi: "HashInfo", old_obj: Optional[Union["HashFile", "Tree"]]
@@ -66,7 +66,7 @@ class ParamsLoadError(ContextError):
    pass


-class KeyNotInContext(ContextError, KeyError):
+class KeyNotInContext(ContextError, KeyError):  # noqa: N818
    def __init__(self, key: str) -> None:
        self.key: str = key
        super().__init__(f"Could not find '{key}'")
@@ -75,7 +75,7 @@ class KeyNotInContext(ContextError, KeyError):
        return self.msg


-class VarsAlreadyLoaded(ContextError):
+class VarsAlreadyLoaded(ContextError):  # noqa: N818
    pass


@@ -272,7 +272,7 @@ class CtxDict(Container, MutableMapping):
        if not isinstance(key, str):
            # limitation for the interpolation
            # ignore other kinds of keys
-            return
+            return None
        return super().__setitem__(key, value)

    def merge_update(self, other, overwrite=False):
@@ -37,13 +37,13 @@ KEYCRE = re.compile(
 def get_parser():
    from pyparsing import CharsNotIn, ParserElement, Suppress, ZeroOrMore

-    ParserElement.enablePackrat()
+    ParserElement.enable_packrat()

    word = CharsNotIn(f"{PERIOD}{LBRACK}{RBRACK}")
    idx = Suppress(LBRACK) + word + Suppress(RBRACK)
    attr = Suppress(PERIOD) + word
    parser = word + ZeroOrMore(attr ^ idx)
-    parser.setParseAction(PERIOD.join)
+    parser.set_parse_action(PERIOD.join)

    return parser

@@ -178,7 +178,7 @@ def parse_expr(s: str):
    from pyparsing import ParseException

    try:
-        result = get_parser().parseString(s, parseAll=True)
+        result = get_parser().parse_string(s, parse_all=True)
    except ParseException as exc:
        format_and_raise_parse_error(exc)
        raise AssertionError("unreachable")  # noqa: B904
@@ -202,6 +202,7 @@ def validate_value(value, key):
        if isinstance(value, dict) and key == "cmd":
            return True
        raise ParseError(f"Cannot interpolate data of type '{type(value).__name__}'")
+    return None


 def str_interpolate(
@@ -167,7 +167,7 @@ class VegaConverter(Converter):
        if isinstance(y, list):
            return "y"
        if not isinstance(y, dict):
-            return
+            return None

        fields = {field for _, field in _file_field(y)}
        if len(fields) == 1:
@@ -2,7 +2,6 @@ import os
 from collections import defaultdict
 from typing import TYPE_CHECKING, NamedTuple, Optional

-import dpath
 import dpath.options
 from funcy import get_in, last

@@ -20,6 +20,7 @@ if TYPE_CHECKING:
    from dvc.fs.data import DataFileSystem
    from dvc.fs.dvc import DVCFileSystem
    from dvc.lock import LockBase
+    from dvc.output import Output
    from dvc.scm import Git, NoSCM
    from dvc.stage import Stage
    from dvc.types import DictStrAny
@@ -544,7 +545,9 @@ class Repo:

        return used

-    def find_outs_by_path(self, path, outs=None, recursive=False, strict=True):
+    def find_outs_by_path(
+        self, path, outs=None, recursive=False, strict=True
+    ) -> list["Output"]:
        # using `outs_graph` to ensure graph checks are run
        outs = outs or self.index.outs_graph

@@ -40,7 +40,7 @@ def check_name_format(name: str) -> None:

 def name_is_compatible(name: str) -> bool:
    """
-    Only needed by DVCLive per iterative/dvclive#715
+    Only needed by DVCLive per treeverse/dvclive#715
    Will be removed in future release.
    """
    from gto.constants import assert_name_is_valid
@@ -1,6 +1,6 @@
 import os
 import posixpath
-from collections import defaultdict
+from collections import defaultdict, deque
 from collections.abc import Iterable, Iterator, Mapping
 from typing import TYPE_CHECKING, Optional, TypedDict, Union

@@ -385,6 +385,26 @@ def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> list[str]
    return [repo.fs.relpath(file, start) for file in files]


+def iter_index(
+    index: Union["BaseDataIndex", "DataIndexView"], shallow: bool = False
+) -> Iterator[tuple["DataIndexKey", Optional["DataIndexEntry"]]]:
+    if not shallow:
+        yield from index.iteritems(shallow=shallow)
+        return
+
+    # only iterate until we find entries with hash_info in shallow mode
+    todo: deque[tuple[DataIndexKey, dict]] = deque([((), index.info(()))])
+    while todo:
+        key, info = todo.popleft()
+        entry = info.get("entry")
+        if info.get("type") == "directory" and not (entry and entry.hash_info):
+            try:
+                todo.extend(index.ls(key, detail=True))
+            except (KeyError, DataIndexDirError):
+                pass
+        yield key, entry
+
+
 def _get_entries_not_in_remote(
    repo: "Repo",
    filter_keys: Optional[Iterable["DataIndexKey"]] = None,
@@ -418,7 +438,7 @@ def _get_entries_not_in_remote(

    n = 0
    with TqdmCallback(size=n, desc="Checking remote", unit="entry") as cb:
-        for key, entry in view.iteritems(shallow=not granular):
+        for key, entry in iter_index(view, shallow=not granular):
            if not (entry and entry.hash_info):
                continue

@@ -436,15 +456,17 @@ def _get_entries_not_in_remote(
            n += 1
            cb.set_size(n)

+        results = storage_map.bulk_remote_exists(
+            list(entries.values()), refresh=remote_refresh, callback=cb
+        )
        for key, entry in entries.items():
            k = (*key, "") if entry.meta and entry.meta.isdir else key
            try:
-                if not storage_map.remote_exists(entry, refresh=remote_refresh):
+                if not results.get(entry, False):
                    missing_entries.append(os.path.sep.join(k))
            except StorageKeyError:
                pass
-            finally:
-                cb.relative_update()
+
    data_index.onerror = orig_data_index_onerror
    return missing_entries

@@ -188,7 +188,7 @@ class Experiments:
        rev_names = self.get_exact_name(revs)
        for rev in revs:
            name = rev_names[rev]
-            names.append(name if name else rev[:7])
+            names.append(name or rev[:7])
        ui.write("\nRan experiment(s): {}".format(", ".join(names)))
        if tmp_dir:
            ui.write(
@@ -58,7 +58,7 @@ def collect_rev(
        if not (rev == "workspace" or param_deps or data.contains_error):
            cache.put(data, force=True)
        return ExpState(rev=rev, data=data)
-    except Exception as exc:  # noqa: BLE001
+    except Exception as exc:
        logger.debug("", exc_info=True)
        error = SerializableError(str(exc), type(exc).__name__)
        return ExpState(rev=rev, error=error)
@@ -328,7 +328,7 @@ def collect(
                name=baseline_names.get(baseline_rev),
                data=baseline_data.data,
                error=baseline_data.error,
-                experiments=experiments if experiments else None,
+                experiments=experiments or None,
            )
        )
    return result
@@ -50,7 +50,7 @@ class MultipleBranchError(DvcException):
        self.ref_infos = ref_infos


-class AmbiguousExpRefInfo(InvalidArgumentError):
+class AmbiguousExpRefInfo(InvalidArgumentError):  # noqa: N818
    def __init__(self, exp_name: str, exp_ref_list: Iterable["ExpRefInfo"]):
        msg = [
            (
@@ -9,6 +9,7 @@ from enum import IntEnum
 from itertools import chain
 from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, Union

+import funcy
 from funcy import nullcontext
 from scmrepo.exceptions import SCMError

@@ -756,7 +757,7 @@ class BaseExecutor(ABC):
            logger.debug("Commit to current experiment branch '%s'", branch)
        else:
            baseline_rev = scm.get_ref(EXEC_BASELINE)
-            name = exp_name if exp_name else f"exp-{exp_hash[:5]}"
+            name = exp_name or f"exp-{exp_hash[:5]}"
            ref_info = ExpRefInfo(baseline_rev, name)
            branch = str(ref_info)
            old_ref = None
@@ -815,11 +816,22 @@ class BaseExecutor(ABC):

    @contextmanager
    def set_temp_refs(self, scm: "Git", temp_dict: dict[str, str]):
+        # Retry ref set, get, and remove operations to handle transient issues during
+        # concurrent Git access.
+        # Dulwich deletes parent directories of refs if they happen to be empty after
+        # removing a ref, which can interfere with `set_ref` in other processes.
+        # `remove_ref` may also fail with a `FileLocked` error when refs are packed,
+        # since multiple processes might attempt to write to the same file.
+        retry = funcy.retry(10, errors=Exception, timeout=0.1)
+        set_ref = retry(scm.set_ref)
+        get_ref = retry(scm.get_ref)
+        remove_ref = retry(scm.remove_ref)
+
        try:
            for ref, rev in temp_dict.items():
-                scm.set_ref(ref, rev)
+                set_ref(ref, rev)
            yield
        finally:
            for ref in temp_dict:
-                if scm.get_ref(ref):
-                    scm.remove_ref(ref)
+                if get_ref(ref):
+                    remove_ref(ref)
@@ -90,7 +90,7 @@ class BaseStashQueue(ABC):
        """Construct a queue.

        Arguments:
-            scm: Git SCM instance for this queue.
+            repo: DVC repo instance for this queue.
            ref: Git stash ref for this queue.
            failed_ref: Failed run Git stash ref for this queue.
        """
@@ -287,6 +287,7 @@ class BaseStashQueue(ABC):
        baseline_rev: Optional[str] = None,
        branch: Optional[str] = None,
        name: Optional[str] = None,
+        no_hydra: bool = False,
        **kwargs,
    ) -> QueueEntry:
        """Stash changes from the workspace as an experiment.
@@ -302,6 +303,7 @@ class BaseStashQueue(ABC):
            name: Optional experiment name. If specified this will be used as
                the human-readable name in the experiment branch ref. Has no
                effect of branch is specified.
+            no_hydra: Disable Hydra from automatically overwriting all params.

        .. _Hydra Override:
            https://hydra.cc/docs/next/advanced/override_grammar/basic/
@@ -318,7 +320,7 @@ class BaseStashQueue(ABC):

                    # update experiment params from command line
                    if params:
-                        self._update_params(params)
+                        self._update_params(params, no_hydra=no_hydra)

                    # DVC commit data deps to preserve state across workspace
                    # & tempdir runs
@@ -441,12 +443,13 @@ class BaseStashQueue(ABC):
            f"from '{config_path}': {param_list}"
        )

-    def _update_params(self, params: dict[str, list[str]]):
+    def _update_params(self, params: dict[str, list[str]], no_hydra: bool = False):
        """Update param files with the provided `Hydra Override`_ patterns.

        Args:
            params: Dict mapping paths to `Hydra Override`_ patterns,
                provided via `exp run --set-param`.
+            no_hydra: Disable Hydra from automatically overwriting all params.

        .. _Hydra Override:
            https://hydra.cc/docs/advanced/override_grammar/basic/
@@ -456,7 +459,7 @@ class BaseStashQueue(ABC):
        logger.debug("Using experiment params '%s'", params)

        hydra_config = self.repo.config.get("hydra", {})
-        hydra_enabled = hydra_config.get("enabled", False)
+        hydra_enabled = hydra_config.get("enabled", False) and not no_hydra
        hydra_output_file = ParamsDependency.DEFAULT_PARAMS_FILE
        for path, overrides in params.items():
            if hydra_enabled and path == hydra_output_file:
@@ -70,7 +70,7 @@ class LocalCeleryQueue(BaseStashQueue):
    def celery(self) -> "FSApp":
        from kombu.transport.filesystem import Channel

-        # related to https://github.com/iterative/dvc-task/issues/61
+        # related to https://github.com/treeverse/dvc-task/issues/61
        Channel.QoS.restore_at_shutdown = False

        from dvc_task.app import FSApp
@@ -84,8 +84,8 @@ def cleanup_exp(executor: TempDirExecutor, infofile: str) -> None:
    """Cleanup after an experiment.

    Arguments:
-        tmp_dir: Temp directory to be removed.
-        entry_dict: Serialized QueueEntry for this experiment.
+        executor: Executor to clean up.
+        infofile: Path to the executor info file.
    """
    executor.cleanup(infofile)

@@ -34,7 +34,7 @@ class ExpRefInfo:

    def __init__(self, baseline_sha: str, name: Optional[str] = None):
        self.baseline_sha = baseline_sha
-        self.name: str = name if name else ""
+        self.name: str = name or ""

    def __str__(self):
        return "/".join(self.parts)
@@ -22,6 +22,7 @@ def run(  # noqa: C901, PLR0912
    queue: bool = False,
    copy_paths: Optional[Iterable[str]] = None,
    message: Optional[str] = None,
+    no_hydra: bool = False,
    **kwargs,
 ) -> dict[str, str]:
    """Reproduce the specified targets as an experiment.
@@ -67,7 +68,7 @@ def run(  # noqa: C901, PLR0912
    else:
        path_overrides = {}

-    hydra_enabled = repo.config.get("hydra", {}).get("enabled", False)
+    hydra_enabled = repo.config.get("hydra", {}).get("enabled", False) and not no_hydra
    hydra_output_file = ParamsDependency.DEFAULT_PARAMS_FILE
    if hydra_enabled and hydra_output_file not in path_overrides:
        # Force `_update_params` even if `--set-param` was not used
@@ -80,6 +81,7 @@ def run(  # noqa: C901, PLR0912
            tmp_dir=tmp_dir,
            copy_paths=copy_paths,
            message=message,
+            no_hydra=no_hydra,
            **kwargs,
        )

@@ -100,6 +102,7 @@ def run(  # noqa: C901, PLR0912
            params=sweep_overrides,
            copy_paths=copy_paths,
            message=message,
+            no_hydra=no_hydra,
            **kwargs,
        )
        if sweep_overrides:
@@ -67,7 +67,7 @@ class ExpStash(Stash):
        branch: Optional[str] = None,
    ) -> str:
        msg = cls.MESSAGE_FORMAT.format(
-            rev=rev, baseline_rev=baseline_rev, name=name if name else ""
+            rev=rev, baseline_rev=baseline_rev, name=name or ""
        )
        branch_msg = f":{branch}" if branch else ""
        return f"{msg}{branch_msg}"
@@ -124,9 +124,7 @@ class ApplyStash(Stash):

    @classmethod
    def format_message(cls, head_rev: str, rev: str, name: Optional[str] = None) -> str:
-        return cls.MESSAGE_FORMAT.format(
-            head_rev=head_rev, rev=rev, name=name if name else ""
-        )
+        return cls.MESSAGE_FORMAT.format(head_rev=head_rev, rev=rev, name=name or "")

    @contextmanager
    def preserve_workspace(
@@ -200,7 +200,7 @@ def exp_commits(
 ) -> Iterable[str]:
    """Iterate over all experiment commits."""
    shas: set[str] = set()
-    refs = ref_infos if ref_infos else exp_refs(scm)
+    refs = ref_infos or exp_refs(scm)
    for ref_info in refs:
        shas.update(scm.branch_revs(str(ref_info), ref_info.baseline_sha))
    yield from shas
@@ -20,6 +20,7 @@ def _make_index_onerror(onerror, rev):
    def _onerror(entry, exc):
        if onerror:
            return onerror(rev, entry, exc)
+        return None

    return _onerror

@@ -28,8 +28,10 @@ def _validate_args(**kwargs):
            "`--all-experiments`, `--all-commits`, `--date` or `--rev` "
            "needs to be set."
        )
-    if kwargs.get("num") and not kwargs.get("rev"):
-        raise InvalidArgumentError("`--num` can only be used alongside `--rev`")
+    if kwargs.get("num") and not (kwargs.get("rev") or kwargs.get("all_branches")):
+        raise InvalidArgumentError(
+            "`--num` can only be used alongside `--rev` or `--all-branches`"
+        )


 def _used_obj_ids_not_in_remote(
@@ -128,9 +130,11 @@ def gc(  # noqa: C901, PLR0912, PLR0913
        used_obj_ids = set()
        used_obj_ids.update(*odb_to_obj_ids.values())

+    seen_odbs = set()
    for scheme, odb in self.cache.by_scheme():
-        if not odb:
+        if not odb or odb in seen_odbs:
            continue
+        seen_odbs.add(odb)
        num_removed = ogc(odb, used_obj_ids, jobs=jobs, dry=dry)
        if num_removed:
            logger.info("Removed %d objects from %s cache.", num_removed, scheme)
@@ -16,7 +16,7 @@ def pre_commit_install(scm: "Git") -> None:
    config_path = os.path.join(scm.root_dir, ".pre-commit-config.yaml")
    with modify_yaml(config_path) as config:
        entry = {
-            "repo": "https://github.com/iterative/dvc",
+            "repo": "https://github.com/treeverse/dvc",
            "rev": ".".join(map(str, version_tuple[:3])),
            "hooks": [
                {
@@ -75,7 +75,7 @@ def install(self: "Repo", use_pre_commit_tool: bool = False) -> None:

    scm = self.scm
    if not isinstance(scm, Git):
-        return
+        return None

    driver = "dvc git-hook merge-driver --ancestor %O --our %A --their %B "
    scm.install_merge_driver("dvc", "DVC merge driver", driver)
@@ -123,7 +123,7 @@ def try_expand_paths(fs: "FileSystem", paths: Iterable[str]) -> Iterator[str]:
            if fs.isdir(path):
                yield from fs.find(path)
                continue
-        except Exception as e:  # noqa: BLE001
+        except Exception as e:
            logger.debug(
                "failed to expand %r: %s",
                path,
@@ -1,10 +1,13 @@
 import os
+from typing import TYPE_CHECKING

-from dvc.exceptions import MoveNotDataSourceError
 from dvc.repo.scm_context import scm_context

 from . import locked

+if TYPE_CHECKING:
+    from . import Repo
+

 def _expand_target_path(from_path, to_path):
    if os.path.isdir(to_path):
@@ -14,7 +17,7 @@ def _expand_target_path(from_path, to_path):

@locked
@scm_context
-def move(self, from_path, to_path):
+def move(self: "Repo", from_path, to_path):
    """
    Renames an output file and modifies the stage associated
    to reflect the change on the pipeline.
@@ -32,9 +35,12 @@ def move(self, from_path, to_path):
    It only works with outputs generated by `add` or `import`,
    also known as data sources.
    """
-    from dvc import output
+    from dvc import dependency, output
    from dvc.dvcfile import DVC_FILE_SUFFIX
+    from dvc.exceptions import MoveNotDataSourceError
    from dvc.stage import Stage
+    from dvc.stage.exceptions import StageFileAlreadyExistsError
+    from dvc_objects.fs.local import LocalFileSystem

    from_out = output.loads_from(Stage(self), [from_path])[0]
    assert from_out.protocol == "local"
@@ -45,6 +51,7 @@ def move(self, from_path, to_path):
    assert len(outs) == 1
    out = outs[0]
    stage = out.stage
+    deps = stage.deps

    if not stage.is_data_source:
        raise MoveNotDataSourceError(stage.addressing)
@@ -58,21 +65,38 @@ def move(self, from_path, to_path):
        )
        new_wdir = os.path.abspath(os.path.join(os.curdir, os.path.dirname(to_path)))
        to_path = os.path.relpath(to_path, new_wdir)
-        new_stage = self.stage.create(
-            single_stage=True,
-            fname=new_fname,
-            wdir=new_wdir,
-            outs=[to_path],
-            meta=stage.meta,
-        )
-
-        os.unlink(stage.path)
-        stage = new_stage
+        try:
+            new_stage = self.stage.create(
+                single_stage=True,
+                fname=new_fname,
+                wdir=new_wdir,
+                outs=[to_path],
+                meta=stage.meta,
+                frozen=stage.frozen,
+                always_changed=stage.always_changed,
+                desc=stage.desc,
+            )
+        except StageFileAlreadyExistsError:
+            # reraise to remove `--force` hint
+            raise StageFileAlreadyExistsError(f"{new_fname!r} already exists") from None
    else:
+        new_stage = stage
        to_path = os.path.relpath(to_path, stage.wdir)

-    to_out = output.loads_from(stage, [to_path], out.use_cache, out.metric)[0]
+    def with_dep_path_adjusted(dep: dependency.Dependency):
+        d = dep.dumpd()
+        if isinstance(dep.fs, LocalFileSystem) and not os.path.isabs(dep.def_path):
+            return d | {"path": os.path.relpath(dep.fspath, new_stage.wdir)}
+        return d

-    out.move(to_out)
-    stage.save()
-    stage.dump()
+    new_stage.outs = output.loadd_from(new_stage, [out.dumpd() | {"path": to_path}])
+    new_stage.deps = dependency.loadd_from(
+        new_stage, [with_dep_path_adjusted(dep) for dep in deps]
+    )
+    out.move(new_stage.outs[0])
+    new_stage.md5 = new_stage.compute_md5()
+    new_stage.dump()
+    if stage != new_stage:
+        stage.dvcfile.remove()
+        self.scm_context.track_file(stage.dvcfile.relpath)
+    return stage, new_stage
@@ -10,6 +10,8 @@ from dvc.utils import relpath
 from dvc.utils.collections import ensure_list

 if TYPE_CHECKING:
+    from typing_extensions import Self
+
    from dvc.repo import Repo
    from dvc.scm import Base

@@ -46,7 +48,7 @@ class SCMContext:
        try:
            add_no_submodules(self.scm, paths)
        except UnsupportedIndexFormat:
-            link = "https://github.com/iterative/dvc/issues/610"
+            link = "https://github.com/treeverse/dvc/issues/610"
            add_cmd = self._make_git_add_cmd([relpath(path) for path in paths])
            logger.info("")
            msg = (
@@ -58,7 +60,7 @@ class SCMContext:
    def track_changed_files(self) -> None:
        """Stage files that have changed."""
        if not self.files_to_track:
-            return
+            return None
        logger.debug("Staging files: %s", self.files_to_track)
        return self.add(self.files_to_track)

@@ -76,6 +78,7 @@ class SCMContext:
            logger.debug("Added '%s' to gitignore file.", path)
            self.track_file(relpath(gitignore_file))
            return self.ignored_paths.append(path)
+        return None

    def ignore_remove(self, path: str) -> None:
        from scmrepo.exceptions import FileNotInRepoError
@@ -90,11 +93,12 @@ class SCMContext:

        if gitignore_file:
            return self.track_file(relpath(gitignore_file))
+        return None

    @contextmanager
    def __call__(
        self, autostage: Optional[bool] = None, quiet: Optional[bool] = None
-    ) -> Iterator["SCMContext"]:
+    ) -> Iterator["Self"]:
        try:
            yield self
        except Exception:
@@ -129,9 +133,10 @@ class SCMContext:

        self.files_to_track = set()

-    def __enter__(self) -> "SCMContext":
+    def __enter__(self) -> "Self":
        self._cm = self()
-        return self._cm.__enter__()
+        self._cm.__enter__()
+        return self

    def __exit__(self, *exc_args) -> None:
        assert self._cm
@@ -45,7 +45,7 @@ class NoSCMError(SCMError):
        super().__init__(msg)


-class InvalidRemoteSCMRepo(SCMError):
+class InvalidRemoteSCMRepo(SCMError):  # noqa: N818
    pass


@@ -234,7 +234,7 @@ def iter_revs(
        results.extend(scm.list_all_commits())
    else:
        if all_branches:
-            results.extend(scm.list_branches())
+            results.extend(_get_n_commits(scm, scm.list_branches(), num))

        if all_tags:
            results.extend(scm.list_tags())
@@ -258,7 +258,10 @@ def iter_revs(
        results.extend(exp_commits(scm))

    rev_resolver = partial(resolve_rev, scm)
-    return group_by(rev_resolver, results)
+    grouped = group_by(rev_resolver, results)
+    for rev, names in grouped.items():
+        grouped[rev] = list(dict.fromkeys(names))
+    return grouped


 def lfs_prefetch(fs: "FileSystem", paths: list[str]):
@@ -32,7 +32,7 @@ from .utils import (
 )

 if TYPE_CHECKING:
-    from dvc.dependency import ParamsDependency
+    from dvc.dependency import Dependency, ParamsDependency
    from dvc.dvcfile import ProjectFile, SingleStageFile
    from dvc.output import Output
    from dvc.repo import Repo
@@ -121,7 +121,7 @@ def restore_fields(stage: "Stage") -> None:

    old_outs = {out.def_path: out for out in old.outs}
    for out in stage.outs:
-        old_out = old_outs.get(out.def_path, None)
+        old_out = old_outs.get(out.def_path)
        if old_out is not None:
            out.restore_fields(old_out)

@@ -153,8 +153,8 @@ class Stage(params.StageParams):
        self._path = path
        self.cmd = cmd
        self.wdir = wdir
-        self.outs = outs
-        self.deps = deps
+        self.outs: list[Output] = outs
+        self.deps: list[Dependency] = deps
        self.md5 = md5
        self.frozen = locked or frozen
        self.always_changed = always_changed
@@ -82,7 +82,7 @@ class DataSourceChanged(DvcException):
        super().__init__(f"data source changed: {path}")


-class StageNotFound(DvcException, KeyError):
+class StageNotFound(DvcException, KeyError):  # noqa: N818
    def __init__(self, file, name):
        self.file = file.relpath
        self.name = name
@@ -51,7 +51,7 @@ def _warn_if_fish(executable):
        "Be aware that it might cause problems by overwriting "
        "your current environment variables with values defined "
        "in 'config.fish', which might affect your command. See "
-        "https://github.com/iterative/dvc/issues/1307. "
+        "https://github.com/treeverse/dvc/issues/1307. "
    )


@@ -99,7 +99,7 @@ def prepare_kwargs(stage, run_env=None):
    #
    # [1] https://github.com/python/cpython/blob/3.7/Lib/subprocess.py
    #                                                            #L1426
-    # [2] https://github.com/iterative/dvc/issues/2506
+    # [2] https://github.com/treeverse/dvc/issues/2506
    #                                           #issuecomment-535396799
    kwargs["shell"] = os.name == "nt"
    return kwargs
@@ -178,7 +178,7 @@ def to_single_stage_lockfile(stage: "Stage", **kwargs) -> dict:
    res = OrderedDict([("cmd", stage.cmd)])
    params, deps = split_params_deps(stage)
    deps, outs = (
-        [_dumpd(item) for item in sorted(items, key=attrgetter("def_path"))]
+        [_dumpd(item) for item in sorted(items, key=attrgetter("def_path"))]  # type: ignore[call-overload]
        for items in [deps, stage.outs]
    )
    params = _serialize_params_values(params)
@@ -4,7 +4,7 @@ dvc.testing.benchmarks
 ======================

 Benchmark test definitions are now part of ``dvc.testing``.
-For CLI usage and `bench.dvc.org <https://bench.dvc.org>`_ details see `dvc-bench <https://github.com/iterative/dvc-bench>`_.
+For CLI usage and `bench.dvc.org <https://bench.dvc.org>`_ details see `dvc-bench <https://github.com/treeverse/dvc-bench>`_.

 ``dvc.testing.benchmarks`` structure:

@@ -2,6 +2,7 @@ import pytest

 from dvc import api
 from dvc.api import DVCFileSystem
+from dvc.testing import matchers as M
 from dvc.utils.fs import remove


@@ -42,7 +43,6 @@ class TestAPI:
    )
    def test_filesystem(
        self,
-        M,
        tmp_dir,
        make_tmp_dir,
        scm,
@@ -6,8 +6,8 @@ import pytest
    "!=3.53.*,!=3.54.0",
    reason="Takes 10 mins to run. Regression in 3.53.0, fixed in 3.54.1",
 )
-# Introduced in https://github.com/iterative/dvc/pull/10388.
-# Fixed in https://github.com/iterative/dvc/pull/10531.
+# Introduced in https://github.com/treeverse/dvc/pull/10388.
+# Fixed in https://github.com/treeverse/dvc/pull/10531.
 def test_import(bench_dvc, tmp_dir, scm, dvc, make_dataset, remote):
    dataset = make_dataset(
        cache=False, files=False, dvcfile=True, commit=True, remote=True
@@ -9,7 +9,7 @@ from typing import Optional
 import pytest
 from dulwich.porcelain import clone
 from funcy import first
-from packaging import version
+from packaging import specifiers, version

 from dvc.types import StrPath

@@ -79,7 +79,7 @@ def dvc_bench_repo(tmp_path_factory, bench_config):
    if url is None:
        pytest.skip(
            "--dvc-bench-repo is not set, "
-            "clone https://github.com/iterative/dvc-bench repository and set its path"
+            "clone https://github.com/treeverse/dvc-bench repository and set its path"
        )

    if os.path.isdir(url):
@@ -109,11 +109,20 @@ def make_dvc_bin(
            else:
                pkg = "dvc"
            packages = [f"{pkg} @ git+file://{dvc_repo}@{dvc_rev}"]
-            try:
-                if version.Version(dvc_rev) < version.Version("3.50.3"):
-                    packages.append("pygit2==1.14.1")
-            except version.InvalidVersion:
-                pass
+
+            version_constraints = [
+                ("<3.50.3", ["pygit2==1.14.1"]),
+                ("<3.44.0", ["dulwich<1.0.0"]),
+                ("<3.67.0", ["pathspec<1"]),
+            ]
+            for spec, pkgs in version_constraints:
+                try:
+                    _dvc_version = version.Version(dvc_rev)
+                except version.InvalidVersion:
+                    continue
+                if _dvc_version in specifiers.SpecifierSet(spec):
+                    packages.extend(pkgs)
+
            venv.install(*packages)

            dvc_venvs[dvc_rev] = venv
@@ -158,7 +167,7 @@ def make_bench(request):
            # pytest >= 8.4.0
            wrapped_func = fixture_function._get_wrapped_function()
        except AttributeError:
-            wrapped_func = fixture_function.__pytest_wrapped__.obj  # ty: ignore[unresolved-attribute]
+            wrapped_func = fixture_function.__pytest_wrapped__.obj  # type: ignore[attr-defined]  # ty: ignore[unresolved-attribute]
        assert inspect.isgeneratorfunction(wrapped_func)

        generator = wrapped_func(request)
@@ -0,0 +1,104 @@
+from collections.abc import Iterable, Mapping
+from typing import TYPE_CHECKING, Any, Optional, Union
+
+if TYPE_CHECKING:
+    import builtins
+
+
+class dict:  # noqa: A001, N801, PLW1641
+    """Special class to eq by matching only presented dict keys.
+
+    Implementation notes:
+
+     - can't inherit from dict because that makes D() == M.dict() to not call
+       our __eq__, if D is a subclass of a dict
+
+     - should not call itself dict or use dict in repr because it creates
+       confusing error messages (shadowing python builtins is bad anyway)
+
+    """
+
+    def __init__(self, d: Optional[Mapping[Any, Any]] = None, **keys: Any) -> None:
+        self.d: builtins.dict[Any, Any] = {}
+        if d:
+            self.d.update(d)
+        self.d.update(keys)
+
+    def __len__(self) -> int:
+        return len(self.d)
+
+    def __repr__(self) -> str:
+        inner = ", ".join(f"{k}={v!r}" for k, v in self.d.items())
+        return f"{self.__class__.__name__}({inner})"
+
+    def __eq__(self, other: object) -> bool:
+        assert isinstance(other, Mapping)
+        return all(other.get(name) == v for name, v in self.d.items())
+
+
+class unordered:  # noqa: N801, PLW1641
+    """Compare list contents, but do not care about ordering.
+
+    (E.g. sort lists first, then compare.)
+    If you care about ordering, then just compare lists directly."""
+
+    def __init__(self, *items: Any) -> None:
+        self.items = items
+
+    def __repr__(self) -> str:
+        inner = ", ".join(map(repr, self.items))
+        return f"{self.__class__.__name__}({inner})"
+
+    def __eq__(self, other: object) -> bool:
+        assert isinstance(other, Iterable)
+        return sorted(self.items) == sorted(other)
+
+
+class attrs:  # noqa: N801, PLW1641
+    def __init__(self, **attribs: Any) -> None:
+        self.attribs = attribs
+
+    def __repr__(self) -> str:
+        inner = ", ".join(f"{k}={v!r}" for k, v in self.attribs.items())
+        return f"{self.__class__.__name__}({inner})"
+
+    def __eq__(self, other: object) -> bool:
+        # Unforturnately this doesn't work with classes with slots
+        # self.__class__ = other.__class__
+        return all(getattr(other, name) == v for name, v in self.attribs.items())
+
+
+class instance_of:  # noqa: N801, PLW1641
+    def __init__(self, expected_type: Union[Any, tuple[Any, ...]]) -> None:
+        self.expected_type = expected_type
+
+    def __repr__(self) -> str:
+        if isinstance(self.expected_type, tuple):
+            inner = f"({', '.join(t.__name__ for t in self.expected_type)})"
+        else:
+            inner = self.expected_type.__name__
+        return f"{self.__class__.__name__}({inner})"
+
+    def __eq__(self, other: object) -> bool:
+        return isinstance(other, self.expected_type)
+
+
+class any_of:  # noqa: N801, PLW1641
+    def __init__(self, *items: Any) -> None:
+        self.items = sorted(items)
+
+    def __repr__(self) -> str:
+        inner = ", ".join(map(repr, self.items))
+        return f"any_of({inner})"
+
+    def __eq__(self, other: object) -> bool:
+        return other in self.items
+
+
+__all__ = [
+    "any_of",
+    "attrs",
+    "dict",
+    "instance_of",
+    "unordered",
+]
@@ -2,7 +2,7 @@ import os
 import pathlib
 import posixpath
 import sys
-from typing import Callable, ClassVar
+from typing import Callable, ClassVar, Union
 from urllib.parse import urlparse

 from dvc.utils import relpath
@@ -79,15 +79,17 @@ class PathInfo(pathlib.PurePath, _BasePath):
            and self._cparts[:n] == other._cparts  # type: ignore[attr-defined]
        )

-    def relative_to(self, other, *args, **kwargs):
+    def relative_to(
+        self, *other: Union[str, "os.PathLike[str]"], **kwargs
+    ) -> "PathInfo":
        # pathlib relative_to raises exception when one path is not a direct
        # descendant of the other when os.path.relpath would return abspath.
        # For DVC PathInfo we only need the relpath behavior.
        # See: https://bugs.python.org/issue40358
        try:
-            path = super().relative_to(other, *args, **kwargs)
+            path = super().relative_to(*other, **kwargs)
        except ValueError:
-            path = relpath(self, other)
+            path = relpath(self, other[0])
        return self.__class__(path)


@@ -44,7 +44,7 @@ class TestRemote:
        _check_status(status_dir, new=dir_hashes)

        # Move cache and check status
-        # See issue https://github.com/iterative/dvc/issues/4383 for details
+        # See issue https://github.com/treeverse/dvc/issues/4383 for details
        backup_dir = dvc.cache.local.path + ".backup"
        shutil.move(dvc.cache.local.path, backup_dir)
        status = dvc.cloud.status(foo_hashes)
@@ -116,7 +116,7 @@ class TestRemote:

    @pytest.mark.xfail(raises=NotImplementedError, strict=False)
    def test_pull_00_prefix(self, tmp_dir, dvc, remote, monkeypatch):
-        # Related: https://github.com/iterative/dvc/issues/6089
+        # Related: https://github.com/treeverse/dvc/issues/6089

        fs_type = type(dvc.cloud.get_remote_odb("upstream").fs)
        monkeypatch.setattr(fs_type, "_ALWAYS_TRAVERSE", True, raising=False)
@@ -146,7 +146,7 @@ class TestRemote:

    @pytest.mark.xfail(raises=NotImplementedError, strict=False)
    def test_pull_no_00_prefix(self, tmp_dir, dvc, remote, monkeypatch):
-        # Related: https://github.com/iterative/dvc/issues/6244
+        # Related: https://github.com/treeverse/dvc/issues/6244

        fs_type = type(dvc.cloud.get_remote_odb("upstream").fs)
        monkeypatch.setattr(fs_type, "_ALWAYS_TRAVERSE", True, raising=False)
@@ -193,7 +193,7 @@ class Console:
        sep = " " if sep is None else sep
        end = "\n" if end is None else end
        if not self._enabled and not force:
-            return
+            return None

        file = file or (sys.stderr if stderr else sys.stdout)
        with Tqdm.external_write_mode(file=file):
@@ -289,7 +289,7 @@ class Console:

        return console.Console(stderr=True)

-    def table(
+    def table(  # noqa: PLR0913
        self,
        data: "TableData",
        headers: Optional["Headers"] = None,
@@ -301,11 +301,12 @@ class Console:
        row_styles: Optional[Sequence["Styles"]] = None,
        borders: Union[bool, str] = False,
        colalign: Optional[tuple[str, ...]] = None,
+        preserve_whitespace: bool = False,
    ) -> None:
        from dvc.ui import table as t

        if not data and not markdown:
-            return
+            return None

        if not markdown and rich_table:
            if force or self._enabled:
@@ -319,7 +320,7 @@ class Console:
                    borders=borders,
                )

-            return
+            return None

        return t.plain_table(
            self,
@@ -329,6 +330,7 @@ class Console:
            pager=pager,
            force=force,
            colalign=colalign,
+            preserve_whitespace=preserve_whitespace,
        )

    def status(self, status: str, **kwargs: Any) -> "Status":
@@ -30,19 +30,37 @@ def plain_table(
    pager: bool = False,
    force: bool = True,
    colalign: Optional[tuple[str, ...]] = None,
+    preserve_whitespace: bool = False,
 ) -> None:
+    import tabulate as tabulate_mod
    from funcy import nullcontext
+    from packaging.version import Version
    from tabulate import tabulate

-    text: str = tabulate(
-        data,
-        headers if headers is not None else (),
-        tablefmt="github" if markdown else "plain",
-        disable_numparse=True,
-        # None will be shown as "" by default, overriding
-        missingval="-",
-        colalign=colalign,
-    )
+    # NOTE: tabulate 0.10.0+ supports preserve_whitespace as a kwarg,
+    # while 0.9.0 only respects the global PRESERVE_WHITESPACE.
+    kwargs: dict = {}
+    _orig = tabulate_mod.PRESERVE_WHITESPACE
+    if preserve_whitespace:
+        if Version(tabulate_mod.__version__) >= Version("0.10"):
+            kwargs["preserve_whitespace"] = True
+        else:
+            tabulate_mod.PRESERVE_WHITESPACE = True
+
+    try:
+        text: str = tabulate(
+            data,
+            headers if headers is not None else (),
+            tablefmt="github" if markdown else "plain",
+            disable_numparse=True,
+            # None will be shown as "" by default, overriding
+            missingval="-",
+            colalign=colalign,
+            **kwargs,
+        )
+    finally:
+        tabulate_mod.PRESERVE_WHITESPACE = _orig
+
    if markdown:
        # NOTE: md table is incomplete without the trailing newline
        text += "\n"
@@ -120,7 +120,7 @@ class Updater:
        from dvc.ui import ui

        if not ui.isatty():
-            return
+            return None

        message = self._get_message(latest, pkg=pkg)
        return ui.error_write(message, styled=True)
@@ -163,7 +163,7 @@ class Updater:
        if pkg not in instructions:
            return (
                "Find the latest release at "
-                "[blue]https://github.com/iterative/dvc/releases/latest[/]."
+                "[blue]https://github.com/treeverse/dvc/releases/latest[/]."
            )

        instruction = instructions[pkg]
@@ -18,7 +18,7 @@ keywords = [
 ]
 license = "Apache-2.0"
 license-files = ["LICENSE"]
-maintainers = [{ name = "Iterative", email = "[email protected]" }]
+maintainers = [{ name = "Treeverse", email = "[email protected]" }]
 authors = [{ name = "Dmitry Petrov", email = "[email protected]" }]
 requires-python = ">=3.9"
 classifiers = [
@@ -29,6 +29,7 @@ classifiers = [
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
 ]
 dynamic = ["version"]

@@ -40,14 +41,14 @@ dependencies = [
    "distro>=1.3",
    "dpath<3,>=2.1.0",
    "dulwich",
-    "dvc-data>=3.16.2,<3.17",
+    "dvc-data>=3.18.2,<3.19.0",
    "dvc-http>=2.29.0",
    "dvc-objects",
    "dvc-render>=1.0.1,<2",
    "dvc-studio-client>=0.21,<1",
    "dvc-task>=0.3.0,<1",
-    "flatten_dict<1,>=0.4.1",
-    "flufl.lock>=8.1.0,<9",
+    "flatten-dict<1,>=0.4.1",
+    "flufl.lock>=8.1.0,<10",
    "fsspec>=2024.2.0",
    "funcy>=1.14",
    "grandalf<1,>=0.7",
@@ -58,12 +59,12 @@ dependencies = [
    "networkx>=2.5",
    "omegaconf",
    "packaging>=19",
-    "pathspec>=0.10.3",
+    "pathspec>=0.10.3,<2",
    "platformdirs<5,>=3.1.1",
    "psutil>=5.8",
    "pydot>=1.2.4",
    "pygtrie>=2.3.2",
-    "pyparsing>=2.4.7",
+    "pyparsing>=3.0.0",
    "requests>=2.22",
    "rich>=12",
    "ruamel.yaml>=0.17.11",
@@ -85,7 +86,7 @@ gdrive = ["dvc-gdrive>=3,<4"]
 gs = ["dvc-gs>=3.0.2,<4"]
 hdfs = ["dvc-hdfs>=3,<4"]
 lint = [
-    "mypy==1.17.1",
+    "mypy==1.19.1",
    "pandas-stubs",
    "types-colorama",
    "types-psutil",
@@ -102,7 +103,6 @@ ssh = ["dvc-ssh>=4,<5"]
 ssh_gssapi = ["dvc-ssh[gssapi]>=4,<5"]
 testing = [
  "pytest-benchmark[histogram]>=5,<6",
-  "pytest-test-utils",
  "uv",
 ]
 tests = [
@@ -110,12 +110,11 @@ tests = [
    "beautifulsoup4>=4.4",
    "dvc-ssh",
    "filelock",
-    "pytest>=7,<9",
-    "pytest-rerunfailures<16.0",
+    "pytest>=7,<10",
+    "pytest-rerunfailures<17.0",
    "pytest-cov>=4.1.0",
    "pytest-docker>=1,<4",
    "pytest-mock",
-    "pytest-test-utils",
    "pytest-timeout>=2",
    "pytest-xdist>=3.2",
    'pywin32>=225; sys_platform == "win32"', # optional test dependency
@@ -123,14 +122,14 @@ tests = [
    "sqlalchemy>=1,<3", # optional dependency for `import-db`
    "pandas>=1", # optional dependency for `import-db`
 ]
-webdav = ["dvc-webdav>=3,<4"]
+webdav = ["dvc-webdav>=3.0.1,<4"]
 webhdfs = ["dvc-webhdfs>=3.1,<4"]
 webhdfs_kerberos = ["dvc-webhdfs[kerberos]>=3.1,<4"]

 [project.urls]
 Documentation = "https://dvc.org/doc"
-Issues = "https://github.com/iterative/dvc/issues"
-Source = "https://github.com/iterative/dvc"
+Issues = "https://github.com/treeverse/dvc/issues"
+Source = "https://github.com/treeverse/dvc"

 [project.scripts]
 dvc = "dvc.cli:main"
@@ -248,7 +247,6 @@ module = [
    "shortuuid",
    "shtab",
    "upath",
-    "virtualenv",
    "viztracer",
    "voluptuous",
    "yappi",
@@ -265,8 +263,16 @@ show-fixes = true

 [tool.ruff.lint]
 ignore = [
-    "N818", "S101", "A005", "PT007", "RET502", "RET503", "SIM105", "SIM108", "SIM117",
-    "TRY003", "TRY300", "PERF203", "PLC0415", "PLR2004",  "PLW2901", "LOG007",
+    "PERF203", # try-except-in-loop
+    "PLC0415",  # import-outside-top-level
+    "PLR2004", # magic-value-comparison
+    "PLW2901", # redefined-loop-name
+    "S101", # assert
+    "SIM105", # suppressible-exception
+    "SIM108",  # if-else-block-instead-of-if-exp
+    "SIM117", # multiple-with-statements
+    "TRY003", # raise-vanilla-args
+    "TRY300", # try-consider-else
 ]
 select = [
    "F", "E", "W", "C90", "I", "N", "UP", "YTT", "ASYNC", "S", "BLE", "B", "A", "C4", "DTZ", "T10",
@@ -304,4 +310,4 @@ max-args = 10
 "dvc/commands/**" = ["N806"]
 "dvc/testing/**" = ["ARG002"]
 "dvc/testing/benchmarks/**" = ["ARG001"]
-"tests/**" = ["S", "ARG001", "ARG002", "TRY002", "TRY301", "PERF"]
+"tests/**" = ["S", "ARG001", "ARG002", "TRY002", "TRY301", "PERF", "PLR2004"]
@@ -1,17 +0,0 @@
-import os
-
-# Increasing fd ulimit for tests
-if os.name == "nt":
-    try:
-        import win32file
-    except ImportError:
-        pass
-    else:
-        win32file._setmaxstdio(4096)
-else:
-    import resource
-
-    resource.setrlimit(resource.RLIMIT_NOFILE, (4096, 4096))
-
-    nproc_soft, nproc_hard = resource.getrlimit(resource.RLIMIT_NPROC)
-    resource.setrlimit(resource.RLIMIT_NPROC, (nproc_hard, nproc_hard))
@@ -1,6 +0,0 @@
-if __name__ == "__main__":
-    import sys
-
-    import pytest
-
-    sys.exit(pytest.main(["-v", "-n=logical", "--dist=worksteal", *sys.argv[1:]]))
@@ -194,6 +194,7 @@ def isolate(tmp_path_factory):
            monkeypatch.setenv(env_var, os.fspath(path))
    else:
        monkeypatch.setenv("HOME", str(home_dir))
+        monkeypatch.setenv("XDG_CONFIG_HOME", str(home_dir / ".config"))

    monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1")
    contents = b"""
@@ -1,3 +1,5 @@
+import re
+
 import pytest

 from dvc import api
@@ -13,7 +15,7 @@ def test_exp_save(tmp_dir, dvc, scm):
    api.exp_save("foo")
    with pytest.raises(
        ExperimentExistsError,
-        match="Experiment conflicts with existing experiment 'foo'.",
+        match=re.escape("Experiment conflicts with existing experiment 'foo'."),
    ):
        api.exp_save("foo")
    api.exp_save("foo", force=True)
@@ -91,7 +91,7 @@ def test_apply_untracked(tmp_dir, scm, dvc, exp_stage):


 def test_apply_unchanged_head(tmp_dir, scm, dvc, exp_stage):
-    # see https://github.com/iterative/dvc/issues/8764
+    # see https://github.com/treeverse/dvc/issues/8764
    tmp_dir.gen("params.yaml", "foo: 2")
    scm.add(["dvc.yaml", "dvc.lock", "params.yaml", "metrics.yaml"])
    scm.commit("commit foo=2")
@@ -4,6 +4,7 @@ import os
 import stat
 from textwrap import dedent

+import dulwich
 import pytest
 from configobj import ConfigObj
 from funcy import first
@@ -19,6 +20,7 @@ from dvc.env import (
    DVC_STUDIO_URL,
 )
 from dvc.exceptions import DvcException, ReproductionError
+from dvc.repo import Repo
 from dvc.repo.experiments.exceptions import ExperimentExistsError
 from dvc.repo.experiments.queue.base import BaseStashQueue
 from dvc.repo.experiments.refs import CELERY_STASH
@@ -51,7 +53,7 @@ def test_new_simple(tmp_dir, scm, dvc, exp_stage, mocker, name, workspace):
    if workspace:
        assert (tmp_dir / "metrics.yaml").read_text().strip() == "foo: 2"

-    exp_name = name if name else ref_info.name
+    exp_name = name or ref_info.name
    assert dvc.experiments.get_exact_name([exp])[exp] == exp_name
    assert resolve_rev(scm, exp_name) == exp

@@ -808,3 +810,26 @@ def test_experiments_run_with_submodule_dependencies(dvc, scm, make_tmp_dir, dep
    dvc.stage.add(cmd="echo foo", deps=[dep], name="foo")

    assert dvc.experiments.run()
+
+
+@pytest.mark.skipif(dulwich.__version__ < (0, 24, 2), reason="requires dulwich>=0.24.2")
+def test_experiments_run_in_linked_git_worktree(
+    dvc, scm, tmp_path_factory: pytest.TempPathFactory, monkeypatch
+):
+    from dulwich.worktree import add_worktree
+
+    wt = tmp_path_factory.mktemp("worktrees") / "worktree"
+    add_worktree(scm.dulwich.repo, wt, branch="wt-main")
+
+    monkeypatch.chdir(wt)
+
+    wt_dvc = Repo(os.fspath(wt))
+    (wt / "foo").write_bytes(b"foo")
+    wt_dvc.stage.add(cmd="cp foo bar", deps=["foo"], outs=["bar"], name="cp")
+
+    results = wt_dvc.experiments.run(name="my-exp")
+    assert results
+    rev = first(results)
+    assert rev
+    # If `bar` exists, we know that the stage was run.
+    assert (wt / "bar").read_bytes() == b"foo"
@@ -26,7 +26,7 @@ def test_celery_logs(tmp_dir, scm, dvc, failed_exp_stage, follow, capsys, test_q


 def test_queue_doesnt_remove_untracked_params_file(tmp_dir, dvc, scm):
-    """Regression test for https://github.com/iterative/dvc/issues/7842"""
+    """Regression test for https://github.com/treeverse/dvc/issues/7842"""
    tmp_dir.gen("params.yaml", "foo: 1")
    stage = dvc.run(cmd="echo ${foo}", params=["foo"], name="echo-foo")
    scm.add(["dvc.yaml", "dvc.lock", ".gitignore"])
@@ -204,10 +204,10 @@ def test_remove_multi_rev(tmp_dir, scm, dvc, exp_stage):
@pytest.mark.parametrize(
    "keep, expected_removed",
    [
-        [["exp1"], ["exp2", "exp3"]],
-        [["exp1", "exp2"], ["exp3"]],
-        [["exp1", "exp2", "exp3"], []],
-        [[], []],  # remove does nothing if no experiments are specified
+        (["exp1"], ["exp2", "exp3"]),
+        (["exp1", "exp2"], ["exp3"]),
+        (["exp1", "exp2", "exp3"], []),
+        ([], []),  # remove does nothing if no experiments are specified
    ],
 )
 def test_keep_selected_by_name(tmp_dir, scm, dvc, exp_stage, keep, expected_removed):
@@ -239,14 +239,14 @@ def test_keep_selected_by_nonexistent_name(tmp_dir, scm, dvc, exp_stage):
@pytest.mark.parametrize(
    "num_exps, rev, num, expected_removed",
    [
-        [2, "exp1", 1, ["exp2"]],
-        [3, "exp3", 1, ["exp1", "exp2"]],
-        [3, "exp3", 2, ["exp1"]],
-        [3, "exp3", 3, []],
-        [3, "exp2", 2, ["exp3"]],
-        [4, "exp2", 2, ["exp3", "exp4"]],
-        [4, "exp4", 2, ["exp1", "exp2"]],
-        [1, None, 1, []],  # remove does nothing if no experiments are specified
+        (2, "exp1", 1, ["exp2"]),
+        (3, "exp3", 1, ["exp1", "exp2"]),
+        (3, "exp3", 2, ["exp1"]),
+        (3, "exp3", 3, []),
+        (3, "exp2", 2, ["exp3"]),
+        (4, "exp2", 2, ["exp3", "exp4"]),
+        (4, "exp4", 2, ["exp1", "exp2"]),
+        (1, None, 1, []),  # remove does nothing if no experiments are specified
    ],
 )
 def test_keep_selected_by_rev(
@@ -18,7 +18,7 @@ def test_exp_save_unchanged(tmp_dir, dvc, scm):
    dvc.experiments.save()


-@pytest.mark.parametrize("name", (None, "test"))
+@pytest.mark.parametrize("name", [None, "test"])
 def test_exp_save(tmp_dir, dvc, scm, name):
    setup_stage(tmp_dir, dvc, scm)
    baseline = scm.get_rev()
@@ -28,7 +28,7 @@ def test_exp_save(tmp_dir, dvc, scm, name):
    assert ref_info
    assert ref_info.baseline_sha == baseline

-    exp_name = name if name else ref_info.name
+    exp_name = name or ref_info.name
    assert dvc.experiments.get_exact_name([exp])[exp] == exp_name
    assert resolve_rev(scm, exp_name) == exp

@@ -47,13 +47,13 @@ def test_exp_save_overwrite_experiment(tmp_dir, dvc, scm):

@pytest.mark.parametrize(
    "name",
-    (
+    [
        "invalid/name",
        "invalid..name",
        "invalid~name",
        "invalid?name",
        "invalidname.",
-    ),
+    ],
 )
 def test_exp_save_invalid_name(tmp_dir, dvc, scm, name):
    setup_stage(tmp_dir, dvc, scm)
@@ -105,7 +105,7 @@ def test_exp_save_include_untracked(tmp_dir, dvc, scm):


 def test_exp_save_include_untracked_warning(tmp_dir, dvc, scm, mocker):
-    """Regression test for https://github.com/iterative/dvc/issues/9061"""
+    """Regression test for https://github.com/treeverse/dvc/issues/9061"""
    setup_stage(tmp_dir, dvc, scm)

    new_dir = tmp_dir / "new_dir"
@@ -7,8 +7,8 @@ from tests.func.utils.test_hydra import hydra_setup
@pytest.mark.parametrize(
    "changes, expected",
    [
-        [["foo=baz"], "foo: baz\ngoo:\n  bag: 3.0\nlorem: false"],
-        [["params.yaml:foo=baz"], "foo: baz\ngoo:\n  bag: 3.0\nlorem: false"],
+        (["foo=baz"], "foo: baz\ngoo:\n  bag: 3.0\nlorem: false"),
+        (["params.yaml:foo=baz"], "foo: baz\ngoo:\n  bag: 3.0\nlorem: false"),
    ],
 )
 def test_modify_params(params_repo, dvc, changes, expected):
@@ -26,8 +26,9 @@ def test_modify_params(params_repo, dvc, changes, expected):
        ("conf", "bar"),
    ],
 )
+@pytest.mark.parametrize("no_hydra", [True, False])
 def test_hydra_compose_and_dump(
-    tmp_dir, params_repo, dvc, hydra_enabled, config_dir, config_name
+    tmp_dir, params_repo, dvc, hydra_enabled, config_dir, config_name, no_hydra
 ):
    hydra_setup(
        tmp_dir,
@@ -50,14 +51,14 @@ def test_hydra_compose_and_dump(
        if config_name is not None:
            conf["hydra"]["config_name"] = config_name

-    dvc.experiments.run()
+    dvc.experiments.run(no_hydra=no_hydra)

-    if hydra_enabled:
+    if hydra_enabled and not no_hydra:
        assert (tmp_dir / "params.yaml").parse() == {
            "db": {"driver": "mysql", "user": "omry", "pass": "secret"},
        }

-        dvc.experiments.run(params=["db=postgresql"])
+        dvc.experiments.run(params=["db=postgresql"], no_hydra=no_hydra)
        assert (tmp_dir / "params.yaml").parse() == {
            "db": {
                "driver": "postgresql",
@@ -117,6 +118,7 @@ def test_hydra_sweep(
            targets=None,
            copy_paths=None,
            message=None,
+            no_hydra=False,
        )


@@ -137,3 +139,76 @@ def test_hydra_sweep_prefix_name(tmp_dir, params_repo, dvc):
    exp_names = [entry.name for entry in dvc.experiments.celery_queue.iter_queued()]
    for name, expected in zip(exp_names, expected_names):
        assert name == expected
+
+
+def test_mixing_no_hydra_and_params_flags(tmp_dir, params_repo, dvc):
+    # Passing no_hydra should not prevent user from
+    # using --set-param on unmodified params.yaml
+    hydra_setup(
+        tmp_dir,
+        config_dir="conf",
+        config_name="config",
+    )
+
+    with dvc.config.edit() as conf:
+        conf["hydra"]["enabled"] = True
+        conf["hydra"]["config_dir"] = "conf"
+        conf["hydra"]["config_name"] = "config"
+
+    dvc.experiments.run(no_hydra=True, params=["goo.bag=10.0"])
+
+    assert (tmp_dir / "params.yaml").parse() == {
+        "foo": [{"bar": 1}, {"baz": 2}],
+        "goo": {"bag": 10.0},
+        "lorem": False,
+    }
+
+
+@pytest.mark.parametrize(
+    "hydra_enabled,overrides,expected",
+    [
+        (
+            True,
+            ["db=mysql,postgresql"],
+            [
+                {"params.yaml": ["db=mysql"]},
+                {"params.yaml": ["db=postgresql"]},
+            ],
+        ),
+        (
+            False,
+            ["foo=bar,baz"],
+            [{"params.yaml": ["foo=bar"]}, {"params.yaml": ["foo=baz"]}],
+        ),
+        (
+            False,
+            [],
+            [{}],
+        ),
+    ],
+)
+@pytest.mark.parametrize("no_hydra", [True, False])
+def test_mixing_no_hydra_and_sweeps(
+    tmp_dir, params_repo, dvc, mocker, hydra_enabled, overrides, expected, no_hydra
+):
+    # Passing no_hydra should not prevent user from
+    # queuing sweeps with --set-param and --queue
+    patched = mocker.patch.object(dvc.experiments, "queue_one")
+
+    if hydra_enabled:
+        hydra_setup(tmp_dir, config_dir="conf", config_name="config")
+        with dvc.config.edit() as conf:
+            conf["hydra"]["enabled"] = True
+
+    dvc.experiments.run(params=overrides, queue=True, no_hydra=no_hydra)
+
+    assert patched.call_count == len(expected)
+    for e in expected:
+        patched.assert_any_call(
+            mocker.ANY,
+            params=e,
+            targets=None,
+            copy_paths=None,
+            message=None,
+            no_hydra=no_hydra,
+        )
@@ -4,6 +4,7 @@ from os.path import join
 import pytest

 from dvc.cli import main
+from dvc.testing import matchers as M
 from dvc.utils import relpath
 from dvc.utils.serialize import JSONFileCorruptedError

@@ -95,7 +96,7 @@ def test_metrics_diff_json_unchanged(tmp_dir, scm, dvc, run_copy_metrics):
    assert dvc.metrics.diff(a_rev="HEAD~2") == {}


-def test_metrics_diff_broken_json(M, tmp_dir, scm, dvc, run_copy_metrics):
+def test_metrics_diff_broken_json(tmp_dir, scm, dvc, run_copy_metrics):
    metrics = {"a": {"b": {"c": 1, "d": 1, "e": "3"}}}
    (tmp_dir / "m_temp.json").dump(metrics)
    run_copy_metrics(
@@ -143,7 +144,7 @@ def test_metrics_diff_new_metric(tmp_dir, scm, dvc, run_copy_metrics):
    }


-def test_metrics_diff_deleted_metric(M, tmp_dir, scm, dvc, run_copy_metrics):
+def test_metrics_diff_deleted_metric(tmp_dir, scm, dvc, run_copy_metrics):
    metrics = {"a": {"b": {"c": 1, "d": 1, "e": "3"}}}
    (tmp_dir / "m_temp.json").dump(metrics)
    run_copy_metrics(
@@ -11,6 +11,7 @@ from dvc.dvcfile import PROJECT_FILE
 from dvc.exceptions import OverlappingOutputPathsError
 from dvc.repo import Repo
 from dvc.repo.metrics.show import FileResult, Result
+from dvc.testing import matchers as M
 from dvc.utils.fs import remove
 from dvc.utils.serialize import JSONFileCorruptedError
 from dvc_data.index import DataIndexDirError
@@ -246,7 +247,7 @@ def test_show_malformed_metric(tmp_dir, scm, dvc, caplog):
    )


-def test_metrics_show_no_target(M, tmp_dir, dvc, capsys):
+def test_metrics_show_no_target(tmp_dir, dvc, capsys):
    assert dvc.metrics.show(targets=["metrics.json"]) == {
        "": {"data": {"metrics.json": {"error": M.instance_of(FileNotFoundError)}}}
    }
@@ -289,14 +290,14 @@ def test_metrics_show_overlap(tmp_dir, dvc, run_copy_metrics, clear_before_run):

@pytest.mark.parametrize(
    "file,error_path,err_type",
-    (
+    [
        (PROJECT_FILE, ["workspace", "error", "type"], "YAMLSyntaxError"),
        (
            "metrics.yaml",
            ["workspace", "data", "metrics.yaml", "error", "type"],
            "YAMLFileCorruptedError",
        ),
-    ),
+    ],
 )
 def test_log_errors(
    tmp_dir, scm, dvc, capsys, run_copy_metrics, file, error_path, err_type
@@ -356,7 +357,7 @@ def test_top_level_parametrized(tmp_dir, dvc):
    }


-def test_metric_in_a_tracked_directory_with_missing_dir_file(M, tmp_dir, dvc):
+def test_metric_in_a_tracked_directory_with_missing_dir_file(tmp_dir, dvc):
    tmp_dir.dvc_gen({"dir": {"file": "2"}})
    (tmp_dir / "dvc.yaml").dump({"metrics": [join("dir", "file")]})
    shutil.rmtree(tmp_dir / "dir")  # remove from workspace
@@ -2,6 +2,7 @@ from os.path import join

 import pytest

+from dvc.testing import matchers as M
 from dvc.utils import relpath


@@ -54,7 +55,7 @@ def test_diff_new(tmp_dir, scm, dvc):
    }


-def test_diff_deleted(M, tmp_dir, scm, dvc):
+def test_diff_deleted(tmp_dir, scm, dvc):
    tmp_dir.gen("params.yaml", "foo: bar")
    dvc.run(cmd="echo params.yaml", params=["foo"], name="echo-params")
    scm.add(["params.yaml", "Dvcfile"])
@@ -6,6 +6,7 @@ import pytest
 from dvc.dvcfile import PROJECT_FILE
 from dvc.repo import Repo
 from dvc.repo.metrics.show import FileResult, Result
+from dvc.testing import matchers as M
 from dvc_data.index import DataIndexDirError


@@ -197,7 +198,7 @@ def test_top_level_parametrized(tmp_dir, dvc):
    }


-def test_param_in_a_tracked_directory_with_missing_dir_file(M, tmp_dir, dvc):
+def test_param_in_a_tracked_directory_with_missing_dir_file(tmp_dir, dvc):
    tmp_dir.dvc_gen({"dir": {"file": "2"}})
    (tmp_dir / "dvc.yaml").dump({"params": [join("dir", "file")]})
    shutil.rmtree(tmp_dir / "dir")  # remove from workspace
@@ -300,13 +300,13 @@ def test_ignore_parsing_error(tmp_dir, dvc, run_copy_metrics):

@pytest.mark.parametrize(
    "file,path_kwargs",
-    (
+    [
        (PROJECT_FILE, {"revision": "workspace", "endkey": "error"}),
        (
            "plot.yaml",
            {"revision": "workspace", "file": "plot.yaml", "endkey": "error"},
        ),
-    ),
+    ],
 )
 def test_log_errors(tmp_dir, scm, dvc, run_copy_metrics, file, path_kwargs, capsys):
    metric = [{"val": 2}, {"val": 3}]
@@ -15,6 +15,7 @@ from dvc.output import Output
 from dvc.stage import PipelineStage, Stage
 from dvc.stage.cache import RunCacheNotSupported
 from dvc.stage.exceptions import StageFileDoesNotExistError, StageNotFound
+from dvc.testing import matchers as M
 from dvc.utils.fs import remove
 from dvc.utils.serialize import modify_yaml
 from dvc_data.hashfile.hash import file_md5
@@ -55,7 +56,7 @@ def test_repro_frozen(tmp_dir, dvc, run_copy):
    assert stages == [data_stage, stage0]


-def test_downstream(M, tmp_dir, dvc):
+def test_downstream(tmp_dir, dvc):
    # The dependency graph should look like this:
    #
    #       E
@@ -148,7 +149,7 @@ def test_downstream(M, tmp_dir, dvc):
    assert [stage.name for stage in evaluation] == ["B-gen", "D-gen", "E-gen"]

    # B, C should be run (in any order) before D
-    # See https://github.com/iterative/dvc/issues/3602
+    # See https://github.com/treeverse/dvc/issues/3602
    evaluation = dvc.reproduce(PROJECT_FILE + ":A-gen", downstream=True, force=True)

    assert len(evaluation) == 5
@@ -36,7 +36,7 @@ def test_repro_allow_missing_and_pull(tmp_dir, dvc, mocker, local_remote):
 def test_repro_allow_missing_upstream_stage_modified(
    tmp_dir, dvc, mocker, local_remote
 ):
-    """https://github.com/iterative/dvc/issues/9530"""
+    """https://github.com/treeverse/dvc/issues/9530"""
    tmp_dir.gen("params.yaml", "param: 1")
    dvc.stage.add(
        name="create-foo", cmd="echo ${param} > foo", params=["param"], outs=["foo"]
@@ -723,7 +723,7 @@ def test_add_symlink_file(tmp_dir, dvc):
    ).is_symlink()

    # Test that subsequent add succeeds
-    # See https://github.com/iterative/dvc/issues/4654
+    # See https://github.com/treeverse/dvc/issues/4654
    dvc.add(os.path.join("dir", "foo"))


--- a/Show More
+++ b/Show More