mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 23:09:28 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f342cc7db5 | ||
|
|
0eb279ea35 | ||
|
|
d086bbbd3a | ||
|
|
046d2fd72e | ||
|
|
cb96cf1487 | ||
|
|
cf5168af74 | ||
|
|
dc123102cc | ||
|
|
438259cc70 | ||
|
|
7b978fec5b | ||
|
|
9f901e57a2 | ||
|
|
ab5ac17925 | ||
|
|
72abd013cf | ||
|
|
fa5c33ed24 | ||
|
|
9dda30b7a0 | ||
|
|
19c146e392 | ||
|
|
749c22bbcb | ||
|
|
19a0bf48f1 | ||
|
|
bda614baa8 | ||
|
|
d6fa8b67ff | ||
|
|
498a4d5d3a | ||
|
|
b74711d5fe | ||
|
|
61d5de54e5 | ||
|
|
ca7454980d | ||
|
|
8912ca7d2b | ||
|
|
f13ae75a45 | ||
|
|
aa090ecf8d | ||
|
|
0c4a5cbbb7 | ||
|
|
2b19c724c5 | ||
|
|
28b92c8b03 | ||
|
|
cd740fed42 | ||
|
|
5f31ed2b08 | ||
|
|
6e896d396c | ||
|
|
336c8695da | ||
|
|
3915fa26aa | ||
|
|
114fd8075c | ||
|
|
b39a888948 | ||
|
|
c1af66c0a6 |
@@ -737,8 +737,8 @@
|
||||
"tests/func/test_config.py::test_config_gdrive_fields": 0.12703910000016094,
|
||||
"tests/func/test_config.py::test_config_get[args0-0-False]": 0.20948869999995168,
|
||||
"tests/func/test_config.py::test_config_get[args1-0-myremote]": 0.1829517000001033,
|
||||
"tests/func/test_config.py::test_config_get[args2-0-iterative]": 0.19686669999998685,
|
||||
"tests/func/test_config.py::test_config_get[args3-0-iterative]": 0.1675059000000374,
|
||||
"tests/func/test_config.py::test_config_get[args2-0-treeverse]": 0.19686669999998685,
|
||||
"tests/func/test_config.py::test_config_get[args3-0-treeverse]": 0.1675059000000374,
|
||||
"tests/func/test_config.py::test_config_get[args4-251-option 'profile' doesn't exist]": 0.18096200000002227,
|
||||
"tests/func/test_config.py::test_config_get[args5-0-gs://bucket/path]": 0.12368970000022728,
|
||||
"tests/func/test_config.py::test_config_get[args6-0-gs://bucket/path]": 0.13007389999995667,
|
||||
@@ -2642,7 +2642,7 @@
|
||||
"tests/unit/test_updater.py::test_is_enabled[config0-True]": 0.13130560000013247,
|
||||
"tests/unit/test_updater.py::test_is_enabled[config1-True]": 0.14342600000009043,
|
||||
"tests/unit/test_updater.py::test_is_enabled[config2-False]": 0.13593219999984285,
|
||||
"tests/unit/test_updater.py::test_notify_message[None-Find the latest release at https://github.com/iterative/dvc/releases/latest.]": 0.014024399999925663,
|
||||
"tests/unit/test_updater.py::test_notify_message[None-Find the latest release at https://github.com/treeverse/dvc/releases/latest.]": 0.014024399999925663,
|
||||
"tests/unit/test_updater.py::test_notify_message[binary-To upgrade, uninstall dvc and reinstall from https://dvc.org.]": 0.013217200000326557,
|
||||
"tests/unit/test_updater.py::test_notify_message[brew-To upgrade, run 'brew upgrade dvc'.]": 0.013467300000229443,
|
||||
"tests/unit/test_updater.py::test_notify_message[choco-To upgrade, run 'choco upgrade dvc'.]": 0.014645900000004985,
|
||||
@@ -2652,7 +2652,7 @@
|
||||
"tests/unit/test_updater.py::test_notify_message[osxpkg-To upgrade, uninstall dvc and reinstall from https://dvc.org.]": 0.013980499999888707,
|
||||
"tests/unit/test_updater.py::test_notify_message[pip-To upgrade, run 'pip install --upgrade dvc'.]": 0.015155299999833005,
|
||||
"tests/unit/test_updater.py::test_notify_message[rpm-To upgrade, run 'yum update dvc'.]": 0.014036500000202068,
|
||||
"tests/unit/test_updater.py::test_notify_message[unknown-Find the latest release at https://github.com/iterative/dvc/releases/latest.]": 0.01609110000003966,
|
||||
"tests/unit/test_updater.py::test_notify_message[unknown-Find the latest release at https://github.com/treeverse/dvc/releases/latest.]": 0.01609110000003966,
|
||||
"tests/unit/ui/test_console.py::test_capsys_works": 0.003642499999841675,
|
||||
"tests/unit/ui/test_console.py::test_write": 0.004320600000255581,
|
||||
"tests/unit/ui/test_console.py::test_write_json[False-{\"hello\": \"world\", \"date\": \"1970-01-01 00:00:00\"}\\n]": 0.004433300000073359,
|
||||
|
||||
@@ -57,7 +57,7 @@ $ dvc doctor
|
||||
**Additional Information (if any):**
|
||||
|
||||
<!--
|
||||
Please check https://github.com/iterative/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.
|
||||
Please check https://github.com/treeverse/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.
|
||||
|
||||
If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.
|
||||
If the issue is regarding the performance, please attach the profiling information and the benchmark comparisons.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
* [ ] ❗ I have followed the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) checklist.
|
||||
|
||||
* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here.
|
||||
* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/treeverse/dvc.org) and linked it here.
|
||||
|
||||
Thank you for the contribution - we'll try to review it as soon as possible. 🙏
|
||||
|
||||
@@ -14,16 +14,16 @@ jobs:
|
||||
name: run benchmarks
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/setup-python@v5
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.13"
|
||||
python-version: "3.14"
|
||||
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: pyproject.toml
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
- name: run benchmarks on base branch
|
||||
run: pytest --benchmark-autosave dvc/testing/benchmarks/ -k 'test_init or test_help'
|
||||
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
clean: false
|
||||
|
||||
@@ -19,15 +19,15 @@ jobs:
|
||||
name: Build & Verify package
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: 3.13
|
||||
python-version: "3.14"
|
||||
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
|
||||
- name: Omit local version for Test PyPI upload
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
- name: Check dist
|
||||
run: uv tool run twine check --strict dist/*
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
- uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: Packages
|
||||
path: dist/
|
||||
@@ -51,7 +51,7 @@ jobs:
|
||||
name: Publish dev package to test.pypi.org
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
if: ${{ github.event.action == 'published' || (github.repository == 'iterative/dvc' && github.event_name == 'push' && github.ref == 'refs/heads/main') }}
|
||||
if: ${{ github.event.action == 'published' || (github.repository == 'treeverse/dvc' && github.event_name == 'push' && github.ref == 'refs/heads/main') }}
|
||||
|
||||
environment:
|
||||
name: test-pypi
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- uses: actions/download-artifact@v5
|
||||
- uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: Packages
|
||||
path: dist
|
||||
@@ -86,7 +86,7 @@ jobs:
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- uses: actions/download-artifact@v5
|
||||
- uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: Packages
|
||||
path: dist
|
||||
|
||||
@@ -28,13 +28,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v5
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v3
|
||||
uses: github/codeql-action/init@v4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
queries: security-extended
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v3
|
||||
uses: github/codeql-action/analyze@v4
|
||||
|
||||
@@ -19,26 +19,26 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest]
|
||||
pyv: ["3.13"]
|
||||
pyv: ["3.14"]
|
||||
plugin: ["dvc-s3"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
path: dvc
|
||||
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
repository: iterative/${{ matrix.plugin }}
|
||||
repository: treeverse/${{ matrix.plugin }}
|
||||
ref: main
|
||||
path: ${{ matrix.plugin }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.pyv }}
|
||||
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: "**/pyproject.toml"
|
||||
|
||||
@@ -26,14 +26,14 @@ jobs:
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.13'
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
python-version: '3.14'
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: pyproject.toml
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest]
|
||||
pyv: ["3.9", "3.10", "3.11", "3.12", "3.13"]
|
||||
pyv: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
|
||||
include:
|
||||
- os: windows-latest
|
||||
pyv: "3.9"
|
||||
@@ -117,15 +117,27 @@ jobs:
|
||||
- os: windows-latest
|
||||
pyv: "3.13"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
|
||||
- os: windows-latest
|
||||
pyv: "3.14"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
|
||||
- os: windows-latest
|
||||
pyv: "3.14"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
|
||||
- os: windows-latest
|
||||
pyv: "3.14"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
|
||||
- os: windows-latest
|
||||
pyv: "3.14"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.pyv }}
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: pyproject.toml
|
||||
|
||||
@@ -22,7 +22,7 @@ repos:
|
||||
- id: sort-simple-yaml
|
||||
- id: trailing-whitespace
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: 'v0.12.11'
|
||||
rev: 'v0.14.7'
|
||||
hooks:
|
||||
- id: ruff-check
|
||||
args: [--fix, --exit-non-zero-on-fix]
|
||||
|
||||
+1
-1
@@ -4,5 +4,5 @@
|
||||
"data-science", "data-version-control", "machine-learning", "git",
|
||||
"developer-tools", "reproducibility", "collaboration", "ai", "python"],
|
||||
"contributors": [
|
||||
{"name": "DVC team", "type": "Other", "affiliation": "Iterative"}]
|
||||
{"name": "DVC team", "type": "Other", "affiliation": "Treeverse"}]
|
||||
}
|
||||
|
||||
+2
-2
@@ -6,8 +6,8 @@ message: >-
|
||||
type: software
|
||||
authors:
|
||||
- name: The DVC team and contributors
|
||||
website: 'https://github.com/iterative'
|
||||
repository-code: 'https://github.com/iterative/dvc'
|
||||
website: 'https://github.com/treeverse'
|
||||
repository-code: 'https://github.com/treeverse/dvc'
|
||||
url: 'https://dvc.org'
|
||||
abstract: >-
|
||||
DVC is a tool for data science that takes advantage of existing software engineering toolset.
|
||||
|
||||
@@ -186,7 +186,7 @@
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2017-2021 Iterative, Inc.
|
||||
Copyright 2025 Treeverse.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
+9
-15
@@ -167,7 +167,7 @@ To install the development version, run:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pip install git+git://github.com/iterative/dvc
|
||||
pip install "dvc @ git+https://github.com/treeverse/dvc"
|
||||
|
||||
Package (Platform-specific)
|
||||
---------------------------
|
||||
@@ -175,7 +175,7 @@ Package (Platform-specific)
|
||||
|Packages|
|
||||
|
||||
Self-contained packages for Linux, Windows, and Mac are available.
|
||||
The latest version of the packages can be found on the GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
|
||||
The latest version of the packages can be found on the GitHub `releases page <https://github.com/treeverse/dvc/releases>`_.
|
||||
|
||||
Ubuntu / Debian (deb)
|
||||
^^^^^^^^^^^^^^^^^^^^^
|
||||
@@ -198,8 +198,6 @@ Fedora / CentOS (rpm)
|
||||
Contributing
|
||||
============
|
||||
|
||||
|Maintainability|
|
||||
|
||||
Contributions are welcome!
|
||||
Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more details.
|
||||
Thanks to all our contributors!
|
||||
@@ -237,23 +235,19 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
|
||||
:target: https://dvc.org
|
||||
:alt: DVC logo
|
||||
|
||||
.. |VS Code Extension Overview| image:: https://raw.githubusercontent.com/iterative/vscode-dvc/main/extension/docs/overview.gif
|
||||
.. |VS Code Extension Overview| image:: https://raw.githubusercontent.com/treeverse/vscode-dvc/main/extension/docs/overview.gif
|
||||
:alt: DVC Extension for VS Code
|
||||
|
||||
.. |CI| image:: https://github.com/iterative/dvc/actions/workflows/tests.yaml/badge.svg
|
||||
:target: https://github.com/iterative/dvc/actions/workflows/tests.yaml
|
||||
.. |CI| image:: https://github.com/treeverse/dvc/actions/workflows/tests.yaml/badge.svg
|
||||
:target: https://github.com/treeverse/dvc/actions/workflows/tests.yaml
|
||||
:alt: GHA Tests
|
||||
|
||||
.. |Maintainability| image:: https://img.shields.io/codeclimate/maintainability/iterative/dvc
|
||||
:target: https://codeclimate.com/github/iterative/dvc
|
||||
:alt: Code Climate
|
||||
|
||||
.. |Python Version| image:: https://img.shields.io/pypi/pyversions/dvc
|
||||
:target: https://pypi.org/project/dvc
|
||||
:alt: Python Version
|
||||
|
||||
.. |Coverage| image:: https://codecov.io/gh/iterative/dvc/branch/main/graph/badge.svg
|
||||
:target: https://codecov.io/gh/iterative/dvc
|
||||
.. |Coverage| image:: https://codecov.io/gh/treeverse/dvc/branch/main/graph/badge.svg
|
||||
:target: https://codecov.io/gh/treeverse/dvc
|
||||
:alt: Codecov
|
||||
|
||||
.. |Snap| image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
|
||||
@@ -292,8 +286,8 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
|
||||
:target: https://dvc.org/img/flow.gif
|
||||
:alt: how_dvc_works
|
||||
|
||||
.. |Contribs| image:: https://contrib.rocks/image?repo=iterative/dvc
|
||||
:target: https://github.com/iterative/dvc/graphs/contributors
|
||||
.. |Contribs| image:: https://contrib.rocks/image?repo=treeverse/dvc
|
||||
:target: https://github.com/treeverse/dvc/graphs/contributors
|
||||
:alt: Contributors
|
||||
|
||||
.. |VS Code| image:: https://img.shields.io/visual-studio-marketplace/v/Iterative.dvc?color=blue&label=VSCode&logo=visualstudiocode&logoColor=blue
|
||||
|
||||
+1
-1
@@ -206,7 +206,7 @@ def add_debugging_flags(parser):
|
||||
from argparse import SUPPRESS
|
||||
|
||||
# For detailed info see:
|
||||
# https://github.com/iterative/dvc/wiki/Debugging,-Profiling-and-Benchmarking-DVC
|
||||
# https://github.com/treeverse/dvc/wiki/Debugging,-Profiling-and-Benchmarking-DVC
|
||||
args, _ = parser.parse_known_args()
|
||||
verbose = args.verbose
|
||||
|
||||
|
||||
@@ -64,3 +64,4 @@ def get(name: str) -> Union[DatachainDataset, DVCDataset, URLDataset]:
|
||||
for file in dataset.lock.files
|
||||
]
|
||||
return URLDataset(type="url", files=files, path=versioned_path)
|
||||
raise AssertionError("unreachable")
|
||||
|
||||
+1
-1
@@ -96,7 +96,7 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
|
||||
f"See {link} for more info."
|
||||
)
|
||||
else:
|
||||
link = format_link("https://github.com/iterative/dvc/issues")
|
||||
link = format_link("https://github.com/treeverse/dvc/issues")
|
||||
hint = f"\nPlease report this bug to {link}. Thank you!"
|
||||
|
||||
logger.exception(
|
||||
|
||||
@@ -27,6 +27,10 @@ _dvc_compgen_remotes() {
|
||||
local _dvc_remotes=($(dvc remote list | cut -d' ' -f1))
|
||||
compgen -W "${_dvc_remotes[*]}" -- $1
|
||||
}
|
||||
|
||||
_dvc_compgen_config_vars() {
|
||||
compgen -W "${_dvc_config_vars[*]}" -- $1
|
||||
}
|
||||
"""
|
||||
|
||||
ZSH_PREAMBLE = """
|
||||
@@ -64,6 +68,10 @@ _dvc_compadd_exps() {
|
||||
_dvc_compadd_remotes() {
|
||||
_describe 'remotes' "($(dvc remote list | cut -d' ' -f1))"
|
||||
}
|
||||
|
||||
_dvc_compadd_config_vars() {
|
||||
_describe 'config_vars' _dvc_config_vars
|
||||
}
|
||||
"""
|
||||
|
||||
PREAMBLE = {
|
||||
@@ -81,3 +89,21 @@ DVCFILES_AND_STAGE = {
|
||||
}
|
||||
EXPERIMENT = {"bash": "_dvc_compgen_exps", "zsh": "_dvc_compadd_exps"}
|
||||
REMOTE = {"bash": "_dvc_compgen_remotes", "zsh": "_dvc_compadd_remotes"}
|
||||
CONFIG_VARS = {"bash": "_dvc_compgen_config_vars", "zsh": "_dvc_compadd_config_vars"}
|
||||
|
||||
|
||||
def get_preamble() -> dict[str, str]:
|
||||
from dvc.config_schema import config_vars_for_completion
|
||||
|
||||
ret: dict[str, str] = {}
|
||||
config_vars = list(config_vars_for_completion())
|
||||
|
||||
nl = "\n\t".expandtabs(4)
|
||||
config_vars_arr = f"""
|
||||
_dvc_config_vars=(
|
||||
{nl.join(config_vars)}
|
||||
)
|
||||
"""
|
||||
for shell, preamble in PREAMBLE.items():
|
||||
ret[shell] = config_vars_arr + preamble
|
||||
return ret
|
||||
|
||||
@@ -16,7 +16,7 @@ class CmdCheckIgnore(CmdBase):
|
||||
|
||||
def _show_results(self, result: "CheckIgnoreResult"):
|
||||
if not result.match and not self.args.details:
|
||||
return
|
||||
return None
|
||||
|
||||
if self.args.details:
|
||||
pattern_infos = result.pattern_infos
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from dvc.cli import formatter
|
||||
from dvc.cli.command import CmdBaseNoRepo
|
||||
from dvc.cli.completion import PREAMBLE
|
||||
from dvc.cli.completion import get_preamble
|
||||
from dvc.cli.utils import append_doc_link
|
||||
from dvc.log import logger
|
||||
from dvc.ui import ui
|
||||
@@ -17,7 +17,7 @@ class CmdCompletion(CmdBaseNoRepo):
|
||||
|
||||
shell = self.args.shell
|
||||
parser = self.args.parser
|
||||
script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
|
||||
script = shtab.complete(parser, shell=shell, preamble=get_preamble())
|
||||
ui.write(script, force=True)
|
||||
return 0
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ import os
|
||||
|
||||
from funcy import set_in
|
||||
|
||||
from dvc.cli import formatter
|
||||
from dvc.cli import completion, formatter
|
||||
from dvc.cli.command import CmdBaseNoRepo
|
||||
from dvc.cli.utils import append_doc_link
|
||||
from dvc.log import logger
|
||||
@@ -224,7 +224,7 @@ def add_parser(subparsers, parent_parser):
|
||||
nargs="?",
|
||||
type=_name_type,
|
||||
help="Option name (section.option or remote.name.option).",
|
||||
)
|
||||
).complete = completion.CONFIG_VARS
|
||||
config_parser.add_argument("value", nargs="?", help="Option value.")
|
||||
config_parser.add_argument(
|
||||
"-l",
|
||||
|
||||
@@ -83,7 +83,7 @@ class CmdDatasetUpdate(CmdBase):
|
||||
return CmdDatasetAdd.display(name, new, action)
|
||||
if dataset == new:
|
||||
ui.write("[yellow]Nothing to update[/]", styled=True)
|
||||
return
|
||||
return None
|
||||
|
||||
assert new.lock
|
||||
|
||||
@@ -112,6 +112,7 @@ class CmdDatasetUpdate(CmdBase):
|
||||
assert new.type == "url"
|
||||
stats = diff_files(dataset.lock.files, new.lock.files)
|
||||
log_changes(stats)
|
||||
return None
|
||||
|
||||
def run(self):
|
||||
from difflib import get_close_matches
|
||||
|
||||
@@ -20,6 +20,7 @@ class CmdExperimentsRun(CmdRepro):
|
||||
tmp_dir=self.args.tmp_dir,
|
||||
copy_paths=self.args.copy_paths,
|
||||
message=self.args.message,
|
||||
no_hydra=self.args.no_hydra,
|
||||
**self._common_kwargs,
|
||||
)
|
||||
|
||||
@@ -107,4 +108,14 @@ def _add_run_common(parser):
|
||||
default=None,
|
||||
help="Custom commit message to use when committing the experiment.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-hydra",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"Disables automatically updating `params.yaml` with Hydra configuration. "
|
||||
" You can still use `--set-param` to update individual params if needed."
|
||||
" Default is False."
|
||||
),
|
||||
)
|
||||
parser.add_argument("-M", dest="message", help=argparse.SUPPRESS) # obsolete
|
||||
|
||||
@@ -29,7 +29,7 @@ def _welcome_message():
|
||||
"{yellow}------------{nc}\n"
|
||||
f"- Check out the documentation: {fmt_link('https://dvc.org/doc')}\n"
|
||||
f"- Get help and share ideas: {fmt_link('https://dvc.org/chat')}\n"
|
||||
f"- Star us on GitHub: {fmt_link('https://github.com/iterative/dvc')}"
|
||||
f"- Star us on GitHub: {fmt_link('https://github.com/treeverse/dvc')}"
|
||||
).format(yellow=colorama.Fore.YELLOW, nc=colorama.Fore.RESET)
|
||||
|
||||
ui.write(msg)
|
||||
|
||||
@@ -139,6 +139,7 @@ class Config(dict):
|
||||
return global_config_dir()
|
||||
if level == "system":
|
||||
return system_config_dir()
|
||||
return None
|
||||
|
||||
@cached_property
|
||||
def files(self) -> dict[str, str]:
|
||||
|
||||
+173
-141
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
from typing import TYPE_CHECKING
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from funcy import once, walk_values
|
||||
@@ -10,6 +11,7 @@ from voluptuous import (
|
||||
Exclusive,
|
||||
Invalid,
|
||||
Lower,
|
||||
Marker,
|
||||
Optional,
|
||||
Range,
|
||||
Schema,
|
||||
@@ -17,6 +19,9 @@ from voluptuous import (
|
||||
|
||||
from dvc.log import logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
Bool = All(
|
||||
@@ -112,17 +117,19 @@ class FeatureSchema(Schema):
|
||||
return ret
|
||||
|
||||
|
||||
DEPRECATED = "==DEPRECATED=="
|
||||
|
||||
REMOTE_COMMON = {
|
||||
"url": str,
|
||||
"checksum_jobs": All(Coerce(int), Range(1)),
|
||||
"jobs": All(Coerce(int), Range(1)),
|
||||
Optional("worktree"): Bool,
|
||||
Optional("no_traverse"): Bool, # obsoleted
|
||||
Optional("no_traverse", description=DEPRECATED): Bool, # obsoleted
|
||||
Optional("version_aware"): Bool,
|
||||
}
|
||||
LOCAL_COMMON = {
|
||||
"type": supported_cache_type,
|
||||
Optional("protected", default=False): Bool, # obsoleted
|
||||
Optional("protected", default=False, description=DEPRECATED): Bool, # obsoleted
|
||||
"shared": All(Lower, Choices("group")),
|
||||
Optional("slow_link_warning", default=True): Bool,
|
||||
Optional("verify", default=False): Bool,
|
||||
@@ -144,6 +151,7 @@ WEBDAV_COMMON = {
|
||||
"password": str,
|
||||
"ask_password": Bool,
|
||||
"token": str,
|
||||
"bearer_token_command": str,
|
||||
"custom_auth_header": str,
|
||||
"cert_path": str,
|
||||
"key_path": str,
|
||||
@@ -152,6 +160,127 @@ WEBDAV_COMMON = {
|
||||
Optional("verify", default=False): Bool,
|
||||
}
|
||||
|
||||
REMOTE_SCHEMAS = {
|
||||
"": LOCAL_COMMON | REMOTE_COMMON,
|
||||
"s3": {
|
||||
"region": str,
|
||||
"profile": str,
|
||||
"credentialpath": str,
|
||||
"configpath": str,
|
||||
"endpointurl": str,
|
||||
"access_key_id": str,
|
||||
"secret_access_key": str,
|
||||
"session_token": str,
|
||||
Optional(
|
||||
"listobjects", default=False, description=DEPRECATED
|
||||
): Bool, # obsoleted
|
||||
Optional("use_ssl", default=True): Bool,
|
||||
Optional("allow_anonymous_login", default=False): Bool,
|
||||
"ssl_verify": Any(Bool, str),
|
||||
"sse": str,
|
||||
"sse_kms_key_id": str,
|
||||
"sse_customer_algorithm": str,
|
||||
"sse_customer_key": str,
|
||||
"acl": str,
|
||||
"grant_read": str,
|
||||
"grant_read_acp": str,
|
||||
"grant_write_acp": str,
|
||||
"grant_full_control": str,
|
||||
"cache_regions": bool,
|
||||
"read_timeout": Coerce(int),
|
||||
"connect_timeout": Coerce(int),
|
||||
Optional("verify", default=False): Bool,
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
"gs": {
|
||||
"projectname": str,
|
||||
"credentialpath": str,
|
||||
"endpointurl": str,
|
||||
Optional("verify", default=False): Bool,
|
||||
Optional("allow_anonymous_login", default=False): Bool,
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
"ssh": {
|
||||
"type": supported_cache_type,
|
||||
"port": Coerce(int),
|
||||
"user": str,
|
||||
"password": str,
|
||||
"ask_password": Bool,
|
||||
"passphrase": str,
|
||||
"ask_passphrase": Bool,
|
||||
"keyfile": str,
|
||||
"timeout": Coerce(int),
|
||||
"gss_auth": Bool,
|
||||
"allow_agent": Bool,
|
||||
"max_sessions": Coerce(int),
|
||||
Optional("verify", default=False): Bool,
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
"hdfs": {
|
||||
"user": str,
|
||||
"kerb_ticket": str,
|
||||
"replication": int,
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
"webhdfs": {
|
||||
"kerberos": Bool,
|
||||
"kerberos_principal": str,
|
||||
"proxy_to": str,
|
||||
"ssl_verify": Any(Bool, str),
|
||||
"token": str,
|
||||
"use_https": Bool,
|
||||
"user": str,
|
||||
"password": str,
|
||||
"data_proxy_target": str,
|
||||
Optional("verify", default=False): Bool,
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
"azure": {
|
||||
"connection_string": str,
|
||||
"sas_token": str,
|
||||
"account_name": str,
|
||||
"account_key": str,
|
||||
"tenant_id": str,
|
||||
"client_id": str,
|
||||
"client_secret": str,
|
||||
"allow_anonymous_login": Bool,
|
||||
"exclude_environment_credential": Bool,
|
||||
"exclude_visual_studio_code_credential": Bool,
|
||||
"exclude_shared_token_cache_credential": Bool,
|
||||
"exclude_managed_identity_credential": Bool,
|
||||
Optional("verify", default=False): Bool,
|
||||
"timeout": Coerce(int),
|
||||
"read_timeout": Coerce(int),
|
||||
"connection_timeout": Coerce(int),
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
"oss": {
|
||||
"oss_key_id": str,
|
||||
"oss_key_secret": str,
|
||||
"oss_endpoint": str,
|
||||
Optional("verify", default=True): Bool,
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
"gdrive": {
|
||||
"profile": str,
|
||||
"gdrive_use_service_account": Bool,
|
||||
"gdrive_client_id": str,
|
||||
"gdrive_client_secret": str,
|
||||
"gdrive_user_credentials_file": str,
|
||||
"gdrive_service_account_user_email": str,
|
||||
"gdrive_service_account_json_file_path": str,
|
||||
Optional("gdrive_trash_only", default=False): Bool,
|
||||
Optional("gdrive_acknowledge_abuse", default=False): Bool,
|
||||
Optional("verify", default=True): Bool,
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
"http": HTTP_COMMON | REMOTE_COMMON,
|
||||
"https": HTTP_COMMON | REMOTE_COMMON,
|
||||
"webdav": WEBDAV_COMMON | REMOTE_COMMON,
|
||||
"webdavs": WEBDAV_COMMON | REMOTE_COMMON,
|
||||
"remote": {str: object}, # Any of the above options are valid
|
||||
}
|
||||
|
||||
SCHEMA = {
|
||||
"core": {
|
||||
"remote": Lower,
|
||||
@@ -161,152 +290,37 @@ SCHEMA = {
|
||||
Optional("hardlink_lock", default=False): Bool,
|
||||
Optional("no_scm", default=False): Bool,
|
||||
Optional("autostage", default=False): Bool,
|
||||
Optional("experiments"): Bool, # obsoleted
|
||||
Optional("experiments", description=DEPRECATED): Bool, # obsoleted
|
||||
Optional("check_update", default=True): Bool,
|
||||
"site_cache_dir": str,
|
||||
"machine": Lower,
|
||||
},
|
||||
"cache": {
|
||||
"local": str, # obsoleted
|
||||
"s3": str, # obsoleted
|
||||
"gs": str, # obsoleted
|
||||
"hdfs": str, # obsoleted
|
||||
"webhdfs": str, # obsoleted
|
||||
"ssh": str, # obsoleted
|
||||
"azure": str, # obsoleted
|
||||
Marker("local", description=DEPRECATED): str, # obsoleted
|
||||
Marker("s3", description=DEPRECATED): str, # obsoleted
|
||||
Marker("gs", description=DEPRECATED): str, # obsoleted
|
||||
Marker("hdfs", description=DEPRECATED): str, # obsoleted
|
||||
Marker("webhdfs", description=DEPRECATED): str, # obsoleted
|
||||
Marker("ssh", description=DEPRECATED): str, # obsoleted
|
||||
Marker("azure", description=DEPRECATED): str, # obsoleted
|
||||
# This is for default local cache
|
||||
"dir": str,
|
||||
**LOCAL_COMMON,
|
||||
},
|
||||
"remote": {
|
||||
str: ByUrl(
|
||||
{
|
||||
"": LOCAL_COMMON | REMOTE_COMMON,
|
||||
"s3": {
|
||||
"region": str,
|
||||
"profile": str,
|
||||
"credentialpath": str,
|
||||
"configpath": str,
|
||||
"endpointurl": str,
|
||||
"access_key_id": str,
|
||||
"secret_access_key": str,
|
||||
"session_token": str,
|
||||
Optional("listobjects", default=False): Bool, # obsoleted
|
||||
Optional("use_ssl", default=True): Bool,
|
||||
Optional("allow_anonymous_login", default=False): Bool,
|
||||
"ssl_verify": Any(Bool, str),
|
||||
"sse": str,
|
||||
"sse_kms_key_id": str,
|
||||
"sse_customer_algorithm": str,
|
||||
"sse_customer_key": str,
|
||||
"acl": str,
|
||||
"grant_read": str,
|
||||
"grant_read_acp": str,
|
||||
"grant_write_acp": str,
|
||||
"grant_full_control": str,
|
||||
"cache_regions": bool,
|
||||
"read_timeout": Coerce(int),
|
||||
"connect_timeout": Coerce(int),
|
||||
Optional("verify", default=False): Bool,
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
"gs": {
|
||||
"projectname": str,
|
||||
"credentialpath": str,
|
||||
"endpointurl": str,
|
||||
Optional("verify", default=False): Bool,
|
||||
Optional("allow_anonymous_login", default=False): Bool,
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
"ssh": {
|
||||
"type": supported_cache_type,
|
||||
"port": Coerce(int),
|
||||
"user": str,
|
||||
"password": str,
|
||||
"ask_password": Bool,
|
||||
"passphrase": str,
|
||||
"ask_passphrase": Bool,
|
||||
"keyfile": str,
|
||||
"timeout": Coerce(int),
|
||||
"gss_auth": Bool,
|
||||
"allow_agent": Bool,
|
||||
"max_sessions": Coerce(int),
|
||||
Optional("verify", default=False): Bool,
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
"hdfs": {
|
||||
"user": str,
|
||||
"kerb_ticket": str,
|
||||
"replication": int,
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
"webhdfs": {
|
||||
"kerberos": Bool,
|
||||
"kerberos_principal": str,
|
||||
"proxy_to": str,
|
||||
"ssl_verify": Any(Bool, str),
|
||||
"token": str,
|
||||
"use_https": Bool,
|
||||
"user": str,
|
||||
"password": str,
|
||||
"data_proxy_target": str,
|
||||
Optional("verify", default=False): Bool,
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
"azure": {
|
||||
"connection_string": str,
|
||||
"sas_token": str,
|
||||
"account_name": str,
|
||||
"account_key": str,
|
||||
"tenant_id": str,
|
||||
"client_id": str,
|
||||
"client_secret": str,
|
||||
"allow_anonymous_login": Bool,
|
||||
"exclude_environment_credential": Bool,
|
||||
"exclude_visual_studio_code_credential": Bool,
|
||||
"exclude_shared_token_cache_credential": Bool,
|
||||
"exclude_managed_identity_credential": Bool,
|
||||
Optional("verify", default=False): Bool,
|
||||
"timeout": Coerce(int),
|
||||
"read_timeout": Coerce(int),
|
||||
"connection_timeout": Coerce(int),
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
"oss": {
|
||||
"oss_key_id": str,
|
||||
"oss_key_secret": str,
|
||||
"oss_endpoint": str,
|
||||
Optional("verify", default=True): Bool,
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
"gdrive": {
|
||||
"profile": str,
|
||||
"gdrive_use_service_account": Bool,
|
||||
"gdrive_client_id": str,
|
||||
"gdrive_client_secret": str,
|
||||
"gdrive_user_credentials_file": str,
|
||||
"gdrive_service_account_user_email": str,
|
||||
"gdrive_service_account_json_file_path": str,
|
||||
Optional("gdrive_trash_only", default=False): Bool,
|
||||
Optional("gdrive_acknowledge_abuse", default=False): Bool,
|
||||
Optional("verify", default=True): Bool,
|
||||
**REMOTE_COMMON,
|
||||
},
|
||||
"http": HTTP_COMMON | REMOTE_COMMON,
|
||||
"https": HTTP_COMMON | REMOTE_COMMON,
|
||||
"webdav": WEBDAV_COMMON | REMOTE_COMMON,
|
||||
"webdavs": WEBDAV_COMMON | REMOTE_COMMON,
|
||||
"remote": {str: object}, # Any of the above options are valid
|
||||
}
|
||||
)
|
||||
str: ByUrl(REMOTE_SCHEMAS),
|
||||
},
|
||||
"state": {
|
||||
"dir": str, # obsoleted
|
||||
"row_limit": All(Coerce(int), Range(1)), # obsoleted
|
||||
"row_cleanup_quota": All(Coerce(int), Range(0, 100)), # obsoleted
|
||||
Marker("dir", description=DEPRECATED): str, # obsoleted
|
||||
Marker("row_limit", description=DEPRECATED): All(
|
||||
Coerce(int), Range(1)
|
||||
), # obsoleted
|
||||
Marker("row_cleanup_quota", description=DEPRECATED): All(
|
||||
Coerce(int), Range(0, 100)
|
||||
), # obsoleted
|
||||
},
|
||||
"index": {
|
||||
"dir": str, # obsoleted
|
||||
Marker("dir", description=DEPRECATED): str, # obsoleted
|
||||
},
|
||||
"machine": {
|
||||
str: {
|
||||
@@ -336,13 +350,13 @@ SCHEMA = {
|
||||
"out_dir": str,
|
||||
},
|
||||
"exp": {
|
||||
"code": str,
|
||||
"data": str,
|
||||
"models": str,
|
||||
"metrics": str,
|
||||
"params": str,
|
||||
"plots": str,
|
||||
"live": str,
|
||||
Marker("code", description=DEPRECATED): str,
|
||||
Marker("data", description=DEPRECATED): str,
|
||||
Marker("models", description=DEPRECATED): str,
|
||||
Marker("metrics", description=DEPRECATED): str,
|
||||
Marker("params", description=DEPRECATED): str,
|
||||
Marker("plots", description=DEPRECATED): str,
|
||||
Marker("live", description=DEPRECATED): str,
|
||||
"auto_push": Bool,
|
||||
"git_remote": str,
|
||||
},
|
||||
@@ -371,3 +385,21 @@ SCHEMA = {
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def config_vars_for_completion(d: dict = SCHEMA, path: str = "") -> "Iterator[str]":
|
||||
for k, v in d.items():
|
||||
if k in ("machine", "feature"):
|
||||
continue
|
||||
if isinstance(k, Marker):
|
||||
if k.description == DEPRECATED:
|
||||
continue
|
||||
k = k.schema
|
||||
if not isinstance(k, str):
|
||||
continue
|
||||
|
||||
keypath = path + k
|
||||
if isinstance(v, dict):
|
||||
yield from config_vars_for_completion(v, keypath + ".")
|
||||
else:
|
||||
yield keypath
|
||||
|
||||
+1
-1
@@ -84,7 +84,7 @@ class Client:
|
||||
except Exception as exc:
|
||||
if callable(onerror):
|
||||
onerror()
|
||||
logger.exception(
|
||||
logger.exception( # noqa: LOG007
|
||||
"Could not connect to the database. "
|
||||
"Check your database credentials and try again.",
|
||||
exc_info=False,
|
||||
|
||||
@@ -41,7 +41,7 @@ def _get(stage, p, info, **kwargs):
|
||||
return Dependency(stage, p, info, **kwargs)
|
||||
|
||||
|
||||
def loadd_from(stage, d_list):
|
||||
def loadd_from(stage, d_list) -> list[Dependency]:
|
||||
ret = []
|
||||
for d in d_list:
|
||||
p = d.pop(Output.PARAM_PATH, None)
|
||||
|
||||
@@ -110,6 +110,7 @@ class ParamsDependency(Dependency):
|
||||
except MissingParamsFile:
|
||||
return {}
|
||||
|
||||
assert self.repo
|
||||
try:
|
||||
return read_param_file(
|
||||
self.repo.fs,
|
||||
|
||||
@@ -150,6 +150,7 @@ class RepoDependency(Dependency):
|
||||
# local cache instead of the remote's cache. This avoids re-streaming of data,
|
||||
# but messes up the call to `_get_remote_config()` downstream, which will need
|
||||
# to ignore cache parameters.
|
||||
assert self.repo
|
||||
config["cache"] = self.repo.config["cache"]
|
||||
config["cache"]["dir"] = self.repo.cache.local_cache_dir
|
||||
|
||||
|
||||
+1
-1
@@ -203,7 +203,7 @@ class SingleStageFile(FileMixin):
|
||||
|
||||
def dump_stages(self, stages, **kwargs) -> None:
|
||||
if not stages:
|
||||
return
|
||||
return None
|
||||
|
||||
assert len(stages) == 1, "SingleStageFile can only dump one stage."
|
||||
return self.dump(stages[0], **kwargs)
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@ if TYPE_CHECKING:
|
||||
from dvc.stage import Stage
|
||||
|
||||
|
||||
class DvcException(Exception):
|
||||
class DvcException(Exception): # noqa: N818
|
||||
"""Base class for all dvc exceptions."""
|
||||
|
||||
def __init__(self, msg, *args):
|
||||
|
||||
+2
-2
@@ -508,7 +508,7 @@ class DvcIgnoreFilter:
|
||||
|
||||
def check_ignore(self, target: str) -> CheckIgnoreResult:
|
||||
# NOTE: can only be used in `dvc check-ignore`, see
|
||||
# https://github.com/iterative/dvc/issues/5046
|
||||
# https://github.com/treeverse/dvc/issues/5046
|
||||
full_target = self.fs.abspath(target)
|
||||
matched_patterns: list[PatternInfo] = []
|
||||
ignore = False
|
||||
@@ -525,7 +525,7 @@ class DvcIgnoreFilter:
|
||||
self, fs: "FileSystem", path: str, ignore_subrepos: bool = True
|
||||
) -> bool:
|
||||
# NOTE: can't use self.check_ignore(path).match for now, see
|
||||
# https://github.com/iterative/dvc/issues/4555
|
||||
# https://github.com/treeverse/dvc/issues/4555
|
||||
if fs.protocol != Schemes.LOCAL:
|
||||
return False
|
||||
if fs.isfile(path):
|
||||
|
||||
+1
-1
@@ -158,7 +158,7 @@ def _get_fs_type(path):
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
# need to follow the symlink: https://github.com/iterative/dvc/issues/5065
|
||||
# need to follow the symlink: https://github.com/treeverse/dvc/issues/5065
|
||||
path = pathlib.Path(path).resolve()
|
||||
|
||||
for parent in itertools.chain([path], path.parents):
|
||||
|
||||
+1
-1
@@ -182,7 +182,7 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
|
||||
# number to generate the resulting lock file name, which is unique
|
||||
# enough for our application.
|
||||
#
|
||||
# [1] https://github.com/iterative/dvc/issues/2582
|
||||
# [1] https://github.com/treeverse/dvc/issues/2582
|
||||
self._hostname = socket.gethostname()
|
||||
|
||||
self._lifetime = timedelta(days=365) # Lock for good by default
|
||||
|
||||
+1
-3
@@ -1,8 +1,6 @@
|
||||
"""Manages logging configuration for DVC repo."""
|
||||
|
||||
import logging
|
||||
import logging.config
|
||||
import logging.handlers
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Iterator
|
||||
@@ -55,7 +53,7 @@ def add_logging_level(level_name, level_num, method_name=None):
|
||||
setattr(logging, method_name, log_to_root)
|
||||
|
||||
|
||||
class LoggingException(Exception):
|
||||
class LoggingException(Exception): # noqa: N818
|
||||
def __init__(self, record):
|
||||
msg = f"failed to log {record!s}"
|
||||
super().__init__(msg)
|
||||
|
||||
+29
-11
@@ -42,6 +42,7 @@ from .utils import relpath
|
||||
from .utils.fs import path_isin
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.repo import Repo
|
||||
from dvc_data.hashfile.obj import HashFile
|
||||
from dvc_data.index import DataIndexKey
|
||||
|
||||
@@ -83,7 +84,7 @@ def _get(stage, path, **kwargs):
|
||||
return Output(stage, path, **kwargs)
|
||||
|
||||
|
||||
def loadd_from(stage, d_list):
|
||||
def loadd_from(stage, d_list) -> list["Output"]:
|
||||
ret = []
|
||||
for d in d_list:
|
||||
p = d.pop(Output.PARAM_PATH)
|
||||
@@ -329,7 +330,7 @@ class Output:
|
||||
self.annot = Annotation(
|
||||
desc=desc, type=type, labels=labels or [], meta=meta or {}
|
||||
)
|
||||
self.repo = stage.repo if not repo and stage else repo
|
||||
self.repo: Optional[Repo] = stage.repo if not repo and stage else repo
|
||||
meta_d = merge_file_meta_from_cloud(info or {})
|
||||
meta = Meta.from_dict(meta_d)
|
||||
# NOTE: when version_aware is not passed into get_cloud_fs, it will be
|
||||
@@ -511,6 +512,7 @@ class Output:
|
||||
from dvc.cachemgr import LEGACY_HASH_NAMES
|
||||
|
||||
assert self.is_in_repo
|
||||
assert self.repo
|
||||
odb_name = "legacy" if self.hash_name in LEGACY_HASH_NAMES else "repo"
|
||||
return getattr(self.repo.cache, odb_name)
|
||||
|
||||
@@ -518,6 +520,7 @@ class Output:
|
||||
def local_cache(self):
|
||||
from dvc.cachemgr import LEGACY_HASH_NAMES
|
||||
|
||||
assert self.repo
|
||||
if self.hash_name in LEGACY_HASH_NAMES:
|
||||
return self.repo.cache.legacy
|
||||
return self.repo.cache.local
|
||||
@@ -585,6 +588,7 @@ class Output:
|
||||
def index_key(self) -> tuple[str, "DataIndexKey"]:
|
||||
if self.is_in_repo:
|
||||
workspace = "repo"
|
||||
assert self.repo
|
||||
key = self.repo.fs.relparts(self.fs_path, self.repo.root_dir)
|
||||
else:
|
||||
workspace = self.fs.protocol
|
||||
@@ -640,6 +644,7 @@ class Output:
|
||||
@property
|
||||
def dvcignore(self) -> Optional["DvcIgnoreFilter"]:
|
||||
if self.fs.protocol == "local":
|
||||
assert self.repo
|
||||
return self.repo.dvcignore
|
||||
return None
|
||||
|
||||
@@ -661,6 +666,7 @@ class Output:
|
||||
if not self.use_scm_ignore:
|
||||
return
|
||||
|
||||
assert self.repo
|
||||
if self.repo.scm.is_tracked(self.fspath):
|
||||
raise OutputAlreadyTrackedError(self)
|
||||
|
||||
@@ -670,6 +676,7 @@ class Output:
|
||||
if not self.use_scm_ignore:
|
||||
return
|
||||
|
||||
assert self.repo
|
||||
self.repo.scm_context.ignore_remove(self.fspath)
|
||||
|
||||
def save(self) -> None:
|
||||
@@ -776,6 +783,7 @@ class Output:
|
||||
callback=cb,
|
||||
)
|
||||
if relink:
|
||||
assert self.repo
|
||||
rel = self.fs.relpath(filter_info or self.fs_path)
|
||||
with CheckoutCallback(desc=f"Checking out {rel}", unit="files") as cb:
|
||||
self._checkout(
|
||||
@@ -957,6 +965,7 @@ class Output:
|
||||
added = not self.exists
|
||||
|
||||
try:
|
||||
assert self.repo
|
||||
modified = self._checkout(
|
||||
filter_info or self.fs_path,
|
||||
self.fs,
|
||||
@@ -987,18 +996,24 @@ class Output:
|
||||
if ignore_remove:
|
||||
self.ignore_remove()
|
||||
|
||||
def move(self, out):
|
||||
def move(self, out: "Output") -> None:
|
||||
src_exists = self.exists
|
||||
if src_exists:
|
||||
self.fs.move(self.fs_path, out.fs_path)
|
||||
else:
|
||||
logger.warning("%r missing", self.fspath)
|
||||
|
||||
if self.protocol == "local" and self.use_scm_ignore:
|
||||
assert self.repo
|
||||
self.repo.scm_context.ignore_remove(self.fspath)
|
||||
|
||||
self.fs.move(self.fs_path, out.fs_path)
|
||||
self.def_path = out.def_path
|
||||
self.fs_path = out.fs_path
|
||||
self.save()
|
||||
self.commit()
|
||||
|
||||
if self.protocol == "local" and self.use_scm_ignore:
|
||||
self.repo.scm_context.ignore(self.fspath)
|
||||
try:
|
||||
self.save()
|
||||
self.commit()
|
||||
except self.DoesNotExistError:
|
||||
self.ignore()
|
||||
|
||||
def transfer(
|
||||
self, source, odb=None, jobs=None, update=False, no_progress_bar=False
|
||||
@@ -1079,6 +1094,7 @@ class Output:
|
||||
if self.remote:
|
||||
kwargs["remote"] = self.remote
|
||||
with suppress(Exception):
|
||||
assert self.repo
|
||||
self.repo.cloud.pull([obj.hash_info], **kwargs)
|
||||
|
||||
if self.obj:
|
||||
@@ -1172,6 +1188,7 @@ class Output:
|
||||
return {}
|
||||
|
||||
if self.remote:
|
||||
assert self.repo
|
||||
remote_odb = self.repo.cloud.get_remote_odb(
|
||||
name=self.remote, hash_name=self.hash_name
|
||||
)
|
||||
@@ -1473,12 +1490,13 @@ class Output:
|
||||
):
|
||||
"""Merge version meta for files which are unchanged from other."""
|
||||
if not self.hash_info:
|
||||
return
|
||||
return None
|
||||
if self.hash_info.isdir:
|
||||
return self._merge_dir_version_meta(old_hi, old_obj)
|
||||
if self.hash_info != old_hi:
|
||||
return
|
||||
return None
|
||||
self.meta = old_meta
|
||||
return None
|
||||
|
||||
def _merge_dir_version_meta(
|
||||
self, old_hi: "HashInfo", old_obj: Optional[Union["HashFile", "Tree"]]
|
||||
|
||||
@@ -66,7 +66,7 @@ class ParamsLoadError(ContextError):
|
||||
pass
|
||||
|
||||
|
||||
class KeyNotInContext(ContextError, KeyError):
|
||||
class KeyNotInContext(ContextError, KeyError): # noqa: N818
|
||||
def __init__(self, key: str) -> None:
|
||||
self.key: str = key
|
||||
super().__init__(f"Could not find '{key}'")
|
||||
@@ -75,7 +75,7 @@ class KeyNotInContext(ContextError, KeyError):
|
||||
return self.msg
|
||||
|
||||
|
||||
class VarsAlreadyLoaded(ContextError):
|
||||
class VarsAlreadyLoaded(ContextError): # noqa: N818
|
||||
pass
|
||||
|
||||
|
||||
@@ -272,7 +272,7 @@ class CtxDict(Container, MutableMapping):
|
||||
if not isinstance(key, str):
|
||||
# limitation for the interpolation
|
||||
# ignore other kinds of keys
|
||||
return
|
||||
return None
|
||||
return super().__setitem__(key, value)
|
||||
|
||||
def merge_update(self, other, overwrite=False):
|
||||
|
||||
@@ -202,6 +202,7 @@ def validate_value(value, key):
|
||||
if isinstance(value, dict) and key == "cmd":
|
||||
return True
|
||||
raise ParseError(f"Cannot interpolate data of type '{type(value).__name__}'")
|
||||
return None
|
||||
|
||||
|
||||
def str_interpolate(
|
||||
|
||||
@@ -167,7 +167,7 @@ class VegaConverter(Converter):
|
||||
if isinstance(y, list):
|
||||
return "y"
|
||||
if not isinstance(y, dict):
|
||||
return
|
||||
return None
|
||||
|
||||
fields = {field for _, field in _file_field(y)}
|
||||
if len(fields) == 1:
|
||||
|
||||
@@ -2,7 +2,6 @@ import os
|
||||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING, NamedTuple, Optional
|
||||
|
||||
import dpath
|
||||
import dpath.options
|
||||
from funcy import get_in, last
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ if TYPE_CHECKING:
|
||||
from dvc.fs.data import DataFileSystem
|
||||
from dvc.fs.dvc import DVCFileSystem
|
||||
from dvc.lock import LockBase
|
||||
from dvc.output import Output
|
||||
from dvc.scm import Git, NoSCM
|
||||
from dvc.stage import Stage
|
||||
from dvc.types import DictStrAny
|
||||
@@ -544,7 +545,9 @@ class Repo:
|
||||
|
||||
return used
|
||||
|
||||
def find_outs_by_path(self, path, outs=None, recursive=False, strict=True):
|
||||
def find_outs_by_path(
|
||||
self, path, outs=None, recursive=False, strict=True
|
||||
) -> list["Output"]:
|
||||
# using `outs_graph` to ensure graph checks are run
|
||||
outs = outs or self.index.outs_graph
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ def check_name_format(name: str) -> None:
|
||||
|
||||
def name_is_compatible(name: str) -> bool:
|
||||
"""
|
||||
Only needed by DVCLive per iterative/dvclive#715
|
||||
Only needed by DVCLive per treeverse/dvclive#715
|
||||
Will be removed in future release.
|
||||
"""
|
||||
from gto.constants import assert_name_is_valid
|
||||
|
||||
+22
-2
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
import posixpath
|
||||
from collections import defaultdict
|
||||
from collections import defaultdict, deque
|
||||
from collections.abc import Iterable, Iterator, Mapping
|
||||
from typing import TYPE_CHECKING, Optional, TypedDict, Union
|
||||
|
||||
@@ -385,6 +385,26 @@ def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> list[str]
|
||||
return [repo.fs.relpath(file, start) for file in files]
|
||||
|
||||
|
||||
def iter_index(
|
||||
index: Union["BaseDataIndex", "DataIndexView"], shallow: bool = False
|
||||
) -> Iterator[tuple["DataIndexKey", Optional["DataIndexEntry"]]]:
|
||||
if not shallow:
|
||||
yield from index.iteritems(shallow=shallow)
|
||||
return
|
||||
|
||||
# only iterate until we find entries with hash_info in shallow mode
|
||||
todo: deque[tuple[DataIndexKey, dict]] = deque([((), index.info(()))])
|
||||
while todo:
|
||||
key, info = todo.popleft()
|
||||
entry = info.get("entry")
|
||||
if info.get("type") == "directory" and not (entry and entry.hash_info):
|
||||
try:
|
||||
todo.extend(index.ls(key, detail=True))
|
||||
except (KeyError, DataIndexDirError):
|
||||
pass
|
||||
yield key, entry
|
||||
|
||||
|
||||
def _get_entries_not_in_remote(
|
||||
repo: "Repo",
|
||||
filter_keys: Optional[Iterable["DataIndexKey"]] = None,
|
||||
@@ -418,7 +438,7 @@ def _get_entries_not_in_remote(
|
||||
|
||||
n = 0
|
||||
with TqdmCallback(size=n, desc="Checking remote", unit="entry") as cb:
|
||||
for key, entry in view.iteritems(shallow=not granular):
|
||||
for key, entry in iter_index(view, shallow=not granular):
|
||||
if not (entry and entry.hash_info):
|
||||
continue
|
||||
|
||||
|
||||
@@ -50,7 +50,7 @@ class MultipleBranchError(DvcException):
|
||||
self.ref_infos = ref_infos
|
||||
|
||||
|
||||
class AmbiguousExpRefInfo(InvalidArgumentError):
|
||||
class AmbiguousExpRefInfo(InvalidArgumentError): # noqa: N818
|
||||
def __init__(self, exp_name: str, exp_ref_list: Iterable["ExpRefInfo"]):
|
||||
msg = [
|
||||
(
|
||||
|
||||
@@ -9,6 +9,7 @@ from enum import IntEnum
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, Union
|
||||
|
||||
import funcy
|
||||
from funcy import nullcontext
|
||||
from scmrepo.exceptions import SCMError
|
||||
|
||||
@@ -815,11 +816,22 @@ class BaseExecutor(ABC):
|
||||
|
||||
@contextmanager
|
||||
def set_temp_refs(self, scm: "Git", temp_dict: dict[str, str]):
|
||||
# Retry ref set, get, and remove operations to handle transient issues during
|
||||
# concurrent Git access.
|
||||
# Dulwich deletes parent directories of refs if they happen to be empty after
|
||||
# removing a ref, which can interfere with `set_ref` in other processes.
|
||||
# `remove_ref` may also fail with a `FileLocked` error when refs are packed,
|
||||
# since multiple processes might attempt to write to the same file.
|
||||
retry = funcy.retry(10, errors=Exception, timeout=0.1)
|
||||
set_ref = retry(scm.set_ref)
|
||||
get_ref = retry(scm.get_ref)
|
||||
remove_ref = retry(scm.remove_ref)
|
||||
|
||||
try:
|
||||
for ref, rev in temp_dict.items():
|
||||
scm.set_ref(ref, rev)
|
||||
set_ref(ref, rev)
|
||||
yield
|
||||
finally:
|
||||
for ref in temp_dict:
|
||||
if scm.get_ref(ref):
|
||||
scm.remove_ref(ref)
|
||||
if get_ref(ref):
|
||||
remove_ref(ref)
|
||||
|
||||
@@ -287,6 +287,7 @@ class BaseStashQueue(ABC):
|
||||
baseline_rev: Optional[str] = None,
|
||||
branch: Optional[str] = None,
|
||||
name: Optional[str] = None,
|
||||
no_hydra: bool = False,
|
||||
**kwargs,
|
||||
) -> QueueEntry:
|
||||
"""Stash changes from the workspace as an experiment.
|
||||
@@ -302,6 +303,7 @@ class BaseStashQueue(ABC):
|
||||
name: Optional experiment name. If specified this will be used as
|
||||
the human-readable name in the experiment branch ref. Has no
|
||||
effect of branch is specified.
|
||||
no_hydra: Disable Hydra from automatically overwriting all params.
|
||||
|
||||
.. _Hydra Override:
|
||||
https://hydra.cc/docs/next/advanced/override_grammar/basic/
|
||||
@@ -318,7 +320,7 @@ class BaseStashQueue(ABC):
|
||||
|
||||
# update experiment params from command line
|
||||
if params:
|
||||
self._update_params(params)
|
||||
self._update_params(params, no_hydra=no_hydra)
|
||||
|
||||
# DVC commit data deps to preserve state across workspace
|
||||
# & tempdir runs
|
||||
@@ -441,12 +443,13 @@ class BaseStashQueue(ABC):
|
||||
f"from '{config_path}': {param_list}"
|
||||
)
|
||||
|
||||
def _update_params(self, params: dict[str, list[str]]):
|
||||
def _update_params(self, params: dict[str, list[str]], no_hydra: bool = False):
|
||||
"""Update param files with the provided `Hydra Override`_ patterns.
|
||||
|
||||
Args:
|
||||
params: Dict mapping paths to `Hydra Override`_ patterns,
|
||||
provided via `exp run --set-param`.
|
||||
no_hydra: Disable Hydra from automatically overwriting all params.
|
||||
|
||||
.. _Hydra Override:
|
||||
https://hydra.cc/docs/advanced/override_grammar/basic/
|
||||
@@ -456,7 +459,7 @@ class BaseStashQueue(ABC):
|
||||
logger.debug("Using experiment params '%s'", params)
|
||||
|
||||
hydra_config = self.repo.config.get("hydra", {})
|
||||
hydra_enabled = hydra_config.get("enabled", False)
|
||||
hydra_enabled = hydra_config.get("enabled", False) and not no_hydra
|
||||
hydra_output_file = ParamsDependency.DEFAULT_PARAMS_FILE
|
||||
for path, overrides in params.items():
|
||||
if hydra_enabled and path == hydra_output_file:
|
||||
|
||||
@@ -70,7 +70,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
def celery(self) -> "FSApp":
|
||||
from kombu.transport.filesystem import Channel
|
||||
|
||||
# related to https://github.com/iterative/dvc-task/issues/61
|
||||
# related to https://github.com/treeverse/dvc-task/issues/61
|
||||
Channel.QoS.restore_at_shutdown = False
|
||||
|
||||
from dvc_task.app import FSApp
|
||||
|
||||
@@ -22,6 +22,7 @@ def run( # noqa: C901, PLR0912
|
||||
queue: bool = False,
|
||||
copy_paths: Optional[Iterable[str]] = None,
|
||||
message: Optional[str] = None,
|
||||
no_hydra: bool = False,
|
||||
**kwargs,
|
||||
) -> dict[str, str]:
|
||||
"""Reproduce the specified targets as an experiment.
|
||||
@@ -67,7 +68,7 @@ def run( # noqa: C901, PLR0912
|
||||
else:
|
||||
path_overrides = {}
|
||||
|
||||
hydra_enabled = repo.config.get("hydra", {}).get("enabled", False)
|
||||
hydra_enabled = repo.config.get("hydra", {}).get("enabled", False) and not no_hydra
|
||||
hydra_output_file = ParamsDependency.DEFAULT_PARAMS_FILE
|
||||
if hydra_enabled and hydra_output_file not in path_overrides:
|
||||
# Force `_update_params` even if `--set-param` was not used
|
||||
@@ -80,6 +81,7 @@ def run( # noqa: C901, PLR0912
|
||||
tmp_dir=tmp_dir,
|
||||
copy_paths=copy_paths,
|
||||
message=message,
|
||||
no_hydra=no_hydra,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@@ -100,6 +102,7 @@ def run( # noqa: C901, PLR0912
|
||||
params=sweep_overrides,
|
||||
copy_paths=copy_paths,
|
||||
message=message,
|
||||
no_hydra=no_hydra,
|
||||
**kwargs,
|
||||
)
|
||||
if sweep_overrides:
|
||||
|
||||
@@ -20,6 +20,7 @@ def _make_index_onerror(onerror, rev):
|
||||
def _onerror(entry, exc):
|
||||
if onerror:
|
||||
return onerror(rev, entry, exc)
|
||||
return None
|
||||
|
||||
return _onerror
|
||||
|
||||
|
||||
+2
-2
@@ -16,7 +16,7 @@ def pre_commit_install(scm: "Git") -> None:
|
||||
config_path = os.path.join(scm.root_dir, ".pre-commit-config.yaml")
|
||||
with modify_yaml(config_path) as config:
|
||||
entry = {
|
||||
"repo": "https://github.com/iterative/dvc",
|
||||
"repo": "https://github.com/treeverse/dvc",
|
||||
"rev": ".".join(map(str, version_tuple[:3])),
|
||||
"hooks": [
|
||||
{
|
||||
@@ -75,7 +75,7 @@ def install(self: "Repo", use_pre_commit_tool: bool = False) -> None:
|
||||
|
||||
scm = self.scm
|
||||
if not isinstance(scm, Git):
|
||||
return
|
||||
return None
|
||||
|
||||
driver = "dvc git-hook merge-driver --ancestor %O --our %A --their %B "
|
||||
scm.install_merge_driver("dvc", "DVC merge driver", driver)
|
||||
|
||||
+41
-17
@@ -1,10 +1,13 @@
|
||||
import os
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from dvc.exceptions import MoveNotDataSourceError
|
||||
from dvc.repo.scm_context import scm_context
|
||||
|
||||
from . import locked
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from . import Repo
|
||||
|
||||
|
||||
def _expand_target_path(from_path, to_path):
|
||||
if os.path.isdir(to_path):
|
||||
@@ -14,7 +17,7 @@ def _expand_target_path(from_path, to_path):
|
||||
|
||||
@locked
|
||||
@scm_context
|
||||
def move(self, from_path, to_path):
|
||||
def move(self: "Repo", from_path, to_path):
|
||||
"""
|
||||
Renames an output file and modifies the stage associated
|
||||
to reflect the change on the pipeline.
|
||||
@@ -32,9 +35,12 @@ def move(self, from_path, to_path):
|
||||
It only works with outputs generated by `add` or `import`,
|
||||
also known as data sources.
|
||||
"""
|
||||
from dvc import output
|
||||
from dvc import dependency, output
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX
|
||||
from dvc.exceptions import MoveNotDataSourceError
|
||||
from dvc.stage import Stage
|
||||
from dvc.stage.exceptions import StageFileAlreadyExistsError
|
||||
from dvc_objects.fs.local import LocalFileSystem
|
||||
|
||||
from_out = output.loads_from(Stage(self), [from_path])[0]
|
||||
assert from_out.protocol == "local"
|
||||
@@ -45,6 +51,7 @@ def move(self, from_path, to_path):
|
||||
assert len(outs) == 1
|
||||
out = outs[0]
|
||||
stage = out.stage
|
||||
deps = stage.deps
|
||||
|
||||
if not stage.is_data_source:
|
||||
raise MoveNotDataSourceError(stage.addressing)
|
||||
@@ -58,21 +65,38 @@ def move(self, from_path, to_path):
|
||||
)
|
||||
new_wdir = os.path.abspath(os.path.join(os.curdir, os.path.dirname(to_path)))
|
||||
to_path = os.path.relpath(to_path, new_wdir)
|
||||
new_stage = self.stage.create(
|
||||
single_stage=True,
|
||||
fname=new_fname,
|
||||
wdir=new_wdir,
|
||||
outs=[to_path],
|
||||
meta=stage.meta,
|
||||
)
|
||||
|
||||
os.unlink(stage.path)
|
||||
stage = new_stage
|
||||
try:
|
||||
new_stage = self.stage.create(
|
||||
single_stage=True,
|
||||
fname=new_fname,
|
||||
wdir=new_wdir,
|
||||
outs=[to_path],
|
||||
meta=stage.meta,
|
||||
frozen=stage.frozen,
|
||||
always_changed=stage.always_changed,
|
||||
desc=stage.desc,
|
||||
)
|
||||
except StageFileAlreadyExistsError:
|
||||
# reraise to remove `--force` hint
|
||||
raise StageFileAlreadyExistsError(f"{new_fname!r} already exists") from None
|
||||
else:
|
||||
new_stage = stage
|
||||
to_path = os.path.relpath(to_path, stage.wdir)
|
||||
|
||||
to_out = output.loads_from(stage, [to_path], out.use_cache, out.metric)[0]
|
||||
def with_dep_path_adjusted(dep: dependency.Dependency):
|
||||
d = dep.dumpd()
|
||||
if isinstance(dep.fs, LocalFileSystem) and not os.path.isabs(dep.def_path):
|
||||
return d | {"path": os.path.relpath(dep.fspath, new_stage.wdir)}
|
||||
return d
|
||||
|
||||
out.move(to_out)
|
||||
stage.save()
|
||||
stage.dump()
|
||||
new_stage.outs = output.loadd_from(new_stage, [out.dumpd() | {"path": to_path}])
|
||||
new_stage.deps = dependency.loadd_from(
|
||||
new_stage, [with_dep_path_adjusted(dep) for dep in deps]
|
||||
)
|
||||
out.move(new_stage.outs[0])
|
||||
new_stage.md5 = new_stage.compute_md5()
|
||||
new_stage.dump()
|
||||
if stage != new_stage:
|
||||
stage.dvcfile.remove()
|
||||
self.scm_context.track_file(stage.dvcfile.relpath)
|
||||
return stage, new_stage
|
||||
|
||||
@@ -46,7 +46,7 @@ class SCMContext:
|
||||
try:
|
||||
add_no_submodules(self.scm, paths)
|
||||
except UnsupportedIndexFormat:
|
||||
link = "https://github.com/iterative/dvc/issues/610"
|
||||
link = "https://github.com/treeverse/dvc/issues/610"
|
||||
add_cmd = self._make_git_add_cmd([relpath(path) for path in paths])
|
||||
logger.info("")
|
||||
msg = (
|
||||
@@ -58,7 +58,7 @@ class SCMContext:
|
||||
def track_changed_files(self) -> None:
|
||||
"""Stage files that have changed."""
|
||||
if not self.files_to_track:
|
||||
return
|
||||
return None
|
||||
logger.debug("Staging files: %s", self.files_to_track)
|
||||
return self.add(self.files_to_track)
|
||||
|
||||
@@ -76,6 +76,7 @@ class SCMContext:
|
||||
logger.debug("Added '%s' to gitignore file.", path)
|
||||
self.track_file(relpath(gitignore_file))
|
||||
return self.ignored_paths.append(path)
|
||||
return None
|
||||
|
||||
def ignore_remove(self, path: str) -> None:
|
||||
from scmrepo.exceptions import FileNotInRepoError
|
||||
@@ -90,6 +91,7 @@ class SCMContext:
|
||||
|
||||
if gitignore_file:
|
||||
return self.track_file(relpath(gitignore_file))
|
||||
return None
|
||||
|
||||
@contextmanager
|
||||
def __call__(
|
||||
|
||||
+1
-1
@@ -45,7 +45,7 @@ class NoSCMError(SCMError):
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
class InvalidRemoteSCMRepo(SCMError):
|
||||
class InvalidRemoteSCMRepo(SCMError): # noqa: N818
|
||||
pass
|
||||
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ from .utils import (
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.dependency import ParamsDependency
|
||||
from dvc.dependency import Dependency, ParamsDependency
|
||||
from dvc.dvcfile import ProjectFile, SingleStageFile
|
||||
from dvc.output import Output
|
||||
from dvc.repo import Repo
|
||||
@@ -121,7 +121,7 @@ def restore_fields(stage: "Stage") -> None:
|
||||
|
||||
old_outs = {out.def_path: out for out in old.outs}
|
||||
for out in stage.outs:
|
||||
old_out = old_outs.get(out.def_path, None)
|
||||
old_out = old_outs.get(out.def_path)
|
||||
if old_out is not None:
|
||||
out.restore_fields(old_out)
|
||||
|
||||
@@ -153,8 +153,8 @@ class Stage(params.StageParams):
|
||||
self._path = path
|
||||
self.cmd = cmd
|
||||
self.wdir = wdir
|
||||
self.outs = outs
|
||||
self.deps = deps
|
||||
self.outs: list[Output] = outs
|
||||
self.deps: list[Dependency] = deps
|
||||
self.md5 = md5
|
||||
self.frozen = locked or frozen
|
||||
self.always_changed = always_changed
|
||||
|
||||
@@ -82,7 +82,7 @@ class DataSourceChanged(DvcException):
|
||||
super().__init__(f"data source changed: {path}")
|
||||
|
||||
|
||||
class StageNotFound(DvcException, KeyError):
|
||||
class StageNotFound(DvcException, KeyError): # noqa: N818
|
||||
def __init__(self, file, name):
|
||||
self.file = file.relpath
|
||||
self.name = name
|
||||
|
||||
+2
-2
@@ -51,7 +51,7 @@ def _warn_if_fish(executable):
|
||||
"Be aware that it might cause problems by overwriting "
|
||||
"your current environment variables with values defined "
|
||||
"in 'config.fish', which might affect your command. See "
|
||||
"https://github.com/iterative/dvc/issues/1307. "
|
||||
"https://github.com/treeverse/dvc/issues/1307. "
|
||||
)
|
||||
|
||||
|
||||
@@ -99,7 +99,7 @@ def prepare_kwargs(stage, run_env=None):
|
||||
#
|
||||
# [1] https://github.com/python/cpython/blob/3.7/Lib/subprocess.py
|
||||
# #L1426
|
||||
# [2] https://github.com/iterative/dvc/issues/2506
|
||||
# [2] https://github.com/treeverse/dvc/issues/2506
|
||||
# #issuecomment-535396799
|
||||
kwargs["shell"] = os.name == "nt"
|
||||
return kwargs
|
||||
|
||||
@@ -178,7 +178,7 @@ def to_single_stage_lockfile(stage: "Stage", **kwargs) -> dict:
|
||||
res = OrderedDict([("cmd", stage.cmd)])
|
||||
params, deps = split_params_deps(stage)
|
||||
deps, outs = (
|
||||
[_dumpd(item) for item in sorted(items, key=attrgetter("def_path"))]
|
||||
[_dumpd(item) for item in sorted(items, key=attrgetter("def_path"))] # type: ignore[call-overload]
|
||||
for items in [deps, stage.outs]
|
||||
)
|
||||
params = _serialize_params_values(params)
|
||||
|
||||
@@ -4,7 +4,7 @@ dvc.testing.benchmarks
|
||||
======================
|
||||
|
||||
Benchmark test definitions are now part of ``dvc.testing``.
|
||||
For CLI usage and `bench.dvc.org <https://bench.dvc.org>`_ details see `dvc-bench <https://github.com/iterative/dvc-bench>`_.
|
||||
For CLI usage and `bench.dvc.org <https://bench.dvc.org>`_ details see `dvc-bench <https://github.com/treeverse/dvc-bench>`_.
|
||||
|
||||
``dvc.testing.benchmarks`` structure:
|
||||
|
||||
|
||||
@@ -6,8 +6,8 @@ import pytest
|
||||
"!=3.53.*,!=3.54.0",
|
||||
reason="Takes 10 mins to run. Regression in 3.53.0, fixed in 3.54.1",
|
||||
)
|
||||
# Introduced in https://github.com/iterative/dvc/pull/10388.
|
||||
# Fixed in https://github.com/iterative/dvc/pull/10531.
|
||||
# Introduced in https://github.com/treeverse/dvc/pull/10388.
|
||||
# Fixed in https://github.com/treeverse/dvc/pull/10531.
|
||||
def test_import(bench_dvc, tmp_dir, scm, dvc, make_dataset, remote):
|
||||
dataset = make_dataset(
|
||||
cache=False, files=False, dvcfile=True, commit=True, remote=True
|
||||
|
||||
@@ -79,7 +79,7 @@ def dvc_bench_repo(tmp_path_factory, bench_config):
|
||||
if url is None:
|
||||
pytest.skip(
|
||||
"--dvc-bench-repo is not set, "
|
||||
"clone https://github.com/iterative/dvc-bench repository and set its path"
|
||||
"clone https://github.com/treeverse/dvc-bench repository and set its path"
|
||||
)
|
||||
|
||||
if os.path.isdir(url):
|
||||
@@ -158,7 +158,7 @@ def make_bench(request):
|
||||
# pytest >= 8.4.0
|
||||
wrapped_func = fixture_function._get_wrapped_function()
|
||||
except AttributeError:
|
||||
wrapped_func = fixture_function.__pytest_wrapped__.obj # ty: ignore[unresolved-attribute]
|
||||
wrapped_func = fixture_function.__pytest_wrapped__.obj # type: ignore[attr-defined] # ty: ignore[unresolved-attribute]
|
||||
assert inspect.isgeneratorfunction(wrapped_func)
|
||||
|
||||
generator = wrapped_func(request)
|
||||
|
||||
@@ -44,7 +44,7 @@ class TestRemote:
|
||||
_check_status(status_dir, new=dir_hashes)
|
||||
|
||||
# Move cache and check status
|
||||
# See issue https://github.com/iterative/dvc/issues/4383 for details
|
||||
# See issue https://github.com/treeverse/dvc/issues/4383 for details
|
||||
backup_dir = dvc.cache.local.path + ".backup"
|
||||
shutil.move(dvc.cache.local.path, backup_dir)
|
||||
status = dvc.cloud.status(foo_hashes)
|
||||
@@ -116,7 +116,7 @@ class TestRemote:
|
||||
|
||||
@pytest.mark.xfail(raises=NotImplementedError, strict=False)
|
||||
def test_pull_00_prefix(self, tmp_dir, dvc, remote, monkeypatch):
|
||||
# Related: https://github.com/iterative/dvc/issues/6089
|
||||
# Related: https://github.com/treeverse/dvc/issues/6089
|
||||
|
||||
fs_type = type(dvc.cloud.get_remote_odb("upstream").fs)
|
||||
monkeypatch.setattr(fs_type, "_ALWAYS_TRAVERSE", True, raising=False)
|
||||
@@ -146,7 +146,7 @@ class TestRemote:
|
||||
|
||||
@pytest.mark.xfail(raises=NotImplementedError, strict=False)
|
||||
def test_pull_no_00_prefix(self, tmp_dir, dvc, remote, monkeypatch):
|
||||
# Related: https://github.com/iterative/dvc/issues/6244
|
||||
# Related: https://github.com/treeverse/dvc/issues/6244
|
||||
|
||||
fs_type = type(dvc.cloud.get_remote_odb("upstream").fs)
|
||||
monkeypatch.setattr(fs_type, "_ALWAYS_TRAVERSE", True, raising=False)
|
||||
|
||||
+3
-3
@@ -193,7 +193,7 @@ class Console:
|
||||
sep = " " if sep is None else sep
|
||||
end = "\n" if end is None else end
|
||||
if not self._enabled and not force:
|
||||
return
|
||||
return None
|
||||
|
||||
file = file or (sys.stderr if stderr else sys.stdout)
|
||||
with Tqdm.external_write_mode(file=file):
|
||||
@@ -305,7 +305,7 @@ class Console:
|
||||
from dvc.ui import table as t
|
||||
|
||||
if not data and not markdown:
|
||||
return
|
||||
return None
|
||||
|
||||
if not markdown and rich_table:
|
||||
if force or self._enabled:
|
||||
@@ -319,7 +319,7 @@ class Console:
|
||||
borders=borders,
|
||||
)
|
||||
|
||||
return
|
||||
return None
|
||||
|
||||
return t.plain_table(
|
||||
self,
|
||||
|
||||
+2
-2
@@ -120,7 +120,7 @@ class Updater:
|
||||
from dvc.ui import ui
|
||||
|
||||
if not ui.isatty():
|
||||
return
|
||||
return None
|
||||
|
||||
message = self._get_message(latest, pkg=pkg)
|
||||
return ui.error_write(message, styled=True)
|
||||
@@ -163,7 +163,7 @@ class Updater:
|
||||
if pkg not in instructions:
|
||||
return (
|
||||
"Find the latest release at "
|
||||
"[blue]https://github.com/iterative/dvc/releases/latest[/]."
|
||||
"[blue]https://github.com/treeverse/dvc/releases/latest[/]."
|
||||
)
|
||||
|
||||
instruction = instructions[pkg]
|
||||
|
||||
+18
-9
@@ -18,7 +18,7 @@ keywords = [
|
||||
]
|
||||
license = "Apache-2.0"
|
||||
license-files = ["LICENSE"]
|
||||
maintainers = [{ name = "Iterative", email = "[email protected]" }]
|
||||
maintainers = [{ name = "Treeverse", email = "[email protected]" }]
|
||||
authors = [{ name = "Dmitry Petrov", email = "[email protected]" }]
|
||||
requires-python = ">=3.9"
|
||||
classifiers = [
|
||||
@@ -29,6 +29,7 @@ classifiers = [
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Programming Language :: Python :: 3.14",
|
||||
]
|
||||
dynamic = ["version"]
|
||||
|
||||
@@ -85,7 +86,7 @@ gdrive = ["dvc-gdrive>=3,<4"]
|
||||
gs = ["dvc-gs>=3.0.2,<4"]
|
||||
hdfs = ["dvc-hdfs>=3,<4"]
|
||||
lint = [
|
||||
"mypy==1.17.1",
|
||||
"mypy==1.19.0",
|
||||
"pandas-stubs",
|
||||
"types-colorama",
|
||||
"types-psutil",
|
||||
@@ -111,7 +112,7 @@ tests = [
|
||||
"dvc-ssh",
|
||||
"filelock",
|
||||
"pytest>=7,<9",
|
||||
"pytest-rerunfailures<16.0",
|
||||
"pytest-rerunfailures<17.0",
|
||||
"pytest-cov>=4.1.0",
|
||||
"pytest-docker>=1,<4",
|
||||
"pytest-mock",
|
||||
@@ -123,14 +124,14 @@ tests = [
|
||||
"sqlalchemy>=1,<3", # optional dependency for `import-db`
|
||||
"pandas>=1", # optional dependency for `import-db`
|
||||
]
|
||||
webdav = ["dvc-webdav>=3,<4"]
|
||||
webdav = ["dvc-webdav>=3.0.1,<4"]
|
||||
webhdfs = ["dvc-webhdfs>=3.1,<4"]
|
||||
webhdfs_kerberos = ["dvc-webhdfs[kerberos]>=3.1,<4"]
|
||||
|
||||
[project.urls]
|
||||
Documentation = "https://dvc.org/doc"
|
||||
Issues = "https://github.com/iterative/dvc/issues"
|
||||
Source = "https://github.com/iterative/dvc"
|
||||
Issues = "https://github.com/treeverse/dvc/issues"
|
||||
Source = "https://github.com/treeverse/dvc"
|
||||
|
||||
[project.scripts]
|
||||
dvc = "dvc.cli:main"
|
||||
@@ -265,8 +266,16 @@ show-fixes = true
|
||||
|
||||
[tool.ruff.lint]
|
||||
ignore = [
|
||||
"N818", "S101", "A005", "PT007", "RET502", "RET503", "SIM105", "SIM108", "SIM117",
|
||||
"TRY003", "TRY300", "PERF203", "PLC0415", "PLR2004", "PLW2901", "LOG007",
|
||||
"PERF203", # try-except-in-loop
|
||||
"PLC0415", # import-outside-top-level
|
||||
"PLR2004", # magic-value-comparison
|
||||
"PLW2901", # redefined-loop-name
|
||||
"S101", # assert
|
||||
"SIM105", # suppressible-exception
|
||||
"SIM108", # if-else-block-instead-of-if-exp
|
||||
"SIM117", # multiple-with-statements
|
||||
"TRY003", # raise-vanilla-args
|
||||
"TRY300", # try-consider-else
|
||||
]
|
||||
select = [
|
||||
"F", "E", "W", "C90", "I", "N", "UP", "YTT", "ASYNC", "S", "BLE", "B", "A", "C4", "DTZ", "T10",
|
||||
@@ -304,4 +313,4 @@ max-args = 10
|
||||
"dvc/commands/**" = ["N806"]
|
||||
"dvc/testing/**" = ["ARG002"]
|
||||
"dvc/testing/benchmarks/**" = ["ARG001"]
|
||||
"tests/**" = ["S", "ARG001", "ARG002", "TRY002", "TRY301", "PERF"]
|
||||
"tests/**" = ["S", "ARG001", "ARG002", "TRY002", "TRY301", "PERF", "PLR2004"]
|
||||
|
||||
@@ -194,6 +194,7 @@ def isolate(tmp_path_factory):
|
||||
monkeypatch.setenv(env_var, os.fspath(path))
|
||||
else:
|
||||
monkeypatch.setenv("HOME", str(home_dir))
|
||||
monkeypatch.setenv("XDG_CONFIG_HOME", str(home_dir / ".config"))
|
||||
|
||||
monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1")
|
||||
contents = b"""
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import re
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc import api
|
||||
@@ -13,7 +15,7 @@ def test_exp_save(tmp_dir, dvc, scm):
|
||||
api.exp_save("foo")
|
||||
with pytest.raises(
|
||||
ExperimentExistsError,
|
||||
match="Experiment conflicts with existing experiment 'foo'.",
|
||||
match=re.escape("Experiment conflicts with existing experiment 'foo'."),
|
||||
):
|
||||
api.exp_save("foo")
|
||||
api.exp_save("foo", force=True)
|
||||
|
||||
@@ -91,7 +91,7 @@ def test_apply_untracked(tmp_dir, scm, dvc, exp_stage):
|
||||
|
||||
|
||||
def test_apply_unchanged_head(tmp_dir, scm, dvc, exp_stage):
|
||||
# see https://github.com/iterative/dvc/issues/8764
|
||||
# see https://github.com/treeverse/dvc/issues/8764
|
||||
tmp_dir.gen("params.yaml", "foo: 2")
|
||||
scm.add(["dvc.yaml", "dvc.lock", "params.yaml", "metrics.yaml"])
|
||||
scm.commit("commit foo=2")
|
||||
|
||||
@@ -4,6 +4,7 @@ import os
|
||||
import stat
|
||||
from textwrap import dedent
|
||||
|
||||
import dulwich
|
||||
import pytest
|
||||
from configobj import ConfigObj
|
||||
from funcy import first
|
||||
@@ -19,6 +20,7 @@ from dvc.env import (
|
||||
DVC_STUDIO_URL,
|
||||
)
|
||||
from dvc.exceptions import DvcException, ReproductionError
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.experiments.exceptions import ExperimentExistsError
|
||||
from dvc.repo.experiments.queue.base import BaseStashQueue
|
||||
from dvc.repo.experiments.refs import CELERY_STASH
|
||||
@@ -808,3 +810,26 @@ def test_experiments_run_with_submodule_dependencies(dvc, scm, make_tmp_dir, dep
|
||||
dvc.stage.add(cmd="echo foo", deps=[dep], name="foo")
|
||||
|
||||
assert dvc.experiments.run()
|
||||
|
||||
|
||||
@pytest.mark.skipif(dulwich.__version__ < (0, 24, 2), reason="requires dulwich>=0.24.2")
|
||||
def test_experiments_run_in_linked_git_worktree(
|
||||
dvc, scm, tmp_path_factory: pytest.TempPathFactory, monkeypatch
|
||||
):
|
||||
from dulwich.worktree import add_worktree
|
||||
|
||||
wt = tmp_path_factory.mktemp("worktrees") / "worktree"
|
||||
add_worktree(scm.dulwich.repo, wt, branch="wt-main")
|
||||
|
||||
monkeypatch.chdir(wt)
|
||||
|
||||
wt_dvc = Repo(os.fspath(wt))
|
||||
(wt / "foo").write_bytes(b"foo")
|
||||
wt_dvc.stage.add(cmd="cp foo bar", deps=["foo"], outs=["bar"], name="cp")
|
||||
|
||||
results = wt_dvc.experiments.run(name="my-exp")
|
||||
assert results
|
||||
rev = first(results)
|
||||
assert rev
|
||||
# If `bar` exists, we know that the stage was run.
|
||||
assert (wt / "bar").read_bytes() == b"foo"
|
||||
|
||||
@@ -26,7 +26,7 @@ def test_celery_logs(tmp_dir, scm, dvc, failed_exp_stage, follow, capsys, test_q
|
||||
|
||||
|
||||
def test_queue_doesnt_remove_untracked_params_file(tmp_dir, dvc, scm):
|
||||
"""Regression test for https://github.com/iterative/dvc/issues/7842"""
|
||||
"""Regression test for https://github.com/treeverse/dvc/issues/7842"""
|
||||
tmp_dir.gen("params.yaml", "foo: 1")
|
||||
stage = dvc.run(cmd="echo ${foo}", params=["foo"], name="echo-foo")
|
||||
scm.add(["dvc.yaml", "dvc.lock", ".gitignore"])
|
||||
|
||||
@@ -204,10 +204,10 @@ def test_remove_multi_rev(tmp_dir, scm, dvc, exp_stage):
|
||||
@pytest.mark.parametrize(
|
||||
"keep, expected_removed",
|
||||
[
|
||||
[["exp1"], ["exp2", "exp3"]],
|
||||
[["exp1", "exp2"], ["exp3"]],
|
||||
[["exp1", "exp2", "exp3"], []],
|
||||
[[], []], # remove does nothing if no experiments are specified
|
||||
(["exp1"], ["exp2", "exp3"]),
|
||||
(["exp1", "exp2"], ["exp3"]),
|
||||
(["exp1", "exp2", "exp3"], []),
|
||||
([], []), # remove does nothing if no experiments are specified
|
||||
],
|
||||
)
|
||||
def test_keep_selected_by_name(tmp_dir, scm, dvc, exp_stage, keep, expected_removed):
|
||||
@@ -239,14 +239,14 @@ def test_keep_selected_by_nonexistent_name(tmp_dir, scm, dvc, exp_stage):
|
||||
@pytest.mark.parametrize(
|
||||
"num_exps, rev, num, expected_removed",
|
||||
[
|
||||
[2, "exp1", 1, ["exp2"]],
|
||||
[3, "exp3", 1, ["exp1", "exp2"]],
|
||||
[3, "exp3", 2, ["exp1"]],
|
||||
[3, "exp3", 3, []],
|
||||
[3, "exp2", 2, ["exp3"]],
|
||||
[4, "exp2", 2, ["exp3", "exp4"]],
|
||||
[4, "exp4", 2, ["exp1", "exp2"]],
|
||||
[1, None, 1, []], # remove does nothing if no experiments are specified
|
||||
(2, "exp1", 1, ["exp2"]),
|
||||
(3, "exp3", 1, ["exp1", "exp2"]),
|
||||
(3, "exp3", 2, ["exp1"]),
|
||||
(3, "exp3", 3, []),
|
||||
(3, "exp2", 2, ["exp3"]),
|
||||
(4, "exp2", 2, ["exp3", "exp4"]),
|
||||
(4, "exp4", 2, ["exp1", "exp2"]),
|
||||
(1, None, 1, []), # remove does nothing if no experiments are specified
|
||||
],
|
||||
)
|
||||
def test_keep_selected_by_rev(
|
||||
|
||||
@@ -18,7 +18,7 @@ def test_exp_save_unchanged(tmp_dir, dvc, scm):
|
||||
dvc.experiments.save()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", (None, "test"))
|
||||
@pytest.mark.parametrize("name", [None, "test"])
|
||||
def test_exp_save(tmp_dir, dvc, scm, name):
|
||||
setup_stage(tmp_dir, dvc, scm)
|
||||
baseline = scm.get_rev()
|
||||
@@ -47,13 +47,13 @@ def test_exp_save_overwrite_experiment(tmp_dir, dvc, scm):
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
(
|
||||
[
|
||||
"invalid/name",
|
||||
"invalid..name",
|
||||
"invalid~name",
|
||||
"invalid?name",
|
||||
"invalidname.",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_exp_save_invalid_name(tmp_dir, dvc, scm, name):
|
||||
setup_stage(tmp_dir, dvc, scm)
|
||||
@@ -105,7 +105,7 @@ def test_exp_save_include_untracked(tmp_dir, dvc, scm):
|
||||
|
||||
|
||||
def test_exp_save_include_untracked_warning(tmp_dir, dvc, scm, mocker):
|
||||
"""Regression test for https://github.com/iterative/dvc/issues/9061"""
|
||||
"""Regression test for https://github.com/treeverse/dvc/issues/9061"""
|
||||
setup_stage(tmp_dir, dvc, scm)
|
||||
|
||||
new_dir = tmp_dir / "new_dir"
|
||||
|
||||
@@ -7,8 +7,8 @@ from tests.func.utils.test_hydra import hydra_setup
|
||||
@pytest.mark.parametrize(
|
||||
"changes, expected",
|
||||
[
|
||||
[["foo=baz"], "foo: baz\ngoo:\n bag: 3.0\nlorem: false"],
|
||||
[["params.yaml:foo=baz"], "foo: baz\ngoo:\n bag: 3.0\nlorem: false"],
|
||||
(["foo=baz"], "foo: baz\ngoo:\n bag: 3.0\nlorem: false"),
|
||||
(["params.yaml:foo=baz"], "foo: baz\ngoo:\n bag: 3.0\nlorem: false"),
|
||||
],
|
||||
)
|
||||
def test_modify_params(params_repo, dvc, changes, expected):
|
||||
@@ -26,8 +26,9 @@ def test_modify_params(params_repo, dvc, changes, expected):
|
||||
("conf", "bar"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("no_hydra", [True, False])
|
||||
def test_hydra_compose_and_dump(
|
||||
tmp_dir, params_repo, dvc, hydra_enabled, config_dir, config_name
|
||||
tmp_dir, params_repo, dvc, hydra_enabled, config_dir, config_name, no_hydra
|
||||
):
|
||||
hydra_setup(
|
||||
tmp_dir,
|
||||
@@ -50,14 +51,14 @@ def test_hydra_compose_and_dump(
|
||||
if config_name is not None:
|
||||
conf["hydra"]["config_name"] = config_name
|
||||
|
||||
dvc.experiments.run()
|
||||
dvc.experiments.run(no_hydra=no_hydra)
|
||||
|
||||
if hydra_enabled:
|
||||
if hydra_enabled and not no_hydra:
|
||||
assert (tmp_dir / "params.yaml").parse() == {
|
||||
"db": {"driver": "mysql", "user": "omry", "pass": "secret"},
|
||||
}
|
||||
|
||||
dvc.experiments.run(params=["db=postgresql"])
|
||||
dvc.experiments.run(params=["db=postgresql"], no_hydra=no_hydra)
|
||||
assert (tmp_dir / "params.yaml").parse() == {
|
||||
"db": {
|
||||
"driver": "postgresql",
|
||||
@@ -117,6 +118,7 @@ def test_hydra_sweep(
|
||||
targets=None,
|
||||
copy_paths=None,
|
||||
message=None,
|
||||
no_hydra=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -137,3 +139,76 @@ def test_hydra_sweep_prefix_name(tmp_dir, params_repo, dvc):
|
||||
exp_names = [entry.name for entry in dvc.experiments.celery_queue.iter_queued()]
|
||||
for name, expected in zip(exp_names, expected_names):
|
||||
assert name == expected
|
||||
|
||||
|
||||
def test_mixing_no_hydra_and_params_flags(tmp_dir, params_repo, dvc):
|
||||
# Passing no_hydra should not prevent user from
|
||||
# using --set-param on unmodified params.yaml
|
||||
hydra_setup(
|
||||
tmp_dir,
|
||||
config_dir="conf",
|
||||
config_name="config",
|
||||
)
|
||||
|
||||
with dvc.config.edit() as conf:
|
||||
conf["hydra"]["enabled"] = True
|
||||
conf["hydra"]["config_dir"] = "conf"
|
||||
conf["hydra"]["config_name"] = "config"
|
||||
|
||||
dvc.experiments.run(no_hydra=True, params=["goo.bag=10.0"])
|
||||
|
||||
assert (tmp_dir / "params.yaml").parse() == {
|
||||
"foo": [{"bar": 1}, {"baz": 2}],
|
||||
"goo": {"bag": 10.0},
|
||||
"lorem": False,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"hydra_enabled,overrides,expected",
|
||||
[
|
||||
(
|
||||
True,
|
||||
["db=mysql,postgresql"],
|
||||
[
|
||||
{"params.yaml": ["db=mysql"]},
|
||||
{"params.yaml": ["db=postgresql"]},
|
||||
],
|
||||
),
|
||||
(
|
||||
False,
|
||||
["foo=bar,baz"],
|
||||
[{"params.yaml": ["foo=bar"]}, {"params.yaml": ["foo=baz"]}],
|
||||
),
|
||||
(
|
||||
False,
|
||||
[],
|
||||
[{}],
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("no_hydra", [True, False])
|
||||
def test_mixing_no_hydra_and_sweeps(
|
||||
tmp_dir, params_repo, dvc, mocker, hydra_enabled, overrides, expected, no_hydra
|
||||
):
|
||||
# Passing no_hydra should not prevent user from
|
||||
# queuing sweeps with --set-param and --queue
|
||||
patched = mocker.patch.object(dvc.experiments, "queue_one")
|
||||
|
||||
if hydra_enabled:
|
||||
hydra_setup(tmp_dir, config_dir="conf", config_name="config")
|
||||
with dvc.config.edit() as conf:
|
||||
conf["hydra"]["enabled"] = True
|
||||
|
||||
dvc.experiments.run(params=overrides, queue=True, no_hydra=no_hydra)
|
||||
|
||||
assert patched.call_count == len(expected)
|
||||
for e in expected:
|
||||
patched.assert_any_call(
|
||||
mocker.ANY,
|
||||
params=e,
|
||||
targets=None,
|
||||
copy_paths=None,
|
||||
message=None,
|
||||
no_hydra=no_hydra,
|
||||
)
|
||||
|
||||
@@ -289,14 +289,14 @@ def test_metrics_show_overlap(tmp_dir, dvc, run_copy_metrics, clear_before_run):
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"file,error_path,err_type",
|
||||
(
|
||||
[
|
||||
(PROJECT_FILE, ["workspace", "error", "type"], "YAMLSyntaxError"),
|
||||
(
|
||||
"metrics.yaml",
|
||||
["workspace", "data", "metrics.yaml", "error", "type"],
|
||||
"YAMLFileCorruptedError",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_log_errors(
|
||||
tmp_dir, scm, dvc, capsys, run_copy_metrics, file, error_path, err_type
|
||||
|
||||
@@ -300,13 +300,13 @@ def test_ignore_parsing_error(tmp_dir, dvc, run_copy_metrics):
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"file,path_kwargs",
|
||||
(
|
||||
[
|
||||
(PROJECT_FILE, {"revision": "workspace", "endkey": "error"}),
|
||||
(
|
||||
"plot.yaml",
|
||||
{"revision": "workspace", "file": "plot.yaml", "endkey": "error"},
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_log_errors(tmp_dir, scm, dvc, run_copy_metrics, file, path_kwargs, capsys):
|
||||
metric = [{"val": 2}, {"val": 3}]
|
||||
|
||||
@@ -148,7 +148,7 @@ def test_downstream(M, tmp_dir, dvc):
|
||||
assert [stage.name for stage in evaluation] == ["B-gen", "D-gen", "E-gen"]
|
||||
|
||||
# B, C should be run (in any order) before D
|
||||
# See https://github.com/iterative/dvc/issues/3602
|
||||
# See https://github.com/treeverse/dvc/issues/3602
|
||||
evaluation = dvc.reproduce(PROJECT_FILE + ":A-gen", downstream=True, force=True)
|
||||
|
||||
assert len(evaluation) == 5
|
||||
|
||||
@@ -36,7 +36,7 @@ def test_repro_allow_missing_and_pull(tmp_dir, dvc, mocker, local_remote):
|
||||
def test_repro_allow_missing_upstream_stage_modified(
|
||||
tmp_dir, dvc, mocker, local_remote
|
||||
):
|
||||
"""https://github.com/iterative/dvc/issues/9530"""
|
||||
"""https://github.com/treeverse/dvc/issues/9530"""
|
||||
tmp_dir.gen("params.yaml", "param: 1")
|
||||
dvc.stage.add(
|
||||
name="create-foo", cmd="echo ${param} > foo", params=["param"], outs=["foo"]
|
||||
|
||||
@@ -723,7 +723,7 @@ def test_add_symlink_file(tmp_dir, dvc):
|
||||
).is_symlink()
|
||||
|
||||
# Test that subsequent add succeeds
|
||||
# See https://github.com/iterative/dvc/issues/4654
|
||||
# See https://github.com/treeverse/dvc/issues/4654
|
||||
dvc.add(os.path.join("dir", "foo"))
|
||||
|
||||
|
||||
|
||||
@@ -88,8 +88,8 @@ def test_config_set_in_non_dvc_repo(tmp_dir, caplog):
|
||||
[
|
||||
(["core.analytics"], 0, "False"),
|
||||
(["core.remote"], 0, "myremote"),
|
||||
(["remote.myremote.profile"], 0, "iterative"),
|
||||
(["remote.myremote.profile", "--local"], 0, "iterative"),
|
||||
(["remote.myremote.profile"], 0, "treeverse"),
|
||||
(["remote.myremote.profile", "--local"], 0, "treeverse"),
|
||||
(
|
||||
["remote.myremote.profile", "--project"],
|
||||
251,
|
||||
@@ -122,7 +122,7 @@ def test_config_get(tmp_dir, dvc, capsys, caplog, args, ret, msg):
|
||||
textwrap.dedent(
|
||||
"""\
|
||||
['remote "myremote"']
|
||||
profile = iterative
|
||||
profile = treeverse
|
||||
['remote "other"']
|
||||
url = gs://bucket/path
|
||||
"""
|
||||
@@ -167,7 +167,7 @@ def test_config_list(tmp_dir, dvc, capsys):
|
||||
textwrap.dedent(
|
||||
"""\
|
||||
['remote "myremote"']
|
||||
profile = iterative
|
||||
profile = treeverse
|
||||
access_key_id = abcde
|
||||
secret_access_key = 123456
|
||||
['remote "other"']
|
||||
@@ -181,7 +181,7 @@ def test_config_list(tmp_dir, dvc, capsys):
|
||||
out, _ = capsys.readouterr()
|
||||
assert "remote.myremote.url=s3://bucket/path" in out
|
||||
assert "remote.myremote.region=us-east-2" in out
|
||||
assert "remote.myremote.profile=iterative" in out
|
||||
assert "remote.myremote.profile=treeverse" in out
|
||||
assert "remote.myremote.access_key_id=abcde" in out
|
||||
assert "remote.myremote.secret_access_key=123456" in out
|
||||
assert "remote.other.url=gs://bucket/path" in out
|
||||
@@ -242,7 +242,7 @@ def test_merging_two_levels(dvc):
|
||||
|
||||
|
||||
def test_config_loads_without_error_for_non_dvc_repo(tmp_dir):
|
||||
# regression testing for https://github.com/iterative/dvc/issues/3328
|
||||
# regression testing for https://github.com/treeverse/dvc/issues/3328
|
||||
Config.from_cwd(validate=True)
|
||||
|
||||
|
||||
|
||||
@@ -730,12 +730,12 @@ def test_pull_allow_missing(tmp_dir, dvc, local_remote):
|
||||
def test_pull_granular_excluding_import_that_cannot_be_pulled(
|
||||
tmp_dir, dvc, local_remote, mocker
|
||||
):
|
||||
"""Regression test for https://github.com/iterative/dvc/issues/10309."""
|
||||
"""Regression test for https://github.com/treeverse/dvc/issues/10309."""
|
||||
|
||||
mocker.patch("dvc.fs.dvc._DVCFileSystem", side_effect=CloneError("SCM error"))
|
||||
(stage,) = tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||||
imp_stage = dvc.imp(
|
||||
"https://user:[email protected]/iterative/dvc.git",
|
||||
"https://user:[email protected]/treeverse/dvc.git",
|
||||
"dir",
|
||||
out="new_dir",
|
||||
rev="HEAD",
|
||||
|
||||
@@ -578,7 +578,7 @@ def test_root_from_file_to_dir(M, tmp_dir, dvc, scm):
|
||||
|
||||
|
||||
def test_empty_dir(tmp_dir, scm, dvc, M):
|
||||
# regression testing for https://github.com/iterative/dvc/issues/8958
|
||||
# regression testing for https://github.com/treeverse/dvc/issues/8958
|
||||
tmp_dir.dvc_gen({"data": {"foo": "foo"}})
|
||||
remove("data")
|
||||
|
||||
@@ -1211,3 +1211,40 @@ def test_renames(
|
||||
dvc.data_status(granular=True, with_renames=True, targets=targets)
|
||||
== EMPTY_STATUS | {"git": M.dict()} | expected_granular
|
||||
)
|
||||
|
||||
|
||||
def test_shallow_should_iterate_upto_tracked_directory(
|
||||
M, tmp_dir, dvc, scm, local_remote
|
||||
):
|
||||
"""Testing regression for https://github.com/treeverse/dvc/issues/10899."""
|
||||
|
||||
tmp_dir.scm_gen({"dir": {".gitkeep": ""}}, commit="mk dir")
|
||||
dataset_root = join("dir", "data", "")
|
||||
tmp_dir.dvc_gen({dataset_root: {"foo": "foo", "bar": "bar"}})
|
||||
files = [dataset_root, join(dataset_root, "foo"), join(dataset_root, "bar")]
|
||||
|
||||
assert dvc.data_status(not_in_remote=True) == {
|
||||
**EMPTY_STATUS,
|
||||
"committed": {"added": [dataset_root]},
|
||||
"not_in_remote": [dataset_root],
|
||||
"git": M.dict(),
|
||||
}
|
||||
assert dvc.data_status(granular=True, not_in_remote=True) == {
|
||||
**EMPTY_STATUS,
|
||||
"committed": {"added": M.unordered(*files)},
|
||||
"not_in_remote": M.unordered(*files),
|
||||
"git": M.dict(),
|
||||
}
|
||||
|
||||
assert dvc.push() == 3
|
||||
|
||||
assert dvc.data_status(not_in_remote=True, remote_refresh=True) == {
|
||||
**EMPTY_STATUS,
|
||||
"committed": {"added": [dataset_root]},
|
||||
"git": M.dict(),
|
||||
}
|
||||
assert dvc.data_status(granular=True, not_in_remote=True, remote_refresh=True) == {
|
||||
**EMPTY_STATUS,
|
||||
"committed": {"added": M.unordered(*files)},
|
||||
"git": M.dict(),
|
||||
}
|
||||
|
||||
@@ -101,7 +101,7 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
|
||||
|
||||
def test_relative_remote(erepo_dir, tmp_dir):
|
||||
# these steps reproduce the script on this issue:
|
||||
# https://github.com/iterative/dvc/issues/2756
|
||||
# https://github.com/treeverse/dvc/issues/2756
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("file", "contents", commit="create file")
|
||||
|
||||
|
||||
@@ -160,7 +160,7 @@ def test_non_cached_output(tmp_dir, erepo_dir):
|
||||
assert (tmp_dir / dst).read_text().strip() == "hello"
|
||||
|
||||
|
||||
# https://github.com/iterative/dvc/pull/2837#discussion_r352123053
|
||||
# https://github.com/treeverse/dvc/pull/2837#discussion_r352123053
|
||||
def test_absolute_file_outside_repo(tmp_dir, erepo_dir):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
Repo.get(os.fspath(erepo_dir), "/root/")
|
||||
|
||||
@@ -405,7 +405,7 @@ def test_import_non_existing(erepo_dir, tmp_dir, dvc):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
tmp_dir.dvc.imp(os.fspath(erepo_dir), "invalid_output")
|
||||
|
||||
# https://github.com/iterative/dvc/pull/2837#discussion_r352123053
|
||||
# https://github.com/treeverse/dvc/pull/2837#discussion_r352123053
|
||||
with pytest.raises(FileNotFoundError):
|
||||
tmp_dir.dvc.imp(os.fspath(erepo_dir), "/root/", "root")
|
||||
|
||||
@@ -628,7 +628,7 @@ def test_chained_import(tmp_dir, dvc, make_tmp_dir, erepo_dir, local_cloud):
|
||||
assert (dst / "bar").read_text() == "bar"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("paths", ([], ["dir"]))
|
||||
@pytest.mark.parametrize("paths", [[], ["dir"]])
|
||||
def test_parameterized_repo(tmp_dir, dvc, scm, erepo_dir, paths):
|
||||
path = erepo_dir.joinpath(*paths)
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -38,7 +38,7 @@ def load_data(file, output_format):
|
||||
return pd.read_csv(file)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output_format", ("csv", "json"))
|
||||
@pytest.mark.parametrize("output_format", ["csv", "json"])
|
||||
@pytest.mark.parametrize(
|
||||
"args,file_name",
|
||||
[
|
||||
|
||||
@@ -109,7 +109,7 @@ def test_gen_dvcignore(tmp_dir):
|
||||
|
||||
|
||||
def test_init_when_ignored_by_git(tmp_dir, scm, caplog):
|
||||
# https://github.com/iterative/dvc/issues/3738
|
||||
# https://github.com/treeverse/dvc/issues/3738
|
||||
tmp_dir.gen({".gitignore": ".*"})
|
||||
with caplog.at_level(logging.ERROR, logger="dvc"):
|
||||
assert main(["init"]) == 1
|
||||
|
||||
+201
-4
@@ -6,14 +6,20 @@ import pytest
|
||||
|
||||
from dvc.cli import main
|
||||
from dvc.exceptions import MoveNotDataSourceError, OutputNotFoundError
|
||||
from dvc.stage.exceptions import StageFileAlreadyExistsError
|
||||
|
||||
|
||||
def test_move(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
dvc.move("foo", "foo1")
|
||||
def test_move(tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen("foo", "bar")
|
||||
assert (tmp_dir / "foo.dvc").exists()
|
||||
dvc.move("foo", "bar")
|
||||
|
||||
assert not (tmp_dir / "foo.dvc").exists()
|
||||
assert (tmp_dir / "bar.dvc").exists()
|
||||
assert not (tmp_dir / "foo").is_file()
|
||||
assert (tmp_dir / "foo1").is_file()
|
||||
assert (tmp_dir / "bar").is_file()
|
||||
# should only have the new path in the .gitignore, and only once
|
||||
assert (tmp_dir / ".gitignore").read_text().splitlines() == ["/bar"]
|
||||
|
||||
|
||||
def test_move_non_existent_file(dvc):
|
||||
@@ -21,6 +27,24 @@ def test_move_non_existent_file(dvc):
|
||||
dvc.move("non_existent_file", "dst")
|
||||
|
||||
|
||||
def test_move_missing_file(tmp_dir, dvc, scm, caplog):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
(tmp_dir / "foo").unlink()
|
||||
contents = (tmp_dir / "foo.dvc").parse()
|
||||
dvc.move("foo", "bar")
|
||||
|
||||
assert not (tmp_dir / "foo.dvc").exists()
|
||||
# only the path should be changed in the dvc file
|
||||
contents["outs"][0]["path"] = "bar"
|
||||
assert contents == (tmp_dir / "bar.dvc").parse()
|
||||
|
||||
# file should not be checked out
|
||||
assert not (tmp_dir / "foo").is_file()
|
||||
assert not (tmp_dir / "bar").is_file()
|
||||
# should only have the new path in the .gitignore, and only once
|
||||
assert (tmp_dir / ".gitignore").read_text().splitlines() == ["/bar"]
|
||||
|
||||
|
||||
def test_move_directory(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen("data", {"foo": "foo", "bar": "bar"})
|
||||
dvc.move("data", "dst")
|
||||
@@ -195,3 +219,176 @@ def test_move_meta(tmp_dir, dvc):
|
||||
custom_key: 42
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def test_import(tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen("foo", "foo", commit="add foo")
|
||||
imp_stage = dvc.imp(os.curdir, "foo", "foo_imported")
|
||||
|
||||
dvc.move("foo_imported", "foo_moved")
|
||||
|
||||
(stage,) = dvc.stage.collect("foo_moved.dvc")
|
||||
assert imp_stage.md5 != stage.md5
|
||||
res = (tmp_dir / "foo_moved.dvc").read_text()
|
||||
assert res == textwrap.dedent(
|
||||
f"""\
|
||||
md5: {stage.md5}
|
||||
frozen: true
|
||||
deps:
|
||||
- path: foo
|
||||
repo:
|
||||
url: {os.curdir}
|
||||
rev_lock: {scm.get_rev()}
|
||||
outs:
|
||||
- md5: acbd18db4cc2f85cedef654fccc4a4d8
|
||||
size: 3
|
||||
hash: md5
|
||||
path: foo_moved
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path_func",
|
||||
[pytest.param(os.path.abspath, id="abs"), pytest.param(os.path.relpath, id="rel")],
|
||||
)
|
||||
def test_import_url_in_repo(tmp_dir, dvc, path_func):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
imp_stage = dvc.imp_url(path_func(tmp_dir / "foo"), "foo_imported")
|
||||
(tmp_dir / "data").mkdir()
|
||||
|
||||
dvc.move("foo_imported", os.path.join("data", "foo_moved"))
|
||||
|
||||
(stage,) = dvc.stage.collect(os.path.join("data", "foo_moved.dvc"))
|
||||
assert imp_stage.md5 != stage.md5
|
||||
res = (tmp_dir / "data" / "foo_moved.dvc").read_text()
|
||||
assert res == textwrap.dedent(
|
||||
f"""\
|
||||
md5: {stage.md5}
|
||||
frozen: true
|
||||
deps:
|
||||
- md5: acbd18db4cc2f85cedef654fccc4a4d8
|
||||
size: 3
|
||||
hash: md5
|
||||
path: ../foo
|
||||
outs:
|
||||
- md5: acbd18db4cc2f85cedef654fccc4a4d8
|
||||
size: 3
|
||||
hash: md5
|
||||
path: foo_moved
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path_func",
|
||||
[pytest.param(os.path.abspath, id="abs"), pytest.param(os.path.relpath, id="rel")],
|
||||
)
|
||||
def test_import_url_out_of_repo(tmp_dir, dvc, scm, path_func, make_tmp_dir):
|
||||
external = make_tmp_dir("external")
|
||||
external.gen("foo", "foo")
|
||||
|
||||
imp_stage = dvc.imp_url(path_func(external / "foo"), "foo_imported")
|
||||
|
||||
data_dir = tmp_dir / "data"
|
||||
data_dir.mkdir()
|
||||
|
||||
new_path = data_dir / "foo_moved"
|
||||
new_dvcfile = new_path.with_suffix(".dvc")
|
||||
dvc.move("foo_imported", os.fspath(new_path))
|
||||
|
||||
(stage,) = dvc.stage.collect(os.fspath(new_dvcfile))
|
||||
assert imp_stage.md5 != stage.md5
|
||||
|
||||
with data_dir.chdir():
|
||||
expected_path = path_func(external / "foo")
|
||||
|
||||
assert new_dvcfile.parse() == {
|
||||
"md5": stage.md5,
|
||||
"frozen": True,
|
||||
"deps": [
|
||||
{
|
||||
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
|
||||
"size": 3,
|
||||
"hash": "md5",
|
||||
"path": expected_path,
|
||||
}
|
||||
],
|
||||
"outs": [
|
||||
{
|
||||
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
|
||||
"size": 3,
|
||||
"hash": "md5",
|
||||
"path": "foo_moved",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path_func",
|
||||
[pytest.param(os.path.abspath, id="abs"), pytest.param(os.path.relpath, id="rel")],
|
||||
)
|
||||
def test_all_metadata_are_preserved(tmp_dir, dvc, make_tmp_dir, path_func):
|
||||
external = make_tmp_dir("external")
|
||||
external.gen("foo", "foo")
|
||||
|
||||
contents = {
|
||||
"md5": "bad", # placeholder, does not matter for the test
|
||||
"frozen": True,
|
||||
"desc": "this is a stage description",
|
||||
"always_changed": True,
|
||||
"meta": {"custom_key": 42},
|
||||
"deps": [
|
||||
{
|
||||
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
|
||||
"size": 3,
|
||||
"hash": "md5",
|
||||
"path": path_func(external / "foo"),
|
||||
}
|
||||
],
|
||||
"outs": [
|
||||
{
|
||||
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
|
||||
"path": "foo_imported",
|
||||
"persist": True,
|
||||
"hash": "md5",
|
||||
"size": 3,
|
||||
"desc": "this is a description",
|
||||
"type": "model",
|
||||
"labels": ["label1", "label2"],
|
||||
"meta": {"custom_key": 42},
|
||||
"cache": False,
|
||||
"remote": "myremote",
|
||||
"push": False,
|
||||
}
|
||||
],
|
||||
}
|
||||
(tmp_dir / "foo_imported.dvc").dump(contents)
|
||||
(tmp_dir / "foo_imported").write_text("foo")
|
||||
|
||||
data_dir = tmp_dir / "data"
|
||||
data_dir.mkdir()
|
||||
|
||||
new_path = data_dir / "foo_moved"
|
||||
new_dvcfile = new_path.with_suffix(".dvc")
|
||||
dvc.move("foo_imported", os.fspath(new_path))
|
||||
|
||||
(stage,) = dvc.stage.collect(os.fspath(new_dvcfile))
|
||||
|
||||
with data_dir.chdir():
|
||||
expected_path = path_func(external / "foo")
|
||||
|
||||
contents["outs"][0] |= {"path": "foo_moved"}
|
||||
contents["deps"][0] |= {"path": expected_path}
|
||||
contents |= {"md5": stage.md5}
|
||||
assert new_dvcfile.parse() == contents
|
||||
|
||||
|
||||
def test_move_dst_stage_file_already_exists(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
|
||||
|
||||
with pytest.raises(StageFileAlreadyExistsError) as exc_info:
|
||||
dvc.move("foo", "bar")
|
||||
assert str(exc_info.value) == "'bar.dvc' already exists"
|
||||
assert exc_info.value.__cause__ is None
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
import re
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -37,7 +38,7 @@ def test_remove_file_target(tmp_dir, dvc):
|
||||
|
||||
with pytest.raises(
|
||||
StageFileIsNotDvcFileError,
|
||||
match="'foo' is not a .dvc file. Do you mean 'foo.dvc'?",
|
||||
match=re.escape("'foo' is not a .dvc file. Do you mean 'foo.dvc'?"),
|
||||
):
|
||||
dvc.remove("foo")
|
||||
|
||||
|
||||
@@ -125,7 +125,7 @@ def assert_index_equal(first, second, strict=True, ordered=True):
|
||||
|
||||
|
||||
def test_skip_graph_checks(dvc, mocker):
|
||||
# See https://github.com/iterative/dvc/issues/2671 for more info
|
||||
# See https://github.com/treeverse/dvc/issues/2671 for more info
|
||||
mock_build_graph = mocker.spy(Index.graph, "fget")
|
||||
|
||||
# sanity check
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
import re
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -228,7 +229,7 @@ def test_parent_repo_collect_stages(tmp_dir, scm, dvc):
|
||||
assert deep_subrepo_stages != []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("with_deps", (False, True))
|
||||
@pytest.mark.parametrize("with_deps", [False, True])
|
||||
def test_collect_symlink(tmp_dir, dvc, with_deps):
|
||||
from dvc.exceptions import StageNotFoundError
|
||||
|
||||
@@ -339,6 +340,8 @@ def test_stage_add_duplicated_output(tmp_dir, dvc):
|
||||
|
||||
with pytest.raises(
|
||||
OutputDuplicationError,
|
||||
match="Use `dvc remove foo.dvc` to stop tracking the overlapping output.",
|
||||
match=re.escape(
|
||||
"Use `dvc remove foo.dvc` to stop tracking the overlapping output."
|
||||
),
|
||||
):
|
||||
dvc.stage.add(name="duplicated", cmd="echo bar > foo", outs=["foo"])
|
||||
|
||||
@@ -156,7 +156,7 @@ def test_update_unchanged(tmp_dir, dvc, erepo_dir, mocker):
|
||||
assert not spy.called
|
||||
|
||||
|
||||
@pytest.mark.parametrize("outs_exist", (False, True))
|
||||
@pytest.mark.parametrize("outs_exist", [False, True])
|
||||
def test_update_no_download(tmp_dir, dvc, erepo_dir, outs_exist, mocker):
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("file", "file content", commit="add file")
|
||||
@@ -204,7 +204,7 @@ def test_update_import_url(tmp_dir, dvc, workspace):
|
||||
assert dst.read_text() == "updated file content"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("outs_exist", (False, True))
|
||||
@pytest.mark.parametrize("outs_exist", [False, True])
|
||||
def test_update_import_url_no_download(tmp_dir, dvc, workspace, outs_exist, mocker):
|
||||
workspace.gen("file", "file content")
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import re
|
||||
from contextlib import nullcontext as does_not_raise
|
||||
|
||||
import pytest
|
||||
@@ -214,7 +215,9 @@ def hydra_setup_dir_basic(tmp_dir, config_subdir, config_name, config_content):
|
||||
None,
|
||||
pytest.raises(
|
||||
ValueError,
|
||||
match="Either `config_dir` or `config_module` should be provided.",
|
||||
match=re.escape(
|
||||
"Either `config_dir` or `config_module` should be provided."
|
||||
),
|
||||
),
|
||||
),
|
||||
],
|
||||
@@ -238,7 +241,7 @@ def test_compose_and_dump_dir_module(
|
||||
|
||||
|
||||
def test_compose_and_dump_yaml_handles_string(tmp_dir):
|
||||
"""Regression test for https://github.com/iterative/dvc/issues/8583"""
|
||||
"""Regression test for https://github.com/treeverse/dvc/issues/8583"""
|
||||
from dvc.utils.hydra import compose_and_dump
|
||||
|
||||
config = tmp_dir / "conf" / "config.yaml"
|
||||
@@ -250,7 +253,7 @@ def test_compose_and_dump_yaml_handles_string(tmp_dir):
|
||||
|
||||
|
||||
def test_compose_and_dump_resolves_interpolation(tmp_dir):
|
||||
"""Regression test for https://github.com/iterative/dvc/issues/9196"""
|
||||
"""Regression test for https://github.com/treeverse/dvc/issues/9196"""
|
||||
from dvc.utils.hydra import compose_and_dump
|
||||
|
||||
config = tmp_dir / "conf" / "config.yaml"
|
||||
|
||||
@@ -59,7 +59,7 @@ def test_remote_missing_deps_are_correctly_reported(tmp_dir, caplog, mocker, pkg
|
||||
else:
|
||||
expected += (
|
||||
"\nPlease report this bug to "
|
||||
"<https://github.com/iterative/dvc/issues>. "
|
||||
"<https://github.com/treeverse/dvc/issues>. "
|
||||
"Thank you!"
|
||||
)
|
||||
assert expected in caplog.text
|
||||
|
||||
@@ -119,7 +119,7 @@ def test_show_ascii(repo):
|
||||
def test_show_dot(repo):
|
||||
# dot file rendering is not deterministic though graph
|
||||
# output doesn't depend upon order of lines. Use sorted values
|
||||
# https://github.com/iterative/dvc/pull/7725
|
||||
# https://github.com/treeverse/dvc/pull/7725
|
||||
expected = [
|
||||
"\"stage: '1'\";",
|
||||
"\"stage: '2'\";",
|
||||
|
||||
@@ -206,20 +206,20 @@ def test_diff_show_markdown_and_hash(mocker, show_hash, dvc):
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"opts",
|
||||
(
|
||||
[
|
||||
[],
|
||||
["a_rev", "b_rev"],
|
||||
["--targets", "."],
|
||||
["--hide-missing"],
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"show, expected",
|
||||
(
|
||||
[
|
||||
([], ""),
|
||||
(["--json"], "{}"),
|
||||
(["--md"], "| Status | Path |\n|----------|--------|"),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_no_changes(mocker, capsys, opts, show, expected, dvc):
|
||||
args = parse_args(["diff", *opts, *show])
|
||||
|
||||
@@ -130,6 +130,7 @@ def test_experiments_run(dvc, scm, mocker):
|
||||
"tmp_dir": False,
|
||||
"copy_paths": [],
|
||||
"message": None,
|
||||
"no_hydra": False,
|
||||
}
|
||||
default_arguments.update(repro_arguments)
|
||||
|
||||
@@ -151,6 +152,7 @@ def test_experiments_run_message(dvc, scm, mocker, flag):
|
||||
"tmp_dir": False,
|
||||
"copy_paths": [],
|
||||
"message": "mymessage",
|
||||
"no_hydra": False,
|
||||
}
|
||||
default_arguments.update(repro_arguments)
|
||||
|
||||
|
||||
@@ -260,7 +260,7 @@ def test_should_pass_template_dir(tmp_dir, dvc, mocker, capsys):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output", ("some_out", os.path.join("to", "subdir"), None))
|
||||
@pytest.mark.parametrize("output", ["some_out", os.path.join("to", "subdir"), None])
|
||||
def test_should_call_render(tmp_dir, mocker, capsys, plots_data, output):
|
||||
cli_args = parse_args(["plots", "diff", "--targets", "plots.csv", "--out", output])
|
||||
cmd = cli_args.func(cli_args)
|
||||
@@ -322,7 +322,7 @@ def test_plots_diff_json(dvc, mocker, capsys):
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target,expected_out,expected_rtn",
|
||||
(("t1", "\"{'t1'}\"", 0), (None, "t1\nt2", 0), ("t3", "", 1)),
|
||||
[("t1", "\"{'t1'}\"", 0), (None, "t1\nt2", 0), ("t3", "", 1)],
|
||||
)
|
||||
def test_plots_templates(dvc, mocker, capsys, target, expected_out, expected_rtn):
|
||||
t1 = mocker.Mock()
|
||||
@@ -352,7 +352,7 @@ def test_plots_templates(dvc, mocker, capsys, target, expected_out, expected_rtn
|
||||
assert rtn == expected_rtn
|
||||
|
||||
|
||||
@pytest.mark.parametrize("split", (True, False))
|
||||
@pytest.mark.parametrize("split", [True, False])
|
||||
def test_show_json(split, mocker, capsys):
|
||||
import dvc.commands.plots
|
||||
|
||||
|
||||
@@ -214,7 +214,7 @@ def test_read_params_py(tmp_dir, dvc):
|
||||
|
||||
|
||||
def test_params_py_tuple_status(tmp_dir, dvc):
|
||||
"""https://github.com/iterative/dvc/issues/8803"""
|
||||
"""https://github.com/treeverse/dvc/issues/8803"""
|
||||
parameters_file = "parameters.py"
|
||||
tmp_dir.gen(parameters_file, "TUPLE = (10, 100)\n")
|
||||
dep = ParamsDependency(Stage(dvc), parameters_file, ["TUPLE"])
|
||||
|
||||
@@ -42,7 +42,7 @@ def test_save_missing(dvc, mocker):
|
||||
), # hdfs checksum
|
||||
# Not much we can do about hex and oct values without writing our own
|
||||
# parser. So listing these test cases just to acknowledge this.
|
||||
# See https://github.com/iterative/dvc/issues/3331.
|
||||
# See https://github.com/treeverse/dvc/issues/3331.
|
||||
(0x3451, "13393"),
|
||||
(0o1244, "676"),
|
||||
],
|
||||
|
||||
@@ -153,3 +153,31 @@ def test_remote_with_jobs(dvc, base_url, fs_cls):
|
||||
assert config["user"] == "admin"
|
||||
assert f"{scheme}://{user}@example.com" in config["host"]
|
||||
assert cls is fs_cls
|
||||
|
||||
|
||||
def test_bearer_token_command(mocker):
|
||||
mock_auth_obj = mocker.Mock()
|
||||
mock_get_auth = mocker.patch(
|
||||
"dvc_webdav.get_bearer_auth", return_value=mock_auth_obj
|
||||
)
|
||||
|
||||
command = "my-token-cmd.sh"
|
||||
timeout = 10
|
||||
|
||||
# Configuration with potential conflicts
|
||||
config = {
|
||||
"url": url,
|
||||
"token": "static-token-value",
|
||||
"user": "conflict-user",
|
||||
"password": "conflict-password",
|
||||
"bearer_token_command": command,
|
||||
"timeout": timeout,
|
||||
}
|
||||
|
||||
fs = WebDAVFileSystem(**config)
|
||||
|
||||
# the factory function was invoked with the correct arguments from the config.
|
||||
mock_get_auth.assert_called_once_with(command, timeout)
|
||||
|
||||
# the generated auth object is correctly injected into `fs_args`.
|
||||
assert fs.fs_args["auth"] == mock_auth_obj
|
||||
|
||||
@@ -69,7 +69,7 @@ def session_app(tmp_path_factory) -> FSApp:
|
||||
"""Session scoped experiments queue celery app."""
|
||||
from kombu.transport.filesystem import Channel
|
||||
|
||||
# related to https://github.com/iterative/dvc-task/issues/61
|
||||
# related to https://github.com/treeverse/dvc-task/issues/61
|
||||
Channel.QoS.restore_at_shutdown = False
|
||||
|
||||
from dvc_task.app import FSApp
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user