mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 23:09:28 +01:00
Compare commits
27
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f342cc7db5 | ||
|
|
0eb279ea35 | ||
|
|
d086bbbd3a | ||
|
|
046d2fd72e | ||
|
|
cb96cf1487 | ||
|
|
cf5168af74 | ||
|
|
dc123102cc | ||
|
|
438259cc70 | ||
|
|
7b978fec5b | ||
|
|
9f901e57a2 | ||
|
|
ab5ac17925 | ||
|
|
72abd013cf | ||
|
|
fa5c33ed24 | ||
|
|
9dda30b7a0 | ||
|
|
19c146e392 | ||
|
|
749c22bbcb | ||
|
|
19a0bf48f1 | ||
|
|
bda614baa8 | ||
|
|
d6fa8b67ff | ||
|
|
498a4d5d3a | ||
|
|
b74711d5fe | ||
|
|
61d5de54e5 | ||
|
|
ca7454980d | ||
|
|
8912ca7d2b | ||
|
|
f13ae75a45 | ||
|
|
aa090ecf8d | ||
|
|
0c4a5cbbb7 |
@@ -737,8 +737,8 @@
|
||||
"tests/func/test_config.py::test_config_gdrive_fields": 0.12703910000016094,
|
||||
"tests/func/test_config.py::test_config_get[args0-0-False]": 0.20948869999995168,
|
||||
"tests/func/test_config.py::test_config_get[args1-0-myremote]": 0.1829517000001033,
|
||||
"tests/func/test_config.py::test_config_get[args2-0-iterative]": 0.19686669999998685,
|
||||
"tests/func/test_config.py::test_config_get[args3-0-iterative]": 0.1675059000000374,
|
||||
"tests/func/test_config.py::test_config_get[args2-0-treeverse]": 0.19686669999998685,
|
||||
"tests/func/test_config.py::test_config_get[args3-0-treeverse]": 0.1675059000000374,
|
||||
"tests/func/test_config.py::test_config_get[args4-251-option 'profile' doesn't exist]": 0.18096200000002227,
|
||||
"tests/func/test_config.py::test_config_get[args5-0-gs://bucket/path]": 0.12368970000022728,
|
||||
"tests/func/test_config.py::test_config_get[args6-0-gs://bucket/path]": 0.13007389999995667,
|
||||
@@ -2642,7 +2642,7 @@
|
||||
"tests/unit/test_updater.py::test_is_enabled[config0-True]": 0.13130560000013247,
|
||||
"tests/unit/test_updater.py::test_is_enabled[config1-True]": 0.14342600000009043,
|
||||
"tests/unit/test_updater.py::test_is_enabled[config2-False]": 0.13593219999984285,
|
||||
"tests/unit/test_updater.py::test_notify_message[None-Find the latest release at https://github.com/iterative/dvc/releases/latest.]": 0.014024399999925663,
|
||||
"tests/unit/test_updater.py::test_notify_message[None-Find the latest release at https://github.com/treeverse/dvc/releases/latest.]": 0.014024399999925663,
|
||||
"tests/unit/test_updater.py::test_notify_message[binary-To upgrade, uninstall dvc and reinstall from https://dvc.org.]": 0.013217200000326557,
|
||||
"tests/unit/test_updater.py::test_notify_message[brew-To upgrade, run 'brew upgrade dvc'.]": 0.013467300000229443,
|
||||
"tests/unit/test_updater.py::test_notify_message[choco-To upgrade, run 'choco upgrade dvc'.]": 0.014645900000004985,
|
||||
@@ -2652,7 +2652,7 @@
|
||||
"tests/unit/test_updater.py::test_notify_message[osxpkg-To upgrade, uninstall dvc and reinstall from https://dvc.org.]": 0.013980499999888707,
|
||||
"tests/unit/test_updater.py::test_notify_message[pip-To upgrade, run 'pip install --upgrade dvc'.]": 0.015155299999833005,
|
||||
"tests/unit/test_updater.py::test_notify_message[rpm-To upgrade, run 'yum update dvc'.]": 0.014036500000202068,
|
||||
"tests/unit/test_updater.py::test_notify_message[unknown-Find the latest release at https://github.com/iterative/dvc/releases/latest.]": 0.01609110000003966,
|
||||
"tests/unit/test_updater.py::test_notify_message[unknown-Find the latest release at https://github.com/treeverse/dvc/releases/latest.]": 0.01609110000003966,
|
||||
"tests/unit/ui/test_console.py::test_capsys_works": 0.003642499999841675,
|
||||
"tests/unit/ui/test_console.py::test_write": 0.004320600000255581,
|
||||
"tests/unit/ui/test_console.py::test_write_json[False-{\"hello\": \"world\", \"date\": \"1970-01-01 00:00:00\"}\\n]": 0.004433300000073359,
|
||||
|
||||
@@ -57,7 +57,7 @@ $ dvc doctor
|
||||
**Additional Information (if any):**
|
||||
|
||||
<!--
|
||||
Please check https://github.com/iterative/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.
|
||||
Please check https://github.com/treeverse/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.
|
||||
|
||||
If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.
|
||||
If the issue is regarding the performance, please attach the profiling information and the benchmark comparisons.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
* [ ] ❗ I have followed the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) checklist.
|
||||
|
||||
* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here.
|
||||
* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/treeverse/dvc.org) and linked it here.
|
||||
|
||||
Thank you for the contribution - we'll try to review it as soon as possible. 🙏
|
||||
|
||||
@@ -16,14 +16,14 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.13"
|
||||
python-version: "3.14"
|
||||
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: pyproject.toml
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
- name: run benchmarks on base branch
|
||||
run: pytest --benchmark-autosave dvc/testing/benchmarks/ -k 'test_init or test_help'
|
||||
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
clean: false
|
||||
|
||||
@@ -19,15 +19,15 @@ jobs:
|
||||
name: Build & Verify package
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: 3.13
|
||||
python-version: "3.14"
|
||||
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
|
||||
- name: Omit local version for Test PyPI upload
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
- name: Check dist
|
||||
run: uv tool run twine check --strict dist/*
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
- uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: Packages
|
||||
path: dist/
|
||||
@@ -51,7 +51,7 @@ jobs:
|
||||
name: Publish dev package to test.pypi.org
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
if: ${{ github.event.action == 'published' || (github.repository == 'iterative/dvc' && github.event_name == 'push' && github.ref == 'refs/heads/main') }}
|
||||
if: ${{ github.event.action == 'published' || (github.repository == 'treeverse/dvc' && github.event_name == 'push' && github.ref == 'refs/heads/main') }}
|
||||
|
||||
environment:
|
||||
name: test-pypi
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- uses: actions/download-artifact@v5
|
||||
- uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: Packages
|
||||
path: dist
|
||||
@@ -86,7 +86,7 @@ jobs:
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- uses: actions/download-artifact@v5
|
||||
- uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: Packages
|
||||
path: dist
|
||||
|
||||
@@ -28,13 +28,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v5
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v3
|
||||
uses: github/codeql-action/init@v4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
queries: security-extended
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v3
|
||||
uses: github/codeql-action/analyze@v4
|
||||
|
||||
@@ -19,17 +19,17 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest]
|
||||
pyv: ["3.13"]
|
||||
pyv: ["3.14"]
|
||||
plugin: ["dvc-s3"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
path: dvc
|
||||
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
repository: iterative/${{ matrix.plugin }}
|
||||
repository: treeverse/${{ matrix.plugin }}
|
||||
ref: main
|
||||
path: ${{ matrix.plugin }}
|
||||
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
with:
|
||||
python-version: ${{ matrix.pyv }}
|
||||
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: "**/pyproject.toml"
|
||||
|
||||
@@ -26,14 +26,14 @@ jobs:
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.13'
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
python-version: '3.14'
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: pyproject.toml
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest]
|
||||
pyv: ["3.9", "3.10", "3.11", "3.12", "3.13"]
|
||||
pyv: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
|
||||
include:
|
||||
- os: windows-latest
|
||||
pyv: "3.9"
|
||||
@@ -117,15 +117,27 @@ jobs:
|
||||
- os: windows-latest
|
||||
pyv: "3.13"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
|
||||
- os: windows-latest
|
||||
pyv: "3.14"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
|
||||
- os: windows-latest
|
||||
pyv: "3.14"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
|
||||
- os: windows-latest
|
||||
pyv: "3.14"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
|
||||
- os: windows-latest
|
||||
pyv: "3.14"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.pyv }}
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: pyproject.toml
|
||||
|
||||
@@ -22,7 +22,7 @@ repos:
|
||||
- id: sort-simple-yaml
|
||||
- id: trailing-whitespace
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: 'v0.13.2'
|
||||
rev: 'v0.14.7'
|
||||
hooks:
|
||||
- id: ruff-check
|
||||
args: [--fix, --exit-non-zero-on-fix]
|
||||
|
||||
+1
-1
@@ -4,5 +4,5 @@
|
||||
"data-science", "data-version-control", "machine-learning", "git",
|
||||
"developer-tools", "reproducibility", "collaboration", "ai", "python"],
|
||||
"contributors": [
|
||||
{"name": "DVC team", "type": "Other", "affiliation": "Iterative"}]
|
||||
{"name": "DVC team", "type": "Other", "affiliation": "Treeverse"}]
|
||||
}
|
||||
|
||||
+2
-2
@@ -6,8 +6,8 @@ message: >-
|
||||
type: software
|
||||
authors:
|
||||
- name: The DVC team and contributors
|
||||
website: 'https://github.com/iterative'
|
||||
repository-code: 'https://github.com/iterative/dvc'
|
||||
website: 'https://github.com/treeverse'
|
||||
repository-code: 'https://github.com/treeverse/dvc'
|
||||
url: 'https://dvc.org'
|
||||
abstract: >-
|
||||
DVC is a tool for data science that takes advantage of existing software engineering toolset.
|
||||
|
||||
@@ -186,7 +186,7 @@
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2017-2021 Iterative, Inc.
|
||||
Copyright 2025 Treeverse.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
+9
-15
@@ -167,7 +167,7 @@ To install the development version, run:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pip install git+git://github.com/iterative/dvc
|
||||
pip install "dvc @ git+https://github.com/treeverse/dvc"
|
||||
|
||||
Package (Platform-specific)
|
||||
---------------------------
|
||||
@@ -175,7 +175,7 @@ Package (Platform-specific)
|
||||
|Packages|
|
||||
|
||||
Self-contained packages for Linux, Windows, and Mac are available.
|
||||
The latest version of the packages can be found on the GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
|
||||
The latest version of the packages can be found on the GitHub `releases page <https://github.com/treeverse/dvc/releases>`_.
|
||||
|
||||
Ubuntu / Debian (deb)
|
||||
^^^^^^^^^^^^^^^^^^^^^
|
||||
@@ -198,8 +198,6 @@ Fedora / CentOS (rpm)
|
||||
Contributing
|
||||
============
|
||||
|
||||
|Maintainability|
|
||||
|
||||
Contributions are welcome!
|
||||
Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more details.
|
||||
Thanks to all our contributors!
|
||||
@@ -237,23 +235,19 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
|
||||
:target: https://dvc.org
|
||||
:alt: DVC logo
|
||||
|
||||
.. |VS Code Extension Overview| image:: https://raw.githubusercontent.com/iterative/vscode-dvc/main/extension/docs/overview.gif
|
||||
.. |VS Code Extension Overview| image:: https://raw.githubusercontent.com/treeverse/vscode-dvc/main/extension/docs/overview.gif
|
||||
:alt: DVC Extension for VS Code
|
||||
|
||||
.. |CI| image:: https://github.com/iterative/dvc/actions/workflows/tests.yaml/badge.svg
|
||||
:target: https://github.com/iterative/dvc/actions/workflows/tests.yaml
|
||||
.. |CI| image:: https://github.com/treeverse/dvc/actions/workflows/tests.yaml/badge.svg
|
||||
:target: https://github.com/treeverse/dvc/actions/workflows/tests.yaml
|
||||
:alt: GHA Tests
|
||||
|
||||
.. |Maintainability| image:: https://img.shields.io/codeclimate/maintainability/iterative/dvc
|
||||
:target: https://codeclimate.com/github/iterative/dvc
|
||||
:alt: Code Climate
|
||||
|
||||
.. |Python Version| image:: https://img.shields.io/pypi/pyversions/dvc
|
||||
:target: https://pypi.org/project/dvc
|
||||
:alt: Python Version
|
||||
|
||||
.. |Coverage| image:: https://codecov.io/gh/iterative/dvc/branch/main/graph/badge.svg
|
||||
:target: https://codecov.io/gh/iterative/dvc
|
||||
.. |Coverage| image:: https://codecov.io/gh/treeverse/dvc/branch/main/graph/badge.svg
|
||||
:target: https://codecov.io/gh/treeverse/dvc
|
||||
:alt: Codecov
|
||||
|
||||
.. |Snap| image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
|
||||
@@ -292,8 +286,8 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
|
||||
:target: https://dvc.org/img/flow.gif
|
||||
:alt: how_dvc_works
|
||||
|
||||
.. |Contribs| image:: https://contrib.rocks/image?repo=iterative/dvc
|
||||
:target: https://github.com/iterative/dvc/graphs/contributors
|
||||
.. |Contribs| image:: https://contrib.rocks/image?repo=treeverse/dvc
|
||||
:target: https://github.com/treeverse/dvc/graphs/contributors
|
||||
:alt: Contributors
|
||||
|
||||
.. |VS Code| image:: https://img.shields.io/visual-studio-marketplace/v/Iterative.dvc?color=blue&label=VSCode&logo=visualstudiocode&logoColor=blue
|
||||
|
||||
+1
-1
@@ -206,7 +206,7 @@ def add_debugging_flags(parser):
|
||||
from argparse import SUPPRESS
|
||||
|
||||
# For detailed info see:
|
||||
# https://github.com/iterative/dvc/wiki/Debugging,-Profiling-and-Benchmarking-DVC
|
||||
# https://github.com/treeverse/dvc/wiki/Debugging,-Profiling-and-Benchmarking-DVC
|
||||
args, _ = parser.parse_known_args()
|
||||
verbose = args.verbose
|
||||
|
||||
|
||||
+1
-1
@@ -96,7 +96,7 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
|
||||
f"See {link} for more info."
|
||||
)
|
||||
else:
|
||||
link = format_link("https://github.com/iterative/dvc/issues")
|
||||
link = format_link("https://github.com/treeverse/dvc/issues")
|
||||
hint = f"\nPlease report this bug to {link}. Thank you!"
|
||||
|
||||
logger.exception(
|
||||
|
||||
+6
-130
@@ -31,27 +31,6 @@ _dvc_compgen_remotes() {
|
||||
_dvc_compgen_config_vars() {
|
||||
compgen -W "${_dvc_config_vars[*]}" -- $1
|
||||
}
|
||||
|
||||
_dvc_compgen_remote_config_vars() {
|
||||
local cur prev words cword remote_name filtered
|
||||
_init_completion || return
|
||||
|
||||
# last non-option word before cursor
|
||||
local nonopts=()
|
||||
for w in "${words[@]:1:$((cword-1))}"; do
|
||||
[[ $w == -* ]] || nonopts+=("$w")
|
||||
done
|
||||
|
||||
remote_name="${nonopts[-1]}"
|
||||
if [[ -n $remote_name ]]; then
|
||||
local _dvc_remote_config_vars=($(
|
||||
dvc config --available-options -q 2> /dev/null \
|
||||
| grep "^remote\\.${remote_name}\\." \
|
||||
| sed "s/^remote\\.${remote_name}\\.//"
|
||||
))
|
||||
compgen -W "${_dvc_remote_config_vars[*]}" -- $1
|
||||
fi
|
||||
}
|
||||
"""
|
||||
|
||||
ZSH_PREAMBLE = """
|
||||
@@ -90,111 +69,27 @@ _dvc_compadd_remotes() {
|
||||
_describe 'remotes' "($(dvc remote list | cut -d' ' -f1))"
|
||||
}
|
||||
|
||||
_dvc_config_config_vars_for_completion() {
|
||||
dvc config --available-options -q 2> /dev/null
|
||||
}
|
||||
|
||||
_dvc_compadd_config_vars() {
|
||||
_describe 'config_vars' _dvc_config_vars
|
||||
}
|
||||
|
||||
_dvc_compadd_remote_config_vars() {
|
||||
local remote_name filtered
|
||||
|
||||
# last non-option word before cursor
|
||||
remote_name=${${words[1,CURRENT-1]##-*}[-1]}
|
||||
if [[ -n $remote_name ]]; then
|
||||
filtered=$(_dvc_config_config_vars_for_completion \\
|
||||
| grep "^remote.${remote_name}\\\\." \\
|
||||
| sed "s/^remote\\\\.${remote_name}\\\\.//"
|
||||
)
|
||||
compadd -- $=filtered
|
||||
fi
|
||||
}
|
||||
"""
|
||||
|
||||
FISH_PREAMBLE = """
|
||||
function __fish_complete_dvc_files
|
||||
__fish_complete_path | string match -re '\\*?.dvc|Dvcfile|dvc\\.yaml'
|
||||
end
|
||||
|
||||
function __fish_complete_dvc_stages
|
||||
for line in (dvc stage list -q)
|
||||
set -l parts (string split -m1 ' ' -- $line)
|
||||
set -l name $parts[1]
|
||||
set -l desc (string trim $parts[2])
|
||||
echo -e "$name $desc"
|
||||
end
|
||||
end
|
||||
|
||||
function __fish_complete_dvc_stages_and_files
|
||||
__fish_complete_dvc_stages
|
||||
__fish_complete_dvc_files
|
||||
end
|
||||
|
||||
function __fish_complete_dvc_experiments
|
||||
dvc exp list -q --all-commits --names-only
|
||||
end
|
||||
|
||||
function __fish_complete_dvc_remotes
|
||||
dvc remote list | cut -d' ' -f1
|
||||
end
|
||||
|
||||
function __fish_complete_dvc_remote_config_vars
|
||||
# last non-option word before cursor
|
||||
set -l remote_name (commandline -opc | string match -rv '^-' | tail -n 1)
|
||||
|
||||
if test -n "$remote_name"
|
||||
dvc config --available-options -q 2>/dev/null |
|
||||
string match -re "^remote\\.$remote_name\\." |
|
||||
string replace -r "^remote\\.$remote_name\\." ''
|
||||
end
|
||||
end
|
||||
"""
|
||||
|
||||
PREAMBLE = {
|
||||
"bash": BASH_PREAMBLE,
|
||||
"zsh": ZSH_PREAMBLE,
|
||||
"fish": FISH_PREAMBLE,
|
||||
}
|
||||
|
||||
FILE = shtab.FILE
|
||||
DIR = shtab.DIRECTORY
|
||||
DVC_FILE = {
|
||||
"bash": "_dvc_compgen_DVCFiles",
|
||||
"zsh": "_dvc_compadd_DVCFiles",
|
||||
"fish": '-f -a "(__fish_complete_dvc_files)"',
|
||||
}
|
||||
STAGE = {
|
||||
"bash": "_dvc_compgen_stages",
|
||||
"zsh": "_dvc_compadd_stages",
|
||||
"fish": '-f -a "(__fish_complete_dvc_stages)"',
|
||||
}
|
||||
DVC_FILE = {"bash": "_dvc_compgen_DVCFiles", "zsh": "_dvc_compadd_DVCFiles"}
|
||||
STAGE = {"bash": "_dvc_compgen_stages", "zsh": "_dvc_compadd_stages"}
|
||||
DVCFILES_AND_STAGE = {
|
||||
"bash": "_dvc_compgen_stages_and_files",
|
||||
"zsh": "_dvc_compadd_stages_and_files",
|
||||
"fish": '-f -a "(__fish_complete_dvc_stages_and_files)"',
|
||||
}
|
||||
EXPERIMENT = {
|
||||
"bash": "_dvc_compgen_exps",
|
||||
"zsh": "_dvc_compadd_exps",
|
||||
"fish": '-f -a "(__fish_complete_dvc_experiments)"',
|
||||
}
|
||||
REMOTE = {
|
||||
"bash": "_dvc_compgen_remotes",
|
||||
"zsh": "_dvc_compadd_remotes",
|
||||
"fish": '-f -a "(__fish_complete_dvc_remotes)"',
|
||||
}
|
||||
CONFIG_VARS = {
|
||||
"bash": "_dvc_compgen_config_vars",
|
||||
"zsh": "_dvc_compadd_config_vars",
|
||||
"fish": '-f -a "(__fish_complete_dvc_config_vars)"',
|
||||
}
|
||||
REMOTE_CONFIG_VARS = {
|
||||
"bash": "_dvc_compgen_remote_config_vars",
|
||||
"zsh": "_dvc_compadd_remote_config_vars",
|
||||
"fish": '-f -a "(__fish_complete_dvc_remote_config_vars)"',
|
||||
}
|
||||
EXPERIMENT = {"bash": "_dvc_compgen_exps", "zsh": "_dvc_compadd_exps"}
|
||||
REMOTE = {"bash": "_dvc_compgen_remotes", "zsh": "_dvc_compadd_remotes"}
|
||||
CONFIG_VARS = {"bash": "_dvc_compgen_config_vars", "zsh": "_dvc_compadd_config_vars"}
|
||||
|
||||
|
||||
def get_preamble() -> dict[str, str]:
|
||||
@@ -208,26 +103,7 @@ def get_preamble() -> dict[str, str]:
|
||||
_dvc_config_vars=(
|
||||
{nl.join(config_vars)}
|
||||
)
|
||||
"""
|
||||
indent = "\t\t".expandtabs(4) # 8 spaces
|
||||
lines = (
|
||||
"\n".join(
|
||||
f"{indent}{c} \\"
|
||||
for c in config_vars[:-1] # all but last
|
||||
)
|
||||
+ "\n"
|
||||
+ f"{indent}{config_vars[-1]}"
|
||||
) # last line without backslash
|
||||
config_vars_arr_fish = f"""
|
||||
function __fish_complete_dvc_config_vars
|
||||
set -l _dvc_config_vars \\
|
||||
{lines}
|
||||
printf %s\\n $_dvc_config_vars
|
||||
end
|
||||
"""
|
||||
for shell, preamble in PREAMBLE.items():
|
||||
if shell != "fish":
|
||||
ret[shell] = config_vars_arr + preamble
|
||||
else:
|
||||
ret[shell] = config_vars_arr_fish + preamble
|
||||
ret[shell] = config_vars_arr + preamble
|
||||
return ret
|
||||
|
||||
@@ -8,7 +8,7 @@ from dvc.ui import ui
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
||||
SUPPORTED_SHELLS = ["bash", "zsh", "fish"]
|
||||
SUPPORTED_SHELLS = ["bash", "zsh"]
|
||||
|
||||
|
||||
class CmdCompletion(CmdBaseNoRepo):
|
||||
|
||||
@@ -40,15 +40,6 @@ class CmdConfig(CmdBaseNoRepo):
|
||||
self.config = Config.from_cwd(validate=False)
|
||||
|
||||
def run(self):
|
||||
if self.args.available_options:
|
||||
from dvc.config_schema import contextual_config_vars_for_completion
|
||||
|
||||
ui.write(
|
||||
"\n".join(list(contextual_config_vars_for_completion(self.config))),
|
||||
force=True,
|
||||
)
|
||||
return 0
|
||||
|
||||
if self.args.show_origin and (self.args.value or self.args.unset):
|
||||
logger.error(
|
||||
"--show-origin can't be used together with any of these "
|
||||
@@ -221,9 +212,6 @@ def add_parser(subparsers, parent_parser):
|
||||
help=CONFIG_HELP,
|
||||
formatter_class=formatter.RawDescriptionHelpFormatter,
|
||||
)
|
||||
config_parser.add_argument(
|
||||
"--available-options", default=False, action="store_true"
|
||||
)
|
||||
config_parser.add_argument(
|
||||
"-u",
|
||||
"--unset",
|
||||
|
||||
@@ -20,6 +20,7 @@ class CmdExperimentsRun(CmdRepro):
|
||||
tmp_dir=self.args.tmp_dir,
|
||||
copy_paths=self.args.copy_paths,
|
||||
message=self.args.message,
|
||||
no_hydra=self.args.no_hydra,
|
||||
**self._common_kwargs,
|
||||
)
|
||||
|
||||
@@ -107,4 +108,14 @@ def _add_run_common(parser):
|
||||
default=None,
|
||||
help="Custom commit message to use when committing the experiment.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-hydra",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"Disables automatically updating `params.yaml` with Hydra configuration. "
|
||||
" You can still use `--set-param` to update individual params if needed."
|
||||
" Default is False."
|
||||
),
|
||||
)
|
||||
parser.add_argument("-M", dest="message", help=argparse.SUPPRESS) # obsolete
|
||||
|
||||
@@ -29,7 +29,7 @@ def _welcome_message():
|
||||
"{yellow}------------{nc}\n"
|
||||
f"- Check out the documentation: {fmt_link('https://dvc.org/doc')}\n"
|
||||
f"- Get help and share ideas: {fmt_link('https://dvc.org/chat')}\n"
|
||||
f"- Star us on GitHub: {fmt_link('https://github.com/iterative/dvc')}"
|
||||
f"- Star us on GitHub: {fmt_link('https://github.com/treeverse/dvc')}"
|
||||
).format(yellow=colorama.Fore.YELLOW, nc=colorama.Fore.RESET)
|
||||
|
||||
ui.write(msg)
|
||||
|
||||
@@ -233,9 +233,7 @@ def add_parser(subparsers, parent_parser):
|
||||
remote_modify_parser.add_argument(
|
||||
"name", help="Name of the remote"
|
||||
).complete = completion.REMOTE
|
||||
remote_modify_parser.add_argument(
|
||||
"option", help="Name of the option to modify."
|
||||
).complete = completion.REMOTE_CONFIG_VARS
|
||||
remote_modify_parser.add_argument("option", help="Name of the option to modify.")
|
||||
remote_modify_parser.add_argument(
|
||||
"value", nargs="?", help="(optional) Value of the option."
|
||||
)
|
||||
|
||||
+15
-34
@@ -60,27 +60,23 @@ def Choices(*choices): # noqa: N802
|
||||
return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
|
||||
|
||||
|
||||
def _get_schema_from_url(url: str) -> str:
|
||||
parsed = urlparse(url)
|
||||
# Windows absolute paths should really have scheme == "" (local)
|
||||
if os.name == "nt" and len(parsed.scheme) == 1 and not parsed.netloc:
|
||||
return ""
|
||||
if not parsed.netloc:
|
||||
return ""
|
||||
return parsed.scheme
|
||||
|
||||
|
||||
def ByUrl(mapping): # noqa: N802
|
||||
schemas: dict[str, Schema] = walk_values(Schema, mapping)
|
||||
schemas = walk_values(Schema, mapping)
|
||||
|
||||
def validate(data):
|
||||
if "url" not in data:
|
||||
raise Invalid("expected 'url'")
|
||||
|
||||
scheme = _get_schema_from_url(data["url"])
|
||||
if scheme not in schemas:
|
||||
raise Invalid(f"Unsupported URL type {scheme}://")
|
||||
return schemas[scheme](data)
|
||||
parsed = urlparse(data["url"])
|
||||
# Windows absolute paths should really have scheme == "" (local)
|
||||
if os.name == "nt" and len(parsed.scheme) == 1 and not parsed.netloc:
|
||||
return schemas[""](data)
|
||||
if not parsed.netloc:
|
||||
return schemas[""](data)
|
||||
if parsed.scheme not in schemas:
|
||||
raise Invalid(f"Unsupported URL type {parsed.scheme}://")
|
||||
|
||||
return schemas[parsed.scheme](data)
|
||||
|
||||
return validate
|
||||
|
||||
@@ -155,6 +151,7 @@ WEBDAV_COMMON = {
|
||||
"password": str,
|
||||
"ask_password": Bool,
|
||||
"token": str,
|
||||
"bearer_token_command": str,
|
||||
"custom_auth_header": str,
|
||||
"cert_path": str,
|
||||
"key_path": str,
|
||||
@@ -400,25 +397,9 @@ def config_vars_for_completion(d: dict = SCHEMA, path: str = "") -> "Iterator[st
|
||||
k = k.schema
|
||||
if not isinstance(k, str):
|
||||
continue
|
||||
keypath = f"{path}.{k}" if path else k
|
||||
|
||||
keypath = path + k
|
||||
if isinstance(v, dict):
|
||||
yield from config_vars_for_completion(v, keypath)
|
||||
yield from config_vars_for_completion(v, keypath + ".")
|
||||
else:
|
||||
yield keypath
|
||||
|
||||
|
||||
def contextual_config_vars_for_completion(config) -> "Iterator[str]":
|
||||
yield from config_vars_for_completion()
|
||||
for name, data in config["remote"].items():
|
||||
if "url" not in data:
|
||||
continue
|
||||
scheme = _get_schema_from_url(data["url"])
|
||||
if scheme not in REMOTE_SCHEMAS:
|
||||
continue
|
||||
schema = REMOTE_SCHEMAS[scheme]
|
||||
if schema:
|
||||
yield from config_vars_for_completion(schema, f"remote.{name}") # type: ignore[arg-type]
|
||||
|
||||
db_schema = SCHEMA["db"][str] # type: ignore[index]
|
||||
for name in config["db"]:
|
||||
yield from config_vars_for_completion(db_schema, f"db.{name}") # type: ignore[arg-type]
|
||||
|
||||
@@ -41,7 +41,7 @@ def _get(stage, p, info, **kwargs):
|
||||
return Dependency(stage, p, info, **kwargs)
|
||||
|
||||
|
||||
def loadd_from(stage, d_list):
|
||||
def loadd_from(stage, d_list) -> list[Dependency]:
|
||||
ret = []
|
||||
for d in d_list:
|
||||
p = d.pop(Output.PARAM_PATH, None)
|
||||
|
||||
@@ -110,6 +110,7 @@ class ParamsDependency(Dependency):
|
||||
except MissingParamsFile:
|
||||
return {}
|
||||
|
||||
assert self.repo
|
||||
try:
|
||||
return read_param_file(
|
||||
self.repo.fs,
|
||||
|
||||
@@ -150,6 +150,7 @@ class RepoDependency(Dependency):
|
||||
# local cache instead of the remote's cache. This avoids re-streaming of data,
|
||||
# but messes up the call to `_get_remote_config()` downstream, which will need
|
||||
# to ignore cache parameters.
|
||||
assert self.repo
|
||||
config["cache"] = self.repo.config["cache"]
|
||||
config["cache"]["dir"] = self.repo.cache.local_cache_dir
|
||||
|
||||
|
||||
+2
-2
@@ -508,7 +508,7 @@ class DvcIgnoreFilter:
|
||||
|
||||
def check_ignore(self, target: str) -> CheckIgnoreResult:
|
||||
# NOTE: can only be used in `dvc check-ignore`, see
|
||||
# https://github.com/iterative/dvc/issues/5046
|
||||
# https://github.com/treeverse/dvc/issues/5046
|
||||
full_target = self.fs.abspath(target)
|
||||
matched_patterns: list[PatternInfo] = []
|
||||
ignore = False
|
||||
@@ -525,7 +525,7 @@ class DvcIgnoreFilter:
|
||||
self, fs: "FileSystem", path: str, ignore_subrepos: bool = True
|
||||
) -> bool:
|
||||
# NOTE: can't use self.check_ignore(path).match for now, see
|
||||
# https://github.com/iterative/dvc/issues/4555
|
||||
# https://github.com/treeverse/dvc/issues/4555
|
||||
if fs.protocol != Schemes.LOCAL:
|
||||
return False
|
||||
if fs.isfile(path):
|
||||
|
||||
+1
-1
@@ -158,7 +158,7 @@ def _get_fs_type(path):
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
# need to follow the symlink: https://github.com/iterative/dvc/issues/5065
|
||||
# need to follow the symlink: https://github.com/treeverse/dvc/issues/5065
|
||||
path = pathlib.Path(path).resolve()
|
||||
|
||||
for parent in itertools.chain([path], path.parents):
|
||||
|
||||
+1
-1
@@ -182,7 +182,7 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
|
||||
# number to generate the resulting lock file name, which is unique
|
||||
# enough for our application.
|
||||
#
|
||||
# [1] https://github.com/iterative/dvc/issues/2582
|
||||
# [1] https://github.com/treeverse/dvc/issues/2582
|
||||
self._hostname = socket.gethostname()
|
||||
|
||||
self._lifetime = timedelta(days=365) # Lock for good by default
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
"""Manages logging configuration for DVC repo."""
|
||||
|
||||
import logging
|
||||
import logging.config
|
||||
import logging.handlers
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Iterator
|
||||
|
||||
+26
-9
@@ -42,6 +42,7 @@ from .utils import relpath
|
||||
from .utils.fs import path_isin
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.repo import Repo
|
||||
from dvc_data.hashfile.obj import HashFile
|
||||
from dvc_data.index import DataIndexKey
|
||||
|
||||
@@ -83,7 +84,7 @@ def _get(stage, path, **kwargs):
|
||||
return Output(stage, path, **kwargs)
|
||||
|
||||
|
||||
def loadd_from(stage, d_list):
|
||||
def loadd_from(stage, d_list) -> list["Output"]:
|
||||
ret = []
|
||||
for d in d_list:
|
||||
p = d.pop(Output.PARAM_PATH)
|
||||
@@ -329,7 +330,7 @@ class Output:
|
||||
self.annot = Annotation(
|
||||
desc=desc, type=type, labels=labels or [], meta=meta or {}
|
||||
)
|
||||
self.repo = stage.repo if not repo and stage else repo
|
||||
self.repo: Optional[Repo] = stage.repo if not repo and stage else repo
|
||||
meta_d = merge_file_meta_from_cloud(info or {})
|
||||
meta = Meta.from_dict(meta_d)
|
||||
# NOTE: when version_aware is not passed into get_cloud_fs, it will be
|
||||
@@ -511,6 +512,7 @@ class Output:
|
||||
from dvc.cachemgr import LEGACY_HASH_NAMES
|
||||
|
||||
assert self.is_in_repo
|
||||
assert self.repo
|
||||
odb_name = "legacy" if self.hash_name in LEGACY_HASH_NAMES else "repo"
|
||||
return getattr(self.repo.cache, odb_name)
|
||||
|
||||
@@ -518,6 +520,7 @@ class Output:
|
||||
def local_cache(self):
|
||||
from dvc.cachemgr import LEGACY_HASH_NAMES
|
||||
|
||||
assert self.repo
|
||||
if self.hash_name in LEGACY_HASH_NAMES:
|
||||
return self.repo.cache.legacy
|
||||
return self.repo.cache.local
|
||||
@@ -585,6 +588,7 @@ class Output:
|
||||
def index_key(self) -> tuple[str, "DataIndexKey"]:
|
||||
if self.is_in_repo:
|
||||
workspace = "repo"
|
||||
assert self.repo
|
||||
key = self.repo.fs.relparts(self.fs_path, self.repo.root_dir)
|
||||
else:
|
||||
workspace = self.fs.protocol
|
||||
@@ -640,6 +644,7 @@ class Output:
|
||||
@property
|
||||
def dvcignore(self) -> Optional["DvcIgnoreFilter"]:
|
||||
if self.fs.protocol == "local":
|
||||
assert self.repo
|
||||
return self.repo.dvcignore
|
||||
return None
|
||||
|
||||
@@ -661,6 +666,7 @@ class Output:
|
||||
if not self.use_scm_ignore:
|
||||
return
|
||||
|
||||
assert self.repo
|
||||
if self.repo.scm.is_tracked(self.fspath):
|
||||
raise OutputAlreadyTrackedError(self)
|
||||
|
||||
@@ -670,6 +676,7 @@ class Output:
|
||||
if not self.use_scm_ignore:
|
||||
return
|
||||
|
||||
assert self.repo
|
||||
self.repo.scm_context.ignore_remove(self.fspath)
|
||||
|
||||
def save(self) -> None:
|
||||
@@ -776,6 +783,7 @@ class Output:
|
||||
callback=cb,
|
||||
)
|
||||
if relink:
|
||||
assert self.repo
|
||||
rel = self.fs.relpath(filter_info or self.fs_path)
|
||||
with CheckoutCallback(desc=f"Checking out {rel}", unit="files") as cb:
|
||||
self._checkout(
|
||||
@@ -957,6 +965,7 @@ class Output:
|
||||
added = not self.exists
|
||||
|
||||
try:
|
||||
assert self.repo
|
||||
modified = self._checkout(
|
||||
filter_info or self.fs_path,
|
||||
self.fs,
|
||||
@@ -987,18 +996,24 @@ class Output:
|
||||
if ignore_remove:
|
||||
self.ignore_remove()
|
||||
|
||||
def move(self, out):
|
||||
def move(self, out: "Output") -> None:
|
||||
src_exists = self.exists
|
||||
if src_exists:
|
||||
self.fs.move(self.fs_path, out.fs_path)
|
||||
else:
|
||||
logger.warning("%r missing", self.fspath)
|
||||
|
||||
if self.protocol == "local" and self.use_scm_ignore:
|
||||
assert self.repo
|
||||
self.repo.scm_context.ignore_remove(self.fspath)
|
||||
|
||||
self.fs.move(self.fs_path, out.fs_path)
|
||||
self.def_path = out.def_path
|
||||
self.fs_path = out.fs_path
|
||||
self.save()
|
||||
self.commit()
|
||||
|
||||
if self.protocol == "local" and self.use_scm_ignore:
|
||||
self.repo.scm_context.ignore(self.fspath)
|
||||
try:
|
||||
self.save()
|
||||
self.commit()
|
||||
except self.DoesNotExistError:
|
||||
self.ignore()
|
||||
|
||||
def transfer(
|
||||
self, source, odb=None, jobs=None, update=False, no_progress_bar=False
|
||||
@@ -1079,6 +1094,7 @@ class Output:
|
||||
if self.remote:
|
||||
kwargs["remote"] = self.remote
|
||||
with suppress(Exception):
|
||||
assert self.repo
|
||||
self.repo.cloud.pull([obj.hash_info], **kwargs)
|
||||
|
||||
if self.obj:
|
||||
@@ -1172,6 +1188,7 @@ class Output:
|
||||
return {}
|
||||
|
||||
if self.remote:
|
||||
assert self.repo
|
||||
remote_odb = self.repo.cloud.get_remote_odb(
|
||||
name=self.remote, hash_name=self.hash_name
|
||||
)
|
||||
|
||||
@@ -2,7 +2,6 @@ import os
|
||||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING, NamedTuple, Optional
|
||||
|
||||
import dpath
|
||||
import dpath.options
|
||||
from funcy import get_in, last
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ if TYPE_CHECKING:
|
||||
from dvc.fs.data import DataFileSystem
|
||||
from dvc.fs.dvc import DVCFileSystem
|
||||
from dvc.lock import LockBase
|
||||
from dvc.output import Output
|
||||
from dvc.scm import Git, NoSCM
|
||||
from dvc.stage import Stage
|
||||
from dvc.types import DictStrAny
|
||||
@@ -544,7 +545,9 @@ class Repo:
|
||||
|
||||
return used
|
||||
|
||||
def find_outs_by_path(self, path, outs=None, recursive=False, strict=True):
|
||||
def find_outs_by_path(
|
||||
self, path, outs=None, recursive=False, strict=True
|
||||
) -> list["Output"]:
|
||||
# using `outs_graph` to ensure graph checks are run
|
||||
outs = outs or self.index.outs_graph
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ def check_name_format(name: str) -> None:
|
||||
|
||||
def name_is_compatible(name: str) -> bool:
|
||||
"""
|
||||
Only needed by DVCLive per iterative/dvclive#715
|
||||
Only needed by DVCLive per treeverse/dvclive#715
|
||||
Will be removed in future release.
|
||||
"""
|
||||
from gto.constants import assert_name_is_valid
|
||||
|
||||
+22
-2
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
import posixpath
|
||||
from collections import defaultdict
|
||||
from collections import defaultdict, deque
|
||||
from collections.abc import Iterable, Iterator, Mapping
|
||||
from typing import TYPE_CHECKING, Optional, TypedDict, Union
|
||||
|
||||
@@ -385,6 +385,26 @@ def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> list[str]
|
||||
return [repo.fs.relpath(file, start) for file in files]
|
||||
|
||||
|
||||
def iter_index(
|
||||
index: Union["BaseDataIndex", "DataIndexView"], shallow: bool = False
|
||||
) -> Iterator[tuple["DataIndexKey", Optional["DataIndexEntry"]]]:
|
||||
if not shallow:
|
||||
yield from index.iteritems(shallow=shallow)
|
||||
return
|
||||
|
||||
# only iterate until we find entries with hash_info in shallow mode
|
||||
todo: deque[tuple[DataIndexKey, dict]] = deque([((), index.info(()))])
|
||||
while todo:
|
||||
key, info = todo.popleft()
|
||||
entry = info.get("entry")
|
||||
if info.get("type") == "directory" and not (entry and entry.hash_info):
|
||||
try:
|
||||
todo.extend(index.ls(key, detail=True))
|
||||
except (KeyError, DataIndexDirError):
|
||||
pass
|
||||
yield key, entry
|
||||
|
||||
|
||||
def _get_entries_not_in_remote(
|
||||
repo: "Repo",
|
||||
filter_keys: Optional[Iterable["DataIndexKey"]] = None,
|
||||
@@ -418,7 +438,7 @@ def _get_entries_not_in_remote(
|
||||
|
||||
n = 0
|
||||
with TqdmCallback(size=n, desc="Checking remote", unit="entry") as cb:
|
||||
for key, entry in view.iteritems(shallow=not granular):
|
||||
for key, entry in iter_index(view, shallow=not granular):
|
||||
if not (entry and entry.hash_info):
|
||||
continue
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ from enum import IntEnum
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, Union
|
||||
|
||||
import funcy
|
||||
from funcy import nullcontext
|
||||
from scmrepo.exceptions import SCMError
|
||||
|
||||
@@ -815,11 +816,22 @@ class BaseExecutor(ABC):
|
||||
|
||||
@contextmanager
|
||||
def set_temp_refs(self, scm: "Git", temp_dict: dict[str, str]):
|
||||
# Retry ref set, get, and remove operations to handle transient issues during
|
||||
# concurrent Git access.
|
||||
# Dulwich deletes parent directories of refs if they happen to be empty after
|
||||
# removing a ref, which can interfere with `set_ref` in other processes.
|
||||
# `remove_ref` may also fail with a `FileLocked` error when refs are packed,
|
||||
# since multiple processes might attempt to write to the same file.
|
||||
retry = funcy.retry(10, errors=Exception, timeout=0.1)
|
||||
set_ref = retry(scm.set_ref)
|
||||
get_ref = retry(scm.get_ref)
|
||||
remove_ref = retry(scm.remove_ref)
|
||||
|
||||
try:
|
||||
for ref, rev in temp_dict.items():
|
||||
scm.set_ref(ref, rev)
|
||||
set_ref(ref, rev)
|
||||
yield
|
||||
finally:
|
||||
for ref in temp_dict:
|
||||
if scm.get_ref(ref):
|
||||
scm.remove_ref(ref)
|
||||
if get_ref(ref):
|
||||
remove_ref(ref)
|
||||
|
||||
@@ -287,6 +287,7 @@ class BaseStashQueue(ABC):
|
||||
baseline_rev: Optional[str] = None,
|
||||
branch: Optional[str] = None,
|
||||
name: Optional[str] = None,
|
||||
no_hydra: bool = False,
|
||||
**kwargs,
|
||||
) -> QueueEntry:
|
||||
"""Stash changes from the workspace as an experiment.
|
||||
@@ -302,6 +303,7 @@ class BaseStashQueue(ABC):
|
||||
name: Optional experiment name. If specified this will be used as
|
||||
the human-readable name in the experiment branch ref. Has no
|
||||
effect of branch is specified.
|
||||
no_hydra: Disable Hydra from automatically overwriting all params.
|
||||
|
||||
.. _Hydra Override:
|
||||
https://hydra.cc/docs/next/advanced/override_grammar/basic/
|
||||
@@ -318,7 +320,7 @@ class BaseStashQueue(ABC):
|
||||
|
||||
# update experiment params from command line
|
||||
if params:
|
||||
self._update_params(params)
|
||||
self._update_params(params, no_hydra=no_hydra)
|
||||
|
||||
# DVC commit data deps to preserve state across workspace
|
||||
# & tempdir runs
|
||||
@@ -441,12 +443,13 @@ class BaseStashQueue(ABC):
|
||||
f"from '{config_path}': {param_list}"
|
||||
)
|
||||
|
||||
def _update_params(self, params: dict[str, list[str]]):
|
||||
def _update_params(self, params: dict[str, list[str]], no_hydra: bool = False):
|
||||
"""Update param files with the provided `Hydra Override`_ patterns.
|
||||
|
||||
Args:
|
||||
params: Dict mapping paths to `Hydra Override`_ patterns,
|
||||
provided via `exp run --set-param`.
|
||||
no_hydra: Disable Hydra from automatically overwriting all params.
|
||||
|
||||
.. _Hydra Override:
|
||||
https://hydra.cc/docs/advanced/override_grammar/basic/
|
||||
@@ -456,7 +459,7 @@ class BaseStashQueue(ABC):
|
||||
logger.debug("Using experiment params '%s'", params)
|
||||
|
||||
hydra_config = self.repo.config.get("hydra", {})
|
||||
hydra_enabled = hydra_config.get("enabled", False)
|
||||
hydra_enabled = hydra_config.get("enabled", False) and not no_hydra
|
||||
hydra_output_file = ParamsDependency.DEFAULT_PARAMS_FILE
|
||||
for path, overrides in params.items():
|
||||
if hydra_enabled and path == hydra_output_file:
|
||||
|
||||
@@ -70,7 +70,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
def celery(self) -> "FSApp":
|
||||
from kombu.transport.filesystem import Channel
|
||||
|
||||
# related to https://github.com/iterative/dvc-task/issues/61
|
||||
# related to https://github.com/treeverse/dvc-task/issues/61
|
||||
Channel.QoS.restore_at_shutdown = False
|
||||
|
||||
from dvc_task.app import FSApp
|
||||
|
||||
@@ -22,6 +22,7 @@ def run( # noqa: C901, PLR0912
|
||||
queue: bool = False,
|
||||
copy_paths: Optional[Iterable[str]] = None,
|
||||
message: Optional[str] = None,
|
||||
no_hydra: bool = False,
|
||||
**kwargs,
|
||||
) -> dict[str, str]:
|
||||
"""Reproduce the specified targets as an experiment.
|
||||
@@ -67,7 +68,7 @@ def run( # noqa: C901, PLR0912
|
||||
else:
|
||||
path_overrides = {}
|
||||
|
||||
hydra_enabled = repo.config.get("hydra", {}).get("enabled", False)
|
||||
hydra_enabled = repo.config.get("hydra", {}).get("enabled", False) and not no_hydra
|
||||
hydra_output_file = ParamsDependency.DEFAULT_PARAMS_FILE
|
||||
if hydra_enabled and hydra_output_file not in path_overrides:
|
||||
# Force `_update_params` even if `--set-param` was not used
|
||||
@@ -80,6 +81,7 @@ def run( # noqa: C901, PLR0912
|
||||
tmp_dir=tmp_dir,
|
||||
copy_paths=copy_paths,
|
||||
message=message,
|
||||
no_hydra=no_hydra,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@@ -100,6 +102,7 @@ def run( # noqa: C901, PLR0912
|
||||
params=sweep_overrides,
|
||||
copy_paths=copy_paths,
|
||||
message=message,
|
||||
no_hydra=no_hydra,
|
||||
**kwargs,
|
||||
)
|
||||
if sweep_overrides:
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ def pre_commit_install(scm: "Git") -> None:
|
||||
config_path = os.path.join(scm.root_dir, ".pre-commit-config.yaml")
|
||||
with modify_yaml(config_path) as config:
|
||||
entry = {
|
||||
"repo": "https://github.com/iterative/dvc",
|
||||
"repo": "https://github.com/treeverse/dvc",
|
||||
"rev": ".".join(map(str, version_tuple[:3])),
|
||||
"hooks": [
|
||||
{
|
||||
|
||||
+41
-17
@@ -1,10 +1,13 @@
|
||||
import os
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from dvc.exceptions import MoveNotDataSourceError
|
||||
from dvc.repo.scm_context import scm_context
|
||||
|
||||
from . import locked
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from . import Repo
|
||||
|
||||
|
||||
def _expand_target_path(from_path, to_path):
|
||||
if os.path.isdir(to_path):
|
||||
@@ -14,7 +17,7 @@ def _expand_target_path(from_path, to_path):
|
||||
|
||||
@locked
|
||||
@scm_context
|
||||
def move(self, from_path, to_path):
|
||||
def move(self: "Repo", from_path, to_path):
|
||||
"""
|
||||
Renames an output file and modifies the stage associated
|
||||
to reflect the change on the pipeline.
|
||||
@@ -32,9 +35,12 @@ def move(self, from_path, to_path):
|
||||
It only works with outputs generated by `add` or `import`,
|
||||
also known as data sources.
|
||||
"""
|
||||
from dvc import output
|
||||
from dvc import dependency, output
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX
|
||||
from dvc.exceptions import MoveNotDataSourceError
|
||||
from dvc.stage import Stage
|
||||
from dvc.stage.exceptions import StageFileAlreadyExistsError
|
||||
from dvc_objects.fs.local import LocalFileSystem
|
||||
|
||||
from_out = output.loads_from(Stage(self), [from_path])[0]
|
||||
assert from_out.protocol == "local"
|
||||
@@ -45,6 +51,7 @@ def move(self, from_path, to_path):
|
||||
assert len(outs) == 1
|
||||
out = outs[0]
|
||||
stage = out.stage
|
||||
deps = stage.deps
|
||||
|
||||
if not stage.is_data_source:
|
||||
raise MoveNotDataSourceError(stage.addressing)
|
||||
@@ -58,21 +65,38 @@ def move(self, from_path, to_path):
|
||||
)
|
||||
new_wdir = os.path.abspath(os.path.join(os.curdir, os.path.dirname(to_path)))
|
||||
to_path = os.path.relpath(to_path, new_wdir)
|
||||
new_stage = self.stage.create(
|
||||
single_stage=True,
|
||||
fname=new_fname,
|
||||
wdir=new_wdir,
|
||||
outs=[to_path],
|
||||
meta=stage.meta,
|
||||
)
|
||||
|
||||
os.unlink(stage.path)
|
||||
stage = new_stage
|
||||
try:
|
||||
new_stage = self.stage.create(
|
||||
single_stage=True,
|
||||
fname=new_fname,
|
||||
wdir=new_wdir,
|
||||
outs=[to_path],
|
||||
meta=stage.meta,
|
||||
frozen=stage.frozen,
|
||||
always_changed=stage.always_changed,
|
||||
desc=stage.desc,
|
||||
)
|
||||
except StageFileAlreadyExistsError:
|
||||
# reraise to remove `--force` hint
|
||||
raise StageFileAlreadyExistsError(f"{new_fname!r} already exists") from None
|
||||
else:
|
||||
new_stage = stage
|
||||
to_path = os.path.relpath(to_path, stage.wdir)
|
||||
|
||||
to_out = output.loads_from(stage, [to_path], out.use_cache, out.metric)[0]
|
||||
def with_dep_path_adjusted(dep: dependency.Dependency):
|
||||
d = dep.dumpd()
|
||||
if isinstance(dep.fs, LocalFileSystem) and not os.path.isabs(dep.def_path):
|
||||
return d | {"path": os.path.relpath(dep.fspath, new_stage.wdir)}
|
||||
return d
|
||||
|
||||
out.move(to_out)
|
||||
stage.save()
|
||||
stage.dump()
|
||||
new_stage.outs = output.loadd_from(new_stage, [out.dumpd() | {"path": to_path}])
|
||||
new_stage.deps = dependency.loadd_from(
|
||||
new_stage, [with_dep_path_adjusted(dep) for dep in deps]
|
||||
)
|
||||
out.move(new_stage.outs[0])
|
||||
new_stage.md5 = new_stage.compute_md5()
|
||||
new_stage.dump()
|
||||
if stage != new_stage:
|
||||
stage.dvcfile.remove()
|
||||
self.scm_context.track_file(stage.dvcfile.relpath)
|
||||
return stage, new_stage
|
||||
|
||||
@@ -46,7 +46,7 @@ class SCMContext:
|
||||
try:
|
||||
add_no_submodules(self.scm, paths)
|
||||
except UnsupportedIndexFormat:
|
||||
link = "https://github.com/iterative/dvc/issues/610"
|
||||
link = "https://github.com/treeverse/dvc/issues/610"
|
||||
add_cmd = self._make_git_add_cmd([relpath(path) for path in paths])
|
||||
logger.info("")
|
||||
msg = (
|
||||
|
||||
@@ -32,7 +32,7 @@ from .utils import (
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.dependency import ParamsDependency
|
||||
from dvc.dependency import Dependency, ParamsDependency
|
||||
from dvc.dvcfile import ProjectFile, SingleStageFile
|
||||
from dvc.output import Output
|
||||
from dvc.repo import Repo
|
||||
@@ -153,8 +153,8 @@ class Stage(params.StageParams):
|
||||
self._path = path
|
||||
self.cmd = cmd
|
||||
self.wdir = wdir
|
||||
self.outs = outs
|
||||
self.deps = deps
|
||||
self.outs: list[Output] = outs
|
||||
self.deps: list[Dependency] = deps
|
||||
self.md5 = md5
|
||||
self.frozen = locked or frozen
|
||||
self.always_changed = always_changed
|
||||
|
||||
+2
-2
@@ -51,7 +51,7 @@ def _warn_if_fish(executable):
|
||||
"Be aware that it might cause problems by overwriting "
|
||||
"your current environment variables with values defined "
|
||||
"in 'config.fish', which might affect your command. See "
|
||||
"https://github.com/iterative/dvc/issues/1307. "
|
||||
"https://github.com/treeverse/dvc/issues/1307. "
|
||||
)
|
||||
|
||||
|
||||
@@ -99,7 +99,7 @@ def prepare_kwargs(stage, run_env=None):
|
||||
#
|
||||
# [1] https://github.com/python/cpython/blob/3.7/Lib/subprocess.py
|
||||
# #L1426
|
||||
# [2] https://github.com/iterative/dvc/issues/2506
|
||||
# [2] https://github.com/treeverse/dvc/issues/2506
|
||||
# #issuecomment-535396799
|
||||
kwargs["shell"] = os.name == "nt"
|
||||
return kwargs
|
||||
|
||||
@@ -178,7 +178,7 @@ def to_single_stage_lockfile(stage: "Stage", **kwargs) -> dict:
|
||||
res = OrderedDict([("cmd", stage.cmd)])
|
||||
params, deps = split_params_deps(stage)
|
||||
deps, outs = (
|
||||
[_dumpd(item) for item in sorted(items, key=attrgetter("def_path"))]
|
||||
[_dumpd(item) for item in sorted(items, key=attrgetter("def_path"))] # type: ignore[call-overload]
|
||||
for items in [deps, stage.outs]
|
||||
)
|
||||
params = _serialize_params_values(params)
|
||||
|
||||
@@ -4,7 +4,7 @@ dvc.testing.benchmarks
|
||||
======================
|
||||
|
||||
Benchmark test definitions are now part of ``dvc.testing``.
|
||||
For CLI usage and `bench.dvc.org <https://bench.dvc.org>`_ details see `dvc-bench <https://github.com/iterative/dvc-bench>`_.
|
||||
For CLI usage and `bench.dvc.org <https://bench.dvc.org>`_ details see `dvc-bench <https://github.com/treeverse/dvc-bench>`_.
|
||||
|
||||
``dvc.testing.benchmarks`` structure:
|
||||
|
||||
|
||||
@@ -6,8 +6,8 @@ import pytest
|
||||
"!=3.53.*,!=3.54.0",
|
||||
reason="Takes 10 mins to run. Regression in 3.53.0, fixed in 3.54.1",
|
||||
)
|
||||
# Introduced in https://github.com/iterative/dvc/pull/10388.
|
||||
# Fixed in https://github.com/iterative/dvc/pull/10531.
|
||||
# Introduced in https://github.com/treeverse/dvc/pull/10388.
|
||||
# Fixed in https://github.com/treeverse/dvc/pull/10531.
|
||||
def test_import(bench_dvc, tmp_dir, scm, dvc, make_dataset, remote):
|
||||
dataset = make_dataset(
|
||||
cache=False, files=False, dvcfile=True, commit=True, remote=True
|
||||
|
||||
@@ -79,7 +79,7 @@ def dvc_bench_repo(tmp_path_factory, bench_config):
|
||||
if url is None:
|
||||
pytest.skip(
|
||||
"--dvc-bench-repo is not set, "
|
||||
"clone https://github.com/iterative/dvc-bench repository and set its path"
|
||||
"clone https://github.com/treeverse/dvc-bench repository and set its path"
|
||||
)
|
||||
|
||||
if os.path.isdir(url):
|
||||
@@ -158,7 +158,7 @@ def make_bench(request):
|
||||
# pytest >= 8.4.0
|
||||
wrapped_func = fixture_function._get_wrapped_function()
|
||||
except AttributeError:
|
||||
wrapped_func = fixture_function.__pytest_wrapped__.obj # ty: ignore[unresolved-attribute]
|
||||
wrapped_func = fixture_function.__pytest_wrapped__.obj # type: ignore[attr-defined] # ty: ignore[unresolved-attribute]
|
||||
assert inspect.isgeneratorfunction(wrapped_func)
|
||||
|
||||
generator = wrapped_func(request)
|
||||
|
||||
@@ -44,7 +44,7 @@ class TestRemote:
|
||||
_check_status(status_dir, new=dir_hashes)
|
||||
|
||||
# Move cache and check status
|
||||
# See issue https://github.com/iterative/dvc/issues/4383 for details
|
||||
# See issue https://github.com/treeverse/dvc/issues/4383 for details
|
||||
backup_dir = dvc.cache.local.path + ".backup"
|
||||
shutil.move(dvc.cache.local.path, backup_dir)
|
||||
status = dvc.cloud.status(foo_hashes)
|
||||
@@ -116,7 +116,7 @@ class TestRemote:
|
||||
|
||||
@pytest.mark.xfail(raises=NotImplementedError, strict=False)
|
||||
def test_pull_00_prefix(self, tmp_dir, dvc, remote, monkeypatch):
|
||||
# Related: https://github.com/iterative/dvc/issues/6089
|
||||
# Related: https://github.com/treeverse/dvc/issues/6089
|
||||
|
||||
fs_type = type(dvc.cloud.get_remote_odb("upstream").fs)
|
||||
monkeypatch.setattr(fs_type, "_ALWAYS_TRAVERSE", True, raising=False)
|
||||
@@ -146,7 +146,7 @@ class TestRemote:
|
||||
|
||||
@pytest.mark.xfail(raises=NotImplementedError, strict=False)
|
||||
def test_pull_no_00_prefix(self, tmp_dir, dvc, remote, monkeypatch):
|
||||
# Related: https://github.com/iterative/dvc/issues/6244
|
||||
# Related: https://github.com/treeverse/dvc/issues/6244
|
||||
|
||||
fs_type = type(dvc.cloud.get_remote_odb("upstream").fs)
|
||||
monkeypatch.setattr(fs_type, "_ALWAYS_TRAVERSE", True, raising=False)
|
||||
|
||||
+1
-1
@@ -163,7 +163,7 @@ class Updater:
|
||||
if pkg not in instructions:
|
||||
return (
|
||||
"Find the latest release at "
|
||||
"[blue]https://github.com/iterative/dvc/releases/latest[/]."
|
||||
"[blue]https://github.com/treeverse/dvc/releases/latest[/]."
|
||||
)
|
||||
|
||||
instruction = instructions[pkg]
|
||||
|
||||
+7
-7
@@ -18,7 +18,7 @@ keywords = [
|
||||
]
|
||||
license = "Apache-2.0"
|
||||
license-files = ["LICENSE"]
|
||||
maintainers = [{ name = "Iterative", email = "[email protected]" }]
|
||||
maintainers = [{ name = "Treeverse", email = "[email protected]" }]
|
||||
authors = [{ name = "Dmitry Petrov", email = "[email protected]" }]
|
||||
requires-python = ">=3.9"
|
||||
classifiers = [
|
||||
@@ -29,6 +29,7 @@ classifiers = [
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Programming Language :: Python :: 3.14",
|
||||
]
|
||||
dynamic = ["version"]
|
||||
|
||||
@@ -69,8 +70,7 @@ dependencies = [
|
||||
"ruamel.yaml>=0.17.11",
|
||||
"scmrepo>=3.5.2,<4",
|
||||
"shortuuid>=0.5",
|
||||
# "shtab<2,>=1.3.4",
|
||||
"shtab @ git+https://github.com/skshetry/shtab.git@fish-shell-v2",
|
||||
"shtab<2,>=1.3.4",
|
||||
"tabulate>=0.8.7",
|
||||
"tomlkit>=0.11.1",
|
||||
"tqdm<5,>=4.63.1",
|
||||
@@ -86,7 +86,7 @@ gdrive = ["dvc-gdrive>=3,<4"]
|
||||
gs = ["dvc-gs>=3.0.2,<4"]
|
||||
hdfs = ["dvc-hdfs>=3,<4"]
|
||||
lint = [
|
||||
"mypy==1.18.2",
|
||||
"mypy==1.19.0",
|
||||
"pandas-stubs",
|
||||
"types-colorama",
|
||||
"types-psutil",
|
||||
@@ -124,14 +124,14 @@ tests = [
|
||||
"sqlalchemy>=1,<3", # optional dependency for `import-db`
|
||||
"pandas>=1", # optional dependency for `import-db`
|
||||
]
|
||||
webdav = ["dvc-webdav>=3,<4"]
|
||||
webdav = ["dvc-webdav>=3.0.1,<4"]
|
||||
webhdfs = ["dvc-webhdfs>=3.1,<4"]
|
||||
webhdfs_kerberos = ["dvc-webhdfs[kerberos]>=3.1,<4"]
|
||||
|
||||
[project.urls]
|
||||
Documentation = "https://dvc.org/doc"
|
||||
Issues = "https://github.com/iterative/dvc/issues"
|
||||
Source = "https://github.com/iterative/dvc"
|
||||
Issues = "https://github.com/treeverse/dvc/issues"
|
||||
Source = "https://github.com/treeverse/dvc"
|
||||
|
||||
[project.scripts]
|
||||
dvc = "dvc.cli:main"
|
||||
|
||||
@@ -194,6 +194,7 @@ def isolate(tmp_path_factory):
|
||||
monkeypatch.setenv(env_var, os.fspath(path))
|
||||
else:
|
||||
monkeypatch.setenv("HOME", str(home_dir))
|
||||
monkeypatch.setenv("XDG_CONFIG_HOME", str(home_dir / ".config"))
|
||||
|
||||
monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1")
|
||||
contents = b"""
|
||||
|
||||
@@ -91,7 +91,7 @@ def test_apply_untracked(tmp_dir, scm, dvc, exp_stage):
|
||||
|
||||
|
||||
def test_apply_unchanged_head(tmp_dir, scm, dvc, exp_stage):
|
||||
# see https://github.com/iterative/dvc/issues/8764
|
||||
# see https://github.com/treeverse/dvc/issues/8764
|
||||
tmp_dir.gen("params.yaml", "foo: 2")
|
||||
scm.add(["dvc.yaml", "dvc.lock", "params.yaml", "metrics.yaml"])
|
||||
scm.commit("commit foo=2")
|
||||
|
||||
@@ -26,7 +26,7 @@ def test_celery_logs(tmp_dir, scm, dvc, failed_exp_stage, follow, capsys, test_q
|
||||
|
||||
|
||||
def test_queue_doesnt_remove_untracked_params_file(tmp_dir, dvc, scm):
|
||||
"""Regression test for https://github.com/iterative/dvc/issues/7842"""
|
||||
"""Regression test for https://github.com/treeverse/dvc/issues/7842"""
|
||||
tmp_dir.gen("params.yaml", "foo: 1")
|
||||
stage = dvc.run(cmd="echo ${foo}", params=["foo"], name="echo-foo")
|
||||
scm.add(["dvc.yaml", "dvc.lock", ".gitignore"])
|
||||
|
||||
@@ -105,7 +105,7 @@ def test_exp_save_include_untracked(tmp_dir, dvc, scm):
|
||||
|
||||
|
||||
def test_exp_save_include_untracked_warning(tmp_dir, dvc, scm, mocker):
|
||||
"""Regression test for https://github.com/iterative/dvc/issues/9061"""
|
||||
"""Regression test for https://github.com/treeverse/dvc/issues/9061"""
|
||||
setup_stage(tmp_dir, dvc, scm)
|
||||
|
||||
new_dir = tmp_dir / "new_dir"
|
||||
|
||||
@@ -26,8 +26,9 @@ def test_modify_params(params_repo, dvc, changes, expected):
|
||||
("conf", "bar"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("no_hydra", [True, False])
|
||||
def test_hydra_compose_and_dump(
|
||||
tmp_dir, params_repo, dvc, hydra_enabled, config_dir, config_name
|
||||
tmp_dir, params_repo, dvc, hydra_enabled, config_dir, config_name, no_hydra
|
||||
):
|
||||
hydra_setup(
|
||||
tmp_dir,
|
||||
@@ -50,14 +51,14 @@ def test_hydra_compose_and_dump(
|
||||
if config_name is not None:
|
||||
conf["hydra"]["config_name"] = config_name
|
||||
|
||||
dvc.experiments.run()
|
||||
dvc.experiments.run(no_hydra=no_hydra)
|
||||
|
||||
if hydra_enabled:
|
||||
if hydra_enabled and not no_hydra:
|
||||
assert (tmp_dir / "params.yaml").parse() == {
|
||||
"db": {"driver": "mysql", "user": "omry", "pass": "secret"},
|
||||
}
|
||||
|
||||
dvc.experiments.run(params=["db=postgresql"])
|
||||
dvc.experiments.run(params=["db=postgresql"], no_hydra=no_hydra)
|
||||
assert (tmp_dir / "params.yaml").parse() == {
|
||||
"db": {
|
||||
"driver": "postgresql",
|
||||
@@ -117,6 +118,7 @@ def test_hydra_sweep(
|
||||
targets=None,
|
||||
copy_paths=None,
|
||||
message=None,
|
||||
no_hydra=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -137,3 +139,76 @@ def test_hydra_sweep_prefix_name(tmp_dir, params_repo, dvc):
|
||||
exp_names = [entry.name for entry in dvc.experiments.celery_queue.iter_queued()]
|
||||
for name, expected in zip(exp_names, expected_names):
|
||||
assert name == expected
|
||||
|
||||
|
||||
def test_mixing_no_hydra_and_params_flags(tmp_dir, params_repo, dvc):
|
||||
# Passing no_hydra should not prevent user from
|
||||
# using --set-param on unmodified params.yaml
|
||||
hydra_setup(
|
||||
tmp_dir,
|
||||
config_dir="conf",
|
||||
config_name="config",
|
||||
)
|
||||
|
||||
with dvc.config.edit() as conf:
|
||||
conf["hydra"]["enabled"] = True
|
||||
conf["hydra"]["config_dir"] = "conf"
|
||||
conf["hydra"]["config_name"] = "config"
|
||||
|
||||
dvc.experiments.run(no_hydra=True, params=["goo.bag=10.0"])
|
||||
|
||||
assert (tmp_dir / "params.yaml").parse() == {
|
||||
"foo": [{"bar": 1}, {"baz": 2}],
|
||||
"goo": {"bag": 10.0},
|
||||
"lorem": False,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"hydra_enabled,overrides,expected",
|
||||
[
|
||||
(
|
||||
True,
|
||||
["db=mysql,postgresql"],
|
||||
[
|
||||
{"params.yaml": ["db=mysql"]},
|
||||
{"params.yaml": ["db=postgresql"]},
|
||||
],
|
||||
),
|
||||
(
|
||||
False,
|
||||
["foo=bar,baz"],
|
||||
[{"params.yaml": ["foo=bar"]}, {"params.yaml": ["foo=baz"]}],
|
||||
),
|
||||
(
|
||||
False,
|
||||
[],
|
||||
[{}],
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("no_hydra", [True, False])
|
||||
def test_mixing_no_hydra_and_sweeps(
|
||||
tmp_dir, params_repo, dvc, mocker, hydra_enabled, overrides, expected, no_hydra
|
||||
):
|
||||
# Passing no_hydra should not prevent user from
|
||||
# queuing sweeps with --set-param and --queue
|
||||
patched = mocker.patch.object(dvc.experiments, "queue_one")
|
||||
|
||||
if hydra_enabled:
|
||||
hydra_setup(tmp_dir, config_dir="conf", config_name="config")
|
||||
with dvc.config.edit() as conf:
|
||||
conf["hydra"]["enabled"] = True
|
||||
|
||||
dvc.experiments.run(params=overrides, queue=True, no_hydra=no_hydra)
|
||||
|
||||
assert patched.call_count == len(expected)
|
||||
for e in expected:
|
||||
patched.assert_any_call(
|
||||
mocker.ANY,
|
||||
params=e,
|
||||
targets=None,
|
||||
copy_paths=None,
|
||||
message=None,
|
||||
no_hydra=no_hydra,
|
||||
)
|
||||
|
||||
@@ -148,7 +148,7 @@ def test_downstream(M, tmp_dir, dvc):
|
||||
assert [stage.name for stage in evaluation] == ["B-gen", "D-gen", "E-gen"]
|
||||
|
||||
# B, C should be run (in any order) before D
|
||||
# See https://github.com/iterative/dvc/issues/3602
|
||||
# See https://github.com/treeverse/dvc/issues/3602
|
||||
evaluation = dvc.reproduce(PROJECT_FILE + ":A-gen", downstream=True, force=True)
|
||||
|
||||
assert len(evaluation) == 5
|
||||
|
||||
@@ -36,7 +36,7 @@ def test_repro_allow_missing_and_pull(tmp_dir, dvc, mocker, local_remote):
|
||||
def test_repro_allow_missing_upstream_stage_modified(
|
||||
tmp_dir, dvc, mocker, local_remote
|
||||
):
|
||||
"""https://github.com/iterative/dvc/issues/9530"""
|
||||
"""https://github.com/treeverse/dvc/issues/9530"""
|
||||
tmp_dir.gen("params.yaml", "param: 1")
|
||||
dvc.stage.add(
|
||||
name="create-foo", cmd="echo ${param} > foo", params=["param"], outs=["foo"]
|
||||
|
||||
@@ -723,7 +723,7 @@ def test_add_symlink_file(tmp_dir, dvc):
|
||||
).is_symlink()
|
||||
|
||||
# Test that subsequent add succeeds
|
||||
# See https://github.com/iterative/dvc/issues/4654
|
||||
# See https://github.com/treeverse/dvc/issues/4654
|
||||
dvc.add(os.path.join("dir", "foo"))
|
||||
|
||||
|
||||
|
||||
@@ -88,8 +88,8 @@ def test_config_set_in_non_dvc_repo(tmp_dir, caplog):
|
||||
[
|
||||
(["core.analytics"], 0, "False"),
|
||||
(["core.remote"], 0, "myremote"),
|
||||
(["remote.myremote.profile"], 0, "iterative"),
|
||||
(["remote.myremote.profile", "--local"], 0, "iterative"),
|
||||
(["remote.myremote.profile"], 0, "treeverse"),
|
||||
(["remote.myremote.profile", "--local"], 0, "treeverse"),
|
||||
(
|
||||
["remote.myremote.profile", "--project"],
|
||||
251,
|
||||
@@ -122,7 +122,7 @@ def test_config_get(tmp_dir, dvc, capsys, caplog, args, ret, msg):
|
||||
textwrap.dedent(
|
||||
"""\
|
||||
['remote "myremote"']
|
||||
profile = iterative
|
||||
profile = treeverse
|
||||
['remote "other"']
|
||||
url = gs://bucket/path
|
||||
"""
|
||||
@@ -167,7 +167,7 @@ def test_config_list(tmp_dir, dvc, capsys):
|
||||
textwrap.dedent(
|
||||
"""\
|
||||
['remote "myremote"']
|
||||
profile = iterative
|
||||
profile = treeverse
|
||||
access_key_id = abcde
|
||||
secret_access_key = 123456
|
||||
['remote "other"']
|
||||
@@ -181,7 +181,7 @@ def test_config_list(tmp_dir, dvc, capsys):
|
||||
out, _ = capsys.readouterr()
|
||||
assert "remote.myremote.url=s3://bucket/path" in out
|
||||
assert "remote.myremote.region=us-east-2" in out
|
||||
assert "remote.myremote.profile=iterative" in out
|
||||
assert "remote.myremote.profile=treeverse" in out
|
||||
assert "remote.myremote.access_key_id=abcde" in out
|
||||
assert "remote.myremote.secret_access_key=123456" in out
|
||||
assert "remote.other.url=gs://bucket/path" in out
|
||||
@@ -242,7 +242,7 @@ def test_merging_two_levels(dvc):
|
||||
|
||||
|
||||
def test_config_loads_without_error_for_non_dvc_repo(tmp_dir):
|
||||
# regression testing for https://github.com/iterative/dvc/issues/3328
|
||||
# regression testing for https://github.com/treeverse/dvc/issues/3328
|
||||
Config.from_cwd(validate=True)
|
||||
|
||||
|
||||
|
||||
@@ -730,12 +730,12 @@ def test_pull_allow_missing(tmp_dir, dvc, local_remote):
|
||||
def test_pull_granular_excluding_import_that_cannot_be_pulled(
|
||||
tmp_dir, dvc, local_remote, mocker
|
||||
):
|
||||
"""Regression test for https://github.com/iterative/dvc/issues/10309."""
|
||||
"""Regression test for https://github.com/treeverse/dvc/issues/10309."""
|
||||
|
||||
mocker.patch("dvc.fs.dvc._DVCFileSystem", side_effect=CloneError("SCM error"))
|
||||
(stage,) = tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||||
imp_stage = dvc.imp(
|
||||
"https://user:[email protected]/iterative/dvc.git",
|
||||
"https://user:[email protected]/treeverse/dvc.git",
|
||||
"dir",
|
||||
out="new_dir",
|
||||
rev="HEAD",
|
||||
|
||||
@@ -578,7 +578,7 @@ def test_root_from_file_to_dir(M, tmp_dir, dvc, scm):
|
||||
|
||||
|
||||
def test_empty_dir(tmp_dir, scm, dvc, M):
|
||||
# regression testing for https://github.com/iterative/dvc/issues/8958
|
||||
# regression testing for https://github.com/treeverse/dvc/issues/8958
|
||||
tmp_dir.dvc_gen({"data": {"foo": "foo"}})
|
||||
remove("data")
|
||||
|
||||
@@ -1211,3 +1211,40 @@ def test_renames(
|
||||
dvc.data_status(granular=True, with_renames=True, targets=targets)
|
||||
== EMPTY_STATUS | {"git": M.dict()} | expected_granular
|
||||
)
|
||||
|
||||
|
||||
def test_shallow_should_iterate_upto_tracked_directory(
|
||||
M, tmp_dir, dvc, scm, local_remote
|
||||
):
|
||||
"""Testing regression for https://github.com/treeverse/dvc/issues/10899."""
|
||||
|
||||
tmp_dir.scm_gen({"dir": {".gitkeep": ""}}, commit="mk dir")
|
||||
dataset_root = join("dir", "data", "")
|
||||
tmp_dir.dvc_gen({dataset_root: {"foo": "foo", "bar": "bar"}})
|
||||
files = [dataset_root, join(dataset_root, "foo"), join(dataset_root, "bar")]
|
||||
|
||||
assert dvc.data_status(not_in_remote=True) == {
|
||||
**EMPTY_STATUS,
|
||||
"committed": {"added": [dataset_root]},
|
||||
"not_in_remote": [dataset_root],
|
||||
"git": M.dict(),
|
||||
}
|
||||
assert dvc.data_status(granular=True, not_in_remote=True) == {
|
||||
**EMPTY_STATUS,
|
||||
"committed": {"added": M.unordered(*files)},
|
||||
"not_in_remote": M.unordered(*files),
|
||||
"git": M.dict(),
|
||||
}
|
||||
|
||||
assert dvc.push() == 3
|
||||
|
||||
assert dvc.data_status(not_in_remote=True, remote_refresh=True) == {
|
||||
**EMPTY_STATUS,
|
||||
"committed": {"added": [dataset_root]},
|
||||
"git": M.dict(),
|
||||
}
|
||||
assert dvc.data_status(granular=True, not_in_remote=True, remote_refresh=True) == {
|
||||
**EMPTY_STATUS,
|
||||
"committed": {"added": M.unordered(*files)},
|
||||
"git": M.dict(),
|
||||
}
|
||||
|
||||
@@ -101,7 +101,7 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
|
||||
|
||||
def test_relative_remote(erepo_dir, tmp_dir):
|
||||
# these steps reproduce the script on this issue:
|
||||
# https://github.com/iterative/dvc/issues/2756
|
||||
# https://github.com/treeverse/dvc/issues/2756
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("file", "contents", commit="create file")
|
||||
|
||||
|
||||
@@ -160,7 +160,7 @@ def test_non_cached_output(tmp_dir, erepo_dir):
|
||||
assert (tmp_dir / dst).read_text().strip() == "hello"
|
||||
|
||||
|
||||
# https://github.com/iterative/dvc/pull/2837#discussion_r352123053
|
||||
# https://github.com/treeverse/dvc/pull/2837#discussion_r352123053
|
||||
def test_absolute_file_outside_repo(tmp_dir, erepo_dir):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
Repo.get(os.fspath(erepo_dir), "/root/")
|
||||
|
||||
@@ -405,7 +405,7 @@ def test_import_non_existing(erepo_dir, tmp_dir, dvc):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
tmp_dir.dvc.imp(os.fspath(erepo_dir), "invalid_output")
|
||||
|
||||
# https://github.com/iterative/dvc/pull/2837#discussion_r352123053
|
||||
# https://github.com/treeverse/dvc/pull/2837#discussion_r352123053
|
||||
with pytest.raises(FileNotFoundError):
|
||||
tmp_dir.dvc.imp(os.fspath(erepo_dir), "/root/", "root")
|
||||
|
||||
|
||||
@@ -109,7 +109,7 @@ def test_gen_dvcignore(tmp_dir):
|
||||
|
||||
|
||||
def test_init_when_ignored_by_git(tmp_dir, scm, caplog):
|
||||
# https://github.com/iterative/dvc/issues/3738
|
||||
# https://github.com/treeverse/dvc/issues/3738
|
||||
tmp_dir.gen({".gitignore": ".*"})
|
||||
with caplog.at_level(logging.ERROR, logger="dvc"):
|
||||
assert main(["init"]) == 1
|
||||
|
||||
+201
-4
@@ -6,14 +6,20 @@ import pytest
|
||||
|
||||
from dvc.cli import main
|
||||
from dvc.exceptions import MoveNotDataSourceError, OutputNotFoundError
|
||||
from dvc.stage.exceptions import StageFileAlreadyExistsError
|
||||
|
||||
|
||||
def test_move(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
dvc.move("foo", "foo1")
|
||||
def test_move(tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen("foo", "bar")
|
||||
assert (tmp_dir / "foo.dvc").exists()
|
||||
dvc.move("foo", "bar")
|
||||
|
||||
assert not (tmp_dir / "foo.dvc").exists()
|
||||
assert (tmp_dir / "bar.dvc").exists()
|
||||
assert not (tmp_dir / "foo").is_file()
|
||||
assert (tmp_dir / "foo1").is_file()
|
||||
assert (tmp_dir / "bar").is_file()
|
||||
# should only have the new path in the .gitignore, and only once
|
||||
assert (tmp_dir / ".gitignore").read_text().splitlines() == ["/bar"]
|
||||
|
||||
|
||||
def test_move_non_existent_file(dvc):
|
||||
@@ -21,6 +27,24 @@ def test_move_non_existent_file(dvc):
|
||||
dvc.move("non_existent_file", "dst")
|
||||
|
||||
|
||||
def test_move_missing_file(tmp_dir, dvc, scm, caplog):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
(tmp_dir / "foo").unlink()
|
||||
contents = (tmp_dir / "foo.dvc").parse()
|
||||
dvc.move("foo", "bar")
|
||||
|
||||
assert not (tmp_dir / "foo.dvc").exists()
|
||||
# only the path should be changed in the dvc file
|
||||
contents["outs"][0]["path"] = "bar"
|
||||
assert contents == (tmp_dir / "bar.dvc").parse()
|
||||
|
||||
# file should not be checked out
|
||||
assert not (tmp_dir / "foo").is_file()
|
||||
assert not (tmp_dir / "bar").is_file()
|
||||
# should only have the new path in the .gitignore, and only once
|
||||
assert (tmp_dir / ".gitignore").read_text().splitlines() == ["/bar"]
|
||||
|
||||
|
||||
def test_move_directory(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen("data", {"foo": "foo", "bar": "bar"})
|
||||
dvc.move("data", "dst")
|
||||
@@ -195,3 +219,176 @@ def test_move_meta(tmp_dir, dvc):
|
||||
custom_key: 42
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def test_import(tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen("foo", "foo", commit="add foo")
|
||||
imp_stage = dvc.imp(os.curdir, "foo", "foo_imported")
|
||||
|
||||
dvc.move("foo_imported", "foo_moved")
|
||||
|
||||
(stage,) = dvc.stage.collect("foo_moved.dvc")
|
||||
assert imp_stage.md5 != stage.md5
|
||||
res = (tmp_dir / "foo_moved.dvc").read_text()
|
||||
assert res == textwrap.dedent(
|
||||
f"""\
|
||||
md5: {stage.md5}
|
||||
frozen: true
|
||||
deps:
|
||||
- path: foo
|
||||
repo:
|
||||
url: {os.curdir}
|
||||
rev_lock: {scm.get_rev()}
|
||||
outs:
|
||||
- md5: acbd18db4cc2f85cedef654fccc4a4d8
|
||||
size: 3
|
||||
hash: md5
|
||||
path: foo_moved
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path_func",
|
||||
[pytest.param(os.path.abspath, id="abs"), pytest.param(os.path.relpath, id="rel")],
|
||||
)
|
||||
def test_import_url_in_repo(tmp_dir, dvc, path_func):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
imp_stage = dvc.imp_url(path_func(tmp_dir / "foo"), "foo_imported")
|
||||
(tmp_dir / "data").mkdir()
|
||||
|
||||
dvc.move("foo_imported", os.path.join("data", "foo_moved"))
|
||||
|
||||
(stage,) = dvc.stage.collect(os.path.join("data", "foo_moved.dvc"))
|
||||
assert imp_stage.md5 != stage.md5
|
||||
res = (tmp_dir / "data" / "foo_moved.dvc").read_text()
|
||||
assert res == textwrap.dedent(
|
||||
f"""\
|
||||
md5: {stage.md5}
|
||||
frozen: true
|
||||
deps:
|
||||
- md5: acbd18db4cc2f85cedef654fccc4a4d8
|
||||
size: 3
|
||||
hash: md5
|
||||
path: ../foo
|
||||
outs:
|
||||
- md5: acbd18db4cc2f85cedef654fccc4a4d8
|
||||
size: 3
|
||||
hash: md5
|
||||
path: foo_moved
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path_func",
|
||||
[pytest.param(os.path.abspath, id="abs"), pytest.param(os.path.relpath, id="rel")],
|
||||
)
|
||||
def test_import_url_out_of_repo(tmp_dir, dvc, scm, path_func, make_tmp_dir):
|
||||
external = make_tmp_dir("external")
|
||||
external.gen("foo", "foo")
|
||||
|
||||
imp_stage = dvc.imp_url(path_func(external / "foo"), "foo_imported")
|
||||
|
||||
data_dir = tmp_dir / "data"
|
||||
data_dir.mkdir()
|
||||
|
||||
new_path = data_dir / "foo_moved"
|
||||
new_dvcfile = new_path.with_suffix(".dvc")
|
||||
dvc.move("foo_imported", os.fspath(new_path))
|
||||
|
||||
(stage,) = dvc.stage.collect(os.fspath(new_dvcfile))
|
||||
assert imp_stage.md5 != stage.md5
|
||||
|
||||
with data_dir.chdir():
|
||||
expected_path = path_func(external / "foo")
|
||||
|
||||
assert new_dvcfile.parse() == {
|
||||
"md5": stage.md5,
|
||||
"frozen": True,
|
||||
"deps": [
|
||||
{
|
||||
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
|
||||
"size": 3,
|
||||
"hash": "md5",
|
||||
"path": expected_path,
|
||||
}
|
||||
],
|
||||
"outs": [
|
||||
{
|
||||
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
|
||||
"size": 3,
|
||||
"hash": "md5",
|
||||
"path": "foo_moved",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path_func",
|
||||
[pytest.param(os.path.abspath, id="abs"), pytest.param(os.path.relpath, id="rel")],
|
||||
)
|
||||
def test_all_metadata_are_preserved(tmp_dir, dvc, make_tmp_dir, path_func):
|
||||
external = make_tmp_dir("external")
|
||||
external.gen("foo", "foo")
|
||||
|
||||
contents = {
|
||||
"md5": "bad", # placeholder, does not matter for the test
|
||||
"frozen": True,
|
||||
"desc": "this is a stage description",
|
||||
"always_changed": True,
|
||||
"meta": {"custom_key": 42},
|
||||
"deps": [
|
||||
{
|
||||
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
|
||||
"size": 3,
|
||||
"hash": "md5",
|
||||
"path": path_func(external / "foo"),
|
||||
}
|
||||
],
|
||||
"outs": [
|
||||
{
|
||||
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
|
||||
"path": "foo_imported",
|
||||
"persist": True,
|
||||
"hash": "md5",
|
||||
"size": 3,
|
||||
"desc": "this is a description",
|
||||
"type": "model",
|
||||
"labels": ["label1", "label2"],
|
||||
"meta": {"custom_key": 42},
|
||||
"cache": False,
|
||||
"remote": "myremote",
|
||||
"push": False,
|
||||
}
|
||||
],
|
||||
}
|
||||
(tmp_dir / "foo_imported.dvc").dump(contents)
|
||||
(tmp_dir / "foo_imported").write_text("foo")
|
||||
|
||||
data_dir = tmp_dir / "data"
|
||||
data_dir.mkdir()
|
||||
|
||||
new_path = data_dir / "foo_moved"
|
||||
new_dvcfile = new_path.with_suffix(".dvc")
|
||||
dvc.move("foo_imported", os.fspath(new_path))
|
||||
|
||||
(stage,) = dvc.stage.collect(os.fspath(new_dvcfile))
|
||||
|
||||
with data_dir.chdir():
|
||||
expected_path = path_func(external / "foo")
|
||||
|
||||
contents["outs"][0] |= {"path": "foo_moved"}
|
||||
contents["deps"][0] |= {"path": expected_path}
|
||||
contents |= {"md5": stage.md5}
|
||||
assert new_dvcfile.parse() == contents
|
||||
|
||||
|
||||
def test_move_dst_stage_file_already_exists(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
|
||||
|
||||
with pytest.raises(StageFileAlreadyExistsError) as exc_info:
|
||||
dvc.move("foo", "bar")
|
||||
assert str(exc_info.value) == "'bar.dvc' already exists"
|
||||
assert exc_info.value.__cause__ is None
|
||||
|
||||
@@ -125,7 +125,7 @@ def assert_index_equal(first, second, strict=True, ordered=True):
|
||||
|
||||
|
||||
def test_skip_graph_checks(dvc, mocker):
|
||||
# See https://github.com/iterative/dvc/issues/2671 for more info
|
||||
# See https://github.com/treeverse/dvc/issues/2671 for more info
|
||||
mock_build_graph = mocker.spy(Index.graph, "fget")
|
||||
|
||||
# sanity check
|
||||
|
||||
@@ -241,7 +241,7 @@ def test_compose_and_dump_dir_module(
|
||||
|
||||
|
||||
def test_compose_and_dump_yaml_handles_string(tmp_dir):
|
||||
"""Regression test for https://github.com/iterative/dvc/issues/8583"""
|
||||
"""Regression test for https://github.com/treeverse/dvc/issues/8583"""
|
||||
from dvc.utils.hydra import compose_and_dump
|
||||
|
||||
config = tmp_dir / "conf" / "config.yaml"
|
||||
@@ -253,7 +253,7 @@ def test_compose_and_dump_yaml_handles_string(tmp_dir):
|
||||
|
||||
|
||||
def test_compose_and_dump_resolves_interpolation(tmp_dir):
|
||||
"""Regression test for https://github.com/iterative/dvc/issues/9196"""
|
||||
"""Regression test for https://github.com/treeverse/dvc/issues/9196"""
|
||||
from dvc.utils.hydra import compose_and_dump
|
||||
|
||||
config = tmp_dir / "conf" / "config.yaml"
|
||||
|
||||
@@ -59,7 +59,7 @@ def test_remote_missing_deps_are_correctly_reported(tmp_dir, caplog, mocker, pkg
|
||||
else:
|
||||
expected += (
|
||||
"\nPlease report this bug to "
|
||||
"<https://github.com/iterative/dvc/issues>. "
|
||||
"<https://github.com/treeverse/dvc/issues>. "
|
||||
"Thank you!"
|
||||
)
|
||||
assert expected in caplog.text
|
||||
|
||||
@@ -119,7 +119,7 @@ def test_show_ascii(repo):
|
||||
def test_show_dot(repo):
|
||||
# dot file rendering is not deterministic though graph
|
||||
# output doesn't depend upon order of lines. Use sorted values
|
||||
# https://github.com/iterative/dvc/pull/7725
|
||||
# https://github.com/treeverse/dvc/pull/7725
|
||||
expected = [
|
||||
"\"stage: '1'\";",
|
||||
"\"stage: '2'\";",
|
||||
|
||||
@@ -130,6 +130,7 @@ def test_experiments_run(dvc, scm, mocker):
|
||||
"tmp_dir": False,
|
||||
"copy_paths": [],
|
||||
"message": None,
|
||||
"no_hydra": False,
|
||||
}
|
||||
default_arguments.update(repro_arguments)
|
||||
|
||||
@@ -151,6 +152,7 @@ def test_experiments_run_message(dvc, scm, mocker, flag):
|
||||
"tmp_dir": False,
|
||||
"copy_paths": [],
|
||||
"message": "mymessage",
|
||||
"no_hydra": False,
|
||||
}
|
||||
default_arguments.update(repro_arguments)
|
||||
|
||||
|
||||
@@ -214,7 +214,7 @@ def test_read_params_py(tmp_dir, dvc):
|
||||
|
||||
|
||||
def test_params_py_tuple_status(tmp_dir, dvc):
|
||||
"""https://github.com/iterative/dvc/issues/8803"""
|
||||
"""https://github.com/treeverse/dvc/issues/8803"""
|
||||
parameters_file = "parameters.py"
|
||||
tmp_dir.gen(parameters_file, "TUPLE = (10, 100)\n")
|
||||
dep = ParamsDependency(Stage(dvc), parameters_file, ["TUPLE"])
|
||||
|
||||
@@ -42,7 +42,7 @@ def test_save_missing(dvc, mocker):
|
||||
), # hdfs checksum
|
||||
# Not much we can do about hex and oct values without writing our own
|
||||
# parser. So listing these test cases just to acknowledge this.
|
||||
# See https://github.com/iterative/dvc/issues/3331.
|
||||
# See https://github.com/treeverse/dvc/issues/3331.
|
||||
(0x3451, "13393"),
|
||||
(0o1244, "676"),
|
||||
],
|
||||
|
||||
@@ -153,3 +153,31 @@ def test_remote_with_jobs(dvc, base_url, fs_cls):
|
||||
assert config["user"] == "admin"
|
||||
assert f"{scheme}://{user}@example.com" in config["host"]
|
||||
assert cls is fs_cls
|
||||
|
||||
|
||||
def test_bearer_token_command(mocker):
|
||||
mock_auth_obj = mocker.Mock()
|
||||
mock_get_auth = mocker.patch(
|
||||
"dvc_webdav.get_bearer_auth", return_value=mock_auth_obj
|
||||
)
|
||||
|
||||
command = "my-token-cmd.sh"
|
||||
timeout = 10
|
||||
|
||||
# Configuration with potential conflicts
|
||||
config = {
|
||||
"url": url,
|
||||
"token": "static-token-value",
|
||||
"user": "conflict-user",
|
||||
"password": "conflict-password",
|
||||
"bearer_token_command": command,
|
||||
"timeout": timeout,
|
||||
}
|
||||
|
||||
fs = WebDAVFileSystem(**config)
|
||||
|
||||
# the factory function was invoked with the correct arguments from the config.
|
||||
mock_get_auth.assert_called_once_with(command, timeout)
|
||||
|
||||
# the generated auth object is correctly injected into `fs_args`.
|
||||
assert fs.fs_args["auth"] == mock_auth_obj
|
||||
|
||||
@@ -69,7 +69,7 @@ def session_app(tmp_path_factory) -> FSApp:
|
||||
"""Session scoped experiments queue celery app."""
|
||||
from kombu.transport.filesystem import Channel
|
||||
|
||||
# related to https://github.com/iterative/dvc-task/issues/61
|
||||
# related to https://github.com/treeverse/dvc-task/issues/61
|
||||
Channel.QoS.restore_at_shutdown = False
|
||||
|
||||
from dvc_task.app import FSApp
|
||||
|
||||
@@ -74,7 +74,7 @@ def test_locked(mocker):
|
||||
|
||||
|
||||
def test_skip_graph_checks(tmp_dir, dvc, mocker, run_copy):
|
||||
# See https://github.com/iterative/dvc/issues/2671 for more info
|
||||
# See https://github.com/treeverse/dvc/issues/2671 for more info
|
||||
from dvc.repo.index import Index
|
||||
|
||||
mock_build_graph = mocker.spy(Index.graph, "fget")
|
||||
|
||||
@@ -167,16 +167,16 @@ def test_system_info():
|
||||
@pytest.mark.parametrize(
|
||||
"git_remote",
|
||||
[
|
||||
"git://github.com/iterative/dvc.git",
|
||||
"[email protected]:iterative/dvc.git",
|
||||
"http://github.com/iterative/dvc.git",
|
||||
"https://github.com/iterative/dvc.git",
|
||||
"ssh://[email protected]/iterative/dvc.git",
|
||||
"git://github.com/treeverse/dvc.git",
|
||||
"[email protected]:treeverse/dvc.git",
|
||||
"http://github.com/treeverse/dvc.git",
|
||||
"https://github.com/treeverse/dvc.git",
|
||||
"ssh://[email protected]/treeverse/dvc.git",
|
||||
],
|
||||
)
|
||||
def test_git_remote_hash(mocker, git_remote):
|
||||
m = mocker.patch("dvc.analytics._git_remote_url", return_value=git_remote)
|
||||
expected = hashlib.md5(b"iterative/dvc.git", usedforsecurity=False).hexdigest()
|
||||
expected = hashlib.md5(b"treeverse/dvc.git", usedforsecurity=False).hexdigest()
|
||||
|
||||
assert analytics._git_remote_path_hash(None) == expected
|
||||
m.assert_called_once_with(None)
|
||||
|
||||
@@ -158,14 +158,14 @@ def test_check(mocker, updater):
|
||||
None,
|
||||
(
|
||||
"Find the latest release at "
|
||||
"https://github.com/iterative/dvc/releases/latest."
|
||||
"https://github.com/treeverse/dvc/releases/latest."
|
||||
),
|
||||
),
|
||||
(
|
||||
"unknown",
|
||||
(
|
||||
"Find the latest release at "
|
||||
"https://github.com/iterative/dvc/releases/latest."
|
||||
"https://github.com/treeverse/dvc/releases/latest."
|
||||
),
|
||||
),
|
||||
],
|
||||
|
||||
@@ -46,7 +46,7 @@ def test_notify_refs(mocker, status_code, side_effect):
|
||||
mock_post = mocker.patch("requests.Session.post", return_value=response)
|
||||
|
||||
notify_refs(
|
||||
"[email protected]:iterative/dvc.git",
|
||||
"[email protected]:treeverse/dvc.git",
|
||||
"TOKEN",
|
||||
pushed=["p1", "p2"],
|
||||
removed=["r1", "r2"],
|
||||
@@ -56,7 +56,7 @@ def test_notify_refs(mocker, status_code, side_effect):
|
||||
assert mock_post.call_args == mocker.call(
|
||||
urljoin(STUDIO_URL, "/webhook/dvc"),
|
||||
json={
|
||||
"repo_url": "[email protected]:iterative/dvc.git",
|
||||
"repo_url": "[email protected]:treeverse/dvc.git",
|
||||
"client": "dvc",
|
||||
"refs": {
|
||||
"pushed": ["p1", "p2"],
|
||||
|
||||
Reference in New Issue
Block a user