mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 23:09:28 +01:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a5ae8b19a4 | ||
|
|
f2b1c41cdf | ||
|
|
ffd32dfe8a |
@@ -737,8 +737,8 @@
|
||||
"tests/func/test_config.py::test_config_gdrive_fields": 0.12703910000016094,
|
||||
"tests/func/test_config.py::test_config_get[args0-0-False]": 0.20948869999995168,
|
||||
"tests/func/test_config.py::test_config_get[args1-0-myremote]": 0.1829517000001033,
|
||||
"tests/func/test_config.py::test_config_get[args2-0-treeverse]": 0.19686669999998685,
|
||||
"tests/func/test_config.py::test_config_get[args3-0-treeverse]": 0.1675059000000374,
|
||||
"tests/func/test_config.py::test_config_get[args2-0-iterative]": 0.19686669999998685,
|
||||
"tests/func/test_config.py::test_config_get[args3-0-iterative]": 0.1675059000000374,
|
||||
"tests/func/test_config.py::test_config_get[args4-251-option 'profile' doesn't exist]": 0.18096200000002227,
|
||||
"tests/func/test_config.py::test_config_get[args5-0-gs://bucket/path]": 0.12368970000022728,
|
||||
"tests/func/test_config.py::test_config_get[args6-0-gs://bucket/path]": 0.13007389999995667,
|
||||
@@ -2642,7 +2642,7 @@
|
||||
"tests/unit/test_updater.py::test_is_enabled[config0-True]": 0.13130560000013247,
|
||||
"tests/unit/test_updater.py::test_is_enabled[config1-True]": 0.14342600000009043,
|
||||
"tests/unit/test_updater.py::test_is_enabled[config2-False]": 0.13593219999984285,
|
||||
"tests/unit/test_updater.py::test_notify_message[None-Find the latest release at https://github.com/treeverse/dvc/releases/latest.]": 0.014024399999925663,
|
||||
"tests/unit/test_updater.py::test_notify_message[None-Find the latest release at https://github.com/iterative/dvc/releases/latest.]": 0.014024399999925663,
|
||||
"tests/unit/test_updater.py::test_notify_message[binary-To upgrade, uninstall dvc and reinstall from https://dvc.org.]": 0.013217200000326557,
|
||||
"tests/unit/test_updater.py::test_notify_message[brew-To upgrade, run 'brew upgrade dvc'.]": 0.013467300000229443,
|
||||
"tests/unit/test_updater.py::test_notify_message[choco-To upgrade, run 'choco upgrade dvc'.]": 0.014645900000004985,
|
||||
@@ -2652,7 +2652,7 @@
|
||||
"tests/unit/test_updater.py::test_notify_message[osxpkg-To upgrade, uninstall dvc and reinstall from https://dvc.org.]": 0.013980499999888707,
|
||||
"tests/unit/test_updater.py::test_notify_message[pip-To upgrade, run 'pip install --upgrade dvc'.]": 0.015155299999833005,
|
||||
"tests/unit/test_updater.py::test_notify_message[rpm-To upgrade, run 'yum update dvc'.]": 0.014036500000202068,
|
||||
"tests/unit/test_updater.py::test_notify_message[unknown-Find the latest release at https://github.com/treeverse/dvc/releases/latest.]": 0.01609110000003966,
|
||||
"tests/unit/test_updater.py::test_notify_message[unknown-Find the latest release at https://github.com/iterative/dvc/releases/latest.]": 0.01609110000003966,
|
||||
"tests/unit/ui/test_console.py::test_capsys_works": 0.003642499999841675,
|
||||
"tests/unit/ui/test_console.py::test_write": 0.004320600000255581,
|
||||
"tests/unit/ui/test_console.py::test_write_json[False-{\"hello\": \"world\", \"date\": \"1970-01-01 00:00:00\"}\\n]": 0.004433300000073359,
|
||||
|
||||
@@ -57,7 +57,7 @@ $ dvc doctor
|
||||
**Additional Information (if any):**
|
||||
|
||||
<!--
|
||||
Please check https://github.com/treeverse/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.
|
||||
Please check https://github.com/iterative/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.
|
||||
|
||||
If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.
|
||||
If the issue is regarding the performance, please attach the profiling information and the benchmark comparisons.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
* [ ] ❗ I have followed the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) checklist.
|
||||
|
||||
* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/treeverse/dvc.org) and linked it here.
|
||||
* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here.
|
||||
|
||||
Thank you for the contribution - we'll try to review it as soon as possible. 🙏
|
||||
|
||||
@@ -14,16 +14,16 @@ jobs:
|
||||
name: run benchmarks
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/setup-python@v7
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.14"
|
||||
python-version: "3.13"
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: pyproject.toml
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
- name: run benchmarks on base branch
|
||||
run: pytest --benchmark-autosave dvc/testing/benchmarks/ -k 'test_init or test_help'
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
clean: false
|
||||
|
||||
@@ -19,15 +19,15 @@ jobs:
|
||||
name: Build & Verify package
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-python@v7
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.14"
|
||||
python-version: 3.13
|
||||
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
|
||||
- name: Omit local version for Test PyPI upload
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
- name: Check dist
|
||||
run: uv tool run twine check --strict dist/*
|
||||
|
||||
- uses: actions/upload-artifact@v7
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: Packages
|
||||
path: dist/
|
||||
@@ -51,7 +51,7 @@ jobs:
|
||||
name: Publish dev package to test.pypi.org
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
if: ${{ github.event.action == 'published' || (github.repository == 'treeverse/dvc' && github.event_name == 'push' && github.ref == 'refs/heads/main') }}
|
||||
if: ${{ github.event.action == 'published' || (github.repository == 'iterative/dvc' && github.event_name == 'push' && github.ref == 'refs/heads/main') }}
|
||||
|
||||
environment:
|
||||
name: test-pypi
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- uses: actions/download-artifact@v8
|
||||
- uses: actions/download-artifact@v5
|
||||
with:
|
||||
name: Packages
|
||||
path: dist
|
||||
@@ -86,7 +86,7 @@ jobs:
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- uses: actions/download-artifact@v8
|
||||
- uses: actions/download-artifact@v5
|
||||
with:
|
||||
name: Packages
|
||||
path: dist
|
||||
|
||||
@@ -28,13 +28,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4.37.4
|
||||
uses: github/codeql-action/init@v3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
queries: security-extended
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v4.37.4
|
||||
uses: github/codeql-action/analyze@v3
|
||||
|
||||
@@ -19,26 +19,26 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest]
|
||||
pyv: ["3.14"]
|
||||
pyv: ["3.13"]
|
||||
plugin: ["dvc-s3"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
path: dvc
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
repository: treeverse/${{ matrix.plugin }}
|
||||
repository: iterative/${{ matrix.plugin }}
|
||||
ref: main
|
||||
path: ${{ matrix.plugin }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.pyv }}
|
||||
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: "**/pyproject.toml"
|
||||
|
||||
@@ -26,14 +26,14 @@ jobs:
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.14'
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
python-version: '3.13'
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: pyproject.toml
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
UV_LINK_MODE: ${{ runner.os == 'Windows' && 'symlink' || 'hardlink' }}
|
||||
|
||||
- name: Cache mypy
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: .mypy_cache
|
||||
key: mypy-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest]
|
||||
pyv: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
|
||||
pyv: ["3.9", "3.10", "3.11", "3.12", "3.13"]
|
||||
include:
|
||||
- os: windows-latest
|
||||
pyv: "3.9"
|
||||
@@ -117,27 +117,15 @@ jobs:
|
||||
- os: windows-latest
|
||||
pyv: "3.13"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
|
||||
- os: windows-latest
|
||||
pyv: "3.14"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
|
||||
- os: windows-latest
|
||||
pyv: "3.14"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
|
||||
- os: windows-latest
|
||||
pyv: "3.14"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
|
||||
- os: windows-latest
|
||||
pyv: "3.14"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.pyv }}
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: pyproject.toml
|
||||
@@ -155,7 +143,7 @@ jobs:
|
||||
pytest ${{ matrix.pytestargs }} -n=logical --dist=worksteal --timeout=300 --durations=0
|
||||
--cov --cov-report=xml --cov-report=term --durations-path=./.github/.test_durations
|
||||
- name: upload coverage report
|
||||
uses: codecov/codecov-action@v7
|
||||
uses: codecov/codecov-action@v5
|
||||
check:
|
||||
if: always()
|
||||
needs: [tests]
|
||||
@@ -170,7 +158,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Slack Notification
|
||||
uses: rtCamp/action-slack-notify@v2.4.0
|
||||
uses: rtCamp/action-slack-notify@v2.3.3
|
||||
env:
|
||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
||||
SLACK_COLOR: ${{ job.status }}
|
||||
|
||||
@@ -22,13 +22,13 @@ repos:
|
||||
- id: sort-simple-yaml
|
||||
- id: trailing-whitespace
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: 'v0.15.20'
|
||||
rev: 'v0.13.2'
|
||||
hooks:
|
||||
- id: ruff-check
|
||||
args: [--fix, --exit-non-zero-on-fix]
|
||||
- id: ruff-format
|
||||
- repo: https://github.com/codespell-project/codespell
|
||||
rev: v2.4.2
|
||||
rev: v2.4.1
|
||||
hooks:
|
||||
- id: codespell
|
||||
additional_dependencies: ["tomli"]
|
||||
|
||||
+1
-1
@@ -4,5 +4,5 @@
|
||||
"data-science", "data-version-control", "machine-learning", "git",
|
||||
"developer-tools", "reproducibility", "collaboration", "ai", "python"],
|
||||
"contributors": [
|
||||
{"name": "DVC team", "type": "Other", "affiliation": "Treeverse"}]
|
||||
{"name": "DVC team", "type": "Other", "affiliation": "Iterative"}]
|
||||
}
|
||||
|
||||
+2
-2
@@ -6,8 +6,8 @@ message: >-
|
||||
type: software
|
||||
authors:
|
||||
- name: The DVC team and contributors
|
||||
website: 'https://github.com/treeverse'
|
||||
repository-code: 'https://github.com/treeverse/dvc'
|
||||
website: 'https://github.com/iterative'
|
||||
repository-code: 'https://github.com/iterative/dvc'
|
||||
url: 'https://dvc.org'
|
||||
abstract: >-
|
||||
DVC is a tool for data science that takes advantage of existing software engineering toolset.
|
||||
|
||||
@@ -186,7 +186,7 @@
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2025 Treeverse.
|
||||
Copyright 2017-2021 Iterative, Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
+15
-9
@@ -167,7 +167,7 @@ To install the development version, run:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pip install "dvc @ git+https://github.com/treeverse/dvc"
|
||||
pip install git+git://github.com/iterative/dvc
|
||||
|
||||
Package (Platform-specific)
|
||||
---------------------------
|
||||
@@ -175,7 +175,7 @@ Package (Platform-specific)
|
||||
|Packages|
|
||||
|
||||
Self-contained packages for Linux, Windows, and Mac are available.
|
||||
The latest version of the packages can be found on the GitHub `releases page <https://github.com/treeverse/dvc/releases>`_.
|
||||
The latest version of the packages can be found on the GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
|
||||
|
||||
Ubuntu / Debian (deb)
|
||||
^^^^^^^^^^^^^^^^^^^^^
|
||||
@@ -198,6 +198,8 @@ Fedora / CentOS (rpm)
|
||||
Contributing
|
||||
============
|
||||
|
||||
|Maintainability|
|
||||
|
||||
Contributions are welcome!
|
||||
Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more details.
|
||||
Thanks to all our contributors!
|
||||
@@ -235,19 +237,23 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
|
||||
:target: https://dvc.org
|
||||
:alt: DVC logo
|
||||
|
||||
.. |VS Code Extension Overview| image:: https://raw.githubusercontent.com/treeverse/vscode-dvc/main/extension/docs/overview.gif
|
||||
.. |VS Code Extension Overview| image:: https://raw.githubusercontent.com/iterative/vscode-dvc/main/extension/docs/overview.gif
|
||||
:alt: DVC Extension for VS Code
|
||||
|
||||
.. |CI| image:: https://github.com/treeverse/dvc/actions/workflows/tests.yaml/badge.svg
|
||||
:target: https://github.com/treeverse/dvc/actions/workflows/tests.yaml
|
||||
.. |CI| image:: https://github.com/iterative/dvc/actions/workflows/tests.yaml/badge.svg
|
||||
:target: https://github.com/iterative/dvc/actions/workflows/tests.yaml
|
||||
:alt: GHA Tests
|
||||
|
||||
.. |Maintainability| image:: https://img.shields.io/codeclimate/maintainability/iterative/dvc
|
||||
:target: https://codeclimate.com/github/iterative/dvc
|
||||
:alt: Code Climate
|
||||
|
||||
.. |Python Version| image:: https://img.shields.io/pypi/pyversions/dvc
|
||||
:target: https://pypi.org/project/dvc
|
||||
:alt: Python Version
|
||||
|
||||
.. |Coverage| image:: https://codecov.io/gh/treeverse/dvc/branch/main/graph/badge.svg
|
||||
:target: https://codecov.io/gh/treeverse/dvc
|
||||
.. |Coverage| image:: https://codecov.io/gh/iterative/dvc/branch/main/graph/badge.svg
|
||||
:target: https://codecov.io/gh/iterative/dvc
|
||||
:alt: Codecov
|
||||
|
||||
.. |Snap| image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
|
||||
@@ -286,8 +292,8 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
|
||||
:target: https://dvc.org/img/flow.gif
|
||||
:alt: how_dvc_works
|
||||
|
||||
.. |Contribs| image:: https://contrib.rocks/image?repo=treeverse/dvc
|
||||
:target: https://github.com/treeverse/dvc/graphs/contributors
|
||||
.. |Contribs| image:: https://contrib.rocks/image?repo=iterative/dvc
|
||||
:target: https://github.com/iterative/dvc/graphs/contributors
|
||||
:alt: Contributors
|
||||
|
||||
.. |VS Code| image:: https://img.shields.io/visual-studio-marketplace/v/Iterative.dvc?color=blue&label=VSCode&logo=visualstudiocode&logoColor=blue
|
||||
|
||||
@@ -8,7 +8,7 @@ from PyInstaller.utils.hooks import collect_submodules, is_module_or_submodule
|
||||
_EXCLUDES = ("celery.bin", "celery.contrib")
|
||||
hiddenimports = collect_submodules(
|
||||
"celery",
|
||||
filter=lambda name: (
|
||||
not any(is_module_or_submodule(name, module) for module in _EXCLUDES)
|
||||
filter=lambda name: not any(
|
||||
is_module_or_submodule(name, module) for module in _EXCLUDES
|
||||
),
|
||||
)
|
||||
|
||||
@@ -8,6 +8,7 @@ datas += copy_metadata("gcsfs")
|
||||
datas += copy_metadata("pyarrow")
|
||||
datas += copy_metadata("pydrive2")
|
||||
datas += copy_metadata("s3fs", recursive=True)
|
||||
datas += copy_metadata("boto3")
|
||||
datas += copy_metadata("ossfs")
|
||||
datas += copy_metadata("sshfs")
|
||||
datas += copy_metadata("webdav4")
|
||||
|
||||
+1
-1
@@ -206,7 +206,7 @@ def add_debugging_flags(parser):
|
||||
from argparse import SUPPRESS
|
||||
|
||||
# For detailed info see:
|
||||
# https://github.com/treeverse/dvc/wiki/Debugging,-Profiling-and-Benchmarking-DVC
|
||||
# https://github.com/iterative/dvc/wiki/Debugging,-Profiling-and-Benchmarking-DVC
|
||||
args, _ = parser.parse_known_args()
|
||||
verbose = args.verbose
|
||||
|
||||
|
||||
+1
-1
@@ -147,7 +147,7 @@ def _git_remote_path_hash(scm: Optional["Base"]) -> Optional[str]:
|
||||
path = _parse_git_remote_path(remote_url)
|
||||
h = hashlib.md5(path.encode("utf-8"), usedforsecurity=False) # for FIPS
|
||||
return h.hexdigest()
|
||||
except Exception:
|
||||
except Exception: # noqa: BLE001
|
||||
logger.debug("Failed to get git remote path", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
+1
-1
@@ -96,7 +96,7 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
|
||||
f"See {link} for more info."
|
||||
)
|
||||
else:
|
||||
link = format_link("https://github.com/treeverse/dvc/issues")
|
||||
link = format_link("https://github.com/iterative/dvc/issues")
|
||||
hint = f"\nPlease report this bug to {link}. Thank you!"
|
||||
|
||||
logger.exception(
|
||||
|
||||
+130
-6
@@ -31,6 +31,27 @@ _dvc_compgen_remotes() {
|
||||
_dvc_compgen_config_vars() {
|
||||
compgen -W "${_dvc_config_vars[*]}" -- $1
|
||||
}
|
||||
|
||||
_dvc_compgen_remote_config_vars() {
|
||||
local cur prev words cword remote_name filtered
|
||||
_init_completion || return
|
||||
|
||||
# last non-option word before cursor
|
||||
local nonopts=()
|
||||
for w in "${words[@]:1:$((cword-1))}"; do
|
||||
[[ $w == -* ]] || nonopts+=("$w")
|
||||
done
|
||||
|
||||
remote_name="${nonopts[-1]}"
|
||||
if [[ -n $remote_name ]]; then
|
||||
local _dvc_remote_config_vars=($(
|
||||
dvc config --available-options -q 2> /dev/null \
|
||||
| grep "^remote\\.${remote_name}\\." \
|
||||
| sed "s/^remote\\.${remote_name}\\.//"
|
||||
))
|
||||
compgen -W "${_dvc_remote_config_vars[*]}" -- $1
|
||||
fi
|
||||
}
|
||||
"""
|
||||
|
||||
ZSH_PREAMBLE = """
|
||||
@@ -69,27 +90,111 @@ _dvc_compadd_remotes() {
|
||||
_describe 'remotes' "($(dvc remote list | cut -d' ' -f1))"
|
||||
}
|
||||
|
||||
_dvc_config_config_vars_for_completion() {
|
||||
dvc config --available-options -q 2> /dev/null
|
||||
}
|
||||
|
||||
_dvc_compadd_config_vars() {
|
||||
_describe 'config_vars' _dvc_config_vars
|
||||
}
|
||||
|
||||
_dvc_compadd_remote_config_vars() {
|
||||
local remote_name filtered
|
||||
|
||||
# last non-option word before cursor
|
||||
remote_name=${${words[1,CURRENT-1]##-*}[-1]}
|
||||
if [[ -n $remote_name ]]; then
|
||||
filtered=$(_dvc_config_config_vars_for_completion \\
|
||||
| grep "^remote.${remote_name}\\\\." \\
|
||||
| sed "s/^remote\\\\.${remote_name}\\\\.//"
|
||||
)
|
||||
compadd -- $=filtered
|
||||
fi
|
||||
}
|
||||
"""
|
||||
|
||||
FISH_PREAMBLE = """
|
||||
function __fish_complete_dvc_files
|
||||
__fish_complete_path | string match -re '\\*?.dvc|Dvcfile|dvc\\.yaml'
|
||||
end
|
||||
|
||||
function __fish_complete_dvc_stages
|
||||
for line in (dvc stage list -q)
|
||||
set -l parts (string split -m1 ' ' -- $line)
|
||||
set -l name $parts[1]
|
||||
set -l desc (string trim $parts[2])
|
||||
echo -e "$name $desc"
|
||||
end
|
||||
end
|
||||
|
||||
function __fish_complete_dvc_stages_and_files
|
||||
__fish_complete_dvc_stages
|
||||
__fish_complete_dvc_files
|
||||
end
|
||||
|
||||
function __fish_complete_dvc_experiments
|
||||
dvc exp list -q --all-commits --names-only
|
||||
end
|
||||
|
||||
function __fish_complete_dvc_remotes
|
||||
dvc remote list | cut -d' ' -f1
|
||||
end
|
||||
|
||||
function __fish_complete_dvc_remote_config_vars
|
||||
# last non-option word before cursor
|
||||
set -l remote_name (commandline -opc | string match -rv '^-' | tail -n 1)
|
||||
|
||||
if test -n "$remote_name"
|
||||
dvc config --available-options -q 2>/dev/null |
|
||||
string match -re "^remote\\.$remote_name\\." |
|
||||
string replace -r "^remote\\.$remote_name\\." ''
|
||||
end
|
||||
end
|
||||
"""
|
||||
|
||||
PREAMBLE = {
|
||||
"bash": BASH_PREAMBLE,
|
||||
"zsh": ZSH_PREAMBLE,
|
||||
"fish": FISH_PREAMBLE,
|
||||
}
|
||||
|
||||
FILE = shtab.FILE
|
||||
DIR = shtab.DIRECTORY
|
||||
DVC_FILE = {"bash": "_dvc_compgen_DVCFiles", "zsh": "_dvc_compadd_DVCFiles"}
|
||||
STAGE = {"bash": "_dvc_compgen_stages", "zsh": "_dvc_compadd_stages"}
|
||||
DVC_FILE = {
|
||||
"bash": "_dvc_compgen_DVCFiles",
|
||||
"zsh": "_dvc_compadd_DVCFiles",
|
||||
"fish": '-f -a "(__fish_complete_dvc_files)"',
|
||||
}
|
||||
STAGE = {
|
||||
"bash": "_dvc_compgen_stages",
|
||||
"zsh": "_dvc_compadd_stages",
|
||||
"fish": '-f -a "(__fish_complete_dvc_stages)"',
|
||||
}
|
||||
DVCFILES_AND_STAGE = {
|
||||
"bash": "_dvc_compgen_stages_and_files",
|
||||
"zsh": "_dvc_compadd_stages_and_files",
|
||||
"fish": '-f -a "(__fish_complete_dvc_stages_and_files)"',
|
||||
}
|
||||
EXPERIMENT = {
|
||||
"bash": "_dvc_compgen_exps",
|
||||
"zsh": "_dvc_compadd_exps",
|
||||
"fish": '-f -a "(__fish_complete_dvc_experiments)"',
|
||||
}
|
||||
REMOTE = {
|
||||
"bash": "_dvc_compgen_remotes",
|
||||
"zsh": "_dvc_compadd_remotes",
|
||||
"fish": '-f -a "(__fish_complete_dvc_remotes)"',
|
||||
}
|
||||
CONFIG_VARS = {
|
||||
"bash": "_dvc_compgen_config_vars",
|
||||
"zsh": "_dvc_compadd_config_vars",
|
||||
"fish": '-f -a "(__fish_complete_dvc_config_vars)"',
|
||||
}
|
||||
REMOTE_CONFIG_VARS = {
|
||||
"bash": "_dvc_compgen_remote_config_vars",
|
||||
"zsh": "_dvc_compadd_remote_config_vars",
|
||||
"fish": '-f -a "(__fish_complete_dvc_remote_config_vars)"',
|
||||
}
|
||||
EXPERIMENT = {"bash": "_dvc_compgen_exps", "zsh": "_dvc_compadd_exps"}
|
||||
REMOTE = {"bash": "_dvc_compgen_remotes", "zsh": "_dvc_compadd_remotes"}
|
||||
CONFIG_VARS = {"bash": "_dvc_compgen_config_vars", "zsh": "_dvc_compadd_config_vars"}
|
||||
|
||||
|
||||
def get_preamble() -> dict[str, str]:
|
||||
@@ -103,7 +208,26 @@ def get_preamble() -> dict[str, str]:
|
||||
_dvc_config_vars=(
|
||||
{nl.join(config_vars)}
|
||||
)
|
||||
"""
|
||||
indent = "\t\t".expandtabs(4) # 8 spaces
|
||||
lines = (
|
||||
"\n".join(
|
||||
f"{indent}{c} \\"
|
||||
for c in config_vars[:-1] # all but last
|
||||
)
|
||||
+ "\n"
|
||||
+ f"{indent}{config_vars[-1]}"
|
||||
) # last line without backslash
|
||||
config_vars_arr_fish = f"""
|
||||
function __fish_complete_dvc_config_vars
|
||||
set -l _dvc_config_vars \\
|
||||
{lines}
|
||||
printf %s\\n $_dvc_config_vars
|
||||
end
|
||||
"""
|
||||
for shell, preamble in PREAMBLE.items():
|
||||
ret[shell] = config_vars_arr + preamble
|
||||
if shell != "fish":
|
||||
ret[shell] = config_vars_arr + preamble
|
||||
else:
|
||||
ret[shell] = config_vars_arr_fish + preamble
|
||||
return ret
|
||||
|
||||
@@ -8,7 +8,7 @@ from dvc.ui import ui
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
||||
SUPPORTED_SHELLS = ["bash", "zsh"]
|
||||
SUPPORTED_SHELLS = ["bash", "zsh", "fish"]
|
||||
|
||||
|
||||
class CmdCompletion(CmdBaseNoRepo):
|
||||
|
||||
@@ -40,6 +40,15 @@ class CmdConfig(CmdBaseNoRepo):
|
||||
self.config = Config.from_cwd(validate=False)
|
||||
|
||||
def run(self):
|
||||
if self.args.available_options:
|
||||
from dvc.config_schema import contextual_config_vars_for_completion
|
||||
|
||||
ui.write(
|
||||
"\n".join(list(contextual_config_vars_for_completion(self.config))),
|
||||
force=True,
|
||||
)
|
||||
return 0
|
||||
|
||||
if self.args.show_origin and (self.args.value or self.args.unset):
|
||||
logger.error(
|
||||
"--show-origin can't be used together with any of these "
|
||||
@@ -212,6 +221,9 @@ def add_parser(subparsers, parent_parser):
|
||||
help=CONFIG_HELP,
|
||||
formatter_class=formatter.RawDescriptionHelpFormatter,
|
||||
)
|
||||
config_parser.add_argument(
|
||||
"--available-options", default=False, action="store_true"
|
||||
)
|
||||
config_parser.add_argument(
|
||||
"-u",
|
||||
"--unset",
|
||||
|
||||
+2
-71
@@ -127,80 +127,19 @@ def _filter(graph, targets, full):
|
||||
return new_graph
|
||||
|
||||
|
||||
def _is_foreach_matrix_stage(node, join_string):
|
||||
if node.endswith(".dvc"):
|
||||
return False
|
||||
return join_string in node
|
||||
|
||||
|
||||
def _collapse_foreach_matrix_get_nodes(graph):
|
||||
from dvc.parsing import JOIN
|
||||
|
||||
new_nodes = set()
|
||||
nodes_to_remove = set()
|
||||
for _node in list(graph.nodes):
|
||||
if not _is_foreach_matrix_stage(_node, JOIN):
|
||||
continue
|
||||
nodes_to_remove.add(_node)
|
||||
new_nodes.add(_node.split(JOIN)[0])
|
||||
return new_nodes, nodes_to_remove
|
||||
|
||||
|
||||
def _collapse_foreach_matrix_get_edges(graph):
|
||||
from dvc.parsing import JOIN
|
||||
|
||||
new_edges = set()
|
||||
edges_to_remove = set()
|
||||
for _e1, _e2 in list(graph.edges):
|
||||
_replace = False
|
||||
_new_e1 = _e1
|
||||
_new_e2 = _e2
|
||||
if _is_foreach_matrix_stage(_e1, JOIN):
|
||||
_new_e1 = _e1.split(JOIN)[0]
|
||||
_replace = True
|
||||
if _is_foreach_matrix_stage(_e2, JOIN):
|
||||
_new_e2 = _e2.split(JOIN)[0]
|
||||
_replace = True
|
||||
if _replace:
|
||||
edges_to_remove.add((_e1, _e2))
|
||||
new_edges.add((_new_e1, _new_e2))
|
||||
return new_edges, edges_to_remove
|
||||
|
||||
|
||||
def _collapse_foreach_matrix(graph):
|
||||
new_nodes, nodes_to_remove = _collapse_foreach_matrix_get_nodes(graph)
|
||||
new_edges, edges_to_remove = _collapse_foreach_matrix_get_edges(graph)
|
||||
new_graph = graph.copy()
|
||||
new_graph.remove_edges_from(edges_to_remove)
|
||||
new_graph.add_nodes_from(new_nodes)
|
||||
new_graph.add_edges_from(new_edges)
|
||||
new_graph.remove_nodes_from(nodes_to_remove)
|
||||
return new_graph
|
||||
|
||||
|
||||
def _build(repo, target=None, full=False, outs=False, collapse_foreach_matrix=False):
|
||||
def _build(repo, target=None, full=False, outs=False):
|
||||
targets = _collect_targets(repo, target, outs)
|
||||
graph = _transform(repo.index, outs)
|
||||
filtered_graph = _filter(graph, targets, full)
|
||||
if collapse_foreach_matrix:
|
||||
filtered_graph = _collapse_foreach_matrix(filtered_graph)
|
||||
return filtered_graph
|
||||
return _filter(graph, targets, full)
|
||||
|
||||
|
||||
class CmdDAG(CmdBase):
|
||||
def run(self):
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
|
||||
if self.args.outs and self.args.collapse_foreach_matrix:
|
||||
raise InvalidArgumentError(
|
||||
"`--outs` and `--collapse-foreach-matrix` are mutually exclusive"
|
||||
)
|
||||
graph = _build(
|
||||
self.repo,
|
||||
target=self.args.target,
|
||||
full=self.args.full,
|
||||
outs=self.args.outs,
|
||||
collapse_foreach_matrix=self.args.collapse_foreach_matrix,
|
||||
)
|
||||
|
||||
if self.args.dot:
|
||||
@@ -242,14 +181,6 @@ def add_parser(subparsers, parent_parser):
|
||||
dest="markdown",
|
||||
help="Print DAG with mermaid format wrapped in Markdown block.",
|
||||
)
|
||||
dag_parser.add_argument(
|
||||
"--collapse-foreach-matrix",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"Collapse stages from each foreach/matrix definition into a single node."
|
||||
),
|
||||
)
|
||||
dag_parser.add_argument(
|
||||
"--full",
|
||||
action="store_true",
|
||||
|
||||
@@ -20,7 +20,6 @@ class CmdExperimentsRun(CmdRepro):
|
||||
tmp_dir=self.args.tmp_dir,
|
||||
copy_paths=self.args.copy_paths,
|
||||
message=self.args.message,
|
||||
no_hydra=self.args.no_hydra,
|
||||
**self._common_kwargs,
|
||||
)
|
||||
|
||||
@@ -108,14 +107,4 @@ def _add_run_common(parser):
|
||||
default=None,
|
||||
help="Custom commit message to use when committing the experiment.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-hydra",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"Disables automatically updating `params.yaml` with Hydra configuration. "
|
||||
" You can still use `--set-param` to update individual params if needed."
|
||||
" Default is False."
|
||||
),
|
||||
)
|
||||
parser.add_argument("-M", dest="message", help=argparse.SUPPRESS) # obsolete
|
||||
|
||||
+4
-11
@@ -40,16 +40,9 @@ class CmdGC(CmdBase):
|
||||
msg += " and all git commits"
|
||||
else:
|
||||
if self.args.all_branches and self.args.all_tags:
|
||||
if self.args.num:
|
||||
msg += f" and last {self.args.num} commits from all git branches"
|
||||
msg += " and all git tags"
|
||||
else:
|
||||
msg += " and all git branches and tags"
|
||||
msg += " and all git branches and tags"
|
||||
elif self.args.all_branches:
|
||||
if self.args.num:
|
||||
msg += f" and last {self.args.num} commits from all git branches"
|
||||
else:
|
||||
msg += " and all git branches"
|
||||
msg += " and all git branches"
|
||||
elif self.args.all_tags:
|
||||
msg += " and all git tags"
|
||||
if self.args.commit_date:
|
||||
@@ -133,8 +126,8 @@ def add_parser(subparsers, parent_parser):
|
||||
metavar="<num>",
|
||||
help=(
|
||||
"Keep data files used in the last `num` commits "
|
||||
"starting from each selected revision root. "
|
||||
"Can be used with `--rev` and `--all-branches`. "
|
||||
"starting from the `--rev` <commit>. "
|
||||
"Only used if `--rev` is also provided. "
|
||||
"Defaults to `1`."
|
||||
),
|
||||
)
|
||||
|
||||
@@ -29,7 +29,7 @@ def _welcome_message():
|
||||
"{yellow}------------{nc}\n"
|
||||
f"- Check out the documentation: {fmt_link('https://dvc.org/doc')}\n"
|
||||
f"- Get help and share ideas: {fmt_link('https://dvc.org/chat')}\n"
|
||||
f"- Star us on GitHub: {fmt_link('https://github.com/treeverse/dvc')}"
|
||||
f"- Star us on GitHub: {fmt_link('https://github.com/iterative/dvc')}"
|
||||
).format(yellow=colorama.Fore.YELLOW, nc=colorama.Fore.RESET)
|
||||
|
||||
ui.write(msg)
|
||||
|
||||
@@ -109,6 +109,8 @@ def _build_tree_structure(
|
||||
|
||||
|
||||
def show_tree(entries, with_color=False, with_size=False, with_hash=False):
|
||||
import tabulate
|
||||
|
||||
rows = _build_tree_structure(
|
||||
entries,
|
||||
with_color=with_color,
|
||||
@@ -117,7 +119,13 @@ def show_tree(entries, with_color=False, with_size=False, with_hash=False):
|
||||
)
|
||||
|
||||
colalign = ("right",) if with_size else None
|
||||
ui.table(rows, colalign=colalign, preserve_whitespace=True)
|
||||
|
||||
_orig = tabulate.PRESERVE_WHITESPACE
|
||||
tabulate.PRESERVE_WHITESPACE = True
|
||||
try:
|
||||
ui.table(rows, colalign=colalign)
|
||||
finally:
|
||||
tabulate.PRESERVE_WHITESPACE = _orig
|
||||
|
||||
|
||||
class CmdList(CmdBaseNoRepo):
|
||||
|
||||
@@ -233,7 +233,9 @@ def add_parser(subparsers, parent_parser):
|
||||
remote_modify_parser.add_argument(
|
||||
"name", help="Name of the remote"
|
||||
).complete = completion.REMOTE
|
||||
remote_modify_parser.add_argument("option", help="Name of the option to modify.")
|
||||
remote_modify_parser.add_argument(
|
||||
"option", help="Name of the option to modify."
|
||||
).complete = completion.REMOTE_CONFIG_VARS
|
||||
remote_modify_parser.add_argument(
|
||||
"value", nargs="?", help="(optional) Value of the option."
|
||||
)
|
||||
|
||||
@@ -41,6 +41,18 @@ class RemoteNotFoundError(RemoteConfigError):
|
||||
pass
|
||||
|
||||
|
||||
class MachineConfigError(ConfigError):
|
||||
pass
|
||||
|
||||
|
||||
class NoMachineError(MachineConfigError):
|
||||
pass
|
||||
|
||||
|
||||
class MachineNotFoundError(MachineConfigError):
|
||||
pass
|
||||
|
||||
|
||||
@memoize
|
||||
def get_compiled_schema():
|
||||
from voluptuous import Schema
|
||||
|
||||
+34
-15
@@ -60,23 +60,27 @@ def Choices(*choices): # noqa: N802
|
||||
return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
|
||||
|
||||
|
||||
def _get_schema_from_url(url: str) -> str:
|
||||
parsed = urlparse(url)
|
||||
# Windows absolute paths should really have scheme == "" (local)
|
||||
if os.name == "nt" and len(parsed.scheme) == 1 and not parsed.netloc:
|
||||
return ""
|
||||
if not parsed.netloc:
|
||||
return ""
|
||||
return parsed.scheme
|
||||
|
||||
|
||||
def ByUrl(mapping): # noqa: N802
|
||||
schemas = walk_values(Schema, mapping)
|
||||
schemas: dict[str, Schema] = walk_values(Schema, mapping)
|
||||
|
||||
def validate(data):
|
||||
if "url" not in data:
|
||||
raise Invalid("expected 'url'")
|
||||
|
||||
parsed = urlparse(data["url"])
|
||||
# Windows absolute paths should really have scheme == "" (local)
|
||||
if os.name == "nt" and len(parsed.scheme) == 1 and not parsed.netloc:
|
||||
return schemas[""](data)
|
||||
if not parsed.netloc:
|
||||
return schemas[""](data)
|
||||
if parsed.scheme not in schemas:
|
||||
raise Invalid(f"Unsupported URL type {parsed.scheme}://")
|
||||
|
||||
return schemas[parsed.scheme](data)
|
||||
scheme = _get_schema_from_url(data["url"])
|
||||
if scheme not in schemas:
|
||||
raise Invalid(f"Unsupported URL type {scheme}://")
|
||||
return schemas[scheme](data)
|
||||
|
||||
return validate
|
||||
|
||||
@@ -151,7 +155,6 @@ WEBDAV_COMMON = {
|
||||
"password": str,
|
||||
"ask_password": Bool,
|
||||
"token": str,
|
||||
"bearer_token_command": str,
|
||||
"custom_auth_header": str,
|
||||
"cert_path": str,
|
||||
"key_path": str,
|
||||
@@ -397,9 +400,25 @@ def config_vars_for_completion(d: dict = SCHEMA, path: str = "") -> "Iterator[st
|
||||
k = k.schema
|
||||
if not isinstance(k, str):
|
||||
continue
|
||||
|
||||
keypath = path + k
|
||||
keypath = f"{path}.{k}" if path else k
|
||||
if isinstance(v, dict):
|
||||
yield from config_vars_for_completion(v, keypath + ".")
|
||||
yield from config_vars_for_completion(v, keypath)
|
||||
else:
|
||||
yield keypath
|
||||
|
||||
|
||||
def contextual_config_vars_for_completion(config) -> "Iterator[str]":
|
||||
yield from config_vars_for_completion()
|
||||
for name, data in config["remote"].items():
|
||||
if "url" not in data:
|
||||
continue
|
||||
scheme = _get_schema_from_url(data["url"])
|
||||
if scheme not in REMOTE_SCHEMAS:
|
||||
continue
|
||||
schema = REMOTE_SCHEMAS[scheme]
|
||||
if schema:
|
||||
yield from config_vars_for_completion(schema, f"remote.{name}") # type: ignore[arg-type]
|
||||
|
||||
db_schema = SCHEMA["db"][str] # type: ignore[index]
|
||||
for name in config["db"]:
|
||||
yield from config_vars_for_completion(db_schema, f"db.{name}") # type: ignore[arg-type]
|
||||
|
||||
@@ -41,7 +41,7 @@ def _get(stage, p, info, **kwargs):
|
||||
return Dependency(stage, p, info, **kwargs)
|
||||
|
||||
|
||||
def loadd_from(stage, d_list) -> list[Dependency]:
|
||||
def loadd_from(stage, d_list):
|
||||
ret = []
|
||||
for d in d_list:
|
||||
p = d.pop(Output.PARAM_PATH, None)
|
||||
|
||||
@@ -110,7 +110,6 @@ class ParamsDependency(Dependency):
|
||||
except MissingParamsFile:
|
||||
return {}
|
||||
|
||||
assert self.repo
|
||||
try:
|
||||
return read_param_file(
|
||||
self.repo.fs,
|
||||
|
||||
@@ -150,7 +150,6 @@ class RepoDependency(Dependency):
|
||||
# local cache instead of the remote's cache. This avoids re-streaming of data,
|
||||
# but messes up the call to `_get_remote_config()` downstream, which will need
|
||||
# to ignore cache parameters.
|
||||
assert self.repo
|
||||
config["cache"] = self.repo.config["cache"]
|
||||
config["cache"]["dir"] = self.repo.cache.local_cache_dir
|
||||
|
||||
|
||||
+42
-68
@@ -5,15 +5,7 @@ from collections.abc import Iterable, Iterator
|
||||
from itertools import chain, groupby, takewhile
|
||||
from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Optional, Union, overload
|
||||
|
||||
try:
|
||||
from pathspec.patterns.gitignore.spec import ( # type: ignore[import-not-found]
|
||||
GitIgnoreSpecPattern,
|
||||
)
|
||||
except ImportError: # pathspec<1
|
||||
from pathspec.patterns import (
|
||||
GitWildMatchPattern as GitIgnoreSpecPattern,
|
||||
)
|
||||
|
||||
from pathspec.patterns import GitWildMatchPattern
|
||||
from pathspec.util import normalize_file
|
||||
from pygtrie import Trie
|
||||
|
||||
@@ -42,14 +34,7 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
def __init__(
|
||||
self, pattern_list: Iterable[Union[PatternInfo, str]], dirname: str, sep: str
|
||||
) -> None:
|
||||
try:
|
||||
from pathspec.patterns.gitignore.spec import ( # type: ignore[import-not-found]
|
||||
_DIR_MARK,
|
||||
)
|
||||
except ImportError: # pathspec<1
|
||||
from pathspec.patterns.gitwildmatch import ( # type: ignore[attr-defined, no-redef]
|
||||
_DIR_MARK,
|
||||
)
|
||||
from pathspec.patterns.gitwildmatch import _DIR_MARK
|
||||
|
||||
pattern_infos = [
|
||||
pattern if isinstance(pattern, PatternInfo) else PatternInfo(pattern, "")
|
||||
@@ -60,16 +45,13 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
self.pattern_list: list[PatternInfo] = []
|
||||
self.dirname = dirname
|
||||
self.find_matching_pattern = functools.cache(self._find_matching_pattern)
|
||||
self._ignore_parents = functools.cache(self._ignore_parents)
|
||||
|
||||
regex_pattern_list: list[tuple[str, bool, bool, PatternInfo]] = []
|
||||
for pattern_info in pattern_infos:
|
||||
regex, ignore = GitIgnoreSpecPattern.pattern_to_regex(pattern_info.patterns)
|
||||
for count, pattern_info in enumerate(pattern_infos):
|
||||
regex, ignore = GitWildMatchPattern.pattern_to_regex(pattern_info.patterns)
|
||||
if regex is not None and ignore is not None:
|
||||
self.pattern_list.append(pattern_info)
|
||||
# DVC resolves directory matches itself; only the outer rule
|
||||
# group is needed to identify a match in the combined regex.
|
||||
regex = regex.replace(f"(?P<{_DIR_MARK}>", "(?:")
|
||||
regex = regex.replace(f"<{_DIR_MARK}>", f"<{_DIR_MARK}{count}>")
|
||||
regex_pattern_list.append(
|
||||
(regex, ignore, pattern_info.patterns.endswith("/"), pattern_info)
|
||||
)
|
||||
@@ -129,20 +111,8 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
def __call__(
|
||||
self, root: str, dirs: list[str], files: list[str]
|
||||
) -> tuple[list[str], list[str]]:
|
||||
# Entries are basenames from one directory listing, so normalize the
|
||||
# shared prefix and check its ancestors only once.
|
||||
prefix = self._get_normalize_path(root, "")
|
||||
if prefix is None:
|
||||
return dirs.copy(), files.copy()
|
||||
if prefix and self._ignore_parents(prefix[:-1])[0]:
|
||||
return [], []
|
||||
|
||||
files = [
|
||||
f for f in files if not self.find_matching_pattern(f"{prefix}{f}", False)[0]
|
||||
]
|
||||
dirs = [
|
||||
d for d in dirs if not self.find_matching_pattern(f"{prefix}{d}", True)[0]
|
||||
]
|
||||
files = [f for f in files if not self.matches(root, f)]
|
||||
dirs = [d for d in dirs if not self.matches(root, d, True)]
|
||||
|
||||
return dirs, files
|
||||
|
||||
@@ -204,51 +174,55 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
_match: list[PatternInfo] = []
|
||||
if path:
|
||||
result, _match = self._ignore(path, is_dir)
|
||||
return (result, _match.copy()) if details else result
|
||||
return (result, _match) if details else result
|
||||
|
||||
def _find_matching_pattern(
|
||||
self, path: str, is_dir: bool
|
||||
) -> tuple[bool, list[PatternInfo]]:
|
||||
paths = [path]
|
||||
if is_dir and not path.endswith("/"):
|
||||
paths.append(f"{path}/")
|
||||
|
||||
for pattern, ignore, dir_only_pattern, pattern_map in reversed(
|
||||
self.ignore_spec
|
||||
):
|
||||
if dir_only_pattern and not is_dir:
|
||||
continue
|
||||
match = pattern.match(path)
|
||||
# Only directory-only patterns need the trailing slash. Probing it
|
||||
# for other patterns would make X/** exclude X itself.
|
||||
if not match and dir_only_pattern and not path.endswith("/"):
|
||||
match = pattern.match(f"{path}/")
|
||||
if not match:
|
||||
continue
|
||||
if ignore:
|
||||
group_name = match.lastgroup
|
||||
else:
|
||||
# unignored patterns are not combined with `|`,
|
||||
# so there are no groups.
|
||||
group_name = None
|
||||
_regex, pattern_info = pattern_map[group_name]
|
||||
return ignore, [pattern_info]
|
||||
for p in paths:
|
||||
match = pattern.match(p)
|
||||
if not match:
|
||||
continue
|
||||
if ignore:
|
||||
group_name, _match = next(
|
||||
(
|
||||
(name, _match)
|
||||
for name, _match in match.groupdict().items()
|
||||
if name.startswith("rule_") and _match is not None
|
||||
)
|
||||
)
|
||||
else:
|
||||
# unignored patterns are not combined with `|`,
|
||||
# so there are no groups.
|
||||
group_name = None
|
||||
_regex, pattern_info = pattern_map[group_name]
|
||||
return ignore, [pattern_info]
|
||||
return False, []
|
||||
|
||||
def _ignore(self, path: str, is_dir: bool) -> tuple[bool, list[PatternInfo]]:
|
||||
parent, sep, _ = path.rpartition("/")
|
||||
if sep:
|
||||
result = self._ignore_parents(parent)
|
||||
if result[0]:
|
||||
return result
|
||||
return self.find_matching_pattern(path, is_dir)
|
||||
|
||||
def _ignore_parents(self, path: str) -> tuple[bool, list[PatternInfo]]:
|
||||
# Files in the same directory share the ancestor verdict, not just
|
||||
# individual regex matches. Cache it to avoid rebuilding every prefix.
|
||||
parts = path.split("/")
|
||||
result = False
|
||||
matches: list[PatternInfo] = []
|
||||
for i in range(1, len(parts) + 1):
|
||||
rel_path = "/".join(parts[:i])
|
||||
result, matches = self.find_matching_pattern(rel_path, True)
|
||||
result, _matches = self.find_matching_pattern(
|
||||
rel_path, is_dir or i < len(parts)
|
||||
)
|
||||
if i < len(parts) and not result:
|
||||
continue
|
||||
matches.extend(_matches)
|
||||
if result:
|
||||
return result, matches
|
||||
return False, []
|
||||
break
|
||||
return result, matches
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash(self.dirname + ":" + str(self.pattern_list))
|
||||
@@ -534,7 +508,7 @@ class DvcIgnoreFilter:
|
||||
|
||||
def check_ignore(self, target: str) -> CheckIgnoreResult:
|
||||
# NOTE: can only be used in `dvc check-ignore`, see
|
||||
# https://github.com/treeverse/dvc/issues/5046
|
||||
# https://github.com/iterative/dvc/issues/5046
|
||||
full_target = self.fs.abspath(target)
|
||||
matched_patterns: list[PatternInfo] = []
|
||||
ignore = False
|
||||
@@ -551,7 +525,7 @@ class DvcIgnoreFilter:
|
||||
self, fs: "FileSystem", path: str, ignore_subrepos: bool = True
|
||||
) -> bool:
|
||||
# NOTE: can't use self.check_ignore(path).match for now, see
|
||||
# https://github.com/treeverse/dvc/issues/4555
|
||||
# https://github.com/iterative/dvc/issues/4555
|
||||
if fs.protocol != Schemes.LOCAL:
|
||||
return False
|
||||
if fs.isfile(path):
|
||||
|
||||
+1
-1
@@ -158,7 +158,7 @@ def _get_fs_type(path):
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
# need to follow the symlink: https://github.com/treeverse/dvc/issues/5065
|
||||
# need to follow the symlink: https://github.com/iterative/dvc/issues/5065
|
||||
path = pathlib.Path(path).resolve()
|
||||
|
||||
for parent in itertools.chain([path], path.parents):
|
||||
|
||||
+1
-1
@@ -182,7 +182,7 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
|
||||
# number to generate the resulting lock file name, which is unique
|
||||
# enough for our application.
|
||||
#
|
||||
# [1] https://github.com/treeverse/dvc/issues/2582
|
||||
# [1] https://github.com/iterative/dvc/issues/2582
|
||||
self._hostname = socket.gethostname()
|
||||
|
||||
self._lifetime = timedelta(days=365) # Lock for good by default
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
"""Manages logging configuration for DVC repo."""
|
||||
|
||||
import logging
|
||||
import logging.config
|
||||
import logging.handlers
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Iterator
|
||||
|
||||
+9
-26
@@ -42,7 +42,6 @@ from .utils import relpath
|
||||
from .utils.fs import path_isin
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.repo import Repo
|
||||
from dvc_data.hashfile.obj import HashFile
|
||||
from dvc_data.index import DataIndexKey
|
||||
|
||||
@@ -84,7 +83,7 @@ def _get(stage, path, **kwargs):
|
||||
return Output(stage, path, **kwargs)
|
||||
|
||||
|
||||
def loadd_from(stage, d_list) -> list["Output"]:
|
||||
def loadd_from(stage, d_list):
|
||||
ret = []
|
||||
for d in d_list:
|
||||
p = d.pop(Output.PARAM_PATH)
|
||||
@@ -330,7 +329,7 @@ class Output:
|
||||
self.annot = Annotation(
|
||||
desc=desc, type=type, labels=labels or [], meta=meta or {}
|
||||
)
|
||||
self.repo: Optional[Repo] = stage.repo if not repo and stage else repo
|
||||
self.repo = stage.repo if not repo and stage else repo
|
||||
meta_d = merge_file_meta_from_cloud(info or {})
|
||||
meta = Meta.from_dict(meta_d)
|
||||
# NOTE: when version_aware is not passed into get_cloud_fs, it will be
|
||||
@@ -512,7 +511,6 @@ class Output:
|
||||
from dvc.cachemgr import LEGACY_HASH_NAMES
|
||||
|
||||
assert self.is_in_repo
|
||||
assert self.repo
|
||||
odb_name = "legacy" if self.hash_name in LEGACY_HASH_NAMES else "repo"
|
||||
return getattr(self.repo.cache, odb_name)
|
||||
|
||||
@@ -520,7 +518,6 @@ class Output:
|
||||
def local_cache(self):
|
||||
from dvc.cachemgr import LEGACY_HASH_NAMES
|
||||
|
||||
assert self.repo
|
||||
if self.hash_name in LEGACY_HASH_NAMES:
|
||||
return self.repo.cache.legacy
|
||||
return self.repo.cache.local
|
||||
@@ -588,7 +585,6 @@ class Output:
|
||||
def index_key(self) -> tuple[str, "DataIndexKey"]:
|
||||
if self.is_in_repo:
|
||||
workspace = "repo"
|
||||
assert self.repo
|
||||
key = self.repo.fs.relparts(self.fs_path, self.repo.root_dir)
|
||||
else:
|
||||
workspace = self.fs.protocol
|
||||
@@ -644,7 +640,6 @@ class Output:
|
||||
@property
|
||||
def dvcignore(self) -> Optional["DvcIgnoreFilter"]:
|
||||
if self.fs.protocol == "local":
|
||||
assert self.repo
|
||||
return self.repo.dvcignore
|
||||
return None
|
||||
|
||||
@@ -666,7 +661,6 @@ class Output:
|
||||
if not self.use_scm_ignore:
|
||||
return
|
||||
|
||||
assert self.repo
|
||||
if self.repo.scm.is_tracked(self.fspath):
|
||||
raise OutputAlreadyTrackedError(self)
|
||||
|
||||
@@ -676,7 +670,6 @@ class Output:
|
||||
if not self.use_scm_ignore:
|
||||
return
|
||||
|
||||
assert self.repo
|
||||
self.repo.scm_context.ignore_remove(self.fspath)
|
||||
|
||||
def save(self) -> None:
|
||||
@@ -783,7 +776,6 @@ class Output:
|
||||
callback=cb,
|
||||
)
|
||||
if relink:
|
||||
assert self.repo
|
||||
rel = self.fs.relpath(filter_info or self.fs_path)
|
||||
with CheckoutCallback(desc=f"Checking out {rel}", unit="files") as cb:
|
||||
self._checkout(
|
||||
@@ -965,7 +957,6 @@ class Output:
|
||||
added = not self.exists
|
||||
|
||||
try:
|
||||
assert self.repo
|
||||
modified = self._checkout(
|
||||
filter_info or self.fs_path,
|
||||
self.fs,
|
||||
@@ -996,24 +987,18 @@ class Output:
|
||||
if ignore_remove:
|
||||
self.ignore_remove()
|
||||
|
||||
def move(self, out: "Output") -> None:
|
||||
src_exists = self.exists
|
||||
if src_exists:
|
||||
self.fs.move(self.fs_path, out.fs_path)
|
||||
else:
|
||||
logger.warning("%r missing", self.fspath)
|
||||
|
||||
def move(self, out):
|
||||
if self.protocol == "local" and self.use_scm_ignore:
|
||||
assert self.repo
|
||||
self.repo.scm_context.ignore_remove(self.fspath)
|
||||
|
||||
self.fs.move(self.fs_path, out.fs_path)
|
||||
self.def_path = out.def_path
|
||||
self.fs_path = out.fs_path
|
||||
try:
|
||||
self.save()
|
||||
self.commit()
|
||||
except self.DoesNotExistError:
|
||||
self.ignore()
|
||||
self.save()
|
||||
self.commit()
|
||||
|
||||
if self.protocol == "local" and self.use_scm_ignore:
|
||||
self.repo.scm_context.ignore(self.fspath)
|
||||
|
||||
def transfer(
|
||||
self, source, odb=None, jobs=None, update=False, no_progress_bar=False
|
||||
@@ -1094,7 +1079,6 @@ class Output:
|
||||
if self.remote:
|
||||
kwargs["remote"] = self.remote
|
||||
with suppress(Exception):
|
||||
assert self.repo
|
||||
self.repo.cloud.pull([obj.hash_info], **kwargs)
|
||||
|
||||
if self.obj:
|
||||
@@ -1188,7 +1172,6 @@ class Output:
|
||||
return {}
|
||||
|
||||
if self.remote:
|
||||
assert self.repo
|
||||
remote_odb = self.repo.cloud.get_remote_odb(
|
||||
name=self.remote, hash_name=self.hash_name
|
||||
)
|
||||
|
||||
@@ -37,13 +37,13 @@ KEYCRE = re.compile(
|
||||
def get_parser():
|
||||
from pyparsing import CharsNotIn, ParserElement, Suppress, ZeroOrMore
|
||||
|
||||
ParserElement.enable_packrat()
|
||||
ParserElement.enablePackrat()
|
||||
|
||||
word = CharsNotIn(f"{PERIOD}{LBRACK}{RBRACK}")
|
||||
idx = Suppress(LBRACK) + word + Suppress(RBRACK)
|
||||
attr = Suppress(PERIOD) + word
|
||||
parser = word + ZeroOrMore(attr ^ idx)
|
||||
parser.set_parse_action(PERIOD.join)
|
||||
parser.setParseAction(PERIOD.join)
|
||||
|
||||
return parser
|
||||
|
||||
@@ -178,7 +178,7 @@ def parse_expr(s: str):
|
||||
from pyparsing import ParseException
|
||||
|
||||
try:
|
||||
result = get_parser().parse_string(s, parse_all=True)
|
||||
result = get_parser().parseString(s, parseAll=True)
|
||||
except ParseException as exc:
|
||||
format_and_raise_parse_error(exc)
|
||||
raise AssertionError("unreachable") # noqa: B904
|
||||
|
||||
@@ -2,6 +2,7 @@ import os
|
||||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING, NamedTuple, Optional
|
||||
|
||||
import dpath
|
||||
import dpath.options
|
||||
from funcy import get_in, last
|
||||
|
||||
|
||||
@@ -20,7 +20,6 @@ if TYPE_CHECKING:
|
||||
from dvc.fs.data import DataFileSystem
|
||||
from dvc.fs.dvc import DVCFileSystem
|
||||
from dvc.lock import LockBase
|
||||
from dvc.output import Output
|
||||
from dvc.scm import Git, NoSCM
|
||||
from dvc.stage import Stage
|
||||
from dvc.types import DictStrAny
|
||||
@@ -545,9 +544,7 @@ class Repo:
|
||||
|
||||
return used
|
||||
|
||||
def find_outs_by_path(
|
||||
self, path, outs=None, recursive=False, strict=True
|
||||
) -> list["Output"]:
|
||||
def find_outs_by_path(self, path, outs=None, recursive=False, strict=True):
|
||||
# using `outs_graph` to ensure graph checks are run
|
||||
outs = outs or self.index.outs_graph
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ def check_name_format(name: str) -> None:
|
||||
|
||||
def name_is_compatible(name: str) -> bool:
|
||||
"""
|
||||
Only needed by DVCLive per treeverse/dvclive#715
|
||||
Only needed by DVCLive per iterative/dvclive#715
|
||||
Will be removed in future release.
|
||||
"""
|
||||
from gto.constants import assert_name_is_valid
|
||||
|
||||
+5
-27
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
import posixpath
|
||||
from collections import defaultdict, deque
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable, Iterator, Mapping
|
||||
from typing import TYPE_CHECKING, Optional, TypedDict, Union
|
||||
|
||||
@@ -385,26 +385,6 @@ def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> list[str]
|
||||
return [repo.fs.relpath(file, start) for file in files]
|
||||
|
||||
|
||||
def iter_index(
|
||||
index: Union["BaseDataIndex", "DataIndexView"], shallow: bool = False
|
||||
) -> Iterator[tuple["DataIndexKey", Optional["DataIndexEntry"]]]:
|
||||
if not shallow:
|
||||
yield from index.iteritems(shallow=shallow)
|
||||
return
|
||||
|
||||
# only iterate until we find entries with hash_info in shallow mode
|
||||
todo: deque[tuple[DataIndexKey, dict]] = deque([((), index.info(()))])
|
||||
while todo:
|
||||
key, info = todo.popleft()
|
||||
entry = info.get("entry")
|
||||
if info.get("type") == "directory" and not (entry and entry.hash_info):
|
||||
try:
|
||||
todo.extend(index.ls(key, detail=True))
|
||||
except (KeyError, DataIndexDirError):
|
||||
pass
|
||||
yield key, entry
|
||||
|
||||
|
||||
def _get_entries_not_in_remote(
|
||||
repo: "Repo",
|
||||
filter_keys: Optional[Iterable["DataIndexKey"]] = None,
|
||||
@@ -438,7 +418,7 @@ def _get_entries_not_in_remote(
|
||||
|
||||
n = 0
|
||||
with TqdmCallback(size=n, desc="Checking remote", unit="entry") as cb:
|
||||
for key, entry in iter_index(view, shallow=not granular):
|
||||
for key, entry in view.iteritems(shallow=not granular):
|
||||
if not (entry and entry.hash_info):
|
||||
continue
|
||||
|
||||
@@ -456,17 +436,15 @@ def _get_entries_not_in_remote(
|
||||
n += 1
|
||||
cb.set_size(n)
|
||||
|
||||
results = storage_map.bulk_remote_exists(
|
||||
list(entries.values()), refresh=remote_refresh, callback=cb
|
||||
)
|
||||
for key, entry in entries.items():
|
||||
k = (*key, "") if entry.meta and entry.meta.isdir else key
|
||||
try:
|
||||
if not results.get(entry, False):
|
||||
if not storage_map.remote_exists(entry, refresh=remote_refresh):
|
||||
missing_entries.append(os.path.sep.join(k))
|
||||
except StorageKeyError:
|
||||
pass
|
||||
|
||||
finally:
|
||||
cb.relative_update()
|
||||
data_index.onerror = orig_data_index_onerror
|
||||
return missing_entries
|
||||
|
||||
|
||||
@@ -188,7 +188,7 @@ class Experiments:
|
||||
rev_names = self.get_exact_name(revs)
|
||||
for rev in revs:
|
||||
name = rev_names[rev]
|
||||
names.append(name or rev[:7])
|
||||
names.append(name if name else rev[:7])
|
||||
ui.write("\nRan experiment(s): {}".format(", ".join(names)))
|
||||
if tmp_dir:
|
||||
ui.write(
|
||||
|
||||
@@ -58,7 +58,7 @@ def collect_rev(
|
||||
if not (rev == "workspace" or param_deps or data.contains_error):
|
||||
cache.put(data, force=True)
|
||||
return ExpState(rev=rev, data=data)
|
||||
except Exception as exc:
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("", exc_info=True)
|
||||
error = SerializableError(str(exc), type(exc).__name__)
|
||||
return ExpState(rev=rev, error=error)
|
||||
@@ -328,7 +328,7 @@ def collect(
|
||||
name=baseline_names.get(baseline_rev),
|
||||
data=baseline_data.data,
|
||||
error=baseline_data.error,
|
||||
experiments=experiments or None,
|
||||
experiments=experiments if experiments else None,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -9,7 +9,6 @@ from enum import IntEnum
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, Union
|
||||
|
||||
import funcy
|
||||
from funcy import nullcontext
|
||||
from scmrepo.exceptions import SCMError
|
||||
|
||||
@@ -757,7 +756,7 @@ class BaseExecutor(ABC):
|
||||
logger.debug("Commit to current experiment branch '%s'", branch)
|
||||
else:
|
||||
baseline_rev = scm.get_ref(EXEC_BASELINE)
|
||||
name = exp_name or f"exp-{exp_hash[:5]}"
|
||||
name = exp_name if exp_name else f"exp-{exp_hash[:5]}"
|
||||
ref_info = ExpRefInfo(baseline_rev, name)
|
||||
branch = str(ref_info)
|
||||
old_ref = None
|
||||
@@ -816,22 +815,11 @@ class BaseExecutor(ABC):
|
||||
|
||||
@contextmanager
|
||||
def set_temp_refs(self, scm: "Git", temp_dict: dict[str, str]):
|
||||
# Retry ref set, get, and remove operations to handle transient issues during
|
||||
# concurrent Git access.
|
||||
# Dulwich deletes parent directories of refs if they happen to be empty after
|
||||
# removing a ref, which can interfere with `set_ref` in other processes.
|
||||
# `remove_ref` may also fail with a `FileLocked` error when refs are packed,
|
||||
# since multiple processes might attempt to write to the same file.
|
||||
retry = funcy.retry(10, errors=Exception, timeout=0.1)
|
||||
set_ref = retry(scm.set_ref)
|
||||
get_ref = retry(scm.get_ref)
|
||||
remove_ref = retry(scm.remove_ref)
|
||||
|
||||
try:
|
||||
for ref, rev in temp_dict.items():
|
||||
set_ref(ref, rev)
|
||||
scm.set_ref(ref, rev)
|
||||
yield
|
||||
finally:
|
||||
for ref in temp_dict:
|
||||
if get_ref(ref):
|
||||
remove_ref(ref)
|
||||
if scm.get_ref(ref):
|
||||
scm.remove_ref(ref)
|
||||
|
||||
@@ -90,7 +90,7 @@ class BaseStashQueue(ABC):
|
||||
"""Construct a queue.
|
||||
|
||||
Arguments:
|
||||
repo: DVC repo instance for this queue.
|
||||
scm: Git SCM instance for this queue.
|
||||
ref: Git stash ref for this queue.
|
||||
failed_ref: Failed run Git stash ref for this queue.
|
||||
"""
|
||||
@@ -287,7 +287,6 @@ class BaseStashQueue(ABC):
|
||||
baseline_rev: Optional[str] = None,
|
||||
branch: Optional[str] = None,
|
||||
name: Optional[str] = None,
|
||||
no_hydra: bool = False,
|
||||
**kwargs,
|
||||
) -> QueueEntry:
|
||||
"""Stash changes from the workspace as an experiment.
|
||||
@@ -303,7 +302,6 @@ class BaseStashQueue(ABC):
|
||||
name: Optional experiment name. If specified this will be used as
|
||||
the human-readable name in the experiment branch ref. Has no
|
||||
effect of branch is specified.
|
||||
no_hydra: Disable Hydra from automatically overwriting all params.
|
||||
|
||||
.. _Hydra Override:
|
||||
https://hydra.cc/docs/next/advanced/override_grammar/basic/
|
||||
@@ -320,7 +318,7 @@ class BaseStashQueue(ABC):
|
||||
|
||||
# update experiment params from command line
|
||||
if params:
|
||||
self._update_params(params, no_hydra=no_hydra)
|
||||
self._update_params(params)
|
||||
|
||||
# DVC commit data deps to preserve state across workspace
|
||||
# & tempdir runs
|
||||
@@ -443,13 +441,12 @@ class BaseStashQueue(ABC):
|
||||
f"from '{config_path}': {param_list}"
|
||||
)
|
||||
|
||||
def _update_params(self, params: dict[str, list[str]], no_hydra: bool = False):
|
||||
def _update_params(self, params: dict[str, list[str]]):
|
||||
"""Update param files with the provided `Hydra Override`_ patterns.
|
||||
|
||||
Args:
|
||||
params: Dict mapping paths to `Hydra Override`_ patterns,
|
||||
provided via `exp run --set-param`.
|
||||
no_hydra: Disable Hydra from automatically overwriting all params.
|
||||
|
||||
.. _Hydra Override:
|
||||
https://hydra.cc/docs/advanced/override_grammar/basic/
|
||||
@@ -459,7 +456,7 @@ class BaseStashQueue(ABC):
|
||||
logger.debug("Using experiment params '%s'", params)
|
||||
|
||||
hydra_config = self.repo.config.get("hydra", {})
|
||||
hydra_enabled = hydra_config.get("enabled", False) and not no_hydra
|
||||
hydra_enabled = hydra_config.get("enabled", False)
|
||||
hydra_output_file = ParamsDependency.DEFAULT_PARAMS_FILE
|
||||
for path, overrides in params.items():
|
||||
if hydra_enabled and path == hydra_output_file:
|
||||
|
||||
@@ -70,7 +70,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
def celery(self) -> "FSApp":
|
||||
from kombu.transport.filesystem import Channel
|
||||
|
||||
# related to https://github.com/treeverse/dvc-task/issues/61
|
||||
# related to https://github.com/iterative/dvc-task/issues/61
|
||||
Channel.QoS.restore_at_shutdown = False
|
||||
|
||||
from dvc_task.app import FSApp
|
||||
|
||||
@@ -84,8 +84,8 @@ def cleanup_exp(executor: TempDirExecutor, infofile: str) -> None:
|
||||
"""Cleanup after an experiment.
|
||||
|
||||
Arguments:
|
||||
executor: Executor to clean up.
|
||||
infofile: Path to the executor info file.
|
||||
tmp_dir: Temp directory to be removed.
|
||||
entry_dict: Serialized QueueEntry for this experiment.
|
||||
"""
|
||||
executor.cleanup(infofile)
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ class ExpRefInfo:
|
||||
|
||||
def __init__(self, baseline_sha: str, name: Optional[str] = None):
|
||||
self.baseline_sha = baseline_sha
|
||||
self.name: str = name or ""
|
||||
self.name: str = name if name else ""
|
||||
|
||||
def __str__(self):
|
||||
return "/".join(self.parts)
|
||||
|
||||
@@ -22,7 +22,6 @@ def run( # noqa: C901, PLR0912
|
||||
queue: bool = False,
|
||||
copy_paths: Optional[Iterable[str]] = None,
|
||||
message: Optional[str] = None,
|
||||
no_hydra: bool = False,
|
||||
**kwargs,
|
||||
) -> dict[str, str]:
|
||||
"""Reproduce the specified targets as an experiment.
|
||||
@@ -68,7 +67,7 @@ def run( # noqa: C901, PLR0912
|
||||
else:
|
||||
path_overrides = {}
|
||||
|
||||
hydra_enabled = repo.config.get("hydra", {}).get("enabled", False) and not no_hydra
|
||||
hydra_enabled = repo.config.get("hydra", {}).get("enabled", False)
|
||||
hydra_output_file = ParamsDependency.DEFAULT_PARAMS_FILE
|
||||
if hydra_enabled and hydra_output_file not in path_overrides:
|
||||
# Force `_update_params` even if `--set-param` was not used
|
||||
@@ -81,7 +80,6 @@ def run( # noqa: C901, PLR0912
|
||||
tmp_dir=tmp_dir,
|
||||
copy_paths=copy_paths,
|
||||
message=message,
|
||||
no_hydra=no_hydra,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@@ -102,7 +100,6 @@ def run( # noqa: C901, PLR0912
|
||||
params=sweep_overrides,
|
||||
copy_paths=copy_paths,
|
||||
message=message,
|
||||
no_hydra=no_hydra,
|
||||
**kwargs,
|
||||
)
|
||||
if sweep_overrides:
|
||||
|
||||
@@ -67,7 +67,7 @@ class ExpStash(Stash):
|
||||
branch: Optional[str] = None,
|
||||
) -> str:
|
||||
msg = cls.MESSAGE_FORMAT.format(
|
||||
rev=rev, baseline_rev=baseline_rev, name=name or ""
|
||||
rev=rev, baseline_rev=baseline_rev, name=name if name else ""
|
||||
)
|
||||
branch_msg = f":{branch}" if branch else ""
|
||||
return f"{msg}{branch_msg}"
|
||||
@@ -124,7 +124,9 @@ class ApplyStash(Stash):
|
||||
|
||||
@classmethod
|
||||
def format_message(cls, head_rev: str, rev: str, name: Optional[str] = None) -> str:
|
||||
return cls.MESSAGE_FORMAT.format(head_rev=head_rev, rev=rev, name=name or "")
|
||||
return cls.MESSAGE_FORMAT.format(
|
||||
head_rev=head_rev, rev=rev, name=name if name else ""
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def preserve_workspace(
|
||||
|
||||
@@ -200,7 +200,7 @@ def exp_commits(
|
||||
) -> Iterable[str]:
|
||||
"""Iterate over all experiment commits."""
|
||||
shas: set[str] = set()
|
||||
refs = ref_infos or exp_refs(scm)
|
||||
refs = ref_infos if ref_infos else exp_refs(scm)
|
||||
for ref_info in refs:
|
||||
shas.update(scm.branch_revs(str(ref_info), ref_info.baseline_sha))
|
||||
yield from shas
|
||||
|
||||
+3
-7
@@ -28,10 +28,8 @@ def _validate_args(**kwargs):
|
||||
"`--all-experiments`, `--all-commits`, `--date` or `--rev` "
|
||||
"needs to be set."
|
||||
)
|
||||
if kwargs.get("num") and not (kwargs.get("rev") or kwargs.get("all_branches")):
|
||||
raise InvalidArgumentError(
|
||||
"`--num` can only be used alongside `--rev` or `--all-branches`"
|
||||
)
|
||||
if kwargs.get("num") and not kwargs.get("rev"):
|
||||
raise InvalidArgumentError("`--num` can only be used alongside `--rev`")
|
||||
|
||||
|
||||
def _used_obj_ids_not_in_remote(
|
||||
@@ -130,11 +128,9 @@ def gc( # noqa: C901, PLR0912, PLR0913
|
||||
used_obj_ids = set()
|
||||
used_obj_ids.update(*odb_to_obj_ids.values())
|
||||
|
||||
seen_odbs = set()
|
||||
for scheme, odb in self.cache.by_scheme():
|
||||
if not odb or odb in seen_odbs:
|
||||
if not odb:
|
||||
continue
|
||||
seen_odbs.add(odb)
|
||||
num_removed = ogc(odb, used_obj_ids, jobs=jobs, dry=dry)
|
||||
if num_removed:
|
||||
logger.info("Removed %d objects from %s cache.", num_removed, scheme)
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ def pre_commit_install(scm: "Git") -> None:
|
||||
config_path = os.path.join(scm.root_dir, ".pre-commit-config.yaml")
|
||||
with modify_yaml(config_path) as config:
|
||||
entry = {
|
||||
"repo": "https://github.com/treeverse/dvc",
|
||||
"repo": "https://github.com/iterative/dvc",
|
||||
"rev": ".".join(map(str, version_tuple[:3])),
|
||||
"hooks": [
|
||||
{
|
||||
|
||||
@@ -123,7 +123,7 @@ def try_expand_paths(fs: "FileSystem", paths: Iterable[str]) -> Iterator[str]:
|
||||
if fs.isdir(path):
|
||||
yield from fs.find(path)
|
||||
continue
|
||||
except Exception as e:
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug(
|
||||
"failed to expand %r: %s",
|
||||
path,
|
||||
|
||||
+17
-41
@@ -1,13 +1,10 @@
|
||||
import os
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from dvc.exceptions import MoveNotDataSourceError
|
||||
from dvc.repo.scm_context import scm_context
|
||||
|
||||
from . import locked
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from . import Repo
|
||||
|
||||
|
||||
def _expand_target_path(from_path, to_path):
|
||||
if os.path.isdir(to_path):
|
||||
@@ -17,7 +14,7 @@ def _expand_target_path(from_path, to_path):
|
||||
|
||||
@locked
|
||||
@scm_context
|
||||
def move(self: "Repo", from_path, to_path):
|
||||
def move(self, from_path, to_path):
|
||||
"""
|
||||
Renames an output file and modifies the stage associated
|
||||
to reflect the change on the pipeline.
|
||||
@@ -35,12 +32,9 @@ def move(self: "Repo", from_path, to_path):
|
||||
It only works with outputs generated by `add` or `import`,
|
||||
also known as data sources.
|
||||
"""
|
||||
from dvc import dependency, output
|
||||
from dvc import output
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX
|
||||
from dvc.exceptions import MoveNotDataSourceError
|
||||
from dvc.stage import Stage
|
||||
from dvc.stage.exceptions import StageFileAlreadyExistsError
|
||||
from dvc_objects.fs.local import LocalFileSystem
|
||||
|
||||
from_out = output.loads_from(Stage(self), [from_path])[0]
|
||||
assert from_out.protocol == "local"
|
||||
@@ -51,7 +45,6 @@ def move(self: "Repo", from_path, to_path):
|
||||
assert len(outs) == 1
|
||||
out = outs[0]
|
||||
stage = out.stage
|
||||
deps = stage.deps
|
||||
|
||||
if not stage.is_data_source:
|
||||
raise MoveNotDataSourceError(stage.addressing)
|
||||
@@ -65,38 +58,21 @@ def move(self: "Repo", from_path, to_path):
|
||||
)
|
||||
new_wdir = os.path.abspath(os.path.join(os.curdir, os.path.dirname(to_path)))
|
||||
to_path = os.path.relpath(to_path, new_wdir)
|
||||
try:
|
||||
new_stage = self.stage.create(
|
||||
single_stage=True,
|
||||
fname=new_fname,
|
||||
wdir=new_wdir,
|
||||
outs=[to_path],
|
||||
meta=stage.meta,
|
||||
frozen=stage.frozen,
|
||||
always_changed=stage.always_changed,
|
||||
desc=stage.desc,
|
||||
)
|
||||
except StageFileAlreadyExistsError:
|
||||
# reraise to remove `--force` hint
|
||||
raise StageFileAlreadyExistsError(f"{new_fname!r} already exists") from None
|
||||
new_stage = self.stage.create(
|
||||
single_stage=True,
|
||||
fname=new_fname,
|
||||
wdir=new_wdir,
|
||||
outs=[to_path],
|
||||
meta=stage.meta,
|
||||
)
|
||||
|
||||
os.unlink(stage.path)
|
||||
stage = new_stage
|
||||
else:
|
||||
new_stage = stage
|
||||
to_path = os.path.relpath(to_path, stage.wdir)
|
||||
|
||||
def with_dep_path_adjusted(dep: dependency.Dependency):
|
||||
d = dep.dumpd()
|
||||
if isinstance(dep.fs, LocalFileSystem) and not os.path.isabs(dep.def_path):
|
||||
return d | {"path": os.path.relpath(dep.fspath, new_stage.wdir)}
|
||||
return d
|
||||
to_out = output.loads_from(stage, [to_path], out.use_cache, out.metric)[0]
|
||||
|
||||
new_stage.outs = output.loadd_from(new_stage, [out.dumpd() | {"path": to_path}])
|
||||
new_stage.deps = dependency.loadd_from(
|
||||
new_stage, [with_dep_path_adjusted(dep) for dep in deps]
|
||||
)
|
||||
out.move(new_stage.outs[0])
|
||||
new_stage.md5 = new_stage.compute_md5()
|
||||
new_stage.dump()
|
||||
if stage != new_stage:
|
||||
stage.dvcfile.remove()
|
||||
self.scm_context.track_file(stage.dvcfile.relpath)
|
||||
return stage, new_stage
|
||||
out.move(to_out)
|
||||
stage.save()
|
||||
stage.dump()
|
||||
|
||||
@@ -10,8 +10,6 @@ from dvc.utils import relpath
|
||||
from dvc.utils.collections import ensure_list
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from typing_extensions import Self
|
||||
|
||||
from dvc.repo import Repo
|
||||
from dvc.scm import Base
|
||||
|
||||
@@ -48,7 +46,7 @@ class SCMContext:
|
||||
try:
|
||||
add_no_submodules(self.scm, paths)
|
||||
except UnsupportedIndexFormat:
|
||||
link = "https://github.com/treeverse/dvc/issues/610"
|
||||
link = "https://github.com/iterative/dvc/issues/610"
|
||||
add_cmd = self._make_git_add_cmd([relpath(path) for path in paths])
|
||||
logger.info("")
|
||||
msg = (
|
||||
@@ -98,7 +96,7 @@ class SCMContext:
|
||||
@contextmanager
|
||||
def __call__(
|
||||
self, autostage: Optional[bool] = None, quiet: Optional[bool] = None
|
||||
) -> Iterator["Self"]:
|
||||
) -> Iterator["SCMContext"]:
|
||||
try:
|
||||
yield self
|
||||
except Exception:
|
||||
@@ -133,10 +131,9 @@ class SCMContext:
|
||||
|
||||
self.files_to_track = set()
|
||||
|
||||
def __enter__(self) -> "Self":
|
||||
def __enter__(self) -> "SCMContext":
|
||||
self._cm = self()
|
||||
self._cm.__enter__()
|
||||
return self
|
||||
return self._cm.__enter__()
|
||||
|
||||
def __exit__(self, *exc_args) -> None:
|
||||
assert self._cm
|
||||
|
||||
+2
-5
@@ -234,7 +234,7 @@ def iter_revs(
|
||||
results.extend(scm.list_all_commits())
|
||||
else:
|
||||
if all_branches:
|
||||
results.extend(_get_n_commits(scm, scm.list_branches(), num))
|
||||
results.extend(scm.list_branches())
|
||||
|
||||
if all_tags:
|
||||
results.extend(scm.list_tags())
|
||||
@@ -258,10 +258,7 @@ def iter_revs(
|
||||
results.extend(exp_commits(scm))
|
||||
|
||||
rev_resolver = partial(resolve_rev, scm)
|
||||
grouped = group_by(rev_resolver, results)
|
||||
for rev, names in grouped.items():
|
||||
grouped[rev] = list(dict.fromkeys(names))
|
||||
return grouped
|
||||
return group_by(rev_resolver, results)
|
||||
|
||||
|
||||
def lfs_prefetch(fs: "FileSystem", paths: list[str]):
|
||||
|
||||
@@ -32,7 +32,7 @@ from .utils import (
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.dependency import Dependency, ParamsDependency
|
||||
from dvc.dependency import ParamsDependency
|
||||
from dvc.dvcfile import ProjectFile, SingleStageFile
|
||||
from dvc.output import Output
|
||||
from dvc.repo import Repo
|
||||
@@ -153,8 +153,8 @@ class Stage(params.StageParams):
|
||||
self._path = path
|
||||
self.cmd = cmd
|
||||
self.wdir = wdir
|
||||
self.outs: list[Output] = outs
|
||||
self.deps: list[Dependency] = deps
|
||||
self.outs = outs
|
||||
self.deps = deps
|
||||
self.md5 = md5
|
||||
self.frozen = locked or frozen
|
||||
self.always_changed = always_changed
|
||||
|
||||
+2
-2
@@ -51,7 +51,7 @@ def _warn_if_fish(executable):
|
||||
"Be aware that it might cause problems by overwriting "
|
||||
"your current environment variables with values defined "
|
||||
"in 'config.fish', which might affect your command. See "
|
||||
"https://github.com/treeverse/dvc/issues/1307. "
|
||||
"https://github.com/iterative/dvc/issues/1307. "
|
||||
)
|
||||
|
||||
|
||||
@@ -99,7 +99,7 @@ def prepare_kwargs(stage, run_env=None):
|
||||
#
|
||||
# [1] https://github.com/python/cpython/blob/3.7/Lib/subprocess.py
|
||||
# #L1426
|
||||
# [2] https://github.com/treeverse/dvc/issues/2506
|
||||
# [2] https://github.com/iterative/dvc/issues/2506
|
||||
# #issuecomment-535396799
|
||||
kwargs["shell"] = os.name == "nt"
|
||||
return kwargs
|
||||
|
||||
@@ -178,7 +178,7 @@ def to_single_stage_lockfile(stage: "Stage", **kwargs) -> dict:
|
||||
res = OrderedDict([("cmd", stage.cmd)])
|
||||
params, deps = split_params_deps(stage)
|
||||
deps, outs = (
|
||||
[_dumpd(item) for item in sorted(items, key=attrgetter("def_path"))] # type: ignore[call-overload]
|
||||
[_dumpd(item) for item in sorted(items, key=attrgetter("def_path"))]
|
||||
for items in [deps, stage.outs]
|
||||
)
|
||||
params = _serialize_params_values(params)
|
||||
|
||||
@@ -4,7 +4,7 @@ dvc.testing.benchmarks
|
||||
======================
|
||||
|
||||
Benchmark test definitions are now part of ``dvc.testing``.
|
||||
For CLI usage and `bench.dvc.org <https://bench.dvc.org>`_ details see `dvc-bench <https://github.com/treeverse/dvc-bench>`_.
|
||||
For CLI usage and `bench.dvc.org <https://bench.dvc.org>`_ details see `dvc-bench <https://github.com/iterative/dvc-bench>`_.
|
||||
|
||||
``dvc.testing.benchmarks`` structure:
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@ import pytest
|
||||
|
||||
from dvc import api
|
||||
from dvc.api import DVCFileSystem
|
||||
from dvc.testing import matchers as M
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
|
||||
@@ -43,6 +42,7 @@ class TestAPI:
|
||||
)
|
||||
def test_filesystem(
|
||||
self,
|
||||
M,
|
||||
tmp_dir,
|
||||
make_tmp_dir,
|
||||
scm,
|
||||
|
||||
@@ -6,8 +6,8 @@ import pytest
|
||||
"!=3.53.*,!=3.54.0",
|
||||
reason="Takes 10 mins to run. Regression in 3.53.0, fixed in 3.54.1",
|
||||
)
|
||||
# Introduced in https://github.com/treeverse/dvc/pull/10388.
|
||||
# Fixed in https://github.com/treeverse/dvc/pull/10531.
|
||||
# Introduced in https://github.com/iterative/dvc/pull/10388.
|
||||
# Fixed in https://github.com/iterative/dvc/pull/10531.
|
||||
def test_import(bench_dvc, tmp_dir, scm, dvc, make_dataset, remote):
|
||||
dataset = make_dataset(
|
||||
cache=False, files=False, dvcfile=True, commit=True, remote=True
|
||||
|
||||
@@ -9,7 +9,7 @@ from typing import Optional
|
||||
import pytest
|
||||
from dulwich.porcelain import clone
|
||||
from funcy import first
|
||||
from packaging import specifiers, version
|
||||
from packaging import version
|
||||
|
||||
from dvc.types import StrPath
|
||||
|
||||
@@ -79,7 +79,7 @@ def dvc_bench_repo(tmp_path_factory, bench_config):
|
||||
if url is None:
|
||||
pytest.skip(
|
||||
"--dvc-bench-repo is not set, "
|
||||
"clone https://github.com/treeverse/dvc-bench repository and set its path"
|
||||
"clone https://github.com/iterative/dvc-bench repository and set its path"
|
||||
)
|
||||
|
||||
if os.path.isdir(url):
|
||||
@@ -109,20 +109,11 @@ def make_dvc_bin(
|
||||
else:
|
||||
pkg = "dvc"
|
||||
packages = [f"{pkg} @ git+file://{dvc_repo}@{dvc_rev}"]
|
||||
|
||||
version_constraints = [
|
||||
("<3.50.3", ["pygit2==1.14.1"]),
|
||||
("<3.44.0", ["dulwich<1.0.0"]),
|
||||
("<3.67.0", ["pathspec<1"]),
|
||||
]
|
||||
for spec, pkgs in version_constraints:
|
||||
try:
|
||||
_dvc_version = version.Version(dvc_rev)
|
||||
except version.InvalidVersion:
|
||||
continue
|
||||
if _dvc_version in specifiers.SpecifierSet(spec):
|
||||
packages.extend(pkgs)
|
||||
|
||||
try:
|
||||
if version.Version(dvc_rev) < version.Version("3.50.3"):
|
||||
packages.append("pygit2==1.14.1")
|
||||
except version.InvalidVersion:
|
||||
pass
|
||||
venv.install(*packages)
|
||||
|
||||
dvc_venvs[dvc_rev] = venv
|
||||
@@ -167,7 +158,7 @@ def make_bench(request):
|
||||
# pytest >= 8.4.0
|
||||
wrapped_func = fixture_function._get_wrapped_function()
|
||||
except AttributeError:
|
||||
wrapped_func = fixture_function.__pytest_wrapped__.obj # type: ignore[attr-defined] # ty: ignore[unresolved-attribute]
|
||||
wrapped_func = fixture_function.__pytest_wrapped__.obj # ty: ignore[unresolved-attribute]
|
||||
assert inspect.isgeneratorfunction(wrapped_func)
|
||||
|
||||
generator = wrapped_func(request)
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import builtins
|
||||
|
||||
|
||||
class dict: # noqa: A001, N801, PLW1641
|
||||
"""Special class to eq by matching only presented dict keys.
|
||||
|
||||
Implementation notes:
|
||||
|
||||
- can't inherit from dict because that makes D() == M.dict() to not call
|
||||
our __eq__, if D is a subclass of a dict
|
||||
|
||||
- should not call itself dict or use dict in repr because it creates
|
||||
confusing error messages (shadowing python builtins is bad anyway)
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, d: Optional[Mapping[Any, Any]] = None, **keys: Any) -> None:
|
||||
self.d: builtins.dict[Any, Any] = {}
|
||||
if d:
|
||||
self.d.update(d)
|
||||
self.d.update(keys)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.d)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
inner = ", ".join(f"{k}={v!r}" for k, v in self.d.items())
|
||||
return f"{self.__class__.__name__}({inner})"
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
assert isinstance(other, Mapping)
|
||||
return all(other.get(name) == v for name, v in self.d.items())
|
||||
|
||||
|
||||
class unordered: # noqa: N801, PLW1641
|
||||
"""Compare list contents, but do not care about ordering.
|
||||
|
||||
(E.g. sort lists first, then compare.)
|
||||
If you care about ordering, then just compare lists directly."""
|
||||
|
||||
def __init__(self, *items: Any) -> None:
|
||||
self.items = items
|
||||
|
||||
def __repr__(self) -> str:
|
||||
inner = ", ".join(map(repr, self.items))
|
||||
return f"{self.__class__.__name__}({inner})"
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
assert isinstance(other, Iterable)
|
||||
return sorted(self.items) == sorted(other)
|
||||
|
||||
|
||||
class attrs: # noqa: N801, PLW1641
|
||||
def __init__(self, **attribs: Any) -> None:
|
||||
self.attribs = attribs
|
||||
|
||||
def __repr__(self) -> str:
|
||||
inner = ", ".join(f"{k}={v!r}" for k, v in self.attribs.items())
|
||||
return f"{self.__class__.__name__}({inner})"
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
# Unforturnately this doesn't work with classes with slots
|
||||
# self.__class__ = other.__class__
|
||||
return all(getattr(other, name) == v for name, v in self.attribs.items())
|
||||
|
||||
|
||||
class instance_of: # noqa: N801, PLW1641
|
||||
def __init__(self, expected_type: Union[Any, tuple[Any, ...]]) -> None:
|
||||
self.expected_type = expected_type
|
||||
|
||||
def __repr__(self) -> str:
|
||||
if isinstance(self.expected_type, tuple):
|
||||
inner = f"({', '.join(t.__name__ for t in self.expected_type)})"
|
||||
else:
|
||||
inner = self.expected_type.__name__
|
||||
return f"{self.__class__.__name__}({inner})"
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
return isinstance(other, self.expected_type)
|
||||
|
||||
|
||||
class any_of: # noqa: N801, PLW1641
|
||||
def __init__(self, *items: Any) -> None:
|
||||
self.items = sorted(items)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
inner = ", ".join(map(repr, self.items))
|
||||
return f"any_of({inner})"
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
return other in self.items
|
||||
|
||||
|
||||
__all__ = [
|
||||
"any_of",
|
||||
"attrs",
|
||||
"dict",
|
||||
"instance_of",
|
||||
"unordered",
|
||||
]
|
||||
@@ -2,7 +2,7 @@ import os
|
||||
import pathlib
|
||||
import posixpath
|
||||
import sys
|
||||
from typing import Callable, ClassVar, Union
|
||||
from typing import Callable, ClassVar
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dvc.utils import relpath
|
||||
@@ -79,17 +79,15 @@ class PathInfo(pathlib.PurePath, _BasePath):
|
||||
and self._cparts[:n] == other._cparts # type: ignore[attr-defined]
|
||||
)
|
||||
|
||||
def relative_to(
|
||||
self, *other: Union[str, "os.PathLike[str]"], **kwargs
|
||||
) -> "PathInfo":
|
||||
def relative_to(self, other, *args, **kwargs):
|
||||
# pathlib relative_to raises exception when one path is not a direct
|
||||
# descendant of the other when os.path.relpath would return abspath.
|
||||
# For DVC PathInfo we only need the relpath behavior.
|
||||
# See: https://bugs.python.org/issue40358
|
||||
try:
|
||||
path = super().relative_to(*other, **kwargs)
|
||||
path = super().relative_to(other, *args, **kwargs)
|
||||
except ValueError:
|
||||
path = relpath(self, other[0])
|
||||
path = relpath(self, other)
|
||||
return self.__class__(path)
|
||||
|
||||
|
||||
|
||||
@@ -44,7 +44,7 @@ class TestRemote:
|
||||
_check_status(status_dir, new=dir_hashes)
|
||||
|
||||
# Move cache and check status
|
||||
# See issue https://github.com/treeverse/dvc/issues/4383 for details
|
||||
# See issue https://github.com/iterative/dvc/issues/4383 for details
|
||||
backup_dir = dvc.cache.local.path + ".backup"
|
||||
shutil.move(dvc.cache.local.path, backup_dir)
|
||||
status = dvc.cloud.status(foo_hashes)
|
||||
@@ -116,7 +116,7 @@ class TestRemote:
|
||||
|
||||
@pytest.mark.xfail(raises=NotImplementedError, strict=False)
|
||||
def test_pull_00_prefix(self, tmp_dir, dvc, remote, monkeypatch):
|
||||
# Related: https://github.com/treeverse/dvc/issues/6089
|
||||
# Related: https://github.com/iterative/dvc/issues/6089
|
||||
|
||||
fs_type = type(dvc.cloud.get_remote_odb("upstream").fs)
|
||||
monkeypatch.setattr(fs_type, "_ALWAYS_TRAVERSE", True, raising=False)
|
||||
@@ -146,7 +146,7 @@ class TestRemote:
|
||||
|
||||
@pytest.mark.xfail(raises=NotImplementedError, strict=False)
|
||||
def test_pull_no_00_prefix(self, tmp_dir, dvc, remote, monkeypatch):
|
||||
# Related: https://github.com/treeverse/dvc/issues/6244
|
||||
# Related: https://github.com/iterative/dvc/issues/6244
|
||||
|
||||
fs_type = type(dvc.cloud.get_remote_odb("upstream").fs)
|
||||
monkeypatch.setattr(fs_type, "_ALWAYS_TRAVERSE", True, raising=False)
|
||||
|
||||
+1
-3
@@ -289,7 +289,7 @@ class Console:
|
||||
|
||||
return console.Console(stderr=True)
|
||||
|
||||
def table( # noqa: PLR0913
|
||||
def table(
|
||||
self,
|
||||
data: "TableData",
|
||||
headers: Optional["Headers"] = None,
|
||||
@@ -301,7 +301,6 @@ class Console:
|
||||
row_styles: Optional[Sequence["Styles"]] = None,
|
||||
borders: Union[bool, str] = False,
|
||||
colalign: Optional[tuple[str, ...]] = None,
|
||||
preserve_whitespace: bool = False,
|
||||
) -> None:
|
||||
from dvc.ui import table as t
|
||||
|
||||
@@ -330,7 +329,6 @@ class Console:
|
||||
pager=pager,
|
||||
force=force,
|
||||
colalign=colalign,
|
||||
preserve_whitespace=preserve_whitespace,
|
||||
)
|
||||
|
||||
def status(self, status: str, **kwargs: Any) -> "Status":
|
||||
|
||||
+9
-27
@@ -30,37 +30,19 @@ def plain_table(
|
||||
pager: bool = False,
|
||||
force: bool = True,
|
||||
colalign: Optional[tuple[str, ...]] = None,
|
||||
preserve_whitespace: bool = False,
|
||||
) -> None:
|
||||
import tabulate as tabulate_mod
|
||||
from funcy import nullcontext
|
||||
from packaging.version import Version
|
||||
from tabulate import tabulate
|
||||
|
||||
# NOTE: tabulate 0.10.0+ supports preserve_whitespace as a kwarg,
|
||||
# while 0.9.0 only respects the global PRESERVE_WHITESPACE.
|
||||
kwargs: dict = {}
|
||||
_orig = tabulate_mod.PRESERVE_WHITESPACE
|
||||
if preserve_whitespace:
|
||||
if Version(tabulate_mod.__version__) >= Version("0.10"):
|
||||
kwargs["preserve_whitespace"] = True
|
||||
else:
|
||||
tabulate_mod.PRESERVE_WHITESPACE = True
|
||||
|
||||
try:
|
||||
text: str = tabulate(
|
||||
data,
|
||||
headers if headers is not None else (),
|
||||
tablefmt="github" if markdown else "plain",
|
||||
disable_numparse=True,
|
||||
# None will be shown as "" by default, overriding
|
||||
missingval="-",
|
||||
colalign=colalign,
|
||||
**kwargs,
|
||||
)
|
||||
finally:
|
||||
tabulate_mod.PRESERVE_WHITESPACE = _orig
|
||||
|
||||
text: str = tabulate(
|
||||
data,
|
||||
headers if headers is not None else (),
|
||||
tablefmt="github" if markdown else "plain",
|
||||
disable_numparse=True,
|
||||
# None will be shown as "" by default, overriding
|
||||
missingval="-",
|
||||
colalign=colalign,
|
||||
)
|
||||
if markdown:
|
||||
# NOTE: md table is incomplete without the trailing newline
|
||||
text += "\n"
|
||||
|
||||
+1
-1
@@ -163,7 +163,7 @@ class Updater:
|
||||
if pkg not in instructions:
|
||||
return (
|
||||
"Find the latest release at "
|
||||
"[blue]https://github.com/treeverse/dvc/releases/latest[/]."
|
||||
"[blue]https://github.com/iterative/dvc/releases/latest[/]."
|
||||
)
|
||||
|
||||
instruction = instructions[pkg]
|
||||
|
||||
+16
-13
@@ -18,7 +18,7 @@ keywords = [
|
||||
]
|
||||
license = "Apache-2.0"
|
||||
license-files = ["LICENSE"]
|
||||
maintainers = [{ name = "Treeverse", email = "[email protected]" }]
|
||||
maintainers = [{ name = "Iterative", email = "[email protected]" }]
|
||||
authors = [{ name = "Dmitry Petrov", email = "[email protected]" }]
|
||||
requires-python = ">=3.9"
|
||||
classifiers = [
|
||||
@@ -29,7 +29,6 @@ classifiers = [
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Programming Language :: Python :: 3.14",
|
||||
]
|
||||
dynamic = ["version"]
|
||||
|
||||
@@ -41,14 +40,14 @@ dependencies = [
|
||||
"distro>=1.3",
|
||||
"dpath<3,>=2.1.0",
|
||||
"dulwich",
|
||||
"dvc-data>=3.18.2,<3.19.0",
|
||||
"dvc-data>=3.16.2,<3.17",
|
||||
"dvc-http>=2.29.0",
|
||||
"dvc-objects",
|
||||
"dvc-render>=1.0.1,<2",
|
||||
"dvc-studio-client>=0.21,<1",
|
||||
"dvc-task>=0.3.0,<1",
|
||||
"flatten-dict<1,>=0.4.1",
|
||||
"flufl.lock>=8.1.0,<10",
|
||||
"flatten_dict<1,>=0.4.1",
|
||||
"flufl.lock>=8.1.0,<9",
|
||||
"fsspec>=2024.2.0",
|
||||
"funcy>=1.14",
|
||||
"grandalf<1,>=0.7",
|
||||
@@ -59,18 +58,19 @@ dependencies = [
|
||||
"networkx>=2.5",
|
||||
"omegaconf",
|
||||
"packaging>=19",
|
||||
"pathspec>=0.10.3,<2",
|
||||
"pathspec>=0.10.3",
|
||||
"platformdirs<5,>=3.1.1",
|
||||
"psutil>=5.8",
|
||||
"pydot>=1.2.4",
|
||||
"pygtrie>=2.3.2",
|
||||
"pyparsing>=3.0.0",
|
||||
"pyparsing>=2.4.7",
|
||||
"requests>=2.22",
|
||||
"rich>=12",
|
||||
"ruamel.yaml>=0.17.11",
|
||||
"scmrepo>=3.5.2,<4",
|
||||
"shortuuid>=0.5",
|
||||
"shtab<2,>=1.3.4",
|
||||
# "shtab<2,>=1.3.4",
|
||||
"shtab @ git+https://github.com/skshetry/shtab.git@fish-shell-v2",
|
||||
"tabulate>=0.8.7",
|
||||
"tomlkit>=0.11.1",
|
||||
"tqdm<5,>=4.63.1",
|
||||
@@ -86,7 +86,7 @@ gdrive = ["dvc-gdrive>=3,<4"]
|
||||
gs = ["dvc-gs>=3.0.2,<4"]
|
||||
hdfs = ["dvc-hdfs>=3,<4"]
|
||||
lint = [
|
||||
"mypy==1.19.1",
|
||||
"mypy==1.18.2",
|
||||
"pandas-stubs",
|
||||
"types-colorama",
|
||||
"types-psutil",
|
||||
@@ -103,6 +103,7 @@ ssh = ["dvc-ssh>=4,<5"]
|
||||
ssh_gssapi = ["dvc-ssh[gssapi]>=4,<5"]
|
||||
testing = [
|
||||
"pytest-benchmark[histogram]>=5,<6",
|
||||
"pytest-test-utils",
|
||||
"uv",
|
||||
]
|
||||
tests = [
|
||||
@@ -110,11 +111,12 @@ tests = [
|
||||
"beautifulsoup4>=4.4",
|
||||
"dvc-ssh",
|
||||
"filelock",
|
||||
"pytest>=7,<10",
|
||||
"pytest>=7,<9",
|
||||
"pytest-rerunfailures<17.0",
|
||||
"pytest-cov>=4.1.0",
|
||||
"pytest-docker>=1,<4",
|
||||
"pytest-mock",
|
||||
"pytest-test-utils",
|
||||
"pytest-timeout>=2",
|
||||
"pytest-xdist>=3.2",
|
||||
'pywin32>=225; sys_platform == "win32"', # optional test dependency
|
||||
@@ -122,14 +124,14 @@ tests = [
|
||||
"sqlalchemy>=1,<3", # optional dependency for `import-db`
|
||||
"pandas>=1", # optional dependency for `import-db`
|
||||
]
|
||||
webdav = ["dvc-webdav>=3.0.1,<4"]
|
||||
webdav = ["dvc-webdav>=3,<4"]
|
||||
webhdfs = ["dvc-webhdfs>=3.1,<4"]
|
||||
webhdfs_kerberos = ["dvc-webhdfs[kerberos]>=3.1,<4"]
|
||||
|
||||
[project.urls]
|
||||
Documentation = "https://dvc.org/doc"
|
||||
Issues = "https://github.com/treeverse/dvc/issues"
|
||||
Source = "https://github.com/treeverse/dvc"
|
||||
Issues = "https://github.com/iterative/dvc/issues"
|
||||
Source = "https://github.com/iterative/dvc"
|
||||
|
||||
[project.scripts]
|
||||
dvc = "dvc.cli:main"
|
||||
@@ -247,6 +249,7 @@ module = [
|
||||
"shortuuid",
|
||||
"shtab",
|
||||
"upath",
|
||||
"virtualenv",
|
||||
"viztracer",
|
||||
"voluptuous",
|
||||
"yappi",
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
import os
|
||||
|
||||
# Increasing fd ulimit for tests
|
||||
if os.name == "nt":
|
||||
try:
|
||||
import win32file
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
win32file._setmaxstdio(4096)
|
||||
else:
|
||||
import resource
|
||||
|
||||
resource.setrlimit(resource.RLIMIT_NOFILE, (4096, 4096))
|
||||
|
||||
nproc_soft, nproc_hard = resource.getrlimit(resource.RLIMIT_NPROC)
|
||||
resource.setrlimit(resource.RLIMIT_NPROC, (nproc_hard, nproc_hard))
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
sys.exit(pytest.main(["-v", "-n=logical", "--dist=worksteal", *sys.argv[1:]]))
|
||||
@@ -194,7 +194,6 @@ def isolate(tmp_path_factory):
|
||||
monkeypatch.setenv(env_var, os.fspath(path))
|
||||
else:
|
||||
monkeypatch.setenv("HOME", str(home_dir))
|
||||
monkeypatch.setenv("XDG_CONFIG_HOME", str(home_dir / ".config"))
|
||||
|
||||
monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1")
|
||||
contents = b"""
|
||||
|
||||
@@ -91,7 +91,7 @@ def test_apply_untracked(tmp_dir, scm, dvc, exp_stage):
|
||||
|
||||
|
||||
def test_apply_unchanged_head(tmp_dir, scm, dvc, exp_stage):
|
||||
# see https://github.com/treeverse/dvc/issues/8764
|
||||
# see https://github.com/iterative/dvc/issues/8764
|
||||
tmp_dir.gen("params.yaml", "foo: 2")
|
||||
scm.add(["dvc.yaml", "dvc.lock", "params.yaml", "metrics.yaml"])
|
||||
scm.commit("commit foo=2")
|
||||
|
||||
@@ -53,7 +53,7 @@ def test_new_simple(tmp_dir, scm, dvc, exp_stage, mocker, name, workspace):
|
||||
if workspace:
|
||||
assert (tmp_dir / "metrics.yaml").read_text().strip() == "foo: 2"
|
||||
|
||||
exp_name = name or ref_info.name
|
||||
exp_name = name if name else ref_info.name
|
||||
assert dvc.experiments.get_exact_name([exp])[exp] == exp_name
|
||||
assert resolve_rev(scm, exp_name) == exp
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ def test_celery_logs(tmp_dir, scm, dvc, failed_exp_stage, follow, capsys, test_q
|
||||
|
||||
|
||||
def test_queue_doesnt_remove_untracked_params_file(tmp_dir, dvc, scm):
|
||||
"""Regression test for https://github.com/treeverse/dvc/issues/7842"""
|
||||
"""Regression test for https://github.com/iterative/dvc/issues/7842"""
|
||||
tmp_dir.gen("params.yaml", "foo: 1")
|
||||
stage = dvc.run(cmd="echo ${foo}", params=["foo"], name="echo-foo")
|
||||
scm.add(["dvc.yaml", "dvc.lock", ".gitignore"])
|
||||
|
||||
@@ -28,7 +28,7 @@ def test_exp_save(tmp_dir, dvc, scm, name):
|
||||
assert ref_info
|
||||
assert ref_info.baseline_sha == baseline
|
||||
|
||||
exp_name = name or ref_info.name
|
||||
exp_name = name if name else ref_info.name
|
||||
assert dvc.experiments.get_exact_name([exp])[exp] == exp_name
|
||||
assert resolve_rev(scm, exp_name) == exp
|
||||
|
||||
@@ -105,7 +105,7 @@ def test_exp_save_include_untracked(tmp_dir, dvc, scm):
|
||||
|
||||
|
||||
def test_exp_save_include_untracked_warning(tmp_dir, dvc, scm, mocker):
|
||||
"""Regression test for https://github.com/treeverse/dvc/issues/9061"""
|
||||
"""Regression test for https://github.com/iterative/dvc/issues/9061"""
|
||||
setup_stage(tmp_dir, dvc, scm)
|
||||
|
||||
new_dir = tmp_dir / "new_dir"
|
||||
|
||||
@@ -26,9 +26,8 @@ def test_modify_params(params_repo, dvc, changes, expected):
|
||||
("conf", "bar"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("no_hydra", [True, False])
|
||||
def test_hydra_compose_and_dump(
|
||||
tmp_dir, params_repo, dvc, hydra_enabled, config_dir, config_name, no_hydra
|
||||
tmp_dir, params_repo, dvc, hydra_enabled, config_dir, config_name
|
||||
):
|
||||
hydra_setup(
|
||||
tmp_dir,
|
||||
@@ -51,14 +50,14 @@ def test_hydra_compose_and_dump(
|
||||
if config_name is not None:
|
||||
conf["hydra"]["config_name"] = config_name
|
||||
|
||||
dvc.experiments.run(no_hydra=no_hydra)
|
||||
dvc.experiments.run()
|
||||
|
||||
if hydra_enabled and not no_hydra:
|
||||
if hydra_enabled:
|
||||
assert (tmp_dir / "params.yaml").parse() == {
|
||||
"db": {"driver": "mysql", "user": "omry", "pass": "secret"},
|
||||
}
|
||||
|
||||
dvc.experiments.run(params=["db=postgresql"], no_hydra=no_hydra)
|
||||
dvc.experiments.run(params=["db=postgresql"])
|
||||
assert (tmp_dir / "params.yaml").parse() == {
|
||||
"db": {
|
||||
"driver": "postgresql",
|
||||
@@ -118,7 +117,6 @@ def test_hydra_sweep(
|
||||
targets=None,
|
||||
copy_paths=None,
|
||||
message=None,
|
||||
no_hydra=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -139,76 +137,3 @@ def test_hydra_sweep_prefix_name(tmp_dir, params_repo, dvc):
|
||||
exp_names = [entry.name for entry in dvc.experiments.celery_queue.iter_queued()]
|
||||
for name, expected in zip(exp_names, expected_names):
|
||||
assert name == expected
|
||||
|
||||
|
||||
def test_mixing_no_hydra_and_params_flags(tmp_dir, params_repo, dvc):
|
||||
# Passing no_hydra should not prevent user from
|
||||
# using --set-param on unmodified params.yaml
|
||||
hydra_setup(
|
||||
tmp_dir,
|
||||
config_dir="conf",
|
||||
config_name="config",
|
||||
)
|
||||
|
||||
with dvc.config.edit() as conf:
|
||||
conf["hydra"]["enabled"] = True
|
||||
conf["hydra"]["config_dir"] = "conf"
|
||||
conf["hydra"]["config_name"] = "config"
|
||||
|
||||
dvc.experiments.run(no_hydra=True, params=["goo.bag=10.0"])
|
||||
|
||||
assert (tmp_dir / "params.yaml").parse() == {
|
||||
"foo": [{"bar": 1}, {"baz": 2}],
|
||||
"goo": {"bag": 10.0},
|
||||
"lorem": False,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"hydra_enabled,overrides,expected",
|
||||
[
|
||||
(
|
||||
True,
|
||||
["db=mysql,postgresql"],
|
||||
[
|
||||
{"params.yaml": ["db=mysql"]},
|
||||
{"params.yaml": ["db=postgresql"]},
|
||||
],
|
||||
),
|
||||
(
|
||||
False,
|
||||
["foo=bar,baz"],
|
||||
[{"params.yaml": ["foo=bar"]}, {"params.yaml": ["foo=baz"]}],
|
||||
),
|
||||
(
|
||||
False,
|
||||
[],
|
||||
[{}],
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("no_hydra", [True, False])
|
||||
def test_mixing_no_hydra_and_sweeps(
|
||||
tmp_dir, params_repo, dvc, mocker, hydra_enabled, overrides, expected, no_hydra
|
||||
):
|
||||
# Passing no_hydra should not prevent user from
|
||||
# queuing sweeps with --set-param and --queue
|
||||
patched = mocker.patch.object(dvc.experiments, "queue_one")
|
||||
|
||||
if hydra_enabled:
|
||||
hydra_setup(tmp_dir, config_dir="conf", config_name="config")
|
||||
with dvc.config.edit() as conf:
|
||||
conf["hydra"]["enabled"] = True
|
||||
|
||||
dvc.experiments.run(params=overrides, queue=True, no_hydra=no_hydra)
|
||||
|
||||
assert patched.call_count == len(expected)
|
||||
for e in expected:
|
||||
patched.assert_any_call(
|
||||
mocker.ANY,
|
||||
params=e,
|
||||
targets=None,
|
||||
copy_paths=None,
|
||||
message=None,
|
||||
no_hydra=no_hydra,
|
||||
)
|
||||
|
||||
@@ -4,7 +4,6 @@ from os.path import join
|
||||
import pytest
|
||||
|
||||
from dvc.cli import main
|
||||
from dvc.testing import matchers as M
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.serialize import JSONFileCorruptedError
|
||||
|
||||
@@ -96,7 +95,7 @@ def test_metrics_diff_json_unchanged(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
assert dvc.metrics.diff(a_rev="HEAD~2") == {}
|
||||
|
||||
|
||||
def test_metrics_diff_broken_json(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
def test_metrics_diff_broken_json(M, tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metrics = {"a": {"b": {"c": 1, "d": 1, "e": "3"}}}
|
||||
(tmp_dir / "m_temp.json").dump(metrics)
|
||||
run_copy_metrics(
|
||||
@@ -144,7 +143,7 @@ def test_metrics_diff_new_metric(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
}
|
||||
|
||||
|
||||
def test_metrics_diff_deleted_metric(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
def test_metrics_diff_deleted_metric(M, tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metrics = {"a": {"b": {"c": 1, "d": 1, "e": "3"}}}
|
||||
(tmp_dir / "m_temp.json").dump(metrics)
|
||||
run_copy_metrics(
|
||||
|
||||
@@ -11,7 +11,6 @@ from dvc.dvcfile import PROJECT_FILE
|
||||
from dvc.exceptions import OverlappingOutputPathsError
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.metrics.show import FileResult, Result
|
||||
from dvc.testing import matchers as M
|
||||
from dvc.utils.fs import remove
|
||||
from dvc.utils.serialize import JSONFileCorruptedError
|
||||
from dvc_data.index import DataIndexDirError
|
||||
@@ -247,7 +246,7 @@ def test_show_malformed_metric(tmp_dir, scm, dvc, caplog):
|
||||
)
|
||||
|
||||
|
||||
def test_metrics_show_no_target(tmp_dir, dvc, capsys):
|
||||
def test_metrics_show_no_target(M, tmp_dir, dvc, capsys):
|
||||
assert dvc.metrics.show(targets=["metrics.json"]) == {
|
||||
"": {"data": {"metrics.json": {"error": M.instance_of(FileNotFoundError)}}}
|
||||
}
|
||||
@@ -357,7 +356,7 @@ def test_top_level_parametrized(tmp_dir, dvc):
|
||||
}
|
||||
|
||||
|
||||
def test_metric_in_a_tracked_directory_with_missing_dir_file(tmp_dir, dvc):
|
||||
def test_metric_in_a_tracked_directory_with_missing_dir_file(M, tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen({"dir": {"file": "2"}})
|
||||
(tmp_dir / "dvc.yaml").dump({"metrics": [join("dir", "file")]})
|
||||
shutil.rmtree(tmp_dir / "dir") # remove from workspace
|
||||
|
||||
@@ -2,7 +2,6 @@ from os.path import join
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.testing import matchers as M
|
||||
from dvc.utils import relpath
|
||||
|
||||
|
||||
@@ -55,7 +54,7 @@ def test_diff_new(tmp_dir, scm, dvc):
|
||||
}
|
||||
|
||||
|
||||
def test_diff_deleted(tmp_dir, scm, dvc):
|
||||
def test_diff_deleted(M, tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo: bar")
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], name="echo-params")
|
||||
scm.add(["params.yaml", "Dvcfile"])
|
||||
|
||||
@@ -6,7 +6,6 @@ import pytest
|
||||
from dvc.dvcfile import PROJECT_FILE
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.metrics.show import FileResult, Result
|
||||
from dvc.testing import matchers as M
|
||||
from dvc_data.index import DataIndexDirError
|
||||
|
||||
|
||||
@@ -198,7 +197,7 @@ def test_top_level_parametrized(tmp_dir, dvc):
|
||||
}
|
||||
|
||||
|
||||
def test_param_in_a_tracked_directory_with_missing_dir_file(tmp_dir, dvc):
|
||||
def test_param_in_a_tracked_directory_with_missing_dir_file(M, tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen({"dir": {"file": "2"}})
|
||||
(tmp_dir / "dvc.yaml").dump({"params": [join("dir", "file")]})
|
||||
shutil.rmtree(tmp_dir / "dir") # remove from workspace
|
||||
|
||||
@@ -15,7 +15,6 @@ from dvc.output import Output
|
||||
from dvc.stage import PipelineStage, Stage
|
||||
from dvc.stage.cache import RunCacheNotSupported
|
||||
from dvc.stage.exceptions import StageFileDoesNotExistError, StageNotFound
|
||||
from dvc.testing import matchers as M
|
||||
from dvc.utils.fs import remove
|
||||
from dvc.utils.serialize import modify_yaml
|
||||
from dvc_data.hashfile.hash import file_md5
|
||||
@@ -56,7 +55,7 @@ def test_repro_frozen(tmp_dir, dvc, run_copy):
|
||||
assert stages == [data_stage, stage0]
|
||||
|
||||
|
||||
def test_downstream(tmp_dir, dvc):
|
||||
def test_downstream(M, tmp_dir, dvc):
|
||||
# The dependency graph should look like this:
|
||||
#
|
||||
# E
|
||||
@@ -149,7 +148,7 @@ def test_downstream(tmp_dir, dvc):
|
||||
assert [stage.name for stage in evaluation] == ["B-gen", "D-gen", "E-gen"]
|
||||
|
||||
# B, C should be run (in any order) before D
|
||||
# See https://github.com/treeverse/dvc/issues/3602
|
||||
# See https://github.com/iterative/dvc/issues/3602
|
||||
evaluation = dvc.reproduce(PROJECT_FILE + ":A-gen", downstream=True, force=True)
|
||||
|
||||
assert len(evaluation) == 5
|
||||
|
||||
@@ -36,7 +36,7 @@ def test_repro_allow_missing_and_pull(tmp_dir, dvc, mocker, local_remote):
|
||||
def test_repro_allow_missing_upstream_stage_modified(
|
||||
tmp_dir, dvc, mocker, local_remote
|
||||
):
|
||||
"""https://github.com/treeverse/dvc/issues/9530"""
|
||||
"""https://github.com/iterative/dvc/issues/9530"""
|
||||
tmp_dir.gen("params.yaml", "param: 1")
|
||||
dvc.stage.add(
|
||||
name="create-foo", cmd="echo ${param} > foo", params=["param"], outs=["foo"]
|
||||
|
||||
@@ -723,7 +723,7 @@ def test_add_symlink_file(tmp_dir, dvc):
|
||||
).is_symlink()
|
||||
|
||||
# Test that subsequent add succeeds
|
||||
# See https://github.com/treeverse/dvc/issues/4654
|
||||
# See https://github.com/iterative/dvc/issues/4654
|
||||
dvc.add(os.path.join("dir", "foo"))
|
||||
|
||||
|
||||
|
||||
@@ -19,36 +19,6 @@ def test_check_ignore(tmp_dir, dvc, file, ret, output, caplog, capsys):
|
||||
assert "Having any troubles?" not in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pattern, exception, ignored",
|
||||
[
|
||||
("volumes/functions/**", "volumes/functions/deno.json", False),
|
||||
("volumes/functions/**", "volumes/functions/deno.json*", False),
|
||||
("volumes/functions/", "volumes/functions/deno.json", True),
|
||||
("ignore.txt", "no-ignore.txt", False),
|
||||
("*ignore.txt", "no-ignore.txt", False),
|
||||
],
|
||||
)
|
||||
def test_check_ignore_reinclude(tmp_dir, dvc, pattern, exception, ignored):
|
||||
tmp_dir.gen(
|
||||
{
|
||||
".dvcignore": f"{pattern}\n!{exception}\n",
|
||||
"volumes": {"functions": {"deno.json": "", "other.py": ""}},
|
||||
"ignore.txt": "",
|
||||
"no-ignore.txt": "",
|
||||
}
|
||||
)
|
||||
file = exception.rstrip("*").replace("/", os.sep)
|
||||
sibling = (
|
||||
os.path.join("volumes", "functions", "other.py")
|
||||
if pattern.startswith("volumes/")
|
||||
else "ignore.txt"
|
||||
)
|
||||
|
||||
assert main(["check-ignore", "-q", file]) == (0 if ignored else 1)
|
||||
assert main(["check-ignore", "-q", sibling]) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"file,ret,output",
|
||||
[
|
||||
|
||||
@@ -88,8 +88,8 @@ def test_config_set_in_non_dvc_repo(tmp_dir, caplog):
|
||||
[
|
||||
(["core.analytics"], 0, "False"),
|
||||
(["core.remote"], 0, "myremote"),
|
||||
(["remote.myremote.profile"], 0, "treeverse"),
|
||||
(["remote.myremote.profile", "--local"], 0, "treeverse"),
|
||||
(["remote.myremote.profile"], 0, "iterative"),
|
||||
(["remote.myremote.profile", "--local"], 0, "iterative"),
|
||||
(
|
||||
["remote.myremote.profile", "--project"],
|
||||
251,
|
||||
@@ -122,7 +122,7 @@ def test_config_get(tmp_dir, dvc, capsys, caplog, args, ret, msg):
|
||||
textwrap.dedent(
|
||||
"""\
|
||||
['remote "myremote"']
|
||||
profile = treeverse
|
||||
profile = iterative
|
||||
['remote "other"']
|
||||
url = gs://bucket/path
|
||||
"""
|
||||
@@ -167,7 +167,7 @@ def test_config_list(tmp_dir, dvc, capsys):
|
||||
textwrap.dedent(
|
||||
"""\
|
||||
['remote "myremote"']
|
||||
profile = treeverse
|
||||
profile = iterative
|
||||
access_key_id = abcde
|
||||
secret_access_key = 123456
|
||||
['remote "other"']
|
||||
@@ -181,7 +181,7 @@ def test_config_list(tmp_dir, dvc, capsys):
|
||||
out, _ = capsys.readouterr()
|
||||
assert "remote.myremote.url=s3://bucket/path" in out
|
||||
assert "remote.myremote.region=us-east-2" in out
|
||||
assert "remote.myremote.profile=treeverse" in out
|
||||
assert "remote.myremote.profile=iterative" in out
|
||||
assert "remote.myremote.access_key_id=abcde" in out
|
||||
assert "remote.myremote.secret_access_key=123456" in out
|
||||
assert "remote.other.url=gs://bucket/path" in out
|
||||
@@ -242,7 +242,7 @@ def test_merging_two_levels(dvc):
|
||||
|
||||
|
||||
def test_config_loads_without_error_for_non_dvc_repo(tmp_dir):
|
||||
# regression testing for https://github.com/treeverse/dvc/issues/3328
|
||||
# regression testing for https://github.com/iterative/dvc/issues/3328
|
||||
Config.from_cwd(validate=True)
|
||||
|
||||
|
||||
|
||||
@@ -730,12 +730,12 @@ def test_pull_allow_missing(tmp_dir, dvc, local_remote):
|
||||
def test_pull_granular_excluding_import_that_cannot_be_pulled(
|
||||
tmp_dir, dvc, local_remote, mocker
|
||||
):
|
||||
"""Regression test for https://github.com/treeverse/dvc/issues/10309."""
|
||||
"""Regression test for https://github.com/iterative/dvc/issues/10309."""
|
||||
|
||||
mocker.patch("dvc.fs.dvc._DVCFileSystem", side_effect=CloneError("SCM error"))
|
||||
(stage,) = tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||||
imp_stage = dvc.imp(
|
||||
"https://user:[email protected]/treeverse/dvc.git",
|
||||
"https://user:[email protected]/iterative/dvc.git",
|
||||
"dir",
|
||||
out="new_dir",
|
||||
rev="HEAD",
|
||||
|
||||
+48
-105
@@ -6,10 +6,10 @@ from os.path import join
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
from pytest_test_utils import matchers as m
|
||||
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.data import _transform_git_paths_to_dvc, posixpath_to_os_path
|
||||
from dvc.testing import matchers as M
|
||||
from dvc.testing.tmp_dir import TmpDir, make_subrepo
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
@@ -60,7 +60,7 @@ def test_git_to_dvc_path_wdir_transformation(tmp_dir, scm, path):
|
||||
]
|
||||
|
||||
|
||||
def test_file(tmp_dir, dvc, scm):
|
||||
def test_file(M, tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen("foo", "foo", commit="add foo")
|
||||
tmp_dir.dvc_gen("foo", "foobar")
|
||||
remove(tmp_dir / "foo")
|
||||
@@ -75,7 +75,7 @@ def test_file(tmp_dir, dvc, scm):
|
||||
assert dvc.data_status(granular=True) == expected
|
||||
|
||||
|
||||
def test_directory(tmp_dir, dvc, scm):
|
||||
def test_directory(M, tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen({"dir": {"foo": "foo"}}, commit="add dir")
|
||||
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar", "foobar": "foobar"}})
|
||||
remove(tmp_dir / "dir")
|
||||
@@ -107,7 +107,7 @@ def test_directory(tmp_dir, dvc, scm):
|
||||
}
|
||||
|
||||
|
||||
def test_tracked_directory_deep(tmp_dir, dvc, scm):
|
||||
def test_tracked_directory_deep(M, tmp_dir, dvc, scm):
|
||||
"""Test for a directory not in cwd, but nested inside other directories."""
|
||||
(tmp_dir / "sub").gen({"dir": {"foo": "foo"}})
|
||||
dvc.add(fspath(tmp_dir / "sub" / "dir"))
|
||||
@@ -140,7 +140,7 @@ def test_tracked_directory_deep(tmp_dir, dvc, scm):
|
||||
|
||||
@pytest.mark.parametrize("git_repo_state", ["unborn", "committed"])
|
||||
@pytest.mark.parametrize("subdir", [True, False])
|
||||
def test_new_dvc_repo(tmp_dir, scm, subdir, git_repo_state):
|
||||
def test_new_dvc_repo(M, tmp_dir, scm, subdir, git_repo_state):
|
||||
if git_repo_state == "committed":
|
||||
tmp_dir.scm_gen("test", "test", commit="init")
|
||||
|
||||
@@ -167,7 +167,7 @@ def test_noscm_repo(tmp_dir, dvc):
|
||||
assert dvc.data_status() == {**EMPTY_STATUS, "unchanged": ["foo"]}
|
||||
|
||||
|
||||
def test_unchanged(tmp_dir, dvc, scm):
|
||||
def test_unchanged(M, tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen({"dir": {"foo": "foo"}}, commit="add dir")
|
||||
tmp_dir.dvc_gen("bar", "bar", commit="add foo")
|
||||
|
||||
@@ -195,7 +195,7 @@ def test_skip_uncached_pipeline_outputs(tmp_dir, dvc, run_copy_metrics):
|
||||
assert dvc.data_status(granular=True, untracked_files="all") == EMPTY_STATUS
|
||||
|
||||
|
||||
def test_outs_with_no_hashes(tmp_dir, dvc, scm):
|
||||
def test_outs_with_no_hashes(M, tmp_dir, dvc, scm):
|
||||
dvc.stage.add(single_stage=True, outs=["bar"])
|
||||
dvc.stage.add(deps=["bar"], outs=["foo"], name="copy", cmd="cp foo bar")
|
||||
|
||||
@@ -204,7 +204,7 @@ def test_outs_with_no_hashes(tmp_dir, dvc, scm):
|
||||
assert dvc.data_status(granular=True) == expected_output
|
||||
|
||||
|
||||
def test_outs_with_no_hashes_and_with_uncommitted_files(tmp_dir, dvc, scm):
|
||||
def test_outs_with_no_hashes_and_with_uncommitted_files(M, tmp_dir, dvc, scm):
|
||||
tmp_dir.gen({"bar": "bar", "foo": "foo"})
|
||||
dvc.stage.add(single_stage=True, outs=["bar"])
|
||||
dvc.stage.add(deps=["bar"], outs=["foo"], name="copy", cmd="cp foo bar")
|
||||
@@ -218,7 +218,7 @@ def test_outs_with_no_hashes_and_with_uncommitted_files(tmp_dir, dvc, scm):
|
||||
assert dvc.data_status(granular=True) == expected_output
|
||||
|
||||
|
||||
def test_subdir(tmp_dir, scm):
|
||||
def test_subdir(M, tmp_dir, scm):
|
||||
subrepo = tmp_dir / "sub"
|
||||
make_subrepo(subrepo, scm)
|
||||
|
||||
@@ -236,7 +236,7 @@ def test_subdir(tmp_dir, scm):
|
||||
}
|
||||
|
||||
|
||||
def test_untracked_newly_added_files(tmp_dir, dvc, scm):
|
||||
def test_untracked_newly_added_files(M, tmp_dir, dvc, scm):
|
||||
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||||
tmp_dir.gen("foobar", "foobar")
|
||||
|
||||
@@ -249,7 +249,7 @@ def test_untracked_newly_added_files(tmp_dir, dvc, scm):
|
||||
assert dvc.data_status(granular=True, untracked_files="all") == expected
|
||||
|
||||
|
||||
def test_missing_cache_workspace_exists(tmp_dir, dvc, scm):
|
||||
def test_missing_cache_workspace_exists(M, tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||||
tmp_dir.dvc_gen("foobar", "foobar")
|
||||
remove(dvc.cache.repo.path)
|
||||
@@ -272,7 +272,7 @@ def test_missing_cache_workspace_exists(tmp_dir, dvc, scm):
|
||||
}
|
||||
|
||||
|
||||
def test_missing_cache_missing_workspace(tmp_dir, dvc, scm):
|
||||
def test_missing_cache_missing_workspace(M, tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||||
tmp_dir.dvc_gen("foobar", "foobar")
|
||||
for path in [dvc.cache.repo.path, "dir", "foobar"]:
|
||||
@@ -297,7 +297,7 @@ def test_missing_cache_missing_workspace(tmp_dir, dvc, scm):
|
||||
}
|
||||
|
||||
|
||||
def test_git_committed_missing_cache_workspace_exists(tmp_dir, dvc, scm):
|
||||
def test_git_committed_missing_cache_workspace_exists(M, tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir")
|
||||
tmp_dir.dvc_gen("foobar", "foobar", commit="add foobar")
|
||||
remove(dvc.cache.local.path)
|
||||
@@ -317,7 +317,7 @@ def test_git_committed_missing_cache_workspace_exists(tmp_dir, dvc, scm):
|
||||
}
|
||||
|
||||
|
||||
def test_git_committed_missing_cache_missing_workspace(tmp_dir, dvc, scm):
|
||||
def test_git_committed_missing_cache_missing_workspace(M, tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir")
|
||||
tmp_dir.dvc_gen("foobar", "foobar", commit="add foobar")
|
||||
for path in [dvc.cache.repo.path, "dir", "foobar"]:
|
||||
@@ -338,7 +338,7 @@ def test_git_committed_missing_cache_missing_workspace(tmp_dir, dvc, scm):
|
||||
}
|
||||
|
||||
|
||||
def test_partial_missing_cache(tmp_dir, dvc, scm):
|
||||
def test_partial_missing_cache(M, tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||||
|
||||
# remove "foo" from cache
|
||||
@@ -362,7 +362,7 @@ def test_partial_missing_cache(tmp_dir, dvc, scm):
|
||||
}
|
||||
|
||||
|
||||
def test_missing_dir_object_from_head(tmp_dir, dvc, scm):
|
||||
def test_missing_dir_object_from_head(M, tmp_dir, dvc, scm):
|
||||
(stage,) = tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir")
|
||||
remove("dir")
|
||||
tmp_dir.dvc_gen({"dir": {"foobar": "foobar"}})
|
||||
@@ -384,7 +384,7 @@ def test_missing_dir_object_from_head(tmp_dir, dvc, scm):
|
||||
}
|
||||
|
||||
|
||||
def test_missing_dir_object_from_index(tmp_dir, dvc, scm):
|
||||
def test_missing_dir_object_from_index(M, tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir")
|
||||
remove("dir")
|
||||
(stage,) = tmp_dir.dvc_gen({"dir": {"foobar": "foobar"}})
|
||||
@@ -406,7 +406,7 @@ def test_missing_dir_object_from_index(tmp_dir, dvc, scm):
|
||||
}
|
||||
|
||||
|
||||
def test_remote_check(tmp_dir, dvc, scm, make_remote):
|
||||
def test_remote_check(M, tmp_dir, dvc, scm, make_remote):
|
||||
make_remote("default", default=True)
|
||||
make_remote("myremote", default=False)
|
||||
|
||||
@@ -442,7 +442,7 @@ def test_remote_check(tmp_dir, dvc, scm, make_remote):
|
||||
assert dvc.data_status(granular=True, **opts) == expected_g
|
||||
|
||||
|
||||
def test_missing_remote_cache(tmp_dir, dvc, scm, local_remote):
|
||||
def test_missing_remote_cache(M, tmp_dir, dvc, scm, local_remote):
|
||||
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||||
tmp_dir.dvc_gen("foobar", "foobar")
|
||||
|
||||
@@ -485,7 +485,7 @@ def test_missing_remote_cache(tmp_dir, dvc, scm, local_remote):
|
||||
|
||||
|
||||
def test_not_in_remote_respects_not_pushable(
|
||||
tmp_dir: TmpDir, dvc: Repo, scm, mocker, local_remote
|
||||
M: type["m.Matcher"], tmp_dir: TmpDir, dvc: Repo, scm, mocker, local_remote
|
||||
):
|
||||
stages: list[Stage] = tmp_dir.dvc_gen({"foo": "foo", "dir": {"foobar": "foobar"}})
|
||||
# Make foo not pushable
|
||||
@@ -529,7 +529,7 @@ def test_not_in_remote_respects_not_pushable(
|
||||
)
|
||||
|
||||
|
||||
def test_root_from_dir_to_file(tmp_dir, dvc, scm):
|
||||
def test_root_from_dir_to_file(M, tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen({"data": {"foo": "foo", "bar": "bar"}})
|
||||
remove("data")
|
||||
tmp_dir.gen("data", "file")
|
||||
@@ -555,7 +555,7 @@ def test_root_from_dir_to_file(tmp_dir, dvc, scm):
|
||||
}
|
||||
|
||||
|
||||
def test_root_from_file_to_dir(tmp_dir, dvc, scm):
|
||||
def test_root_from_file_to_dir(M, tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen("data", "file")
|
||||
remove("data")
|
||||
tmp_dir.gen({"data": {"foo": "foo", "bar": "bar"}})
|
||||
@@ -577,8 +577,8 @@ def test_root_from_file_to_dir(tmp_dir, dvc, scm):
|
||||
}
|
||||
|
||||
|
||||
def test_empty_dir(tmp_dir, scm, dvc):
|
||||
# regression testing for https://github.com/treeverse/dvc/issues/8958
|
||||
def test_empty_dir(tmp_dir, scm, dvc, M):
|
||||
# regression testing for https://github.com/iterative/dvc/issues/8958
|
||||
tmp_dir.dvc_gen({"data": {"foo": "foo"}})
|
||||
remove("data")
|
||||
|
||||
@@ -592,7 +592,7 @@ def test_empty_dir(tmp_dir, scm, dvc):
|
||||
}
|
||||
|
||||
|
||||
def test_untracked_files_filter_targets(tmp_dir, scm, dvc):
|
||||
def test_untracked_files_filter_targets(M, tmp_dir, scm, dvc):
|
||||
tmp_dir.gen(
|
||||
{"spam": "spam", "ham": "ham", "dir": {"eggs": "eggs", "bacon": "bacon"}}
|
||||
)
|
||||
@@ -644,13 +644,13 @@ def param(*values):
|
||||
param(
|
||||
["foo", "foobar"],
|
||||
{
|
||||
"committed": {"added": M.unordered("foo", "foobar")},
|
||||
"committed": {"added": m.unordered("foo", "foobar")},
|
||||
"uncommitted": {"deleted": ["foo"]},
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_filter_targets_files_after_dvc_commit(tmp_dir, dvc, scm, targets, expected):
|
||||
def test_filter_targets_files_after_dvc_commit(M, tmp_dir, dvc, scm, targets, expected):
|
||||
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar", "foobar": "foobar"})
|
||||
(tmp_dir / "foo").unlink() # deleted
|
||||
tmp_dir.gen({"bar": "bar modified", "baz": "baz new"})
|
||||
@@ -677,7 +677,7 @@ def test_filter_targets_files_after_dvc_commit(tmp_dir, dvc, scm, targets, expec
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_filter_targets_after_git_commit(tmp_dir, dvc, scm, targets, expected):
|
||||
def test_filter_targets_after_git_commit(M, tmp_dir, dvc, scm, targets, expected):
|
||||
tmp_dir.dvc_gen(
|
||||
{"foo": "foo", "bar": "bar", "foobar": "foobar", "baz": "baz"},
|
||||
commit="add files",
|
||||
@@ -718,7 +718,7 @@ def with_aliases(values, aliases):
|
||||
},
|
||||
{
|
||||
"committed": {
|
||||
"added": M.unordered(
|
||||
"added": m.unordered(
|
||||
join("dir", ""),
|
||||
join("dir", "foo"),
|
||||
join("dir", "sub", "bar"),
|
||||
@@ -779,7 +779,7 @@ def with_aliases(values, aliases):
|
||||
{},
|
||||
{
|
||||
"committed": {
|
||||
"added": M.unordered(join("dir", "foo"), join("dir", "foobar"))
|
||||
"added": m.unordered(join("dir", "foo"), join("dir", "foobar"))
|
||||
},
|
||||
"uncommitted": {"deleted": [join("dir", "foo")]},
|
||||
},
|
||||
@@ -794,7 +794,7 @@ def with_aliases(values, aliases):
|
||||
),
|
||||
)
|
||||
def test_filter_targets_inside_directory_after_dvc_commit(
|
||||
tmp_dir, dvc, scm, targets, expected_ng, expected_g
|
||||
M, tmp_dir, dvc, scm, targets, expected_ng, expected_g
|
||||
):
|
||||
tmp_dir.dvc_gen({"dir": {"foo": "foo", "sub": {"bar": "bar"}, "foobar": "foobar"}})
|
||||
(tmp_dir / "dir" / "foo").unlink() # deleted
|
||||
@@ -827,7 +827,7 @@ def test_filter_targets_inside_directory_after_dvc_commit(
|
||||
"unchanged": [join("dir", "foobar")],
|
||||
"committed": {
|
||||
"added": [join("dir", "baz")],
|
||||
"modified": M.unordered(
|
||||
"modified": m.unordered(
|
||||
join("dir", ""), join("dir", "sub", "bar")
|
||||
),
|
||||
"deleted": [join("dir", "foo")],
|
||||
@@ -861,7 +861,7 @@ def test_filter_targets_inside_directory_after_dvc_commit(
|
||||
),
|
||||
)
|
||||
def test_filter_targets_inside_directory_after_git_commit(
|
||||
tmp_dir, dvc, scm, targets, expected_ng, expected_g
|
||||
M, tmp_dir, dvc, scm, targets, expected_ng, expected_g
|
||||
):
|
||||
tmp_dir.dvc_gen(
|
||||
{"dir": {"foo": "foo", "sub": {"bar": "bar"}, "foobar": "foobar"}},
|
||||
@@ -893,22 +893,22 @@ def test_filter_targets_inside_directory_after_git_commit(
|
||||
param(
|
||||
(join("dir", "bar"), "foo"),
|
||||
["foo"],
|
||||
M.unordered(join("dir", "bar"), "foo"),
|
||||
m.unordered(join("dir", "bar"), "foo"),
|
||||
),
|
||||
param(
|
||||
(join("dir", "bar"), "dir"),
|
||||
[join("dir", "")],
|
||||
M.unordered(join("dir", ""), join("dir", "bar")),
|
||||
m.unordered(join("dir", ""), join("dir", "bar")),
|
||||
),
|
||||
param(
|
||||
("dir", "foo"),
|
||||
M.unordered(join("dir", ""), "foo"),
|
||||
M.unordered(join("dir", ""), join("dir", "bar"), "foo"),
|
||||
m.unordered(join("dir", ""), "foo"),
|
||||
m.unordered(join("dir", ""), join("dir", "bar"), "foo"),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_filter_targets_not_in_cache(
|
||||
local_remote, tmp_dir, scm, dvc, to_check, targets, non_granular, granular
|
||||
M, local_remote, tmp_dir, scm, dvc, to_check, targets, non_granular, granular
|
||||
):
|
||||
tmp_dir.dvc_gen({"foo": "foo", "dir": {"bar": "bar"}})
|
||||
|
||||
@@ -927,7 +927,7 @@ def test_filter_targets_not_in_cache(
|
||||
}
|
||||
|
||||
|
||||
def test_compat_legacy_new_cache_types(tmp_dir, dvc, scm):
|
||||
def test_compat_legacy_new_cache_types(M, tmp_dir, dvc, scm):
|
||||
tmp_dir.gen({"foo": "foo", "bar": "bar"})
|
||||
(tmp_dir / "foo.dvc").dump(
|
||||
{
|
||||
@@ -964,7 +964,7 @@ def test_compat_legacy_new_cache_types(tmp_dir, dvc, scm):
|
||||
}
|
||||
|
||||
|
||||
def test_missing_cache_remote_check(tmp_dir, dvc, scm, local_remote):
|
||||
def test_missing_cache_remote_check(M, tmp_dir, dvc, scm, local_remote):
|
||||
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||||
tmp_dir.dvc_gen("foobar", "foobar")
|
||||
remove(dvc.cache.repo.path)
|
||||
@@ -1046,7 +1046,7 @@ def test_missing_cache_remote_check(tmp_dir, dvc, scm, local_remote):
|
||||
{"committed": {"deleted": [join("dir", "")]}},
|
||||
{
|
||||
"committed": {
|
||||
"deleted": M.unordered(
|
||||
"deleted": m.unordered(
|
||||
join("dir", ""), join("dir", "bar"), join("dir", "foo")
|
||||
)
|
||||
}
|
||||
@@ -1060,7 +1060,7 @@ def test_missing_cache_remote_check(tmp_dir, dvc, scm, local_remote):
|
||||
},
|
||||
{
|
||||
"committed": {
|
||||
"added": M.unordered(
|
||||
"added": m.unordered(
|
||||
join("dir2", ""),
|
||||
join("dir2", "bar"),
|
||||
join("dir2", "foo"),
|
||||
@@ -1150,12 +1150,12 @@ def test_missing_cache_remote_check(tmp_dir, dvc, scm, local_remote):
|
||||
param(
|
||||
["dir2", "file2"],
|
||||
{
|
||||
"committed": {"added": M.unordered(join("dir2", ""), "file2")},
|
||||
"committed": {"added": m.unordered(join("dir2", ""), "file2")},
|
||||
"uncommitted": {"modified": [join("dir2", "")]},
|
||||
},
|
||||
{
|
||||
"committed": {
|
||||
"added": M.unordered(
|
||||
"added": m.unordered(
|
||||
join("dir2", ""),
|
||||
join("dir2", "bar"),
|
||||
join("dir2", "foo"),
|
||||
@@ -1178,7 +1178,7 @@ def test_missing_cache_remote_check(tmp_dir, dvc, scm, local_remote):
|
||||
},
|
||||
{
|
||||
"committed": {
|
||||
"added": M.unordered(
|
||||
"added": m.unordered(
|
||||
join("dir2", ""), join("dir2", "bar"), join("dir2", "foo")
|
||||
),
|
||||
"deleted": ["file"],
|
||||
@@ -1193,7 +1193,9 @@ def test_missing_cache_remote_check(tmp_dir, dvc, scm, local_remote):
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_renames(tmp_dir, scm, dvc, targets, expected_non_granular, expected_granular):
|
||||
def test_renames(
|
||||
M, tmp_dir, scm, dvc, targets, expected_non_granular, expected_granular
|
||||
):
|
||||
tmp_dir.dvc_gen(
|
||||
{"dir": {"foo": "foo", "bar": "bar"}, "file": "file"}, commit="add dir and file"
|
||||
)
|
||||
@@ -1209,62 +1211,3 @@ def test_renames(tmp_dir, scm, dvc, targets, expected_non_granular, expected_gra
|
||||
dvc.data_status(granular=True, with_renames=True, targets=targets)
|
||||
== EMPTY_STATUS | {"git": M.dict()} | expected_granular
|
||||
)
|
||||
|
||||
|
||||
def test_shallow_should_iterate_upto_tracked_directory(tmp_dir, dvc, scm, local_remote):
|
||||
"""Testing regression for https://github.com/treeverse/dvc/issues/10899."""
|
||||
|
||||
tmp_dir.scm_gen({"dir": {".gitkeep": ""}}, commit="mk dir")
|
||||
dataset_root = join("dir", "data", "")
|
||||
tmp_dir.dvc_gen({dataset_root: {"foo": "foo", "bar": "bar"}})
|
||||
files = [dataset_root, join(dataset_root, "foo"), join(dataset_root, "bar")]
|
||||
|
||||
assert dvc.data_status(not_in_remote=True) == {
|
||||
**EMPTY_STATUS,
|
||||
"committed": {"added": [dataset_root]},
|
||||
"not_in_remote": [dataset_root],
|
||||
"git": M.dict(),
|
||||
}
|
||||
assert dvc.data_status(granular=True, not_in_remote=True) == {
|
||||
**EMPTY_STATUS,
|
||||
"committed": {"added": M.unordered(*files)},
|
||||
"not_in_remote": M.unordered(*files),
|
||||
"git": M.dict(),
|
||||
}
|
||||
|
||||
assert dvc.push() == 3
|
||||
|
||||
assert dvc.data_status(not_in_remote=True, remote_refresh=True) == {
|
||||
**EMPTY_STATUS,
|
||||
"committed": {"added": [dataset_root]},
|
||||
"git": M.dict(),
|
||||
}
|
||||
assert dvc.data_status(granular=True, not_in_remote=True, remote_refresh=True) == {
|
||||
**EMPTY_STATUS,
|
||||
"committed": {"added": M.unordered(*files)},
|
||||
"git": M.dict(),
|
||||
}
|
||||
|
||||
|
||||
def test_duplicate_hashes_not_in_remote(tmp_dir, dvc, scm, local_remote):
|
||||
"""Test that files with identical content (same hash) are all correctly reported.
|
||||
|
||||
Regression test for https://github.com/iterative/dvc/issues/10959
|
||||
"""
|
||||
tmp_dir.dvc_gen(
|
||||
{"foo": "content", "bar": "content", "foobar": "foobar"}, commit="add files"
|
||||
)
|
||||
assert dvc.data_status(not_in_remote=True, granular=True, remote_refresh=True) == {
|
||||
**EMPTY_STATUS,
|
||||
"unchanged": M.unordered("foo", "bar", "foobar"),
|
||||
"not_in_remote": M.unordered("foo", "bar", "foobar"),
|
||||
"git": M.dict(),
|
||||
}
|
||||
|
||||
dvc.push()
|
||||
assert dvc.data_status(not_in_remote=True, granular=True, remote_refresh=True) == {
|
||||
**EMPTY_STATUS,
|
||||
"unchanged": M.unordered("foo", "bar", "foobar"),
|
||||
"not_in_remote": [],
|
||||
"git": M.dict(),
|
||||
}
|
||||
|
||||
@@ -101,7 +101,7 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
|
||||
|
||||
def test_relative_remote(erepo_dir, tmp_dir):
|
||||
# these steps reproduce the script on this issue:
|
||||
# https://github.com/treeverse/dvc/issues/2756
|
||||
# https://github.com/iterative/dvc/issues/2756
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("file", "contents", commit="create file")
|
||||
|
||||
|
||||
+2
-49
@@ -92,27 +92,6 @@ def test_gc_multiple_dvc_repos(tmp_dir, scm, dvc, erepo_dir):
|
||||
assert len(list(odb.all())) == 2
|
||||
|
||||
|
||||
def test_gc_multiple_dvc_repos_all_branches_num(tmp_dir, scm, dvc, erepo_dir):
|
||||
tmp_dir.dvc_gen("main", "main", commit="main")
|
||||
|
||||
erepo_dir.dvc.cache.local.path = dvc.cache.local.path
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.scm_gen("base", "base", commit="base")
|
||||
with erepo_dir.branch("feature", new=True):
|
||||
erepo_dir.dvc_gen("shared", "feature-v1", commit="feature-v1")
|
||||
erepo_dir.dvc.remove("shared.dvc")
|
||||
erepo_dir.dvc_gen("shared", "feature-v2", commit="feature-v2")
|
||||
|
||||
odb = dvc.cache.local
|
||||
assert len(list(odb.all())) == 3
|
||||
|
||||
dvc.gc(all_branches=True, num=2, repos=[erepo_dir])
|
||||
assert len(list(odb.all())) == 3
|
||||
|
||||
dvc.gc(all_branches=True, repos=[erepo_dir])
|
||||
assert len(list(odb.all())) == 2
|
||||
|
||||
|
||||
def test_all_commits(tmp_dir, scm, dvc):
|
||||
tmp_dir.dvc_gen("testfile", "uncommitted")
|
||||
tmp_dir.dvc_gen("testfile", "committed", commit="committed")
|
||||
@@ -458,16 +437,8 @@ def test_gc_logging(caplog, dvc, good_and_bad_cache):
|
||||
with caplog.at_level(logging.INFO, logger="dvc"):
|
||||
dvc.gc(workspace=True)
|
||||
|
||||
# Check that 3 objects were removed from either repo or local cache
|
||||
# (they point to the same ODB, so only one will be logged due to deduplication)
|
||||
has_repo_log = "Removed 3 objects from repo cache." in caplog.text
|
||||
has_local_log = "Removed 3 objects from local cache." in caplog.text
|
||||
|
||||
# Exactly one of repo or local should have the log, not both (deduplication)
|
||||
assert has_repo_log ^ has_local_log, (
|
||||
"Should have exactly one log for repo/local cache"
|
||||
)
|
||||
|
||||
assert "Removed 3 objects from repo cache." in caplog.text
|
||||
assert "No unused 'local' cache to remove." in caplog.text
|
||||
assert "No unused 'legacy' cache to remove." in caplog.text
|
||||
|
||||
|
||||
@@ -479,21 +450,3 @@ def test_gc_skip_failed(tmp_dir, dvc):
|
||||
dvc.gc(force=True, workspace=True)
|
||||
|
||||
dvc.gc(force=True, workspace=True, skip_failed=True)
|
||||
|
||||
|
||||
def test_gc_dry_logs_paths(caplog, tmp_dir, dvc):
|
||||
"""Test that dry run logs paths of objects to be removed."""
|
||||
odb = dvc.cache.local
|
||||
|
||||
# Add some unused objects directly to cache
|
||||
unused_hashes = ["test123", "test456", "test789"]
|
||||
for hash_val in unused_hashes:
|
||||
odb.add_bytes(hash_val, hash_val.encode("utf8"))
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="dvc_data.hashfile.gc"):
|
||||
dvc.gc(workspace=True, dry=True)
|
||||
|
||||
# Verify that paths are logged by dvc-data layer
|
||||
for hash_val in unused_hashes:
|
||||
expected_path = odb.oid_to_path(hash_val)
|
||||
assert f"Removing {expected_path}" in caplog.text
|
||||
|
||||
@@ -160,7 +160,7 @@ def test_non_cached_output(tmp_dir, erepo_dir):
|
||||
assert (tmp_dir / dst).read_text().strip() == "hello"
|
||||
|
||||
|
||||
# https://github.com/treeverse/dvc/pull/2837#discussion_r352123053
|
||||
# https://github.com/iterative/dvc/pull/2837#discussion_r352123053
|
||||
def test_absolute_file_outside_repo(tmp_dir, erepo_dir):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
Repo.get(os.fspath(erepo_dir), "/root/")
|
||||
|
||||
@@ -31,50 +31,6 @@ def test_ignore(tmp_dir, dvc, filename):
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("suffix, ignored", [("/**", False), ("/", True), ("", True)])
|
||||
@pytest.mark.parametrize("exception", ["deno.json", "deno.json*"])
|
||||
def test_reinclude_directory_contents(tmp_dir, dvc, suffix, ignored, exception):
|
||||
tmp_dir.gen(
|
||||
{
|
||||
".dvcignore": (
|
||||
f"volumes/functions{suffix}\n!volumes/functions/{exception}\n"
|
||||
),
|
||||
"volumes": {"functions": {"deno.json": "", "other.py": ""}},
|
||||
}
|
||||
)
|
||||
dvc._reset()
|
||||
|
||||
directory = tmp_dir / "volumes" / "functions"
|
||||
assert dvc.dvcignore.is_ignored_dir(str(directory)) is ignored
|
||||
assert dvc.dvcignore.is_ignored_file(str(directory / "deno.json")) is ignored
|
||||
assert dvc.dvcignore.is_ignored_file(str(directory / "other.py"))
|
||||
|
||||
files = {
|
||||
os.path.join(root, name)
|
||||
for root, _, names in dvc.dvcignore.walk(dvc.fs, tmp_dir)
|
||||
for name in names
|
||||
}
|
||||
expected = {str(tmp_dir / ".dvcignore")}
|
||||
if not ignored:
|
||||
expected.add(str(directory / "deno.json"))
|
||||
assert files == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pattern", ["ignore.txt", "*ignore.txt"])
|
||||
def test_reinclude_root_file(tmp_dir, dvc, pattern):
|
||||
tmp_dir.gen(
|
||||
{
|
||||
".dvcignore": f"{pattern}\n!no-ignore.txt\n",
|
||||
"ignore.txt": "",
|
||||
"no-ignore.txt": "",
|
||||
}
|
||||
)
|
||||
dvc._reset()
|
||||
|
||||
assert dvc.dvcignore.is_ignored_file(str(tmp_dir / "ignore.txt"))
|
||||
assert not dvc.dvcignore.is_ignored_file(str(tmp_dir / "no-ignore.txt"))
|
||||
|
||||
|
||||
def test_walk(tmp_dir, dvc):
|
||||
tmp_dir.gen(
|
||||
{
|
||||
|
||||
@@ -405,7 +405,7 @@ def test_import_non_existing(erepo_dir, tmp_dir, dvc):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
tmp_dir.dvc.imp(os.fspath(erepo_dir), "invalid_output")
|
||||
|
||||
# https://github.com/treeverse/dvc/pull/2837#discussion_r352123053
|
||||
# https://github.com/iterative/dvc/pull/2837#discussion_r352123053
|
||||
with pytest.raises(FileNotFoundError):
|
||||
tmp_dir.dvc.imp(os.fspath(erepo_dir), "/root/", "root")
|
||||
|
||||
|
||||
@@ -6,8 +6,6 @@ import pandas as pd
|
||||
import pytest
|
||||
from funcy import compact
|
||||
|
||||
from dvc.testing import matchers as M
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db_path(tmp_dir):
|
||||
@@ -48,7 +46,7 @@ def load_data(file, output_format):
|
||||
({"table": "model"}, "model"),
|
||||
],
|
||||
)
|
||||
def test(tmp_dir, scm, dvc, db_connection, seed_db, output_format, args, file_name):
|
||||
def test(M, tmp_dir, scm, dvc, db_connection, seed_db, output_format, args, file_name):
|
||||
seed_db(values=range(5))
|
||||
if output_format == "json":
|
||||
file_size = 96, 192
|
||||
|
||||
@@ -109,7 +109,7 @@ def test_gen_dvcignore(tmp_dir):
|
||||
|
||||
|
||||
def test_init_when_ignored_by_git(tmp_dir, scm, caplog):
|
||||
# https://github.com/treeverse/dvc/issues/3738
|
||||
# https://github.com/iterative/dvc/issues/3738
|
||||
tmp_dir.gen({".gitignore": ".*"})
|
||||
with caplog.at_level(logging.ERROR, logger="dvc"):
|
||||
assert main(["init"]) == 1
|
||||
|
||||
@@ -10,7 +10,6 @@ from dvc.fs import MemoryFileSystem
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.ls import _ls_tree, ls_tree
|
||||
from dvc.scm import CloneError
|
||||
from dvc.testing import matchers as M
|
||||
|
||||
FS_STRUCTURE = {
|
||||
"README.md": "content",
|
||||
@@ -461,7 +460,7 @@ def test_ls_shows_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
|
||||
match_files(files, ((("bar",), True),))
|
||||
|
||||
|
||||
def test_ls_granular(erepo_dir):
|
||||
def test_ls_granular(erepo_dir, M):
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen(
|
||||
{
|
||||
@@ -625,7 +624,7 @@ def test_subrepo(request, dvc_top_level, erepo_type):
|
||||
assert _list_files(subrepo, "dvc_dir") == {"lorem"}
|
||||
|
||||
|
||||
def test_broken_symlink(tmp_dir, dvc):
|
||||
def test_broken_symlink(tmp_dir, dvc, M):
|
||||
from dvc.fs import system
|
||||
|
||||
tmp_dir.gen("file", "content")
|
||||
@@ -655,7 +654,7 @@ def test_broken_symlink(tmp_dir, dvc):
|
||||
]
|
||||
|
||||
|
||||
def test_ls_broken_dir(tmp_dir, dvc):
|
||||
def test_ls_broken_dir(tmp_dir, dvc, M):
|
||||
from dvc_data.index import DataIndexDirError
|
||||
|
||||
tmp_dir.dvc_gen(
|
||||
@@ -837,7 +836,7 @@ def _simplify_tree(files):
|
||||
return ret
|
||||
|
||||
|
||||
def test_ls_tree(tmp_dir, scm, dvc):
|
||||
def test_ls_tree(M, tmp_dir, scm, dvc):
|
||||
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
|
||||
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
|
||||
|
||||
@@ -888,7 +887,7 @@ def test_ls_tree(tmp_dir, scm, dvc):
|
||||
}
|
||||
|
||||
|
||||
def test_ls_tree_dvc_only(tmp_dir, scm, dvc):
|
||||
def test_ls_tree_dvc_only(M, tmp_dir, scm, dvc):
|
||||
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
|
||||
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
|
||||
|
||||
@@ -906,7 +905,7 @@ def test_ls_tree_dvc_only(tmp_dir, scm, dvc):
|
||||
assert _simplify_tree(files) == expected
|
||||
|
||||
|
||||
def test_ls_tree_maxdepth(tmp_dir, scm, dvc):
|
||||
def test_ls_tree_maxdepth(M, tmp_dir, scm, dvc):
|
||||
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
|
||||
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
|
||||
|
||||
|
||||
+4
-201
@@ -6,20 +6,14 @@ import pytest
|
||||
|
||||
from dvc.cli import main
|
||||
from dvc.exceptions import MoveNotDataSourceError, OutputNotFoundError
|
||||
from dvc.stage.exceptions import StageFileAlreadyExistsError
|
||||
|
||||
|
||||
def test_move(tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen("foo", "bar")
|
||||
assert (tmp_dir / "foo.dvc").exists()
|
||||
dvc.move("foo", "bar")
|
||||
def test_move(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
dvc.move("foo", "foo1")
|
||||
|
||||
assert not (tmp_dir / "foo.dvc").exists()
|
||||
assert (tmp_dir / "bar.dvc").exists()
|
||||
assert not (tmp_dir / "foo").is_file()
|
||||
assert (tmp_dir / "bar").is_file()
|
||||
# should only have the new path in the .gitignore, and only once
|
||||
assert (tmp_dir / ".gitignore").read_text().splitlines() == ["/bar"]
|
||||
assert (tmp_dir / "foo1").is_file()
|
||||
|
||||
|
||||
def test_move_non_existent_file(dvc):
|
||||
@@ -27,24 +21,6 @@ def test_move_non_existent_file(dvc):
|
||||
dvc.move("non_existent_file", "dst")
|
||||
|
||||
|
||||
def test_move_missing_file(tmp_dir, dvc, scm, caplog):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
(tmp_dir / "foo").unlink()
|
||||
contents = (tmp_dir / "foo.dvc").parse()
|
||||
dvc.move("foo", "bar")
|
||||
|
||||
assert not (tmp_dir / "foo.dvc").exists()
|
||||
# only the path should be changed in the dvc file
|
||||
contents["outs"][0]["path"] = "bar"
|
||||
assert contents == (tmp_dir / "bar.dvc").parse()
|
||||
|
||||
# file should not be checked out
|
||||
assert not (tmp_dir / "foo").is_file()
|
||||
assert not (tmp_dir / "bar").is_file()
|
||||
# should only have the new path in the .gitignore, and only once
|
||||
assert (tmp_dir / ".gitignore").read_text().splitlines() == ["/bar"]
|
||||
|
||||
|
||||
def test_move_directory(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen("data", {"foo": "foo", "bar": "bar"})
|
||||
dvc.move("data", "dst")
|
||||
@@ -219,176 +195,3 @@ def test_move_meta(tmp_dir, dvc):
|
||||
custom_key: 42
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def test_import(tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen("foo", "foo", commit="add foo")
|
||||
imp_stage = dvc.imp(os.curdir, "foo", "foo_imported")
|
||||
|
||||
dvc.move("foo_imported", "foo_moved")
|
||||
|
||||
(stage,) = dvc.stage.collect("foo_moved.dvc")
|
||||
assert imp_stage.md5 != stage.md5
|
||||
res = (tmp_dir / "foo_moved.dvc").read_text()
|
||||
assert res == textwrap.dedent(
|
||||
f"""\
|
||||
md5: {stage.md5}
|
||||
frozen: true
|
||||
deps:
|
||||
- path: foo
|
||||
repo:
|
||||
url: {os.curdir}
|
||||
rev_lock: {scm.get_rev()}
|
||||
outs:
|
||||
- md5: acbd18db4cc2f85cedef654fccc4a4d8
|
||||
size: 3
|
||||
hash: md5
|
||||
path: foo_moved
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path_func",
|
||||
[pytest.param(os.path.abspath, id="abs"), pytest.param(os.path.relpath, id="rel")],
|
||||
)
|
||||
def test_import_url_in_repo(tmp_dir, dvc, path_func):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
imp_stage = dvc.imp_url(path_func(tmp_dir / "foo"), "foo_imported")
|
||||
(tmp_dir / "data").mkdir()
|
||||
|
||||
dvc.move("foo_imported", os.path.join("data", "foo_moved"))
|
||||
|
||||
(stage,) = dvc.stage.collect(os.path.join("data", "foo_moved.dvc"))
|
||||
assert imp_stage.md5 != stage.md5
|
||||
res = (tmp_dir / "data" / "foo_moved.dvc").read_text()
|
||||
assert res == textwrap.dedent(
|
||||
f"""\
|
||||
md5: {stage.md5}
|
||||
frozen: true
|
||||
deps:
|
||||
- md5: acbd18db4cc2f85cedef654fccc4a4d8
|
||||
size: 3
|
||||
hash: md5
|
||||
path: ../foo
|
||||
outs:
|
||||
- md5: acbd18db4cc2f85cedef654fccc4a4d8
|
||||
size: 3
|
||||
hash: md5
|
||||
path: foo_moved
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path_func",
|
||||
[pytest.param(os.path.abspath, id="abs"), pytest.param(os.path.relpath, id="rel")],
|
||||
)
|
||||
def test_import_url_out_of_repo(tmp_dir, dvc, scm, path_func, make_tmp_dir):
|
||||
external = make_tmp_dir("external")
|
||||
external.gen("foo", "foo")
|
||||
|
||||
imp_stage = dvc.imp_url(path_func(external / "foo"), "foo_imported")
|
||||
|
||||
data_dir = tmp_dir / "data"
|
||||
data_dir.mkdir()
|
||||
|
||||
new_path = data_dir / "foo_moved"
|
||||
new_dvcfile = new_path.with_suffix(".dvc")
|
||||
dvc.move("foo_imported", os.fspath(new_path))
|
||||
|
||||
(stage,) = dvc.stage.collect(os.fspath(new_dvcfile))
|
||||
assert imp_stage.md5 != stage.md5
|
||||
|
||||
with data_dir.chdir():
|
||||
expected_path = path_func(external / "foo")
|
||||
|
||||
assert new_dvcfile.parse() == {
|
||||
"md5": stage.md5,
|
||||
"frozen": True,
|
||||
"deps": [
|
||||
{
|
||||
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
|
||||
"size": 3,
|
||||
"hash": "md5",
|
||||
"path": expected_path,
|
||||
}
|
||||
],
|
||||
"outs": [
|
||||
{
|
||||
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
|
||||
"size": 3,
|
||||
"hash": "md5",
|
||||
"path": "foo_moved",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path_func",
|
||||
[pytest.param(os.path.abspath, id="abs"), pytest.param(os.path.relpath, id="rel")],
|
||||
)
|
||||
def test_all_metadata_are_preserved(tmp_dir, dvc, make_tmp_dir, path_func):
|
||||
external = make_tmp_dir("external")
|
||||
external.gen("foo", "foo")
|
||||
|
||||
contents = {
|
||||
"md5": "bad", # placeholder, does not matter for the test
|
||||
"frozen": True,
|
||||
"desc": "this is a stage description",
|
||||
"always_changed": True,
|
||||
"meta": {"custom_key": 42},
|
||||
"deps": [
|
||||
{
|
||||
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
|
||||
"size": 3,
|
||||
"hash": "md5",
|
||||
"path": path_func(external / "foo"),
|
||||
}
|
||||
],
|
||||
"outs": [
|
||||
{
|
||||
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
|
||||
"path": "foo_imported",
|
||||
"persist": True,
|
||||
"hash": "md5",
|
||||
"size": 3,
|
||||
"desc": "this is a description",
|
||||
"type": "model",
|
||||
"labels": ["label1", "label2"],
|
||||
"meta": {"custom_key": 42},
|
||||
"cache": False,
|
||||
"remote": "myremote",
|
||||
"push": False,
|
||||
}
|
||||
],
|
||||
}
|
||||
(tmp_dir / "foo_imported.dvc").dump(contents)
|
||||
(tmp_dir / "foo_imported").write_text("foo")
|
||||
|
||||
data_dir = tmp_dir / "data"
|
||||
data_dir.mkdir()
|
||||
|
||||
new_path = data_dir / "foo_moved"
|
||||
new_dvcfile = new_path.with_suffix(".dvc")
|
||||
dvc.move("foo_imported", os.fspath(new_path))
|
||||
|
||||
(stage,) = dvc.stage.collect(os.fspath(new_dvcfile))
|
||||
|
||||
with data_dir.chdir():
|
||||
expected_path = path_func(external / "foo")
|
||||
|
||||
contents["outs"][0] |= {"path": "foo_moved"}
|
||||
contents["deps"][0] |= {"path": expected_path}
|
||||
contents |= {"md5": stage.md5}
|
||||
assert new_dvcfile.parse() == contents
|
||||
|
||||
|
||||
def test_move_dst_stage_file_already_exists(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
|
||||
|
||||
with pytest.raises(StageFileAlreadyExistsError) as exc_info:
|
||||
dvc.move("foo", "bar")
|
||||
assert str(exc_info.value) == "'bar.dvc' already exists"
|
||||
assert exc_info.value.__cause__ is None
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user