Compare commits

..
98 changed files with 463 additions and 980 deletions
+4 -4
View File
@@ -737,8 +737,8 @@
"tests/func/test_config.py::test_config_gdrive_fields": 0.12703910000016094,
"tests/func/test_config.py::test_config_get[args0-0-False]": 0.20948869999995168,
"tests/func/test_config.py::test_config_get[args1-0-myremote]": 0.1829517000001033,
"tests/func/test_config.py::test_config_get[args2-0-treeverse]": 0.19686669999998685,
"tests/func/test_config.py::test_config_get[args3-0-treeverse]": 0.1675059000000374,
"tests/func/test_config.py::test_config_get[args2-0-iterative]": 0.19686669999998685,
"tests/func/test_config.py::test_config_get[args3-0-iterative]": 0.1675059000000374,
"tests/func/test_config.py::test_config_get[args4-251-option 'profile' doesn't exist]": 0.18096200000002227,
"tests/func/test_config.py::test_config_get[args5-0-gs://bucket/path]": 0.12368970000022728,
"tests/func/test_config.py::test_config_get[args6-0-gs://bucket/path]": 0.13007389999995667,
@@ -2642,7 +2642,7 @@
"tests/unit/test_updater.py::test_is_enabled[config0-True]": 0.13130560000013247,
"tests/unit/test_updater.py::test_is_enabled[config1-True]": 0.14342600000009043,
"tests/unit/test_updater.py::test_is_enabled[config2-False]": 0.13593219999984285,
"tests/unit/test_updater.py::test_notify_message[None-Find the latest release at https://github.com/treeverse/dvc/releases/latest.]": 0.014024399999925663,
"tests/unit/test_updater.py::test_notify_message[None-Find the latest release at https://github.com/iterative/dvc/releases/latest.]": 0.014024399999925663,
"tests/unit/test_updater.py::test_notify_message[binary-To upgrade, uninstall dvc and reinstall from https://dvc.org.]": 0.013217200000326557,
"tests/unit/test_updater.py::test_notify_message[brew-To upgrade, run 'brew upgrade dvc'.]": 0.013467300000229443,
"tests/unit/test_updater.py::test_notify_message[choco-To upgrade, run 'choco upgrade dvc'.]": 0.014645900000004985,
@@ -2652,7 +2652,7 @@
"tests/unit/test_updater.py::test_notify_message[osxpkg-To upgrade, uninstall dvc and reinstall from https://dvc.org.]": 0.013980499999888707,
"tests/unit/test_updater.py::test_notify_message[pip-To upgrade, run 'pip install --upgrade dvc'.]": 0.015155299999833005,
"tests/unit/test_updater.py::test_notify_message[rpm-To upgrade, run 'yum update dvc'.]": 0.014036500000202068,
"tests/unit/test_updater.py::test_notify_message[unknown-Find the latest release at https://github.com/treeverse/dvc/releases/latest.]": 0.01609110000003966,
"tests/unit/test_updater.py::test_notify_message[unknown-Find the latest release at https://github.com/iterative/dvc/releases/latest.]": 0.01609110000003966,
"tests/unit/ui/test_console.py::test_capsys_works": 0.003642499999841675,
"tests/unit/ui/test_console.py::test_write": 0.004320600000255581,
"tests/unit/ui/test_console.py::test_write_json[False-{\"hello\": \"world\", \"date\": \"1970-01-01 00:00:00\"}\\n]": 0.004433300000073359,
+1 -1
View File
@@ -57,7 +57,7 @@ $ dvc doctor
**Additional Information (if any):**
<!--
Please check https://github.com/treeverse/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.
Please check https://github.com/iterative/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.
If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.
If the issue is regarding the performance, please attach the profiling information and the benchmark comparisons.
+1 -1
View File
@@ -1,5 +1,5 @@
* [ ] ❗ I have followed the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) checklist.
* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/treeverse/dvc.org) and linked it here.
* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here.
Thank you for the contribution - we'll try to review it as soon as possible. 🙏
+4 -4
View File
@@ -16,14 +16,14 @@ jobs:
steps:
- uses: actions/setup-python@v6
with:
python-version: "3.14"
python-version: "3.13"
- uses: actions/checkout@v6
- uses: actions/checkout@v5
with:
ref: ${{ github.event.pull_request.base.sha }}
fetch-depth: 0
- uses: astral-sh/setup-uv@v7
- uses: astral-sh/setup-uv@v6
with:
enable-cache: true
cache-dependency-glob: pyproject.toml
@@ -32,7 +32,7 @@ jobs:
- name: run benchmarks on base branch
run: pytest --benchmark-autosave dvc/testing/benchmarks/ -k 'test_init or test_help'
- uses: actions/checkout@v6
- uses: actions/checkout@v5
with:
fetch-depth: 0
clean: false
+7 -7
View File
@@ -19,15 +19,15 @@ jobs:
name: Build & Verify package
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v5
with:
fetch-depth: 0
- uses: actions/setup-python@v6
with:
python-version: "3.14"
python-version: 3.13
- uses: astral-sh/setup-uv@v7
- uses: astral-sh/setup-uv@v6
- name: Omit local version for Test PyPI upload
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
@@ -41,7 +41,7 @@ jobs:
- name: Check dist
run: uv tool run twine check --strict dist/*
- uses: actions/upload-artifact@v6
- uses: actions/upload-artifact@v4
with:
name: Packages
path: dist/
@@ -51,7 +51,7 @@ jobs:
name: Publish dev package to test.pypi.org
runs-on: ubuntu-latest
needs: build
if: ${{ github.event.action == 'published' || (github.repository == 'treeverse/dvc' && github.event_name == 'push' && github.ref == 'refs/heads/main') }}
if: ${{ github.event.action == 'published' || (github.repository == 'iterative/dvc' && github.event_name == 'push' && github.ref == 'refs/heads/main') }}
environment:
name: test-pypi
@@ -61,7 +61,7 @@ jobs:
id-token: write
steps:
- uses: actions/download-artifact@v7
- uses: actions/download-artifact@v5
with:
name: Packages
path: dist
@@ -86,7 +86,7 @@ jobs:
id-token: write
steps:
- uses: actions/download-artifact@v7
- uses: actions/download-artifact@v5
with:
name: Packages
path: dist
+3 -3
View File
@@ -28,13 +28,13 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Initialize CodeQL
uses: github/codeql-action/init@v4
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
queries: security-extended
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v4
uses: github/codeql-action/analyze@v3
+5 -5
View File
@@ -19,17 +19,17 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest]
pyv: ["3.14"]
pyv: ["3.13"]
plugin: ["dvc-s3"]
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v5
with:
path: dvc
- uses: actions/checkout@v6
- uses: actions/checkout@v5
with:
repository: treeverse/${{ matrix.plugin }}
repository: iterative/${{ matrix.plugin }}
ref: main
path: ${{ matrix.plugin }}
@@ -38,7 +38,7 @@ jobs:
with:
python-version: ${{ matrix.pyv }}
- uses: astral-sh/setup-uv@v7
- uses: astral-sh/setup-uv@v6
with:
enable-cache: true
cache-dependency-glob: "**/pyproject.toml"
+7 -19
View File
@@ -26,14 +26,14 @@ jobs:
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v5
with:
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.14'
- uses: astral-sh/setup-uv@v7
python-version: '3.13'
- uses: astral-sh/setup-uv@v6
with:
enable-cache: true
cache-dependency-glob: pyproject.toml
@@ -42,7 +42,7 @@ jobs:
UV_LINK_MODE: ${{ runner.os == 'Windows' && 'symlink' || 'hardlink' }}
- name: Cache mypy
uses: actions/cache@v5
uses: actions/cache@v4
with:
path: .mypy_cache
key: mypy-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}
@@ -55,7 +55,7 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
pyv: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
pyv: ["3.9", "3.10", "3.11", "3.12", "3.13"]
include:
- os: windows-latest
pyv: "3.9"
@@ -117,27 +117,15 @@ jobs:
- os: windows-latest
pyv: "3.13"
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
- os: windows-latest
pyv: "3.14"
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
- os: windows-latest
pyv: "3.14"
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
- os: windows-latest
pyv: "3.14"
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
- os: windows-latest
pyv: "3.14"
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v5
with:
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.pyv }}
- uses: astral-sh/setup-uv@v7
- uses: astral-sh/setup-uv@v6
with:
enable-cache: true
cache-dependency-glob: pyproject.toml
+1 -1
View File
@@ -22,7 +22,7 @@ repos:
- id: sort-simple-yaml
- id: trailing-whitespace
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: 'v0.14.10'
rev: 'v0.13.2'
hooks:
- id: ruff-check
args: [--fix, --exit-non-zero-on-fix]
+1 -1
View File
@@ -4,5 +4,5 @@
"data-science", "data-version-control", "machine-learning", "git",
"developer-tools", "reproducibility", "collaboration", "ai", "python"],
"contributors": [
{"name": "DVC team", "type": "Other", "affiliation": "Treeverse"}]
{"name": "DVC team", "type": "Other", "affiliation": "Iterative"}]
}
+2 -2
View File
@@ -6,8 +6,8 @@ message: >-
type: software
authors:
- name: The DVC team and contributors
website: 'https://github.com/treeverse'
repository-code: 'https://github.com/treeverse/dvc'
website: 'https://github.com/iterative'
repository-code: 'https://github.com/iterative/dvc'
url: 'https://dvc.org'
abstract: >-
DVC is a tool for data science that takes advantage of existing software engineering toolset.
+1 -1
View File
@@ -186,7 +186,7 @@
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2025 Treeverse.
Copyright 2017-2021 Iterative, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
+15 -9
View File
@@ -167,7 +167,7 @@ To install the development version, run:
.. code-block:: bash
pip install "dvc @ git+https://github.com/treeverse/dvc"
pip install git+git://github.com/iterative/dvc
Package (Platform-specific)
---------------------------
@@ -175,7 +175,7 @@ Package (Platform-specific)
|Packages|
Self-contained packages for Linux, Windows, and Mac are available.
The latest version of the packages can be found on the GitHub `releases page <https://github.com/treeverse/dvc/releases>`_.
The latest version of the packages can be found on the GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
Ubuntu / Debian (deb)
^^^^^^^^^^^^^^^^^^^^^
@@ -198,6 +198,8 @@ Fedora / CentOS (rpm)
Contributing
============
|Maintainability|
Contributions are welcome!
Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more details.
Thanks to all our contributors!
@@ -235,19 +237,23 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
:target: https://dvc.org
:alt: DVC logo
.. |VS Code Extension Overview| image:: https://raw.githubusercontent.com/treeverse/vscode-dvc/main/extension/docs/overview.gif
.. |VS Code Extension Overview| image:: https://raw.githubusercontent.com/iterative/vscode-dvc/main/extension/docs/overview.gif
:alt: DVC Extension for VS Code
.. |CI| image:: https://github.com/treeverse/dvc/actions/workflows/tests.yaml/badge.svg
:target: https://github.com/treeverse/dvc/actions/workflows/tests.yaml
.. |CI| image:: https://github.com/iterative/dvc/actions/workflows/tests.yaml/badge.svg
:target: https://github.com/iterative/dvc/actions/workflows/tests.yaml
:alt: GHA Tests
.. |Maintainability| image:: https://img.shields.io/codeclimate/maintainability/iterative/dvc
:target: https://codeclimate.com/github/iterative/dvc
:alt: Code Climate
.. |Python Version| image:: https://img.shields.io/pypi/pyversions/dvc
:target: https://pypi.org/project/dvc
:alt: Python Version
.. |Coverage| image:: https://codecov.io/gh/treeverse/dvc/branch/main/graph/badge.svg
:target: https://codecov.io/gh/treeverse/dvc
.. |Coverage| image:: https://codecov.io/gh/iterative/dvc/branch/main/graph/badge.svg
:target: https://codecov.io/gh/iterative/dvc
:alt: Codecov
.. |Snap| image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
@@ -286,8 +292,8 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
:target: https://dvc.org/img/flow.gif
:alt: how_dvc_works
.. |Contribs| image:: https://contrib.rocks/image?repo=treeverse/dvc
:target: https://github.com/treeverse/dvc/graphs/contributors
.. |Contribs| image:: https://contrib.rocks/image?repo=iterative/dvc
:target: https://github.com/iterative/dvc/graphs/contributors
:alt: Contributors
.. |VS Code| image:: https://img.shields.io/visual-studio-marketplace/v/Iterative.dvc?color=blue&label=VSCode&logo=visualstudiocode&logoColor=blue
+1 -1
View File
@@ -206,7 +206,7 @@ def add_debugging_flags(parser):
from argparse import SUPPRESS
# For detailed info see:
# https://github.com/treeverse/dvc/wiki/Debugging,-Profiling-and-Benchmarking-DVC
# https://github.com/iterative/dvc/wiki/Debugging,-Profiling-and-Benchmarking-DVC
args, _ = parser.parse_known_args()
verbose = args.verbose
+1 -1
View File
@@ -96,7 +96,7 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
f"See {link} for more info."
)
else:
link = format_link("https://github.com/treeverse/dvc/issues")
link = format_link("https://github.com/iterative/dvc/issues")
hint = f"\nPlease report this bug to {link}. Thank you!"
logger.exception(
+130 -6
View File
@@ -31,6 +31,27 @@ _dvc_compgen_remotes() {
_dvc_compgen_config_vars() {
compgen -W "${_dvc_config_vars[*]}" -- $1
}
_dvc_compgen_remote_config_vars() {
local cur prev words cword remote_name filtered
_init_completion || return
# last non-option word before cursor
local nonopts=()
for w in "${words[@]:1:$((cword-1))}"; do
[[ $w == -* ]] || nonopts+=("$w")
done
remote_name="${nonopts[-1]}"
if [[ -n $remote_name ]]; then
local _dvc_remote_config_vars=($(
dvc config --available-options -q 2> /dev/null \
| grep "^remote\\.${remote_name}\\." \
| sed "s/^remote\\.${remote_name}\\.//"
))
compgen -W "${_dvc_remote_config_vars[*]}" -- $1
fi
}
"""
ZSH_PREAMBLE = """
@@ -69,27 +90,111 @@ _dvc_compadd_remotes() {
_describe 'remotes' "($(dvc remote list | cut -d' ' -f1))"
}
_dvc_config_config_vars_for_completion() {
dvc config --available-options -q 2> /dev/null
}
_dvc_compadd_config_vars() {
_describe 'config_vars' _dvc_config_vars
}
_dvc_compadd_remote_config_vars() {
local remote_name filtered
# last non-option word before cursor
remote_name=${${words[1,CURRENT-1]##-*}[-1]}
if [[ -n $remote_name ]]; then
filtered=$(_dvc_config_config_vars_for_completion \\
| grep "^remote.${remote_name}\\\\." \\
| sed "s/^remote\\\\.${remote_name}\\\\.//"
)
compadd -- $=filtered
fi
}
"""
FISH_PREAMBLE = """
function __fish_complete_dvc_files
__fish_complete_path | string match -re '\\*?.dvc|Dvcfile|dvc\\.yaml'
end
function __fish_complete_dvc_stages
for line in (dvc stage list -q)
set -l parts (string split -m1 ' ' -- $line)
set -l name $parts[1]
set -l desc (string trim $parts[2])
echo -e "$name $desc"
end
end
function __fish_complete_dvc_stages_and_files
__fish_complete_dvc_stages
__fish_complete_dvc_files
end
function __fish_complete_dvc_experiments
dvc exp list -q --all-commits --names-only
end
function __fish_complete_dvc_remotes
dvc remote list | cut -d' ' -f1
end
function __fish_complete_dvc_remote_config_vars
# last non-option word before cursor
set -l remote_name (commandline -opc | string match -rv '^-' | tail -n 1)
if test -n "$remote_name"
dvc config --available-options -q 2>/dev/null |
string match -re "^remote\\.$remote_name\\." |
string replace -r "^remote\\.$remote_name\\." ''
end
end
"""
PREAMBLE = {
"bash": BASH_PREAMBLE,
"zsh": ZSH_PREAMBLE,
"fish": FISH_PREAMBLE,
}
FILE = shtab.FILE
DIR = shtab.DIRECTORY
DVC_FILE = {"bash": "_dvc_compgen_DVCFiles", "zsh": "_dvc_compadd_DVCFiles"}
STAGE = {"bash": "_dvc_compgen_stages", "zsh": "_dvc_compadd_stages"}
DVC_FILE = {
"bash": "_dvc_compgen_DVCFiles",
"zsh": "_dvc_compadd_DVCFiles",
"fish": '-f -a "(__fish_complete_dvc_files)"',
}
STAGE = {
"bash": "_dvc_compgen_stages",
"zsh": "_dvc_compadd_stages",
"fish": '-f -a "(__fish_complete_dvc_stages)"',
}
DVCFILES_AND_STAGE = {
"bash": "_dvc_compgen_stages_and_files",
"zsh": "_dvc_compadd_stages_and_files",
"fish": '-f -a "(__fish_complete_dvc_stages_and_files)"',
}
EXPERIMENT = {
"bash": "_dvc_compgen_exps",
"zsh": "_dvc_compadd_exps",
"fish": '-f -a "(__fish_complete_dvc_experiments)"',
}
REMOTE = {
"bash": "_dvc_compgen_remotes",
"zsh": "_dvc_compadd_remotes",
"fish": '-f -a "(__fish_complete_dvc_remotes)"',
}
CONFIG_VARS = {
"bash": "_dvc_compgen_config_vars",
"zsh": "_dvc_compadd_config_vars",
"fish": '-f -a "(__fish_complete_dvc_config_vars)"',
}
REMOTE_CONFIG_VARS = {
"bash": "_dvc_compgen_remote_config_vars",
"zsh": "_dvc_compadd_remote_config_vars",
"fish": '-f -a "(__fish_complete_dvc_remote_config_vars)"',
}
EXPERIMENT = {"bash": "_dvc_compgen_exps", "zsh": "_dvc_compadd_exps"}
REMOTE = {"bash": "_dvc_compgen_remotes", "zsh": "_dvc_compadd_remotes"}
CONFIG_VARS = {"bash": "_dvc_compgen_config_vars", "zsh": "_dvc_compadd_config_vars"}
def get_preamble() -> dict[str, str]:
@@ -103,7 +208,26 @@ def get_preamble() -> dict[str, str]:
_dvc_config_vars=(
{nl.join(config_vars)}
)
"""
indent = "\t\t".expandtabs(4) # 8 spaces
lines = (
"\n".join(
f"{indent}{c} \\"
for c in config_vars[:-1] # all but last
)
+ "\n"
+ f"{indent}{config_vars[-1]}"
) # last line without backslash
config_vars_arr_fish = f"""
function __fish_complete_dvc_config_vars
set -l _dvc_config_vars \\
{lines}
printf %s\\n $_dvc_config_vars
end
"""
for shell, preamble in PREAMBLE.items():
ret[shell] = config_vars_arr + preamble
if shell != "fish":
ret[shell] = config_vars_arr + preamble
else:
ret[shell] = config_vars_arr_fish + preamble
return ret
+1 -1
View File
@@ -8,7 +8,7 @@ from dvc.ui import ui
logger = logger.getChild(__name__)
SUPPORTED_SHELLS = ["bash", "zsh"]
SUPPORTED_SHELLS = ["bash", "zsh", "fish"]
class CmdCompletion(CmdBaseNoRepo):
+12
View File
@@ -40,6 +40,15 @@ class CmdConfig(CmdBaseNoRepo):
self.config = Config.from_cwd(validate=False)
def run(self):
if self.args.available_options:
from dvc.config_schema import contextual_config_vars_for_completion
ui.write(
"\n".join(list(contextual_config_vars_for_completion(self.config))),
force=True,
)
return 0
if self.args.show_origin and (self.args.value or self.args.unset):
logger.error(
"--show-origin can't be used together with any of these "
@@ -212,6 +221,9 @@ def add_parser(subparsers, parent_parser):
help=CONFIG_HELP,
formatter_class=formatter.RawDescriptionHelpFormatter,
)
config_parser.add_argument(
"--available-options", default=False, action="store_true"
)
config_parser.add_argument(
"-u",
"--unset",
+2 -71
View File
@@ -127,80 +127,19 @@ def _filter(graph, targets, full):
return new_graph
def _is_foreach_matrix_stage(node, join_string):
if node.endswith(".dvc"):
return False
return join_string in node
def _collapse_foreach_matrix_get_nodes(graph):
from dvc.parsing import JOIN
new_nodes = set()
nodes_to_remove = set()
for _node in list(graph.nodes):
if not _is_foreach_matrix_stage(_node, JOIN):
continue
nodes_to_remove.add(_node)
new_nodes.add(_node.split(JOIN)[0])
return new_nodes, nodes_to_remove
def _collapse_foreach_matrix_get_edges(graph):
from dvc.parsing import JOIN
new_edges = set()
edges_to_remove = set()
for _e1, _e2 in list(graph.edges):
_replace = False
_new_e1 = _e1
_new_e2 = _e2
if _is_foreach_matrix_stage(_e1, JOIN):
_new_e1 = _e1.split(JOIN)[0]
_replace = True
if _is_foreach_matrix_stage(_e2, JOIN):
_new_e2 = _e2.split(JOIN)[0]
_replace = True
if _replace:
edges_to_remove.add((_e1, _e2))
new_edges.add((_new_e1, _new_e2))
return new_edges, edges_to_remove
def _collapse_foreach_matrix(graph):
new_nodes, nodes_to_remove = _collapse_foreach_matrix_get_nodes(graph)
new_edges, edges_to_remove = _collapse_foreach_matrix_get_edges(graph)
new_graph = graph.copy()
new_graph.remove_edges_from(edges_to_remove)
new_graph.add_nodes_from(new_nodes)
new_graph.add_edges_from(new_edges)
new_graph.remove_nodes_from(nodes_to_remove)
return new_graph
def _build(repo, target=None, full=False, outs=False, collapse_foreach_matrix=False):
def _build(repo, target=None, full=False, outs=False):
targets = _collect_targets(repo, target, outs)
graph = _transform(repo.index, outs)
filtered_graph = _filter(graph, targets, full)
if collapse_foreach_matrix:
filtered_graph = _collapse_foreach_matrix(filtered_graph)
return filtered_graph
return _filter(graph, targets, full)
class CmdDAG(CmdBase):
def run(self):
from dvc.exceptions import InvalidArgumentError
if self.args.outs and self.args.collapse_foreach_matrix:
raise InvalidArgumentError(
"`--outs` and `--collapse-foreach-matrix` are mutually exclusive"
)
graph = _build(
self.repo,
target=self.args.target,
full=self.args.full,
outs=self.args.outs,
collapse_foreach_matrix=self.args.collapse_foreach_matrix,
)
if self.args.dot:
@@ -242,14 +181,6 @@ def add_parser(subparsers, parent_parser):
dest="markdown",
help="Print DAG with mermaid format wrapped in Markdown block.",
)
dag_parser.add_argument(
"--collapse-foreach-matrix",
action="store_true",
default=False,
help=(
"Collapse stages from each foreach/matrix definition into a single node."
),
)
dag_parser.add_argument(
"--full",
action="store_true",
-11
View File
@@ -20,7 +20,6 @@ class CmdExperimentsRun(CmdRepro):
tmp_dir=self.args.tmp_dir,
copy_paths=self.args.copy_paths,
message=self.args.message,
no_hydra=self.args.no_hydra,
**self._common_kwargs,
)
@@ -108,14 +107,4 @@ def _add_run_common(parser):
default=None,
help="Custom commit message to use when committing the experiment.",
)
parser.add_argument(
"--no-hydra",
action="store_true",
default=False,
help=(
"Disables automatically updating `params.yaml` with Hydra configuration. "
" You can still use `--set-param` to update individual params if needed."
" Default is False."
),
)
parser.add_argument("-M", dest="message", help=argparse.SUPPRESS) # obsolete
+1 -1
View File
@@ -29,7 +29,7 @@ def _welcome_message():
"{yellow}------------{nc}\n"
f"- Check out the documentation: {fmt_link('https://dvc.org/doc')}\n"
f"- Get help and share ideas: {fmt_link('https://dvc.org/chat')}\n"
f"- Star us on GitHub: {fmt_link('https://github.com/treeverse/dvc')}"
f"- Star us on GitHub: {fmt_link('https://github.com/iterative/dvc')}"
).format(yellow=colorama.Fore.YELLOW, nc=colorama.Fore.RESET)
ui.write(msg)
+3 -1
View File
@@ -233,7 +233,9 @@ def add_parser(subparsers, parent_parser):
remote_modify_parser.add_argument(
"name", help="Name of the remote"
).complete = completion.REMOTE
remote_modify_parser.add_argument("option", help="Name of the option to modify.")
remote_modify_parser.add_argument(
"option", help="Name of the option to modify."
).complete = completion.REMOTE_CONFIG_VARS
remote_modify_parser.add_argument(
"value", nargs="?", help="(optional) Value of the option."
)
+34 -15
View File
@@ -60,23 +60,27 @@ def Choices(*choices): # noqa: N802
return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
def _get_schema_from_url(url: str) -> str:
parsed = urlparse(url)
# Windows absolute paths should really have scheme == "" (local)
if os.name == "nt" and len(parsed.scheme) == 1 and not parsed.netloc:
return ""
if not parsed.netloc:
return ""
return parsed.scheme
def ByUrl(mapping): # noqa: N802
schemas = walk_values(Schema, mapping)
schemas: dict[str, Schema] = walk_values(Schema, mapping)
def validate(data):
if "url" not in data:
raise Invalid("expected 'url'")
parsed = urlparse(data["url"])
# Windows absolute paths should really have scheme == "" (local)
if os.name == "nt" and len(parsed.scheme) == 1 and not parsed.netloc:
return schemas[""](data)
if not parsed.netloc:
return schemas[""](data)
if parsed.scheme not in schemas:
raise Invalid(f"Unsupported URL type {parsed.scheme}://")
return schemas[parsed.scheme](data)
scheme = _get_schema_from_url(data["url"])
if scheme not in schemas:
raise Invalid(f"Unsupported URL type {scheme}://")
return schemas[scheme](data)
return validate
@@ -151,7 +155,6 @@ WEBDAV_COMMON = {
"password": str,
"ask_password": Bool,
"token": str,
"bearer_token_command": str,
"custom_auth_header": str,
"cert_path": str,
"key_path": str,
@@ -397,9 +400,25 @@ def config_vars_for_completion(d: dict = SCHEMA, path: str = "") -> "Iterator[st
k = k.schema
if not isinstance(k, str):
continue
keypath = path + k
keypath = f"{path}.{k}" if path else k
if isinstance(v, dict):
yield from config_vars_for_completion(v, keypath + ".")
yield from config_vars_for_completion(v, keypath)
else:
yield keypath
def contextual_config_vars_for_completion(config) -> "Iterator[str]":
yield from config_vars_for_completion()
for name, data in config["remote"].items():
if "url" not in data:
continue
scheme = _get_schema_from_url(data["url"])
if scheme not in REMOTE_SCHEMAS:
continue
schema = REMOTE_SCHEMAS[scheme]
if schema:
yield from config_vars_for_completion(schema, f"remote.{name}") # type: ignore[arg-type]
db_schema = SCHEMA["db"][str] # type: ignore[index]
for name in config["db"]:
yield from config_vars_for_completion(db_schema, f"db.{name}") # type: ignore[arg-type]
+1 -1
View File
@@ -41,7 +41,7 @@ def _get(stage, p, info, **kwargs):
return Dependency(stage, p, info, **kwargs)
def loadd_from(stage, d_list) -> list[Dependency]:
def loadd_from(stage, d_list):
ret = []
for d in d_list:
p = d.pop(Output.PARAM_PATH, None)
-1
View File
@@ -110,7 +110,6 @@ class ParamsDependency(Dependency):
except MissingParamsFile:
return {}
assert self.repo
try:
return read_param_file(
self.repo.fs,
-1
View File
@@ -150,7 +150,6 @@ class RepoDependency(Dependency):
# local cache instead of the remote's cache. This avoids re-streaming of data,
# but messes up the call to `_get_remote_config()` downstream, which will need
# to ignore cache parameters.
assert self.repo
config["cache"] = self.repo.config["cache"]
config["cache"]["dir"] = self.repo.cache.local_cache_dir
+2 -2
View File
@@ -508,7 +508,7 @@ class DvcIgnoreFilter:
def check_ignore(self, target: str) -> CheckIgnoreResult:
# NOTE: can only be used in `dvc check-ignore`, see
# https://github.com/treeverse/dvc/issues/5046
# https://github.com/iterative/dvc/issues/5046
full_target = self.fs.abspath(target)
matched_patterns: list[PatternInfo] = []
ignore = False
@@ -525,7 +525,7 @@ class DvcIgnoreFilter:
self, fs: "FileSystem", path: str, ignore_subrepos: bool = True
) -> bool:
# NOTE: can't use self.check_ignore(path).match for now, see
# https://github.com/treeverse/dvc/issues/4555
# https://github.com/iterative/dvc/issues/4555
if fs.protocol != Schemes.LOCAL:
return False
if fs.isfile(path):
+1 -1
View File
@@ -158,7 +158,7 @@ def _get_fs_type(path):
except PermissionError:
pass
# need to follow the symlink: https://github.com/treeverse/dvc/issues/5065
# need to follow the symlink: https://github.com/iterative/dvc/issues/5065
path = pathlib.Path(path).resolve()
for parent in itertools.chain([path], path.parents):
+1 -1
View File
@@ -182,7 +182,7 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
# number to generate the resulting lock file name, which is unique
# enough for our application.
#
# [1] https://github.com/treeverse/dvc/issues/2582
# [1] https://github.com/iterative/dvc/issues/2582
self._hostname = socket.gethostname()
self._lifetime = timedelta(days=365) # Lock for good by default
+2
View File
@@ -1,6 +1,8 @@
"""Manages logging configuration for DVC repo."""
import logging
import logging.config
import logging.handlers
import os
import sys
from collections.abc import Iterator
+9 -26
View File
@@ -42,7 +42,6 @@ from .utils import relpath
from .utils.fs import path_isin
if TYPE_CHECKING:
from dvc.repo import Repo
from dvc_data.hashfile.obj import HashFile
from dvc_data.index import DataIndexKey
@@ -84,7 +83,7 @@ def _get(stage, path, **kwargs):
return Output(stage, path, **kwargs)
def loadd_from(stage, d_list) -> list["Output"]:
def loadd_from(stage, d_list):
ret = []
for d in d_list:
p = d.pop(Output.PARAM_PATH)
@@ -330,7 +329,7 @@ class Output:
self.annot = Annotation(
desc=desc, type=type, labels=labels or [], meta=meta or {}
)
self.repo: Optional[Repo] = stage.repo if not repo and stage else repo
self.repo = stage.repo if not repo and stage else repo
meta_d = merge_file_meta_from_cloud(info or {})
meta = Meta.from_dict(meta_d)
# NOTE: when version_aware is not passed into get_cloud_fs, it will be
@@ -512,7 +511,6 @@ class Output:
from dvc.cachemgr import LEGACY_HASH_NAMES
assert self.is_in_repo
assert self.repo
odb_name = "legacy" if self.hash_name in LEGACY_HASH_NAMES else "repo"
return getattr(self.repo.cache, odb_name)
@@ -520,7 +518,6 @@ class Output:
def local_cache(self):
from dvc.cachemgr import LEGACY_HASH_NAMES
assert self.repo
if self.hash_name in LEGACY_HASH_NAMES:
return self.repo.cache.legacy
return self.repo.cache.local
@@ -588,7 +585,6 @@ class Output:
def index_key(self) -> tuple[str, "DataIndexKey"]:
if self.is_in_repo:
workspace = "repo"
assert self.repo
key = self.repo.fs.relparts(self.fs_path, self.repo.root_dir)
else:
workspace = self.fs.protocol
@@ -644,7 +640,6 @@ class Output:
@property
def dvcignore(self) -> Optional["DvcIgnoreFilter"]:
if self.fs.protocol == "local":
assert self.repo
return self.repo.dvcignore
return None
@@ -666,7 +661,6 @@ class Output:
if not self.use_scm_ignore:
return
assert self.repo
if self.repo.scm.is_tracked(self.fspath):
raise OutputAlreadyTrackedError(self)
@@ -676,7 +670,6 @@ class Output:
if not self.use_scm_ignore:
return
assert self.repo
self.repo.scm_context.ignore_remove(self.fspath)
def save(self) -> None:
@@ -783,7 +776,6 @@ class Output:
callback=cb,
)
if relink:
assert self.repo
rel = self.fs.relpath(filter_info or self.fs_path)
with CheckoutCallback(desc=f"Checking out {rel}", unit="files") as cb:
self._checkout(
@@ -965,7 +957,6 @@ class Output:
added = not self.exists
try:
assert self.repo
modified = self._checkout(
filter_info or self.fs_path,
self.fs,
@@ -996,24 +987,18 @@ class Output:
if ignore_remove:
self.ignore_remove()
def move(self, out: "Output") -> None:
src_exists = self.exists
if src_exists:
self.fs.move(self.fs_path, out.fs_path)
else:
logger.warning("%r missing", self.fspath)
def move(self, out):
if self.protocol == "local" and self.use_scm_ignore:
assert self.repo
self.repo.scm_context.ignore_remove(self.fspath)
self.fs.move(self.fs_path, out.fs_path)
self.def_path = out.def_path
self.fs_path = out.fs_path
try:
self.save()
self.commit()
except self.DoesNotExistError:
self.ignore()
self.save()
self.commit()
if self.protocol == "local" and self.use_scm_ignore:
self.repo.scm_context.ignore(self.fspath)
def transfer(
self, source, odb=None, jobs=None, update=False, no_progress_bar=False
@@ -1094,7 +1079,6 @@ class Output:
if self.remote:
kwargs["remote"] = self.remote
with suppress(Exception):
assert self.repo
self.repo.cloud.pull([obj.hash_info], **kwargs)
if self.obj:
@@ -1188,7 +1172,6 @@ class Output:
return {}
if self.remote:
assert self.repo
remote_odb = self.repo.cloud.get_remote_odb(
name=self.remote, hash_name=self.hash_name
)
+1
View File
@@ -2,6 +2,7 @@ import os
from collections import defaultdict
from typing import TYPE_CHECKING, NamedTuple, Optional
import dpath
import dpath.options
from funcy import get_in, last
+1 -4
View File
@@ -20,7 +20,6 @@ if TYPE_CHECKING:
from dvc.fs.data import DataFileSystem
from dvc.fs.dvc import DVCFileSystem
from dvc.lock import LockBase
from dvc.output import Output
from dvc.scm import Git, NoSCM
from dvc.stage import Stage
from dvc.types import DictStrAny
@@ -545,9 +544,7 @@ class Repo:
return used
def find_outs_by_path(
self, path, outs=None, recursive=False, strict=True
) -> list["Output"]:
def find_outs_by_path(self, path, outs=None, recursive=False, strict=True):
# using `outs_graph` to ensure graph checks are run
outs = outs or self.index.outs_graph
+1 -1
View File
@@ -40,7 +40,7 @@ def check_name_format(name: str) -> None:
def name_is_compatible(name: str) -> bool:
"""
Only needed by DVCLive per treeverse/dvclive#715
Only needed by DVCLive per iterative/dvclive#715
Will be removed in future release.
"""
from gto.constants import assert_name_is_valid
+5 -27
View File
@@ -1,6 +1,6 @@
import os
import posixpath
from collections import defaultdict, deque
from collections import defaultdict
from collections.abc import Iterable, Iterator, Mapping
from typing import TYPE_CHECKING, Optional, TypedDict, Union
@@ -385,26 +385,6 @@ def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> list[str]
return [repo.fs.relpath(file, start) for file in files]
def iter_index(
index: Union["BaseDataIndex", "DataIndexView"], shallow: bool = False
) -> Iterator[tuple["DataIndexKey", Optional["DataIndexEntry"]]]:
if not shallow:
yield from index.iteritems(shallow=shallow)
return
# only iterate until we find entries with hash_info in shallow mode
todo: deque[tuple[DataIndexKey, dict]] = deque([((), index.info(()))])
while todo:
key, info = todo.popleft()
entry = info.get("entry")
if info.get("type") == "directory" and not (entry and entry.hash_info):
try:
todo.extend(index.ls(key, detail=True))
except (KeyError, DataIndexDirError):
pass
yield key, entry
def _get_entries_not_in_remote(
repo: "Repo",
filter_keys: Optional[Iterable["DataIndexKey"]] = None,
@@ -438,7 +418,7 @@ def _get_entries_not_in_remote(
n = 0
with TqdmCallback(size=n, desc="Checking remote", unit="entry") as cb:
for key, entry in iter_index(view, shallow=not granular):
for key, entry in view.iteritems(shallow=not granular):
if not (entry and entry.hash_info):
continue
@@ -456,17 +436,15 @@ def _get_entries_not_in_remote(
n += 1
cb.set_size(n)
results = storage_map.bulk_remote_exists(
list(entries.values()), refresh=remote_refresh, callback=cb
)
for key, entry in entries.items():
k = (*key, "") if entry.meta and entry.meta.isdir else key
try:
if not results.get(entry, False):
if not storage_map.remote_exists(entry, refresh=remote_refresh):
missing_entries.append(os.path.sep.join(k))
except StorageKeyError:
pass
finally:
cb.relative_update()
data_index.onerror = orig_data_index_onerror
return missing_entries
+3 -15
View File
@@ -9,7 +9,6 @@ from enum import IntEnum
from itertools import chain
from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, Union
import funcy
from funcy import nullcontext
from scmrepo.exceptions import SCMError
@@ -816,22 +815,11 @@ class BaseExecutor(ABC):
@contextmanager
def set_temp_refs(self, scm: "Git", temp_dict: dict[str, str]):
# Retry ref set, get, and remove operations to handle transient issues during
# concurrent Git access.
# Dulwich deletes parent directories of refs if they happen to be empty after
# removing a ref, which can interfere with `set_ref` in other processes.
# `remove_ref` may also fail with a `FileLocked` error when refs are packed,
# since multiple processes might attempt to write to the same file.
retry = funcy.retry(10, errors=Exception, timeout=0.1)
set_ref = retry(scm.set_ref)
get_ref = retry(scm.get_ref)
remove_ref = retry(scm.remove_ref)
try:
for ref, rev in temp_dict.items():
set_ref(ref, rev)
scm.set_ref(ref, rev)
yield
finally:
for ref in temp_dict:
if get_ref(ref):
remove_ref(ref)
if scm.get_ref(ref):
scm.remove_ref(ref)
+3 -6
View File
@@ -287,7 +287,6 @@ class BaseStashQueue(ABC):
baseline_rev: Optional[str] = None,
branch: Optional[str] = None,
name: Optional[str] = None,
no_hydra: bool = False,
**kwargs,
) -> QueueEntry:
"""Stash changes from the workspace as an experiment.
@@ -303,7 +302,6 @@ class BaseStashQueue(ABC):
name: Optional experiment name. If specified this will be used as
the human-readable name in the experiment branch ref. Has no
effect of branch is specified.
no_hydra: Disable Hydra from automatically overwriting all params.
.. _Hydra Override:
https://hydra.cc/docs/next/advanced/override_grammar/basic/
@@ -320,7 +318,7 @@ class BaseStashQueue(ABC):
# update experiment params from command line
if params:
self._update_params(params, no_hydra=no_hydra)
self._update_params(params)
# DVC commit data deps to preserve state across workspace
# & tempdir runs
@@ -443,13 +441,12 @@ class BaseStashQueue(ABC):
f"from '{config_path}': {param_list}"
)
def _update_params(self, params: dict[str, list[str]], no_hydra: bool = False):
def _update_params(self, params: dict[str, list[str]]):
"""Update param files with the provided `Hydra Override`_ patterns.
Args:
params: Dict mapping paths to `Hydra Override`_ patterns,
provided via `exp run --set-param`.
no_hydra: Disable Hydra from automatically overwriting all params.
.. _Hydra Override:
https://hydra.cc/docs/advanced/override_grammar/basic/
@@ -459,7 +456,7 @@ class BaseStashQueue(ABC):
logger.debug("Using experiment params '%s'", params)
hydra_config = self.repo.config.get("hydra", {})
hydra_enabled = hydra_config.get("enabled", False) and not no_hydra
hydra_enabled = hydra_config.get("enabled", False)
hydra_output_file = ParamsDependency.DEFAULT_PARAMS_FILE
for path, overrides in params.items():
if hydra_enabled and path == hydra_output_file:
+1 -1
View File
@@ -70,7 +70,7 @@ class LocalCeleryQueue(BaseStashQueue):
def celery(self) -> "FSApp":
from kombu.transport.filesystem import Channel
# related to https://github.com/treeverse/dvc-task/issues/61
# related to https://github.com/iterative/dvc-task/issues/61
Channel.QoS.restore_at_shutdown = False
from dvc_task.app import FSApp
+1 -4
View File
@@ -22,7 +22,6 @@ def run( # noqa: C901, PLR0912
queue: bool = False,
copy_paths: Optional[Iterable[str]] = None,
message: Optional[str] = None,
no_hydra: bool = False,
**kwargs,
) -> dict[str, str]:
"""Reproduce the specified targets as an experiment.
@@ -68,7 +67,7 @@ def run( # noqa: C901, PLR0912
else:
path_overrides = {}
hydra_enabled = repo.config.get("hydra", {}).get("enabled", False) and not no_hydra
hydra_enabled = repo.config.get("hydra", {}).get("enabled", False)
hydra_output_file = ParamsDependency.DEFAULT_PARAMS_FILE
if hydra_enabled and hydra_output_file not in path_overrides:
# Force `_update_params` even if `--set-param` was not used
@@ -81,7 +80,6 @@ def run( # noqa: C901, PLR0912
tmp_dir=tmp_dir,
copy_paths=copy_paths,
message=message,
no_hydra=no_hydra,
**kwargs,
)
@@ -102,7 +100,6 @@ def run( # noqa: C901, PLR0912
params=sweep_overrides,
copy_paths=copy_paths,
message=message,
no_hydra=no_hydra,
**kwargs,
)
if sweep_overrides:
+1 -3
View File
@@ -128,11 +128,9 @@ def gc( # noqa: C901, PLR0912, PLR0913
used_obj_ids = set()
used_obj_ids.update(*odb_to_obj_ids.values())
seen_odbs = set()
for scheme, odb in self.cache.by_scheme():
if not odb or odb in seen_odbs:
if not odb:
continue
seen_odbs.add(odb)
num_removed = ogc(odb, used_obj_ids, jobs=jobs, dry=dry)
if num_removed:
logger.info("Removed %d objects from %s cache.", num_removed, scheme)
+1 -1
View File
@@ -16,7 +16,7 @@ def pre_commit_install(scm: "Git") -> None:
config_path = os.path.join(scm.root_dir, ".pre-commit-config.yaml")
with modify_yaml(config_path) as config:
entry = {
"repo": "https://github.com/treeverse/dvc",
"repo": "https://github.com/iterative/dvc",
"rev": ".".join(map(str, version_tuple[:3])),
"hooks": [
{
+17 -41
View File
@@ -1,13 +1,10 @@
import os
from typing import TYPE_CHECKING
from dvc.exceptions import MoveNotDataSourceError
from dvc.repo.scm_context import scm_context
from . import locked
if TYPE_CHECKING:
from . import Repo
def _expand_target_path(from_path, to_path):
if os.path.isdir(to_path):
@@ -17,7 +14,7 @@ def _expand_target_path(from_path, to_path):
@locked
@scm_context
def move(self: "Repo", from_path, to_path):
def move(self, from_path, to_path):
"""
Renames an output file and modifies the stage associated
to reflect the change on the pipeline.
@@ -35,12 +32,9 @@ def move(self: "Repo", from_path, to_path):
It only works with outputs generated by `add` or `import`,
also known as data sources.
"""
from dvc import dependency, output
from dvc import output
from dvc.dvcfile import DVC_FILE_SUFFIX
from dvc.exceptions import MoveNotDataSourceError
from dvc.stage import Stage
from dvc.stage.exceptions import StageFileAlreadyExistsError
from dvc_objects.fs.local import LocalFileSystem
from_out = output.loads_from(Stage(self), [from_path])[0]
assert from_out.protocol == "local"
@@ -51,7 +45,6 @@ def move(self: "Repo", from_path, to_path):
assert len(outs) == 1
out = outs[0]
stage = out.stage
deps = stage.deps
if not stage.is_data_source:
raise MoveNotDataSourceError(stage.addressing)
@@ -65,38 +58,21 @@ def move(self: "Repo", from_path, to_path):
)
new_wdir = os.path.abspath(os.path.join(os.curdir, os.path.dirname(to_path)))
to_path = os.path.relpath(to_path, new_wdir)
try:
new_stage = self.stage.create(
single_stage=True,
fname=new_fname,
wdir=new_wdir,
outs=[to_path],
meta=stage.meta,
frozen=stage.frozen,
always_changed=stage.always_changed,
desc=stage.desc,
)
except StageFileAlreadyExistsError:
# reraise to remove `--force` hint
raise StageFileAlreadyExistsError(f"{new_fname!r} already exists") from None
new_stage = self.stage.create(
single_stage=True,
fname=new_fname,
wdir=new_wdir,
outs=[to_path],
meta=stage.meta,
)
os.unlink(stage.path)
stage = new_stage
else:
new_stage = stage
to_path = os.path.relpath(to_path, stage.wdir)
def with_dep_path_adjusted(dep: dependency.Dependency):
d = dep.dumpd()
if isinstance(dep.fs, LocalFileSystem) and not os.path.isabs(dep.def_path):
return d | {"path": os.path.relpath(dep.fspath, new_stage.wdir)}
return d
to_out = output.loads_from(stage, [to_path], out.use_cache, out.metric)[0]
new_stage.outs = output.loadd_from(new_stage, [out.dumpd() | {"path": to_path}])
new_stage.deps = dependency.loadd_from(
new_stage, [with_dep_path_adjusted(dep) for dep in deps]
)
out.move(new_stage.outs[0])
new_stage.md5 = new_stage.compute_md5()
new_stage.dump()
if stage != new_stage:
stage.dvcfile.remove()
self.scm_context.track_file(stage.dvcfile.relpath)
return stage, new_stage
out.move(to_out)
stage.save()
stage.dump()
+1 -1
View File
@@ -46,7 +46,7 @@ class SCMContext:
try:
add_no_submodules(self.scm, paths)
except UnsupportedIndexFormat:
link = "https://github.com/treeverse/dvc/issues/610"
link = "https://github.com/iterative/dvc/issues/610"
add_cmd = self._make_git_add_cmd([relpath(path) for path in paths])
logger.info("")
msg = (
+3 -3
View File
@@ -32,7 +32,7 @@ from .utils import (
)
if TYPE_CHECKING:
from dvc.dependency import Dependency, ParamsDependency
from dvc.dependency import ParamsDependency
from dvc.dvcfile import ProjectFile, SingleStageFile
from dvc.output import Output
from dvc.repo import Repo
@@ -153,8 +153,8 @@ class Stage(params.StageParams):
self._path = path
self.cmd = cmd
self.wdir = wdir
self.outs: list[Output] = outs
self.deps: list[Dependency] = deps
self.outs = outs
self.deps = deps
self.md5 = md5
self.frozen = locked or frozen
self.always_changed = always_changed
+2 -2
View File
@@ -51,7 +51,7 @@ def _warn_if_fish(executable):
"Be aware that it might cause problems by overwriting "
"your current environment variables with values defined "
"in 'config.fish', which might affect your command. See "
"https://github.com/treeverse/dvc/issues/1307. "
"https://github.com/iterative/dvc/issues/1307. "
)
@@ -99,7 +99,7 @@ def prepare_kwargs(stage, run_env=None):
#
# [1] https://github.com/python/cpython/blob/3.7/Lib/subprocess.py
# #L1426
# [2] https://github.com/treeverse/dvc/issues/2506
# [2] https://github.com/iterative/dvc/issues/2506
# #issuecomment-535396799
kwargs["shell"] = os.name == "nt"
return kwargs
+1 -1
View File
@@ -178,7 +178,7 @@ def to_single_stage_lockfile(stage: "Stage", **kwargs) -> dict:
res = OrderedDict([("cmd", stage.cmd)])
params, deps = split_params_deps(stage)
deps, outs = (
[_dumpd(item) for item in sorted(items, key=attrgetter("def_path"))] # type: ignore[call-overload]
[_dumpd(item) for item in sorted(items, key=attrgetter("def_path"))]
for items in [deps, stage.outs]
)
params = _serialize_params_values(params)
+1 -1
View File
@@ -4,7 +4,7 @@ dvc.testing.benchmarks
======================
Benchmark test definitions are now part of ``dvc.testing``.
For CLI usage and `bench.dvc.org <https://bench.dvc.org>`_ details see `dvc-bench <https://github.com/treeverse/dvc-bench>`_.
For CLI usage and `bench.dvc.org <https://bench.dvc.org>`_ details see `dvc-bench <https://github.com/iterative/dvc-bench>`_.
``dvc.testing.benchmarks`` structure:
+1 -1
View File
@@ -2,7 +2,6 @@ import pytest
from dvc import api
from dvc.api import DVCFileSystem
from dvc.testing import matchers as M
from dvc.utils.fs import remove
@@ -43,6 +42,7 @@ class TestAPI:
)
def test_filesystem(
self,
M,
tmp_dir,
make_tmp_dir,
scm,
@@ -6,8 +6,8 @@ import pytest
"!=3.53.*,!=3.54.0",
reason="Takes 10 mins to run. Regression in 3.53.0, fixed in 3.54.1",
)
# Introduced in https://github.com/treeverse/dvc/pull/10388.
# Fixed in https://github.com/treeverse/dvc/pull/10531.
# Introduced in https://github.com/iterative/dvc/pull/10388.
# Fixed in https://github.com/iterative/dvc/pull/10531.
def test_import(bench_dvc, tmp_dir, scm, dvc, make_dataset, remote):
dataset = make_dataset(
cache=False, files=False, dvcfile=True, commit=True, remote=True
+2 -2
View File
@@ -79,7 +79,7 @@ def dvc_bench_repo(tmp_path_factory, bench_config):
if url is None:
pytest.skip(
"--dvc-bench-repo is not set, "
"clone https://github.com/treeverse/dvc-bench repository and set its path"
"clone https://github.com/iterative/dvc-bench repository and set its path"
)
if os.path.isdir(url):
@@ -158,7 +158,7 @@ def make_bench(request):
# pytest >= 8.4.0
wrapped_func = fixture_function._get_wrapped_function()
except AttributeError:
wrapped_func = fixture_function.__pytest_wrapped__.obj # type: ignore[attr-defined] # ty: ignore[unresolved-attribute]
wrapped_func = fixture_function.__pytest_wrapped__.obj # ty: ignore[unresolved-attribute]
assert inspect.isgeneratorfunction(wrapped_func)
generator = wrapped_func(request)
-104
View File
@@ -1,104 +0,0 @@
from collections.abc import Iterable, Mapping
from typing import TYPE_CHECKING, Any, Optional, Union
if TYPE_CHECKING:
import builtins
class dict: # noqa: A001, N801, PLW1641
"""Special class to eq by matching only presented dict keys.
Implementation notes:
- can't inherit from dict because that makes D() == M.dict() to not call
our __eq__, if D is a subclass of a dict
- should not call itself dict or use dict in repr because it creates
confusing error messages (shadowing python builtins is bad anyway)
"""
def __init__(self, d: Optional[Mapping[Any, Any]] = None, **keys: Any) -> None:
self.d: builtins.dict[Any, Any] = {}
if d:
self.d.update(d)
self.d.update(keys)
def __len__(self) -> int:
return len(self.d)
def __repr__(self) -> str:
inner = ", ".join(f"{k}={v!r}" for k, v in self.d.items())
return f"{self.__class__.__name__}({inner})"
def __eq__(self, other: object) -> bool:
assert isinstance(other, Mapping)
return all(other.get(name) == v for name, v in self.d.items())
class unordered: # noqa: N801, PLW1641
"""Compare list contents, but do not care about ordering.
(E.g. sort lists first, then compare.)
If you care about ordering, then just compare lists directly."""
def __init__(self, *items: Any) -> None:
self.items = items
def __repr__(self) -> str:
inner = ", ".join(map(repr, self.items))
return f"{self.__class__.__name__}({inner})"
def __eq__(self, other: object) -> bool:
assert isinstance(other, Iterable)
return sorted(self.items) == sorted(other)
class attrs: # noqa: N801, PLW1641
def __init__(self, **attribs: Any) -> None:
self.attribs = attribs
def __repr__(self) -> str:
inner = ", ".join(f"{k}={v!r}" for k, v in self.attribs.items())
return f"{self.__class__.__name__}({inner})"
def __eq__(self, other: object) -> bool:
# Unforturnately this doesn't work with classes with slots
# self.__class__ = other.__class__
return all(getattr(other, name) == v for name, v in self.attribs.items())
class instance_of: # noqa: N801, PLW1641
def __init__(self, expected_type: Union[Any, tuple[Any, ...]]) -> None:
self.expected_type = expected_type
def __repr__(self) -> str:
if isinstance(self.expected_type, tuple):
inner = f"({', '.join(t.__name__ for t in self.expected_type)})"
else:
inner = self.expected_type.__name__
return f"{self.__class__.__name__}({inner})"
def __eq__(self, other: object) -> bool:
return isinstance(other, self.expected_type)
class any_of: # noqa: N801, PLW1641
def __init__(self, *items: Any) -> None:
self.items = sorted(items)
def __repr__(self) -> str:
inner = ", ".join(map(repr, self.items))
return f"any_of({inner})"
def __eq__(self, other: object) -> bool:
return other in self.items
__all__ = [
"any_of",
"attrs",
"dict",
"instance_of",
"unordered",
]
+3 -3
View File
@@ -44,7 +44,7 @@ class TestRemote:
_check_status(status_dir, new=dir_hashes)
# Move cache and check status
# See issue https://github.com/treeverse/dvc/issues/4383 for details
# See issue https://github.com/iterative/dvc/issues/4383 for details
backup_dir = dvc.cache.local.path + ".backup"
shutil.move(dvc.cache.local.path, backup_dir)
status = dvc.cloud.status(foo_hashes)
@@ -116,7 +116,7 @@ class TestRemote:
@pytest.mark.xfail(raises=NotImplementedError, strict=False)
def test_pull_00_prefix(self, tmp_dir, dvc, remote, monkeypatch):
# Related: https://github.com/treeverse/dvc/issues/6089
# Related: https://github.com/iterative/dvc/issues/6089
fs_type = type(dvc.cloud.get_remote_odb("upstream").fs)
monkeypatch.setattr(fs_type, "_ALWAYS_TRAVERSE", True, raising=False)
@@ -146,7 +146,7 @@ class TestRemote:
@pytest.mark.xfail(raises=NotImplementedError, strict=False)
def test_pull_no_00_prefix(self, tmp_dir, dvc, remote, monkeypatch):
# Related: https://github.com/treeverse/dvc/issues/6244
# Related: https://github.com/iterative/dvc/issues/6244
fs_type = type(dvc.cloud.get_remote_odb("upstream").fs)
monkeypatch.setattr(fs_type, "_ALWAYS_TRAVERSE", True, raising=False)
+1 -1
View File
@@ -163,7 +163,7 @@ class Updater:
if pkg not in instructions:
return (
"Find the latest release at "
"[blue]https://github.com/treeverse/dvc/releases/latest[/]."
"[blue]https://github.com/iterative/dvc/releases/latest[/]."
)
instruction = instructions[pkg]
+15 -12
View File
@@ -18,7 +18,7 @@ keywords = [
]
license = "Apache-2.0"
license-files = ["LICENSE"]
maintainers = [{ name = "Treeverse", email = "[email protected]" }]
maintainers = [{ name = "Iterative", email = "[email protected]" }]
authors = [{ name = "Dmitry Petrov", email = "[email protected]" }]
requires-python = ">=3.9"
classifiers = [
@@ -29,7 +29,6 @@ classifiers = [
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: 3.14",
]
dynamic = ["version"]
@@ -41,14 +40,14 @@ dependencies = [
"distro>=1.3",
"dpath<3,>=2.1.0",
"dulwich",
"dvc-data>=3.18.0,<3.19.0",
"dvc-data>=3.16.2,<3.17",
"dvc-http>=2.29.0",
"dvc-objects",
"dvc-render>=1.0.1,<2",
"dvc-studio-client>=0.21,<1",
"dvc-task>=0.3.0,<1",
"flatten-dict<1,>=0.4.1",
"flufl.lock>=8.1.0,<10",
"flatten_dict<1,>=0.4.1",
"flufl.lock>=8.1.0,<9",
"fsspec>=2024.2.0",
"funcy>=1.14",
"grandalf<1,>=0.7",
@@ -59,7 +58,7 @@ dependencies = [
"networkx>=2.5",
"omegaconf",
"packaging>=19",
"pathspec>=0.10.3,<1",
"pathspec>=0.10.3",
"platformdirs<5,>=3.1.1",
"psutil>=5.8",
"pydot>=1.2.4",
@@ -70,7 +69,8 @@ dependencies = [
"ruamel.yaml>=0.17.11",
"scmrepo>=3.5.2,<4",
"shortuuid>=0.5",
"shtab<2,>=1.3.4",
# "shtab<2,>=1.3.4",
"shtab @ git+https://github.com/skshetry/shtab.git@fish-shell-v2",
"tabulate>=0.8.7",
"tomlkit>=0.11.1",
"tqdm<5,>=4.63.1",
@@ -86,7 +86,7 @@ gdrive = ["dvc-gdrive>=3,<4"]
gs = ["dvc-gs>=3.0.2,<4"]
hdfs = ["dvc-hdfs>=3,<4"]
lint = [
"mypy==1.19.1",
"mypy==1.18.2",
"pandas-stubs",
"types-colorama",
"types-psutil",
@@ -103,6 +103,7 @@ ssh = ["dvc-ssh>=4,<5"]
ssh_gssapi = ["dvc-ssh[gssapi]>=4,<5"]
testing = [
"pytest-benchmark[histogram]>=5,<6",
"pytest-test-utils",
"uv",
]
tests = [
@@ -110,11 +111,12 @@ tests = [
"beautifulsoup4>=4.4",
"dvc-ssh",
"filelock",
"pytest>=7,<10",
"pytest>=7,<9",
"pytest-rerunfailures<17.0",
"pytest-cov>=4.1.0",
"pytest-docker>=1,<4",
"pytest-mock",
"pytest-test-utils",
"pytest-timeout>=2",
"pytest-xdist>=3.2",
'pywin32>=225; sys_platform == "win32"', # optional test dependency
@@ -122,14 +124,14 @@ tests = [
"sqlalchemy>=1,<3", # optional dependency for `import-db`
"pandas>=1", # optional dependency for `import-db`
]
webdav = ["dvc-webdav>=3.0.1,<4"]
webdav = ["dvc-webdav>=3,<4"]
webhdfs = ["dvc-webhdfs>=3.1,<4"]
webhdfs_kerberos = ["dvc-webhdfs[kerberos]>=3.1,<4"]
[project.urls]
Documentation = "https://dvc.org/doc"
Issues = "https://github.com/treeverse/dvc/issues"
Source = "https://github.com/treeverse/dvc"
Issues = "https://github.com/iterative/dvc/issues"
Source = "https://github.com/iterative/dvc"
[project.scripts]
dvc = "dvc.cli:main"
@@ -247,6 +249,7 @@ module = [
"shortuuid",
"shtab",
"upath",
"virtualenv",
"viztracer",
"voluptuous",
"yappi",
+17
View File
@@ -0,0 +1,17 @@
import os
# Increasing fd ulimit for tests
if os.name == "nt":
try:
import win32file
except ImportError:
pass
else:
win32file._setmaxstdio(4096)
else:
import resource
resource.setrlimit(resource.RLIMIT_NOFILE, (4096, 4096))
nproc_soft, nproc_hard = resource.getrlimit(resource.RLIMIT_NPROC)
resource.setrlimit(resource.RLIMIT_NPROC, (nproc_hard, nproc_hard))
+6
View File
@@ -0,0 +1,6 @@
if __name__ == "__main__":
import sys
import pytest
sys.exit(pytest.main(["-v", "-n=logical", "--dist=worksteal", *sys.argv[1:]]))
-1
View File
@@ -194,7 +194,6 @@ def isolate(tmp_path_factory):
monkeypatch.setenv(env_var, os.fspath(path))
else:
monkeypatch.setenv("HOME", str(home_dir))
monkeypatch.setenv("XDG_CONFIG_HOME", str(home_dir / ".config"))
monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1")
contents = b"""
+1 -1
View File
@@ -91,7 +91,7 @@ def test_apply_untracked(tmp_dir, scm, dvc, exp_stage):
def test_apply_unchanged_head(tmp_dir, scm, dvc, exp_stage):
# see https://github.com/treeverse/dvc/issues/8764
# see https://github.com/iterative/dvc/issues/8764
tmp_dir.gen("params.yaml", "foo: 2")
scm.add(["dvc.yaml", "dvc.lock", "params.yaml", "metrics.yaml"])
scm.commit("commit foo=2")
+1 -1
View File
@@ -26,7 +26,7 @@ def test_celery_logs(tmp_dir, scm, dvc, failed_exp_stage, follow, capsys, test_q
def test_queue_doesnt_remove_untracked_params_file(tmp_dir, dvc, scm):
"""Regression test for https://github.com/treeverse/dvc/issues/7842"""
"""Regression test for https://github.com/iterative/dvc/issues/7842"""
tmp_dir.gen("params.yaml", "foo: 1")
stage = dvc.run(cmd="echo ${foo}", params=["foo"], name="echo-foo")
scm.add(["dvc.yaml", "dvc.lock", ".gitignore"])
+1 -1
View File
@@ -105,7 +105,7 @@ def test_exp_save_include_untracked(tmp_dir, dvc, scm):
def test_exp_save_include_untracked_warning(tmp_dir, dvc, scm, mocker):
"""Regression test for https://github.com/treeverse/dvc/issues/9061"""
"""Regression test for https://github.com/iterative/dvc/issues/9061"""
setup_stage(tmp_dir, dvc, scm)
new_dir = tmp_dir / "new_dir"
+4 -79
View File
@@ -26,9 +26,8 @@ def test_modify_params(params_repo, dvc, changes, expected):
("conf", "bar"),
],
)
@pytest.mark.parametrize("no_hydra", [True, False])
def test_hydra_compose_and_dump(
tmp_dir, params_repo, dvc, hydra_enabled, config_dir, config_name, no_hydra
tmp_dir, params_repo, dvc, hydra_enabled, config_dir, config_name
):
hydra_setup(
tmp_dir,
@@ -51,14 +50,14 @@ def test_hydra_compose_and_dump(
if config_name is not None:
conf["hydra"]["config_name"] = config_name
dvc.experiments.run(no_hydra=no_hydra)
dvc.experiments.run()
if hydra_enabled and not no_hydra:
if hydra_enabled:
assert (tmp_dir / "params.yaml").parse() == {
"db": {"driver": "mysql", "user": "omry", "pass": "secret"},
}
dvc.experiments.run(params=["db=postgresql"], no_hydra=no_hydra)
dvc.experiments.run(params=["db=postgresql"])
assert (tmp_dir / "params.yaml").parse() == {
"db": {
"driver": "postgresql",
@@ -118,7 +117,6 @@ def test_hydra_sweep(
targets=None,
copy_paths=None,
message=None,
no_hydra=False,
)
@@ -139,76 +137,3 @@ def test_hydra_sweep_prefix_name(tmp_dir, params_repo, dvc):
exp_names = [entry.name for entry in dvc.experiments.celery_queue.iter_queued()]
for name, expected in zip(exp_names, expected_names):
assert name == expected
def test_mixing_no_hydra_and_params_flags(tmp_dir, params_repo, dvc):
# Passing no_hydra should not prevent user from
# using --set-param on unmodified params.yaml
hydra_setup(
tmp_dir,
config_dir="conf",
config_name="config",
)
with dvc.config.edit() as conf:
conf["hydra"]["enabled"] = True
conf["hydra"]["config_dir"] = "conf"
conf["hydra"]["config_name"] = "config"
dvc.experiments.run(no_hydra=True, params=["goo.bag=10.0"])
assert (tmp_dir / "params.yaml").parse() == {
"foo": [{"bar": 1}, {"baz": 2}],
"goo": {"bag": 10.0},
"lorem": False,
}
@pytest.mark.parametrize(
"hydra_enabled,overrides,expected",
[
(
True,
["db=mysql,postgresql"],
[
{"params.yaml": ["db=mysql"]},
{"params.yaml": ["db=postgresql"]},
],
),
(
False,
["foo=bar,baz"],
[{"params.yaml": ["foo=bar"]}, {"params.yaml": ["foo=baz"]}],
),
(
False,
[],
[{}],
),
],
)
@pytest.mark.parametrize("no_hydra", [True, False])
def test_mixing_no_hydra_and_sweeps(
tmp_dir, params_repo, dvc, mocker, hydra_enabled, overrides, expected, no_hydra
):
# Passing no_hydra should not prevent user from
# queuing sweeps with --set-param and --queue
patched = mocker.patch.object(dvc.experiments, "queue_one")
if hydra_enabled:
hydra_setup(tmp_dir, config_dir="conf", config_name="config")
with dvc.config.edit() as conf:
conf["hydra"]["enabled"] = True
dvc.experiments.run(params=overrides, queue=True, no_hydra=no_hydra)
assert patched.call_count == len(expected)
for e in expected:
patched.assert_any_call(
mocker.ANY,
params=e,
targets=None,
copy_paths=None,
message=None,
no_hydra=no_hydra,
)
+2 -3
View File
@@ -4,7 +4,6 @@ from os.path import join
import pytest
from dvc.cli import main
from dvc.testing import matchers as M
from dvc.utils import relpath
from dvc.utils.serialize import JSONFileCorruptedError
@@ -96,7 +95,7 @@ def test_metrics_diff_json_unchanged(tmp_dir, scm, dvc, run_copy_metrics):
assert dvc.metrics.diff(a_rev="HEAD~2") == {}
def test_metrics_diff_broken_json(tmp_dir, scm, dvc, run_copy_metrics):
def test_metrics_diff_broken_json(M, tmp_dir, scm, dvc, run_copy_metrics):
metrics = {"a": {"b": {"c": 1, "d": 1, "e": "3"}}}
(tmp_dir / "m_temp.json").dump(metrics)
run_copy_metrics(
@@ -144,7 +143,7 @@ def test_metrics_diff_new_metric(tmp_dir, scm, dvc, run_copy_metrics):
}
def test_metrics_diff_deleted_metric(tmp_dir, scm, dvc, run_copy_metrics):
def test_metrics_diff_deleted_metric(M, tmp_dir, scm, dvc, run_copy_metrics):
metrics = {"a": {"b": {"c": 1, "d": 1, "e": "3"}}}
(tmp_dir / "m_temp.json").dump(metrics)
run_copy_metrics(
+2 -3
View File
@@ -11,7 +11,6 @@ from dvc.dvcfile import PROJECT_FILE
from dvc.exceptions import OverlappingOutputPathsError
from dvc.repo import Repo
from dvc.repo.metrics.show import FileResult, Result
from dvc.testing import matchers as M
from dvc.utils.fs import remove
from dvc.utils.serialize import JSONFileCorruptedError
from dvc_data.index import DataIndexDirError
@@ -247,7 +246,7 @@ def test_show_malformed_metric(tmp_dir, scm, dvc, caplog):
)
def test_metrics_show_no_target(tmp_dir, dvc, capsys):
def test_metrics_show_no_target(M, tmp_dir, dvc, capsys):
assert dvc.metrics.show(targets=["metrics.json"]) == {
"": {"data": {"metrics.json": {"error": M.instance_of(FileNotFoundError)}}}
}
@@ -357,7 +356,7 @@ def test_top_level_parametrized(tmp_dir, dvc):
}
def test_metric_in_a_tracked_directory_with_missing_dir_file(tmp_dir, dvc):
def test_metric_in_a_tracked_directory_with_missing_dir_file(M, tmp_dir, dvc):
tmp_dir.dvc_gen({"dir": {"file": "2"}})
(tmp_dir / "dvc.yaml").dump({"metrics": [join("dir", "file")]})
shutil.rmtree(tmp_dir / "dir") # remove from workspace
+1 -2
View File
@@ -2,7 +2,6 @@ from os.path import join
import pytest
from dvc.testing import matchers as M
from dvc.utils import relpath
@@ -55,7 +54,7 @@ def test_diff_new(tmp_dir, scm, dvc):
}
def test_diff_deleted(tmp_dir, scm, dvc):
def test_diff_deleted(M, tmp_dir, scm, dvc):
tmp_dir.gen("params.yaml", "foo: bar")
dvc.run(cmd="echo params.yaml", params=["foo"], name="echo-params")
scm.add(["params.yaml", "Dvcfile"])
+1 -2
View File
@@ -6,7 +6,6 @@ import pytest
from dvc.dvcfile import PROJECT_FILE
from dvc.repo import Repo
from dvc.repo.metrics.show import FileResult, Result
from dvc.testing import matchers as M
from dvc_data.index import DataIndexDirError
@@ -198,7 +197,7 @@ def test_top_level_parametrized(tmp_dir, dvc):
}
def test_param_in_a_tracked_directory_with_missing_dir_file(tmp_dir, dvc):
def test_param_in_a_tracked_directory_with_missing_dir_file(M, tmp_dir, dvc):
tmp_dir.dvc_gen({"dir": {"file": "2"}})
(tmp_dir / "dvc.yaml").dump({"params": [join("dir", "file")]})
shutil.rmtree(tmp_dir / "dir") # remove from workspace
+2 -3
View File
@@ -15,7 +15,6 @@ from dvc.output import Output
from dvc.stage import PipelineStage, Stage
from dvc.stage.cache import RunCacheNotSupported
from dvc.stage.exceptions import StageFileDoesNotExistError, StageNotFound
from dvc.testing import matchers as M
from dvc.utils.fs import remove
from dvc.utils.serialize import modify_yaml
from dvc_data.hashfile.hash import file_md5
@@ -56,7 +55,7 @@ def test_repro_frozen(tmp_dir, dvc, run_copy):
assert stages == [data_stage, stage0]
def test_downstream(tmp_dir, dvc):
def test_downstream(M, tmp_dir, dvc):
# The dependency graph should look like this:
#
# E
@@ -149,7 +148,7 @@ def test_downstream(tmp_dir, dvc):
assert [stage.name for stage in evaluation] == ["B-gen", "D-gen", "E-gen"]
# B, C should be run (in any order) before D
# See https://github.com/treeverse/dvc/issues/3602
# See https://github.com/iterative/dvc/issues/3602
evaluation = dvc.reproduce(PROJECT_FILE + ":A-gen", downstream=True, force=True)
assert len(evaluation) == 5
+1 -1
View File
@@ -36,7 +36,7 @@ def test_repro_allow_missing_and_pull(tmp_dir, dvc, mocker, local_remote):
def test_repro_allow_missing_upstream_stage_modified(
tmp_dir, dvc, mocker, local_remote
):
"""https://github.com/treeverse/dvc/issues/9530"""
"""https://github.com/iterative/dvc/issues/9530"""
tmp_dir.gen("params.yaml", "param: 1")
dvc.stage.add(
name="create-foo", cmd="echo ${param} > foo", params=["param"], outs=["foo"]
+1 -1
View File
@@ -723,7 +723,7 @@ def test_add_symlink_file(tmp_dir, dvc):
).is_symlink()
# Test that subsequent add succeeds
# See https://github.com/treeverse/dvc/issues/4654
# See https://github.com/iterative/dvc/issues/4654
dvc.add(os.path.join("dir", "foo"))
+6 -6
View File
@@ -88,8 +88,8 @@ def test_config_set_in_non_dvc_repo(tmp_dir, caplog):
[
(["core.analytics"], 0, "False"),
(["core.remote"], 0, "myremote"),
(["remote.myremote.profile"], 0, "treeverse"),
(["remote.myremote.profile", "--local"], 0, "treeverse"),
(["remote.myremote.profile"], 0, "iterative"),
(["remote.myremote.profile", "--local"], 0, "iterative"),
(
["remote.myremote.profile", "--project"],
251,
@@ -122,7 +122,7 @@ def test_config_get(tmp_dir, dvc, capsys, caplog, args, ret, msg):
textwrap.dedent(
"""\
['remote "myremote"']
profile = treeverse
profile = iterative
['remote "other"']
url = gs://bucket/path
"""
@@ -167,7 +167,7 @@ def test_config_list(tmp_dir, dvc, capsys):
textwrap.dedent(
"""\
['remote "myremote"']
profile = treeverse
profile = iterative
access_key_id = abcde
secret_access_key = 123456
['remote "other"']
@@ -181,7 +181,7 @@ def test_config_list(tmp_dir, dvc, capsys):
out, _ = capsys.readouterr()
assert "remote.myremote.url=s3://bucket/path" in out
assert "remote.myremote.region=us-east-2" in out
assert "remote.myremote.profile=treeverse" in out
assert "remote.myremote.profile=iterative" in out
assert "remote.myremote.access_key_id=abcde" in out
assert "remote.myremote.secret_access_key=123456" in out
assert "remote.other.url=gs://bucket/path" in out
@@ -242,7 +242,7 @@ def test_merging_two_levels(dvc):
def test_config_loads_without_error_for_non_dvc_repo(tmp_dir):
# regression testing for https://github.com/treeverse/dvc/issues/3328
# regression testing for https://github.com/iterative/dvc/issues/3328
Config.from_cwd(validate=True)
+2 -2
View File
@@ -730,12 +730,12 @@ def test_pull_allow_missing(tmp_dir, dvc, local_remote):
def test_pull_granular_excluding_import_that_cannot_be_pulled(
tmp_dir, dvc, local_remote, mocker
):
"""Regression test for https://github.com/treeverse/dvc/issues/10309."""
"""Regression test for https://github.com/iterative/dvc/issues/10309."""
mocker.patch("dvc.fs.dvc._DVCFileSystem", side_effect=CloneError("SCM error"))
(stage,) = tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
imp_stage = dvc.imp(
"https://user:[email protected]/treeverse/dvc.git",
"https://user:[email protected]/iterative/dvc.git",
"dir",
out="new_dir",
rev="HEAD",
+48 -81
View File
@@ -6,10 +6,10 @@ from os.path import join
from typing import TYPE_CHECKING
import pytest
from pytest_test_utils import matchers as m
from dvc.repo import Repo
from dvc.repo.data import _transform_git_paths_to_dvc, posixpath_to_os_path
from dvc.testing import matchers as M
from dvc.testing.tmp_dir import TmpDir, make_subrepo
from dvc.utils.fs import remove
@@ -60,7 +60,7 @@ def test_git_to_dvc_path_wdir_transformation(tmp_dir, scm, path):
]
def test_file(tmp_dir, dvc, scm):
def test_file(M, tmp_dir, dvc, scm):
tmp_dir.dvc_gen("foo", "foo", commit="add foo")
tmp_dir.dvc_gen("foo", "foobar")
remove(tmp_dir / "foo")
@@ -75,7 +75,7 @@ def test_file(tmp_dir, dvc, scm):
assert dvc.data_status(granular=True) == expected
def test_directory(tmp_dir, dvc, scm):
def test_directory(M, tmp_dir, dvc, scm):
tmp_dir.dvc_gen({"dir": {"foo": "foo"}}, commit="add dir")
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar", "foobar": "foobar"}})
remove(tmp_dir / "dir")
@@ -107,7 +107,7 @@ def test_directory(tmp_dir, dvc, scm):
}
def test_tracked_directory_deep(tmp_dir, dvc, scm):
def test_tracked_directory_deep(M, tmp_dir, dvc, scm):
"""Test for a directory not in cwd, but nested inside other directories."""
(tmp_dir / "sub").gen({"dir": {"foo": "foo"}})
dvc.add(fspath(tmp_dir / "sub" / "dir"))
@@ -140,7 +140,7 @@ def test_tracked_directory_deep(tmp_dir, dvc, scm):
@pytest.mark.parametrize("git_repo_state", ["unborn", "committed"])
@pytest.mark.parametrize("subdir", [True, False])
def test_new_dvc_repo(tmp_dir, scm, subdir, git_repo_state):
def test_new_dvc_repo(M, tmp_dir, scm, subdir, git_repo_state):
if git_repo_state == "committed":
tmp_dir.scm_gen("test", "test", commit="init")
@@ -167,7 +167,7 @@ def test_noscm_repo(tmp_dir, dvc):
assert dvc.data_status() == {**EMPTY_STATUS, "unchanged": ["foo"]}
def test_unchanged(tmp_dir, dvc, scm):
def test_unchanged(M, tmp_dir, dvc, scm):
tmp_dir.dvc_gen({"dir": {"foo": "foo"}}, commit="add dir")
tmp_dir.dvc_gen("bar", "bar", commit="add foo")
@@ -195,7 +195,7 @@ def test_skip_uncached_pipeline_outputs(tmp_dir, dvc, run_copy_metrics):
assert dvc.data_status(granular=True, untracked_files="all") == EMPTY_STATUS
def test_outs_with_no_hashes(tmp_dir, dvc, scm):
def test_outs_with_no_hashes(M, tmp_dir, dvc, scm):
dvc.stage.add(single_stage=True, outs=["bar"])
dvc.stage.add(deps=["bar"], outs=["foo"], name="copy", cmd="cp foo bar")
@@ -204,7 +204,7 @@ def test_outs_with_no_hashes(tmp_dir, dvc, scm):
assert dvc.data_status(granular=True) == expected_output
def test_outs_with_no_hashes_and_with_uncommitted_files(tmp_dir, dvc, scm):
def test_outs_with_no_hashes_and_with_uncommitted_files(M, tmp_dir, dvc, scm):
tmp_dir.gen({"bar": "bar", "foo": "foo"})
dvc.stage.add(single_stage=True, outs=["bar"])
dvc.stage.add(deps=["bar"], outs=["foo"], name="copy", cmd="cp foo bar")
@@ -218,7 +218,7 @@ def test_outs_with_no_hashes_and_with_uncommitted_files(tmp_dir, dvc, scm):
assert dvc.data_status(granular=True) == expected_output
def test_subdir(tmp_dir, scm):
def test_subdir(M, tmp_dir, scm):
subrepo = tmp_dir / "sub"
make_subrepo(subrepo, scm)
@@ -236,7 +236,7 @@ def test_subdir(tmp_dir, scm):
}
def test_untracked_newly_added_files(tmp_dir, dvc, scm):
def test_untracked_newly_added_files(M, tmp_dir, dvc, scm):
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
tmp_dir.gen("foobar", "foobar")
@@ -249,7 +249,7 @@ def test_untracked_newly_added_files(tmp_dir, dvc, scm):
assert dvc.data_status(granular=True, untracked_files="all") == expected
def test_missing_cache_workspace_exists(tmp_dir, dvc, scm):
def test_missing_cache_workspace_exists(M, tmp_dir, dvc, scm):
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
tmp_dir.dvc_gen("foobar", "foobar")
remove(dvc.cache.repo.path)
@@ -272,7 +272,7 @@ def test_missing_cache_workspace_exists(tmp_dir, dvc, scm):
}
def test_missing_cache_missing_workspace(tmp_dir, dvc, scm):
def test_missing_cache_missing_workspace(M, tmp_dir, dvc, scm):
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
tmp_dir.dvc_gen("foobar", "foobar")
for path in [dvc.cache.repo.path, "dir", "foobar"]:
@@ -297,7 +297,7 @@ def test_missing_cache_missing_workspace(tmp_dir, dvc, scm):
}
def test_git_committed_missing_cache_workspace_exists(tmp_dir, dvc, scm):
def test_git_committed_missing_cache_workspace_exists(M, tmp_dir, dvc, scm):
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir")
tmp_dir.dvc_gen("foobar", "foobar", commit="add foobar")
remove(dvc.cache.local.path)
@@ -317,7 +317,7 @@ def test_git_committed_missing_cache_workspace_exists(tmp_dir, dvc, scm):
}
def test_git_committed_missing_cache_missing_workspace(tmp_dir, dvc, scm):
def test_git_committed_missing_cache_missing_workspace(M, tmp_dir, dvc, scm):
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir")
tmp_dir.dvc_gen("foobar", "foobar", commit="add foobar")
for path in [dvc.cache.repo.path, "dir", "foobar"]:
@@ -338,7 +338,7 @@ def test_git_committed_missing_cache_missing_workspace(tmp_dir, dvc, scm):
}
def test_partial_missing_cache(tmp_dir, dvc, scm):
def test_partial_missing_cache(M, tmp_dir, dvc, scm):
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
# remove "foo" from cache
@@ -362,7 +362,7 @@ def test_partial_missing_cache(tmp_dir, dvc, scm):
}
def test_missing_dir_object_from_head(tmp_dir, dvc, scm):
def test_missing_dir_object_from_head(M, tmp_dir, dvc, scm):
(stage,) = tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir")
remove("dir")
tmp_dir.dvc_gen({"dir": {"foobar": "foobar"}})
@@ -384,7 +384,7 @@ def test_missing_dir_object_from_head(tmp_dir, dvc, scm):
}
def test_missing_dir_object_from_index(tmp_dir, dvc, scm):
def test_missing_dir_object_from_index(M, tmp_dir, dvc, scm):
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir")
remove("dir")
(stage,) = tmp_dir.dvc_gen({"dir": {"foobar": "foobar"}})
@@ -406,7 +406,7 @@ def test_missing_dir_object_from_index(tmp_dir, dvc, scm):
}
def test_remote_check(tmp_dir, dvc, scm, make_remote):
def test_remote_check(M, tmp_dir, dvc, scm, make_remote):
make_remote("default", default=True)
make_remote("myremote", default=False)
@@ -442,7 +442,7 @@ def test_remote_check(tmp_dir, dvc, scm, make_remote):
assert dvc.data_status(granular=True, **opts) == expected_g
def test_missing_remote_cache(tmp_dir, dvc, scm, local_remote):
def test_missing_remote_cache(M, tmp_dir, dvc, scm, local_remote):
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
tmp_dir.dvc_gen("foobar", "foobar")
@@ -485,7 +485,7 @@ def test_missing_remote_cache(tmp_dir, dvc, scm, local_remote):
def test_not_in_remote_respects_not_pushable(
tmp_dir: TmpDir, dvc: Repo, scm, mocker, local_remote
M: type["m.Matcher"], tmp_dir: TmpDir, dvc: Repo, scm, mocker, local_remote
):
stages: list[Stage] = tmp_dir.dvc_gen({"foo": "foo", "dir": {"foobar": "foobar"}})
# Make foo not pushable
@@ -529,7 +529,7 @@ def test_not_in_remote_respects_not_pushable(
)
def test_root_from_dir_to_file(tmp_dir, dvc, scm):
def test_root_from_dir_to_file(M, tmp_dir, dvc, scm):
tmp_dir.dvc_gen({"data": {"foo": "foo", "bar": "bar"}})
remove("data")
tmp_dir.gen("data", "file")
@@ -555,7 +555,7 @@ def test_root_from_dir_to_file(tmp_dir, dvc, scm):
}
def test_root_from_file_to_dir(tmp_dir, dvc, scm):
def test_root_from_file_to_dir(M, tmp_dir, dvc, scm):
tmp_dir.dvc_gen("data", "file")
remove("data")
tmp_dir.gen({"data": {"foo": "foo", "bar": "bar"}})
@@ -577,8 +577,8 @@ def test_root_from_file_to_dir(tmp_dir, dvc, scm):
}
def test_empty_dir(tmp_dir, scm, dvc):
# regression testing for https://github.com/treeverse/dvc/issues/8958
def test_empty_dir(tmp_dir, scm, dvc, M):
# regression testing for https://github.com/iterative/dvc/issues/8958
tmp_dir.dvc_gen({"data": {"foo": "foo"}})
remove("data")
@@ -592,7 +592,7 @@ def test_empty_dir(tmp_dir, scm, dvc):
}
def test_untracked_files_filter_targets(tmp_dir, scm, dvc):
def test_untracked_files_filter_targets(M, tmp_dir, scm, dvc):
tmp_dir.gen(
{"spam": "spam", "ham": "ham", "dir": {"eggs": "eggs", "bacon": "bacon"}}
)
@@ -644,13 +644,13 @@ def param(*values):
param(
["foo", "foobar"],
{
"committed": {"added": M.unordered("foo", "foobar")},
"committed": {"added": m.unordered("foo", "foobar")},
"uncommitted": {"deleted": ["foo"]},
},
),
],
)
def test_filter_targets_files_after_dvc_commit(tmp_dir, dvc, scm, targets, expected):
def test_filter_targets_files_after_dvc_commit(M, tmp_dir, dvc, scm, targets, expected):
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar", "foobar": "foobar"})
(tmp_dir / "foo").unlink() # deleted
tmp_dir.gen({"bar": "bar modified", "baz": "baz new"})
@@ -677,7 +677,7 @@ def test_filter_targets_files_after_dvc_commit(tmp_dir, dvc, scm, targets, expec
),
],
)
def test_filter_targets_after_git_commit(tmp_dir, dvc, scm, targets, expected):
def test_filter_targets_after_git_commit(M, tmp_dir, dvc, scm, targets, expected):
tmp_dir.dvc_gen(
{"foo": "foo", "bar": "bar", "foobar": "foobar", "baz": "baz"},
commit="add files",
@@ -718,7 +718,7 @@ def with_aliases(values, aliases):
},
{
"committed": {
"added": M.unordered(
"added": m.unordered(
join("dir", ""),
join("dir", "foo"),
join("dir", "sub", "bar"),
@@ -779,7 +779,7 @@ def with_aliases(values, aliases):
{},
{
"committed": {
"added": M.unordered(join("dir", "foo"), join("dir", "foobar"))
"added": m.unordered(join("dir", "foo"), join("dir", "foobar"))
},
"uncommitted": {"deleted": [join("dir", "foo")]},
},
@@ -794,7 +794,7 @@ def with_aliases(values, aliases):
),
)
def test_filter_targets_inside_directory_after_dvc_commit(
tmp_dir, dvc, scm, targets, expected_ng, expected_g
M, tmp_dir, dvc, scm, targets, expected_ng, expected_g
):
tmp_dir.dvc_gen({"dir": {"foo": "foo", "sub": {"bar": "bar"}, "foobar": "foobar"}})
(tmp_dir / "dir" / "foo").unlink() # deleted
@@ -827,7 +827,7 @@ def test_filter_targets_inside_directory_after_dvc_commit(
"unchanged": [join("dir", "foobar")],
"committed": {
"added": [join("dir", "baz")],
"modified": M.unordered(
"modified": m.unordered(
join("dir", ""), join("dir", "sub", "bar")
),
"deleted": [join("dir", "foo")],
@@ -861,7 +861,7 @@ def test_filter_targets_inside_directory_after_dvc_commit(
),
)
def test_filter_targets_inside_directory_after_git_commit(
tmp_dir, dvc, scm, targets, expected_ng, expected_g
M, tmp_dir, dvc, scm, targets, expected_ng, expected_g
):
tmp_dir.dvc_gen(
{"dir": {"foo": "foo", "sub": {"bar": "bar"}, "foobar": "foobar"}},
@@ -893,22 +893,22 @@ def test_filter_targets_inside_directory_after_git_commit(
param(
(join("dir", "bar"), "foo"),
["foo"],
M.unordered(join("dir", "bar"), "foo"),
m.unordered(join("dir", "bar"), "foo"),
),
param(
(join("dir", "bar"), "dir"),
[join("dir", "")],
M.unordered(join("dir", ""), join("dir", "bar")),
m.unordered(join("dir", ""), join("dir", "bar")),
),
param(
("dir", "foo"),
M.unordered(join("dir", ""), "foo"),
M.unordered(join("dir", ""), join("dir", "bar"), "foo"),
m.unordered(join("dir", ""), "foo"),
m.unordered(join("dir", ""), join("dir", "bar"), "foo"),
),
],
)
def test_filter_targets_not_in_cache(
local_remote, tmp_dir, scm, dvc, to_check, targets, non_granular, granular
M, local_remote, tmp_dir, scm, dvc, to_check, targets, non_granular, granular
):
tmp_dir.dvc_gen({"foo": "foo", "dir": {"bar": "bar"}})
@@ -927,7 +927,7 @@ def test_filter_targets_not_in_cache(
}
def test_compat_legacy_new_cache_types(tmp_dir, dvc, scm):
def test_compat_legacy_new_cache_types(M, tmp_dir, dvc, scm):
tmp_dir.gen({"foo": "foo", "bar": "bar"})
(tmp_dir / "foo.dvc").dump(
{
@@ -964,7 +964,7 @@ def test_compat_legacy_new_cache_types(tmp_dir, dvc, scm):
}
def test_missing_cache_remote_check(tmp_dir, dvc, scm, local_remote):
def test_missing_cache_remote_check(M, tmp_dir, dvc, scm, local_remote):
tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
tmp_dir.dvc_gen("foobar", "foobar")
remove(dvc.cache.repo.path)
@@ -1046,7 +1046,7 @@ def test_missing_cache_remote_check(tmp_dir, dvc, scm, local_remote):
{"committed": {"deleted": [join("dir", "")]}},
{
"committed": {
"deleted": M.unordered(
"deleted": m.unordered(
join("dir", ""), join("dir", "bar"), join("dir", "foo")
)
}
@@ -1060,7 +1060,7 @@ def test_missing_cache_remote_check(tmp_dir, dvc, scm, local_remote):
},
{
"committed": {
"added": M.unordered(
"added": m.unordered(
join("dir2", ""),
join("dir2", "bar"),
join("dir2", "foo"),
@@ -1150,12 +1150,12 @@ def test_missing_cache_remote_check(tmp_dir, dvc, scm, local_remote):
param(
["dir2", "file2"],
{
"committed": {"added": M.unordered(join("dir2", ""), "file2")},
"committed": {"added": m.unordered(join("dir2", ""), "file2")},
"uncommitted": {"modified": [join("dir2", "")]},
},
{
"committed": {
"added": M.unordered(
"added": m.unordered(
join("dir2", ""),
join("dir2", "bar"),
join("dir2", "foo"),
@@ -1178,7 +1178,7 @@ def test_missing_cache_remote_check(tmp_dir, dvc, scm, local_remote):
},
{
"committed": {
"added": M.unordered(
"added": m.unordered(
join("dir2", ""), join("dir2", "bar"), join("dir2", "foo")
),
"deleted": ["file"],
@@ -1193,7 +1193,9 @@ def test_missing_cache_remote_check(tmp_dir, dvc, scm, local_remote):
),
],
)
def test_renames(tmp_dir, scm, dvc, targets, expected_non_granular, expected_granular):
def test_renames(
M, tmp_dir, scm, dvc, targets, expected_non_granular, expected_granular
):
tmp_dir.dvc_gen(
{"dir": {"foo": "foo", "bar": "bar"}, "file": "file"}, commit="add dir and file"
)
@@ -1209,38 +1211,3 @@ def test_renames(tmp_dir, scm, dvc, targets, expected_non_granular, expected_gra
dvc.data_status(granular=True, with_renames=True, targets=targets)
== EMPTY_STATUS | {"git": M.dict()} | expected_granular
)
def test_shallow_should_iterate_upto_tracked_directory(tmp_dir, dvc, scm, local_remote):
"""Testing regression for https://github.com/treeverse/dvc/issues/10899."""
tmp_dir.scm_gen({"dir": {".gitkeep": ""}}, commit="mk dir")
dataset_root = join("dir", "data", "")
tmp_dir.dvc_gen({dataset_root: {"foo": "foo", "bar": "bar"}})
files = [dataset_root, join(dataset_root, "foo"), join(dataset_root, "bar")]
assert dvc.data_status(not_in_remote=True) == {
**EMPTY_STATUS,
"committed": {"added": [dataset_root]},
"not_in_remote": [dataset_root],
"git": M.dict(),
}
assert dvc.data_status(granular=True, not_in_remote=True) == {
**EMPTY_STATUS,
"committed": {"added": M.unordered(*files)},
"not_in_remote": M.unordered(*files),
"git": M.dict(),
}
assert dvc.push() == 3
assert dvc.data_status(not_in_remote=True, remote_refresh=True) == {
**EMPTY_STATUS,
"committed": {"added": [dataset_root]},
"git": M.dict(),
}
assert dvc.data_status(granular=True, not_in_remote=True, remote_refresh=True) == {
**EMPTY_STATUS,
"committed": {"added": M.unordered(*files)},
"git": M.dict(),
}
+1 -1
View File
@@ -101,7 +101,7 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
def test_relative_remote(erepo_dir, tmp_dir):
# these steps reproduce the script on this issue:
# https://github.com/treeverse/dvc/issues/2756
# https://github.com/iterative/dvc/issues/2756
with erepo_dir.chdir():
erepo_dir.dvc_gen("file", "contents", commit="create file")
+2 -28
View File
@@ -437,16 +437,8 @@ def test_gc_logging(caplog, dvc, good_and_bad_cache):
with caplog.at_level(logging.INFO, logger="dvc"):
dvc.gc(workspace=True)
# Check that 3 objects were removed from either repo or local cache
# (they point to the same ODB, so only one will be logged due to deduplication)
has_repo_log = "Removed 3 objects from repo cache." in caplog.text
has_local_log = "Removed 3 objects from local cache." in caplog.text
# Exactly one of repo or local should have the log, not both (deduplication)
assert has_repo_log ^ has_local_log, (
"Should have exactly one log for repo/local cache"
)
assert "Removed 3 objects from repo cache." in caplog.text
assert "No unused 'local' cache to remove." in caplog.text
assert "No unused 'legacy' cache to remove." in caplog.text
@@ -458,21 +450,3 @@ def test_gc_skip_failed(tmp_dir, dvc):
dvc.gc(force=True, workspace=True)
dvc.gc(force=True, workspace=True, skip_failed=True)
def test_gc_dry_logs_paths(caplog, tmp_dir, dvc):
"""Test that dry run logs paths of objects to be removed."""
odb = dvc.cache.local
# Add some unused objects directly to cache
unused_hashes = ["test123", "test456", "test789"]
for hash_val in unused_hashes:
odb.add_bytes(hash_val, hash_val.encode("utf8"))
with caplog.at_level(logging.INFO, logger="dvc_data.hashfile.gc"):
dvc.gc(workspace=True, dry=True)
# Verify that paths are logged by dvc-data layer
for hash_val in unused_hashes:
expected_path = odb.oid_to_path(hash_val)
assert f"Removing {expected_path}" in caplog.text
+1 -1
View File
@@ -160,7 +160,7 @@ def test_non_cached_output(tmp_dir, erepo_dir):
assert (tmp_dir / dst).read_text().strip() == "hello"
# https://github.com/treeverse/dvc/pull/2837#discussion_r352123053
# https://github.com/iterative/dvc/pull/2837#discussion_r352123053
def test_absolute_file_outside_repo(tmp_dir, erepo_dir):
with pytest.raises(FileNotFoundError):
Repo.get(os.fspath(erepo_dir), "/root/")
+1 -1
View File
@@ -405,7 +405,7 @@ def test_import_non_existing(erepo_dir, tmp_dir, dvc):
with pytest.raises(FileNotFoundError):
tmp_dir.dvc.imp(os.fspath(erepo_dir), "invalid_output")
# https://github.com/treeverse/dvc/pull/2837#discussion_r352123053
# https://github.com/iterative/dvc/pull/2837#discussion_r352123053
with pytest.raises(FileNotFoundError):
tmp_dir.dvc.imp(os.fspath(erepo_dir), "/root/", "root")
+1 -3
View File
@@ -6,8 +6,6 @@ import pandas as pd
import pytest
from funcy import compact
from dvc.testing import matchers as M
@pytest.fixture
def db_path(tmp_dir):
@@ -48,7 +46,7 @@ def load_data(file, output_format):
({"table": "model"}, "model"),
],
)
def test(tmp_dir, scm, dvc, db_connection, seed_db, output_format, args, file_name):
def test(M, tmp_dir, scm, dvc, db_connection, seed_db, output_format, args, file_name):
seed_db(values=range(5))
if output_format == "json":
file_size = 96, 192
+1 -1
View File
@@ -109,7 +109,7 @@ def test_gen_dvcignore(tmp_dir):
def test_init_when_ignored_by_git(tmp_dir, scm, caplog):
# https://github.com/treeverse/dvc/issues/3738
# https://github.com/iterative/dvc/issues/3738
tmp_dir.gen({".gitignore": ".*"})
with caplog.at_level(logging.ERROR, logger="dvc"):
assert main(["init"]) == 1
+6 -7
View File
@@ -10,7 +10,6 @@ from dvc.fs import MemoryFileSystem
from dvc.repo import Repo
from dvc.repo.ls import _ls_tree, ls_tree
from dvc.scm import CloneError
from dvc.testing import matchers as M
FS_STRUCTURE = {
"README.md": "content",
@@ -461,7 +460,7 @@ def test_ls_shows_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
match_files(files, ((("bar",), True),))
def test_ls_granular(erepo_dir):
def test_ls_granular(erepo_dir, M):
with erepo_dir.chdir():
erepo_dir.dvc_gen(
{
@@ -625,7 +624,7 @@ def test_subrepo(request, dvc_top_level, erepo_type):
assert _list_files(subrepo, "dvc_dir") == {"lorem"}
def test_broken_symlink(tmp_dir, dvc):
def test_broken_symlink(tmp_dir, dvc, M):
from dvc.fs import system
tmp_dir.gen("file", "content")
@@ -655,7 +654,7 @@ def test_broken_symlink(tmp_dir, dvc):
]
def test_ls_broken_dir(tmp_dir, dvc):
def test_ls_broken_dir(tmp_dir, dvc, M):
from dvc_data.index import DataIndexDirError
tmp_dir.dvc_gen(
@@ -837,7 +836,7 @@ def _simplify_tree(files):
return ret
def test_ls_tree(tmp_dir, scm, dvc):
def test_ls_tree(M, tmp_dir, scm, dvc):
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
@@ -888,7 +887,7 @@ def test_ls_tree(tmp_dir, scm, dvc):
}
def test_ls_tree_dvc_only(tmp_dir, scm, dvc):
def test_ls_tree_dvc_only(M, tmp_dir, scm, dvc):
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
@@ -906,7 +905,7 @@ def test_ls_tree_dvc_only(tmp_dir, scm, dvc):
assert _simplify_tree(files) == expected
def test_ls_tree_maxdepth(tmp_dir, scm, dvc):
def test_ls_tree_maxdepth(M, tmp_dir, scm, dvc):
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
+4 -201
View File
@@ -6,20 +6,14 @@ import pytest
from dvc.cli import main
from dvc.exceptions import MoveNotDataSourceError, OutputNotFoundError
from dvc.stage.exceptions import StageFileAlreadyExistsError
def test_move(tmp_dir, dvc, scm):
tmp_dir.dvc_gen("foo", "bar")
assert (tmp_dir / "foo.dvc").exists()
dvc.move("foo", "bar")
def test_move(tmp_dir, dvc):
tmp_dir.dvc_gen("foo", "foo")
dvc.move("foo", "foo1")
assert not (tmp_dir / "foo.dvc").exists()
assert (tmp_dir / "bar.dvc").exists()
assert not (tmp_dir / "foo").is_file()
assert (tmp_dir / "bar").is_file()
# should only have the new path in the .gitignore, and only once
assert (tmp_dir / ".gitignore").read_text().splitlines() == ["/bar"]
assert (tmp_dir / "foo1").is_file()
def test_move_non_existent_file(dvc):
@@ -27,24 +21,6 @@ def test_move_non_existent_file(dvc):
dvc.move("non_existent_file", "dst")
def test_move_missing_file(tmp_dir, dvc, scm, caplog):
tmp_dir.dvc_gen("foo", "foo")
(tmp_dir / "foo").unlink()
contents = (tmp_dir / "foo.dvc").parse()
dvc.move("foo", "bar")
assert not (tmp_dir / "foo.dvc").exists()
# only the path should be changed in the dvc file
contents["outs"][0]["path"] = "bar"
assert contents == (tmp_dir / "bar.dvc").parse()
# file should not be checked out
assert not (tmp_dir / "foo").is_file()
assert not (tmp_dir / "bar").is_file()
# should only have the new path in the .gitignore, and only once
assert (tmp_dir / ".gitignore").read_text().splitlines() == ["/bar"]
def test_move_directory(tmp_dir, dvc):
tmp_dir.dvc_gen("data", {"foo": "foo", "bar": "bar"})
dvc.move("data", "dst")
@@ -219,176 +195,3 @@ def test_move_meta(tmp_dir, dvc):
custom_key: 42
"""
)
def test_import(tmp_dir, dvc, scm):
tmp_dir.dvc_gen("foo", "foo", commit="add foo")
imp_stage = dvc.imp(os.curdir, "foo", "foo_imported")
dvc.move("foo_imported", "foo_moved")
(stage,) = dvc.stage.collect("foo_moved.dvc")
assert imp_stage.md5 != stage.md5
res = (tmp_dir / "foo_moved.dvc").read_text()
assert res == textwrap.dedent(
f"""\
md5: {stage.md5}
frozen: true
deps:
- path: foo
repo:
url: {os.curdir}
rev_lock: {scm.get_rev()}
outs:
- md5: acbd18db4cc2f85cedef654fccc4a4d8
size: 3
hash: md5
path: foo_moved
"""
)
@pytest.mark.parametrize(
"path_func",
[pytest.param(os.path.abspath, id="abs"), pytest.param(os.path.relpath, id="rel")],
)
def test_import_url_in_repo(tmp_dir, dvc, path_func):
tmp_dir.gen("foo", "foo")
imp_stage = dvc.imp_url(path_func(tmp_dir / "foo"), "foo_imported")
(tmp_dir / "data").mkdir()
dvc.move("foo_imported", os.path.join("data", "foo_moved"))
(stage,) = dvc.stage.collect(os.path.join("data", "foo_moved.dvc"))
assert imp_stage.md5 != stage.md5
res = (tmp_dir / "data" / "foo_moved.dvc").read_text()
assert res == textwrap.dedent(
f"""\
md5: {stage.md5}
frozen: true
deps:
- md5: acbd18db4cc2f85cedef654fccc4a4d8
size: 3
hash: md5
path: ../foo
outs:
- md5: acbd18db4cc2f85cedef654fccc4a4d8
size: 3
hash: md5
path: foo_moved
"""
)
@pytest.mark.parametrize(
"path_func",
[pytest.param(os.path.abspath, id="abs"), pytest.param(os.path.relpath, id="rel")],
)
def test_import_url_out_of_repo(tmp_dir, dvc, scm, path_func, make_tmp_dir):
external = make_tmp_dir("external")
external.gen("foo", "foo")
imp_stage = dvc.imp_url(path_func(external / "foo"), "foo_imported")
data_dir = tmp_dir / "data"
data_dir.mkdir()
new_path = data_dir / "foo_moved"
new_dvcfile = new_path.with_suffix(".dvc")
dvc.move("foo_imported", os.fspath(new_path))
(stage,) = dvc.stage.collect(os.fspath(new_dvcfile))
assert imp_stage.md5 != stage.md5
with data_dir.chdir():
expected_path = path_func(external / "foo")
assert new_dvcfile.parse() == {
"md5": stage.md5,
"frozen": True,
"deps": [
{
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
"size": 3,
"hash": "md5",
"path": expected_path,
}
],
"outs": [
{
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
"size": 3,
"hash": "md5",
"path": "foo_moved",
}
],
}
@pytest.mark.parametrize(
"path_func",
[pytest.param(os.path.abspath, id="abs"), pytest.param(os.path.relpath, id="rel")],
)
def test_all_metadata_are_preserved(tmp_dir, dvc, make_tmp_dir, path_func):
external = make_tmp_dir("external")
external.gen("foo", "foo")
contents = {
"md5": "bad", # placeholder, does not matter for the test
"frozen": True,
"desc": "this is a stage description",
"always_changed": True,
"meta": {"custom_key": 42},
"deps": [
{
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
"size": 3,
"hash": "md5",
"path": path_func(external / "foo"),
}
],
"outs": [
{
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
"path": "foo_imported",
"persist": True,
"hash": "md5",
"size": 3,
"desc": "this is a description",
"type": "model",
"labels": ["label1", "label2"],
"meta": {"custom_key": 42},
"cache": False,
"remote": "myremote",
"push": False,
}
],
}
(tmp_dir / "foo_imported.dvc").dump(contents)
(tmp_dir / "foo_imported").write_text("foo")
data_dir = tmp_dir / "data"
data_dir.mkdir()
new_path = data_dir / "foo_moved"
new_dvcfile = new_path.with_suffix(".dvc")
dvc.move("foo_imported", os.fspath(new_path))
(stage,) = dvc.stage.collect(os.fspath(new_dvcfile))
with data_dir.chdir():
expected_path = path_func(external / "foo")
contents["outs"][0] |= {"path": "foo_moved"}
contents["deps"][0] |= {"path": expected_path}
contents |= {"md5": stage.md5}
assert new_dvcfile.parse() == contents
def test_move_dst_stage_file_already_exists(tmp_dir, dvc):
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
with pytest.raises(StageFileAlreadyExistsError) as exc_info:
dvc.move("foo", "bar")
assert str(exc_info.value) == "'bar.dvc' already exists"
assert exc_info.value.__cause__ is None
+1 -1
View File
@@ -125,7 +125,7 @@ def assert_index_equal(first, second, strict=True, ordered=True):
def test_skip_graph_checks(dvc, mocker):
# See https://github.com/treeverse/dvc/issues/2671 for more info
# See https://github.com/iterative/dvc/issues/2671 for more info
mock_build_graph = mocker.spy(Index.graph, "fget")
# sanity check
+1 -1
View File
@@ -166,7 +166,7 @@ def test_virtual_update_noop(tmp_dir, dvc, remote):
assert not dvc.push()
def test_partial_checkout_and_update(tmp_dir, dvc, remote):
def test_partial_checkout_and_update(M, tmp_dir, dvc, remote):
dir1 = {f"{i}.txt": f"dir1 {i}" for i in range(10)}
dir2 = {f"{i}.txt": f"dir2 {i}" for i in range(10)}
tmp_dir.gen({"dir": {"foo": "foo", "subdir": dir1, "subdir2": dir2}})
+2 -2
View File
@@ -241,7 +241,7 @@ def test_compose_and_dump_dir_module(
def test_compose_and_dump_yaml_handles_string(tmp_dir):
"""Regression test for https://github.com/treeverse/dvc/issues/8583"""
"""Regression test for https://github.com/iterative/dvc/issues/8583"""
from dvc.utils.hydra import compose_and_dump
config = tmp_dir / "conf" / "config.yaml"
@@ -253,7 +253,7 @@ def test_compose_and_dump_yaml_handles_string(tmp_dir):
def test_compose_and_dump_resolves_interpolation(tmp_dir):
"""Regression test for https://github.com/treeverse/dvc/issues/9196"""
"""Regression test for https://github.com/iterative/dvc/issues/9196"""
from dvc.utils.hydra import compose_and_dump
config = tmp_dir / "conf" / "config.yaml"
+1 -1
View File
@@ -59,7 +59,7 @@ def test_remote_missing_deps_are_correctly_reported(tmp_dir, caplog, mocker, pkg
else:
expected += (
"\nPlease report this bug to "
"<https://github.com/treeverse/dvc/issues>. "
"<https://github.com/iterative/dvc/issues>. "
"Thank you!"
)
assert expected in caplog.text
+2 -53
View File
@@ -2,15 +2,7 @@ import networkx as nx
import pytest
from dvc.cli import main, parse_args
from dvc.commands.dag import (
CmdDAG,
_build,
_collapse_foreach_matrix,
_show_ascii,
_show_dot,
_show_mermaid,
)
from dvc.parsing import JOIN
from dvc.commands.dag import CmdDAG, _build, _show_ascii, _show_dot, _show_mermaid
@pytest.mark.parametrize(
@@ -57,53 +49,10 @@ def repo(tmp_dir, dvc):
return dvc
def test_collapse_foreach_matrix(repo):
graph = nx.DiGraph(
[
("2", "1"),
("3@a", "2"),
("3@a", "1"),
("4", "1"),
("3@b", "4"),
("3@b", "1"),
("5", "3@a"),
("6", "3@b"),
("7", "5"),
("7", "6"),
]
)
expected_graph = nx.DiGraph(
[
("2", "1"),
("3", "2"),
("4", "1"),
("3", "4"),
("3", "1"),
("5", "3"),
("6", "3"),
("7", "5"),
("7", "6"),
]
)
collapsed_graph = _collapse_foreach_matrix(graph)
for node in collapsed_graph.nodes:
assert JOIN not in node
for n1, n2 in collapsed_graph.edges:
assert JOIN not in n1
assert JOIN not in n2
assert nx.is_isomorphic(collapsed_graph, expected_graph)
def test_build(repo):
assert nx.is_isomorphic(_build(repo), repo.index.graph)
def test_build_collapse(repo):
assert nx.is_isomorphic(
_build(repo, collapse_foreach_matrix=True), repo.index.graph
)
def test_build_target(repo):
graph = _build(repo, target="3")
assert set(graph.nodes()) == {"3", "b.dvc", "a.dvc"}
@@ -170,7 +119,7 @@ def test_show_ascii(repo):
def test_show_dot(repo):
# dot file rendering is not deterministic though graph
# output doesn't depend upon order of lines. Use sorted values
# https://github.com/treeverse/dvc/pull/7725
# https://github.com/iterative/dvc/pull/7725
expected = [
"\"stage: '1'\";",
"\"stage: '2'\";",
-2
View File
@@ -130,7 +130,6 @@ def test_experiments_run(dvc, scm, mocker):
"tmp_dir": False,
"copy_paths": [],
"message": None,
"no_hydra": False,
}
default_arguments.update(repro_arguments)
@@ -152,7 +151,6 @@ def test_experiments_run_message(dvc, scm, mocker, flag):
"tmp_dir": False,
"copy_paths": [],
"message": "mymessage",
"no_hydra": False,
}
default_arguments.update(repro_arguments)
+1 -2
View File
@@ -1,10 +1,9 @@
from dvc.cli import parse_args
from dvc.commands.get_url import CmdGetUrl
from dvc.config import Config
from dvc.testing import matchers as M
def test_get_url(mocker):
def test_get_url(mocker, M):
cli_args = parse_args(["get-url", "src", "out", "-j", "5"])
assert cli_args.func == CmdGetUrl
+3 -4
View File
@@ -2,10 +2,9 @@ from dvc.cli import parse_args
from dvc.commands.ls_url import CmdListUrl
from dvc.config import Config
from dvc.fs import LocalFileSystem
from dvc.testing import matchers as M
def test_ls_url(mocker):
def test_ls_url(mocker, M):
cli_args = parse_args(["ls-url", "src"])
assert cli_args.func == CmdListUrl
cmd = cli_args.func(cli_args)
@@ -22,7 +21,7 @@ def test_ls_url(mocker):
)
def test_recursive(mocker):
def test_recursive(mocker, M):
cli_args = parse_args(["ls-url", "-R", "-L", "2", "src"])
assert cli_args.func == CmdListUrl
cmd = cli_args.func(cli_args)
@@ -35,7 +34,7 @@ def test_recursive(mocker):
)
def test_tree(mocker):
def test_tree(mocker, M):
cli_args = parse_args(["ls-url", "--tree", "--level", "2", "src"])
assert cli_args.func == CmdListUrl
cmd = cli_args.func(cli_args)
+2 -2
View File
@@ -7,9 +7,9 @@ from dvc_studio_client.auth import AuthorizationExpiredError
@pytest.fixture(autouse=True)
def global_config_dir(monkeypatch, tmp_path_factory):
def global_config_dir(monkeypatch, tmp_dir_factory):
monkeypatch.setenv(
env.DVC_GLOBAL_CONFIG_DIR, str(tmp_path_factory.mktemp("studio-login"))
env.DVC_GLOBAL_CONFIG_DIR, str(tmp_dir_factory.mktemp("studio-login"))
)
+1 -1
View File
@@ -214,7 +214,7 @@ def test_read_params_py(tmp_dir, dvc):
def test_params_py_tuple_status(tmp_dir, dvc):
"""https://github.com/treeverse/dvc/issues/8803"""
"""https://github.com/iterative/dvc/issues/8803"""
parameters_file = "parameters.py"
tmp_dir.gen(parameters_file, "TUPLE = (10, 100)\n")
dep = ParamsDependency(Stage(dvc), parameters_file, ["TUPLE"])
+1 -1
View File
@@ -42,7 +42,7 @@ def test_save_missing(dvc, mocker):
), # hdfs checksum
# Not much we can do about hex and oct values without writing our own
# parser. So listing these test cases just to acknowledge this.
# See https://github.com/treeverse/dvc/issues/3331.
# See https://github.com/iterative/dvc/issues/3331.
(0x3451, "13393"),
(0o1244, "676"),
],
-28
View File
@@ -153,31 +153,3 @@ def test_remote_with_jobs(dvc, base_url, fs_cls):
assert config["user"] == "admin"
assert f"{scheme}://{user}@example.com" in config["host"]
assert cls is fs_cls
def test_bearer_token_command(mocker):
mock_auth_obj = mocker.Mock()
mock_get_auth = mocker.patch(
"dvc_webdav.get_bearer_auth", return_value=mock_auth_obj
)
command = "my-token-cmd.sh"
timeout = 10
# Configuration with potential conflicts
config = {
"url": url,
"token": "static-token-value",
"user": "conflict-user",
"password": "conflict-password",
"bearer_token_command": command,
"timeout": timeout,
}
fs = WebDAVFileSystem(**config)
# the factory function was invoked with the correct arguments from the config.
mock_get_auth.assert_called_once_with(command, timeout)
# the generated auth object is correctly injected into `fs_args`.
assert fs.fs_args["auth"] == mock_auth_obj
+2 -3
View File
@@ -4,7 +4,6 @@ from funcy import set_in
from dvc.render import FIELD, FILENAME, REVISION
from dvc.render.converter.vega import VegaConverter
from dvc.render.match import PlotsData, _squash_plots_properties, match_defs_renderers
from dvc.testing import matchers as M
@pytest.mark.parametrize(
@@ -119,7 +118,7 @@ def test_group_definitions(data, expected):
assert grouped == expected
def test_match_renderers():
def test_match_renderers(M):
data = {
"v1": {
"definitions": {
@@ -188,7 +187,7 @@ def test_match_renderers():
assert not renderer_with_errors.definition_errors
def test_flat_datapoints_errors_are_caught(mocker):
def test_flat_datapoints_errors_are_caught(M, mocker):
d = {}
d = set_in(
d,
+1 -1
View File
@@ -69,7 +69,7 @@ def session_app(tmp_path_factory) -> FSApp:
"""Session scoped experiments queue celery app."""
from kombu.transport.filesystem import Channel
# related to https://github.com/treeverse/dvc-task/issues/61
# related to https://github.com/iterative/dvc-task/issues/61
Channel.QoS.restore_at_shutdown = False
from dvc_task.app import FSApp
+1 -1
View File
@@ -74,7 +74,7 @@ def test_locked(mocker):
def test_skip_graph_checks(tmp_dir, dvc, mocker, run_copy):
# See https://github.com/treeverse/dvc/issues/2671 for more info
# See https://github.com/iterative/dvc/issues/2671 for more info
from dvc.repo.index import Index
mock_build_graph = mocker.spy(Index.graph, "fget")
+1 -2
View File
@@ -4,7 +4,6 @@ from networkx import DiGraph
from networkx.utils import graphs_equal
from dvc.repo.reproduce import get_active_graph, plan_repro
from dvc.testing import matchers as M
def test_active_graph(mocker):
@@ -27,7 +26,7 @@ def test_active_graph(mocker):
assert not graphs_equal(g, active)
def test_repro_plan():
def test_repro_plan(M):
r"""
1
/ \
+6 -6
View File
@@ -167,16 +167,16 @@ def test_system_info():
@pytest.mark.parametrize(
"git_remote",
[
"git://github.com/treeverse/dvc.git",
"[email protected]:treeverse/dvc.git",
"http://github.com/treeverse/dvc.git",
"https://github.com/treeverse/dvc.git",
"ssh://[email protected]/treeverse/dvc.git",
"git://github.com/iterative/dvc.git",
"[email protected]:iterative/dvc.git",
"http://github.com/iterative/dvc.git",
"https://github.com/iterative/dvc.git",
"ssh://[email protected]/iterative/dvc.git",
],
)
def test_git_remote_hash(mocker, git_remote):
m = mocker.patch("dvc.analytics._git_remote_url", return_value=git_remote)
expected = hashlib.md5(b"treeverse/dvc.git", usedforsecurity=False).hexdigest()
expected = hashlib.md5(b"iterative/dvc.git", usedforsecurity=False).hexdigest()
assert analytics._git_remote_path_hash(None) == expected
m.assert_called_once_with(None)
+2 -2
View File
@@ -158,14 +158,14 @@ def test_check(mocker, updater):
None,
(
"Find the latest release at "
"https://github.com/treeverse/dvc/releases/latest."
"https://github.com/iterative/dvc/releases/latest."
),
),
(
"unknown",
(
"Find the latest release at "
"https://github.com/treeverse/dvc/releases/latest."
"https://github.com/iterative/dvc/releases/latest."
),
),
],
+2 -2
View File
@@ -46,7 +46,7 @@ def test_notify_refs(mocker, status_code, side_effect):
mock_post = mocker.patch("requests.Session.post", return_value=response)
notify_refs(
"[email protected]:treeverse/dvc.git",
"[email protected]:iterative/dvc.git",
"TOKEN",
pushed=["p1", "p2"],
removed=["r1", "r2"],
@@ -56,7 +56,7 @@ def test_notify_refs(mocker, status_code, side_effect):
assert mock_post.call_args == mocker.call(
urljoin(STUDIO_URL, "/webhook/dvc"),
json={
"repo_url": "[email protected]:treeverse/dvc.git",
"repo_url": "[email protected]:iterative/dvc.git",
"client": "dvc",
"refs": {
"pushed": ["p1", "p2"],