Compare commits

...
16 Commits
Author SHA1 Message Date
pre-commit-ci[bot] 72abd013cf [pre-commit.ci] pre-commit autoupdate (#10908)
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
2025-11-11 10:21:29 +05:45
pre-commit-ci[bot] fa5c33ed24 [pre-commit.ci] pre-commit autoupdate (#10906)
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
2025-11-04 13:56:29 +05:45
pre-commit-ci[bot] 9dda30b7a0 [pre-commit.ci] pre-commit autoupdate (#10901)
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
2025-10-28 09:18:51 +05:45
skshetry 19c146e392 fix(data-status): iterate/expand index upto tracked directory in shallow mode (#10900) 2025-10-27 17:06:07 +00:00
dependabot[bot] 749c22bbcb build(deps): bump actions/upload-artifact from 4 to 5 (#10897)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-10-27 09:18:20 +05:45
dependabot[bot] 19a0bf48f1 build(deps): bump actions/download-artifact from 5 to 6 (#10898)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-10-27 09:18:08 +05:45
skshetry bda614baa8 support Python 3.14 (#10896) 2025-10-24 19:26:50 +05:45
pre-commit-ci[bot] d6fa8b67ff [pre-commit.ci] pre-commit autoupdate (#10894)
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
2025-10-21 09:30:04 +05:45
skshetry 498a4d5d3a move: fix error message when dst stage file exists (#10893) 2025-10-17 19:07:51 +05:45
pre-commit-ci[bot] b74711d5fe [pre-commit.ci] pre-commit autoupdate (#10892)
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
2025-10-14 13:05:03 +05:45
skshetry 61d5de54e5 support moving missing file keeping the metadata intact (#10890) 2025-10-13 05:34:14 +00:00
skshetry ca7454980d move: support moving output from import/import-url stages (#10889) 2025-10-12 17:28:41 +00:00
skshetry 8912ca7d2b exp: add a retry when removing temporary refs to avoid FileLocked errors (#10840) 2025-10-12 10:05:21 +05:45
dependabot[bot] f13ae75a45 build(deps): bump astral-sh/setup-uv from 6 to 7 (#10883)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-10-08 12:51:45 +05:45
dependabot[bot] aa090ecf8d build(deps): bump github/codeql-action from 3 to 4 (#10884)
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-10-08 12:40:20 +05:45
pre-commit-ci[bot] 0c4a5cbbb7 [pre-commit.ci] pre-commit autoupdate (#10882)
* [pre-commit.ci] pre-commit autoupdate

updates:
- [github.com/astral-sh/ruff-pre-commit: v0.13.2 → v0.13.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.13.2...v0.13.3)

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
2025-10-07 09:05:01 +05:45
23 changed files with 384 additions and 61 deletions
+2 -2
View File
@@ -16,14 +16,14 @@ jobs:
steps:
- uses: actions/setup-python@v6
with:
python-version: "3.13"
python-version: "3.14"
- uses: actions/checkout@v5
with:
ref: ${{ github.event.pull_request.base.sha }}
fetch-depth: 0
- uses: astral-sh/setup-uv@v6
- uses: astral-sh/setup-uv@v7
with:
enable-cache: true
cache-dependency-glob: pyproject.toml
+5 -5
View File
@@ -25,9 +25,9 @@ jobs:
- uses: actions/setup-python@v6
with:
python-version: 3.13
python-version: "3.14"
- uses: astral-sh/setup-uv@v6
- uses: astral-sh/setup-uv@v7
- name: Omit local version for Test PyPI upload
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
@@ -41,7 +41,7 @@ jobs:
- name: Check dist
run: uv tool run twine check --strict dist/*
- uses: actions/upload-artifact@v4
- uses: actions/upload-artifact@v5
with:
name: Packages
path: dist/
@@ -61,7 +61,7 @@ jobs:
id-token: write
steps:
- uses: actions/download-artifact@v5
- uses: actions/download-artifact@v6
with:
name: Packages
path: dist
@@ -86,7 +86,7 @@ jobs:
id-token: write
steps:
- uses: actions/download-artifact@v5
- uses: actions/download-artifact@v6
with:
name: Packages
path: dist
+2 -2
View File
@@ -31,10 +31,10 @@ jobs:
uses: actions/checkout@v5
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
uses: github/codeql-action/init@v4
with:
languages: ${{ matrix.language }}
queries: security-extended
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
uses: github/codeql-action/analyze@v4
+2 -2
View File
@@ -19,7 +19,7 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest]
pyv: ["3.13"]
pyv: ["3.14"]
plugin: ["dvc-s3"]
steps:
@@ -38,7 +38,7 @@ jobs:
with:
python-version: ${{ matrix.pyv }}
- uses: astral-sh/setup-uv@v6
- uses: astral-sh/setup-uv@v7
with:
enable-cache: true
cache-dependency-glob: "**/pyproject.toml"
+16 -4
View File
@@ -32,8 +32,8 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.13'
- uses: astral-sh/setup-uv@v6
python-version: '3.14'
- uses: astral-sh/setup-uv@v7
with:
enable-cache: true
cache-dependency-glob: pyproject.toml
@@ -55,7 +55,7 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
pyv: ["3.9", "3.10", "3.11", "3.12", "3.13"]
pyv: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
include:
- os: windows-latest
pyv: "3.9"
@@ -117,6 +117,18 @@ jobs:
- os: windows-latest
pyv: "3.13"
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
- os: windows-latest
pyv: "3.14"
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
- os: windows-latest
pyv: "3.14"
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
- os: windows-latest
pyv: "3.14"
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
- os: windows-latest
pyv: "3.14"
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
steps:
- uses: actions/checkout@v5
with:
@@ -125,7 +137,7 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.pyv }}
- uses: astral-sh/setup-uv@v6
- uses: astral-sh/setup-uv@v7
with:
enable-cache: true
cache-dependency-glob: pyproject.toml
+1 -1
View File
@@ -22,7 +22,7 @@ repos:
- id: sort-simple-yaml
- id: trailing-whitespace
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: 'v0.13.2'
rev: 'v0.14.4'
hooks:
- id: ruff-check
args: [--fix, --exit-non-zero-on-fix]
+1 -1
View File
@@ -41,7 +41,7 @@ def _get(stage, p, info, **kwargs):
return Dependency(stage, p, info, **kwargs)
def loadd_from(stage, d_list):
def loadd_from(stage, d_list) -> list[Dependency]:
ret = []
for d in d_list:
p = d.pop(Output.PARAM_PATH, None)
+1
View File
@@ -110,6 +110,7 @@ class ParamsDependency(Dependency):
except MissingParamsFile:
return {}
assert self.repo
try:
return read_param_file(
self.repo.fs,
+1
View File
@@ -150,6 +150,7 @@ class RepoDependency(Dependency):
# local cache instead of the remote's cache. This avoids re-streaming of data,
# but messes up the call to `_get_remote_config()` downstream, which will need
# to ignore cache parameters.
assert self.repo
config["cache"] = self.repo.config["cache"]
config["cache"]["dir"] = self.repo.cache.local_cache_dir
-2
View File
@@ -1,8 +1,6 @@
"""Manages logging configuration for DVC repo."""
import logging
import logging.config
import logging.handlers
import os
import sys
from collections.abc import Iterator
+26 -9
View File
@@ -42,6 +42,7 @@ from .utils import relpath
from .utils.fs import path_isin
if TYPE_CHECKING:
from dvc.repo import Repo
from dvc_data.hashfile.obj import HashFile
from dvc_data.index import DataIndexKey
@@ -83,7 +84,7 @@ def _get(stage, path, **kwargs):
return Output(stage, path, **kwargs)
def loadd_from(stage, d_list):
def loadd_from(stage, d_list) -> list["Output"]:
ret = []
for d in d_list:
p = d.pop(Output.PARAM_PATH)
@@ -329,7 +330,7 @@ class Output:
self.annot = Annotation(
desc=desc, type=type, labels=labels or [], meta=meta or {}
)
self.repo = stage.repo if not repo and stage else repo
self.repo: Optional[Repo] = stage.repo if not repo and stage else repo
meta_d = merge_file_meta_from_cloud(info or {})
meta = Meta.from_dict(meta_d)
# NOTE: when version_aware is not passed into get_cloud_fs, it will be
@@ -511,6 +512,7 @@ class Output:
from dvc.cachemgr import LEGACY_HASH_NAMES
assert self.is_in_repo
assert self.repo
odb_name = "legacy" if self.hash_name in LEGACY_HASH_NAMES else "repo"
return getattr(self.repo.cache, odb_name)
@@ -518,6 +520,7 @@ class Output:
def local_cache(self):
from dvc.cachemgr import LEGACY_HASH_NAMES
assert self.repo
if self.hash_name in LEGACY_HASH_NAMES:
return self.repo.cache.legacy
return self.repo.cache.local
@@ -585,6 +588,7 @@ class Output:
def index_key(self) -> tuple[str, "DataIndexKey"]:
if self.is_in_repo:
workspace = "repo"
assert self.repo
key = self.repo.fs.relparts(self.fs_path, self.repo.root_dir)
else:
workspace = self.fs.protocol
@@ -640,6 +644,7 @@ class Output:
@property
def dvcignore(self) -> Optional["DvcIgnoreFilter"]:
if self.fs.protocol == "local":
assert self.repo
return self.repo.dvcignore
return None
@@ -661,6 +666,7 @@ class Output:
if not self.use_scm_ignore:
return
assert self.repo
if self.repo.scm.is_tracked(self.fspath):
raise OutputAlreadyTrackedError(self)
@@ -670,6 +676,7 @@ class Output:
if not self.use_scm_ignore:
return
assert self.repo
self.repo.scm_context.ignore_remove(self.fspath)
def save(self) -> None:
@@ -776,6 +783,7 @@ class Output:
callback=cb,
)
if relink:
assert self.repo
rel = self.fs.relpath(filter_info or self.fs_path)
with CheckoutCallback(desc=f"Checking out {rel}", unit="files") as cb:
self._checkout(
@@ -957,6 +965,7 @@ class Output:
added = not self.exists
try:
assert self.repo
modified = self._checkout(
filter_info or self.fs_path,
self.fs,
@@ -987,18 +996,24 @@ class Output:
if ignore_remove:
self.ignore_remove()
def move(self, out):
def move(self, out: "Output") -> None:
src_exists = self.exists
if src_exists:
self.fs.move(self.fs_path, out.fs_path)
else:
logger.warning("%r missing", self.fspath)
if self.protocol == "local" and self.use_scm_ignore:
assert self.repo
self.repo.scm_context.ignore_remove(self.fspath)
self.fs.move(self.fs_path, out.fs_path)
self.def_path = out.def_path
self.fs_path = out.fs_path
self.save()
self.commit()
if self.protocol == "local" and self.use_scm_ignore:
self.repo.scm_context.ignore(self.fspath)
try:
self.save()
self.commit()
except self.DoesNotExistError:
self.ignore()
def transfer(
self, source, odb=None, jobs=None, update=False, no_progress_bar=False
@@ -1079,6 +1094,7 @@ class Output:
if self.remote:
kwargs["remote"] = self.remote
with suppress(Exception):
assert self.repo
self.repo.cloud.pull([obj.hash_info], **kwargs)
if self.obj:
@@ -1172,6 +1188,7 @@ class Output:
return {}
if self.remote:
assert self.repo
remote_odb = self.repo.cloud.get_remote_odb(
name=self.remote, hash_name=self.hash_name
)
-1
View File
@@ -2,7 +2,6 @@ import os
from collections import defaultdict
from typing import TYPE_CHECKING, NamedTuple, Optional
import dpath
import dpath.options
from funcy import get_in, last
+4 -1
View File
@@ -20,6 +20,7 @@ if TYPE_CHECKING:
from dvc.fs.data import DataFileSystem
from dvc.fs.dvc import DVCFileSystem
from dvc.lock import LockBase
from dvc.output import Output
from dvc.scm import Git, NoSCM
from dvc.stage import Stage
from dvc.types import DictStrAny
@@ -544,7 +545,9 @@ class Repo:
return used
def find_outs_by_path(self, path, outs=None, recursive=False, strict=True):
def find_outs_by_path(
self, path, outs=None, recursive=False, strict=True
) -> list["Output"]:
# using `outs_graph` to ensure graph checks are run
outs = outs or self.index.outs_graph
+22 -2
View File
@@ -1,6 +1,6 @@
import os
import posixpath
from collections import defaultdict
from collections import defaultdict, deque
from collections.abc import Iterable, Iterator, Mapping
from typing import TYPE_CHECKING, Optional, TypedDict, Union
@@ -385,6 +385,26 @@ def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> list[str]
return [repo.fs.relpath(file, start) for file in files]
def iter_index(
index: Union["BaseDataIndex", "DataIndexView"], shallow: bool = False
) -> Iterator[tuple["DataIndexKey", Optional["DataIndexEntry"]]]:
if not shallow:
yield from index.iteritems(shallow=shallow)
return
# only iterate until we find entries with hash_info in shallow mode
todo: deque[tuple[DataIndexKey, dict]] = deque([((), index.info(()))])
while todo:
key, info = todo.popleft()
entry = info.get("entry")
if info.get("type") == "directory" and not (entry and entry.hash_info):
try:
todo.extend(index.ls(key, detail=True))
except (KeyError, DataIndexDirError):
pass
yield key, entry
def _get_entries_not_in_remote(
repo: "Repo",
filter_keys: Optional[Iterable["DataIndexKey"]] = None,
@@ -418,7 +438,7 @@ def _get_entries_not_in_remote(
n = 0
with TqdmCallback(size=n, desc="Checking remote", unit="entry") as cb:
for key, entry in view.iteritems(shallow=not granular):
for key, entry in iter_index(view, shallow=not granular):
if not (entry and entry.hash_info):
continue
+15 -3
View File
@@ -9,6 +9,7 @@ from enum import IntEnum
from itertools import chain
from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, Union
import funcy
from funcy import nullcontext
from scmrepo.exceptions import SCMError
@@ -815,11 +816,22 @@ class BaseExecutor(ABC):
@contextmanager
def set_temp_refs(self, scm: "Git", temp_dict: dict[str, str]):
# Retry ref set, get, and remove operations to handle transient issues during
# concurrent Git access.
# Dulwich deletes parent directories of refs if they happen to be empty after
# removing a ref, which can interfere with `set_ref` in other processes.
# `remove_ref` may also fail with a `FileLocked` error when refs are packed,
# since multiple processes might attempt to write to the same file.
retry = funcy.retry(10, errors=Exception, timeout=0.1)
set_ref = retry(scm.set_ref)
get_ref = retry(scm.get_ref)
remove_ref = retry(scm.remove_ref)
try:
for ref, rev in temp_dict.items():
scm.set_ref(ref, rev)
set_ref(ref, rev)
yield
finally:
for ref in temp_dict:
if scm.get_ref(ref):
scm.remove_ref(ref)
if get_ref(ref):
remove_ref(ref)
+41 -17
View File
@@ -1,10 +1,13 @@
import os
from typing import TYPE_CHECKING
from dvc.exceptions import MoveNotDataSourceError
from dvc.repo.scm_context import scm_context
from . import locked
if TYPE_CHECKING:
from . import Repo
def _expand_target_path(from_path, to_path):
if os.path.isdir(to_path):
@@ -14,7 +17,7 @@ def _expand_target_path(from_path, to_path):
@locked
@scm_context
def move(self, from_path, to_path):
def move(self: "Repo", from_path, to_path):
"""
Renames an output file and modifies the stage associated
to reflect the change on the pipeline.
@@ -32,9 +35,12 @@ def move(self, from_path, to_path):
It only works with outputs generated by `add` or `import`,
also known as data sources.
"""
from dvc import output
from dvc import dependency, output
from dvc.dvcfile import DVC_FILE_SUFFIX
from dvc.exceptions import MoveNotDataSourceError
from dvc.stage import Stage
from dvc.stage.exceptions import StageFileAlreadyExistsError
from dvc_objects.fs.local import LocalFileSystem
from_out = output.loads_from(Stage(self), [from_path])[0]
assert from_out.protocol == "local"
@@ -45,6 +51,7 @@ def move(self, from_path, to_path):
assert len(outs) == 1
out = outs[0]
stage = out.stage
deps = stage.deps
if not stage.is_data_source:
raise MoveNotDataSourceError(stage.addressing)
@@ -58,21 +65,38 @@ def move(self, from_path, to_path):
)
new_wdir = os.path.abspath(os.path.join(os.curdir, os.path.dirname(to_path)))
to_path = os.path.relpath(to_path, new_wdir)
new_stage = self.stage.create(
single_stage=True,
fname=new_fname,
wdir=new_wdir,
outs=[to_path],
meta=stage.meta,
)
os.unlink(stage.path)
stage = new_stage
try:
new_stage = self.stage.create(
single_stage=True,
fname=new_fname,
wdir=new_wdir,
outs=[to_path],
meta=stage.meta,
frozen=stage.frozen,
always_changed=stage.always_changed,
desc=stage.desc,
)
except StageFileAlreadyExistsError:
# reraise to remove `--force` hint
raise StageFileAlreadyExistsError(f"{new_fname!r} already exists") from None
else:
new_stage = stage
to_path = os.path.relpath(to_path, stage.wdir)
to_out = output.loads_from(stage, [to_path], out.use_cache, out.metric)[0]
def with_dep_path_adjusted(dep: dependency.Dependency):
d = dep.dumpd()
if isinstance(dep.fs, LocalFileSystem) and not os.path.isabs(dep.def_path):
return d | {"path": os.path.relpath(dep.fspath, new_stage.wdir)}
return d
out.move(to_out)
stage.save()
stage.dump()
new_stage.outs = output.loadd_from(new_stage, [out.dumpd() | {"path": to_path}])
new_stage.deps = dependency.loadd_from(
new_stage, [with_dep_path_adjusted(dep) for dep in deps]
)
out.move(new_stage.outs[0])
new_stage.md5 = new_stage.compute_md5()
new_stage.dump()
if stage != new_stage:
stage.dvcfile.remove()
self.scm_context.track_file(stage.dvcfile.relpath)
return stage, new_stage
+3 -3
View File
@@ -32,7 +32,7 @@ from .utils import (
)
if TYPE_CHECKING:
from dvc.dependency import ParamsDependency
from dvc.dependency import Dependency, ParamsDependency
from dvc.dvcfile import ProjectFile, SingleStageFile
from dvc.output import Output
from dvc.repo import Repo
@@ -153,8 +153,8 @@ class Stage(params.StageParams):
self._path = path
self.cmd = cmd
self.wdir = wdir
self.outs = outs
self.deps = deps
self.outs: list[Output] = outs
self.deps: list[Dependency] = deps
self.md5 = md5
self.frozen = locked or frozen
self.always_changed = always_changed
+1 -1
View File
@@ -178,7 +178,7 @@ def to_single_stage_lockfile(stage: "Stage", **kwargs) -> dict:
res = OrderedDict([("cmd", stage.cmd)])
params, deps = split_params_deps(stage)
deps, outs = (
[_dumpd(item) for item in sorted(items, key=attrgetter("def_path"))]
[_dumpd(item) for item in sorted(items, key=attrgetter("def_path"))] # type: ignore[call-overload]
for items in [deps, stage.outs]
)
params = _serialize_params_values(params)
+1 -1
View File
@@ -158,7 +158,7 @@ def make_bench(request):
# pytest >= 8.4.0
wrapped_func = fixture_function._get_wrapped_function()
except AttributeError:
wrapped_func = fixture_function.__pytest_wrapped__.obj # ty: ignore[unresolved-attribute]
wrapped_func = fixture_function.__pytest_wrapped__.obj # type: ignore[attr-defined] # ty: ignore[unresolved-attribute]
assert inspect.isgeneratorfunction(wrapped_func)
generator = wrapped_func(request)
+1
View File
@@ -29,6 +29,7 @@ classifiers = [
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: 3.14",
]
dynamic = ["version"]
+1
View File
@@ -194,6 +194,7 @@ def isolate(tmp_path_factory):
monkeypatch.setenv(env_var, os.fspath(path))
else:
monkeypatch.setenv("HOME", str(home_dir))
monkeypatch.setenv("XDG_CONFIG_HOME", str(home_dir / ".config"))
monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1")
contents = b"""
+37
View File
@@ -1211,3 +1211,40 @@ def test_renames(
dvc.data_status(granular=True, with_renames=True, targets=targets)
== EMPTY_STATUS | {"git": M.dict()} | expected_granular
)
def test_shallow_should_iterate_upto_tracked_directory(
M, tmp_dir, dvc, scm, local_remote
):
"""Testing regression for https://github.com/iterative/dvc/issues/10899."""
tmp_dir.scm_gen({"dir": {".gitkeep": ""}}, commit="mk dir")
dataset_root = join("dir", "data", "")
tmp_dir.dvc_gen({dataset_root: {"foo": "foo", "bar": "bar"}})
files = [dataset_root, join(dataset_root, "foo"), join(dataset_root, "bar")]
assert dvc.data_status(not_in_remote=True) == {
**EMPTY_STATUS,
"committed": {"added": [dataset_root]},
"not_in_remote": [dataset_root],
"git": M.dict(),
}
assert dvc.data_status(granular=True, not_in_remote=True) == {
**EMPTY_STATUS,
"committed": {"added": M.unordered(*files)},
"not_in_remote": M.unordered(*files),
"git": M.dict(),
}
assert dvc.push() == 3
assert dvc.data_status(not_in_remote=True, remote_refresh=True) == {
**EMPTY_STATUS,
"committed": {"added": [dataset_root]},
"git": M.dict(),
}
assert dvc.data_status(granular=True, not_in_remote=True, remote_refresh=True) == {
**EMPTY_STATUS,
"committed": {"added": M.unordered(*files)},
"git": M.dict(),
}
+201 -4
View File
@@ -6,14 +6,20 @@ import pytest
from dvc.cli import main
from dvc.exceptions import MoveNotDataSourceError, OutputNotFoundError
from dvc.stage.exceptions import StageFileAlreadyExistsError
def test_move(tmp_dir, dvc):
tmp_dir.dvc_gen("foo", "foo")
dvc.move("foo", "foo1")
def test_move(tmp_dir, dvc, scm):
tmp_dir.dvc_gen("foo", "bar")
assert (tmp_dir / "foo.dvc").exists()
dvc.move("foo", "bar")
assert not (tmp_dir / "foo.dvc").exists()
assert (tmp_dir / "bar.dvc").exists()
assert not (tmp_dir / "foo").is_file()
assert (tmp_dir / "foo1").is_file()
assert (tmp_dir / "bar").is_file()
# should only have the new path in the .gitignore, and only once
assert (tmp_dir / ".gitignore").read_text().splitlines() == ["/bar"]
def test_move_non_existent_file(dvc):
@@ -21,6 +27,24 @@ def test_move_non_existent_file(dvc):
dvc.move("non_existent_file", "dst")
def test_move_missing_file(tmp_dir, dvc, scm, caplog):
tmp_dir.dvc_gen("foo", "foo")
(tmp_dir / "foo").unlink()
contents = (tmp_dir / "foo.dvc").parse()
dvc.move("foo", "bar")
assert not (tmp_dir / "foo.dvc").exists()
# only the path should be changed in the dvc file
contents["outs"][0]["path"] = "bar"
assert contents == (tmp_dir / "bar.dvc").parse()
# file should not be checked out
assert not (tmp_dir / "foo").is_file()
assert not (tmp_dir / "bar").is_file()
# should only have the new path in the .gitignore, and only once
assert (tmp_dir / ".gitignore").read_text().splitlines() == ["/bar"]
def test_move_directory(tmp_dir, dvc):
tmp_dir.dvc_gen("data", {"foo": "foo", "bar": "bar"})
dvc.move("data", "dst")
@@ -195,3 +219,176 @@ def test_move_meta(tmp_dir, dvc):
custom_key: 42
"""
)
def test_import(tmp_dir, dvc, scm):
tmp_dir.dvc_gen("foo", "foo", commit="add foo")
imp_stage = dvc.imp(os.curdir, "foo", "foo_imported")
dvc.move("foo_imported", "foo_moved")
(stage,) = dvc.stage.collect("foo_moved.dvc")
assert imp_stage.md5 != stage.md5
res = (tmp_dir / "foo_moved.dvc").read_text()
assert res == textwrap.dedent(
f"""\
md5: {stage.md5}
frozen: true
deps:
- path: foo
repo:
url: {os.curdir}
rev_lock: {scm.get_rev()}
outs:
- md5: acbd18db4cc2f85cedef654fccc4a4d8
size: 3
hash: md5
path: foo_moved
"""
)
@pytest.mark.parametrize(
"path_func",
[pytest.param(os.path.abspath, id="abs"), pytest.param(os.path.relpath, id="rel")],
)
def test_import_url_in_repo(tmp_dir, dvc, path_func):
tmp_dir.gen("foo", "foo")
imp_stage = dvc.imp_url(path_func(tmp_dir / "foo"), "foo_imported")
(tmp_dir / "data").mkdir()
dvc.move("foo_imported", os.path.join("data", "foo_moved"))
(stage,) = dvc.stage.collect(os.path.join("data", "foo_moved.dvc"))
assert imp_stage.md5 != stage.md5
res = (tmp_dir / "data" / "foo_moved.dvc").read_text()
assert res == textwrap.dedent(
f"""\
md5: {stage.md5}
frozen: true
deps:
- md5: acbd18db4cc2f85cedef654fccc4a4d8
size: 3
hash: md5
path: ../foo
outs:
- md5: acbd18db4cc2f85cedef654fccc4a4d8
size: 3
hash: md5
path: foo_moved
"""
)
@pytest.mark.parametrize(
"path_func",
[pytest.param(os.path.abspath, id="abs"), pytest.param(os.path.relpath, id="rel")],
)
def test_import_url_out_of_repo(tmp_dir, dvc, scm, path_func, make_tmp_dir):
external = make_tmp_dir("external")
external.gen("foo", "foo")
imp_stage = dvc.imp_url(path_func(external / "foo"), "foo_imported")
data_dir = tmp_dir / "data"
data_dir.mkdir()
new_path = data_dir / "foo_moved"
new_dvcfile = new_path.with_suffix(".dvc")
dvc.move("foo_imported", os.fspath(new_path))
(stage,) = dvc.stage.collect(os.fspath(new_dvcfile))
assert imp_stage.md5 != stage.md5
with data_dir.chdir():
expected_path = path_func(external / "foo")
assert new_dvcfile.parse() == {
"md5": stage.md5,
"frozen": True,
"deps": [
{
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
"size": 3,
"hash": "md5",
"path": expected_path,
}
],
"outs": [
{
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
"size": 3,
"hash": "md5",
"path": "foo_moved",
}
],
}
@pytest.mark.parametrize(
"path_func",
[pytest.param(os.path.abspath, id="abs"), pytest.param(os.path.relpath, id="rel")],
)
def test_all_metadata_are_preserved(tmp_dir, dvc, make_tmp_dir, path_func):
external = make_tmp_dir("external")
external.gen("foo", "foo")
contents = {
"md5": "bad", # placeholder, does not matter for the test
"frozen": True,
"desc": "this is a stage description",
"always_changed": True,
"meta": {"custom_key": 42},
"deps": [
{
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
"size": 3,
"hash": "md5",
"path": path_func(external / "foo"),
}
],
"outs": [
{
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
"path": "foo_imported",
"persist": True,
"hash": "md5",
"size": 3,
"desc": "this is a description",
"type": "model",
"labels": ["label1", "label2"],
"meta": {"custom_key": 42},
"cache": False,
"remote": "myremote",
"push": False,
}
],
}
(tmp_dir / "foo_imported.dvc").dump(contents)
(tmp_dir / "foo_imported").write_text("foo")
data_dir = tmp_dir / "data"
data_dir.mkdir()
new_path = data_dir / "foo_moved"
new_dvcfile = new_path.with_suffix(".dvc")
dvc.move("foo_imported", os.fspath(new_path))
(stage,) = dvc.stage.collect(os.fspath(new_dvcfile))
with data_dir.chdir():
expected_path = path_func(external / "foo")
contents["outs"][0] |= {"path": "foo_moved"}
contents["deps"][0] |= {"path": expected_path}
contents |= {"md5": stage.md5}
assert new_dvcfile.parse() == contents
def test_move_dst_stage_file_already_exists(tmp_dir, dvc):
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
with pytest.raises(StageFileAlreadyExistsError) as exc_info:
dvc.move("foo", "bar")
assert str(exc_info.value) == "'bar.dvc' already exists"
assert exc_info.value.__cause__ is None