mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 23:09:28 +01:00
Compare commits
16
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
72abd013cf | ||
|
|
fa5c33ed24 | ||
|
|
9dda30b7a0 | ||
|
|
19c146e392 | ||
|
|
749c22bbcb | ||
|
|
19a0bf48f1 | ||
|
|
bda614baa8 | ||
|
|
d6fa8b67ff | ||
|
|
498a4d5d3a | ||
|
|
b74711d5fe | ||
|
|
61d5de54e5 | ||
|
|
ca7454980d | ||
|
|
8912ca7d2b | ||
|
|
f13ae75a45 | ||
|
|
aa090ecf8d | ||
|
|
0c4a5cbbb7 |
@@ -16,14 +16,14 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.13"
|
||||
python-version: "3.14"
|
||||
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: pyproject.toml
|
||||
|
||||
@@ -25,9 +25,9 @@ jobs:
|
||||
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: 3.13
|
||||
python-version: "3.14"
|
||||
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
|
||||
- name: Omit local version for Test PyPI upload
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
- name: Check dist
|
||||
run: uv tool run twine check --strict dist/*
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
- uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: Packages
|
||||
path: dist/
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- uses: actions/download-artifact@v5
|
||||
- uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: Packages
|
||||
path: dist
|
||||
@@ -86,7 +86,7 @@ jobs:
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- uses: actions/download-artifact@v5
|
||||
- uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: Packages
|
||||
path: dist
|
||||
|
||||
@@ -31,10 +31,10 @@ jobs:
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v3
|
||||
uses: github/codeql-action/init@v4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
queries: security-extended
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v3
|
||||
uses: github/codeql-action/analyze@v4
|
||||
|
||||
@@ -19,7 +19,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest]
|
||||
pyv: ["3.13"]
|
||||
pyv: ["3.14"]
|
||||
plugin: ["dvc-s3"]
|
||||
|
||||
steps:
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
with:
|
||||
python-version: ${{ matrix.pyv }}
|
||||
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: "**/pyproject.toml"
|
||||
|
||||
@@ -32,8 +32,8 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.13'
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
python-version: '3.14'
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: pyproject.toml
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest]
|
||||
pyv: ["3.9", "3.10", "3.11", "3.12", "3.13"]
|
||||
pyv: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
|
||||
include:
|
||||
- os: windows-latest
|
||||
pyv: "3.9"
|
||||
@@ -117,6 +117,18 @@ jobs:
|
||||
- os: windows-latest
|
||||
pyv: "3.13"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
|
||||
- os: windows-latest
|
||||
pyv: "3.14"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 1"
|
||||
- os: windows-latest
|
||||
pyv: "3.14"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 2"
|
||||
- os: windows-latest
|
||||
pyv: "3.14"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 3"
|
||||
- os: windows-latest
|
||||
pyv: "3.14"
|
||||
pytestargs: "--splitting-algorithm=least_duration --splits 4 --group 4"
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
@@ -125,7 +137,7 @@ jobs:
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.pyv }}
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: pyproject.toml
|
||||
|
||||
@@ -22,7 +22,7 @@ repos:
|
||||
- id: sort-simple-yaml
|
||||
- id: trailing-whitespace
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: 'v0.13.2'
|
||||
rev: 'v0.14.4'
|
||||
hooks:
|
||||
- id: ruff-check
|
||||
args: [--fix, --exit-non-zero-on-fix]
|
||||
|
||||
@@ -41,7 +41,7 @@ def _get(stage, p, info, **kwargs):
|
||||
return Dependency(stage, p, info, **kwargs)
|
||||
|
||||
|
||||
def loadd_from(stage, d_list):
|
||||
def loadd_from(stage, d_list) -> list[Dependency]:
|
||||
ret = []
|
||||
for d in d_list:
|
||||
p = d.pop(Output.PARAM_PATH, None)
|
||||
|
||||
@@ -110,6 +110,7 @@ class ParamsDependency(Dependency):
|
||||
except MissingParamsFile:
|
||||
return {}
|
||||
|
||||
assert self.repo
|
||||
try:
|
||||
return read_param_file(
|
||||
self.repo.fs,
|
||||
|
||||
@@ -150,6 +150,7 @@ class RepoDependency(Dependency):
|
||||
# local cache instead of the remote's cache. This avoids re-streaming of data,
|
||||
# but messes up the call to `_get_remote_config()` downstream, which will need
|
||||
# to ignore cache parameters.
|
||||
assert self.repo
|
||||
config["cache"] = self.repo.config["cache"]
|
||||
config["cache"]["dir"] = self.repo.cache.local_cache_dir
|
||||
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
"""Manages logging configuration for DVC repo."""
|
||||
|
||||
import logging
|
||||
import logging.config
|
||||
import logging.handlers
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Iterator
|
||||
|
||||
+26
-9
@@ -42,6 +42,7 @@ from .utils import relpath
|
||||
from .utils.fs import path_isin
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.repo import Repo
|
||||
from dvc_data.hashfile.obj import HashFile
|
||||
from dvc_data.index import DataIndexKey
|
||||
|
||||
@@ -83,7 +84,7 @@ def _get(stage, path, **kwargs):
|
||||
return Output(stage, path, **kwargs)
|
||||
|
||||
|
||||
def loadd_from(stage, d_list):
|
||||
def loadd_from(stage, d_list) -> list["Output"]:
|
||||
ret = []
|
||||
for d in d_list:
|
||||
p = d.pop(Output.PARAM_PATH)
|
||||
@@ -329,7 +330,7 @@ class Output:
|
||||
self.annot = Annotation(
|
||||
desc=desc, type=type, labels=labels or [], meta=meta or {}
|
||||
)
|
||||
self.repo = stage.repo if not repo and stage else repo
|
||||
self.repo: Optional[Repo] = stage.repo if not repo and stage else repo
|
||||
meta_d = merge_file_meta_from_cloud(info or {})
|
||||
meta = Meta.from_dict(meta_d)
|
||||
# NOTE: when version_aware is not passed into get_cloud_fs, it will be
|
||||
@@ -511,6 +512,7 @@ class Output:
|
||||
from dvc.cachemgr import LEGACY_HASH_NAMES
|
||||
|
||||
assert self.is_in_repo
|
||||
assert self.repo
|
||||
odb_name = "legacy" if self.hash_name in LEGACY_HASH_NAMES else "repo"
|
||||
return getattr(self.repo.cache, odb_name)
|
||||
|
||||
@@ -518,6 +520,7 @@ class Output:
|
||||
def local_cache(self):
|
||||
from dvc.cachemgr import LEGACY_HASH_NAMES
|
||||
|
||||
assert self.repo
|
||||
if self.hash_name in LEGACY_HASH_NAMES:
|
||||
return self.repo.cache.legacy
|
||||
return self.repo.cache.local
|
||||
@@ -585,6 +588,7 @@ class Output:
|
||||
def index_key(self) -> tuple[str, "DataIndexKey"]:
|
||||
if self.is_in_repo:
|
||||
workspace = "repo"
|
||||
assert self.repo
|
||||
key = self.repo.fs.relparts(self.fs_path, self.repo.root_dir)
|
||||
else:
|
||||
workspace = self.fs.protocol
|
||||
@@ -640,6 +644,7 @@ class Output:
|
||||
@property
|
||||
def dvcignore(self) -> Optional["DvcIgnoreFilter"]:
|
||||
if self.fs.protocol == "local":
|
||||
assert self.repo
|
||||
return self.repo.dvcignore
|
||||
return None
|
||||
|
||||
@@ -661,6 +666,7 @@ class Output:
|
||||
if not self.use_scm_ignore:
|
||||
return
|
||||
|
||||
assert self.repo
|
||||
if self.repo.scm.is_tracked(self.fspath):
|
||||
raise OutputAlreadyTrackedError(self)
|
||||
|
||||
@@ -670,6 +676,7 @@ class Output:
|
||||
if not self.use_scm_ignore:
|
||||
return
|
||||
|
||||
assert self.repo
|
||||
self.repo.scm_context.ignore_remove(self.fspath)
|
||||
|
||||
def save(self) -> None:
|
||||
@@ -776,6 +783,7 @@ class Output:
|
||||
callback=cb,
|
||||
)
|
||||
if relink:
|
||||
assert self.repo
|
||||
rel = self.fs.relpath(filter_info or self.fs_path)
|
||||
with CheckoutCallback(desc=f"Checking out {rel}", unit="files") as cb:
|
||||
self._checkout(
|
||||
@@ -957,6 +965,7 @@ class Output:
|
||||
added = not self.exists
|
||||
|
||||
try:
|
||||
assert self.repo
|
||||
modified = self._checkout(
|
||||
filter_info or self.fs_path,
|
||||
self.fs,
|
||||
@@ -987,18 +996,24 @@ class Output:
|
||||
if ignore_remove:
|
||||
self.ignore_remove()
|
||||
|
||||
def move(self, out):
|
||||
def move(self, out: "Output") -> None:
|
||||
src_exists = self.exists
|
||||
if src_exists:
|
||||
self.fs.move(self.fs_path, out.fs_path)
|
||||
else:
|
||||
logger.warning("%r missing", self.fspath)
|
||||
|
||||
if self.protocol == "local" and self.use_scm_ignore:
|
||||
assert self.repo
|
||||
self.repo.scm_context.ignore_remove(self.fspath)
|
||||
|
||||
self.fs.move(self.fs_path, out.fs_path)
|
||||
self.def_path = out.def_path
|
||||
self.fs_path = out.fs_path
|
||||
self.save()
|
||||
self.commit()
|
||||
|
||||
if self.protocol == "local" and self.use_scm_ignore:
|
||||
self.repo.scm_context.ignore(self.fspath)
|
||||
try:
|
||||
self.save()
|
||||
self.commit()
|
||||
except self.DoesNotExistError:
|
||||
self.ignore()
|
||||
|
||||
def transfer(
|
||||
self, source, odb=None, jobs=None, update=False, no_progress_bar=False
|
||||
@@ -1079,6 +1094,7 @@ class Output:
|
||||
if self.remote:
|
||||
kwargs["remote"] = self.remote
|
||||
with suppress(Exception):
|
||||
assert self.repo
|
||||
self.repo.cloud.pull([obj.hash_info], **kwargs)
|
||||
|
||||
if self.obj:
|
||||
@@ -1172,6 +1188,7 @@ class Output:
|
||||
return {}
|
||||
|
||||
if self.remote:
|
||||
assert self.repo
|
||||
remote_odb = self.repo.cloud.get_remote_odb(
|
||||
name=self.remote, hash_name=self.hash_name
|
||||
)
|
||||
|
||||
@@ -2,7 +2,6 @@ import os
|
||||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING, NamedTuple, Optional
|
||||
|
||||
import dpath
|
||||
import dpath.options
|
||||
from funcy import get_in, last
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ if TYPE_CHECKING:
|
||||
from dvc.fs.data import DataFileSystem
|
||||
from dvc.fs.dvc import DVCFileSystem
|
||||
from dvc.lock import LockBase
|
||||
from dvc.output import Output
|
||||
from dvc.scm import Git, NoSCM
|
||||
from dvc.stage import Stage
|
||||
from dvc.types import DictStrAny
|
||||
@@ -544,7 +545,9 @@ class Repo:
|
||||
|
||||
return used
|
||||
|
||||
def find_outs_by_path(self, path, outs=None, recursive=False, strict=True):
|
||||
def find_outs_by_path(
|
||||
self, path, outs=None, recursive=False, strict=True
|
||||
) -> list["Output"]:
|
||||
# using `outs_graph` to ensure graph checks are run
|
||||
outs = outs or self.index.outs_graph
|
||||
|
||||
|
||||
+22
-2
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
import posixpath
|
||||
from collections import defaultdict
|
||||
from collections import defaultdict, deque
|
||||
from collections.abc import Iterable, Iterator, Mapping
|
||||
from typing import TYPE_CHECKING, Optional, TypedDict, Union
|
||||
|
||||
@@ -385,6 +385,26 @@ def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> list[str]
|
||||
return [repo.fs.relpath(file, start) for file in files]
|
||||
|
||||
|
||||
def iter_index(
|
||||
index: Union["BaseDataIndex", "DataIndexView"], shallow: bool = False
|
||||
) -> Iterator[tuple["DataIndexKey", Optional["DataIndexEntry"]]]:
|
||||
if not shallow:
|
||||
yield from index.iteritems(shallow=shallow)
|
||||
return
|
||||
|
||||
# only iterate until we find entries with hash_info in shallow mode
|
||||
todo: deque[tuple[DataIndexKey, dict]] = deque([((), index.info(()))])
|
||||
while todo:
|
||||
key, info = todo.popleft()
|
||||
entry = info.get("entry")
|
||||
if info.get("type") == "directory" and not (entry and entry.hash_info):
|
||||
try:
|
||||
todo.extend(index.ls(key, detail=True))
|
||||
except (KeyError, DataIndexDirError):
|
||||
pass
|
||||
yield key, entry
|
||||
|
||||
|
||||
def _get_entries_not_in_remote(
|
||||
repo: "Repo",
|
||||
filter_keys: Optional[Iterable["DataIndexKey"]] = None,
|
||||
@@ -418,7 +438,7 @@ def _get_entries_not_in_remote(
|
||||
|
||||
n = 0
|
||||
with TqdmCallback(size=n, desc="Checking remote", unit="entry") as cb:
|
||||
for key, entry in view.iteritems(shallow=not granular):
|
||||
for key, entry in iter_index(view, shallow=not granular):
|
||||
if not (entry and entry.hash_info):
|
||||
continue
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ from enum import IntEnum
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, Union
|
||||
|
||||
import funcy
|
||||
from funcy import nullcontext
|
||||
from scmrepo.exceptions import SCMError
|
||||
|
||||
@@ -815,11 +816,22 @@ class BaseExecutor(ABC):
|
||||
|
||||
@contextmanager
|
||||
def set_temp_refs(self, scm: "Git", temp_dict: dict[str, str]):
|
||||
# Retry ref set, get, and remove operations to handle transient issues during
|
||||
# concurrent Git access.
|
||||
# Dulwich deletes parent directories of refs if they happen to be empty after
|
||||
# removing a ref, which can interfere with `set_ref` in other processes.
|
||||
# `remove_ref` may also fail with a `FileLocked` error when refs are packed,
|
||||
# since multiple processes might attempt to write to the same file.
|
||||
retry = funcy.retry(10, errors=Exception, timeout=0.1)
|
||||
set_ref = retry(scm.set_ref)
|
||||
get_ref = retry(scm.get_ref)
|
||||
remove_ref = retry(scm.remove_ref)
|
||||
|
||||
try:
|
||||
for ref, rev in temp_dict.items():
|
||||
scm.set_ref(ref, rev)
|
||||
set_ref(ref, rev)
|
||||
yield
|
||||
finally:
|
||||
for ref in temp_dict:
|
||||
if scm.get_ref(ref):
|
||||
scm.remove_ref(ref)
|
||||
if get_ref(ref):
|
||||
remove_ref(ref)
|
||||
|
||||
+41
-17
@@ -1,10 +1,13 @@
|
||||
import os
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from dvc.exceptions import MoveNotDataSourceError
|
||||
from dvc.repo.scm_context import scm_context
|
||||
|
||||
from . import locked
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from . import Repo
|
||||
|
||||
|
||||
def _expand_target_path(from_path, to_path):
|
||||
if os.path.isdir(to_path):
|
||||
@@ -14,7 +17,7 @@ def _expand_target_path(from_path, to_path):
|
||||
|
||||
@locked
|
||||
@scm_context
|
||||
def move(self, from_path, to_path):
|
||||
def move(self: "Repo", from_path, to_path):
|
||||
"""
|
||||
Renames an output file and modifies the stage associated
|
||||
to reflect the change on the pipeline.
|
||||
@@ -32,9 +35,12 @@ def move(self, from_path, to_path):
|
||||
It only works with outputs generated by `add` or `import`,
|
||||
also known as data sources.
|
||||
"""
|
||||
from dvc import output
|
||||
from dvc import dependency, output
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX
|
||||
from dvc.exceptions import MoveNotDataSourceError
|
||||
from dvc.stage import Stage
|
||||
from dvc.stage.exceptions import StageFileAlreadyExistsError
|
||||
from dvc_objects.fs.local import LocalFileSystem
|
||||
|
||||
from_out = output.loads_from(Stage(self), [from_path])[0]
|
||||
assert from_out.protocol == "local"
|
||||
@@ -45,6 +51,7 @@ def move(self, from_path, to_path):
|
||||
assert len(outs) == 1
|
||||
out = outs[0]
|
||||
stage = out.stage
|
||||
deps = stage.deps
|
||||
|
||||
if not stage.is_data_source:
|
||||
raise MoveNotDataSourceError(stage.addressing)
|
||||
@@ -58,21 +65,38 @@ def move(self, from_path, to_path):
|
||||
)
|
||||
new_wdir = os.path.abspath(os.path.join(os.curdir, os.path.dirname(to_path)))
|
||||
to_path = os.path.relpath(to_path, new_wdir)
|
||||
new_stage = self.stage.create(
|
||||
single_stage=True,
|
||||
fname=new_fname,
|
||||
wdir=new_wdir,
|
||||
outs=[to_path],
|
||||
meta=stage.meta,
|
||||
)
|
||||
|
||||
os.unlink(stage.path)
|
||||
stage = new_stage
|
||||
try:
|
||||
new_stage = self.stage.create(
|
||||
single_stage=True,
|
||||
fname=new_fname,
|
||||
wdir=new_wdir,
|
||||
outs=[to_path],
|
||||
meta=stage.meta,
|
||||
frozen=stage.frozen,
|
||||
always_changed=stage.always_changed,
|
||||
desc=stage.desc,
|
||||
)
|
||||
except StageFileAlreadyExistsError:
|
||||
# reraise to remove `--force` hint
|
||||
raise StageFileAlreadyExistsError(f"{new_fname!r} already exists") from None
|
||||
else:
|
||||
new_stage = stage
|
||||
to_path = os.path.relpath(to_path, stage.wdir)
|
||||
|
||||
to_out = output.loads_from(stage, [to_path], out.use_cache, out.metric)[0]
|
||||
def with_dep_path_adjusted(dep: dependency.Dependency):
|
||||
d = dep.dumpd()
|
||||
if isinstance(dep.fs, LocalFileSystem) and not os.path.isabs(dep.def_path):
|
||||
return d | {"path": os.path.relpath(dep.fspath, new_stage.wdir)}
|
||||
return d
|
||||
|
||||
out.move(to_out)
|
||||
stage.save()
|
||||
stage.dump()
|
||||
new_stage.outs = output.loadd_from(new_stage, [out.dumpd() | {"path": to_path}])
|
||||
new_stage.deps = dependency.loadd_from(
|
||||
new_stage, [with_dep_path_adjusted(dep) for dep in deps]
|
||||
)
|
||||
out.move(new_stage.outs[0])
|
||||
new_stage.md5 = new_stage.compute_md5()
|
||||
new_stage.dump()
|
||||
if stage != new_stage:
|
||||
stage.dvcfile.remove()
|
||||
self.scm_context.track_file(stage.dvcfile.relpath)
|
||||
return stage, new_stage
|
||||
|
||||
@@ -32,7 +32,7 @@ from .utils import (
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.dependency import ParamsDependency
|
||||
from dvc.dependency import Dependency, ParamsDependency
|
||||
from dvc.dvcfile import ProjectFile, SingleStageFile
|
||||
from dvc.output import Output
|
||||
from dvc.repo import Repo
|
||||
@@ -153,8 +153,8 @@ class Stage(params.StageParams):
|
||||
self._path = path
|
||||
self.cmd = cmd
|
||||
self.wdir = wdir
|
||||
self.outs = outs
|
||||
self.deps = deps
|
||||
self.outs: list[Output] = outs
|
||||
self.deps: list[Dependency] = deps
|
||||
self.md5 = md5
|
||||
self.frozen = locked or frozen
|
||||
self.always_changed = always_changed
|
||||
|
||||
@@ -178,7 +178,7 @@ def to_single_stage_lockfile(stage: "Stage", **kwargs) -> dict:
|
||||
res = OrderedDict([("cmd", stage.cmd)])
|
||||
params, deps = split_params_deps(stage)
|
||||
deps, outs = (
|
||||
[_dumpd(item) for item in sorted(items, key=attrgetter("def_path"))]
|
||||
[_dumpd(item) for item in sorted(items, key=attrgetter("def_path"))] # type: ignore[call-overload]
|
||||
for items in [deps, stage.outs]
|
||||
)
|
||||
params = _serialize_params_values(params)
|
||||
|
||||
@@ -158,7 +158,7 @@ def make_bench(request):
|
||||
# pytest >= 8.4.0
|
||||
wrapped_func = fixture_function._get_wrapped_function()
|
||||
except AttributeError:
|
||||
wrapped_func = fixture_function.__pytest_wrapped__.obj # ty: ignore[unresolved-attribute]
|
||||
wrapped_func = fixture_function.__pytest_wrapped__.obj # type: ignore[attr-defined] # ty: ignore[unresolved-attribute]
|
||||
assert inspect.isgeneratorfunction(wrapped_func)
|
||||
|
||||
generator = wrapped_func(request)
|
||||
|
||||
@@ -29,6 +29,7 @@ classifiers = [
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Programming Language :: Python :: 3.14",
|
||||
]
|
||||
dynamic = ["version"]
|
||||
|
||||
|
||||
@@ -194,6 +194,7 @@ def isolate(tmp_path_factory):
|
||||
monkeypatch.setenv(env_var, os.fspath(path))
|
||||
else:
|
||||
monkeypatch.setenv("HOME", str(home_dir))
|
||||
monkeypatch.setenv("XDG_CONFIG_HOME", str(home_dir / ".config"))
|
||||
|
||||
monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1")
|
||||
contents = b"""
|
||||
|
||||
@@ -1211,3 +1211,40 @@ def test_renames(
|
||||
dvc.data_status(granular=True, with_renames=True, targets=targets)
|
||||
== EMPTY_STATUS | {"git": M.dict()} | expected_granular
|
||||
)
|
||||
|
||||
|
||||
def test_shallow_should_iterate_upto_tracked_directory(
|
||||
M, tmp_dir, dvc, scm, local_remote
|
||||
):
|
||||
"""Testing regression for https://github.com/iterative/dvc/issues/10899."""
|
||||
|
||||
tmp_dir.scm_gen({"dir": {".gitkeep": ""}}, commit="mk dir")
|
||||
dataset_root = join("dir", "data", "")
|
||||
tmp_dir.dvc_gen({dataset_root: {"foo": "foo", "bar": "bar"}})
|
||||
files = [dataset_root, join(dataset_root, "foo"), join(dataset_root, "bar")]
|
||||
|
||||
assert dvc.data_status(not_in_remote=True) == {
|
||||
**EMPTY_STATUS,
|
||||
"committed": {"added": [dataset_root]},
|
||||
"not_in_remote": [dataset_root],
|
||||
"git": M.dict(),
|
||||
}
|
||||
assert dvc.data_status(granular=True, not_in_remote=True) == {
|
||||
**EMPTY_STATUS,
|
||||
"committed": {"added": M.unordered(*files)},
|
||||
"not_in_remote": M.unordered(*files),
|
||||
"git": M.dict(),
|
||||
}
|
||||
|
||||
assert dvc.push() == 3
|
||||
|
||||
assert dvc.data_status(not_in_remote=True, remote_refresh=True) == {
|
||||
**EMPTY_STATUS,
|
||||
"committed": {"added": [dataset_root]},
|
||||
"git": M.dict(),
|
||||
}
|
||||
assert dvc.data_status(granular=True, not_in_remote=True, remote_refresh=True) == {
|
||||
**EMPTY_STATUS,
|
||||
"committed": {"added": M.unordered(*files)},
|
||||
"git": M.dict(),
|
||||
}
|
||||
|
||||
+201
-4
@@ -6,14 +6,20 @@ import pytest
|
||||
|
||||
from dvc.cli import main
|
||||
from dvc.exceptions import MoveNotDataSourceError, OutputNotFoundError
|
||||
from dvc.stage.exceptions import StageFileAlreadyExistsError
|
||||
|
||||
|
||||
def test_move(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
dvc.move("foo", "foo1")
|
||||
def test_move(tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen("foo", "bar")
|
||||
assert (tmp_dir / "foo.dvc").exists()
|
||||
dvc.move("foo", "bar")
|
||||
|
||||
assert not (tmp_dir / "foo.dvc").exists()
|
||||
assert (tmp_dir / "bar.dvc").exists()
|
||||
assert not (tmp_dir / "foo").is_file()
|
||||
assert (tmp_dir / "foo1").is_file()
|
||||
assert (tmp_dir / "bar").is_file()
|
||||
# should only have the new path in the .gitignore, and only once
|
||||
assert (tmp_dir / ".gitignore").read_text().splitlines() == ["/bar"]
|
||||
|
||||
|
||||
def test_move_non_existent_file(dvc):
|
||||
@@ -21,6 +27,24 @@ def test_move_non_existent_file(dvc):
|
||||
dvc.move("non_existent_file", "dst")
|
||||
|
||||
|
||||
def test_move_missing_file(tmp_dir, dvc, scm, caplog):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
(tmp_dir / "foo").unlink()
|
||||
contents = (tmp_dir / "foo.dvc").parse()
|
||||
dvc.move("foo", "bar")
|
||||
|
||||
assert not (tmp_dir / "foo.dvc").exists()
|
||||
# only the path should be changed in the dvc file
|
||||
contents["outs"][0]["path"] = "bar"
|
||||
assert contents == (tmp_dir / "bar.dvc").parse()
|
||||
|
||||
# file should not be checked out
|
||||
assert not (tmp_dir / "foo").is_file()
|
||||
assert not (tmp_dir / "bar").is_file()
|
||||
# should only have the new path in the .gitignore, and only once
|
||||
assert (tmp_dir / ".gitignore").read_text().splitlines() == ["/bar"]
|
||||
|
||||
|
||||
def test_move_directory(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen("data", {"foo": "foo", "bar": "bar"})
|
||||
dvc.move("data", "dst")
|
||||
@@ -195,3 +219,176 @@ def test_move_meta(tmp_dir, dvc):
|
||||
custom_key: 42
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def test_import(tmp_dir, dvc, scm):
|
||||
tmp_dir.dvc_gen("foo", "foo", commit="add foo")
|
||||
imp_stage = dvc.imp(os.curdir, "foo", "foo_imported")
|
||||
|
||||
dvc.move("foo_imported", "foo_moved")
|
||||
|
||||
(stage,) = dvc.stage.collect("foo_moved.dvc")
|
||||
assert imp_stage.md5 != stage.md5
|
||||
res = (tmp_dir / "foo_moved.dvc").read_text()
|
||||
assert res == textwrap.dedent(
|
||||
f"""\
|
||||
md5: {stage.md5}
|
||||
frozen: true
|
||||
deps:
|
||||
- path: foo
|
||||
repo:
|
||||
url: {os.curdir}
|
||||
rev_lock: {scm.get_rev()}
|
||||
outs:
|
||||
- md5: acbd18db4cc2f85cedef654fccc4a4d8
|
||||
size: 3
|
||||
hash: md5
|
||||
path: foo_moved
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path_func",
|
||||
[pytest.param(os.path.abspath, id="abs"), pytest.param(os.path.relpath, id="rel")],
|
||||
)
|
||||
def test_import_url_in_repo(tmp_dir, dvc, path_func):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
imp_stage = dvc.imp_url(path_func(tmp_dir / "foo"), "foo_imported")
|
||||
(tmp_dir / "data").mkdir()
|
||||
|
||||
dvc.move("foo_imported", os.path.join("data", "foo_moved"))
|
||||
|
||||
(stage,) = dvc.stage.collect(os.path.join("data", "foo_moved.dvc"))
|
||||
assert imp_stage.md5 != stage.md5
|
||||
res = (tmp_dir / "data" / "foo_moved.dvc").read_text()
|
||||
assert res == textwrap.dedent(
|
||||
f"""\
|
||||
md5: {stage.md5}
|
||||
frozen: true
|
||||
deps:
|
||||
- md5: acbd18db4cc2f85cedef654fccc4a4d8
|
||||
size: 3
|
||||
hash: md5
|
||||
path: ../foo
|
||||
outs:
|
||||
- md5: acbd18db4cc2f85cedef654fccc4a4d8
|
||||
size: 3
|
||||
hash: md5
|
||||
path: foo_moved
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path_func",
|
||||
[pytest.param(os.path.abspath, id="abs"), pytest.param(os.path.relpath, id="rel")],
|
||||
)
|
||||
def test_import_url_out_of_repo(tmp_dir, dvc, scm, path_func, make_tmp_dir):
|
||||
external = make_tmp_dir("external")
|
||||
external.gen("foo", "foo")
|
||||
|
||||
imp_stage = dvc.imp_url(path_func(external / "foo"), "foo_imported")
|
||||
|
||||
data_dir = tmp_dir / "data"
|
||||
data_dir.mkdir()
|
||||
|
||||
new_path = data_dir / "foo_moved"
|
||||
new_dvcfile = new_path.with_suffix(".dvc")
|
||||
dvc.move("foo_imported", os.fspath(new_path))
|
||||
|
||||
(stage,) = dvc.stage.collect(os.fspath(new_dvcfile))
|
||||
assert imp_stage.md5 != stage.md5
|
||||
|
||||
with data_dir.chdir():
|
||||
expected_path = path_func(external / "foo")
|
||||
|
||||
assert new_dvcfile.parse() == {
|
||||
"md5": stage.md5,
|
||||
"frozen": True,
|
||||
"deps": [
|
||||
{
|
||||
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
|
||||
"size": 3,
|
||||
"hash": "md5",
|
||||
"path": expected_path,
|
||||
}
|
||||
],
|
||||
"outs": [
|
||||
{
|
||||
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
|
||||
"size": 3,
|
||||
"hash": "md5",
|
||||
"path": "foo_moved",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path_func",
|
||||
[pytest.param(os.path.abspath, id="abs"), pytest.param(os.path.relpath, id="rel")],
|
||||
)
|
||||
def test_all_metadata_are_preserved(tmp_dir, dvc, make_tmp_dir, path_func):
|
||||
external = make_tmp_dir("external")
|
||||
external.gen("foo", "foo")
|
||||
|
||||
contents = {
|
||||
"md5": "bad", # placeholder, does not matter for the test
|
||||
"frozen": True,
|
||||
"desc": "this is a stage description",
|
||||
"always_changed": True,
|
||||
"meta": {"custom_key": 42},
|
||||
"deps": [
|
||||
{
|
||||
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
|
||||
"size": 3,
|
||||
"hash": "md5",
|
||||
"path": path_func(external / "foo"),
|
||||
}
|
||||
],
|
||||
"outs": [
|
||||
{
|
||||
"md5": "acbd18db4cc2f85cedef654fccc4a4d8",
|
||||
"path": "foo_imported",
|
||||
"persist": True,
|
||||
"hash": "md5",
|
||||
"size": 3,
|
||||
"desc": "this is a description",
|
||||
"type": "model",
|
||||
"labels": ["label1", "label2"],
|
||||
"meta": {"custom_key": 42},
|
||||
"cache": False,
|
||||
"remote": "myremote",
|
||||
"push": False,
|
||||
}
|
||||
],
|
||||
}
|
||||
(tmp_dir / "foo_imported.dvc").dump(contents)
|
||||
(tmp_dir / "foo_imported").write_text("foo")
|
||||
|
||||
data_dir = tmp_dir / "data"
|
||||
data_dir.mkdir()
|
||||
|
||||
new_path = data_dir / "foo_moved"
|
||||
new_dvcfile = new_path.with_suffix(".dvc")
|
||||
dvc.move("foo_imported", os.fspath(new_path))
|
||||
|
||||
(stage,) = dvc.stage.collect(os.fspath(new_dvcfile))
|
||||
|
||||
with data_dir.chdir():
|
||||
expected_path = path_func(external / "foo")
|
||||
|
||||
contents["outs"][0] |= {"path": "foo_moved"}
|
||||
contents["deps"][0] |= {"path": expected_path}
|
||||
contents |= {"md5": stage.md5}
|
||||
assert new_dvcfile.parse() == contents
|
||||
|
||||
|
||||
def test_move_dst_stage_file_already_exists(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
|
||||
|
||||
with pytest.raises(StageFileAlreadyExistsError) as exc_info:
|
||||
dvc.move("foo", "bar")
|
||||
assert str(exc_info.value) == "'bar.dvc' already exists"
|
||||
assert exc_info.value.__cause__ is None
|
||||
|
||||
Reference in New Issue
Block a user