Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-10 07:18:27 +01:00
Code Issues Packages Projects Releases Wiki Activity

dvcfile: rename PipelineFile to ProjectFile, get rid of Dvcfile class, cleanups, etc. (#8812)

Browse Source 
dvcfile: rename PipelineFile to ProjectFile, get rid of Dvcfile class, cleanups etc
This commit is contained in:
skshetry
2023-01-15 12:45:10 +05:45
committed by GitHub
parent aabdddc2b7
commit da27174e02
44 changed files with 283 additions and 307 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/commands/git_hook.py
+4 -4
View File
@@ -64,15 +64,15 @@ class CmdPrePush(CmdHookBase):
class CmdMergeDriver(CmdHookBase):
def _run(self):
from dvc.dvcfile import Dvcfile
from dvc.dvcfile import load_file
from dvc.repo import Repo
dvc = Repo()
try:
ancestor = Dvcfile(dvc, self.args.ancestor, verify=False)
our = Dvcfile(dvc, self.args.our, verify=False)
their = Dvcfile(dvc, self.args.their, verify=False)
ancestor = load_file(dvc, self.args.ancestor, verify=False)
our = load_file(dvc, self.args.our, verify=False)
their = load_file(dvc, self.args.their, verify=False)
our.merge(ancestor, their, allowed=["add", "remove", "change"])
dvc/dvcfile.py
+14 -24
View File
@@ -20,7 +20,7 @@ from dvc.stage.exceptions import (
StageFileDoesNotExistError,
StageFileIsNotDvcFileError,
)
from dvc.types import AnyPath
from dvc.types import StrOrBytesPath
from dvc.utils import relpath
from dvc.utils.collections import apply_diff
from dvc.utils.objects import cached_property
@@ -36,8 +36,8 @@ _T = TypeVar("_T")
DVC_FILE = "Dvcfile"
DVC_FILE_SUFFIX = ".dvc"
PIPELINE_FILE = "dvc.yaml"
PIPELINE_LOCK = "dvc.lock"
PROJECT_FILE = "dvc.yaml"
LOCK_FILE = "dvc.lock"
class FileIsGitIgnored(DvcException):
@@ -56,7 +56,7 @@ class ParametrizedDumpError(DvcException):
def is_valid_filename(path):
return path.endswith(DVC_FILE_SUFFIX) or os.path.basename(path) in [
DVC_FILE,
PIPELINE_FILE,
PROJECT_FILE,
]
@@ -67,7 +67,7 @@ def is_dvc_file(path):
def is_lock_file(path):
return os.path.basename(path) == PIPELINE_LOCK
return os.path.basename(path) == LOCK_FILE
def is_git_ignored(repo, path):
@@ -87,7 +87,7 @@ def check_dvcfile_path(repo, path):
raise StageFileBadNameError(
"bad DVC file name '{}'. DVC files should be named "
"'{}' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
relpath(path), PIPELINE_FILE, os.path.basename(path)
relpath(path), PROJECT_FILE, os.path.basename(path)
)
)
@@ -229,7 +229,7 @@ class SingleStageFile(FileMixin):
self.dump(stage)
class PipelineFile(FileMixin):
class ProjectFile(FileMixin):
"""Abstraction for pipelines file, .yaml + .lock combined."""
from dvc.schema import COMPILED_MULTI_STAGE_SCHEMA as SCHEMA
@@ -297,7 +297,7 @@ class PipelineFile(FileMixin):
@property
def stage(self):
raise DvcException(
"PipelineFile has multiple stages. Please specify it's name."
"ProjectFile has multiple stages. Please specify it's name."
)
@cached_property
@@ -462,20 +462,10 @@ class Lockfile(FileMixin):
raise NotImplementedError
class Dvcfile:
def __new__(cls, repo: "Repo", path: AnyPath, **kwargs: Any):
assert path
assert repo
return make_dvcfile(repo, path, **kwargs)
DVCFile = Union["PipelineFile", "SingleStageFile"]
def make_dvcfile(repo: "Repo", path: AnyPath, **kwargs: Any) -> DVCFile:
_, ext = os.path.splitext(str(path))
if ext in [".yaml", ".yml"]:
return PipelineFile(repo, path, **kwargs)
# fallback to single stage file for better error messages
def load_file(
repo: "Repo", path: StrOrBytesPath, **kwargs: Any
) -> Union[ProjectFile, SingleStageFile]:
_, ext = os.path.splitext(path)
if ext in (".yaml", ".yml"):
return ProjectFile(repo, path, **kwargs)
return SingleStageFile(repo, path, **kwargs)
dvc/repo/collect.py
-3
View File
@@ -1,8 +1,6 @@
import logging
from typing import TYPE_CHECKING, Callable, Iterable, List, Tuple
from dvc.types import AnyPath
if TYPE_CHECKING:
from dvc.output import Output
from dvc.repo import Repo
@@ -12,7 +10,6 @@ logger = logging.getLogger(__name__)
FilterFn = Callable[["Output"], bool]
Outputs = List["Output"]
AnyPaths = List[AnyPath]
StrPaths = List[str]
dvc/repo/commit.py
+1 -4
View File
@@ -45,8 +45,6 @@ def commit(
allow_missing=False,
data_only=False,
):
from dvc.dvcfile import Dvcfile
stages_info = [
info
for info in self.stage.collect_granular(
@@ -66,6 +64,5 @@ def commit(
stage.commit(
filter_info=stage_info.filter_info, allow_missing=allow_missing
)
Dvcfile(self, stage.path).dump(stage, update_pipeline=False)
stage.dump(update_pipeline=False)
return [s.stage for s in stages_info]
dvc/repo/experiments/init.py
+6 -4
View File
@@ -22,8 +22,8 @@ from dvc.stage import PipelineStage
from dvc.types import OptStr
if TYPE_CHECKING:
from dvc.dvcfile import ProjectFile, SingleStageFile
from dvc.repo import Repo
from dvc.dvcfile import DVCFile
from dvc.dependency import Dependency
from dvc.ui import ui
@@ -112,7 +112,9 @@ def init_interactive(
def _check_stage_exists(
dvcfile: "DVCFile", name: str, force: bool = False
dvcfile: Union["ProjectFile", "SingleStageFile"],
name: str,
force: bool = False,
) -> None:
if not force and dvcfile.exists() and name in dvcfile.stages:
from dvc.stage.exceptions import DuplicateStageName
@@ -206,9 +208,9 @@ def init(
force: bool = False,
stream: Optional[TextIO] = None,
) -> Tuple[PipelineStage, List["Dependency"], List[str]]:
from dvc.dvcfile import make_dvcfile
from dvc.dvcfile import PROJECT_FILE, load_file
dvcfile = make_dvcfile(repo, "dvc.yaml")
dvcfile = load_file(repo, PROJECT_FILE)
_check_stage_exists(dvcfile, name, force=force)
defaults = defaults.copy() if defaults else {}
dvc/repo/imp_url.py
+2 -2
View File
@@ -33,7 +33,7 @@ def imp_url( # noqa: C901
fs_config=None,
version_aware: bool = False,
):
from dvc.dvcfile import make_dvcfile
from dvc.dvcfile import load_file
from dvc.stage import Stage, create_stage, restore_fields
out = resolve_output(url, out)
@@ -78,7 +78,7 @@ def imp_url( # noqa: C901
out_obj = stage.outs[0]
out_obj.annot.update(desc=desc, type=type, labels=labels, meta=meta)
dvcfile = make_dvcfile(self, stage.path)
dvcfile = load_file(self, stage.path)
dvcfile.remove()
try:
dvc/repo/index.py
+2 -2
View File
@@ -158,9 +158,9 @@ class Index:
@classmethod
def from_file(cls, repo: "Repo", path: str) -> "Index":
from dvc.dvcfile import make_dvcfile
from dvc.dvcfile import load_file
dvcfile = make_dvcfile(repo, path)
dvcfile = load_file(repo, path)
return cls(
repo,
stages=list(dvcfile.stages.values()),
dvc/repo/move.py
+2 -3
View File
@@ -33,7 +33,7 @@ def move(self, from_path, to_path):
also known as data sources.
"""
import dvc.output as Output
from dvc.dvcfile import DVC_FILE_SUFFIX, Dvcfile
from dvc.dvcfile import DVC_FILE_SUFFIX
from dvc.stage import Stage
from_out = Output.loads_from(Stage(self), [from_path])[0]
@@ -77,5 +77,4 @@ def move(self, from_path, to_path):
out.move(to_out)
stage.save()
Dvcfile(self, stage.path).dump(stage)
stage.dump()
dvc/repo/plots/__init__.py
+1 -4
View File
@@ -230,7 +230,6 @@ class Plots:
out.plot.pop(prop)
def modify(self, path, props=None, unset=None):
from dvc.dvcfile import Dvcfile
from dvc_render.vega_templates import get_template
props = props or {}
@@ -256,9 +255,7 @@ class Plots:
out.plot = True
out.verify_metric()
dvcfile = Dvcfile(self.repo, out.stage.path)
dvcfile.dump(out.stage, update_lock=False)
out.stage.dump(update_lock=False)
@cached_property
def templates_dir(self) -> Optional[str]:
dvc/repo/reproduce.py
+4 -11
View File
@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__)
def _reproduce_stage(stage: "Stage", **kwargs):
def _run_callback(repro_callback):
_dump_stage(stage)
stage.dump(update_pipeline=False)
_track_stage(stage)
repro_callback([stage])
@@ -45,20 +45,13 @@ def _reproduce_stage(stage: "Stage", **kwargs):
if not kwargs.get("dry", False):
track = checkpoint_func is not None
_dump_stage(stage)
stage.dump(update_pipeline=False)
if track:
_track_stage(stage)
return [stage]
def _dump_stage(stage):
from dvc.dvcfile import Dvcfile
dvcfile = Dvcfile(stage.repo, stage.path)
dvcfile.dump(stage, update_pipeline=False)
def _get_stage_files(stage: "Stage") -> Iterator[str]:
yield stage.dvcfile.relpath
for dep in stage.deps:
@@ -102,9 +95,9 @@ def reproduce( # noqa: C901
targets = [targets]
if not all_pipelines and not targets:
from dvc.dvcfile import PIPELINE_FILE
from dvc.dvcfile import PROJECT_FILE
targets = [PIPELINE_FILE]
targets = [PROJECT_FILE]
interactive = kwargs.get("interactive", False)
if not interactive:
dvc/repo/stage.py
+10 -11
View File
@@ -20,7 +20,7 @@ if typing.TYPE_CHECKING:
from dvc.stage.loader import StageLoader
from dvc.types import OptStr
PIPELINE_FILE = "dvc.yaml"
PROJECT_FILE = "dvc.yaml"
class StageInfo(NamedTuple):
@@ -48,9 +48,9 @@ def _maybe_collect_from_dvc_yaml(
from dvc.stage.exceptions import StageNotFound
stages: StageList = []
if loader.fs.exists(PIPELINE_FILE):
if loader.fs.exists(PROJECT_FILE):
with suppress(StageNotFound):
stages = loader.load_all(PIPELINE_FILE, target, **load_kwargs)
stages = loader.load_all(PROJECT_FILE, target, **load_kwargs)
if with_deps:
return _collect_with_deps(stages, loader.repo.index.graph)
return stages
@@ -74,7 +74,7 @@ def _collect_specific_target(
# `dvc.yaml` stage name here. If it exists, then we move on.
# else, we assume it's a output name in the `collect_granular()` below
msg = "Checking if stage '%s' is in '%s'"
logger.debug(msg, target, PIPELINE_FILE)
logger.debug(msg, target, PROJECT_FILE)
if not (recursive and loader.fs.isdir(target)):
stages = _maybe_collect_from_dvc_yaml(loader, target, with_deps)
if stages:
@@ -170,7 +170,7 @@ class StageLoad:
stage_cls = Stage
path = fname or prepare_file_path(stage_data)
else:
path = PIPELINE_FILE
path = PROJECT_FILE
stage_cls = PipelineStage
stage_name = stage_data["name"]
if not (stage_name and is_valid_name(stage_name)):
@@ -214,7 +214,7 @@ class StageLoad:
if path:
return self.repo.fs.path.realpath(path)
path = PIPELINE_FILE
path = PROJECT_FILE
logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
return path
@@ -260,11 +260,11 @@ class StageLoad:
glob: if true, `name` is considered as a glob, which is
used to filter list of stages from the given `path`.
"""
from dvc.dvcfile import Dvcfile
from dvc.dvcfile import load_file
from dvc.stage.loader import SingleStageLoader, StageLoader
path = self._get_filepath(path, name)
dvcfile = Dvcfile(self.repo, path)
dvcfile = load_file(self.repo, path)
# `dvcfile.stages` is not cached
stages = dvcfile.stages # type: ignore[attr-defined]
@@ -283,11 +283,10 @@ class StageLoad:
path: if not provided, default `dvc.yaml` is assumed.
name: required for `dvc.yaml` files, ignored for `.dvc` files.
"""
from dvc.dvcfile import Dvcfile
from dvc.dvcfile import load_file
path = self._get_filepath(path, name)
dvcfile = Dvcfile(self.repo, path)
dvcfile = load_file(self.repo, path)
stages = dvcfile.stages # type: ignore[attr-defined]
return stages[name]
dvc/repo/update.py
+1 -4
View File
@@ -20,8 +20,6 @@ def update( # noqa: C901
remote=None,
jobs=None,
):
from dvc.dvcfile import Dvcfile
from .worktree import update_worktree_stages
if not targets:
@@ -61,8 +59,7 @@ def update( # noqa: C901
no_download=no_download,
jobs=jobs,
)
dvcfile = Dvcfile(self, stage.path)
dvcfile.dump(stage)
stage.dump()
if other_stage_infos:
remote_obj = self.cloud.get_remote(name=remote)
dvc/stage/__init__.py
+10 -8
View File
@@ -4,7 +4,7 @@ import string
from collections import defaultdict
from contextlib import suppress
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Dict, Optional, Set
from typing import TYPE_CHECKING, Any, Dict, Optional, Set, Union
from funcy import project
@@ -36,7 +36,7 @@ from .utils import (
)
if TYPE_CHECKING:
from dvc.dvcfile import DVCFile
from dvc.dvcfile import ProjectFile, SingleStageFile
from dvc_data.hashfile.hash_info import HashInfo
from dvc_objects.db.base import ObjectDB
@@ -173,7 +173,7 @@ class Stage(params.StageParams):
self.__dict__.pop("relpath", None)
@property
def dvcfile(self) -> "DVCFile":
def dvcfile(self) -> Union["ProjectFile", "SingleStageFile"]:
if self.path and self._dvcfile and self.path == self._dvcfile.path:
return self._dvcfile
@@ -183,13 +183,15 @@ class Stage(params.StageParams):
"and is detached from dvcfile."
)
from dvc.dvcfile import make_dvcfile
from dvc.dvcfile import load_file
self._dvcfile = make_dvcfile(self.repo, self.path)
self._dvcfile = load_file(self.repo, self.path)
return self._dvcfile
@dvcfile.setter
def dvcfile(self, dvcfile: "DVCFile") -> None:
def dvcfile(
self, dvcfile: Union["ProjectFile", "SingleStageFile"]
) -> None:
self._dvcfile = dvcfile
def __repr__(self):
@@ -785,9 +787,9 @@ class PipelineStage(Stage):
@property
def addressing(self):
from dvc.dvcfile import PIPELINE_FILE
from dvc.dvcfile import PROJECT_FILE
if self.path and self.relpath == PIPELINE_FILE:
if self.path and self.relpath == PROJECT_FILE:
return self.name
return f"{super().addressing}:{self.name}"
dvc/stage/utils.py
+2 -2
View File
@@ -256,14 +256,14 @@ def prepare_file_path(kwargs):
def check_stage_exists(
repo: "Repo", stage: Union["Stage", "PipelineStage"], path: str
):
from dvc.dvcfile import make_dvcfile
from dvc.dvcfile import load_file
from dvc.stage import PipelineStage
from dvc.stage.exceptions import (
DuplicateStageName,
StageFileAlreadyExistsError,
)
dvcfile = make_dvcfile(repo, path)
dvcfile = load_file(repo, path)
if not dvcfile.exists():
return
dvc/types.py
+4 -2
View File
@@ -1,10 +1,12 @@
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
from typing import TYPE_CHECKING, Any, AnyStr, Dict, List, Optional, Union
if TYPE_CHECKING:
from os import PathLike
StrPath = Union[str, "PathLike[str]"]
AnyPath = Union[str, StrPath]
BytesPath = Union[bytes, "PathLike[bytes]"]
GenericPath = Union[AnyStr, "PathLike[AnyStr]"]
StrOrBytesPath = Union[str, bytes, "PathLike[str]", "PathLike[bytes]"]
OptStr = Optional[str]
TargetType = Union[List[str], str]
dvc/utils/__init__.py
+3 -3
View File
@@ -333,14 +333,14 @@ def error_link(name):
def parse_target(
target: str, default: str = None, isa_glob: bool = False
) -> Tuple[Optional[str], Optional[str]]:
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK, is_valid_filename
from dvc.dvcfile import LOCK_FILE, PROJECT_FILE, is_valid_filename
from dvc.exceptions import DvcException
from dvc.parsing import JOIN
if not target:
return None, None
default = default or PIPELINE_FILE
default = default or PROJECT_FILE
if isa_glob:
path, _, glob = target.rpartition(":")
return path or default, glob or None
@@ -360,7 +360,7 @@ def parse_target(
name += f"{JOIN}{key}"
if path:
if os.path.basename(path) == PIPELINE_LOCK:
if os.path.basename(path) == LOCK_FILE:
raise DvcException(
"Did you mean: `{}`?".format(
target.replace(".lock", ".yaml", 1)
dvc/utils/serialize/_common.py
+9 -9
View File
@@ -17,36 +17,36 @@ from dvc.exceptions import DvcException
if TYPE_CHECKING:
from dvc.fs import FileSystem
from dvc.types import AnyPath
from dvc.types import StrPath
class DumperFn(Protocol):
def __call__(
self, path: "AnyPath", data: Any, fs: "FileSystem" = None
self, path: "StrPath", data: Any, fs: "FileSystem" = None
) -> Any:
...
class ModifierFn(Protocol):
def __call__(
self, path: "AnyPath", fs: "FileSystem" = None
self, path: "StrPath", fs: "FileSystem" = None
) -> ContextManager[Dict]:
...
class LoaderFn(Protocol):
def __call__(self, path: "AnyPath", fs: "FileSystem" = None) -> Any:
def __call__(self, path: "StrPath", fs: "FileSystem" = None) -> Any:
...
ReadType = Union[bytes, None, str]
ParserFn = Callable[[ReadType, "AnyPath"], dict]
ParserFn = Callable[[ReadType, "StrPath"], dict]
class ParseError(DvcException):
"""Errors while parsing files"""
def __init__(self, path: "AnyPath", message: str):
def __init__(self, path: "StrPath", message: str):
from dvc.utils import relpath
path = relpath(path)
@@ -57,12 +57,12 @@ class ParseError(DvcException):
class EncodingError(ParseError):
"""We could not read a file with the given encoding"""
def __init__(self, path: "AnyPath", encoding: str):
def __init__(self, path: "StrPath", encoding: str):
self.encoding = encoding
super().__init__(path, f"is not valid {encoding}")
def _load_data(path: "AnyPath", parser: ParserFn, fs: "FileSystem" = None):
def _load_data(path: "StrPath", parser: ParserFn, fs: "FileSystem" = None):
open_fn = fs.open if fs else open
encoding = "utf-8"
with open_fn(path, encoding=encoding) as fd:
@@ -84,7 +84,7 @@ def _dump_data(
@contextmanager
def _modify_data(
path: "AnyPath",
path: "StrPath",
parser: ParserFn,
dumper: DumperFn,
fs: "FileSystem" = None,
tests/func/experiments/test_experiments.py
+3 -3
View File
@@ -9,7 +9,7 @@ import dulwich.file
import pytest
from funcy import first
from dvc.dvcfile import PIPELINE_FILE
from dvc.dvcfile import PROJECT_FILE
from dvc.exceptions import ReproductionError
from dvc.repo.experiments.exceptions import ExperimentExistsError
from dvc.repo.experiments.queue.base import BaseStashQueue
@@ -505,7 +505,7 @@ def test_subdir(tmp_dir, scm, dvc, workspace):
scm.commit("init")
results = dvc.experiments.run(
PIPELINE_FILE, params=["foo=2"], tmp_dir=not workspace
PROJECT_FILE, params=["foo=2"], tmp_dir=not workspace
)
assert results
@@ -549,7 +549,7 @@ def test_subrepo(tmp_dir, scm, workspace):
scm.commit("init")
results = subrepo.dvc.experiments.run(
PIPELINE_FILE, params=["foo=2"], tmp_dir=not workspace
PROJECT_FILE, params=["foo=2"], tmp_dir=not workspace
)
assert results
tests/func/metrics/test_show.py
+2 -2
View File
@@ -3,7 +3,7 @@ import os
import pytest
from funcy import get_in
from dvc.dvcfile import PIPELINE_FILE
from dvc.dvcfile import PROJECT_FILE
from dvc.exceptions import OverlappingOutputPathsError
from dvc.repo import Repo
from dvc.utils.fs import remove
@@ -303,7 +303,7 @@ def test_metrics_show_overlap(
@pytest.mark.parametrize(
"file,error_path",
(
(PIPELINE_FILE, ["workspace", "error"]),
(PROJECT_FILE, ["workspace", "error"]),
("metrics.yaml", ["workspace", "data", "metrics.yaml", "error"]),
),
)
tests/func/params/test_diff.py
+4 -4
View File
@@ -113,12 +113,12 @@ def test_diff_with_unchanged(tmp_dir, scm, dvc):
def test_pipeline_tracked_params(tmp_dir, scm, dvc, run_copy):
from dvc.dvcfile import PIPELINE_FILE
from dvc.dvcfile import PROJECT_FILE
tmp_dir.gen({"foo": "foo", "params.yaml": "foo: bar\nxyz: val"})
run_copy("foo", "bar", name="copy-foo-bar", params=["foo,xyz"])
scm.add(["params.yaml", PIPELINE_FILE])
scm.add(["params.yaml", PROJECT_FILE])
scm.commit("add stage")
tmp_dir.scm_gen("params.yaml", "foo: baz\nxyz: val", commit="baz")
@@ -182,7 +182,7 @@ def test_vars_shows_on_params_diff(tmp_dir, scm, dvc):
def test_diff_targeted(tmp_dir, scm, dvc, run_copy):
from dvc.dvcfile import PIPELINE_FILE
from dvc.dvcfile import PROJECT_FILE
tmp_dir.gen(
{
@@ -198,7 +198,7 @@ def test_diff_targeted(tmp_dir, scm, dvc, run_copy):
params=["foo", "other_params.yaml:xyz"],
)
scm.add(["params.yaml", "other_params.yaml", PIPELINE_FILE])
scm.add(["params.yaml", "other_params.yaml", PROJECT_FILE])
scm.commit("add stage")
tmp_dir.scm_gen(
tests/func/params/test_show.py
+7 -7
View File
@@ -4,7 +4,7 @@ from functools import reduce
import pytest
from dvc.repo import Repo
from dvc.repo.stage import PIPELINE_FILE
from dvc.repo.stage import PROJECT_FILE
from dvc.utils.serialize import YAMLFileCorruptedError
@@ -112,7 +112,7 @@ def test_pipeline_params(tmp_dir, scm, dvc, run_copy):
{"foo": "foo", "params.yaml": "foo: bar\nxyz: val\nabc: ignore"}
)
run_copy("foo", "bar", name="copy-foo-bar", params=["foo,xyz"])
scm.add(["params.yaml", PIPELINE_FILE])
scm.add(["params.yaml", PROJECT_FILE])
scm.commit("add stage")
tmp_dir.scm_gen(
@@ -155,7 +155,7 @@ def test_show_no_repo(tmp_dir):
@pytest.mark.parametrize(
"file,error_path",
(
(PIPELINE_FILE, ["v1", "error"]),
(PROJECT_FILE, ["v1", "error"]),
("params_other.yaml", ["v1", "data", "params_other.yaml", "error"]),
),
)
@@ -171,7 +171,7 @@ def test_log_errors(tmp_dir, scm, dvc, capsys, file, error_path):
with open(tmp_dir / file, "a", encoding="utf-8") as fd:
fd.write("\nmalformed!")
scm.add([PIPELINE_FILE, "params_other.yaml"])
scm.add([PROJECT_FILE, "params_other.yaml"])
scm.commit("init")
scm.tag("v1")
@@ -211,7 +211,7 @@ def test_deps_multi_stage(tmp_dir, scm, dvc, run_copy):
run_copy("foo", "bar", name="copy-foo-bar", params=["foo"])
run_copy("foo", "bar1", name="copy-foo-bar-1", params=["xyz"])
scm.add(["params.yaml", PIPELINE_FILE])
scm.add(["params.yaml", PROJECT_FILE])
scm.commit("add stage")
assert dvc.params.show(revs=["master"], deps=True) == {
@@ -228,7 +228,7 @@ def test_deps_with_targets(tmp_dir, scm, dvc, run_copy):
run_copy("foo", "bar", name="copy-foo-bar", params=["foo"])
run_copy("foo", "bar1", name="copy-foo-bar-1", params=["xyz"])
scm.add(["params.yaml", PIPELINE_FILE])
scm.add(["params.yaml", PROJECT_FILE])
scm.commit("add stage")
assert dvc.params.show(targets=["params.yaml"], deps=True) == {
@@ -246,6 +246,6 @@ def test_deps_with_bad_target(tmp_dir, scm, dvc, run_copy):
)
run_copy("foo", "bar", name="copy-foo-bar", params=["foo"])
run_copy("foo", "bar1", name="copy-foo-bar-1", params=["xyz"])
scm.add(["params.yaml", PIPELINE_FILE])
scm.add(["params.yaml", PROJECT_FILE])
scm.commit("add stage")
assert dvc.params.show(targets=["foobar"], deps=True) == {}
tests/func/plots/test_modify.py
+3 -3
View File
@@ -1,6 +1,6 @@
import pytest
from dvc.dvcfile import PIPELINE_LOCK
from dvc.dvcfile import LOCK_FILE
from dvc.repo.plots import PropsNotFoundError
from dvc.utils import relpath
from tests.utils.plots import get_plot
@@ -37,11 +37,11 @@ def test_plots_modify_should_not_change_lockfile(
single_stage=False,
)
(tmp_dir / PIPELINE_LOCK).unlink()
(tmp_dir / LOCK_FILE).unlink()
dvc.plots.modify(
"metric.json", props={"template": relpath(custom_template)}
)
assert not (tmp_dir / PIPELINE_LOCK).exists()
assert not (tmp_dir / LOCK_FILE).exists()
def test_plots_modify_not_existing_template(dvc):
tests/func/plots/test_show.py
+2 -2
View File
@@ -3,7 +3,7 @@ import os
import pytest
from dvc.cli import main
from dvc.dvcfile import PIPELINE_FILE
from dvc.dvcfile import PROJECT_FILE
from dvc.exceptions import OverlappingOutputPathsError
from dvc.repo import Repo
from dvc.repo.plots import PlotMetricTypeError
@@ -227,7 +227,7 @@ def test_ignore_parsing_error(tmp_dir, dvc, run_copy_metrics):
@pytest.mark.parametrize(
"file,path_kwargs",
(
(PIPELINE_FILE, {"revision": "workspace", "endkey": "error"}),
(PROJECT_FILE, {"revision": "workspace", "endkey": "error"}),
(
"plot.yaml",
{"revision": "workspace", "file": "plot.yaml", "endkey": "error"},
tests/func/test_add.py
+2 -2
View File
@@ -762,13 +762,13 @@ def test_output_duplication_for_pipeline_tracked(tmp_dir, dvc, run_copy):
def test_add_pipeline_file(tmp_dir, dvc, run_copy):
from dvc.dvcfile import PIPELINE_FILE
from dvc.dvcfile import PROJECT_FILE
tmp_dir.dvc_gen("foo", "foo")
run_copy("foo", "bar", name="copy-foo-bar")
with pytest.raises(OutputIsStageFileError):
dvc.add(PIPELINE_FILE)
dvc.add(PROJECT_FILE)
def test_add_symlink_file(tmp_dir, dvc):
tests/func/test_checkout.py
+4 -4
View File
@@ -7,7 +7,7 @@ import textwrap
import pytest
from dvc.cli import main
from dvc.dvcfile import PIPELINE_FILE, Dvcfile
from dvc.dvcfile import PROJECT_FILE, load_file
from dvc.exceptions import (
CheckoutError,
CheckoutErrorSuggestGit,
@@ -496,7 +496,7 @@ def test_checkout_stats_on_failure(tmp_dir, dvc, scm):
{"foo": "foo", "dir": {"subdir": {"file": "file"}}, "other": "other"},
commit="initial",
)
stage = Dvcfile(dvc, "foo.dvc").stage
stage = load_file(dvc, "foo.dvc").stage
tmp_dir.dvc_gen({"foo": "foobar", "other": "other other"}, commit="second")
# corrupt cache
@@ -673,7 +673,7 @@ def test_checkouts_with_different_addressing(tmp_dir, dvc, run_copy):
(tmp_dir / "bar").unlink()
(tmp_dir / "ipsum").unlink()
assert set(dvc.checkout(PIPELINE_FILE)["added"]) == {"bar", "ipsum"}
assert set(dvc.checkout(PROJECT_FILE)["added"]) == {"bar", "ipsum"}
(tmp_dir / "bar").unlink()
(tmp_dir / "ipsum").unlink()
@@ -721,7 +721,7 @@ def test_checkouts_for_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
assert dvc.checkout(["bar"])["added"] == ["bar"]
(tmp_dir / "bar").unlink()
assert set(dvc.checkout([PIPELINE_FILE])["added"]) == {"bar", "ipsum"}
assert set(dvc.checkout([PROJECT_FILE])["added"]) == {"bar", "ipsum"}
for out in ["bar", "ipsum"]:
(tmp_dir / out).unlink()
tests/func/test_commit.py
+3 -3
View File
@@ -4,7 +4,7 @@ import textwrap
import pytest
from dvc.dependency.base import DependencyDoesNotExistError
from dvc.dvcfile import PIPELINE_FILE
from dvc.dvcfile import PROJECT_FILE
from dvc.output import OutputDoesNotExistError
from dvc.stage.exceptions import StageCommitError
@@ -234,8 +234,8 @@ def test_commit_pipeline_stage(tmp_dir, dvc, run_copy):
# just to confirm different variants work
assert dvc.commit(f":{stage.addressing}") == [stage]
assert dvc.commit(f"{PIPELINE_FILE}:{stage.addressing}") == [stage]
assert dvc.commit(PIPELINE_FILE) == [stage]
assert dvc.commit(f"{PROJECT_FILE}:{stage.addressing}") == [stage]
assert dvc.commit(PROJECT_FILE) == [stage]
def test_imported_entries_unchanged(tmp_dir, dvc, erepo_dir):
tests/func/test_dvcfile.py
+28 -25
View File
@@ -5,11 +5,11 @@ import pytest
from dvc.annotations import Annotation
from dvc.dvcfile import (
PIPELINE_FILE,
PIPELINE_LOCK,
Dvcfile,
LOCK_FILE,
PROJECT_FILE,
ParametrizedDumpError,
SingleStageFile,
load_file,
)
from dvc.stage.exceptions import StageFileDoesNotExistError
from dvc.stage.loader import StageNotFound
@@ -35,7 +35,7 @@ def test_run_load_one_for_multistage(tmp_dir, dvc):
outs_persist_no_cache=["foo2"],
always_changed=True,
)
stage2 = Dvcfile(dvc, PIPELINE_FILE).stages["copy-foo-foo2"]
stage2 = load_file(dvc, PROJECT_FILE).stages["copy-foo-foo2"]
assert stage1 == stage2
foo_out = stage2.outs[0]
assert stage2.cmd == "cp foo foo2"
@@ -49,7 +49,7 @@ def test_run_load_one_for_multistage(tmp_dir, dvc):
def test_run_load_one_for_multistage_non_existing(tmp_dir, dvc):
with pytest.raises(StageFileDoesNotExistError):
assert Dvcfile(dvc, PIPELINE_FILE).stages.get("copy-foo-foo2")
assert load_file(dvc, PROJECT_FILE).stages.get("copy-foo-foo2")
def test_run_load_one_for_multistage_non_existing_stage_name(tmp_dir, dvc):
@@ -62,7 +62,7 @@ def test_run_load_one_for_multistage_non_existing_stage_name(tmp_dir, dvc):
always_changed=True,
)
with pytest.raises(StageNotFound):
assert Dvcfile(dvc, stage.path).stages["random-name"]
assert load_file(dvc, stage.path).stages["random-name"]
def test_run_load_one_on_single_stage(tmp_dir, dvc):
@@ -74,9 +74,9 @@ def test_run_load_one_on_single_stage(tmp_dir, dvc):
always_changed=True,
single_stage=True,
)
assert isinstance(Dvcfile(dvc, stage.path), SingleStageFile)
assert Dvcfile(dvc, stage.path).stages.get("random-name") == stage
assert Dvcfile(dvc, stage.path).stage == stage
assert isinstance(load_file(dvc, stage.path), SingleStageFile)
assert load_file(dvc, stage.path).stages.get("random-name") == stage
assert load_file(dvc, stage.path).stage == stage
def test_has_stage_with_name(tmp_dir, dvc):
@@ -88,7 +88,7 @@ def test_has_stage_with_name(tmp_dir, dvc):
metrics=["foo2"],
always_changed=True,
)
dvcfile = Dvcfile(dvc, PIPELINE_FILE)
dvcfile = load_file(dvc, PROJECT_FILE)
assert "copy-foo-foo2" in dvcfile.stages
assert "copy" not in dvcfile.stages
@@ -102,7 +102,7 @@ def test_load_all_multistage(tmp_dir, dvc):
metrics=["foo2"],
always_changed=True,
)
stages = Dvcfile(dvc, PIPELINE_FILE).stages.values()
stages = load_file(dvc, PROJECT_FILE).stages.values()
assert len(stages) == 1
assert list(stages) == [stage1]
@@ -114,7 +114,10 @@ def test_load_all_multistage(tmp_dir, dvc):
metrics=["bar2"],
always_changed=True,
)
assert set(Dvcfile(dvc, PIPELINE_FILE).stages.values()) == {stage2, stage1}
assert set(load_file(dvc, PROJECT_FILE).stages.values()) == {
stage2,
stage1,
}
def test_load_all_singlestage(tmp_dir, dvc):
@@ -126,7 +129,7 @@ def test_load_all_singlestage(tmp_dir, dvc):
always_changed=True,
single_stage=True,
)
dvcfile = Dvcfile(dvc, "foo2.dvc")
dvcfile = load_file(dvc, "foo2.dvc")
assert isinstance(dvcfile, SingleStageFile)
assert len(dvcfile.stages) == 1
stages = dvcfile.stages.values()
@@ -146,7 +149,7 @@ def test_try_get_single_stage_from_pipeline_file(tmp_dir, dvc):
always_changed=True,
)
with pytest.raises(DvcException):
assert Dvcfile(dvc, PIPELINE_FILE).stage
assert load_file(dvc, PROJECT_FILE).stage
def test_stage_collection(tmp_dir, dvc):
@@ -180,7 +183,7 @@ def test_remove_stage(tmp_dir, dvc, run_copy):
stage = run_copy("foo", "bar", name="copy-foo-bar")
stage2 = run_copy("bar", "foobar", name="copy-bar-foobar")
dvc_file = Dvcfile(dvc, PIPELINE_FILE)
dvc_file = load_file(dvc, PROJECT_FILE)
assert dvc_file.exists()
assert {"copy-bar-foobar", "copy-foo-bar"} == set(
dvc_file._load()[0]["stages"].keys()
@@ -205,7 +208,7 @@ def test_remove_stage_lockfile(tmp_dir, dvc, run_copy):
stage = run_copy("foo", "bar", name="copy-foo-bar")
stage2 = run_copy("bar", "foobar", name="copy-bar-foobar")
dvc_file = Dvcfile(dvc, PIPELINE_FILE)
dvc_file = load_file(dvc, PROJECT_FILE)
lock_file = dvc_file._lockfile
assert dvc_file.exists()
assert lock_file.exists()
@@ -230,7 +233,7 @@ def test_remove_stage_dvcfiles(tmp_dir, dvc, run_copy):
tmp_dir.gen("foo", "foo")
stage = run_copy("foo", "bar", single_stage=True)
dvc_file = Dvcfile(dvc, stage.path)
dvc_file = load_file(dvc, stage.path)
assert dvc_file.exists()
dvc_file.remove_stage(stage)
assert not dvc_file.exists()
@@ -246,7 +249,7 @@ def test_remove_stage_dvcfiles(tmp_dir, dvc, run_copy):
def test_remove_stage_on_lockfile_format_error(tmp_dir, dvc, run_copy):
tmp_dir.gen("foo", "foo")
stage = run_copy("foo", "bar", name="copy-foo-bar")
dvc_file = Dvcfile(dvc, stage.path)
dvc_file = load_file(dvc, stage.path)
lock_file = dvc_file._lockfile
data = dvc_file._load()[0]
@@ -285,19 +288,19 @@ def test_remove_stage_preserves_comment(tmp_dir, dvc, run_copy):
),
)
dvc.reproduce(PIPELINE_FILE)
dvc.reproduce(PROJECT_FILE)
dvc_file = Dvcfile(dvc, PIPELINE_FILE)
dvc_file = load_file(dvc, PROJECT_FILE)
assert dvc_file.exists()
assert (tmp_dir / PIPELINE_LOCK).exists()
assert (tmp_dir / LOCK_FILE).exists()
assert (tmp_dir / "foo").exists()
assert (tmp_dir / "bar").exists()
dvc_file.remove_stage(dvc_file.stages["copy-foo-bar"])
assert (
"# This copies 'foo' text to 'foo' file."
in (tmp_dir / PIPELINE_FILE).read_text()
in (tmp_dir / PROJECT_FILE).read_text()
)
@@ -307,15 +310,15 @@ def test_remove_stage_removes_dvcfiles_if_no_stages_left(
tmp_dir.gen("foo", "foo")
run_copy("foo", "bar", name="run_copy")
dvc_file = Dvcfile(dvc, PIPELINE_FILE)
dvc_file = load_file(dvc, PROJECT_FILE)
assert dvc_file.exists()
assert (tmp_dir / PIPELINE_LOCK).exists()
assert (tmp_dir / LOCK_FILE).exists()
assert (tmp_dir / "foo").exists()
dvc_file.remove_stage(dvc_file.stages["run_copy"])
assert not dvc_file.exists()
assert not (tmp_dir / PIPELINE_LOCK).exists()
assert not (tmp_dir / LOCK_FILE).exists()
def test_dvcfile_dump_preserves_meta(tmp_dir, dvc, run_copy):
tests/func/test_gc.py
+2 -2
View File
@@ -227,7 +227,7 @@ def test_gc_cloud_remove_order(tmp_dir, scm, dvc, tmp_path_factory, mocker):
def test_gc_not_collect_pipeline_tracked_files(tmp_dir, dvc, run_copy):
from dvc.dvcfile import PIPELINE_FILE, Dvcfile
from dvc.dvcfile import PROJECT_FILE, load_file
tmp_dir.gen("foo", "foo")
tmp_dir.gen("bar", "bar")
@@ -239,7 +239,7 @@ def test_gc_not_collect_pipeline_tracked_files(tmp_dir, dvc, run_copy):
assert _count_files(dvc.odb.local.path) == 1
# remove pipeline file and lockfile and check
Dvcfile(dvc, PIPELINE_FILE).remove(force=True)
load_file(dvc, PROJECT_FILE).remove(force=True)
dvc.gc(workspace=True, force=True)
assert _count_files(dvc.odb.local.path) == 0
tests/func/test_get.py
+2 -2
View File
@@ -255,13 +255,13 @@ def test_get_url_git_only_repo(tmp_dir, scm, caplog):
def test_get_pipeline_tracked_outs(
tmp_dir, dvc, scm, git_dir, run_copy, local_remote
):
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
from dvc.dvcfile import LOCK_FILE, PROJECT_FILE
tmp_dir.gen("foo", "foo")
run_copy("foo", "bar", name="copy-foo-bar")
dvc.push()
dvc.scm.add([PIPELINE_FILE, PIPELINE_LOCK])
dvc.scm.add([PROJECT_FILE, LOCK_FILE])
dvc.scm.commit("add pipeline stage")
with git_dir.chdir():
tests/func/test_import.py
+8 -8
View File
@@ -7,7 +7,7 @@ from scmrepo.git import Git
from dvc.annotations import Annotation
from dvc.config import NoRemoteError
from dvc.dvcfile import Dvcfile
from dvc.dvcfile import load_file
from dvc.exceptions import DownloadError, PathMissingError
from dvc.fs import system
from dvc.odbmgr import ODBManager
@@ -205,7 +205,7 @@ def test_pull_imported_stage(tmp_dir, dvc, erepo_dir):
erepo_dir.dvc_gen("foo", "foo content", commit="create foo")
dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported")
dst_stage = Dvcfile(dvc, "foo_imported.dvc").stage
dst_stage = load_file(dvc, "foo_imported.dvc").stage
dst_cache = dst_stage.outs[0].cache_path
remove("foo_imported")
@@ -224,7 +224,7 @@ def test_import_no_download(tmp_dir, scm, dvc, erepo_dir):
assert not os.path.exists("foo_imported")
dst_stage = Dvcfile(dvc, "foo_imported.dvc").stage
dst_stage = load_file(dvc, "foo_imported.dvc").stage
assert dst_stage.deps[0].def_repo == {
"url": os.fspath(erepo_dir),
@@ -250,7 +250,7 @@ def test_pull_import_no_download(tmp_dir, scm, dvc, erepo_dir):
dvc.pull(["foo_imported.dvc"])
assert os.path.exists("foo_imported")
stage = Dvcfile(dvc, "foo_imported.dvc").stage
stage = load_file(dvc, "foo_imported.dvc").stage
assert (
stage.outs[0].hash_info.value == "bdb8641831d8fcb03939637e09011c21.dir"
@@ -363,7 +363,7 @@ def test_download_error_pulling_imported_stage(
erepo_dir.dvc_gen("foo", "foo content", commit="create foo")
dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported")
dst_stage = Dvcfile(dvc, "foo_imported.dvc").stage
dst_stage = load_file(dvc, "foo_imported.dvc").stage
dst_cache = dst_stage.outs[0].cache_path
remove("foo_imported")
@@ -427,7 +427,7 @@ def test_pull_no_rev_lock(erepo_dir, tmp_dir, dvc):
assert "rev" not in stage.deps[0].def_repo
stage.deps[0].def_repo.pop("rev_lock")
Dvcfile(dvc, stage.path).dump(stage)
load_file(dvc, stage.path).dump(stage)
remove(stage.outs[0].cache_path)
(tmp_dir / "foo_imported").unlink()
@@ -459,13 +459,13 @@ def test_import_from_bare_git_repo(
def test_import_pipeline_tracked_outs(
tmp_dir, dvc, scm, erepo_dir, run_copy, local_remote
):
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
from dvc.dvcfile import LOCK_FILE, PROJECT_FILE
tmp_dir.gen("foo", "foo")
run_copy("foo", "bar", name="copy-foo-bar")
dvc.push()
dvc.scm.add([PIPELINE_FILE, PIPELINE_LOCK])
dvc.scm.add([PROJECT_FILE, LOCK_FILE])
dvc.scm.commit("add pipeline stage")
with erepo_dir.chdir():
tests/func/test_import_url.py
+2 -2
View File
@@ -8,7 +8,7 @@ import pytest
from dvc.annotations import Annotation
from dvc.cli import main
from dvc.dependency.base import Dependency, DependencyDoesNotExistError
from dvc.dvcfile import Dvcfile
from dvc.dvcfile import load_file
from dvc.exceptions import InvalidArgumentError
from dvc.stage import Stage
from dvc.testing.workspace_tests import TestImport as _TestImport
@@ -304,7 +304,7 @@ def test_partial_import_pull(tmp_dir, scm, dvc, local_workspace):
assert dst.exists()
stage = Dvcfile(dvc, "file.dvc").stage
stage = load_file(dvc, "file.dvc").stage
assert stage.outs[0].hash_info.value == "d10b4c3ff123b26dc068d43a8bef2d23"
assert stage.outs[0].meta.size == 12
tests/func/test_lockfile.py
+5 -5
View File
@@ -5,7 +5,7 @@ from operator import itemgetter
import pytest
from dvc.dvcfile import PIPELINE_LOCK, Lockfile
from dvc.dvcfile import LOCK_FILE, Lockfile
from dvc.stage.utils import split_params_deps
from dvc.utils.fs import remove
from dvc.utils.serialize import dumps_yaml, parse_yaml_for_update
@@ -22,7 +22,7 @@ FS_STRUCTURE = {
}
def read_lock_file(file=PIPELINE_LOCK):
def read_lock_file(file=LOCK_FILE):
with open(file, encoding="utf-8") as f:
data = parse_yaml_for_update(f.read(), file)
assert isinstance(data, OrderedDict)
@@ -86,7 +86,7 @@ def test_order_is_preserved_when_pipeline_order_changes(
new_lock_content = read_lock_file()
assert_eq_lockfile(new_lock_content, initial_content)
(tmp_dir / PIPELINE_LOCK).unlink()
(tmp_dir / LOCK_FILE).unlink()
assert dvc.reproduce(stage.addressing) == [stage]
new_lock_content = read_lock_file()
assert_eq_lockfile(new_lock_content, initial_content)
@@ -152,12 +152,12 @@ def test_params_dump(tmp_dir, dvc, run_head):
stage.dvcfile._dump_pipeline_file(stage)
assert not dvc.reproduce(stage.addressing)
(tmp_dir / PIPELINE_LOCK).unlink()
(tmp_dir / LOCK_FILE).unlink()
assert dvc.reproduce(stage.addressing) == [stage]
assert_eq_lockfile(initial_content, read_lock_file())
# remove build-cache and check if the same structure is built
for item in [dvc.stage_cache.cache_dir, PIPELINE_LOCK]:
for item in [dvc.stage_cache.cache_dir, LOCK_FILE]:
remove(item)
assert dvc.reproduce(stage.addressing) == [stage]
assert_eq_lockfile(initial_content, read_lock_file())
tests/func/test_ls.py
+2 -2
View File
@@ -458,11 +458,11 @@ def test_ls_not_existed_url():
def test_ls_shows_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
from dvc.dvcfile import LOCK_FILE, PROJECT_FILE
tmp_dir.gen("foo", "foo")
run_copy("foo", "bar", name="copy-foo-bar")
dvc.scm.add([PIPELINE_FILE, PIPELINE_LOCK])
dvc.scm.add([PROJECT_FILE, LOCK_FILE])
dvc.scm.commit("add pipeline stage")
files = Repo.ls(os.curdir, dvc_only=True)
tests/func/test_repo.py
+3 -3
View File
@@ -1,4 +1,4 @@
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
from dvc.dvcfile import LOCK_FILE, PROJECT_FILE
from dvc.fs import system
from dvc.odbmgr import ODBManager
@@ -23,8 +23,8 @@ def test_destroy(tmp_dir, dvc, run_copy):
"file.dvc",
"file2.dvc",
"dir.dvc",
PIPELINE_FILE,
PIPELINE_LOCK,
PROJECT_FILE,
LOCK_FILE,
]:
assert not (tmp_dir / path).exists()
tests/func/test_repro.py
+2 -2
View File
@@ -7,7 +7,7 @@ from pathlib import Path
import pytest
from dvc.cli import main
from dvc.dvcfile import DVC_FILE, Dvcfile
from dvc.dvcfile import DVC_FILE, load_file
from dvc.exceptions import (
CyclicGraphError,
ReproductionError,
@@ -170,7 +170,7 @@ class TestReproWorkingDirectoryAsOutput:
dvc.index = dvc.index.update(
[
nested_stage,
Dvcfile(dvc, error_stage_path).stage,
load_file(dvc, error_stage_path).stage,
]
)
tests/func/test_repro_multistage.py
+26 -26
View File
@@ -6,7 +6,7 @@ import pytest
from funcy import lsplit
from dvc.cli import main
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
from dvc.dvcfile import LOCK_FILE, PROJECT_FILE
from dvc.exceptions import CyclicGraphError, ReproductionError
from dvc.stage import PipelineStage
@@ -86,18 +86,18 @@ def test_downstream(tmp_dir, dvc):
# B
#
evaluation = dvc.reproduce(
PIPELINE_FILE + ":B-gen", downstream=True, force=True
PROJECT_FILE + ":B-gen", downstream=True, force=True
)
assert len(evaluation) == 3
assert (
isinstance(evaluation[0], PipelineStage)
and evaluation[0].relpath == PIPELINE_FILE
and evaluation[0].relpath == PROJECT_FILE
and evaluation[0].name == "B-gen"
)
assert (
isinstance(evaluation[1], PipelineStage)
and evaluation[1].relpath == PIPELINE_FILE
and evaluation[1].relpath == PROJECT_FILE
and evaluation[1].name == "D-gen"
)
assert (
@@ -108,26 +108,26 @@ def test_downstream(tmp_dir, dvc):
# B, C should be run (in any order) before D
# See https://github.com/iterative/dvc/issues/3602
evaluation = dvc.reproduce(
PIPELINE_FILE + ":A-gen", downstream=True, force=True
PROJECT_FILE + ":A-gen", downstream=True, force=True
)
assert len(evaluation) == 5
assert (
isinstance(evaluation[0], PipelineStage)
and evaluation[0].relpath == PIPELINE_FILE
and evaluation[0].relpath == PROJECT_FILE
and evaluation[0].name == "A-gen"
)
names = set()
for stage in evaluation[1:3]:
if isinstance(stage, PipelineStage):
assert stage.relpath == PIPELINE_FILE
assert stage.relpath == PROJECT_FILE
names.add(stage.name)
else:
names.add(stage.relpath)
assert names == {"B-gen", "C.dvc"}
assert (
isinstance(evaluation[3], PipelineStage)
and evaluation[3].relpath == PIPELINE_FILE
and evaluation[3].relpath == PROJECT_FILE
and evaluation[3].name == "D-gen"
)
assert (
@@ -137,7 +137,7 @@ def test_downstream(tmp_dir, dvc):
def test_repro_when_cmd_changes(tmp_dir, dvc, run_copy, mocker):
from dvc.dvcfile import PipelineFile
from dvc.dvcfile import ProjectFile
tmp_dir.gen("foo", "foo")
stage = run_copy("foo", "bar", name="copy-file")
@@ -148,7 +148,7 @@ def test_repro_when_cmd_changes(tmp_dir, dvc, run_copy, mocker):
m = mocker.patch("dvc.stage.run.cmd_run", wraps=cmd_run)
stage.cmd = " ".join(stage.cmd.split()) # change cmd spacing by two
PipelineFile(dvc, PIPELINE_FILE)._dump_pipeline_file(stage)
ProjectFile(dvc, PROJECT_FILE)._dump_pipeline_file(stage)
assert dvc.status([target]) == {target: ["changed command"]}
assert dvc.reproduce(target)[0] == stage
@@ -160,7 +160,7 @@ def test_repro_when_cmd_changes(tmp_dir, dvc, run_copy, mocker):
def test_repro_when_new_deps_is_added_in_dvcfile(
tmp_dir, dvc, run_copy, copy_script
):
from dvc.dvcfile import Dvcfile
from dvc.dvcfile import load_file
tmp_dir.gen({"foo": "foo", "bar": "bar"})
stage = dvc.run(
@@ -169,10 +169,10 @@ def test_repro_when_new_deps_is_added_in_dvcfile(
deps=["foo"],
name="copy-file",
)
target = PIPELINE_FILE + ":copy-file"
target = PROJECT_FILE + ":copy-file"
assert not dvc.reproduce(target)
dvcfile = Dvcfile(dvc, stage.path)
dvcfile = load_file(dvc, stage.path)
data, _ = dvcfile._load()
data["stages"]["copy-file"]["deps"] += ["copy.py"]
(tmp_dir / stage.path).dump(data)
@@ -181,7 +181,7 @@ def test_repro_when_new_deps_is_added_in_dvcfile(
def test_repro_when_new_outs_is_added_in_dvcfile(tmp_dir, dvc, copy_script):
from dvc.dvcfile import Dvcfile
from dvc.dvcfile import load_file
tmp_dir.gen({"foo": "foo", "bar": "bar"})
stage = dvc.run(
@@ -193,7 +193,7 @@ def test_repro_when_new_outs_is_added_in_dvcfile(tmp_dir, dvc, copy_script):
target = ":copy-file"
assert not dvc.reproduce(target)
dvcfile = Dvcfile(dvc, stage.path)
dvcfile = load_file(dvc, stage.path)
data, _ = dvcfile._load()
data["stages"]["copy-file"]["outs"] = ["foobar"]
(tmp_dir / stage.path).dump(data)
@@ -202,7 +202,7 @@ def test_repro_when_new_outs_is_added_in_dvcfile(tmp_dir, dvc, copy_script):
def test_repro_when_new_deps_is_moved(tmp_dir, dvc, copy_script):
from dvc.dvcfile import Dvcfile
from dvc.dvcfile import load_file
tmp_dir.gen({"foo": "foo", "bar": "foo"})
stage = dvc.run(
@@ -227,7 +227,7 @@ shutil.copyfile('bar', 'foobar')
move("foo", "bar")
dvcfile = Dvcfile(dvc, stage.path)
dvcfile = load_file(dvc, stage.path)
data, _ = dvcfile._load()
data["stages"]["copy-file"]["deps"] = ["bar"]
(tmp_dir / stage.path).dump(data)
@@ -240,7 +240,7 @@ def test_repro_when_new_out_overlaps_others_stage_outs(tmp_dir, dvc):
tmp_dir.gen({"dir": {"file1": "file1"}, "foo": "foo"})
dvc.add("dir")
(tmp_dir / PIPELINE_FILE).dump(
(tmp_dir / PROJECT_FILE).dump(
{
"stages": {
"run-copy": {
@@ -257,7 +257,7 @@ def test_repro_when_new_out_overlaps_others_stage_outs(tmp_dir, dvc):
def test_repro_when_new_deps_added_does_not_exist(tmp_dir, dvc, copy_script):
tmp_dir.gen("foo", "foo")
(tmp_dir / PIPELINE_FILE).dump(
(tmp_dir / PROJECT_FILE).dump(
{
"stages": {
"run-copy": {
@@ -274,7 +274,7 @@ def test_repro_when_new_deps_added_does_not_exist(tmp_dir, dvc, copy_script):
def test_repro_when_new_outs_added_does_not_exist(tmp_dir, dvc, copy_script):
tmp_dir.gen("foo", "foo")
(tmp_dir / PIPELINE_FILE).dump(
(tmp_dir / PROJECT_FILE).dump(
{
"stages": {
"run-copy": {
@@ -291,7 +291,7 @@ def test_repro_when_new_outs_added_does_not_exist(tmp_dir, dvc, copy_script):
def test_repro_when_lockfile_gets_deleted(tmp_dir, dvc, copy_script):
tmp_dir.gen("foo", "foo")
(tmp_dir / PIPELINE_FILE).dump(
(tmp_dir / PROJECT_FILE).dump(
{
"stages": {
"run-copy": {
@@ -303,14 +303,14 @@ def test_repro_when_lockfile_gets_deleted(tmp_dir, dvc, copy_script):
},
)
assert dvc.reproduce(":run-copy")
assert os.path.exists(PIPELINE_LOCK)
assert os.path.exists(LOCK_FILE)
assert not dvc.reproduce(":run-copy")
os.unlink(PIPELINE_LOCK)
os.unlink(LOCK_FILE)
stages = dvc.reproduce(":run-copy")
assert (
stages
and stages[0].relpath == PIPELINE_FILE
and stages[0].relpath == PROJECT_FILE
and stages[0].name == "run-copy"
)
@@ -321,13 +321,13 @@ def test_cyclic_graph_error(tmp_dir, dvc, run_copy):
run_copy("bar", "baz", name="copy-bar-baz")
run_copy("baz", "foobar", name="copy-baz-foobar")
data = (tmp_dir / PIPELINE_FILE).parse()
data = (tmp_dir / PROJECT_FILE).parse()
data["stages"]["copy-baz-foo"] = {
"cmd": "echo baz > foo",
"deps": ["baz"],
"outs": ["foo"],
}
(tmp_dir / PIPELINE_FILE).dump(data)
(tmp_dir / PROJECT_FILE).dump(data)
with pytest.raises(CyclicGraphError):
dvc.reproduce(":copy-baz-foo")
tests/func/test_run_cache.py
+12 -12
View File
@@ -2,7 +2,7 @@ import os
import pytest
from dvc.dvcfile import PIPELINE_LOCK
from dvc.dvcfile import LOCK_FILE
from dvc.utils.fs import remove
@@ -30,14 +30,14 @@ def test_restore(tmp_dir, dvc, run_copy, mocker):
# removing any information that `dvc` could use to re-generate from
(tmp_dir / "bar").unlink()
(tmp_dir / PIPELINE_LOCK).unlink()
(tmp_dir / LOCK_FILE).unlink()
(stage,) = dvc.reproduce("copy-foo-bar")
mock_restore.assert_called_once_with(stage, dry=False)
mock_run.assert_not_called()
assert (tmp_dir / "bar").exists() and not (tmp_dir / "foo").unlink()
assert (tmp_dir / PIPELINE_LOCK).exists()
assert (tmp_dir / LOCK_FILE).exists()
def test_save(tmp_dir, dvc, run_copy):
@@ -103,16 +103,16 @@ def test_memory_for_multiple_runs_of_same_stage(
mock_run = mocker.spy(_run, "cmd_run")
(tmp_dir / "bar").unlink()
(tmp_dir / PIPELINE_LOCK).unlink()
(tmp_dir / LOCK_FILE).unlink()
(stage,) = dvc.reproduce("copy-foo-bar")
assert (tmp_dir / PIPELINE_LOCK).exists()
assert (tmp_dir / LOCK_FILE).exists()
assert (tmp_dir / "bar").read_text() == "foobar"
mock_run.assert_not_called()
mock_restore.assert_called_once_with(stage, dry=False)
mock_restore.reset_mock()
(tmp_dir / PIPELINE_LOCK).unlink()
(tmp_dir / LOCK_FILE).unlink()
tmp_dir.gen("foo", "foo")
dvc.reproduce("copy-foo-bar")
@@ -120,7 +120,7 @@ def test_memory_for_multiple_runs_of_same_stage(
mock_run.assert_not_called()
mock_restore.assert_called_once_with(stage, dry=False)
assert (tmp_dir / "bar").exists() and not (tmp_dir / "foo").unlink()
assert (tmp_dir / PIPELINE_LOCK).exists()
assert (tmp_dir / LOCK_FILE).exists()
def test_memory_runs_of_multiple_stages(tmp_dir, dvc, run_copy, mocker):
@@ -141,11 +141,11 @@ def test_memory_runs_of_multiple_stages(tmp_dir, dvc, run_copy, mocker):
(tmp_dir / "foo.bak").unlink()
(tmp_dir / "bar.bak").unlink()
(tmp_dir / PIPELINE_LOCK).unlink()
(tmp_dir / LOCK_FILE).unlink()
(stage,) = dvc.reproduce("backup-foo")
assert (tmp_dir / "foo.bak").read_text() == "foo"
assert (tmp_dir / PIPELINE_LOCK).exists()
assert (tmp_dir / LOCK_FILE).exists()
mock_run.assert_not_called()
mock_restore.assert_called_once_with(stage, dry=False)
mock_restore.reset_mock()
@@ -153,7 +153,7 @@ def test_memory_runs_of_multiple_stages(tmp_dir, dvc, run_copy, mocker):
(stage,) = dvc.reproduce("backup-bar")
assert (tmp_dir / "bar.bak").read_text() == "bar"
assert (tmp_dir / PIPELINE_LOCK).exists()
assert (tmp_dir / LOCK_FILE).exists()
mock_run.assert_not_called()
mock_restore.assert_called_once_with(stage, dry=False)
@@ -172,7 +172,7 @@ def test_restore_pull(tmp_dir, dvc, run_copy, mocker, local_remote):
# removing any information that `dvc` could use to re-generate from
(tmp_dir / "bar").unlink()
(tmp_dir / PIPELINE_LOCK).unlink()
(tmp_dir / LOCK_FILE).unlink()
remove(stage.outs[0].cache_path)
(stage,) = dvc.reproduce("copy-foo-bar", pull=True)
@@ -181,4 +181,4 @@ def test_restore_pull(tmp_dir, dvc, run_copy, mocker, local_remote):
mock_run.assert_not_called()
assert mock_checkout.call_count == 2
assert (tmp_dir / "bar").exists() and not (tmp_dir / "foo").unlink()
assert (tmp_dir / PIPELINE_LOCK).exists()
assert (tmp_dir / LOCK_FILE).exists()
tests/func/test_run_multistage.py
+23 -23
View File
@@ -8,29 +8,29 @@ from dvc.stage.exceptions import DuplicateStageName, InvalidStageName
def test_run_with_name(tmp_dir, dvc, run_copy):
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
from dvc.dvcfile import LOCK_FILE, PROJECT_FILE
from dvc.stage import PipelineStage
tmp_dir.dvc_gen("foo", "foo")
assert not os.path.exists(PIPELINE_FILE)
assert not os.path.exists(PROJECT_FILE)
stage = run_copy("foo", "bar", name="copy-foo-to-bar")
assert isinstance(stage, PipelineStage)
assert stage.name == "copy-foo-to-bar"
assert os.path.exists(PIPELINE_FILE)
assert os.path.exists(PIPELINE_LOCK)
assert os.path.exists(PROJECT_FILE)
assert os.path.exists(LOCK_FILE)
def test_run_no_exec(tmp_dir, dvc, run_copy):
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
from dvc.dvcfile import LOCK_FILE, PROJECT_FILE
from dvc.stage import PipelineStage
tmp_dir.dvc_gen("foo", "foo")
assert not os.path.exists(PIPELINE_FILE)
assert not os.path.exists(PROJECT_FILE)
stage = run_copy("foo", "bar", name="copy-foo-to-bar", no_exec=True)
assert isinstance(stage, PipelineStage)
assert stage.name == "copy-foo-to-bar"
assert os.path.exists(PIPELINE_FILE)
assert not os.path.exists(PIPELINE_LOCK)
assert os.path.exists(PROJECT_FILE)
assert not os.path.exists(LOCK_FILE)
data, _ = stage.dvcfile._load()
assert data["stages"]["copy-foo-to-bar"] == {
@@ -55,7 +55,7 @@ def test_run_with_multistage_and_single_stage(tmp_dir, dvc, run_copy):
def test_run_multi_stage_repeat(tmp_dir, dvc, run_copy):
from dvc.dvcfile import PIPELINE_FILE, Dvcfile
from dvc.dvcfile import PROJECT_FILE, load_file
from dvc.stage import PipelineStage
tmp_dir.dvc_gen("foo", "foo")
@@ -63,7 +63,7 @@ def test_run_multi_stage_repeat(tmp_dir, dvc, run_copy):
run_copy("foo1", "foo2", name="copy-foo1-foo2")
run_copy("foo2", "foo3", single_stage=True)
stages = list(Dvcfile(dvc, PIPELINE_FILE).stages.values())
stages = list(load_file(dvc, PROJECT_FILE).stages.values())
assert len(stages) == 2
assert all(isinstance(stage, PipelineStage) for stage in stages)
assert {stage.name for stage in stages} == {
@@ -94,7 +94,7 @@ def test_multistage_dump_on_non_cached_outputs(tmp_dir, dvc):
def test_multistage_with_wdir(tmp_dir, dvc):
from dvc.dvcfile import Dvcfile
from dvc.dvcfile import load_file
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
stage = dvc.run(
@@ -105,12 +105,12 @@ def test_multistage_with_wdir(tmp_dir, dvc):
wdir="dir",
)
data, _ = Dvcfile(dvc, stage.path)._load()
data, _ = load_file(dvc, stage.path)._load()
assert data["stages"]["copy-foo1-foo2"]["wdir"] == "dir"
def test_multistage_always_changed(tmp_dir, dvc):
from dvc.dvcfile import Dvcfile
from dvc.dvcfile import load_file
tmp_dir.gen({"foo": "foo", "bar": "bar"})
stage = dvc.run(
@@ -121,7 +121,7 @@ def test_multistage_always_changed(tmp_dir, dvc):
always_changed=True,
)
data, _ = Dvcfile(dvc, stage.path)._load()
data, _ = load_file(dvc, stage.path)._load()
assert data["stages"]["copy-foo1-foo2"]["always_changed"]
@@ -139,7 +139,7 @@ def test_graph(tmp_dir, dvc):
def test_run_dump_on_multistage(tmp_dir, dvc, run_head):
from dvc.dvcfile import PIPELINE_FILE, Dvcfile
from dvc.dvcfile import PROJECT_FILE, load_file
tmp_dir.gen(
{
@@ -159,7 +159,7 @@ def test_run_dump_on_multistage(tmp_dir, dvc, run_head):
outs_persist=["foo2"],
always_changed=True,
)
data = Dvcfile(dvc, PIPELINE_FILE)._load()[0]
data = load_file(dvc, PROJECT_FILE)._load()[0]
assert data == {
"stages": {
"copy-foo-foo2": {
@@ -182,7 +182,7 @@ def test_run_dump_on_multistage(tmp_dir, dvc, run_head):
metrics_no_cache=["foobar-1"],
wdir="dir",
)
assert Dvcfile(dvc, PIPELINE_FILE)._load()[0] == {
assert load_file(dvc, PROJECT_FILE)._load()[0] == {
"stages": {
"head-files": {
"cmd": "python {} foo bar foobar".format(
@@ -309,7 +309,7 @@ def test_run_without_cmd(tmp_dir, dvc, kwargs):
def test_run_overwrite_order(tmp_dir, dvc, run_copy):
from dvc.dvcfile import PIPELINE_FILE
from dvc.dvcfile import PROJECT_FILE
tmp_dir.gen({"foo": "foo", "foo1": "foo1"})
run_copy("foo", "bar", name="copy-foo-bar")
@@ -317,12 +317,12 @@ def test_run_overwrite_order(tmp_dir, dvc, run_copy):
run_copy("foo1", "bar1", name="copy-foo-bar", force=True)
data = (tmp_dir / PIPELINE_FILE).parse()
data = (tmp_dir / PROJECT_FILE).parse()
assert list(data["stages"].keys()) == ["copy-foo-bar", "copy-bar-foobar"]
def test_run_overwrite_preserves_meta_and_comment(tmp_dir, dvc, run_copy):
from dvc.dvcfile import PIPELINE_FILE
from dvc.dvcfile import PROJECT_FILE
tmp_dir.gen({"foo": "foo", "foo1": "foo1"})
text = textwrap.dedent(
@@ -340,12 +340,12 @@ def test_run_overwrite_preserves_meta_and_comment(tmp_dir, dvc, run_copy):
name: meta is preserved too
"""
)
(tmp_dir / PIPELINE_FILE).write_text(text.format(src="foo", dest="bar"))
assert dvc.reproduce(PIPELINE_FILE)
(tmp_dir / PROJECT_FILE).write_text(text.format(src="foo", dest="bar"))
assert dvc.reproduce(PROJECT_FILE)
assert run_copy("foo1", "bar1", name="copy-foo-bar", force=True)
assert (tmp_dir / PIPELINE_FILE).read_text() == text.format(
assert (tmp_dir / PROJECT_FILE).read_text() == text.format(
src="foo1", dest="bar1"
)
tests/func/test_stage_load.py
+12 -14
View File
@@ -4,7 +4,7 @@ from operator import itemgetter
import pytest
from funcy import raiser
from dvc.dvcfile import PIPELINE_FILE, FileIsGitIgnored
from dvc.dvcfile import PROJECT_FILE, FileIsGitIgnored
from dvc.exceptions import NoOutputOrStageError
from dvc.repo import Repo
from dvc.stage.exceptions import (
@@ -186,7 +186,7 @@ def test_collect_granular_with_target(tmp_dir, dvc, stages):
assert dvc.stage.collect_granular("bar.dvc") == [
(stages["copy-foo-bar"], None)
]
assert dvc.stage.collect_granular(PIPELINE_FILE) == [
assert dvc.stage.collect_granular(PROJECT_FILE) == [
(stages["copy-bar-foobar"], None),
(stages["copy-lorem-ipsum"], None),
]
@@ -206,7 +206,7 @@ def test_collect_granular_with_target(tmp_dir, dvc, stages):
with (tmp_dir / dvc.DVC_DIR).chdir():
assert dvc.stage.collect_granular(
relpath(tmp_dir / PIPELINE_FILE) + ":copy-bar-foobar"
relpath(tmp_dir / PROJECT_FILE) + ":copy-bar-foobar"
) == [(stages["copy-bar-foobar"], None)]
assert dvc.stage.collect_granular("foobar") == [
@@ -257,7 +257,7 @@ def test_collect_granular_with_deps(tmp_dir, dvc, stages):
assert set(
map(
itemgetter(0),
dvc.stage.collect_granular(PIPELINE_FILE, with_deps=True),
dvc.stage.collect_granular(PROJECT_FILE, with_deps=True),
)
) == set(stages.values())
@@ -333,17 +333,17 @@ def test_get_stages(tmp_dir, dvc, run_copy):
stage2 = run_copy("bar", "foobar", name="copy-bar-foobar")
assert set(dvc.stage.load_all()) == {stage1, stage2}
assert set(dvc.stage.load_all(path=PIPELINE_FILE)) == {stage1, stage2}
assert set(dvc.stage.load_all(path=PROJECT_FILE)) == {stage1, stage2}
assert set(dvc.stage.load_all(name="copy-bar-foobar")) == {stage2}
assert set(
dvc.stage.load_all(path=PIPELINE_FILE, name="copy-bar-foobar")
dvc.stage.load_all(path=PROJECT_FILE, name="copy-bar-foobar")
) == {stage2}
with pytest.raises(StageFileDoesNotExistError):
dvc.stage.load_all(path=relpath(tmp_dir / ".." / PIPELINE_FILE))
dvc.stage.load_all(path=relpath(tmp_dir / ".." / PROJECT_FILE))
with pytest.raises(StageNotFound):
dvc.stage.load_all(path=PIPELINE_FILE, name="copy")
dvc.stage.load_all(path=PROJECT_FILE, name="copy")
def test_get_stages_old_dvcfile(tmp_dir, dvc):
@@ -363,11 +363,9 @@ def test_get_stage(tmp_dir, dvc, run_copy):
dvc.stage.load_one()
with pytest.raises(StageNameUnspecified):
dvc.stage.load_one(path=PIPELINE_FILE)
dvc.stage.load_one(path=PROJECT_FILE)
assert (
dvc.stage.load_one(path=PIPELINE_FILE, name="copy-foo-bar") == stage1
)
assert dvc.stage.load_one(path=PROJECT_FILE, name="copy-foo-bar") == stage1
assert dvc.stage.load_one(name="copy-foo-bar") == stage1
with pytest.raises(StageFileDoesNotExistError):
@@ -420,14 +418,14 @@ def test_collect_repo_callback(tmp_dir, dvc, mocker):
dvc.stage_collection_error_handler = mock
(stage,) = tmp_dir.dvc_gen("foo", "foo")
(tmp_dir / PIPELINE_FILE).dump({"stages": {"cmd": "echo hello world"}})
(tmp_dir / PROJECT_FILE).dump({"stages": {"cmd": "echo hello world"}})
dvc._reset()
assert dvc.index.stages == [stage]
mock.assert_called_once()
file_path, exc = mock.call_args[0]
assert file_path == PIPELINE_FILE
assert file_path == PROJECT_FILE
assert isinstance(exc, YAMLValidationError)
tests/func/test_update.py
+5 -5
View File
@@ -3,7 +3,7 @@ import os
import pytest
from dvc.dependency import base
from dvc.dvcfile import Dvcfile
from dvc.dvcfile import load_file
from dvc.exceptions import InvalidArgumentError
from dvc.testing.tmp_dir import make_subrepo
@@ -117,7 +117,7 @@ def test_update_import_after_remote_updates_to_dvc(tmp_dir, dvc, erepo_dir):
assert imported.is_file()
assert imported.read_text() == "updated"
stage = Dvcfile(dvc, stage.path).stage
stage = load_file(dvc, stage.path).stage
assert stage.deps[0].def_repo == {
"url": os.fspath(erepo_dir),
"rev": "branch",
@@ -344,9 +344,9 @@ def test_update_recursive(tmp_dir, dvc, erepo_dir):
dvc.update(["dir"], recursive=True)
stage1 = Dvcfile(dvc, stage1.path).stage
stage2 = Dvcfile(dvc, stage2.path).stage
stage3 = Dvcfile(dvc, stage3.path).stage
stage1 = load_file(dvc, stage1.path).stage
stage2 = load_file(dvc, stage2.path).stage
stage3 = load_file(dvc, stage3.path).stage
assert stage1.deps[0].def_repo["rev_lock"] == new_rev
assert stage2.deps[0].def_repo["rev_lock"] == new_rev
assert stage3.deps[0].def_repo["rev_lock"] == new_rev
tests/unit/stage/test_loader_pipeline_file.py
+20 -20
View File
@@ -4,7 +4,7 @@ from itertools import chain
import pytest
from dvc.dvcfile import PIPELINE_FILE, Dvcfile
from dvc.dvcfile import PROJECT_FILE, load_file
from dvc.stage import PipelineStage, create_stage
from dvc.stage.loader import StageLoader
from dvc.stage.serialize import split_params_deps
@@ -27,7 +27,7 @@ def lock_data():
def test_fill_from_lock_deps_outs(dvc, lock_data):
stage = create_stage(
PipelineStage, dvc, PIPELINE_FILE, deps=["foo"], outs=["bar"]
PipelineStage, dvc, PROJECT_FILE, deps=["foo"], outs=["bar"]
)
for item in chain(stage.deps, stage.outs):
@@ -40,7 +40,7 @@ def test_fill_from_lock_deps_outs(dvc, lock_data):
def test_fill_from_lock_outs_isexec(dvc):
stage = create_stage(PipelineStage, dvc, PIPELINE_FILE, outs=["foo"])
stage = create_stage(PipelineStage, dvc, PROJECT_FILE, outs=["foo"])
assert not stage.outs[0].meta.isexec
@@ -61,7 +61,7 @@ def test_fill_from_lock_params(dvc, lock_data):
stage = create_stage(
PipelineStage,
dvc,
PIPELINE_FILE,
PROJECT_FILE,
deps=["foo"],
outs=["bar"],
params=[
@@ -97,7 +97,7 @@ def test_fill_from_lock_missing_params_section(dvc, lock_data):
stage = create_stage(
PipelineStage,
dvc,
PIPELINE_FILE,
PROJECT_FILE,
deps=["foo"],
outs=["bar"],
params=["lorem", "lorem.ipsum", {"myparams.yaml": ["ipsum"]}],
@@ -111,7 +111,7 @@ def test_fill_from_lock_missing_checksums(dvc, lock_data):
stage = create_stage(
PipelineStage,
dvc,
PIPELINE_FILE,
PROJECT_FILE,
deps=["foo", "foo1"],
outs=["bar", "bar1"],
)
@@ -127,7 +127,7 @@ def test_fill_from_lock_use_appropriate_checksum(dvc, lock_data):
stage = create_stage(
PipelineStage,
dvc,
PIPELINE_FILE,
PROJECT_FILE,
deps=["s3://dvc-temp/foo"],
outs=["bar"],
)
@@ -139,7 +139,7 @@ def test_fill_from_lock_use_appropriate_checksum(dvc, lock_data):
def test_fill_from_lock_with_missing_sections(dvc, lock_data):
stage = create_stage(
PipelineStage, dvc, PIPELINE_FILE, deps=["foo"], outs=["bar"]
PipelineStage, dvc, PROJECT_FILE, deps=["foo"], outs=["bar"]
)
lock = deepcopy(lock_data)
del lock["deps"]
@@ -156,7 +156,7 @@ def test_fill_from_lock_with_missing_sections(dvc, lock_data):
def test_fill_from_lock_empty_data(dvc):
stage = create_stage(
PipelineStage, dvc, PIPELINE_FILE, deps=["foo"], outs=["bar"]
PipelineStage, dvc, PROJECT_FILE, deps=["foo"], outs=["bar"]
)
StageLoader.fill_from_lock(stage, None)
assert not stage.deps[0].hash_info and not stage.outs[0].hash_info
@@ -165,13 +165,13 @@ def test_fill_from_lock_empty_data(dvc):
def test_load_stage(dvc, stage_data, lock_data):
dvcfile = Dvcfile(dvc, PIPELINE_FILE)
dvcfile = load_file(dvc, PROJECT_FILE)
stage = StageLoader.load_stage(dvcfile, "stage-1", stage_data, lock_data)
assert stage.wdir == os.path.abspath(os.curdir)
assert stage.name == "stage-1"
assert stage.cmd == "command"
assert stage.path == os.path.abspath(PIPELINE_FILE)
assert stage.path == os.path.abspath(PROJECT_FILE)
assert stage.deps[0].def_path == "foo"
assert stage.deps[0].hash_info == HashInfo("md5", "foo_checksum")
assert stage.outs[0].def_path == "bar"
@@ -180,20 +180,20 @@ def test_load_stage(dvc, stage_data, lock_data):
def test_load_stage_cmd_with_list(dvc, stage_data, lock_data):
stage_data["cmd"] = ["cmd-0", "cmd-1"]
dvcfile = Dvcfile(dvc, PIPELINE_FILE)
dvcfile = load_file(dvc, PROJECT_FILE)
stage = StageLoader.load_stage(dvcfile, "stage-1", stage_data, lock_data)
assert stage.cmd == ["cmd-0", "cmd-1"]
def test_load_stage_outs_with_flags(dvc, stage_data, lock_data):
stage_data["outs"] = [{"foo": {"cache": False}}]
dvcfile = Dvcfile(dvc, PIPELINE_FILE)
dvcfile = load_file(dvc, PROJECT_FILE)
stage = StageLoader.load_stage(dvcfile, "stage-1", stage_data, lock_data)
assert stage.outs[0].use_cache is False
def test_load_stage_no_lock(dvc, stage_data):
dvcfile = Dvcfile(dvc, PIPELINE_FILE)
dvcfile = load_file(dvc, PROJECT_FILE)
stage = StageLoader.load_stage(dvcfile, "stage-1", stage_data)
assert stage.deps[0].def_path == "foo" and stage.outs[0].def_path == "bar"
assert not stage.deps[0].hash_info
@@ -203,7 +203,7 @@ def test_load_stage_no_lock(dvc, stage_data):
def test_load_stage_with_params(dvc, stage_data, lock_data):
lock_data["params"] = {"params.yaml": {"lorem": "ipsum"}}
stage_data["params"] = ["lorem"]
dvcfile = Dvcfile(dvc, PIPELINE_FILE)
dvcfile = load_file(dvc, PROJECT_FILE)
stage = StageLoader.load_stage(dvcfile, "stage-1", stage_data, lock_data)
params, deps = split_params_deps(stage)
@@ -217,7 +217,7 @@ def test_load_stage_with_params(dvc, stage_data, lock_data):
@pytest.mark.parametrize("typ", ["metrics", "plots"])
def test_load_stage_with_metrics_and_plots(dvc, stage_data, lock_data, typ):
stage_data[typ] = stage_data.pop("outs")
dvcfile = Dvcfile(dvc, PIPELINE_FILE)
dvcfile = load_file(dvc, PROJECT_FILE)
stage = StageLoader.load_stage(dvcfile, "stage-1", stage_data, lock_data)
assert stage.outs[0].def_path == "bar"
@@ -225,7 +225,7 @@ def test_load_stage_with_metrics_and_plots(dvc, stage_data, lock_data, typ):
def test_load_changed_command(dvc, stage_data, lock_data):
dvcfile = Dvcfile(dvc, PIPELINE_FILE)
dvcfile = load_file(dvc, PROJECT_FILE)
stage = StageLoader.load_stage(dvcfile, "stage-1", stage_data)
assert not stage.cmd_changed
assert stage.cmd == "command"
@@ -238,15 +238,15 @@ def test_load_changed_command(dvc, stage_data, lock_data):
def test_load_stage_wdir_and_path_correctly(dvc, stage_data, lock_data):
stage_data["wdir"] = "dir"
dvcfile = Dvcfile(dvc, PIPELINE_FILE)
dvcfile = load_file(dvc, PROJECT_FILE)
stage = StageLoader.load_stage(dvcfile, "stage-1", stage_data, lock_data)
assert stage.wdir == os.path.abspath("dir")
assert stage.path == os.path.abspath(PIPELINE_FILE)
assert stage.path == os.path.abspath(PROJECT_FILE)
def test_load_stage_mapping(dvc, stage_data, lock_data):
dvcfile = Dvcfile(dvc, PIPELINE_FILE)
dvcfile = load_file(dvc, PROJECT_FILE)
dvcfile.contents = {"stages": {"stage": stage_data}}
dvcfile.lockfile_contents = {"stage": lock_data}
tests/unit/stage/test_serialize_pipeline_file.py
+2 -2
View File
@@ -4,12 +4,12 @@ import pytest
from voluptuous import Schema as _Schema
from dvc import output
from dvc.dvcfile import PIPELINE_FILE
from dvc.dvcfile import PROJECT_FILE
from dvc.schema import SINGLE_PIPELINE_STAGE_SCHEMA
from dvc.stage import PipelineStage, create_stage
from dvc.stage.serialize import to_pipeline_file as _to_pipeline_file
kwargs = {"name": "something", "cmd": "command", "path": PIPELINE_FILE}
kwargs = {"name": "something", "cmd": "command", "path": PROJECT_FILE}
Schema = _Schema(SINGLE_PIPELINE_STAGE_SCHEMA)
tests/unit/stage/test_serialize_pipeline_lock.py
+2 -2
View File
@@ -3,7 +3,7 @@ from collections import OrderedDict
import pytest
from voluptuous import Schema as _Schema
from dvc.dvcfile import PIPELINE_FILE
from dvc.dvcfile import PROJECT_FILE
from dvc.schema import LOCK_FILE_STAGE_SCHEMA, LOCKFILE_STAGES_SCHEMA
from dvc.stage import PipelineStage, create_stage
from dvc.stage.serialize import DEFAULT_PARAMS_FILE, to_lockfile
@@ -13,7 +13,7 @@ from dvc.stage.serialize import (
from dvc.stage.utils import split_params_deps
from dvc_data.hashfile.hash_info import HashInfo
kwargs = {"name": "something", "cmd": "command", "path": PIPELINE_FILE}
kwargs = {"name": "something", "cmd": "command", "path": PROJECT_FILE}
Schema = _Schema(LOCK_FILE_STAGE_SCHEMA)
tests/unit/test_dvcfile.py
+22 -22
View File
@@ -1,12 +1,12 @@
import pytest
from dvc.dvcfile import (
PIPELINE_FILE,
PIPELINE_LOCK,
Dvcfile,
LOCK_FILE,
PROJECT_FILE,
FileIsGitIgnored,
PipelineFile,
ProjectFile,
SingleStageFile,
load_file,
)
from dvc.stage import PipelineStage
from dvc.stage.exceptions import (
@@ -29,22 +29,22 @@ from dvc.utils.strictyaml import YAMLValidationError
],
)
def test_pipelines_file(path):
file_obj = Dvcfile(object(), path)
assert isinstance(file_obj, PipelineFile)
file_obj = load_file(object(), path)
assert isinstance(file_obj, ProjectFile)
@pytest.mark.parametrize(
"path", ["Dvcfile", "stage.dvc", "../models/stage.dvc"]
)
def test_pipelines_single_stage_file(path):
file_obj = Dvcfile(object(), path)
file_obj = load_file(object(), path)
assert isinstance(file_obj, SingleStageFile)
@pytest.mark.parametrize("file", ["stage.dvc", "dvc.yaml"])
@pytest.mark.parametrize("is_dvcignored", [True, False])
def test_stage_load_on_not_existing_file(tmp_dir, dvc, file, is_dvcignored):
dvcfile = Dvcfile(dvc, file)
dvcfile = load_file(dvc, file)
if is_dvcignored:
(tmp_dir / ".dvcignore").write_text(file)
@@ -58,7 +58,7 @@ def test_stage_load_on_not_existing_file(tmp_dir, dvc, file, is_dvcignored):
@pytest.mark.parametrize("file", ["stage.dvc", "dvc.yaml"])
def test_stage_load_on_non_file(tmp_dir, dvc, file):
(tmp_dir / file).mkdir()
dvcfile = Dvcfile(dvc, file)
dvcfile = load_file(dvc, file)
with pytest.raises(StageFileIsNotDvcFileError):
assert dvcfile.stages.values()
@@ -67,7 +67,7 @@ def test_stage_load_on_non_file(tmp_dir, dvc, file):
def test_stage_load_on_invalid_data(tmp_dir, dvc, file):
data = {"is_this_a_valid_dvcfile": False}
(tmp_dir / file).dump(data)
dvcfile = Dvcfile(dvc, file)
dvcfile = load_file(dvc, file)
with pytest.raises(YAMLValidationError):
assert dvcfile.stages
with pytest.raises(YAMLValidationError):
@@ -78,22 +78,22 @@ def test_dump_stage(tmp_dir, dvc):
stage = PipelineStage(
dvc, cmd="command", name="stage_name", path="dvc.yaml"
)
dvcfile = Dvcfile(dvc, "dvc.yaml")
dvcfile = load_file(dvc, "dvc.yaml")
dvcfile.dump(stage, update_lock=False, update_pipeline=False)
assert not (tmp_dir / PIPELINE_FILE).exists()
assert not (tmp_dir / PIPELINE_LOCK).exists()
assert not (tmp_dir / PROJECT_FILE).exists()
assert not (tmp_dir / LOCK_FILE).exists()
dvcfile.dump(stage, update_pipeline=False)
assert not (tmp_dir / PIPELINE_FILE).exists()
assert (tmp_dir / PIPELINE_LOCK).exists()
assert not (tmp_dir / PROJECT_FILE).exists()
assert (tmp_dir / LOCK_FILE).exists()
assert dvcfile._lockfile.load()
remove(tmp_dir / PIPELINE_LOCK)
remove(tmp_dir / LOCK_FILE)
dvcfile.dump(stage)
assert (tmp_dir / PIPELINE_FILE).exists()
assert (tmp_dir / PIPELINE_LOCK).exists()
assert (tmp_dir / PROJECT_FILE).exists()
assert (tmp_dir / LOCK_FILE).exists()
assert list(dvcfile.stages.values()) == [stage]
@@ -103,7 +103,7 @@ def test_stage_load_file_exists_but_dvcignored(tmp_dir, dvc, scm, file):
(tmp_dir / ".dvcignore").write_text(file)
dvc._reset()
dvcfile = Dvcfile(dvc, file)
dvcfile = load_file(dvc, file)
with pytest.raises(StageFileDoesNotExistError) as exc_info:
assert dvcfile.stages.values()
@@ -119,7 +119,7 @@ def test_try_loading_dvcfile_that_is_gitignored(tmp_dir, dvc, scm, file):
(tmp_dir / file).write_text("")
scm._reset()
dvcfile = Dvcfile(dvc, file)
dvcfile = load_file(dvc, file)
with pytest.raises(FileIsGitIgnored) as exc_info:
dvcfile._load()
@@ -127,8 +127,8 @@ def test_try_loading_dvcfile_that_is_gitignored(tmp_dir, dvc, scm, file):
def test_dvcfile_encoding_error(tmp_dir, dvc):
tmp_dir.gen(PIPELINE_FILE, b"\x80some: stuff")
tmp_dir.gen(PROJECT_FILE, b"\x80some: stuff")
dvcfile = Dvcfile(dvc, PIPELINE_FILE)
dvcfile = load_file(dvc, PROJECT_FILE)
with pytest.raises(EncodingError):
dvcfile._load()