From da27174e02595cd4ffdfea1b4bef940a6588b8c9 Mon Sep 17 00:00:00 2001 From: skshetry <18718008+skshetry@users.noreply.github.com> Date: Sun, 15 Jan 2023 12:45:10 +0545 Subject: [PATCH] dvcfile: rename PipelineFile to ProjectFile, get rid of Dvcfile class, cleanups, etc. (#8812) dvcfile: rename PipelineFile to ProjectFile, get rid of Dvcfile class, cleanups etc --- dvc/commands/git_hook.py | 8 +-- dvc/dvcfile.py | 38 +++++-------- dvc/repo/collect.py | 3 -- dvc/repo/commit.py | 5 +- dvc/repo/experiments/init.py | 10 ++-- dvc/repo/imp_url.py | 4 +- dvc/repo/index.py | 4 +- dvc/repo/move.py | 5 +- dvc/repo/plots/__init__.py | 5 +- dvc/repo/reproduce.py | 15 ++---- dvc/repo/stage.py | 21 ++++---- dvc/repo/update.py | 5 +- dvc/stage/__init__.py | 18 ++++--- dvc/stage/utils.py | 4 +- dvc/types.py | 6 ++- dvc/utils/__init__.py | 6 +-- dvc/utils/serialize/_common.py | 18 +++---- tests/func/experiments/test_experiments.py | 6 +-- tests/func/metrics/test_show.py | 4 +- tests/func/params/test_diff.py | 8 +-- tests/func/params/test_show.py | 14 ++--- tests/func/plots/test_modify.py | 6 +-- tests/func/plots/test_show.py | 4 +- tests/func/test_add.py | 4 +- tests/func/test_checkout.py | 8 +-- tests/func/test_commit.py | 6 +-- tests/func/test_dvcfile.py | 53 ++++++++++--------- tests/func/test_gc.py | 4 +- tests/func/test_get.py | 4 +- tests/func/test_import.py | 16 +++--- tests/func/test_import_url.py | 4 +- tests/func/test_lockfile.py | 10 ++-- tests/func/test_ls.py | 4 +- tests/func/test_repo.py | 6 +-- tests/func/test_repro.py | 4 +- tests/func/test_repro_multistage.py | 52 +++++++++--------- tests/func/test_run_cache.py | 24 ++++----- tests/func/test_run_multistage.py | 46 ++++++++-------- tests/func/test_stage_load.py | 26 +++++---- tests/func/test_update.py | 10 ++-- tests/unit/stage/test_loader_pipeline_file.py | 40 +++++++------- .../stage/test_serialize_pipeline_file.py | 4 +- .../stage/test_serialize_pipeline_lock.py | 4 +- tests/unit/test_dvcfile.py | 44 +++++++-------- 44 files changed, 283 insertions(+), 307 deletions(-) diff --git a/dvc/commands/git_hook.py b/dvc/commands/git_hook.py index 948a217b7..9c8cead6d 100644 --- a/dvc/commands/git_hook.py +++ b/dvc/commands/git_hook.py @@ -64,15 +64,15 @@ class CmdPrePush(CmdHookBase): class CmdMergeDriver(CmdHookBase): def _run(self): - from dvc.dvcfile import Dvcfile + from dvc.dvcfile import load_file from dvc.repo import Repo dvc = Repo() try: - ancestor = Dvcfile(dvc, self.args.ancestor, verify=False) - our = Dvcfile(dvc, self.args.our, verify=False) - their = Dvcfile(dvc, self.args.their, verify=False) + ancestor = load_file(dvc, self.args.ancestor, verify=False) + our = load_file(dvc, self.args.our, verify=False) + their = load_file(dvc, self.args.their, verify=False) our.merge(ancestor, their, allowed=["add", "remove", "change"]) diff --git a/dvc/dvcfile.py b/dvc/dvcfile.py index 08dcf9fe7..babee1431 100644 --- a/dvc/dvcfile.py +++ b/dvc/dvcfile.py @@ -20,7 +20,7 @@ from dvc.stage.exceptions import ( StageFileDoesNotExistError, StageFileIsNotDvcFileError, ) -from dvc.types import AnyPath +from dvc.types import StrOrBytesPath from dvc.utils import relpath from dvc.utils.collections import apply_diff from dvc.utils.objects import cached_property @@ -36,8 +36,8 @@ _T = TypeVar("_T") DVC_FILE = "Dvcfile" DVC_FILE_SUFFIX = ".dvc" -PIPELINE_FILE = "dvc.yaml" -PIPELINE_LOCK = "dvc.lock" +PROJECT_FILE = "dvc.yaml" +LOCK_FILE = "dvc.lock" class FileIsGitIgnored(DvcException): @@ -56,7 +56,7 @@ class ParametrizedDumpError(DvcException): def is_valid_filename(path): return path.endswith(DVC_FILE_SUFFIX) or os.path.basename(path) in [ DVC_FILE, - PIPELINE_FILE, + PROJECT_FILE, ] @@ -67,7 +67,7 @@ def is_dvc_file(path): def is_lock_file(path): - return os.path.basename(path) == PIPELINE_LOCK + return os.path.basename(path) == LOCK_FILE def is_git_ignored(repo, path): @@ -87,7 +87,7 @@ def check_dvcfile_path(repo, path): raise StageFileBadNameError( "bad DVC file name '{}'. DVC files should be named " "'{}' or have a '.dvc' suffix (e.g. '{}.dvc').".format( - relpath(path), PIPELINE_FILE, os.path.basename(path) + relpath(path), PROJECT_FILE, os.path.basename(path) ) ) @@ -229,7 +229,7 @@ class SingleStageFile(FileMixin): self.dump(stage) -class PipelineFile(FileMixin): +class ProjectFile(FileMixin): """Abstraction for pipelines file, .yaml + .lock combined.""" from dvc.schema import COMPILED_MULTI_STAGE_SCHEMA as SCHEMA @@ -297,7 +297,7 @@ class PipelineFile(FileMixin): @property def stage(self): raise DvcException( - "PipelineFile has multiple stages. Please specify it's name." + "ProjectFile has multiple stages. Please specify it's name." ) @cached_property @@ -462,20 +462,10 @@ class Lockfile(FileMixin): raise NotImplementedError -class Dvcfile: - def __new__(cls, repo: "Repo", path: AnyPath, **kwargs: Any): - assert path - assert repo - - return make_dvcfile(repo, path, **kwargs) - - -DVCFile = Union["PipelineFile", "SingleStageFile"] - - -def make_dvcfile(repo: "Repo", path: AnyPath, **kwargs: Any) -> DVCFile: - _, ext = os.path.splitext(str(path)) - if ext in [".yaml", ".yml"]: - return PipelineFile(repo, path, **kwargs) - # fallback to single stage file for better error messages +def load_file( + repo: "Repo", path: StrOrBytesPath, **kwargs: Any +) -> Union[ProjectFile, SingleStageFile]: + _, ext = os.path.splitext(path) + if ext in (".yaml", ".yml"): + return ProjectFile(repo, path, **kwargs) return SingleStageFile(repo, path, **kwargs) diff --git a/dvc/repo/collect.py b/dvc/repo/collect.py index b79bb73bb..f51b883cf 100644 --- a/dvc/repo/collect.py +++ b/dvc/repo/collect.py @@ -1,8 +1,6 @@ import logging from typing import TYPE_CHECKING, Callable, Iterable, List, Tuple -from dvc.types import AnyPath - if TYPE_CHECKING: from dvc.output import Output from dvc.repo import Repo @@ -12,7 +10,6 @@ logger = logging.getLogger(__name__) FilterFn = Callable[["Output"], bool] Outputs = List["Output"] -AnyPaths = List[AnyPath] StrPaths = List[str] diff --git a/dvc/repo/commit.py b/dvc/repo/commit.py index c6840a9af..4747eac3e 100644 --- a/dvc/repo/commit.py +++ b/dvc/repo/commit.py @@ -45,8 +45,6 @@ def commit( allow_missing=False, data_only=False, ): - from dvc.dvcfile import Dvcfile - stages_info = [ info for info in self.stage.collect_granular( @@ -66,6 +64,5 @@ def commit( stage.commit( filter_info=stage_info.filter_info, allow_missing=allow_missing ) - - Dvcfile(self, stage.path).dump(stage, update_pipeline=False) + stage.dump(update_pipeline=False) return [s.stage for s in stages_info] diff --git a/dvc/repo/experiments/init.py b/dvc/repo/experiments/init.py index b16c6e9af..c26172527 100644 --- a/dvc/repo/experiments/init.py +++ b/dvc/repo/experiments/init.py @@ -22,8 +22,8 @@ from dvc.stage import PipelineStage from dvc.types import OptStr if TYPE_CHECKING: + from dvc.dvcfile import ProjectFile, SingleStageFile from dvc.repo import Repo - from dvc.dvcfile import DVCFile from dvc.dependency import Dependency from dvc.ui import ui @@ -112,7 +112,9 @@ def init_interactive( def _check_stage_exists( - dvcfile: "DVCFile", name: str, force: bool = False + dvcfile: Union["ProjectFile", "SingleStageFile"], + name: str, + force: bool = False, ) -> None: if not force and dvcfile.exists() and name in dvcfile.stages: from dvc.stage.exceptions import DuplicateStageName @@ -206,9 +208,9 @@ def init( force: bool = False, stream: Optional[TextIO] = None, ) -> Tuple[PipelineStage, List["Dependency"], List[str]]: - from dvc.dvcfile import make_dvcfile + from dvc.dvcfile import PROJECT_FILE, load_file - dvcfile = make_dvcfile(repo, "dvc.yaml") + dvcfile = load_file(repo, PROJECT_FILE) _check_stage_exists(dvcfile, name, force=force) defaults = defaults.copy() if defaults else {} diff --git a/dvc/repo/imp_url.py b/dvc/repo/imp_url.py index 40773f4bc..1237f92c5 100644 --- a/dvc/repo/imp_url.py +++ b/dvc/repo/imp_url.py @@ -33,7 +33,7 @@ def imp_url( # noqa: C901 fs_config=None, version_aware: bool = False, ): - from dvc.dvcfile import make_dvcfile + from dvc.dvcfile import load_file from dvc.stage import Stage, create_stage, restore_fields out = resolve_output(url, out) @@ -78,7 +78,7 @@ def imp_url( # noqa: C901 out_obj = stage.outs[0] out_obj.annot.update(desc=desc, type=type, labels=labels, meta=meta) - dvcfile = make_dvcfile(self, stage.path) + dvcfile = load_file(self, stage.path) dvcfile.remove() try: diff --git a/dvc/repo/index.py b/dvc/repo/index.py index a51d20df1..f8be71510 100644 --- a/dvc/repo/index.py +++ b/dvc/repo/index.py @@ -158,9 +158,9 @@ class Index: @classmethod def from_file(cls, repo: "Repo", path: str) -> "Index": - from dvc.dvcfile import make_dvcfile + from dvc.dvcfile import load_file - dvcfile = make_dvcfile(repo, path) + dvcfile = load_file(repo, path) return cls( repo, stages=list(dvcfile.stages.values()), diff --git a/dvc/repo/move.py b/dvc/repo/move.py index 245b4e26a..78ca603a2 100644 --- a/dvc/repo/move.py +++ b/dvc/repo/move.py @@ -33,7 +33,7 @@ def move(self, from_path, to_path): also known as data sources. """ import dvc.output as Output - from dvc.dvcfile import DVC_FILE_SUFFIX, Dvcfile + from dvc.dvcfile import DVC_FILE_SUFFIX from dvc.stage import Stage from_out = Output.loads_from(Stage(self), [from_path])[0] @@ -77,5 +77,4 @@ def move(self, from_path, to_path): out.move(to_out) stage.save() - - Dvcfile(self, stage.path).dump(stage) + stage.dump() diff --git a/dvc/repo/plots/__init__.py b/dvc/repo/plots/__init__.py index 05de9fbba..5ade3eaf3 100644 --- a/dvc/repo/plots/__init__.py +++ b/dvc/repo/plots/__init__.py @@ -230,7 +230,6 @@ class Plots: out.plot.pop(prop) def modify(self, path, props=None, unset=None): - from dvc.dvcfile import Dvcfile from dvc_render.vega_templates import get_template props = props or {} @@ -256,9 +255,7 @@ class Plots: out.plot = True out.verify_metric() - - dvcfile = Dvcfile(self.repo, out.stage.path) - dvcfile.dump(out.stage, update_lock=False) + out.stage.dump(update_lock=False) @cached_property def templates_dir(self) -> Optional[str]: diff --git a/dvc/repo/reproduce.py b/dvc/repo/reproduce.py index 16545cd81..37f1c0dd1 100644 --- a/dvc/repo/reproduce.py +++ b/dvc/repo/reproduce.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) def _reproduce_stage(stage: "Stage", **kwargs): def _run_callback(repro_callback): - _dump_stage(stage) + stage.dump(update_pipeline=False) _track_stage(stage) repro_callback([stage]) @@ -45,20 +45,13 @@ def _reproduce_stage(stage: "Stage", **kwargs): if not kwargs.get("dry", False): track = checkpoint_func is not None - _dump_stage(stage) + stage.dump(update_pipeline=False) if track: _track_stage(stage) return [stage] -def _dump_stage(stage): - from dvc.dvcfile import Dvcfile - - dvcfile = Dvcfile(stage.repo, stage.path) - dvcfile.dump(stage, update_pipeline=False) - - def _get_stage_files(stage: "Stage") -> Iterator[str]: yield stage.dvcfile.relpath for dep in stage.deps: @@ -102,9 +95,9 @@ def reproduce( # noqa: C901 targets = [targets] if not all_pipelines and not targets: - from dvc.dvcfile import PIPELINE_FILE + from dvc.dvcfile import PROJECT_FILE - targets = [PIPELINE_FILE] + targets = [PROJECT_FILE] interactive = kwargs.get("interactive", False) if not interactive: diff --git a/dvc/repo/stage.py b/dvc/repo/stage.py index 7163f924d..f6d8f8a9c 100644 --- a/dvc/repo/stage.py +++ b/dvc/repo/stage.py @@ -20,7 +20,7 @@ if typing.TYPE_CHECKING: from dvc.stage.loader import StageLoader from dvc.types import OptStr -PIPELINE_FILE = "dvc.yaml" +PROJECT_FILE = "dvc.yaml" class StageInfo(NamedTuple): @@ -48,9 +48,9 @@ def _maybe_collect_from_dvc_yaml( from dvc.stage.exceptions import StageNotFound stages: StageList = [] - if loader.fs.exists(PIPELINE_FILE): + if loader.fs.exists(PROJECT_FILE): with suppress(StageNotFound): - stages = loader.load_all(PIPELINE_FILE, target, **load_kwargs) + stages = loader.load_all(PROJECT_FILE, target, **load_kwargs) if with_deps: return _collect_with_deps(stages, loader.repo.index.graph) return stages @@ -74,7 +74,7 @@ def _collect_specific_target( # `dvc.yaml` stage name here. If it exists, then we move on. # else, we assume it's a output name in the `collect_granular()` below msg = "Checking if stage '%s' is in '%s'" - logger.debug(msg, target, PIPELINE_FILE) + logger.debug(msg, target, PROJECT_FILE) if not (recursive and loader.fs.isdir(target)): stages = _maybe_collect_from_dvc_yaml(loader, target, with_deps) if stages: @@ -170,7 +170,7 @@ class StageLoad: stage_cls = Stage path = fname or prepare_file_path(stage_data) else: - path = PIPELINE_FILE + path = PROJECT_FILE stage_cls = PipelineStage stage_name = stage_data["name"] if not (stage_name and is_valid_name(stage_name)): @@ -214,7 +214,7 @@ class StageLoad: if path: return self.repo.fs.path.realpath(path) - path = PIPELINE_FILE + path = PROJECT_FILE logger.debug("Assuming '%s' to be a stage inside '%s'", name, path) return path @@ -260,11 +260,11 @@ class StageLoad: glob: if true, `name` is considered as a glob, which is used to filter list of stages from the given `path`. """ - from dvc.dvcfile import Dvcfile + from dvc.dvcfile import load_file from dvc.stage.loader import SingleStageLoader, StageLoader path = self._get_filepath(path, name) - dvcfile = Dvcfile(self.repo, path) + dvcfile = load_file(self.repo, path) # `dvcfile.stages` is not cached stages = dvcfile.stages # type: ignore[attr-defined] @@ -283,11 +283,10 @@ class StageLoad: path: if not provided, default `dvc.yaml` is assumed. name: required for `dvc.yaml` files, ignored for `.dvc` files. """ - from dvc.dvcfile import Dvcfile + from dvc.dvcfile import load_file path = self._get_filepath(path, name) - dvcfile = Dvcfile(self.repo, path) - + dvcfile = load_file(self.repo, path) stages = dvcfile.stages # type: ignore[attr-defined] return stages[name] diff --git a/dvc/repo/update.py b/dvc/repo/update.py index 19a4df8df..74a7522df 100644 --- a/dvc/repo/update.py +++ b/dvc/repo/update.py @@ -20,8 +20,6 @@ def update( # noqa: C901 remote=None, jobs=None, ): - from dvc.dvcfile import Dvcfile - from .worktree import update_worktree_stages if not targets: @@ -61,8 +59,7 @@ def update( # noqa: C901 no_download=no_download, jobs=jobs, ) - dvcfile = Dvcfile(self, stage.path) - dvcfile.dump(stage) + stage.dump() if other_stage_infos: remote_obj = self.cloud.get_remote(name=remote) diff --git a/dvc/stage/__init__.py b/dvc/stage/__init__.py index 808682782..6fa16d0b8 100644 --- a/dvc/stage/__init__.py +++ b/dvc/stage/__init__.py @@ -4,7 +4,7 @@ import string from collections import defaultdict from contextlib import suppress from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Dict, Optional, Set +from typing import TYPE_CHECKING, Any, Dict, Optional, Set, Union from funcy import project @@ -36,7 +36,7 @@ from .utils import ( ) if TYPE_CHECKING: - from dvc.dvcfile import DVCFile + from dvc.dvcfile import ProjectFile, SingleStageFile from dvc_data.hashfile.hash_info import HashInfo from dvc_objects.db.base import ObjectDB @@ -173,7 +173,7 @@ class Stage(params.StageParams): self.__dict__.pop("relpath", None) @property - def dvcfile(self) -> "DVCFile": + def dvcfile(self) -> Union["ProjectFile", "SingleStageFile"]: if self.path and self._dvcfile and self.path == self._dvcfile.path: return self._dvcfile @@ -183,13 +183,15 @@ class Stage(params.StageParams): "and is detached from dvcfile." ) - from dvc.dvcfile import make_dvcfile + from dvc.dvcfile import load_file - self._dvcfile = make_dvcfile(self.repo, self.path) + self._dvcfile = load_file(self.repo, self.path) return self._dvcfile @dvcfile.setter - def dvcfile(self, dvcfile: "DVCFile") -> None: + def dvcfile( + self, dvcfile: Union["ProjectFile", "SingleStageFile"] + ) -> None: self._dvcfile = dvcfile def __repr__(self): @@ -785,9 +787,9 @@ class PipelineStage(Stage): @property def addressing(self): - from dvc.dvcfile import PIPELINE_FILE + from dvc.dvcfile import PROJECT_FILE - if self.path and self.relpath == PIPELINE_FILE: + if self.path and self.relpath == PROJECT_FILE: return self.name return f"{super().addressing}:{self.name}" diff --git a/dvc/stage/utils.py b/dvc/stage/utils.py index 428a13b89..d2da7b9f9 100644 --- a/dvc/stage/utils.py +++ b/dvc/stage/utils.py @@ -256,14 +256,14 @@ def prepare_file_path(kwargs): def check_stage_exists( repo: "Repo", stage: Union["Stage", "PipelineStage"], path: str ): - from dvc.dvcfile import make_dvcfile + from dvc.dvcfile import load_file from dvc.stage import PipelineStage from dvc.stage.exceptions import ( DuplicateStageName, StageFileAlreadyExistsError, ) - dvcfile = make_dvcfile(repo, path) + dvcfile = load_file(repo, path) if not dvcfile.exists(): return diff --git a/dvc/types.py b/dvc/types.py index cca8e17a0..0b6966975 100644 --- a/dvc/types.py +++ b/dvc/types.py @@ -1,10 +1,12 @@ -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, AnyStr, Dict, List, Optional, Union if TYPE_CHECKING: from os import PathLike StrPath = Union[str, "PathLike[str]"] -AnyPath = Union[str, StrPath] +BytesPath = Union[bytes, "PathLike[bytes]"] +GenericPath = Union[AnyStr, "PathLike[AnyStr]"] +StrOrBytesPath = Union[str, bytes, "PathLike[str]", "PathLike[bytes]"] OptStr = Optional[str] TargetType = Union[List[str], str] diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 04f0542e2..ae57b541a 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -333,14 +333,14 @@ def error_link(name): def parse_target( target: str, default: str = None, isa_glob: bool = False ) -> Tuple[Optional[str], Optional[str]]: - from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK, is_valid_filename + from dvc.dvcfile import LOCK_FILE, PROJECT_FILE, is_valid_filename from dvc.exceptions import DvcException from dvc.parsing import JOIN if not target: return None, None - default = default or PIPELINE_FILE + default = default or PROJECT_FILE if isa_glob: path, _, glob = target.rpartition(":") return path or default, glob or None @@ -360,7 +360,7 @@ def parse_target( name += f"{JOIN}{key}" if path: - if os.path.basename(path) == PIPELINE_LOCK: + if os.path.basename(path) == LOCK_FILE: raise DvcException( "Did you mean: `{}`?".format( target.replace(".lock", ".yaml", 1) diff --git a/dvc/utils/serialize/_common.py b/dvc/utils/serialize/_common.py index 5fab60943..606393531 100644 --- a/dvc/utils/serialize/_common.py +++ b/dvc/utils/serialize/_common.py @@ -17,36 +17,36 @@ from dvc.exceptions import DvcException if TYPE_CHECKING: from dvc.fs import FileSystem - from dvc.types import AnyPath + from dvc.types import StrPath class DumperFn(Protocol): def __call__( - self, path: "AnyPath", data: Any, fs: "FileSystem" = None + self, path: "StrPath", data: Any, fs: "FileSystem" = None ) -> Any: ... class ModifierFn(Protocol): def __call__( - self, path: "AnyPath", fs: "FileSystem" = None + self, path: "StrPath", fs: "FileSystem" = None ) -> ContextManager[Dict]: ... class LoaderFn(Protocol): - def __call__(self, path: "AnyPath", fs: "FileSystem" = None) -> Any: + def __call__(self, path: "StrPath", fs: "FileSystem" = None) -> Any: ... ReadType = Union[bytes, None, str] -ParserFn = Callable[[ReadType, "AnyPath"], dict] +ParserFn = Callable[[ReadType, "StrPath"], dict] class ParseError(DvcException): """Errors while parsing files""" - def __init__(self, path: "AnyPath", message: str): + def __init__(self, path: "StrPath", message: str): from dvc.utils import relpath path = relpath(path) @@ -57,12 +57,12 @@ class ParseError(DvcException): class EncodingError(ParseError): """We could not read a file with the given encoding""" - def __init__(self, path: "AnyPath", encoding: str): + def __init__(self, path: "StrPath", encoding: str): self.encoding = encoding super().__init__(path, f"is not valid {encoding}") -def _load_data(path: "AnyPath", parser: ParserFn, fs: "FileSystem" = None): +def _load_data(path: "StrPath", parser: ParserFn, fs: "FileSystem" = None): open_fn = fs.open if fs else open encoding = "utf-8" with open_fn(path, encoding=encoding) as fd: @@ -84,7 +84,7 @@ def _dump_data( @contextmanager def _modify_data( - path: "AnyPath", + path: "StrPath", parser: ParserFn, dumper: DumperFn, fs: "FileSystem" = None, diff --git a/tests/func/experiments/test_experiments.py b/tests/func/experiments/test_experiments.py index 5a6ecbe53..fb94ee4b0 100644 --- a/tests/func/experiments/test_experiments.py +++ b/tests/func/experiments/test_experiments.py @@ -9,7 +9,7 @@ import dulwich.file import pytest from funcy import first -from dvc.dvcfile import PIPELINE_FILE +from dvc.dvcfile import PROJECT_FILE from dvc.exceptions import ReproductionError from dvc.repo.experiments.exceptions import ExperimentExistsError from dvc.repo.experiments.queue.base import BaseStashQueue @@ -505,7 +505,7 @@ def test_subdir(tmp_dir, scm, dvc, workspace): scm.commit("init") results = dvc.experiments.run( - PIPELINE_FILE, params=["foo=2"], tmp_dir=not workspace + PROJECT_FILE, params=["foo=2"], tmp_dir=not workspace ) assert results @@ -549,7 +549,7 @@ def test_subrepo(tmp_dir, scm, workspace): scm.commit("init") results = subrepo.dvc.experiments.run( - PIPELINE_FILE, params=["foo=2"], tmp_dir=not workspace + PROJECT_FILE, params=["foo=2"], tmp_dir=not workspace ) assert results diff --git a/tests/func/metrics/test_show.py b/tests/func/metrics/test_show.py index 1b4c207d3..9816037b7 100644 --- a/tests/func/metrics/test_show.py +++ b/tests/func/metrics/test_show.py @@ -3,7 +3,7 @@ import os import pytest from funcy import get_in -from dvc.dvcfile import PIPELINE_FILE +from dvc.dvcfile import PROJECT_FILE from dvc.exceptions import OverlappingOutputPathsError from dvc.repo import Repo from dvc.utils.fs import remove @@ -303,7 +303,7 @@ def test_metrics_show_overlap( @pytest.mark.parametrize( "file,error_path", ( - (PIPELINE_FILE, ["workspace", "error"]), + (PROJECT_FILE, ["workspace", "error"]), ("metrics.yaml", ["workspace", "data", "metrics.yaml", "error"]), ), ) diff --git a/tests/func/params/test_diff.py b/tests/func/params/test_diff.py index c694e7a43..a910468e5 100644 --- a/tests/func/params/test_diff.py +++ b/tests/func/params/test_diff.py @@ -113,12 +113,12 @@ def test_diff_with_unchanged(tmp_dir, scm, dvc): def test_pipeline_tracked_params(tmp_dir, scm, dvc, run_copy): - from dvc.dvcfile import PIPELINE_FILE + from dvc.dvcfile import PROJECT_FILE tmp_dir.gen({"foo": "foo", "params.yaml": "foo: bar\nxyz: val"}) run_copy("foo", "bar", name="copy-foo-bar", params=["foo,xyz"]) - scm.add(["params.yaml", PIPELINE_FILE]) + scm.add(["params.yaml", PROJECT_FILE]) scm.commit("add stage") tmp_dir.scm_gen("params.yaml", "foo: baz\nxyz: val", commit="baz") @@ -182,7 +182,7 @@ def test_vars_shows_on_params_diff(tmp_dir, scm, dvc): def test_diff_targeted(tmp_dir, scm, dvc, run_copy): - from dvc.dvcfile import PIPELINE_FILE + from dvc.dvcfile import PROJECT_FILE tmp_dir.gen( { @@ -198,7 +198,7 @@ def test_diff_targeted(tmp_dir, scm, dvc, run_copy): params=["foo", "other_params.yaml:xyz"], ) - scm.add(["params.yaml", "other_params.yaml", PIPELINE_FILE]) + scm.add(["params.yaml", "other_params.yaml", PROJECT_FILE]) scm.commit("add stage") tmp_dir.scm_gen( diff --git a/tests/func/params/test_show.py b/tests/func/params/test_show.py index 1c11e41a7..effa3b890 100644 --- a/tests/func/params/test_show.py +++ b/tests/func/params/test_show.py @@ -4,7 +4,7 @@ from functools import reduce import pytest from dvc.repo import Repo -from dvc.repo.stage import PIPELINE_FILE +from dvc.repo.stage import PROJECT_FILE from dvc.utils.serialize import YAMLFileCorruptedError @@ -112,7 +112,7 @@ def test_pipeline_params(tmp_dir, scm, dvc, run_copy): {"foo": "foo", "params.yaml": "foo: bar\nxyz: val\nabc: ignore"} ) run_copy("foo", "bar", name="copy-foo-bar", params=["foo,xyz"]) - scm.add(["params.yaml", PIPELINE_FILE]) + scm.add(["params.yaml", PROJECT_FILE]) scm.commit("add stage") tmp_dir.scm_gen( @@ -155,7 +155,7 @@ def test_show_no_repo(tmp_dir): @pytest.mark.parametrize( "file,error_path", ( - (PIPELINE_FILE, ["v1", "error"]), + (PROJECT_FILE, ["v1", "error"]), ("params_other.yaml", ["v1", "data", "params_other.yaml", "error"]), ), ) @@ -171,7 +171,7 @@ def test_log_errors(tmp_dir, scm, dvc, capsys, file, error_path): with open(tmp_dir / file, "a", encoding="utf-8") as fd: fd.write("\nmalformed!") - scm.add([PIPELINE_FILE, "params_other.yaml"]) + scm.add([PROJECT_FILE, "params_other.yaml"]) scm.commit("init") scm.tag("v1") @@ -211,7 +211,7 @@ def test_deps_multi_stage(tmp_dir, scm, dvc, run_copy): run_copy("foo", "bar", name="copy-foo-bar", params=["foo"]) run_copy("foo", "bar1", name="copy-foo-bar-1", params=["xyz"]) - scm.add(["params.yaml", PIPELINE_FILE]) + scm.add(["params.yaml", PROJECT_FILE]) scm.commit("add stage") assert dvc.params.show(revs=["master"], deps=True) == { @@ -228,7 +228,7 @@ def test_deps_with_targets(tmp_dir, scm, dvc, run_copy): run_copy("foo", "bar", name="copy-foo-bar", params=["foo"]) run_copy("foo", "bar1", name="copy-foo-bar-1", params=["xyz"]) - scm.add(["params.yaml", PIPELINE_FILE]) + scm.add(["params.yaml", PROJECT_FILE]) scm.commit("add stage") assert dvc.params.show(targets=["params.yaml"], deps=True) == { @@ -246,6 +246,6 @@ def test_deps_with_bad_target(tmp_dir, scm, dvc, run_copy): ) run_copy("foo", "bar", name="copy-foo-bar", params=["foo"]) run_copy("foo", "bar1", name="copy-foo-bar-1", params=["xyz"]) - scm.add(["params.yaml", PIPELINE_FILE]) + scm.add(["params.yaml", PROJECT_FILE]) scm.commit("add stage") assert dvc.params.show(targets=["foobar"], deps=True) == {} diff --git a/tests/func/plots/test_modify.py b/tests/func/plots/test_modify.py index 66b71d744..566953517 100644 --- a/tests/func/plots/test_modify.py +++ b/tests/func/plots/test_modify.py @@ -1,6 +1,6 @@ import pytest -from dvc.dvcfile import PIPELINE_LOCK +from dvc.dvcfile import LOCK_FILE from dvc.repo.plots import PropsNotFoundError from dvc.utils import relpath from tests.utils.plots import get_plot @@ -37,11 +37,11 @@ def test_plots_modify_should_not_change_lockfile( single_stage=False, ) - (tmp_dir / PIPELINE_LOCK).unlink() + (tmp_dir / LOCK_FILE).unlink() dvc.plots.modify( "metric.json", props={"template": relpath(custom_template)} ) - assert not (tmp_dir / PIPELINE_LOCK).exists() + assert not (tmp_dir / LOCK_FILE).exists() def test_plots_modify_not_existing_template(dvc): diff --git a/tests/func/plots/test_show.py b/tests/func/plots/test_show.py index def95b874..2e52bbe97 100644 --- a/tests/func/plots/test_show.py +++ b/tests/func/plots/test_show.py @@ -3,7 +3,7 @@ import os import pytest from dvc.cli import main -from dvc.dvcfile import PIPELINE_FILE +from dvc.dvcfile import PROJECT_FILE from dvc.exceptions import OverlappingOutputPathsError from dvc.repo import Repo from dvc.repo.plots import PlotMetricTypeError @@ -227,7 +227,7 @@ def test_ignore_parsing_error(tmp_dir, dvc, run_copy_metrics): @pytest.mark.parametrize( "file,path_kwargs", ( - (PIPELINE_FILE, {"revision": "workspace", "endkey": "error"}), + (PROJECT_FILE, {"revision": "workspace", "endkey": "error"}), ( "plot.yaml", {"revision": "workspace", "file": "plot.yaml", "endkey": "error"}, diff --git a/tests/func/test_add.py b/tests/func/test_add.py index c13514da9..0cff482a6 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -762,13 +762,13 @@ def test_output_duplication_for_pipeline_tracked(tmp_dir, dvc, run_copy): def test_add_pipeline_file(tmp_dir, dvc, run_copy): - from dvc.dvcfile import PIPELINE_FILE + from dvc.dvcfile import PROJECT_FILE tmp_dir.dvc_gen("foo", "foo") run_copy("foo", "bar", name="copy-foo-bar") with pytest.raises(OutputIsStageFileError): - dvc.add(PIPELINE_FILE) + dvc.add(PROJECT_FILE) def test_add_symlink_file(tmp_dir, dvc): diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index 598a271a1..d64b82a35 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -7,7 +7,7 @@ import textwrap import pytest from dvc.cli import main -from dvc.dvcfile import PIPELINE_FILE, Dvcfile +from dvc.dvcfile import PROJECT_FILE, load_file from dvc.exceptions import ( CheckoutError, CheckoutErrorSuggestGit, @@ -496,7 +496,7 @@ def test_checkout_stats_on_failure(tmp_dir, dvc, scm): {"foo": "foo", "dir": {"subdir": {"file": "file"}}, "other": "other"}, commit="initial", ) - stage = Dvcfile(dvc, "foo.dvc").stage + stage = load_file(dvc, "foo.dvc").stage tmp_dir.dvc_gen({"foo": "foobar", "other": "other other"}, commit="second") # corrupt cache @@ -673,7 +673,7 @@ def test_checkouts_with_different_addressing(tmp_dir, dvc, run_copy): (tmp_dir / "bar").unlink() (tmp_dir / "ipsum").unlink() - assert set(dvc.checkout(PIPELINE_FILE)["added"]) == {"bar", "ipsum"} + assert set(dvc.checkout(PROJECT_FILE)["added"]) == {"bar", "ipsum"} (tmp_dir / "bar").unlink() (tmp_dir / "ipsum").unlink() @@ -721,7 +721,7 @@ def test_checkouts_for_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy): assert dvc.checkout(["bar"])["added"] == ["bar"] (tmp_dir / "bar").unlink() - assert set(dvc.checkout([PIPELINE_FILE])["added"]) == {"bar", "ipsum"} + assert set(dvc.checkout([PROJECT_FILE])["added"]) == {"bar", "ipsum"} for out in ["bar", "ipsum"]: (tmp_dir / out).unlink() diff --git a/tests/func/test_commit.py b/tests/func/test_commit.py index e24330e1d..d58bce908 100644 --- a/tests/func/test_commit.py +++ b/tests/func/test_commit.py @@ -4,7 +4,7 @@ import textwrap import pytest from dvc.dependency.base import DependencyDoesNotExistError -from dvc.dvcfile import PIPELINE_FILE +from dvc.dvcfile import PROJECT_FILE from dvc.output import OutputDoesNotExistError from dvc.stage.exceptions import StageCommitError @@ -234,8 +234,8 @@ def test_commit_pipeline_stage(tmp_dir, dvc, run_copy): # just to confirm different variants work assert dvc.commit(f":{stage.addressing}") == [stage] - assert dvc.commit(f"{PIPELINE_FILE}:{stage.addressing}") == [stage] - assert dvc.commit(PIPELINE_FILE) == [stage] + assert dvc.commit(f"{PROJECT_FILE}:{stage.addressing}") == [stage] + assert dvc.commit(PROJECT_FILE) == [stage] def test_imported_entries_unchanged(tmp_dir, dvc, erepo_dir): diff --git a/tests/func/test_dvcfile.py b/tests/func/test_dvcfile.py index adcd8f4aa..75ff86b31 100644 --- a/tests/func/test_dvcfile.py +++ b/tests/func/test_dvcfile.py @@ -5,11 +5,11 @@ import pytest from dvc.annotations import Annotation from dvc.dvcfile import ( - PIPELINE_FILE, - PIPELINE_LOCK, - Dvcfile, + LOCK_FILE, + PROJECT_FILE, ParametrizedDumpError, SingleStageFile, + load_file, ) from dvc.stage.exceptions import StageFileDoesNotExistError from dvc.stage.loader import StageNotFound @@ -35,7 +35,7 @@ def test_run_load_one_for_multistage(tmp_dir, dvc): outs_persist_no_cache=["foo2"], always_changed=True, ) - stage2 = Dvcfile(dvc, PIPELINE_FILE).stages["copy-foo-foo2"] + stage2 = load_file(dvc, PROJECT_FILE).stages["copy-foo-foo2"] assert stage1 == stage2 foo_out = stage2.outs[0] assert stage2.cmd == "cp foo foo2" @@ -49,7 +49,7 @@ def test_run_load_one_for_multistage(tmp_dir, dvc): def test_run_load_one_for_multistage_non_existing(tmp_dir, dvc): with pytest.raises(StageFileDoesNotExistError): - assert Dvcfile(dvc, PIPELINE_FILE).stages.get("copy-foo-foo2") + assert load_file(dvc, PROJECT_FILE).stages.get("copy-foo-foo2") def test_run_load_one_for_multistage_non_existing_stage_name(tmp_dir, dvc): @@ -62,7 +62,7 @@ def test_run_load_one_for_multistage_non_existing_stage_name(tmp_dir, dvc): always_changed=True, ) with pytest.raises(StageNotFound): - assert Dvcfile(dvc, stage.path).stages["random-name"] + assert load_file(dvc, stage.path).stages["random-name"] def test_run_load_one_on_single_stage(tmp_dir, dvc): @@ -74,9 +74,9 @@ def test_run_load_one_on_single_stage(tmp_dir, dvc): always_changed=True, single_stage=True, ) - assert isinstance(Dvcfile(dvc, stage.path), SingleStageFile) - assert Dvcfile(dvc, stage.path).stages.get("random-name") == stage - assert Dvcfile(dvc, stage.path).stage == stage + assert isinstance(load_file(dvc, stage.path), SingleStageFile) + assert load_file(dvc, stage.path).stages.get("random-name") == stage + assert load_file(dvc, stage.path).stage == stage def test_has_stage_with_name(tmp_dir, dvc): @@ -88,7 +88,7 @@ def test_has_stage_with_name(tmp_dir, dvc): metrics=["foo2"], always_changed=True, ) - dvcfile = Dvcfile(dvc, PIPELINE_FILE) + dvcfile = load_file(dvc, PROJECT_FILE) assert "copy-foo-foo2" in dvcfile.stages assert "copy" not in dvcfile.stages @@ -102,7 +102,7 @@ def test_load_all_multistage(tmp_dir, dvc): metrics=["foo2"], always_changed=True, ) - stages = Dvcfile(dvc, PIPELINE_FILE).stages.values() + stages = load_file(dvc, PROJECT_FILE).stages.values() assert len(stages) == 1 assert list(stages) == [stage1] @@ -114,7 +114,10 @@ def test_load_all_multistage(tmp_dir, dvc): metrics=["bar2"], always_changed=True, ) - assert set(Dvcfile(dvc, PIPELINE_FILE).stages.values()) == {stage2, stage1} + assert set(load_file(dvc, PROJECT_FILE).stages.values()) == { + stage2, + stage1, + } def test_load_all_singlestage(tmp_dir, dvc): @@ -126,7 +129,7 @@ def test_load_all_singlestage(tmp_dir, dvc): always_changed=True, single_stage=True, ) - dvcfile = Dvcfile(dvc, "foo2.dvc") + dvcfile = load_file(dvc, "foo2.dvc") assert isinstance(dvcfile, SingleStageFile) assert len(dvcfile.stages) == 1 stages = dvcfile.stages.values() @@ -146,7 +149,7 @@ def test_try_get_single_stage_from_pipeline_file(tmp_dir, dvc): always_changed=True, ) with pytest.raises(DvcException): - assert Dvcfile(dvc, PIPELINE_FILE).stage + assert load_file(dvc, PROJECT_FILE).stage def test_stage_collection(tmp_dir, dvc): @@ -180,7 +183,7 @@ def test_remove_stage(tmp_dir, dvc, run_copy): stage = run_copy("foo", "bar", name="copy-foo-bar") stage2 = run_copy("bar", "foobar", name="copy-bar-foobar") - dvc_file = Dvcfile(dvc, PIPELINE_FILE) + dvc_file = load_file(dvc, PROJECT_FILE) assert dvc_file.exists() assert {"copy-bar-foobar", "copy-foo-bar"} == set( dvc_file._load()[0]["stages"].keys() @@ -205,7 +208,7 @@ def test_remove_stage_lockfile(tmp_dir, dvc, run_copy): stage = run_copy("foo", "bar", name="copy-foo-bar") stage2 = run_copy("bar", "foobar", name="copy-bar-foobar") - dvc_file = Dvcfile(dvc, PIPELINE_FILE) + dvc_file = load_file(dvc, PROJECT_FILE) lock_file = dvc_file._lockfile assert dvc_file.exists() assert lock_file.exists() @@ -230,7 +233,7 @@ def test_remove_stage_dvcfiles(tmp_dir, dvc, run_copy): tmp_dir.gen("foo", "foo") stage = run_copy("foo", "bar", single_stage=True) - dvc_file = Dvcfile(dvc, stage.path) + dvc_file = load_file(dvc, stage.path) assert dvc_file.exists() dvc_file.remove_stage(stage) assert not dvc_file.exists() @@ -246,7 +249,7 @@ def test_remove_stage_dvcfiles(tmp_dir, dvc, run_copy): def test_remove_stage_on_lockfile_format_error(tmp_dir, dvc, run_copy): tmp_dir.gen("foo", "foo") stage = run_copy("foo", "bar", name="copy-foo-bar") - dvc_file = Dvcfile(dvc, stage.path) + dvc_file = load_file(dvc, stage.path) lock_file = dvc_file._lockfile data = dvc_file._load()[0] @@ -285,19 +288,19 @@ def test_remove_stage_preserves_comment(tmp_dir, dvc, run_copy): ), ) - dvc.reproduce(PIPELINE_FILE) + dvc.reproduce(PROJECT_FILE) - dvc_file = Dvcfile(dvc, PIPELINE_FILE) + dvc_file = load_file(dvc, PROJECT_FILE) assert dvc_file.exists() - assert (tmp_dir / PIPELINE_LOCK).exists() + assert (tmp_dir / LOCK_FILE).exists() assert (tmp_dir / "foo").exists() assert (tmp_dir / "bar").exists() dvc_file.remove_stage(dvc_file.stages["copy-foo-bar"]) assert ( "# This copies 'foo' text to 'foo' file." - in (tmp_dir / PIPELINE_FILE).read_text() + in (tmp_dir / PROJECT_FILE).read_text() ) @@ -307,15 +310,15 @@ def test_remove_stage_removes_dvcfiles_if_no_stages_left( tmp_dir.gen("foo", "foo") run_copy("foo", "bar", name="run_copy") - dvc_file = Dvcfile(dvc, PIPELINE_FILE) + dvc_file = load_file(dvc, PROJECT_FILE) assert dvc_file.exists() - assert (tmp_dir / PIPELINE_LOCK).exists() + assert (tmp_dir / LOCK_FILE).exists() assert (tmp_dir / "foo").exists() dvc_file.remove_stage(dvc_file.stages["run_copy"]) assert not dvc_file.exists() - assert not (tmp_dir / PIPELINE_LOCK).exists() + assert not (tmp_dir / LOCK_FILE).exists() def test_dvcfile_dump_preserves_meta(tmp_dir, dvc, run_copy): diff --git a/tests/func/test_gc.py b/tests/func/test_gc.py index f810d3b77..f9f4bfd0a 100644 --- a/tests/func/test_gc.py +++ b/tests/func/test_gc.py @@ -227,7 +227,7 @@ def test_gc_cloud_remove_order(tmp_dir, scm, dvc, tmp_path_factory, mocker): def test_gc_not_collect_pipeline_tracked_files(tmp_dir, dvc, run_copy): - from dvc.dvcfile import PIPELINE_FILE, Dvcfile + from dvc.dvcfile import PROJECT_FILE, load_file tmp_dir.gen("foo", "foo") tmp_dir.gen("bar", "bar") @@ -239,7 +239,7 @@ def test_gc_not_collect_pipeline_tracked_files(tmp_dir, dvc, run_copy): assert _count_files(dvc.odb.local.path) == 1 # remove pipeline file and lockfile and check - Dvcfile(dvc, PIPELINE_FILE).remove(force=True) + load_file(dvc, PROJECT_FILE).remove(force=True) dvc.gc(workspace=True, force=True) assert _count_files(dvc.odb.local.path) == 0 diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 0ce2259ae..b1d83342e 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -255,13 +255,13 @@ def test_get_url_git_only_repo(tmp_dir, scm, caplog): def test_get_pipeline_tracked_outs( tmp_dir, dvc, scm, git_dir, run_copy, local_remote ): - from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK + from dvc.dvcfile import LOCK_FILE, PROJECT_FILE tmp_dir.gen("foo", "foo") run_copy("foo", "bar", name="copy-foo-bar") dvc.push() - dvc.scm.add([PIPELINE_FILE, PIPELINE_LOCK]) + dvc.scm.add([PROJECT_FILE, LOCK_FILE]) dvc.scm.commit("add pipeline stage") with git_dir.chdir(): diff --git a/tests/func/test_import.py b/tests/func/test_import.py index 9f51b1a15..1ac692173 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -7,7 +7,7 @@ from scmrepo.git import Git from dvc.annotations import Annotation from dvc.config import NoRemoteError -from dvc.dvcfile import Dvcfile +from dvc.dvcfile import load_file from dvc.exceptions import DownloadError, PathMissingError from dvc.fs import system from dvc.odbmgr import ODBManager @@ -205,7 +205,7 @@ def test_pull_imported_stage(tmp_dir, dvc, erepo_dir): erepo_dir.dvc_gen("foo", "foo content", commit="create foo") dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported") - dst_stage = Dvcfile(dvc, "foo_imported.dvc").stage + dst_stage = load_file(dvc, "foo_imported.dvc").stage dst_cache = dst_stage.outs[0].cache_path remove("foo_imported") @@ -224,7 +224,7 @@ def test_import_no_download(tmp_dir, scm, dvc, erepo_dir): assert not os.path.exists("foo_imported") - dst_stage = Dvcfile(dvc, "foo_imported.dvc").stage + dst_stage = load_file(dvc, "foo_imported.dvc").stage assert dst_stage.deps[0].def_repo == { "url": os.fspath(erepo_dir), @@ -250,7 +250,7 @@ def test_pull_import_no_download(tmp_dir, scm, dvc, erepo_dir): dvc.pull(["foo_imported.dvc"]) assert os.path.exists("foo_imported") - stage = Dvcfile(dvc, "foo_imported.dvc").stage + stage = load_file(dvc, "foo_imported.dvc").stage assert ( stage.outs[0].hash_info.value == "bdb8641831d8fcb03939637e09011c21.dir" @@ -363,7 +363,7 @@ def test_download_error_pulling_imported_stage( erepo_dir.dvc_gen("foo", "foo content", commit="create foo") dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported") - dst_stage = Dvcfile(dvc, "foo_imported.dvc").stage + dst_stage = load_file(dvc, "foo_imported.dvc").stage dst_cache = dst_stage.outs[0].cache_path remove("foo_imported") @@ -427,7 +427,7 @@ def test_pull_no_rev_lock(erepo_dir, tmp_dir, dvc): assert "rev" not in stage.deps[0].def_repo stage.deps[0].def_repo.pop("rev_lock") - Dvcfile(dvc, stage.path).dump(stage) + load_file(dvc, stage.path).dump(stage) remove(stage.outs[0].cache_path) (tmp_dir / "foo_imported").unlink() @@ -459,13 +459,13 @@ def test_import_from_bare_git_repo( def test_import_pipeline_tracked_outs( tmp_dir, dvc, scm, erepo_dir, run_copy, local_remote ): - from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK + from dvc.dvcfile import LOCK_FILE, PROJECT_FILE tmp_dir.gen("foo", "foo") run_copy("foo", "bar", name="copy-foo-bar") dvc.push() - dvc.scm.add([PIPELINE_FILE, PIPELINE_LOCK]) + dvc.scm.add([PROJECT_FILE, LOCK_FILE]) dvc.scm.commit("add pipeline stage") with erepo_dir.chdir(): diff --git a/tests/func/test_import_url.py b/tests/func/test_import_url.py index 05bde6a8c..bb2b502c9 100644 --- a/tests/func/test_import_url.py +++ b/tests/func/test_import_url.py @@ -8,7 +8,7 @@ import pytest from dvc.annotations import Annotation from dvc.cli import main from dvc.dependency.base import Dependency, DependencyDoesNotExistError -from dvc.dvcfile import Dvcfile +from dvc.dvcfile import load_file from dvc.exceptions import InvalidArgumentError from dvc.stage import Stage from dvc.testing.workspace_tests import TestImport as _TestImport @@ -304,7 +304,7 @@ def test_partial_import_pull(tmp_dir, scm, dvc, local_workspace): assert dst.exists() - stage = Dvcfile(dvc, "file.dvc").stage + stage = load_file(dvc, "file.dvc").stage assert stage.outs[0].hash_info.value == "d10b4c3ff123b26dc068d43a8bef2d23" assert stage.outs[0].meta.size == 12 diff --git a/tests/func/test_lockfile.py b/tests/func/test_lockfile.py index 22d403363..899f96fb5 100644 --- a/tests/func/test_lockfile.py +++ b/tests/func/test_lockfile.py @@ -5,7 +5,7 @@ from operator import itemgetter import pytest -from dvc.dvcfile import PIPELINE_LOCK, Lockfile +from dvc.dvcfile import LOCK_FILE, Lockfile from dvc.stage.utils import split_params_deps from dvc.utils.fs import remove from dvc.utils.serialize import dumps_yaml, parse_yaml_for_update @@ -22,7 +22,7 @@ FS_STRUCTURE = { } -def read_lock_file(file=PIPELINE_LOCK): +def read_lock_file(file=LOCK_FILE): with open(file, encoding="utf-8") as f: data = parse_yaml_for_update(f.read(), file) assert isinstance(data, OrderedDict) @@ -86,7 +86,7 @@ def test_order_is_preserved_when_pipeline_order_changes( new_lock_content = read_lock_file() assert_eq_lockfile(new_lock_content, initial_content) - (tmp_dir / PIPELINE_LOCK).unlink() + (tmp_dir / LOCK_FILE).unlink() assert dvc.reproduce(stage.addressing) == [stage] new_lock_content = read_lock_file() assert_eq_lockfile(new_lock_content, initial_content) @@ -152,12 +152,12 @@ def test_params_dump(tmp_dir, dvc, run_head): stage.dvcfile._dump_pipeline_file(stage) assert not dvc.reproduce(stage.addressing) - (tmp_dir / PIPELINE_LOCK).unlink() + (tmp_dir / LOCK_FILE).unlink() assert dvc.reproduce(stage.addressing) == [stage] assert_eq_lockfile(initial_content, read_lock_file()) # remove build-cache and check if the same structure is built - for item in [dvc.stage_cache.cache_dir, PIPELINE_LOCK]: + for item in [dvc.stage_cache.cache_dir, LOCK_FILE]: remove(item) assert dvc.reproduce(stage.addressing) == [stage] assert_eq_lockfile(initial_content, read_lock_file()) diff --git a/tests/func/test_ls.py b/tests/func/test_ls.py index bb63d688f..4d709d57d 100644 --- a/tests/func/test_ls.py +++ b/tests/func/test_ls.py @@ -458,11 +458,11 @@ def test_ls_not_existed_url(): def test_ls_shows_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy): - from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK + from dvc.dvcfile import LOCK_FILE, PROJECT_FILE tmp_dir.gen("foo", "foo") run_copy("foo", "bar", name="copy-foo-bar") - dvc.scm.add([PIPELINE_FILE, PIPELINE_LOCK]) + dvc.scm.add([PROJECT_FILE, LOCK_FILE]) dvc.scm.commit("add pipeline stage") files = Repo.ls(os.curdir, dvc_only=True) diff --git a/tests/func/test_repo.py b/tests/func/test_repo.py index d149bbedc..15e4a9e8d 100644 --- a/tests/func/test_repo.py +++ b/tests/func/test_repo.py @@ -1,4 +1,4 @@ -from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK +from dvc.dvcfile import LOCK_FILE, PROJECT_FILE from dvc.fs import system from dvc.odbmgr import ODBManager @@ -23,8 +23,8 @@ def test_destroy(tmp_dir, dvc, run_copy): "file.dvc", "file2.dvc", "dir.dvc", - PIPELINE_FILE, - PIPELINE_LOCK, + PROJECT_FILE, + LOCK_FILE, ]: assert not (tmp_dir / path).exists() diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py index e563dd73a..76770bb4d 100644 --- a/tests/func/test_repro.py +++ b/tests/func/test_repro.py @@ -7,7 +7,7 @@ from pathlib import Path import pytest from dvc.cli import main -from dvc.dvcfile import DVC_FILE, Dvcfile +from dvc.dvcfile import DVC_FILE, load_file from dvc.exceptions import ( CyclicGraphError, ReproductionError, @@ -170,7 +170,7 @@ class TestReproWorkingDirectoryAsOutput: dvc.index = dvc.index.update( [ nested_stage, - Dvcfile(dvc, error_stage_path).stage, + load_file(dvc, error_stage_path).stage, ] ) diff --git a/tests/func/test_repro_multistage.py b/tests/func/test_repro_multistage.py index 02dce839e..154546fa7 100644 --- a/tests/func/test_repro_multistage.py +++ b/tests/func/test_repro_multistage.py @@ -6,7 +6,7 @@ import pytest from funcy import lsplit from dvc.cli import main -from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK +from dvc.dvcfile import LOCK_FILE, PROJECT_FILE from dvc.exceptions import CyclicGraphError, ReproductionError from dvc.stage import PipelineStage @@ -86,18 +86,18 @@ def test_downstream(tmp_dir, dvc): # B # evaluation = dvc.reproduce( - PIPELINE_FILE + ":B-gen", downstream=True, force=True + PROJECT_FILE + ":B-gen", downstream=True, force=True ) assert len(evaluation) == 3 assert ( isinstance(evaluation[0], PipelineStage) - and evaluation[0].relpath == PIPELINE_FILE + and evaluation[0].relpath == PROJECT_FILE and evaluation[0].name == "B-gen" ) assert ( isinstance(evaluation[1], PipelineStage) - and evaluation[1].relpath == PIPELINE_FILE + and evaluation[1].relpath == PROJECT_FILE and evaluation[1].name == "D-gen" ) assert ( @@ -108,26 +108,26 @@ def test_downstream(tmp_dir, dvc): # B, C should be run (in any order) before D # See https://github.com/iterative/dvc/issues/3602 evaluation = dvc.reproduce( - PIPELINE_FILE + ":A-gen", downstream=True, force=True + PROJECT_FILE + ":A-gen", downstream=True, force=True ) assert len(evaluation) == 5 assert ( isinstance(evaluation[0], PipelineStage) - and evaluation[0].relpath == PIPELINE_FILE + and evaluation[0].relpath == PROJECT_FILE and evaluation[0].name == "A-gen" ) names = set() for stage in evaluation[1:3]: if isinstance(stage, PipelineStage): - assert stage.relpath == PIPELINE_FILE + assert stage.relpath == PROJECT_FILE names.add(stage.name) else: names.add(stage.relpath) assert names == {"B-gen", "C.dvc"} assert ( isinstance(evaluation[3], PipelineStage) - and evaluation[3].relpath == PIPELINE_FILE + and evaluation[3].relpath == PROJECT_FILE and evaluation[3].name == "D-gen" ) assert ( @@ -137,7 +137,7 @@ def test_downstream(tmp_dir, dvc): def test_repro_when_cmd_changes(tmp_dir, dvc, run_copy, mocker): - from dvc.dvcfile import PipelineFile + from dvc.dvcfile import ProjectFile tmp_dir.gen("foo", "foo") stage = run_copy("foo", "bar", name="copy-file") @@ -148,7 +148,7 @@ def test_repro_when_cmd_changes(tmp_dir, dvc, run_copy, mocker): m = mocker.patch("dvc.stage.run.cmd_run", wraps=cmd_run) stage.cmd = " ".join(stage.cmd.split()) # change cmd spacing by two - PipelineFile(dvc, PIPELINE_FILE)._dump_pipeline_file(stage) + ProjectFile(dvc, PROJECT_FILE)._dump_pipeline_file(stage) assert dvc.status([target]) == {target: ["changed command"]} assert dvc.reproduce(target)[0] == stage @@ -160,7 +160,7 @@ def test_repro_when_cmd_changes(tmp_dir, dvc, run_copy, mocker): def test_repro_when_new_deps_is_added_in_dvcfile( tmp_dir, dvc, run_copy, copy_script ): - from dvc.dvcfile import Dvcfile + from dvc.dvcfile import load_file tmp_dir.gen({"foo": "foo", "bar": "bar"}) stage = dvc.run( @@ -169,10 +169,10 @@ def test_repro_when_new_deps_is_added_in_dvcfile( deps=["foo"], name="copy-file", ) - target = PIPELINE_FILE + ":copy-file" + target = PROJECT_FILE + ":copy-file" assert not dvc.reproduce(target) - dvcfile = Dvcfile(dvc, stage.path) + dvcfile = load_file(dvc, stage.path) data, _ = dvcfile._load() data["stages"]["copy-file"]["deps"] += ["copy.py"] (tmp_dir / stage.path).dump(data) @@ -181,7 +181,7 @@ def test_repro_when_new_deps_is_added_in_dvcfile( def test_repro_when_new_outs_is_added_in_dvcfile(tmp_dir, dvc, copy_script): - from dvc.dvcfile import Dvcfile + from dvc.dvcfile import load_file tmp_dir.gen({"foo": "foo", "bar": "bar"}) stage = dvc.run( @@ -193,7 +193,7 @@ def test_repro_when_new_outs_is_added_in_dvcfile(tmp_dir, dvc, copy_script): target = ":copy-file" assert not dvc.reproduce(target) - dvcfile = Dvcfile(dvc, stage.path) + dvcfile = load_file(dvc, stage.path) data, _ = dvcfile._load() data["stages"]["copy-file"]["outs"] = ["foobar"] (tmp_dir / stage.path).dump(data) @@ -202,7 +202,7 @@ def test_repro_when_new_outs_is_added_in_dvcfile(tmp_dir, dvc, copy_script): def test_repro_when_new_deps_is_moved(tmp_dir, dvc, copy_script): - from dvc.dvcfile import Dvcfile + from dvc.dvcfile import load_file tmp_dir.gen({"foo": "foo", "bar": "foo"}) stage = dvc.run( @@ -227,7 +227,7 @@ shutil.copyfile('bar', 'foobar') move("foo", "bar") - dvcfile = Dvcfile(dvc, stage.path) + dvcfile = load_file(dvc, stage.path) data, _ = dvcfile._load() data["stages"]["copy-file"]["deps"] = ["bar"] (tmp_dir / stage.path).dump(data) @@ -240,7 +240,7 @@ def test_repro_when_new_out_overlaps_others_stage_outs(tmp_dir, dvc): tmp_dir.gen({"dir": {"file1": "file1"}, "foo": "foo"}) dvc.add("dir") - (tmp_dir / PIPELINE_FILE).dump( + (tmp_dir / PROJECT_FILE).dump( { "stages": { "run-copy": { @@ -257,7 +257,7 @@ def test_repro_when_new_out_overlaps_others_stage_outs(tmp_dir, dvc): def test_repro_when_new_deps_added_does_not_exist(tmp_dir, dvc, copy_script): tmp_dir.gen("foo", "foo") - (tmp_dir / PIPELINE_FILE).dump( + (tmp_dir / PROJECT_FILE).dump( { "stages": { "run-copy": { @@ -274,7 +274,7 @@ def test_repro_when_new_deps_added_does_not_exist(tmp_dir, dvc, copy_script): def test_repro_when_new_outs_added_does_not_exist(tmp_dir, dvc, copy_script): tmp_dir.gen("foo", "foo") - (tmp_dir / PIPELINE_FILE).dump( + (tmp_dir / PROJECT_FILE).dump( { "stages": { "run-copy": { @@ -291,7 +291,7 @@ def test_repro_when_new_outs_added_does_not_exist(tmp_dir, dvc, copy_script): def test_repro_when_lockfile_gets_deleted(tmp_dir, dvc, copy_script): tmp_dir.gen("foo", "foo") - (tmp_dir / PIPELINE_FILE).dump( + (tmp_dir / PROJECT_FILE).dump( { "stages": { "run-copy": { @@ -303,14 +303,14 @@ def test_repro_when_lockfile_gets_deleted(tmp_dir, dvc, copy_script): }, ) assert dvc.reproduce(":run-copy") - assert os.path.exists(PIPELINE_LOCK) + assert os.path.exists(LOCK_FILE) assert not dvc.reproduce(":run-copy") - os.unlink(PIPELINE_LOCK) + os.unlink(LOCK_FILE) stages = dvc.reproduce(":run-copy") assert ( stages - and stages[0].relpath == PIPELINE_FILE + and stages[0].relpath == PROJECT_FILE and stages[0].name == "run-copy" ) @@ -321,13 +321,13 @@ def test_cyclic_graph_error(tmp_dir, dvc, run_copy): run_copy("bar", "baz", name="copy-bar-baz") run_copy("baz", "foobar", name="copy-baz-foobar") - data = (tmp_dir / PIPELINE_FILE).parse() + data = (tmp_dir / PROJECT_FILE).parse() data["stages"]["copy-baz-foo"] = { "cmd": "echo baz > foo", "deps": ["baz"], "outs": ["foo"], } - (tmp_dir / PIPELINE_FILE).dump(data) + (tmp_dir / PROJECT_FILE).dump(data) with pytest.raises(CyclicGraphError): dvc.reproduce(":copy-baz-foo") diff --git a/tests/func/test_run_cache.py b/tests/func/test_run_cache.py index 25387f1c2..0a64ad1f4 100644 --- a/tests/func/test_run_cache.py +++ b/tests/func/test_run_cache.py @@ -2,7 +2,7 @@ import os import pytest -from dvc.dvcfile import PIPELINE_LOCK +from dvc.dvcfile import LOCK_FILE from dvc.utils.fs import remove @@ -30,14 +30,14 @@ def test_restore(tmp_dir, dvc, run_copy, mocker): # removing any information that `dvc` could use to re-generate from (tmp_dir / "bar").unlink() - (tmp_dir / PIPELINE_LOCK).unlink() + (tmp_dir / LOCK_FILE).unlink() (stage,) = dvc.reproduce("copy-foo-bar") mock_restore.assert_called_once_with(stage, dry=False) mock_run.assert_not_called() assert (tmp_dir / "bar").exists() and not (tmp_dir / "foo").unlink() - assert (tmp_dir / PIPELINE_LOCK).exists() + assert (tmp_dir / LOCK_FILE).exists() def test_save(tmp_dir, dvc, run_copy): @@ -103,16 +103,16 @@ def test_memory_for_multiple_runs_of_same_stage( mock_run = mocker.spy(_run, "cmd_run") (tmp_dir / "bar").unlink() - (tmp_dir / PIPELINE_LOCK).unlink() + (tmp_dir / LOCK_FILE).unlink() (stage,) = dvc.reproduce("copy-foo-bar") - assert (tmp_dir / PIPELINE_LOCK).exists() + assert (tmp_dir / LOCK_FILE).exists() assert (tmp_dir / "bar").read_text() == "foobar" mock_run.assert_not_called() mock_restore.assert_called_once_with(stage, dry=False) mock_restore.reset_mock() - (tmp_dir / PIPELINE_LOCK).unlink() + (tmp_dir / LOCK_FILE).unlink() tmp_dir.gen("foo", "foo") dvc.reproduce("copy-foo-bar") @@ -120,7 +120,7 @@ def test_memory_for_multiple_runs_of_same_stage( mock_run.assert_not_called() mock_restore.assert_called_once_with(stage, dry=False) assert (tmp_dir / "bar").exists() and not (tmp_dir / "foo").unlink() - assert (tmp_dir / PIPELINE_LOCK).exists() + assert (tmp_dir / LOCK_FILE).exists() def test_memory_runs_of_multiple_stages(tmp_dir, dvc, run_copy, mocker): @@ -141,11 +141,11 @@ def test_memory_runs_of_multiple_stages(tmp_dir, dvc, run_copy, mocker): (tmp_dir / "foo.bak").unlink() (tmp_dir / "bar.bak").unlink() - (tmp_dir / PIPELINE_LOCK).unlink() + (tmp_dir / LOCK_FILE).unlink() (stage,) = dvc.reproduce("backup-foo") assert (tmp_dir / "foo.bak").read_text() == "foo" - assert (tmp_dir / PIPELINE_LOCK).exists() + assert (tmp_dir / LOCK_FILE).exists() mock_run.assert_not_called() mock_restore.assert_called_once_with(stage, dry=False) mock_restore.reset_mock() @@ -153,7 +153,7 @@ def test_memory_runs_of_multiple_stages(tmp_dir, dvc, run_copy, mocker): (stage,) = dvc.reproduce("backup-bar") assert (tmp_dir / "bar.bak").read_text() == "bar" - assert (tmp_dir / PIPELINE_LOCK).exists() + assert (tmp_dir / LOCK_FILE).exists() mock_run.assert_not_called() mock_restore.assert_called_once_with(stage, dry=False) @@ -172,7 +172,7 @@ def test_restore_pull(tmp_dir, dvc, run_copy, mocker, local_remote): # removing any information that `dvc` could use to re-generate from (tmp_dir / "bar").unlink() - (tmp_dir / PIPELINE_LOCK).unlink() + (tmp_dir / LOCK_FILE).unlink() remove(stage.outs[0].cache_path) (stage,) = dvc.reproduce("copy-foo-bar", pull=True) @@ -181,4 +181,4 @@ def test_restore_pull(tmp_dir, dvc, run_copy, mocker, local_remote): mock_run.assert_not_called() assert mock_checkout.call_count == 2 assert (tmp_dir / "bar").exists() and not (tmp_dir / "foo").unlink() - assert (tmp_dir / PIPELINE_LOCK).exists() + assert (tmp_dir / LOCK_FILE).exists() diff --git a/tests/func/test_run_multistage.py b/tests/func/test_run_multistage.py index b0c52e6c0..802756ac3 100644 --- a/tests/func/test_run_multistage.py +++ b/tests/func/test_run_multistage.py @@ -8,29 +8,29 @@ from dvc.stage.exceptions import DuplicateStageName, InvalidStageName def test_run_with_name(tmp_dir, dvc, run_copy): - from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK + from dvc.dvcfile import LOCK_FILE, PROJECT_FILE from dvc.stage import PipelineStage tmp_dir.dvc_gen("foo", "foo") - assert not os.path.exists(PIPELINE_FILE) + assert not os.path.exists(PROJECT_FILE) stage = run_copy("foo", "bar", name="copy-foo-to-bar") assert isinstance(stage, PipelineStage) assert stage.name == "copy-foo-to-bar" - assert os.path.exists(PIPELINE_FILE) - assert os.path.exists(PIPELINE_LOCK) + assert os.path.exists(PROJECT_FILE) + assert os.path.exists(LOCK_FILE) def test_run_no_exec(tmp_dir, dvc, run_copy): - from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK + from dvc.dvcfile import LOCK_FILE, PROJECT_FILE from dvc.stage import PipelineStage tmp_dir.dvc_gen("foo", "foo") - assert not os.path.exists(PIPELINE_FILE) + assert not os.path.exists(PROJECT_FILE) stage = run_copy("foo", "bar", name="copy-foo-to-bar", no_exec=True) assert isinstance(stage, PipelineStage) assert stage.name == "copy-foo-to-bar" - assert os.path.exists(PIPELINE_FILE) - assert not os.path.exists(PIPELINE_LOCK) + assert os.path.exists(PROJECT_FILE) + assert not os.path.exists(LOCK_FILE) data, _ = stage.dvcfile._load() assert data["stages"]["copy-foo-to-bar"] == { @@ -55,7 +55,7 @@ def test_run_with_multistage_and_single_stage(tmp_dir, dvc, run_copy): def test_run_multi_stage_repeat(tmp_dir, dvc, run_copy): - from dvc.dvcfile import PIPELINE_FILE, Dvcfile + from dvc.dvcfile import PROJECT_FILE, load_file from dvc.stage import PipelineStage tmp_dir.dvc_gen("foo", "foo") @@ -63,7 +63,7 @@ def test_run_multi_stage_repeat(tmp_dir, dvc, run_copy): run_copy("foo1", "foo2", name="copy-foo1-foo2") run_copy("foo2", "foo3", single_stage=True) - stages = list(Dvcfile(dvc, PIPELINE_FILE).stages.values()) + stages = list(load_file(dvc, PROJECT_FILE).stages.values()) assert len(stages) == 2 assert all(isinstance(stage, PipelineStage) for stage in stages) assert {stage.name for stage in stages} == { @@ -94,7 +94,7 @@ def test_multistage_dump_on_non_cached_outputs(tmp_dir, dvc): def test_multistage_with_wdir(tmp_dir, dvc): - from dvc.dvcfile import Dvcfile + from dvc.dvcfile import load_file tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}}) stage = dvc.run( @@ -105,12 +105,12 @@ def test_multistage_with_wdir(tmp_dir, dvc): wdir="dir", ) - data, _ = Dvcfile(dvc, stage.path)._load() + data, _ = load_file(dvc, stage.path)._load() assert data["stages"]["copy-foo1-foo2"]["wdir"] == "dir" def test_multistage_always_changed(tmp_dir, dvc): - from dvc.dvcfile import Dvcfile + from dvc.dvcfile import load_file tmp_dir.gen({"foo": "foo", "bar": "bar"}) stage = dvc.run( @@ -121,7 +121,7 @@ def test_multistage_always_changed(tmp_dir, dvc): always_changed=True, ) - data, _ = Dvcfile(dvc, stage.path)._load() + data, _ = load_file(dvc, stage.path)._load() assert data["stages"]["copy-foo1-foo2"]["always_changed"] @@ -139,7 +139,7 @@ def test_graph(tmp_dir, dvc): def test_run_dump_on_multistage(tmp_dir, dvc, run_head): - from dvc.dvcfile import PIPELINE_FILE, Dvcfile + from dvc.dvcfile import PROJECT_FILE, load_file tmp_dir.gen( { @@ -159,7 +159,7 @@ def test_run_dump_on_multistage(tmp_dir, dvc, run_head): outs_persist=["foo2"], always_changed=True, ) - data = Dvcfile(dvc, PIPELINE_FILE)._load()[0] + data = load_file(dvc, PROJECT_FILE)._load()[0] assert data == { "stages": { "copy-foo-foo2": { @@ -182,7 +182,7 @@ def test_run_dump_on_multistage(tmp_dir, dvc, run_head): metrics_no_cache=["foobar-1"], wdir="dir", ) - assert Dvcfile(dvc, PIPELINE_FILE)._load()[0] == { + assert load_file(dvc, PROJECT_FILE)._load()[0] == { "stages": { "head-files": { "cmd": "python {} foo bar foobar".format( @@ -309,7 +309,7 @@ def test_run_without_cmd(tmp_dir, dvc, kwargs): def test_run_overwrite_order(tmp_dir, dvc, run_copy): - from dvc.dvcfile import PIPELINE_FILE + from dvc.dvcfile import PROJECT_FILE tmp_dir.gen({"foo": "foo", "foo1": "foo1"}) run_copy("foo", "bar", name="copy-foo-bar") @@ -317,12 +317,12 @@ def test_run_overwrite_order(tmp_dir, dvc, run_copy): run_copy("foo1", "bar1", name="copy-foo-bar", force=True) - data = (tmp_dir / PIPELINE_FILE).parse() + data = (tmp_dir / PROJECT_FILE).parse() assert list(data["stages"].keys()) == ["copy-foo-bar", "copy-bar-foobar"] def test_run_overwrite_preserves_meta_and_comment(tmp_dir, dvc, run_copy): - from dvc.dvcfile import PIPELINE_FILE + from dvc.dvcfile import PROJECT_FILE tmp_dir.gen({"foo": "foo", "foo1": "foo1"}) text = textwrap.dedent( @@ -340,12 +340,12 @@ def test_run_overwrite_preserves_meta_and_comment(tmp_dir, dvc, run_copy): name: meta is preserved too """ ) - (tmp_dir / PIPELINE_FILE).write_text(text.format(src="foo", dest="bar")) - assert dvc.reproduce(PIPELINE_FILE) + (tmp_dir / PROJECT_FILE).write_text(text.format(src="foo", dest="bar")) + assert dvc.reproduce(PROJECT_FILE) assert run_copy("foo1", "bar1", name="copy-foo-bar", force=True) - assert (tmp_dir / PIPELINE_FILE).read_text() == text.format( + assert (tmp_dir / PROJECT_FILE).read_text() == text.format( src="foo1", dest="bar1" ) diff --git a/tests/func/test_stage_load.py b/tests/func/test_stage_load.py index 49c78c0da..b5238ab37 100644 --- a/tests/func/test_stage_load.py +++ b/tests/func/test_stage_load.py @@ -4,7 +4,7 @@ from operator import itemgetter import pytest from funcy import raiser -from dvc.dvcfile import PIPELINE_FILE, FileIsGitIgnored +from dvc.dvcfile import PROJECT_FILE, FileIsGitIgnored from dvc.exceptions import NoOutputOrStageError from dvc.repo import Repo from dvc.stage.exceptions import ( @@ -186,7 +186,7 @@ def test_collect_granular_with_target(tmp_dir, dvc, stages): assert dvc.stage.collect_granular("bar.dvc") == [ (stages["copy-foo-bar"], None) ] - assert dvc.stage.collect_granular(PIPELINE_FILE) == [ + assert dvc.stage.collect_granular(PROJECT_FILE) == [ (stages["copy-bar-foobar"], None), (stages["copy-lorem-ipsum"], None), ] @@ -206,7 +206,7 @@ def test_collect_granular_with_target(tmp_dir, dvc, stages): with (tmp_dir / dvc.DVC_DIR).chdir(): assert dvc.stage.collect_granular( - relpath(tmp_dir / PIPELINE_FILE) + ":copy-bar-foobar" + relpath(tmp_dir / PROJECT_FILE) + ":copy-bar-foobar" ) == [(stages["copy-bar-foobar"], None)] assert dvc.stage.collect_granular("foobar") == [ @@ -257,7 +257,7 @@ def test_collect_granular_with_deps(tmp_dir, dvc, stages): assert set( map( itemgetter(0), - dvc.stage.collect_granular(PIPELINE_FILE, with_deps=True), + dvc.stage.collect_granular(PROJECT_FILE, with_deps=True), ) ) == set(stages.values()) @@ -333,17 +333,17 @@ def test_get_stages(tmp_dir, dvc, run_copy): stage2 = run_copy("bar", "foobar", name="copy-bar-foobar") assert set(dvc.stage.load_all()) == {stage1, stage2} - assert set(dvc.stage.load_all(path=PIPELINE_FILE)) == {stage1, stage2} + assert set(dvc.stage.load_all(path=PROJECT_FILE)) == {stage1, stage2} assert set(dvc.stage.load_all(name="copy-bar-foobar")) == {stage2} assert set( - dvc.stage.load_all(path=PIPELINE_FILE, name="copy-bar-foobar") + dvc.stage.load_all(path=PROJECT_FILE, name="copy-bar-foobar") ) == {stage2} with pytest.raises(StageFileDoesNotExistError): - dvc.stage.load_all(path=relpath(tmp_dir / ".." / PIPELINE_FILE)) + dvc.stage.load_all(path=relpath(tmp_dir / ".." / PROJECT_FILE)) with pytest.raises(StageNotFound): - dvc.stage.load_all(path=PIPELINE_FILE, name="copy") + dvc.stage.load_all(path=PROJECT_FILE, name="copy") def test_get_stages_old_dvcfile(tmp_dir, dvc): @@ -363,11 +363,9 @@ def test_get_stage(tmp_dir, dvc, run_copy): dvc.stage.load_one() with pytest.raises(StageNameUnspecified): - dvc.stage.load_one(path=PIPELINE_FILE) + dvc.stage.load_one(path=PROJECT_FILE) - assert ( - dvc.stage.load_one(path=PIPELINE_FILE, name="copy-foo-bar") == stage1 - ) + assert dvc.stage.load_one(path=PROJECT_FILE, name="copy-foo-bar") == stage1 assert dvc.stage.load_one(name="copy-foo-bar") == stage1 with pytest.raises(StageFileDoesNotExistError): @@ -420,14 +418,14 @@ def test_collect_repo_callback(tmp_dir, dvc, mocker): dvc.stage_collection_error_handler = mock (stage,) = tmp_dir.dvc_gen("foo", "foo") - (tmp_dir / PIPELINE_FILE).dump({"stages": {"cmd": "echo hello world"}}) + (tmp_dir / PROJECT_FILE).dump({"stages": {"cmd": "echo hello world"}}) dvc._reset() assert dvc.index.stages == [stage] mock.assert_called_once() file_path, exc = mock.call_args[0] - assert file_path == PIPELINE_FILE + assert file_path == PROJECT_FILE assert isinstance(exc, YAMLValidationError) diff --git a/tests/func/test_update.py b/tests/func/test_update.py index ab25d3363..ce27d53df 100644 --- a/tests/func/test_update.py +++ b/tests/func/test_update.py @@ -3,7 +3,7 @@ import os import pytest from dvc.dependency import base -from dvc.dvcfile import Dvcfile +from dvc.dvcfile import load_file from dvc.exceptions import InvalidArgumentError from dvc.testing.tmp_dir import make_subrepo @@ -117,7 +117,7 @@ def test_update_import_after_remote_updates_to_dvc(tmp_dir, dvc, erepo_dir): assert imported.is_file() assert imported.read_text() == "updated" - stage = Dvcfile(dvc, stage.path).stage + stage = load_file(dvc, stage.path).stage assert stage.deps[0].def_repo == { "url": os.fspath(erepo_dir), "rev": "branch", @@ -344,9 +344,9 @@ def test_update_recursive(tmp_dir, dvc, erepo_dir): dvc.update(["dir"], recursive=True) - stage1 = Dvcfile(dvc, stage1.path).stage - stage2 = Dvcfile(dvc, stage2.path).stage - stage3 = Dvcfile(dvc, stage3.path).stage + stage1 = load_file(dvc, stage1.path).stage + stage2 = load_file(dvc, stage2.path).stage + stage3 = load_file(dvc, stage3.path).stage assert stage1.deps[0].def_repo["rev_lock"] == new_rev assert stage2.deps[0].def_repo["rev_lock"] == new_rev assert stage3.deps[0].def_repo["rev_lock"] == new_rev diff --git a/tests/unit/stage/test_loader_pipeline_file.py b/tests/unit/stage/test_loader_pipeline_file.py index 8d7705972..8fdb42910 100644 --- a/tests/unit/stage/test_loader_pipeline_file.py +++ b/tests/unit/stage/test_loader_pipeline_file.py @@ -4,7 +4,7 @@ from itertools import chain import pytest -from dvc.dvcfile import PIPELINE_FILE, Dvcfile +from dvc.dvcfile import PROJECT_FILE, load_file from dvc.stage import PipelineStage, create_stage from dvc.stage.loader import StageLoader from dvc.stage.serialize import split_params_deps @@ -27,7 +27,7 @@ def lock_data(): def test_fill_from_lock_deps_outs(dvc, lock_data): stage = create_stage( - PipelineStage, dvc, PIPELINE_FILE, deps=["foo"], outs=["bar"] + PipelineStage, dvc, PROJECT_FILE, deps=["foo"], outs=["bar"] ) for item in chain(stage.deps, stage.outs): @@ -40,7 +40,7 @@ def test_fill_from_lock_deps_outs(dvc, lock_data): def test_fill_from_lock_outs_isexec(dvc): - stage = create_stage(PipelineStage, dvc, PIPELINE_FILE, outs=["foo"]) + stage = create_stage(PipelineStage, dvc, PROJECT_FILE, outs=["foo"]) assert not stage.outs[0].meta.isexec @@ -61,7 +61,7 @@ def test_fill_from_lock_params(dvc, lock_data): stage = create_stage( PipelineStage, dvc, - PIPELINE_FILE, + PROJECT_FILE, deps=["foo"], outs=["bar"], params=[ @@ -97,7 +97,7 @@ def test_fill_from_lock_missing_params_section(dvc, lock_data): stage = create_stage( PipelineStage, dvc, - PIPELINE_FILE, + PROJECT_FILE, deps=["foo"], outs=["bar"], params=["lorem", "lorem.ipsum", {"myparams.yaml": ["ipsum"]}], @@ -111,7 +111,7 @@ def test_fill_from_lock_missing_checksums(dvc, lock_data): stage = create_stage( PipelineStage, dvc, - PIPELINE_FILE, + PROJECT_FILE, deps=["foo", "foo1"], outs=["bar", "bar1"], ) @@ -127,7 +127,7 @@ def test_fill_from_lock_use_appropriate_checksum(dvc, lock_data): stage = create_stage( PipelineStage, dvc, - PIPELINE_FILE, + PROJECT_FILE, deps=["s3://dvc-temp/foo"], outs=["bar"], ) @@ -139,7 +139,7 @@ def test_fill_from_lock_use_appropriate_checksum(dvc, lock_data): def test_fill_from_lock_with_missing_sections(dvc, lock_data): stage = create_stage( - PipelineStage, dvc, PIPELINE_FILE, deps=["foo"], outs=["bar"] + PipelineStage, dvc, PROJECT_FILE, deps=["foo"], outs=["bar"] ) lock = deepcopy(lock_data) del lock["deps"] @@ -156,7 +156,7 @@ def test_fill_from_lock_with_missing_sections(dvc, lock_data): def test_fill_from_lock_empty_data(dvc): stage = create_stage( - PipelineStage, dvc, PIPELINE_FILE, deps=["foo"], outs=["bar"] + PipelineStage, dvc, PROJECT_FILE, deps=["foo"], outs=["bar"] ) StageLoader.fill_from_lock(stage, None) assert not stage.deps[0].hash_info and not stage.outs[0].hash_info @@ -165,13 +165,13 @@ def test_fill_from_lock_empty_data(dvc): def test_load_stage(dvc, stage_data, lock_data): - dvcfile = Dvcfile(dvc, PIPELINE_FILE) + dvcfile = load_file(dvc, PROJECT_FILE) stage = StageLoader.load_stage(dvcfile, "stage-1", stage_data, lock_data) assert stage.wdir == os.path.abspath(os.curdir) assert stage.name == "stage-1" assert stage.cmd == "command" - assert stage.path == os.path.abspath(PIPELINE_FILE) + assert stage.path == os.path.abspath(PROJECT_FILE) assert stage.deps[0].def_path == "foo" assert stage.deps[0].hash_info == HashInfo("md5", "foo_checksum") assert stage.outs[0].def_path == "bar" @@ -180,20 +180,20 @@ def test_load_stage(dvc, stage_data, lock_data): def test_load_stage_cmd_with_list(dvc, stage_data, lock_data): stage_data["cmd"] = ["cmd-0", "cmd-1"] - dvcfile = Dvcfile(dvc, PIPELINE_FILE) + dvcfile = load_file(dvc, PROJECT_FILE) stage = StageLoader.load_stage(dvcfile, "stage-1", stage_data, lock_data) assert stage.cmd == ["cmd-0", "cmd-1"] def test_load_stage_outs_with_flags(dvc, stage_data, lock_data): stage_data["outs"] = [{"foo": {"cache": False}}] - dvcfile = Dvcfile(dvc, PIPELINE_FILE) + dvcfile = load_file(dvc, PROJECT_FILE) stage = StageLoader.load_stage(dvcfile, "stage-1", stage_data, lock_data) assert stage.outs[0].use_cache is False def test_load_stage_no_lock(dvc, stage_data): - dvcfile = Dvcfile(dvc, PIPELINE_FILE) + dvcfile = load_file(dvc, PROJECT_FILE) stage = StageLoader.load_stage(dvcfile, "stage-1", stage_data) assert stage.deps[0].def_path == "foo" and stage.outs[0].def_path == "bar" assert not stage.deps[0].hash_info @@ -203,7 +203,7 @@ def test_load_stage_no_lock(dvc, stage_data): def test_load_stage_with_params(dvc, stage_data, lock_data): lock_data["params"] = {"params.yaml": {"lorem": "ipsum"}} stage_data["params"] = ["lorem"] - dvcfile = Dvcfile(dvc, PIPELINE_FILE) + dvcfile = load_file(dvc, PROJECT_FILE) stage = StageLoader.load_stage(dvcfile, "stage-1", stage_data, lock_data) params, deps = split_params_deps(stage) @@ -217,7 +217,7 @@ def test_load_stage_with_params(dvc, stage_data, lock_data): @pytest.mark.parametrize("typ", ["metrics", "plots"]) def test_load_stage_with_metrics_and_plots(dvc, stage_data, lock_data, typ): stage_data[typ] = stage_data.pop("outs") - dvcfile = Dvcfile(dvc, PIPELINE_FILE) + dvcfile = load_file(dvc, PROJECT_FILE) stage = StageLoader.load_stage(dvcfile, "stage-1", stage_data, lock_data) assert stage.outs[0].def_path == "bar" @@ -225,7 +225,7 @@ def test_load_stage_with_metrics_and_plots(dvc, stage_data, lock_data, typ): def test_load_changed_command(dvc, stage_data, lock_data): - dvcfile = Dvcfile(dvc, PIPELINE_FILE) + dvcfile = load_file(dvc, PROJECT_FILE) stage = StageLoader.load_stage(dvcfile, "stage-1", stage_data) assert not stage.cmd_changed assert stage.cmd == "command" @@ -238,15 +238,15 @@ def test_load_changed_command(dvc, stage_data, lock_data): def test_load_stage_wdir_and_path_correctly(dvc, stage_data, lock_data): stage_data["wdir"] = "dir" - dvcfile = Dvcfile(dvc, PIPELINE_FILE) + dvcfile = load_file(dvc, PROJECT_FILE) stage = StageLoader.load_stage(dvcfile, "stage-1", stage_data, lock_data) assert stage.wdir == os.path.abspath("dir") - assert stage.path == os.path.abspath(PIPELINE_FILE) + assert stage.path == os.path.abspath(PROJECT_FILE) def test_load_stage_mapping(dvc, stage_data, lock_data): - dvcfile = Dvcfile(dvc, PIPELINE_FILE) + dvcfile = load_file(dvc, PROJECT_FILE) dvcfile.contents = {"stages": {"stage": stage_data}} dvcfile.lockfile_contents = {"stage": lock_data} diff --git a/tests/unit/stage/test_serialize_pipeline_file.py b/tests/unit/stage/test_serialize_pipeline_file.py index f0e97abfc..075d65a2e 100644 --- a/tests/unit/stage/test_serialize_pipeline_file.py +++ b/tests/unit/stage/test_serialize_pipeline_file.py @@ -4,12 +4,12 @@ import pytest from voluptuous import Schema as _Schema from dvc import output -from dvc.dvcfile import PIPELINE_FILE +from dvc.dvcfile import PROJECT_FILE from dvc.schema import SINGLE_PIPELINE_STAGE_SCHEMA from dvc.stage import PipelineStage, create_stage from dvc.stage.serialize import to_pipeline_file as _to_pipeline_file -kwargs = {"name": "something", "cmd": "command", "path": PIPELINE_FILE} +kwargs = {"name": "something", "cmd": "command", "path": PROJECT_FILE} Schema = _Schema(SINGLE_PIPELINE_STAGE_SCHEMA) diff --git a/tests/unit/stage/test_serialize_pipeline_lock.py b/tests/unit/stage/test_serialize_pipeline_lock.py index c20fc19f4..53905fe37 100644 --- a/tests/unit/stage/test_serialize_pipeline_lock.py +++ b/tests/unit/stage/test_serialize_pipeline_lock.py @@ -3,7 +3,7 @@ from collections import OrderedDict import pytest from voluptuous import Schema as _Schema -from dvc.dvcfile import PIPELINE_FILE +from dvc.dvcfile import PROJECT_FILE from dvc.schema import LOCK_FILE_STAGE_SCHEMA, LOCKFILE_STAGES_SCHEMA from dvc.stage import PipelineStage, create_stage from dvc.stage.serialize import DEFAULT_PARAMS_FILE, to_lockfile @@ -13,7 +13,7 @@ from dvc.stage.serialize import ( from dvc.stage.utils import split_params_deps from dvc_data.hashfile.hash_info import HashInfo -kwargs = {"name": "something", "cmd": "command", "path": PIPELINE_FILE} +kwargs = {"name": "something", "cmd": "command", "path": PROJECT_FILE} Schema = _Schema(LOCK_FILE_STAGE_SCHEMA) diff --git a/tests/unit/test_dvcfile.py b/tests/unit/test_dvcfile.py index 8291aaaca..a252987b0 100644 --- a/tests/unit/test_dvcfile.py +++ b/tests/unit/test_dvcfile.py @@ -1,12 +1,12 @@ import pytest from dvc.dvcfile import ( - PIPELINE_FILE, - PIPELINE_LOCK, - Dvcfile, + LOCK_FILE, + PROJECT_FILE, FileIsGitIgnored, - PipelineFile, + ProjectFile, SingleStageFile, + load_file, ) from dvc.stage import PipelineStage from dvc.stage.exceptions import ( @@ -29,22 +29,22 @@ from dvc.utils.strictyaml import YAMLValidationError ], ) def test_pipelines_file(path): - file_obj = Dvcfile(object(), path) - assert isinstance(file_obj, PipelineFile) + file_obj = load_file(object(), path) + assert isinstance(file_obj, ProjectFile) @pytest.mark.parametrize( "path", ["Dvcfile", "stage.dvc", "../models/stage.dvc"] ) def test_pipelines_single_stage_file(path): - file_obj = Dvcfile(object(), path) + file_obj = load_file(object(), path) assert isinstance(file_obj, SingleStageFile) @pytest.mark.parametrize("file", ["stage.dvc", "dvc.yaml"]) @pytest.mark.parametrize("is_dvcignored", [True, False]) def test_stage_load_on_not_existing_file(tmp_dir, dvc, file, is_dvcignored): - dvcfile = Dvcfile(dvc, file) + dvcfile = load_file(dvc, file) if is_dvcignored: (tmp_dir / ".dvcignore").write_text(file) @@ -58,7 +58,7 @@ def test_stage_load_on_not_existing_file(tmp_dir, dvc, file, is_dvcignored): @pytest.mark.parametrize("file", ["stage.dvc", "dvc.yaml"]) def test_stage_load_on_non_file(tmp_dir, dvc, file): (tmp_dir / file).mkdir() - dvcfile = Dvcfile(dvc, file) + dvcfile = load_file(dvc, file) with pytest.raises(StageFileIsNotDvcFileError): assert dvcfile.stages.values() @@ -67,7 +67,7 @@ def test_stage_load_on_non_file(tmp_dir, dvc, file): def test_stage_load_on_invalid_data(tmp_dir, dvc, file): data = {"is_this_a_valid_dvcfile": False} (tmp_dir / file).dump(data) - dvcfile = Dvcfile(dvc, file) + dvcfile = load_file(dvc, file) with pytest.raises(YAMLValidationError): assert dvcfile.stages with pytest.raises(YAMLValidationError): @@ -78,22 +78,22 @@ def test_dump_stage(tmp_dir, dvc): stage = PipelineStage( dvc, cmd="command", name="stage_name", path="dvc.yaml" ) - dvcfile = Dvcfile(dvc, "dvc.yaml") + dvcfile = load_file(dvc, "dvc.yaml") dvcfile.dump(stage, update_lock=False, update_pipeline=False) - assert not (tmp_dir / PIPELINE_FILE).exists() - assert not (tmp_dir / PIPELINE_LOCK).exists() + assert not (tmp_dir / PROJECT_FILE).exists() + assert not (tmp_dir / LOCK_FILE).exists() dvcfile.dump(stage, update_pipeline=False) - assert not (tmp_dir / PIPELINE_FILE).exists() - assert (tmp_dir / PIPELINE_LOCK).exists() + assert not (tmp_dir / PROJECT_FILE).exists() + assert (tmp_dir / LOCK_FILE).exists() assert dvcfile._lockfile.load() - remove(tmp_dir / PIPELINE_LOCK) + remove(tmp_dir / LOCK_FILE) dvcfile.dump(stage) - assert (tmp_dir / PIPELINE_FILE).exists() - assert (tmp_dir / PIPELINE_LOCK).exists() + assert (tmp_dir / PROJECT_FILE).exists() + assert (tmp_dir / LOCK_FILE).exists() assert list(dvcfile.stages.values()) == [stage] @@ -103,7 +103,7 @@ def test_stage_load_file_exists_but_dvcignored(tmp_dir, dvc, scm, file): (tmp_dir / ".dvcignore").write_text(file) dvc._reset() - dvcfile = Dvcfile(dvc, file) + dvcfile = load_file(dvc, file) with pytest.raises(StageFileDoesNotExistError) as exc_info: assert dvcfile.stages.values() @@ -119,7 +119,7 @@ def test_try_loading_dvcfile_that_is_gitignored(tmp_dir, dvc, scm, file): (tmp_dir / file).write_text("") scm._reset() - dvcfile = Dvcfile(dvc, file) + dvcfile = load_file(dvc, file) with pytest.raises(FileIsGitIgnored) as exc_info: dvcfile._load() @@ -127,8 +127,8 @@ def test_try_loading_dvcfile_that_is_gitignored(tmp_dir, dvc, scm, file): def test_dvcfile_encoding_error(tmp_dir, dvc): - tmp_dir.gen(PIPELINE_FILE, b"\x80some: stuff") + tmp_dir.gen(PROJECT_FILE, b"\x80some: stuff") - dvcfile = Dvcfile(dvc, PIPELINE_FILE) + dvcfile = load_file(dvc, PROJECT_FILE) with pytest.raises(EncodingError): dvcfile._load()