mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
dvc: implement multistage dvcfile (#3676)
* dvc: use lockfile as yaml instead of json * refactor Dvcfile into Pipeline file and Single stage file * fix tests * dvc: fix outputs * add more tests for collection of outputs * add tests for data cloud/get/import/ls * tests: test for checkouts * Allow other checksums other than md5 * tests: use iterdir instead of os.listdir * cleanup errors reported by cc and ds * utils: throw DvcException instead of plain Exception * tests: use yaml.load instead of json.load * run: split assignments * Update dvc/stage/exceptions.py Co-authored-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
co-authored by
Ruslan Kuprieiev
parent
e03128a8de
commit
1937527249
+9
-19
@@ -8,25 +8,15 @@ from dvc.exceptions import DvcException
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _stage_repr(stage):
|
||||
from dvc.stage import PipelineStage
|
||||
|
||||
return (
|
||||
"{}:{}".format(stage.relpath, stage.name)
|
||||
if isinstance(stage, PipelineStage)
|
||||
else stage.relpath
|
||||
)
|
||||
|
||||
|
||||
class CmdPipelineShow(CmdBase):
|
||||
def _show(self, target, commands, outs, locked):
|
||||
import networkx
|
||||
from dvc import dvcfile
|
||||
from dvc.utils import parse_target
|
||||
|
||||
path, name = parse_target(target)
|
||||
stage = dvcfile.Dvcfile(self.repo, path).stages[name]
|
||||
G = self.repo.pipeline_graph
|
||||
path, name, tag = parse_target(target)
|
||||
stage = dvcfile.Dvcfile(self.repo, path, tag=tag).stages[name]
|
||||
G = self.repo.graph
|
||||
stages = networkx.dfs_postorder_nodes(G, stage)
|
||||
if locked:
|
||||
stages = [s for s in stages if s.locked]
|
||||
@@ -40,7 +30,7 @@ class CmdPipelineShow(CmdBase):
|
||||
for out in stage.outs:
|
||||
logger.info(str(out))
|
||||
else:
|
||||
logger.info(_stage_repr(stage))
|
||||
logger.info(stage.addressing)
|
||||
|
||||
def _build_graph(self, target, commands=False, outs=False):
|
||||
import networkx
|
||||
@@ -48,8 +38,8 @@ class CmdPipelineShow(CmdBase):
|
||||
from dvc.repo.graph import get_pipeline
|
||||
from dvc.utils import parse_target
|
||||
|
||||
path, name = parse_target(target)
|
||||
target_stage = dvcfile.Dvcfile(self.repo, path).stages[name]
|
||||
path, name, tag = parse_target(target)
|
||||
target_stage = dvcfile.Dvcfile(self.repo, path, tag=tag).stages[name]
|
||||
G = get_pipeline(self.repo.pipelines, target_stage)
|
||||
|
||||
nodes = set()
|
||||
@@ -62,7 +52,7 @@ class CmdPipelineShow(CmdBase):
|
||||
for out in stage.outs:
|
||||
nodes.add(str(out))
|
||||
else:
|
||||
nodes.add(_stage_repr(stage))
|
||||
nodes.add(stage.addressing)
|
||||
|
||||
edges = []
|
||||
for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
|
||||
@@ -75,7 +65,7 @@ class CmdPipelineShow(CmdBase):
|
||||
for to_out in to_stage.outs:
|
||||
edges.append((str(from_out), str(to_out)))
|
||||
else:
|
||||
edges.append((_stage_repr(from_stage), _stage_repr(to_stage)))
|
||||
edges.append((from_stage.addressing, to_stage.addressing))
|
||||
|
||||
return list(nodes), edges, networkx.is_tree(G)
|
||||
|
||||
@@ -163,7 +153,7 @@ class CmdPipelineList(CmdBase):
|
||||
pipelines = self.repo.pipelines
|
||||
for pipeline in pipelines:
|
||||
for stage in pipeline:
|
||||
logger.info(_stage_repr(stage))
|
||||
logger.info(stage.addressing)
|
||||
if len(pipeline) != 0:
|
||||
logger.info("=" * 80)
|
||||
logger.info("{} pipelines total".format(len(pipelines)))
|
||||
|
||||
+197
-152
@@ -1,5 +1,5 @@
|
||||
import contextlib
|
||||
import os
|
||||
import re
|
||||
import logging
|
||||
|
||||
import dvc.prompt as prompt
|
||||
@@ -7,11 +7,7 @@ import dvc.prompt as prompt
|
||||
from voluptuous import MultipleInvalid
|
||||
from dvc import serialize
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.loader import SingleStageLoader, StageLoader
|
||||
from dvc.schema import (
|
||||
COMPILED_SINGLE_STAGE_SCHEMA,
|
||||
COMPILED_MULTI_STAGE_SCHEMA,
|
||||
)
|
||||
from dvc.stage.loader import SingleStageLoader, StageLoader
|
||||
from dvc.stage.exceptions import (
|
||||
StageFileBadNameError,
|
||||
StageFileDoesNotExistError,
|
||||
@@ -20,6 +16,7 @@ from dvc.stage.exceptions import (
|
||||
StageFileAlreadyExistsError,
|
||||
)
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.collections import apply_diff
|
||||
from dvc.utils.stage import (
|
||||
dump_stage_file,
|
||||
parse_stage,
|
||||
@@ -30,186 +27,132 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
DVC_FILE = "Dvcfile"
|
||||
DVC_FILE_SUFFIX = ".dvc"
|
||||
TAG_REGEX = r"^(?P<path>.*)@(?P<tag>[^\\/@:]*)$"
|
||||
PIPELINE_FILE = "pipelines.yaml"
|
||||
PIPELINE_LOCK = "pipelines.lock"
|
||||
|
||||
|
||||
class MultiStageFileLoadError(DvcException):
|
||||
def __init__(self, file):
|
||||
super().__init__("Cannot load multi-stage file: '{}'".format(file))
|
||||
class LockfileCorruptedError(DvcException):
|
||||
def __init__(self, path):
|
||||
super().__init__("Lockfile '{}' is corrupted.".format(path))
|
||||
|
||||
|
||||
class Dvcfile:
|
||||
def __init__(self, repo, path):
|
||||
def is_valid_filename(path):
|
||||
return path.endswith(DVC_FILE_SUFFIX) or os.path.basename(path) in [
|
||||
DVC_FILE,
|
||||
PIPELINE_FILE,
|
||||
]
|
||||
|
||||
|
||||
def is_dvc_file(path):
|
||||
return os.path.isfile(path) and (
|
||||
is_valid_filename(path) or os.path.basename(path) == PIPELINE_LOCK
|
||||
)
|
||||
|
||||
|
||||
def check_dvc_filename(path):
|
||||
if not is_valid_filename(path):
|
||||
raise StageFileBadNameError(
|
||||
"bad DVC-file name '{}'. DVC-files should be named "
|
||||
"'Dvcfile' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
|
||||
relpath(path), os.path.basename(path)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class FileMixin:
|
||||
SCHEMA = None
|
||||
|
||||
def __init__(self, repo, path, **kwargs):
|
||||
self.repo = repo
|
||||
self.path, self.tag = self._get_path_tag(path)
|
||||
self.path = path
|
||||
|
||||
def __repr__(self):
|
||||
return "{}: {}".format(
|
||||
DVC_FILE, relpath(self.path, self.repo.root_dir)
|
||||
self.__class__.__name__, relpath(self.path, self.repo.root_dir)
|
||||
)
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.path)
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.repo == other.repo and os.path.abspath(
|
||||
self.path
|
||||
) == os.path.abspath(other.path)
|
||||
|
||||
def __str__(self):
|
||||
return "{}: {}".format(DVC_FILE, self.relpath)
|
||||
return "{}: {}".format(self.__class__.__name__, self.relpath)
|
||||
|
||||
@property
|
||||
def relpath(self):
|
||||
return relpath(self.path)
|
||||
|
||||
@property
|
||||
def stage(self):
|
||||
data, raw = self._load()
|
||||
if not self.is_multi_stage(data):
|
||||
return SingleStageLoader.load_stage(self, data, raw)
|
||||
raise MultiStageFileLoadError(self.path)
|
||||
|
||||
@property
|
||||
def lockfile(self):
|
||||
return os.path.splitext(self.path)[0] + ".lock"
|
||||
|
||||
@property
|
||||
def stages(self):
|
||||
from . import lockfile
|
||||
|
||||
data, raw = self._load()
|
||||
if self.is_multi_stage(data):
|
||||
lockfile_data = lockfile.load(self.repo, self.lockfile)
|
||||
return StageLoader(self, data.get("stages", {}), lockfile_data)
|
||||
return SingleStageLoader(self, data, raw)
|
||||
|
||||
@classmethod
|
||||
def is_valid_filename(cls, path):
|
||||
return (
|
||||
path.endswith(DVC_FILE_SUFFIX)
|
||||
or os.path.basename(path) == DVC_FILE
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def is_stage_file(cls, path):
|
||||
return os.path.isfile(path) and cls.is_valid_filename(path)
|
||||
|
||||
@classmethod
|
||||
def check_dvc_filename(cls, path):
|
||||
if not cls.is_valid_filename(path):
|
||||
raise StageFileBadNameError(
|
||||
"bad DVC-file name '{}'. DVC-files should be named "
|
||||
"'Dvcfile' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
|
||||
relpath(path), os.path.basename(path)
|
||||
)
|
||||
)
|
||||
|
||||
def exists(self):
|
||||
return self.repo.tree.exists(self.path)
|
||||
|
||||
def check_file_exists(self):
|
||||
if not self.exists():
|
||||
raise StageFileDoesNotExistError(self.path)
|
||||
|
||||
def check_isfile(self):
|
||||
if not self.repo.tree.isfile(self.path):
|
||||
raise StageFileIsNotDvcFileError(self.path)
|
||||
|
||||
@staticmethod
|
||||
def _get_path_tag(s):
|
||||
regex = re.compile(TAG_REGEX)
|
||||
match = regex.match(s)
|
||||
if not match:
|
||||
return s, None
|
||||
return match.group("path"), match.group("tag")
|
||||
|
||||
def dump(self, stage, update_dvcfile=False):
|
||||
"""Dumps given stage appropriately in the dvcfile."""
|
||||
from dvc.stage import create_stage, PipelineStage, Stage
|
||||
|
||||
if not isinstance(stage, PipelineStage):
|
||||
self.dump_single_stage(stage)
|
||||
return
|
||||
|
||||
self.dump_lockfile(stage)
|
||||
if update_dvcfile and not stage.is_data_source:
|
||||
self.dump_multistage_dvcfile(stage)
|
||||
|
||||
for out in filter(lambda o: o.use_cache, stage.outs):
|
||||
s = create_stage(
|
||||
Stage,
|
||||
stage.repo,
|
||||
os.path.join(stage.wdir, out.def_path + DVC_FILE_SUFFIX),
|
||||
wdir=stage.wdir,
|
||||
)
|
||||
s.outs = [out]
|
||||
s.md5 = s._compute_md5()
|
||||
Dvcfile(s.repo, s.path).dump_single_stage(s)
|
||||
|
||||
def dump_lockfile(self, stage):
|
||||
from . import lockfile
|
||||
|
||||
lockfile.dump(self.repo, self.lockfile, serialize.to_lockfile(stage))
|
||||
self.repo.scm.track_file(relpath(self.lockfile))
|
||||
|
||||
def dump_multistage_dvcfile(self, stage):
|
||||
data = {}
|
||||
if self.exists():
|
||||
with open(self.path, "r") as fd:
|
||||
data = parse_stage_for_update(fd.read(), self.path)
|
||||
if not self.is_multi_stage(data):
|
||||
raise MultiStageFileLoadError(self.path)
|
||||
else:
|
||||
open(self.path, "w+").close()
|
||||
|
||||
data["stages"] = data.get("stages", {})
|
||||
data["stages"].update(serialize.to_dvcfile(stage))
|
||||
|
||||
dump_stage_file(self.path, COMPILED_MULTI_STAGE_SCHEMA(data))
|
||||
self.repo.scm.track_file(relpath(self.path))
|
||||
|
||||
def dump_single_stage(self, stage):
|
||||
self.check_dvc_filename(self.path)
|
||||
|
||||
logger.debug(
|
||||
"Saving information to '{file}'.".format(file=relpath(self.path))
|
||||
)
|
||||
|
||||
dump_stage_file(self.path, serialize.to_single_stage_file(stage))
|
||||
self.repo.scm.track_file(relpath(self.path))
|
||||
|
||||
def _load(self):
|
||||
# it raises the proper exceptions by priority:
|
||||
# 1. when the file doesn't exists
|
||||
# 2. filename is not a DVC-file
|
||||
# 3. path doesn't represent a regular file
|
||||
self.check_file_exists()
|
||||
self.check_dvc_filename(self.path)
|
||||
self.check_isfile()
|
||||
if not self.exists():
|
||||
raise StageFileDoesNotExistError(self.path)
|
||||
check_dvc_filename(self.path)
|
||||
if not self.repo.tree.isfile(self.path):
|
||||
raise StageFileIsNotDvcFileError(self.path)
|
||||
|
||||
with self.repo.tree.open(self.path) as fd:
|
||||
stage_text = fd.read()
|
||||
d = parse_stage(stage_text, self.path)
|
||||
self.validate(d, self.path)
|
||||
return d, stage_text
|
||||
|
||||
@staticmethod
|
||||
def validate_single_stage(d, fname=None):
|
||||
Dvcfile._validate(COMPILED_SINGLE_STAGE_SCHEMA, d, fname)
|
||||
|
||||
@staticmethod
|
||||
def validate_multi_stage(d, fname=None):
|
||||
Dvcfile._validate(COMPILED_MULTI_STAGE_SCHEMA, d, fname)
|
||||
|
||||
@staticmethod
|
||||
def _validate(schema, d, fname=None):
|
||||
@classmethod
|
||||
def validate(cls, d, fname=None):
|
||||
assert cls.SCHEMA
|
||||
try:
|
||||
schema(d)
|
||||
cls.SCHEMA(d)
|
||||
except MultipleInvalid as exc:
|
||||
raise StageFileFormatError(fname, exc)
|
||||
|
||||
def is_multi_stage(self, d=None):
|
||||
if d is None:
|
||||
d = self._load()[0]
|
||||
check_multi_stage = d.get("stages")
|
||||
if check_multi_stage:
|
||||
self.validate_multi_stage(d, self.path)
|
||||
return True
|
||||
def remove_with_prompt(self, force=False):
|
||||
raise NotImplementedError
|
||||
|
||||
self.validate_single_stage(d, self.path)
|
||||
return False
|
||||
def remove(self, force=False):
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(self.path)
|
||||
|
||||
def dump(self, stage, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class SingleStageFile(FileMixin):
|
||||
from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
|
||||
|
||||
def __init__(self, repo, path, tag=None):
|
||||
super().__init__(repo, path)
|
||||
self.tag = tag
|
||||
|
||||
@property
|
||||
def stage(self):
|
||||
data, raw = self._load()
|
||||
return SingleStageLoader.load_stage(self, data, raw, tag=self.tag)
|
||||
|
||||
@property
|
||||
def stages(self):
|
||||
data, raw = self._load()
|
||||
return SingleStageLoader(self, data, raw, tag=self.tag)
|
||||
|
||||
def dump(self, stage, **kwargs):
|
||||
"""Dumps given stage appropriately in the dvcfile."""
|
||||
from dvc.stage import PipelineStage
|
||||
|
||||
assert not isinstance(stage, PipelineStage)
|
||||
check_dvc_filename(self.path)
|
||||
logger.debug(
|
||||
"Saving information to '{file}'.".format(file=relpath(self.path))
|
||||
)
|
||||
dump_stage_file(self.path, serialize.to_single_stage_file(stage))
|
||||
self.repo.scm.track_file(relpath(self.path))
|
||||
|
||||
def remove_with_prompt(self, force=False):
|
||||
if not self.exists():
|
||||
@@ -222,4 +165,106 @@ class Dvcfile:
|
||||
if not (force or prompt.confirm(msg)):
|
||||
raise StageFileAlreadyExistsError(self.path)
|
||||
|
||||
os.unlink(self.path)
|
||||
self.remove()
|
||||
|
||||
|
||||
class PipelineFile(FileMixin):
|
||||
"""Abstraction for pipelines file, .yaml + .lock combined."""
|
||||
|
||||
from dvc.schema import COMPILED_MULTI_STAGE_SCHEMA as SCHEMA
|
||||
|
||||
@property
|
||||
def _lockfile(self):
|
||||
return Lockfile(self.repo, os.path.splitext(self.path)[0] + ".lock")
|
||||
|
||||
def dump(self, stage, update_pipeline=False, **kwargs):
|
||||
"""Dumps given stage appropriately in the dvcfile."""
|
||||
from dvc.stage import PipelineStage
|
||||
|
||||
assert isinstance(stage, PipelineStage)
|
||||
check_dvc_filename(self.path)
|
||||
self._dump_lockfile(stage)
|
||||
if update_pipeline and not stage.is_data_source:
|
||||
self._dump_pipeline_file(stage)
|
||||
|
||||
def _dump_lockfile(self, stage):
|
||||
self._lockfile.dump(stage)
|
||||
|
||||
def _dump_pipeline_file(self, stage):
|
||||
data = {}
|
||||
if self.exists():
|
||||
with open(self.path, "r") as fd:
|
||||
data = parse_stage_for_update(fd.read(), self.path)
|
||||
else:
|
||||
open(self.path, "w+").close()
|
||||
|
||||
data["stages"] = data.get("stages", {})
|
||||
stage_data = serialize.to_pipeline_file(stage)
|
||||
if data["stages"].get(stage.name):
|
||||
orig_stage_data = data["stages"][stage.name]
|
||||
apply_diff(stage_data[stage.name], orig_stage_data)
|
||||
else:
|
||||
data["stages"].update(stage_data)
|
||||
|
||||
dump_stage_file(self.path, data)
|
||||
self.repo.scm.track_file(relpath(self.path))
|
||||
|
||||
@property
|
||||
def stage(self):
|
||||
raise DvcException(
|
||||
"PipelineFile has multiple stages. Please specify it's name."
|
||||
)
|
||||
|
||||
@property
|
||||
def stages(self):
|
||||
data, _ = self._load()
|
||||
lockfile_data = self._lockfile.load()
|
||||
return StageLoader(self, data.get("stages", {}), lockfile_data)
|
||||
|
||||
def remove(self, force=False):
|
||||
if not force:
|
||||
logger.warning("Cannot remove pipeline file.")
|
||||
return
|
||||
|
||||
super().remove()
|
||||
self._lockfile.remove()
|
||||
|
||||
|
||||
class Lockfile(FileMixin):
|
||||
from dvc.schema import COMPILED_LOCKFILE_SCHEMA as SCHEMA
|
||||
|
||||
def load(self):
|
||||
if not self.exists():
|
||||
return {}
|
||||
with self.repo.tree.open(self.path) as fd:
|
||||
data = parse_stage(fd.read(), self.path)
|
||||
try:
|
||||
self.validate(data, fname=self.path)
|
||||
except StageFileFormatError:
|
||||
raise LockfileCorruptedError(self.path)
|
||||
return data
|
||||
|
||||
def dump(self, stage, **kwargs):
|
||||
stage_data = serialize.to_lockfile(stage)
|
||||
if not self.exists():
|
||||
data = stage_data
|
||||
open(self.path, "w+").close()
|
||||
else:
|
||||
with self.repo.tree.open(self.path, "r") as fd:
|
||||
data = parse_stage_for_update(fd.read(), self.path)
|
||||
data.update(stage_data)
|
||||
|
||||
dump_stage_file(self.path, data)
|
||||
self.repo.scm.track_file(relpath(self.path))
|
||||
|
||||
|
||||
class Dvcfile:
|
||||
def __new__(cls, repo, path, **kwargs):
|
||||
assert path
|
||||
assert repo
|
||||
|
||||
_, ext = os.path.splitext(path)
|
||||
if ext in [".yaml", ".yml"]:
|
||||
return PipelineFile(repo, path, **kwargs)
|
||||
# fallback to single stage file for better error messages
|
||||
return SingleStageFile(repo, path, **kwargs)
|
||||
|
||||
+11
-5
@@ -25,10 +25,14 @@ class OutputDuplicationError(DvcException):
|
||||
assert all(hasattr(stage, "relpath") for stage in stages)
|
||||
msg = (
|
||||
"file/directory '{}' is specified as an output in more than one "
|
||||
"stage: {}\n"
|
||||
"stage: \n{}\n"
|
||||
"This is not allowed. Consider using a different output name."
|
||||
).format(output, "\n ".join(s.relpath for s in stages))
|
||||
).format(
|
||||
output, "\n".join("\t{}".format(s.addressing) for s in stages)
|
||||
)
|
||||
super().__init__(msg)
|
||||
self.stages = stages
|
||||
self.output = output
|
||||
|
||||
|
||||
class OutputNotFoundError(DvcException):
|
||||
@@ -60,8 +64,8 @@ class StagePathAsOutputError(DvcException):
|
||||
def __init__(self, stage, output):
|
||||
assert isinstance(output, str)
|
||||
super().__init__(
|
||||
"'{stage}' is within an output '{output}' of another stage".format(
|
||||
stage=stage.relpath, output=output
|
||||
"{stage} is within an output '{output}' of another stage".format(
|
||||
stage=stage, output=output
|
||||
)
|
||||
)
|
||||
|
||||
@@ -129,7 +133,9 @@ class DvcParserError(DvcException):
|
||||
class CyclicGraphError(DvcException):
|
||||
def __init__(self, stages):
|
||||
assert isinstance(stages, list)
|
||||
stages = "\n".join("\t- {}".format(stage.relpath) for stage in stages)
|
||||
stages = "\n".join(
|
||||
"\t- {}".format(stage.addressing) for stage in stages
|
||||
)
|
||||
msg = (
|
||||
"you've introduced a cycle in your pipeline that involves "
|
||||
"the following stages:"
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
import json
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
from collections import OrderedDict
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.schema import COMPILED_LOCKFILE_SCHEMA
|
||||
from voluptuous import MultipleInvalid
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.repo import Repo
|
||||
|
||||
|
||||
class LockfileCorruptedError(DvcException):
|
||||
def __init__(self, path):
|
||||
super().__init__("Lockfile '{}' is corrupted.".format(path))
|
||||
|
||||
|
||||
def exists(repo: "Repo", path: str) -> bool:
|
||||
return repo.tree.exists(path)
|
||||
|
||||
|
||||
def read(repo: "Repo", path: str) -> dict:
|
||||
with repo.tree.open(path) as f:
|
||||
return json.load(f, object_pairs_hook=OrderedDict)
|
||||
|
||||
|
||||
def write(repo: "Repo", path: str, data: dict) -> None:
|
||||
with repo.tree.open(path, "w+") as f:
|
||||
json.dump(data, f)
|
||||
|
||||
|
||||
def load(repo: "Repo", path: str) -> dict:
|
||||
if not exists(repo, path):
|
||||
return {}
|
||||
try:
|
||||
return COMPILED_LOCKFILE_SCHEMA(read(repo, path))
|
||||
except MultipleInvalid:
|
||||
raise LockfileCorruptedError(path)
|
||||
|
||||
|
||||
def dump(repo: "Repo", path: str, stage_data: dict):
|
||||
if not exists(repo, path):
|
||||
data = stage_data
|
||||
else:
|
||||
data = read(repo, path)
|
||||
data.update(stage_data)
|
||||
|
||||
write(repo, path, COMPILED_LOCKFILE_SCHEMA(data))
|
||||
+13
-6
@@ -148,6 +148,10 @@ class OutputBase(object):
|
||||
def cache_path(self):
|
||||
return self.cache.checksum_to_path_info(self.checksum).url
|
||||
|
||||
@property
|
||||
def checksum_type(self):
|
||||
return self.remote.PARAM_CHECKSUM
|
||||
|
||||
@property
|
||||
def checksum(self):
|
||||
return self.info.get(self.remote.PARAM_CHECKSUM)
|
||||
@@ -156,6 +160,9 @@ class OutputBase(object):
|
||||
def checksum(self, checksum):
|
||||
self.info[self.remote.PARAM_CHECKSUM] = checksum
|
||||
|
||||
def get_checksum(self):
|
||||
return self.remote.get_checksum(self.path_info)
|
||||
|
||||
@property
|
||||
def is_dir_checksum(self):
|
||||
return self.remote.is_dir_checksum(self.checksum)
|
||||
@@ -168,7 +175,7 @@ class OutputBase(object):
|
||||
return self.remote.save_info(self.path_info)
|
||||
|
||||
def changed_checksum(self):
|
||||
return self.checksum != self.remote.get_checksum(self.path_info)
|
||||
return self.checksum != self.get_checksum()
|
||||
|
||||
def changed_cache(self, filter_info=None):
|
||||
if not self.use_cache or not self.checksum:
|
||||
@@ -438,9 +445,9 @@ class OutputBase(object):
|
||||
if self.exists:
|
||||
msg += (
|
||||
"\n"
|
||||
"You can also use `dvc commit {stage}` to associate "
|
||||
"existing '{out}' with '{stage}'.".format(
|
||||
out=self, stage=self.stage.relpath
|
||||
"You can also use `dvc commit {stage.addressing}` "
|
||||
"to associate existing '{out}' with {stage}.".format(
|
||||
out=self, stage=self.stage
|
||||
)
|
||||
)
|
||||
logger.warning(msg)
|
||||
@@ -459,7 +466,7 @@ class OutputBase(object):
|
||||
|
||||
@classmethod
|
||||
def _validate_output_path(cls, path):
|
||||
from dvc.dvcfile import Dvcfile
|
||||
from dvc.dvcfile import is_valid_filename
|
||||
|
||||
if Dvcfile.is_valid_filename(path):
|
||||
if is_valid_filename(path):
|
||||
raise cls.IsStageFileError(path)
|
||||
|
||||
+40
-85
@@ -18,6 +18,7 @@ from dvc.path_info import PathInfo
|
||||
from dvc.remote.base import RemoteActionNotImplemented
|
||||
from dvc.utils.fs import path_isin
|
||||
from .graph import check_acyclic, get_pipeline, get_pipelines
|
||||
from ..utils import parse_target
|
||||
|
||||
|
||||
def locked(f):
|
||||
@@ -175,7 +176,7 @@ class Repo(object):
|
||||
|
||||
self.scm.ignore_list(flist)
|
||||
|
||||
def check_modified_graph(self, new_stages, old_stages=None):
|
||||
def check_modified_graph(self, new_stages):
|
||||
"""Generate graph including the new stage to check for errors"""
|
||||
# Building graph might be costly for the ones with many DVC-files,
|
||||
# so we provide this undocumented hack to skip it. See [1] for
|
||||
@@ -190,30 +191,14 @@ class Repo(object):
|
||||
#
|
||||
# [1] https://github.com/iterative/dvc/issues/2671
|
||||
if not getattr(self, "_skip_graph_checks", False):
|
||||
self._collect_graph((old_stages or self.stages) + new_stages)
|
||||
self._collect_graph(self.stages + new_stages)
|
||||
|
||||
def _collect_inside(self, path, graph):
|
||||
import networkx as nx
|
||||
|
||||
stages = nx.dfs_postorder_nodes(graph or self.pipeline_graph)
|
||||
stages = nx.dfs_postorder_nodes(graph)
|
||||
return [stage for stage in stages if path_isin(stage.path, path)]
|
||||
|
||||
def collect_for_pipelines(
|
||||
self, path=None, name=None, recursive=False, graph=None
|
||||
):
|
||||
from ..dvcfile import Dvcfile
|
||||
|
||||
if not path:
|
||||
return list(graph) if graph else self.pipeline_stages
|
||||
|
||||
path = os.path.abspath(path)
|
||||
if recursive and os.path.isdir(path):
|
||||
return self._collect_inside(path, graph or self.pipeline_graph)
|
||||
|
||||
dvcfile = Dvcfile(self, path)
|
||||
dvcfile.check_file_exists()
|
||||
return [dvcfile.stages[name]]
|
||||
|
||||
def collect(self, target, with_deps=False, recursive=False, graph=None):
|
||||
import networkx as nx
|
||||
from ..dvcfile import Dvcfile
|
||||
@@ -221,33 +206,38 @@ class Repo(object):
|
||||
if not target:
|
||||
return list(graph) if graph else self.stages
|
||||
|
||||
target = os.path.abspath(target)
|
||||
|
||||
if recursive and os.path.isdir(target):
|
||||
return self._collect_inside(target, graph or self.graph)
|
||||
return self._collect_inside(
|
||||
os.path.abspath(target), graph or self.graph
|
||||
)
|
||||
|
||||
stage = Dvcfile(self, target).stage
|
||||
|
||||
# Optimization: do not collect the graph for a specific target
|
||||
file, name, tag = parse_target(target)
|
||||
dvcfile = Dvcfile(self, file, tag=tag)
|
||||
stages = list(dvcfile.stages.filter(name).values())
|
||||
if not with_deps:
|
||||
return [stage]
|
||||
return stages
|
||||
|
||||
pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
|
||||
return list(nx.dfs_postorder_nodes(pipeline, stage))
|
||||
res = set()
|
||||
for stage in stages:
|
||||
pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
|
||||
res.update(nx.dfs_postorder_nodes(pipeline, stage))
|
||||
return res
|
||||
|
||||
def collect_granular(self, target, *args, **kwargs):
|
||||
from ..dvcfile import Dvcfile
|
||||
from ..dvcfile import Dvcfile, is_valid_filename
|
||||
|
||||
if not target:
|
||||
return [(stage, None) for stage in self.stages]
|
||||
|
||||
# Optimization: do not collect the graph for a specific .dvc target
|
||||
if Dvcfile.is_valid_filename(target) and not kwargs.get("with_deps"):
|
||||
return [(Dvcfile(self, target).stage, None)]
|
||||
file, name, tag = parse_target(target)
|
||||
if is_valid_filename(file) and not kwargs.get("with_deps"):
|
||||
# Optimization: do not collect the graph for a specific .dvc target
|
||||
stages = Dvcfile(self, file, tag=tag).stages.filter(name)
|
||||
return [(stage, None) for stage in stages.values()]
|
||||
|
||||
try:
|
||||
(out,) = self.find_outs_by_path(target, strict=False)
|
||||
filter_info = PathInfo(os.path.abspath(target))
|
||||
(out,) = self.find_outs_by_path(file, strict=False)
|
||||
filter_info = PathInfo(os.path.abspath(file))
|
||||
return [(out.stage, filter_info)]
|
||||
except OutputNotFoundError:
|
||||
stages = self.collect(target, *args, **kwargs)
|
||||
@@ -383,9 +373,9 @@ class Repo(object):
|
||||
"rerun command with non overlapping outs paths."
|
||||
).format(
|
||||
str(parent),
|
||||
parent.stage.relpath,
|
||||
parent.stage.addressing,
|
||||
str(overlapping),
|
||||
overlapping.stage.relpath,
|
||||
overlapping.stage.addressing,
|
||||
)
|
||||
raise OverlappingOutputPathsError(parent, overlapping, msg)
|
||||
|
||||
@@ -419,7 +409,7 @@ class Repo(object):
|
||||
|
||||
@cached_property
|
||||
def pipelines(self):
|
||||
return get_pipelines(self.pipeline_graph)
|
||||
return get_pipelines(self.graph)
|
||||
|
||||
@cached_property
|
||||
def stages(self):
|
||||
@@ -432,60 +422,27 @@ class Repo(object):
|
||||
NOTE: For large repos, this could be an expensive
|
||||
operation. Consider using some memoization.
|
||||
"""
|
||||
return self._collect_stages()[0]
|
||||
|
||||
@cached_property
|
||||
def pipeline_stages(self):
|
||||
return self._collect_stages()[1]
|
||||
|
||||
@cached_property
|
||||
def pipeline_graph(self):
|
||||
return self._collect_graph(self.pipeline_stages)
|
||||
return self._collect_stages()
|
||||
|
||||
def _collect_stages(self):
|
||||
from dvc.dvcfile import Dvcfile
|
||||
from dvc.stage import PipelineStage
|
||||
from dvc.dvcfile import Dvcfile, is_valid_filename
|
||||
|
||||
pipeline_stages = []
|
||||
output_stages = []
|
||||
ignored_outs = set()
|
||||
stages = []
|
||||
outs = set()
|
||||
|
||||
for root, dirs, files in self.tree.walk(self.root_dir):
|
||||
for fname in files:
|
||||
path = os.path.join(root, fname)
|
||||
if not Dvcfile.is_valid_filename(path):
|
||||
continue
|
||||
dvcfile = Dvcfile(self, path)
|
||||
for stage in dvcfile.stages.values():
|
||||
if isinstance(stage, PipelineStage):
|
||||
ignored_outs.update(
|
||||
out.path_info for out in stage.outs
|
||||
)
|
||||
pipeline_stages.append(stage)
|
||||
else:
|
||||
# Old single-stages are used for both
|
||||
# outputs and pipelines.
|
||||
output_stages.append(stage)
|
||||
|
||||
outs.update(
|
||||
out.fspath
|
||||
for out in stage.outs
|
||||
if out.scheme == "local"
|
||||
for file_name in filter(is_valid_filename, files):
|
||||
path = os.path.join(root, file_name)
|
||||
stages.extend(list(Dvcfile(self, path).stages.values()))
|
||||
outs.update(
|
||||
out.fspath
|
||||
for stage in stages
|
||||
for out in (
|
||||
out for out in stage.outs if out.scheme == "local"
|
||||
)
|
||||
|
||||
)
|
||||
dirs[:] = [d for d in dirs if os.path.join(root, d) not in outs]
|
||||
|
||||
# DVC files are generated by multi-stage for data management.
|
||||
# We need to ignore those stages for pipelines_stages, but still
|
||||
# should be collected for output stages.
|
||||
pipeline_stages.extend(
|
||||
stage
|
||||
for stage in output_stages
|
||||
if not stage.outs
|
||||
or all(out.path_info not in ignored_outs for out in stage.outs)
|
||||
)
|
||||
return output_stages, pipeline_stages
|
||||
return stages
|
||||
|
||||
def find_outs_by_path(self, path, outs=None, recursive=False, strict=True):
|
||||
if not outs:
|
||||
@@ -582,5 +539,3 @@ class Repo(object):
|
||||
self.__dict__.pop("stages", None)
|
||||
self.__dict__.pop("pipelines", None)
|
||||
self.__dict__.pop("dvcignore", None)
|
||||
self.__dict__.pop("pipeline_graph", None)
|
||||
self.__dict__.pop("pipeline_stages", None)
|
||||
|
||||
+8
-3
@@ -4,10 +4,11 @@ import os
|
||||
import colorama
|
||||
|
||||
from . import locked
|
||||
from ..dvcfile import Dvcfile
|
||||
from dvc.dvcfile import Dvcfile, is_dvc_file
|
||||
from ..exceptions import (
|
||||
RecursiveAddingWhileUsingFilename,
|
||||
OverlappingOutputPathsError,
|
||||
OutputDuplicationError,
|
||||
)
|
||||
from ..output.base import OutputDoesNotExistError
|
||||
from ..progress import Tqdm
|
||||
@@ -63,11 +64,15 @@ def add(repo, targets, recursive=False, no_commit=False, fname=None):
|
||||
).format(
|
||||
out=exc.overlapping_out.path_info,
|
||||
parent=exc.parent.path_info,
|
||||
parent_stage=exc.parent.stage.relpath,
|
||||
parent_stage=exc.parent.stage.addressing,
|
||||
)
|
||||
raise OverlappingOutputPathsError(
|
||||
exc.parent, exc.overlapping_out, msg
|
||||
)
|
||||
except OutputDuplicationError as exc:
|
||||
raise OutputDuplicationError(
|
||||
exc.output, set(exc.stages) - set(stages)
|
||||
)
|
||||
|
||||
with Tqdm(
|
||||
total=len(stages),
|
||||
@@ -107,7 +112,7 @@ def _find_all_targets(repo, target, recursive):
|
||||
unit="file",
|
||||
)
|
||||
if not repo.is_dvc_internal(fname)
|
||||
if not Dvcfile.is_stage_file(fname)
|
||||
if not is_dvc_file(fname)
|
||||
if not repo.scm.belongs_to_scm(fname)
|
||||
if not repo.scm.is_tracked(fname)
|
||||
]
|
||||
|
||||
+2
-1
@@ -5,7 +5,8 @@ from . import locked
|
||||
@locked
|
||||
def _destroy_stages(repo):
|
||||
for stage in repo.stages:
|
||||
stage.remove(remove_outs=False)
|
||||
stage.unprotect_outs()
|
||||
stage.dvcfile.remove(force=True)
|
||||
|
||||
|
||||
# NOTE: not locking `destroy`, as `remove` will need to delete `.dvc` dir,
|
||||
|
||||
+2
-2
@@ -22,11 +22,11 @@ class GetDVCFileError(DvcException):
|
||||
@staticmethod
|
||||
def get(url, path, out=None, rev=None):
|
||||
from dvc.external_repo import external_repo
|
||||
from dvc.dvcfile import Dvcfile
|
||||
from dvc.dvcfile import is_valid_filename
|
||||
|
||||
out = resolve_output(path, out)
|
||||
|
||||
if Dvcfile.is_valid_filename(out):
|
||||
if is_valid_filename(out):
|
||||
raise GetDVCFileError()
|
||||
|
||||
# Creating a directory right beside the output to make sure that they
|
||||
|
||||
+5
-1
@@ -4,6 +4,7 @@ from . import locked as locked_repo
|
||||
from dvc.repo.scm_context import scm_context
|
||||
from dvc.utils import resolve_output, resolve_paths, relpath
|
||||
from dvc.utils.fs import path_isin
|
||||
from ..exceptions import OutputDuplicationError
|
||||
|
||||
|
||||
@locked_repo
|
||||
@@ -35,7 +36,10 @@ def imp_url(self, url, out=None, fname=None, erepo=None, locked=True):
|
||||
dvcfile = Dvcfile(self, stage.path)
|
||||
dvcfile.remove_with_prompt(force=True)
|
||||
|
||||
self.check_modified_graph([stage])
|
||||
try:
|
||||
self.check_modified_graph([stage])
|
||||
except OutputDuplicationError as exc:
|
||||
raise OutputDuplicationError(exc.output, set(exc.stages) - {stage})
|
||||
|
||||
stage.run()
|
||||
|
||||
|
||||
+4
-4
@@ -6,10 +6,10 @@ def lock(self, target, unlock=False):
|
||||
from .. import dvcfile
|
||||
from dvc.utils import parse_target
|
||||
|
||||
path, target = parse_target(target)
|
||||
dvcfile = dvcfile.Dvcfile(self, path)
|
||||
stage = dvcfile.stages[target]
|
||||
path, name, tag = parse_target(target)
|
||||
dvcfile = dvcfile.Dvcfile(self, path, tag=tag)
|
||||
stage = dvcfile.stages[name]
|
||||
stage.locked = False if unlock else True
|
||||
dvcfile.dump(stage, update_dvcfile=True)
|
||||
dvcfile.dump(stage, update_pipeline=True)
|
||||
|
||||
return stage
|
||||
|
||||
+1
-1
@@ -46,7 +46,7 @@ def move(self, from_path, to_path):
|
||||
stage = out.stage
|
||||
|
||||
if not stage.is_data_source:
|
||||
raise MoveNotDataSourceError(stage.relpath)
|
||||
raise MoveNotDataSourceError(stage.addressing)
|
||||
|
||||
stage_name = os.path.splitext(os.path.basename(stage.path))[0]
|
||||
from_name = os.path.basename(from_out.fspath)
|
||||
|
||||
+13
-5
@@ -1,14 +1,22 @@
|
||||
import logging
|
||||
|
||||
from . import locked
|
||||
from ..utils import parse_target
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@locked
|
||||
def remove(self, target, outs_only=False):
|
||||
from ..dvcfile import Dvcfile
|
||||
|
||||
stage = Dvcfile(self, target).stage
|
||||
if outs_only:
|
||||
path, name, _ = parse_target(target)
|
||||
dvcfile = Dvcfile(self, path)
|
||||
stages = list(dvcfile.stages.filter(name).values())
|
||||
for stage in stages:
|
||||
stage.remove_outs(force=True)
|
||||
else:
|
||||
stage.remove(force=True)
|
||||
|
||||
return stage
|
||||
if not outs_only:
|
||||
dvcfile.remove()
|
||||
|
||||
return stages
|
||||
|
||||
@@ -12,8 +12,8 @@ logger = logging.getLogger(__name__)
|
||||
def _reproduce_stage(stage, **kwargs):
|
||||
if stage.locked:
|
||||
logger.warning(
|
||||
"DVC-file '{path}' is locked. Its dependencies are"
|
||||
" not going to be reproduced.".format(path=stage.relpath)
|
||||
"{} is locked. Its dependencies are"
|
||||
" not going to be reproduced.".format(stage)
|
||||
)
|
||||
|
||||
stage = stage.reproduce(**kwargs)
|
||||
@@ -73,15 +73,15 @@ def reproduce(
|
||||
if not interactive:
|
||||
kwargs["interactive"] = self.config["core"].get("interactive", False)
|
||||
|
||||
active_graph = _get_active_graph(self.pipeline_graph)
|
||||
active_graph = _get_active_graph(self.graph)
|
||||
active_pipelines = get_pipelines(active_graph)
|
||||
|
||||
path, name = parse_target(target)
|
||||
path, name, tag = parse_target(target)
|
||||
if pipeline or all_pipelines:
|
||||
if all_pipelines:
|
||||
pipelines = active_pipelines
|
||||
else:
|
||||
dvcfile = Dvcfile(self, path)
|
||||
dvcfile = Dvcfile(self, path, tag=tag)
|
||||
stage = dvcfile.stages[name]
|
||||
pipelines = [get_pipeline(active_pipelines, stage)]
|
||||
|
||||
@@ -91,9 +91,7 @@ def reproduce(
|
||||
if pipeline.in_degree(stage) == 0:
|
||||
targets.append(stage)
|
||||
else:
|
||||
targets = self.collect_for_pipelines(
|
||||
path, name=name, recursive=recursive, graph=active_graph
|
||||
)
|
||||
targets = self.collect(target, recursive=recursive, graph=active_graph)
|
||||
|
||||
return _reproduce_stages(active_graph, targets, **kwargs)
|
||||
|
||||
|
||||
+30
-19
@@ -2,13 +2,18 @@ import os
|
||||
|
||||
from . import locked
|
||||
from .scm_context import scm_context
|
||||
from ..exceptions import DvcException
|
||||
from ..utils import relpath
|
||||
from dvc.stage.exceptions import DuplicateStageName, InvalidStageName
|
||||
|
||||
from funcy import first, concat
|
||||
|
||||
from ..exceptions import OutputDuplicationError
|
||||
|
||||
def _get_file_path(**kwargs):
|
||||
|
||||
def is_valid_name(name: str):
|
||||
return not {"\\", "/", "@", ":"} & set(name)
|
||||
|
||||
|
||||
def _get_file_path(kwargs):
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX, DVC_FILE
|
||||
|
||||
out = first(
|
||||
@@ -33,30 +38,36 @@ def _get_file_path(**kwargs):
|
||||
@scm_context
|
||||
def run(self, fname=None, no_exec=False, **kwargs):
|
||||
from dvc.stage import PipelineStage, Stage, create_stage
|
||||
from dvc.dvcfile import DVC_FILE, Dvcfile
|
||||
from dvc.dvcfile import Dvcfile, PIPELINE_FILE
|
||||
|
||||
stage_cls, path = PipelineStage, fname or DVC_FILE
|
||||
if not kwargs.get("name"):
|
||||
stage_cls = PipelineStage
|
||||
path = PIPELINE_FILE
|
||||
stage_name = kwargs.get("name")
|
||||
if not stage_name:
|
||||
kwargs.pop("name", None)
|
||||
stage_cls, path = Stage, fname or _get_file_path(**kwargs)
|
||||
stage_cls = Stage
|
||||
path = fname or _get_file_path(kwargs)
|
||||
else:
|
||||
if not is_valid_name(stage_name):
|
||||
raise InvalidStageName
|
||||
|
||||
stage = create_stage(stage_cls, repo=self, path=path, **kwargs)
|
||||
if stage is None:
|
||||
return None
|
||||
|
||||
dvcfile = Dvcfile(self, path)
|
||||
if dvcfile.exists() and not dvcfile.is_multi_stage():
|
||||
if stage_cls == PipelineStage:
|
||||
raise DvcException(
|
||||
"'{}' is a single-stage dvcfile. Please use "
|
||||
"`-f <different-filename>` and try again.`.".format(
|
||||
relpath(dvcfile.path)
|
||||
)
|
||||
)
|
||||
dvcfile.remove_with_prompt(force=kwargs.get("overwrite", True))
|
||||
dvcfile = Dvcfile(self, stage.path)
|
||||
if dvcfile.exists():
|
||||
if stage_name and stage_name in dvcfile.stages:
|
||||
raise DuplicateStageName(stage_name, dvcfile)
|
||||
if stage_cls != PipelineStage:
|
||||
dvcfile.remove_with_prompt(force=kwargs.get("overwrite", True))
|
||||
|
||||
try:
|
||||
self.check_modified_graph([stage])
|
||||
except OutputDuplicationError as exc:
|
||||
raise OutputDuplicationError(exc.output, set(exc.stages) - {stage})
|
||||
|
||||
self.check_modified_graph([stage], self.pipeline_stages)
|
||||
if not no_exec:
|
||||
stage.run(no_commit=kwargs.get("no_commit", False))
|
||||
dvcfile.dump(stage, update_dvcfile=True)
|
||||
dvcfile.dump(stage, update_pipeline=True)
|
||||
return stage
|
||||
|
||||
+2
-4
@@ -16,10 +16,8 @@ def _joint_status(stages):
|
||||
for stage in stages:
|
||||
if stage.locked and not stage.is_repo_import:
|
||||
logger.warning(
|
||||
"DVC-file '{path}' is locked. Its dependencies are"
|
||||
" not going to be shown in the status output.".format(
|
||||
path=stage.relpath
|
||||
)
|
||||
"{} is locked. Its dependencies are"
|
||||
" not going to be shown in the status output.".format(stage)
|
||||
)
|
||||
|
||||
status.update(stage.status(check_updates=True))
|
||||
|
||||
+4
-2
@@ -1,4 +1,5 @@
|
||||
from dvc.stage.params import StageParams, OutputParams
|
||||
from dvc.output import CHECKSUMS_SCHEMA
|
||||
from dvc import dependency, output
|
||||
|
||||
from voluptuous import Any, Schema, Optional, Required
|
||||
@@ -16,10 +17,11 @@ SINGLE_STAGE_SCHEMA = {
|
||||
StageParams.PARAM_ALWAYS_CHANGED: bool,
|
||||
}
|
||||
|
||||
DATA_SCHEMA = {**CHECKSUMS_SCHEMA, Required("path"): str}
|
||||
LOCK_FILE_STAGE_SCHEMA = {
|
||||
Required(StageParams.PARAM_CMD): str,
|
||||
Required(StageParams.PARAM_DEPS): {str: output.CHECKSUM_SCHEMA},
|
||||
Required(StageParams.PARAM_OUTS): {str: output.CHECKSUM_SCHEMA},
|
||||
StageParams.PARAM_DEPS: [DATA_SCHEMA],
|
||||
StageParams.PARAM_OUTS: [DATA_SCHEMA],
|
||||
}
|
||||
LOCKFILE_SCHEMA = {str: LOCK_FILE_STAGE_SCHEMA}
|
||||
|
||||
|
||||
+17
-27
@@ -1,4 +1,3 @@
|
||||
from collections import OrderedDict
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from dvc.utils.collections import apply_diff
|
||||
@@ -22,7 +21,7 @@ def _get_outs(stage: "PipelineStage"):
|
||||
return outs_bucket
|
||||
|
||||
|
||||
def to_dvcfile(stage: "PipelineStage"):
|
||||
def to_pipeline_file(stage: "PipelineStage"):
|
||||
return {
|
||||
stage.name: {
|
||||
key: value
|
||||
@@ -39,34 +38,25 @@ def to_dvcfile(stage: "PipelineStage"):
|
||||
}
|
||||
|
||||
|
||||
def to_lockfile(stage: "PipelineStage") -> OrderedDict:
|
||||
def to_lockfile(stage: "PipelineStage") -> dict:
|
||||
assert stage.cmd
|
||||
assert stage.name
|
||||
|
||||
deps = OrderedDict(
|
||||
[
|
||||
(dep.def_path, dep.remote.get_checksum(dep.path_info),)
|
||||
for dep in stage.deps
|
||||
if dep.remote.get_checksum(dep.path_info)
|
||||
]
|
||||
)
|
||||
outs = OrderedDict(
|
||||
[
|
||||
(out.def_path, out.remote.get_checksum(out.path_info),)
|
||||
for out in stage.outs
|
||||
if out.remote.get_checksum(out.path_info)
|
||||
]
|
||||
)
|
||||
return OrderedDict(
|
||||
[
|
||||
(
|
||||
stage.name,
|
||||
OrderedDict(
|
||||
[("cmd", stage.cmd), ("deps", deps,), ("outs", outs)]
|
||||
),
|
||||
)
|
||||
]
|
||||
)
|
||||
res = {"cmd": stage.cmd}
|
||||
deps = [
|
||||
{"path": dep.def_path, dep.checksum_type: dep.get_checksum()}
|
||||
for dep in stage.deps
|
||||
]
|
||||
outs = [
|
||||
{"path": out.def_path, out.checksum_type: out.get_checksum()}
|
||||
for out in stage.outs
|
||||
]
|
||||
if stage.deps:
|
||||
res["deps"] = deps
|
||||
if stage.outs:
|
||||
res["outs"] = outs
|
||||
|
||||
return {stage.name: res}
|
||||
|
||||
|
||||
def to_single_stage_file(stage: "Stage"):
|
||||
|
||||
+67
-42
@@ -54,11 +54,11 @@ def loads_from(cls, repo, path, wdir, data):
|
||||
|
||||
|
||||
def create_stage(cls, repo, path, **kwargs):
|
||||
from dvc.dvcfile import Dvcfile
|
||||
from dvc.dvcfile import check_dvc_filename
|
||||
|
||||
wdir = os.path.abspath(kwargs.get("wdir", None) or os.curdir)
|
||||
path = os.path.abspath(path)
|
||||
Dvcfile.check_dvc_filename(path)
|
||||
check_dvc_filename(path)
|
||||
cls._check_stage_path(repo, wdir, is_wdir=kwargs.get("wdir"))
|
||||
cls._check_stage_path(repo, os.path.dirname(path))
|
||||
|
||||
@@ -147,9 +147,22 @@ class Stage(params.StageParams):
|
||||
|
||||
def __repr__(self):
|
||||
return "Stage: '{path}'".format(
|
||||
path=self.path_in_repo if self.path else "No path"
|
||||
)
|
||||
|
||||
def __str__(self):
|
||||
return "stage: '{path}'".format(
|
||||
path=self.relpath if self.path else "No path"
|
||||
)
|
||||
|
||||
@property
|
||||
def addressing(self):
|
||||
"""
|
||||
Useful for alternative presentations where we don't need
|
||||
`Stage:` prefix.
|
||||
"""
|
||||
return self.relpath
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.path_in_repo)
|
||||
|
||||
@@ -173,9 +186,6 @@ class Stage(params.StageParams):
|
||||
"""Whether the DVC-file was created with `dvc add` or `dvc import`"""
|
||||
return self.cmd is None
|
||||
|
||||
def changed_md5(self):
|
||||
return self.md5 != self._compute_md5()
|
||||
|
||||
@property
|
||||
def is_callback(self):
|
||||
"""
|
||||
@@ -202,9 +212,9 @@ class Stage(params.StageParams):
|
||||
|
||||
if self.is_callback:
|
||||
logger.warning(
|
||||
"DVC-file '{fname}' is a \"callback\" stage "
|
||||
'{stage} is a "callback" stage '
|
||||
"(has a command and no dependencies) and thus always "
|
||||
"considered as changed.".format(fname=self.relpath)
|
||||
"considered as changed.".format(stage=self)
|
||||
)
|
||||
return True
|
||||
|
||||
@@ -215,9 +225,9 @@ class Stage(params.StageParams):
|
||||
status = dep.status()
|
||||
if status:
|
||||
logger.debug(
|
||||
"Dependency '{dep}' of '{stage}' changed because it is "
|
||||
"Dependency '{dep}' of {stage} changed because it is "
|
||||
"'{status}'.".format(
|
||||
dep=dep, stage=self.relpath, status=status[str(dep)]
|
||||
dep=dep, stage=self, status=status[str(dep)]
|
||||
)
|
||||
)
|
||||
return True
|
||||
@@ -229,20 +239,20 @@ class Stage(params.StageParams):
|
||||
status = out.status()
|
||||
if status:
|
||||
logger.debug(
|
||||
"Output '{out}' of '{stage}' changed because it is "
|
||||
"Output '{out}' of {stage} changed because it is "
|
||||
"'{status}'".format(
|
||||
out=out, stage=self.relpath, status=status[str(out)]
|
||||
out=out, stage=self, status=status[str(out)]
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _changed_md5(self):
|
||||
if self.changed_md5():
|
||||
def stage_changed(self, warn=False):
|
||||
changed = self.md5 != self._compute_md5()
|
||||
if changed and warn:
|
||||
logger.warning("DVC-file '{}' changed.".format(self.relpath))
|
||||
return True
|
||||
return False
|
||||
return changed
|
||||
|
||||
@rwlocked(read=["deps", "outs"])
|
||||
def changed(self):
|
||||
@@ -256,7 +266,9 @@ class Stage(params.StageParams):
|
||||
def _changed(self):
|
||||
# Short-circuit order: stage md5 is fast, deps are expected to change
|
||||
return (
|
||||
self._changed_md5() or self._changed_deps() or self._changed_outs()
|
||||
self.stage_changed(warn=True)
|
||||
or self._changed_deps()
|
||||
or self._changed_outs()
|
||||
)
|
||||
|
||||
@rwlocked(write=["outs"])
|
||||
@@ -267,8 +279,8 @@ class Stage(params.StageParams):
|
||||
out.unprotect()
|
||||
else:
|
||||
logger.debug(
|
||||
"Removing output '{out}' of '{stage}'.".format(
|
||||
out=out, stage=self.relpath
|
||||
"Removing output '{out}' of {stage}.".format(
|
||||
out=out, stage=self
|
||||
)
|
||||
)
|
||||
out.remove(ignore_remove=ignore_remove)
|
||||
@@ -283,7 +295,7 @@ class Stage(params.StageParams):
|
||||
self.remove_outs(ignore_remove=True, force=force)
|
||||
else:
|
||||
self.unprotect_outs()
|
||||
os.unlink(self.path)
|
||||
self.dvcfile.remove()
|
||||
|
||||
@rwlocked(read=["deps"], write=["outs"])
|
||||
def reproduce(self, interactive=False, **kwargs):
|
||||
@@ -292,8 +304,8 @@ class Stage(params.StageParams):
|
||||
return None
|
||||
|
||||
msg = (
|
||||
"Going to reproduce '{stage}'. "
|
||||
"Are you sure you want to continue?".format(stage=self.relpath)
|
||||
"Going to reproduce {stage}. "
|
||||
"Are you sure you want to continue?".format(stage=self)
|
||||
)
|
||||
|
||||
if interactive and not prompt.confirm(msg):
|
||||
@@ -301,7 +313,7 @@ class Stage(params.StageParams):
|
||||
|
||||
self.run(**kwargs)
|
||||
|
||||
logger.debug("'{stage}' was reproduced".format(stage=self.relpath))
|
||||
logger.debug("{stage} was reproduced".format(stage=self))
|
||||
|
||||
return self
|
||||
|
||||
@@ -470,7 +482,7 @@ class Stage(params.StageParams):
|
||||
OutputBase.PARAM_PERSIST,
|
||||
],
|
||||
)
|
||||
logger.debug("Computed stage '{}' md5: '{}'".format(self.relpath, m))
|
||||
logger.debug("Computed {} md5: '{}'".format(self, m))
|
||||
return m
|
||||
|
||||
def save(self):
|
||||
@@ -494,19 +506,19 @@ class Stage(params.StageParams):
|
||||
changed_deps = self._changed_entries(self.deps)
|
||||
changed_outs = self._changed_entries(self.outs)
|
||||
|
||||
if changed_deps or changed_outs or self.changed_md5():
|
||||
if changed_deps or changed_outs or self.stage_changed():
|
||||
msg = (
|
||||
"dependencies {}".format(changed_deps) if changed_deps else ""
|
||||
)
|
||||
msg += " and " if (changed_deps and changed_outs) else ""
|
||||
msg += "outputs {}".format(changed_outs) if changed_outs else ""
|
||||
msg += "md5" if not (changed_deps or changed_outs) else ""
|
||||
msg += " of '{}' changed. ".format(self.relpath)
|
||||
msg += " of {} changed. ".format(self)
|
||||
msg += "Are you sure you want to commit it?"
|
||||
if not force and not prompt.confirm(msg):
|
||||
raise StageCommitError(
|
||||
"unable to commit changed '{}'. Use `-f|--force` to "
|
||||
"force.".format(self.relpath)
|
||||
"unable to commit changed {}. Use `-f|--force` to "
|
||||
"force.".format(self)
|
||||
)
|
||||
self.save()
|
||||
|
||||
@@ -614,9 +626,7 @@ class Stage(params.StageParams):
|
||||
|
||||
if self.locked:
|
||||
logger.info(
|
||||
"Verifying outputs in locked stage '{stage}'".format(
|
||||
stage=self.relpath
|
||||
)
|
||||
"Verifying outputs in locked {stage}".format(stage=self)
|
||||
)
|
||||
if not dry:
|
||||
self.check_missing_outputs()
|
||||
@@ -630,14 +640,14 @@ class Stage(params.StageParams):
|
||||
if not dry:
|
||||
if (
|
||||
not force
|
||||
and not self._changed_md5()
|
||||
and not self.stage_changed(warn=True)
|
||||
and self._already_cached()
|
||||
):
|
||||
self.outs[0].checkout()
|
||||
else:
|
||||
self.deps[0].download(self.outs[0])
|
||||
elif self.is_data_source:
|
||||
msg = "Verifying data sources in '{}'".format(self.relpath)
|
||||
msg = "Verifying data sources in {}".format(self)
|
||||
logger.info(msg)
|
||||
if not dry:
|
||||
self.check_missing_outputs()
|
||||
@@ -708,6 +718,9 @@ class Stage(params.StageParams):
|
||||
|
||||
return ret
|
||||
|
||||
def stage_status(self):
|
||||
return ["changed checksum"] if self.stage_changed() else []
|
||||
|
||||
@rwlocked(read=["deps", "outs"])
|
||||
def status(self, check_updates=False):
|
||||
ret = []
|
||||
@@ -723,14 +736,12 @@ class Stage(params.StageParams):
|
||||
if outs_status:
|
||||
ret.append({"changed outs": outs_status})
|
||||
|
||||
if self.changed_md5():
|
||||
ret.append("changed checksum")
|
||||
|
||||
ret.extend(self.stage_status())
|
||||
if self.is_callback or self.always_changed:
|
||||
ret.append("always changed")
|
||||
|
||||
if ret:
|
||||
return {self.relpath: ret}
|
||||
return {self.addressing: ret}
|
||||
|
||||
return {}
|
||||
|
||||
@@ -757,10 +768,12 @@ class Stage(params.StageParams):
|
||||
|
||||
|
||||
class PipelineStage(Stage):
|
||||
def __init__(self, name=None, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
def __init__(self, *args, name=None, meta=None, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.name = name
|
||||
self.cmd_changed = False
|
||||
# This is how the Stage will discover any discrepancies
|
||||
self.meta = meta or {}
|
||||
|
||||
def __eq__(self, other):
|
||||
return super().__eq__(other) and self.name == other.name
|
||||
@@ -773,10 +786,14 @@ class PipelineStage(Stage):
|
||||
path=self.relpath if self.path else "No path", name=self.name
|
||||
)
|
||||
|
||||
def _changed(self):
|
||||
if self.cmd_changed:
|
||||
logger.warning("'cmd' of {} has changed.".format(self))
|
||||
return self.cmd_changed or self._changed_deps() or self._changed_outs()
|
||||
def __str__(self):
|
||||
return "stage: '{path}:{name}'".format(
|
||||
path=self.relpath if self.path else "No path", name=self.name
|
||||
)
|
||||
|
||||
@property
|
||||
def addressing(self):
|
||||
return super().addressing + ":" + self.name
|
||||
|
||||
def reload(self):
|
||||
return self.dvcfile.stages[self.name]
|
||||
@@ -784,3 +801,11 @@ class PipelineStage(Stage):
|
||||
@property
|
||||
def is_cached(self):
|
||||
return self.name in self.dvcfile.stages and super().is_cached
|
||||
|
||||
def stage_status(self):
|
||||
return ["changed command"] if self.cmd_changed else []
|
||||
|
||||
def stage_changed(self, warn=False):
|
||||
if self.cmd_changed and warn:
|
||||
logger.warning("'cmd' of {} has changed.".format(self))
|
||||
return self.cmd_changed
|
||||
|
||||
+41
-5
@@ -3,7 +3,7 @@ from dvc.exceptions import DvcException
|
||||
|
||||
class StageCmdFailedError(DvcException):
|
||||
def __init__(self, stage):
|
||||
msg = "stage '{}' cmd '{}' failed".format(stage.relpath, stage.cmd)
|
||||
msg = "{} cmd '{}' failed".format(stage, stage.cmd)
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
@@ -15,12 +15,12 @@ class StageFileFormatError(DvcException):
|
||||
|
||||
class StageFileDoesNotExistError(DvcException):
|
||||
def __init__(self, fname):
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX, Dvcfile
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX, is_dvc_file
|
||||
|
||||
msg = "'{}' does not exist.".format(fname)
|
||||
|
||||
sname = fname + DVC_FILE_SUFFIX
|
||||
if Dvcfile.is_stage_file(sname):
|
||||
if is_dvc_file(sname):
|
||||
msg += " Do you mean '{}'?".format(sname)
|
||||
|
||||
super().__init__(msg)
|
||||
@@ -34,12 +34,12 @@ class StageFileAlreadyExistsError(DvcException):
|
||||
|
||||
class StageFileIsNotDvcFileError(DvcException):
|
||||
def __init__(self, fname):
|
||||
from dvc.dvcfile import Dvcfile, DVC_FILE_SUFFIX
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX, is_dvc_file
|
||||
|
||||
msg = "'{}' is not a DVC-file".format(fname)
|
||||
|
||||
sname = fname + DVC_FILE_SUFFIX
|
||||
if Dvcfile.is_stage_file(sname):
|
||||
if is_dvc_file(sname):
|
||||
msg += " Do you mean '{}'?".format(sname)
|
||||
|
||||
super().__init__(msg)
|
||||
@@ -92,3 +92,39 @@ class MissingDataSource(DvcException):
|
||||
|
||||
msg = "missing data '{}': {}".format(source, ", ".join(missing_files))
|
||||
super().__init__(msg)
|
||||
|
||||
|
||||
class StageNotFound(KeyError, DvcException):
|
||||
def __init__(self, file, name):
|
||||
super().__init__(
|
||||
"Stage with '{}' name not found inside '{}' file".format(
|
||||
name, file.relpath
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class StageNameUnspecified(DvcException):
|
||||
def __init__(self, file):
|
||||
super().__init__(
|
||||
"Stage name not provided."
|
||||
"Please specify the name as: `{0}:stage_name`".format(
|
||||
file.relpath
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class DuplicateStageName(DvcException):
|
||||
def __init__(self, name, file):
|
||||
super().__init__(
|
||||
"Stage with name '{name}' already exists in '{relpath}'.".format(
|
||||
name=name, relpath=file.relpath
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class InvalidStageName(DvcException):
|
||||
def __init__(self,):
|
||||
super().__init__(
|
||||
"Stage name cannot contain invalid characters: "
|
||||
"'\\', '/', '@' and ':'."
|
||||
)
|
||||
|
||||
@@ -6,21 +6,11 @@ from copy import deepcopy
|
||||
from itertools import chain
|
||||
|
||||
from dvc import dependency, output
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.utils import relpath
|
||||
from .exceptions import StageNameUnspecified, StageNotFound
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StageNotFound(KeyError, DvcException):
|
||||
def __init__(self, file, name):
|
||||
super().__init__(
|
||||
"Stage with '{}' name not found inside '{}' file".format(
|
||||
name, relpath(file)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def resolve_paths(path, wdir=None):
|
||||
path = os.path.abspath(path)
|
||||
wdir = wdir or os.curdir
|
||||
@@ -34,21 +24,39 @@ class StageLoader(collections.abc.Mapping):
|
||||
self.stages_data = stages_data or {}
|
||||
self.lockfile_data = lockfile_data or {}
|
||||
|
||||
def filter(self, item=None):
|
||||
if not item:
|
||||
return self
|
||||
|
||||
if item not in self:
|
||||
raise StageNotFound(self.dvcfile, item)
|
||||
|
||||
data = {item: self.stages_data[item]} if item in self else {}
|
||||
return self.__class__(self.dvcfile, data, self.lockfile_data)
|
||||
|
||||
@staticmethod
|
||||
def _fill_lock_checksums(stage, lock_data):
|
||||
from .stage import Stage
|
||||
from .params import StageParams
|
||||
|
||||
outs = stage.outs if not stage.cmd_changed else []
|
||||
items = chain(
|
||||
((Stage.PARAM_DEPS, dep) for dep in stage.deps),
|
||||
((Stage.PARAM_OUTS, out) for out in outs),
|
||||
((StageParams.PARAM_DEPS, dep) for dep in stage.deps),
|
||||
((StageParams.PARAM_OUTS, out) for out in stage.outs),
|
||||
)
|
||||
|
||||
checksums = {
|
||||
key: {item["path"]: item for item in lock_data.get(key, {})}
|
||||
for key in [StageParams.PARAM_DEPS, StageParams.PARAM_OUTS]
|
||||
}
|
||||
for key, item in items:
|
||||
item.checksum = lock_data.get(key, {}).get(item.def_path)
|
||||
item.checksum = (
|
||||
checksums.get(key, {})
|
||||
.get(item.def_path, {})
|
||||
.get(item.checksum_type)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def load_stage(cls, dvcfile, name, stage_data, lock_data):
|
||||
from .stage import PipelineStage, Stage, loads_from
|
||||
from . import PipelineStage, Stage, loads_from
|
||||
|
||||
path, wdir = resolve_paths(
|
||||
dvcfile.path, stage_data.get(Stage.PARAM_WDIR)
|
||||
@@ -66,13 +74,17 @@ class StageLoader(collections.abc.Mapping):
|
||||
return stage
|
||||
|
||||
def __getitem__(self, name):
|
||||
if not name:
|
||||
raise StageNameUnspecified(self.dvcfile)
|
||||
|
||||
if name not in self:
|
||||
raise StageNotFound(self.dvcfile.path, name)
|
||||
raise StageNotFound(self.dvcfile, name)
|
||||
|
||||
if not self.lockfile_data.get(name):
|
||||
logger.warning(
|
||||
"No lock entry found for '%s:%s'", self.dvcfile.relpath, name
|
||||
)
|
||||
|
||||
return self.load_stage(
|
||||
self.dvcfile,
|
||||
name,
|
||||
@@ -91,10 +103,14 @@ class StageLoader(collections.abc.Mapping):
|
||||
|
||||
|
||||
class SingleStageLoader(collections.abc.Mapping):
|
||||
def __init__(self, dvcfile, stage_data, stage_text=None):
|
||||
def __init__(self, dvcfile, stage_data, stage_text=None, tag=None):
|
||||
self.dvcfile = dvcfile
|
||||
self.stage_data = stage_data or {}
|
||||
self.stage_text = stage_text
|
||||
self.tag = tag
|
||||
|
||||
def filter(self, item=None):
|
||||
return self
|
||||
|
||||
def __getitem__(self, item):
|
||||
if item:
|
||||
@@ -106,16 +122,16 @@ class SingleStageLoader(collections.abc.Mapping):
|
||||
# during `load`, we remove attributes from stage data, so as to
|
||||
# not duplicate, therefore, for MappingView, we need to deepcopy.
|
||||
return self.load_stage(
|
||||
self.dvcfile, deepcopy(self.stage_data), self.stage_text
|
||||
self.dvcfile, deepcopy(self.stage_data), self.stage_text, self.tag
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def load_stage(cls, dvcfile, d, stage_text):
|
||||
def load_stage(cls, dvcfile, d, stage_text, tag=None):
|
||||
from dvc.stage import Stage, loads_from
|
||||
|
||||
path, wdir = resolve_paths(dvcfile.path, d.get(Stage.PARAM_WDIR))
|
||||
stage = loads_from(Stage, dvcfile.repo, path, wdir, d)
|
||||
stage._stage_text, stage.tag = stage_text, dvcfile.tag
|
||||
stage._stage_text, stage.tag = stage_text, tag
|
||||
stage.deps = dependency.loadd_from(
|
||||
stage, d.get(Stage.PARAM_DEPS) or []
|
||||
)
|
||||
+24
-7
@@ -23,7 +23,9 @@ logger = logging.getLogger(__name__)
|
||||
LOCAL_CHUNK_SIZE = 2 ** 20 # 1 MB
|
||||
LARGE_FILE_SIZE = 2 ** 30 # 1 GB
|
||||
LARGE_DIR_SIZE = 100
|
||||
TARGET_REGEX = re.compile(r"^(?P<path>.*):(?P<name>[^\\/@:]*)$")
|
||||
TARGET_REGEX = re.compile(
|
||||
r"(?P<path>.*?)(:(?P<name>[^\\/@:]*))??(@(?P<tag>[^\\/@:]*))??$"
|
||||
)
|
||||
|
||||
|
||||
def dos2unix(data):
|
||||
@@ -377,14 +379,29 @@ def format_link(link):
|
||||
)
|
||||
|
||||
|
||||
def parse_target(target, default="Dvcfile"):
|
||||
def parse_target(target, default=None):
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
if not target:
|
||||
return None, None
|
||||
return None, None, None
|
||||
|
||||
match = TARGET_REGEX.match(target)
|
||||
if not match:
|
||||
return target, None
|
||||
path, name = match.group("path"), match.group("name")
|
||||
return target, None, None
|
||||
|
||||
path, name, tag = (
|
||||
match.group("path"),
|
||||
match.group("name"),
|
||||
match.group("tag"),
|
||||
)
|
||||
if not path:
|
||||
logger.warning("Assuming file to be '%s'", default)
|
||||
return path or default, name
|
||||
path = default or PIPELINE_FILE
|
||||
logger.warning("Assuming file to be '%s'", path)
|
||||
|
||||
if os.path.basename(path) == PIPELINE_LOCK:
|
||||
raise DvcException(
|
||||
"Did you mean: `{}`?".format(target.replace(".lock", ".yaml", 1))
|
||||
)
|
||||
|
||||
return path, name, tag
|
||||
|
||||
@@ -60,6 +60,7 @@ __all__ = [
|
||||
"tmp_dir",
|
||||
"scm",
|
||||
"dvc",
|
||||
"local_remote",
|
||||
"run_copy",
|
||||
"erepo_dir",
|
||||
"git_dir",
|
||||
@@ -256,6 +257,17 @@ def _git_init(path):
|
||||
git.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def local_remote(request, tmp_dir, dvc, make_tmp_dir):
|
||||
path = make_tmp_dir("local-remote")
|
||||
with dvc.config.edit() as conf:
|
||||
conf["remote"]["upstream"] = {"url": fspath(path)}
|
||||
conf["core"]["remote"] = "upstream"
|
||||
if "scm" in request.fixturenames:
|
||||
tmp_dir.scm_add([dvc.config.files["repo"]], commit="add remote")
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_copy(tmp_dir, dvc):
|
||||
tmp_dir.gen(
|
||||
|
||||
+25
-4
@@ -12,11 +12,15 @@ from mock import patch, call
|
||||
import dvc as dvc_module
|
||||
from dvc.cache import Cache
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX
|
||||
from dvc.exceptions import DvcException, OverlappingOutputPathsError
|
||||
from dvc.exceptions import RecursiveAddingWhileUsingFilename
|
||||
from dvc.exceptions import StageFileCorruptedError
|
||||
from dvc.exceptions import (
|
||||
DvcException,
|
||||
OverlappingOutputPathsError,
|
||||
OutputDuplicationError,
|
||||
RecursiveAddingWhileUsingFilename,
|
||||
StageFileCorruptedError,
|
||||
)
|
||||
from dvc.main import main
|
||||
from dvc.output.base import OutputAlreadyTrackedError
|
||||
from dvc.output.base import OutputAlreadyTrackedError, OutputIsStageFileError
|
||||
from dvc.remote import LocalRemote
|
||||
from dvc.repo import Repo as DvcRepo
|
||||
from dvc.stage import Stage
|
||||
@@ -703,3 +707,20 @@ def test_add_optimization_for_hardlink_on_empty_files(tmp_dir, dvc, mocker):
|
||||
for stage in stages:
|
||||
assert os.path.exists(stage.path)
|
||||
assert os.path.exists(stage.outs[0].cache_path)
|
||||
|
||||
|
||||
def test_output_duplication_for_pipeline_tracked(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
with pytest.raises(OutputDuplicationError):
|
||||
dvc.add("bar")
|
||||
|
||||
|
||||
def test_add_pipeline_file(tmp_dir, dvc, run_copy):
|
||||
from dvc.dvcfile import PIPELINE_FILE
|
||||
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
|
||||
with pytest.raises(OutputIsStageFileError):
|
||||
dvc.add(PIPELINE_FILE)
|
||||
|
||||
@@ -758,3 +758,29 @@ def test_checkout_for_external_outputs(tmp_dir, dvc):
|
||||
)
|
||||
stats = dvc.checkout(force=True)
|
||||
assert stats == {**empty_checkout, "modified": [str(file_path)]}
|
||||
|
||||
|
||||
def test_checkouts_for_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
|
||||
from dvc.dvcfile import PIPELINE_FILE
|
||||
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage1 = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
tmp_dir.gen("lorem", "lorem")
|
||||
stage2 = run_copy("lorem", "ipsum", name="copy-lorem-ipsum")
|
||||
|
||||
for out in ["bar", "ipsum"]:
|
||||
(tmp_dir / out).unlink()
|
||||
assert dvc.checkout(["bar"])["added"] == ["bar"]
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
assert set(dvc.checkout([PIPELINE_FILE])["added"]) == {"bar", "ipsum"}
|
||||
|
||||
for out in ["bar", "ipsum"]:
|
||||
(tmp_dir / out).unlink()
|
||||
assert set(dvc.checkout([stage1.addressing])["added"]) == {"bar"}
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
assert set(dvc.checkout([stage2.addressing])["added"]) == {"ipsum"}
|
||||
|
||||
(tmp_dir / "ipsum").unlink()
|
||||
assert set(dvc.checkout()["added"]) == {"bar", "ipsum"}
|
||||
|
||||
@@ -38,13 +38,14 @@ def test_commit_force(tmp_dir, dvc):
|
||||
assert dvc.status([stage.path]) == {}
|
||||
|
||||
|
||||
def test_commit_with_deps(tmp_dir, dvc, run_copy):
|
||||
@pytest.mark.parametrize("run_kw", [{}, {"name": "copy"}])
|
||||
def test_commit_with_deps(tmp_dir, dvc, run_copy, run_kw):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
(foo_stage,) = dvc.add("foo", no_commit=True)
|
||||
assert foo_stage is not None
|
||||
assert len(foo_stage.outs) == 1
|
||||
|
||||
stage = run_copy("foo", "file", no_commit=True)
|
||||
stage = run_copy("foo", "file", no_commit=True, **run_kw)
|
||||
assert stage is not None
|
||||
assert len(stage.outs) == 1
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ from unittest import SkipTest
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.compat import fspath
|
||||
from dvc.compat import fspath, fspath_py35
|
||||
from dvc.cache import NamedCache
|
||||
from dvc.data_cloud import DataCloud
|
||||
from dvc.main import main
|
||||
@@ -21,6 +21,7 @@ from dvc.remote import OSSRemote
|
||||
from dvc.remote import S3Remote
|
||||
from dvc.remote import SSHRemote
|
||||
from dvc.remote.base import STATUS_DELETED, STATUS_NEW, STATUS_OK
|
||||
from dvc.stage.exceptions import StageNotFound
|
||||
from dvc.utils import file_md5
|
||||
from dvc.utils.fs import remove
|
||||
from dvc.utils.stage import dump_stage_file, load_stage_file
|
||||
@@ -531,7 +532,7 @@ class TestWarnOnOutdatedStage(TestDvc):
|
||||
self._caplog.clear()
|
||||
self.main(["status", "-c"])
|
||||
expected_warning = (
|
||||
"Output 'bar'(Stage: 'bar.dvc') is missing version info. "
|
||||
"Output 'bar'(stage: 'bar.dvc') is missing version info. "
|
||||
"Cache for it will not be collected. "
|
||||
"Use `dvc repro` to get your pipeline up to date."
|
||||
)
|
||||
@@ -752,10 +753,7 @@ def test_pull_external_dvc_imports(tmp_dir, dvc, scm, erepo_dir):
|
||||
|
||||
assert dvc.pull()["downloaded"] == 0
|
||||
|
||||
for item in ["foo", "new_dir", dvc.cache.local.cache_dir]:
|
||||
remove(item)
|
||||
os.makedirs(dvc.cache.local.cache_dir, exist_ok=True)
|
||||
clean_repos()
|
||||
clean(["foo", "new_dir"], dvc)
|
||||
|
||||
assert dvc.pull(force=True)["downloaded"] == 2
|
||||
|
||||
@@ -764,3 +762,88 @@ def test_pull_external_dvc_imports(tmp_dir, dvc, scm, erepo_dir):
|
||||
|
||||
assert (tmp_dir / "new_dir").exists()
|
||||
assert (tmp_dir / "new_dir" / "bar").read_text() == "bar"
|
||||
|
||||
|
||||
def clean(outs, dvc=None):
|
||||
if dvc:
|
||||
outs = outs + [dvc.cache.local.cache_dir]
|
||||
for path in outs:
|
||||
print(path)
|
||||
remove(path)
|
||||
if dvc:
|
||||
os.makedirs(dvc.cache.local.cache_dir, exist_ok=True)
|
||||
clean_repos()
|
||||
|
||||
|
||||
def recurse_list_dir(d):
|
||||
return [
|
||||
os.path.join(d, f) for _, _, filenames in os.walk(d) for f in filenames
|
||||
]
|
||||
|
||||
|
||||
def test_dvc_pull_pipeline_stages(tmp_dir, dvc, local_remote, run_copy):
|
||||
(stage0,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
stage1 = run_copy("foo", "bar")
|
||||
stage2 = run_copy("bar", "foobar", name="copy-bar-foobar")
|
||||
outs = ["foo", "bar", "foobar"]
|
||||
|
||||
dvc.push()
|
||||
clean(outs, dvc)
|
||||
dvc.pull()
|
||||
assert all((tmp_dir / file).exists() for file in outs)
|
||||
|
||||
for out, stage in zip(outs, [stage0, stage1, stage2]):
|
||||
for target in [stage.addressing, out]:
|
||||
clean(outs, dvc)
|
||||
stats = dvc.pull([target])
|
||||
assert stats["downloaded"] == 1
|
||||
assert stats["added"] == [out]
|
||||
assert os.path.exists(out)
|
||||
assert not any(os.path.exists(out) for out in set(outs) - {out})
|
||||
|
||||
clean(outs, dvc)
|
||||
stats = dvc.pull([stage2.addressing], with_deps=True)
|
||||
assert len(stats["added"]) == 3
|
||||
assert set(stats["added"]) == set(outs)
|
||||
|
||||
clean(outs, dvc)
|
||||
stats = dvc.pull([os.curdir], recursive=True)
|
||||
assert set(stats["added"]) == set(outs)
|
||||
|
||||
|
||||
def test_pipeline_file_target_ops(tmp_dir, dvc, local_remote, run_copy):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_copy("foo", "bar")
|
||||
|
||||
tmp_dir.dvc_gen("lorem", "lorem")
|
||||
run_copy("lorem", "lorem2", name="copy-lorem-lorem2")
|
||||
|
||||
tmp_dir.dvc_gen("ipsum", "ipsum")
|
||||
run_copy("ipsum", "baz", name="copy-ipsum-baz")
|
||||
|
||||
outs = ["foo", "bar", "lorem", "ipsum", "baz", "lorem2"]
|
||||
|
||||
dvc.push()
|
||||
# each one's a copy of other, hence 3
|
||||
assert len(recurse_list_dir(fspath_py35(local_remote))) == 3
|
||||
|
||||
clean(outs, dvc)
|
||||
assert set(dvc.pull(["pipelines.yaml"])["added"]) == {"lorem2", "baz"}
|
||||
|
||||
clean(outs, dvc)
|
||||
assert set(dvc.pull()["added"]) == set(outs)
|
||||
|
||||
# clean everything in remote and push
|
||||
clean(local_remote.iterdir())
|
||||
dvc.push(["pipelines.yaml:copy-ipsum-baz"])
|
||||
assert len(recurse_list_dir(fspath_py35(local_remote))) == 1
|
||||
|
||||
clean(local_remote.iterdir())
|
||||
dvc.push(["pipelines.yaml"])
|
||||
assert len(recurse_list_dir(fspath_py35(local_remote))) == 2
|
||||
|
||||
with pytest.raises(StageNotFound):
|
||||
dvc.push(["pipelines.yaml:StageThatDoesNotExist"])
|
||||
|
||||
with pytest.raises(StageNotFound):
|
||||
dvc.pull(["pipelines.yaml:StageThatDoesNotExist"])
|
||||
|
||||
+50
-31
@@ -1,9 +1,11 @@
|
||||
import pytest
|
||||
|
||||
from dvc.dvcfile import Dvcfile
|
||||
from dvc.stage import Stage
|
||||
from dvc.loader import StageNotFound
|
||||
from dvc.stage.exceptions import StageFileDoesNotExistError
|
||||
from dvc.dvcfile import Dvcfile, PIPELINE_FILE
|
||||
from dvc.stage.loader import StageNotFound
|
||||
from dvc.stage.exceptions import (
|
||||
StageFileDoesNotExistError,
|
||||
StageNameUnspecified,
|
||||
)
|
||||
|
||||
|
||||
def test_run_load_one_for_multistage(tmp_dir, dvc):
|
||||
@@ -15,7 +17,7 @@ def test_run_load_one_for_multistage(tmp_dir, dvc):
|
||||
outs_persist_no_cache=["foo2"],
|
||||
always_changed=True,
|
||||
)
|
||||
stage2 = Dvcfile(dvc, "Dvcfile").stages["copy-foo-foo2"]
|
||||
stage2 = Dvcfile(dvc, PIPELINE_FILE).stages["copy-foo-foo2"]
|
||||
assert stage1 == stage2
|
||||
foo_out = stage2.outs[0]
|
||||
assert stage2.cmd == "cp foo foo2"
|
||||
@@ -29,7 +31,7 @@ def test_run_load_one_for_multistage(tmp_dir, dvc):
|
||||
|
||||
def test_run_load_one_for_multistage_non_existing(tmp_dir, dvc):
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
assert Dvcfile(dvc, "Dvcfile").stages.get("copy-foo-foo2")
|
||||
assert Dvcfile(dvc, PIPELINE_FILE).stages.get("copy-foo-foo2")
|
||||
|
||||
|
||||
def test_run_load_one_for_multistage_non_existing_stage_name(tmp_dir, dvc):
|
||||
@@ -63,7 +65,7 @@ def test_has_stage_with_name(tmp_dir, dvc):
|
||||
metrics=["foo2"],
|
||||
always_changed=True,
|
||||
)
|
||||
dvcfile = Dvcfile(dvc, "Dvcfile")
|
||||
dvcfile = Dvcfile(dvc, PIPELINE_FILE)
|
||||
assert "copy-foo-foo2" in dvcfile.stages
|
||||
assert "copy" not in dvcfile.stages
|
||||
|
||||
@@ -77,7 +79,7 @@ def test_load_all_multistage(tmp_dir, dvc):
|
||||
metrics=["foo2"],
|
||||
always_changed=True,
|
||||
)
|
||||
stages = Dvcfile(dvc, "Dvcfile").stages.values()
|
||||
stages = Dvcfile(dvc, PIPELINE_FILE).stages.values()
|
||||
assert len(stages) == 1
|
||||
assert list(stages) == [stage1]
|
||||
|
||||
@@ -89,7 +91,7 @@ def test_load_all_multistage(tmp_dir, dvc):
|
||||
metrics=["bar2"],
|
||||
always_changed=True,
|
||||
)
|
||||
assert set(Dvcfile(dvc, "Dvcfile").stages.values()) == {stage2, stage1}
|
||||
assert set(Dvcfile(dvc, PIPELINE_FILE).stages.values()) == {stage2, stage1}
|
||||
|
||||
|
||||
def test_load_all_singlestage(tmp_dir, dvc):
|
||||
@@ -110,8 +112,8 @@ def test_load_singlestage(tmp_dir, dvc):
|
||||
assert Dvcfile(dvc, "foo2.dvc").stage == stage1
|
||||
|
||||
|
||||
def test_load_multistage(tmp_dir, dvc):
|
||||
from dvc.dvcfile import MultiStageFileLoadError
|
||||
def test_try_get_single_stage_from_pipeline_file(tmp_dir, dvc):
|
||||
from dvc.dvcfile import DvcException
|
||||
|
||||
tmp_dir.gen("foo", "foo")
|
||||
dvc.run(
|
||||
@@ -121,24 +123,8 @@ def test_load_multistage(tmp_dir, dvc):
|
||||
metrics=["foo2"],
|
||||
always_changed=True,
|
||||
)
|
||||
with pytest.raises(MultiStageFileLoadError):
|
||||
Dvcfile(dvc, "Dvcfile").stage
|
||||
|
||||
|
||||
def test_is_multistage(tmp_dir, dvc):
|
||||
tmp_dir.gen({"foo": "foo", "bar": "bar"})
|
||||
stage1 = dvc.run(
|
||||
cmd="cp foo foo2",
|
||||
deps=["foo"],
|
||||
name="copy-foo-foo2",
|
||||
metrics=["foo2"],
|
||||
always_changed=True,
|
||||
)
|
||||
assert Dvcfile(dvc, stage1.path).is_multi_stage()
|
||||
stage2 = dvc.run(
|
||||
cmd="cp bar bar2", deps=["bar"], metrics=["bar2"], always_changed=True,
|
||||
)
|
||||
assert not Dvcfile(dvc, stage2.path).is_multi_stage()
|
||||
with pytest.raises(DvcException):
|
||||
assert Dvcfile(dvc, PIPELINE_FILE).stage
|
||||
|
||||
|
||||
def test_stage_collection(tmp_dir, dvc):
|
||||
@@ -160,5 +146,38 @@ def test_stage_collection(tmp_dir, dvc):
|
||||
stage3 = dvc.run(
|
||||
cmd="cp bar bar2", deps=["bar"], metrics=["bar2"], always_changed=True,
|
||||
)
|
||||
assert {s for s in dvc.pipeline_stages} == {stage3, stage2, stage1}
|
||||
assert {s for s in dvc.stages} == {Stage(dvc, "foo2.dvc"), stage1, stage3}
|
||||
assert {s for s in dvc.stages} == {stage1, stage3, stage2}
|
||||
|
||||
|
||||
def test_stage_filter(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage1 = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
stage2 = run_copy("bar", "bax", name="copy-bar-bax")
|
||||
stage3 = run_copy("bax", "baz", name="copy-bax-baz")
|
||||
stages = Dvcfile(dvc, PIPELINE_FILE).stages
|
||||
assert set(stages.filter(None).values()) == {
|
||||
stage1,
|
||||
stage2,
|
||||
stage3,
|
||||
}
|
||||
assert set(stages.filter("copy-bar-bax").values()) == {stage2}
|
||||
assert stages.filter("copy-bar-bax").get("copy-bar-bax") == stage2
|
||||
with pytest.raises(StageNameUnspecified):
|
||||
stages.get(None)
|
||||
|
||||
with pytest.raises(StageNotFound):
|
||||
assert stages["unknown"]
|
||||
|
||||
with pytest.raises(StageNotFound):
|
||||
assert stages.filter("unknown")
|
||||
|
||||
|
||||
def test_stage_filter_in_singlestage_file(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage = run_copy("foo", "bar")
|
||||
dvcfile = Dvcfile(dvc, stage.path)
|
||||
assert set(dvcfile.stages.filter(None).values()) == {stage}
|
||||
assert dvcfile.stages.filter(None).get(None) == stage
|
||||
assert dvcfile.stages.filter("copy-bar-bax").get("copy-bar-bax") == stage
|
||||
assert dvcfile.stages.filter("copy-bar-bax").get(None) == stage
|
||||
assert set(dvcfile.stages.filter("unknown").values()) == {stage}
|
||||
|
||||
@@ -332,3 +332,20 @@ def test_gc_cloud_remove_order(tmp_dir, scm, dvc, tmp_path_factory, mocker):
|
||||
for args in mocked_remove.call_args_list[:4]:
|
||||
checksum = str(args[0][1])
|
||||
assert checksum.endswith(".dir") or checksum.endswith(".dir.unpacked")
|
||||
|
||||
|
||||
def test_gc_not_collect_pipeline_tracked_files(tmp_dir, dvc, run_copy):
|
||||
from dvc.dvcfile import PIPELINE_FILE, Dvcfile
|
||||
|
||||
tmp_dir.gen("foo", "foo")
|
||||
tmp_dir.gen("bar", "bar")
|
||||
|
||||
run_copy("foo", "foo2", name="copy")
|
||||
assert _count_files(dvc.cache.local.cache_dir) == 1
|
||||
dvc.gc(workspace=True, force=True)
|
||||
assert _count_files(dvc.cache.local.cache_dir) == 1
|
||||
|
||||
# remove pipeline file and lockfile and check
|
||||
Dvcfile(dvc, PIPELINE_FILE).remove(force=True)
|
||||
dvc.gc(workspace=True, force=True)
|
||||
assert _count_files(dvc.cache.local.cache_dir) == 0
|
||||
|
||||
@@ -220,3 +220,19 @@ def test_get_url_git_only_repo(tmp_dir, scm, caplog):
|
||||
with caplog.at_level(logging.ERROR):
|
||||
assert main(["get", fspath(tmp_dir), "foo", "--show-url"]) == 1
|
||||
assert "failed to show URL" in caplog.text
|
||||
|
||||
|
||||
def test_get_pipeline_tracked_outs(
|
||||
tmp_dir, dvc, scm, git_dir, local_remote, run_copy
|
||||
):
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
|
||||
tmp_dir.gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
dvc.scm.add([PIPELINE_FILE, PIPELINE_LOCK])
|
||||
dvc.scm.commit("add pipeline stage")
|
||||
dvc.push()
|
||||
|
||||
with git_dir.chdir():
|
||||
Repo.get("file:///{}".format(fspath(tmp_dir)), "bar", out="baz")
|
||||
assert (git_dir / "baz").read_text() == "foo"
|
||||
|
||||
@@ -322,3 +322,21 @@ def test_import_from_bare_git_repo(tmp_dir, make_tmp_dir, erepo_dir):
|
||||
dvc_repo = make_tmp_dir("dvc-repo", scm=True, dvc=True)
|
||||
with dvc_repo.chdir():
|
||||
dvc_repo.dvc.imp(fspath(tmp_dir), "foo")
|
||||
|
||||
|
||||
def test_import_pipeline_tracked_outs(
|
||||
tmp_dir, dvc, scm, erepo_dir, local_remote, run_copy
|
||||
):
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
|
||||
tmp_dir.gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
dvc.scm.add([PIPELINE_FILE, PIPELINE_LOCK])
|
||||
dvc.scm.commit("add pipeline stage")
|
||||
dvc.push()
|
||||
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc.imp(
|
||||
"file:///{}".format(fspath(tmp_dir)), "bar", out="baz"
|
||||
)
|
||||
assert (erepo_dir / "baz").read_text() == "foo"
|
||||
|
||||
@@ -429,3 +429,15 @@ def test_ls_not_existed_url():
|
||||
dirname = "__{}_{}".format("not_existed", time())
|
||||
with pytest.raises(CloneError):
|
||||
Repo.ls(dirname, recursive=True)
|
||||
|
||||
|
||||
def test_ls_shows_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
|
||||
tmp_dir.gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
dvc.scm.add([PIPELINE_FILE, PIPELINE_LOCK])
|
||||
dvc.scm.commit("add pipeline stage")
|
||||
|
||||
files = Repo.ls(os.curdir, outs_only=True)
|
||||
match_files(files, ((("bar",), True),))
|
||||
|
||||
@@ -278,20 +278,20 @@ def test_pipeline_list_show_multistage(tmp_dir, dvc, run_copy, caplog):
|
||||
with caplog.at_level(logging.INFO, "dvc"):
|
||||
command._show("foobar.dvc", False, False, False)
|
||||
output = caplog.text.splitlines()
|
||||
assert "Dvcfile:copy-foo-bar" in output[0]
|
||||
assert "pipelines.yaml:copy-foo-bar" in output[0]
|
||||
assert "foobar.dvc" in output[1]
|
||||
|
||||
caplog.clear()
|
||||
with caplog.at_level(logging.INFO, "dvc"):
|
||||
command._show("Dvcfile:copy-foo-bar", False, False, False)
|
||||
assert "Dvcfile:copy-foo-bar" in caplog.text
|
||||
command._show("pipelines.yaml:copy-foo-bar", False, False, False)
|
||||
assert "pipelines.yaml:copy-foo-bar" in caplog.text
|
||||
assert "foobar.dvc" not in caplog.text
|
||||
|
||||
command = CmdPipelineList([])
|
||||
caplog.clear()
|
||||
with caplog.at_level(logging.INFO, "dvc"):
|
||||
command.run()
|
||||
assert "Dvcfile:copy-foo-bar" in caplog.text
|
||||
assert "pipelines.yaml:copy-foo-bar" in caplog.text
|
||||
assert "foobar.dvc" in caplog.text
|
||||
assert "1 pipelines in total"
|
||||
|
||||
@@ -302,10 +302,10 @@ def test_pipeline_ascii_multistage(tmp_dir, dvc, run_copy):
|
||||
run_copy("bar", "foobar")
|
||||
command = CmdPipelineShow([])
|
||||
nodes, edges, is_tree = command._build_graph("foobar.dvc")
|
||||
assert set(nodes) == {"Dvcfile:copy-foo-bar", "foobar.dvc"}
|
||||
assert set(nodes) == {"pipelines.yaml:copy-foo-bar", "foobar.dvc"}
|
||||
assert set(edges) == {
|
||||
("foobar.dvc", "Dvcfile:copy-foo-bar"),
|
||||
("foobar.dvc", "pipelines.yaml:copy-foo-bar"),
|
||||
}
|
||||
|
||||
nodes, edges, is_tree = command._build_graph("Dvcfile:copy-foo-bar")
|
||||
assert set(nodes) == {"Dvcfile:copy-foo-bar"}
|
||||
nodes, edges, is_tree = command._build_graph("pipelines.yaml:copy-foo-bar")
|
||||
assert set(nodes) == {"pipelines.yaml:copy-foo-bar"}
|
||||
|
||||
@@ -17,13 +17,13 @@ class TestRemove(TestDvc):
|
||||
self.assertEqual(len(stages), 1)
|
||||
stage = stages[0]
|
||||
self.assertTrue(stage is not None)
|
||||
stage_removed = self.dvc.remove(stage.path, outs_only=True)
|
||||
(stage_removed,) = self.dvc.remove(stage.path, outs_only=True)
|
||||
|
||||
self.assertIsInstance(stage_removed, Stage)
|
||||
self.assertEqual(stage.path, stage_removed.path)
|
||||
self.assertFalse(os.path.isfile(self.FOO))
|
||||
|
||||
stage_removed = self.dvc.remove(stage.path)
|
||||
(stage_removed,) = self.dvc.remove(stage.path)
|
||||
self.assertIsInstance(stage_removed, Stage)
|
||||
self.assertEqual(stage.path, stage_removed.path)
|
||||
self.assertFalse(os.path.isfile(self.FOO))
|
||||
@@ -60,7 +60,7 @@ class TestRemoveDirectory(TestDvc):
|
||||
self.assertEqual(len(stages), 1)
|
||||
stage_add = stages[0]
|
||||
self.assertTrue(stage_add is not None)
|
||||
stage_removed = self.dvc.remove(stage_add.path)
|
||||
(stage_removed,) = self.dvc.remove(stage_add.path)
|
||||
self.assertEqual(stage_add.path, stage_removed.path)
|
||||
self.assertFalse(os.path.exists(self.DATA_DIR))
|
||||
self.assertFalse(os.path.exists(stage_removed.path))
|
||||
@@ -96,3 +96,34 @@ class TestRemovePurge(TestDvc):
|
||||
mock_prompt.assert_called()
|
||||
self.assertEqual(ret, 1)
|
||||
self.assertRaises(DvcException)
|
||||
|
||||
|
||||
def test_remove_pipeline_outs(tmp_dir, dvc, run_copy):
|
||||
from dvc.dvcfile import PIPELINE_FILE
|
||||
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
|
||||
assert main(["remove", stage.path]) == 0
|
||||
assert not (tmp_dir / "bar").exists()
|
||||
assert (tmp_dir / PIPELINE_FILE).exists()
|
||||
|
||||
stage = run_copy("foo", "foobar", name="copy-foo-foobar")
|
||||
|
||||
assert main(["remove", stage.addressing]) == 0
|
||||
assert not (tmp_dir / "foobar").exists()
|
||||
assert (tmp_dir / "foo").exists()
|
||||
|
||||
stage = run_copy("foo", "baz", name="copy-foo-baz")
|
||||
assert main(["remove", "--purge", "-f", stage.addressing]) == 0
|
||||
assert not (tmp_dir / "baz").exists()
|
||||
assert (tmp_dir / "foo").exists()
|
||||
assert (tmp_dir / PIPELINE_FILE).exists()
|
||||
|
||||
dvc.reproduce(PIPELINE_FILE)
|
||||
assert main(["remove", PIPELINE_FILE]) == 0
|
||||
for file in ["bar", "foobar", "baz"]:
|
||||
assert not (tmp_dir / file).exists()
|
||||
|
||||
assert (tmp_dir / "foo").exists()
|
||||
assert (tmp_dir / PIPELINE_FILE).exists()
|
||||
|
||||
+51
-8
@@ -1,31 +1,59 @@
|
||||
import os
|
||||
|
||||
from dvc.scm.git.tree import GitTree
|
||||
from dvc.cache import Cache
|
||||
from dvc.repo import Repo
|
||||
from dvc.system import System
|
||||
from dvc.compat import fspath
|
||||
|
||||
|
||||
def test_destroy(tmp_dir, dvc):
|
||||
def test_destroy(tmp_dir, dvc, run_copy):
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
|
||||
dvc.config["cache"]["type"] = ["symlink"]
|
||||
dvc.cache = Cache(dvc)
|
||||
|
||||
tmp_dir.dvc_gen("file", "text")
|
||||
tmp_dir.dvc_gen({"dir": {"file": "lorem", "subdir/file": "ipsum"}})
|
||||
|
||||
run_copy("file", "file2")
|
||||
run_copy("file2", "file3", name="copy-file2-file3")
|
||||
run_copy("file3", "file4", name="copy-file3-file4")
|
||||
|
||||
dvc.destroy()
|
||||
|
||||
# Remove all the files related to DVC
|
||||
for path in [".dvc", "file.dvc", "dir.dvc"]:
|
||||
for path in [
|
||||
".dvc",
|
||||
"file.dvc",
|
||||
"file2.dvc",
|
||||
"dir.dvc",
|
||||
PIPELINE_FILE,
|
||||
PIPELINE_LOCK,
|
||||
]:
|
||||
assert not (tmp_dir / path).exists()
|
||||
|
||||
# Leave the rest of the files
|
||||
for path in ["file", "dir/file", "dir/subdir/file"]:
|
||||
for path in [
|
||||
"file",
|
||||
"file2",
|
||||
"file3",
|
||||
"file4",
|
||||
"dir/file",
|
||||
"dir/subdir/file",
|
||||
]:
|
||||
assert (tmp_dir / path).is_file()
|
||||
|
||||
# Make sure that data was unprotected after `destroy`
|
||||
for path in ["file", "dir", "dir/file", "dir/subdir", "dir/subdir/file"]:
|
||||
for path in [
|
||||
"file",
|
||||
"file2",
|
||||
"file3",
|
||||
"file4",
|
||||
"dir",
|
||||
"dir/file",
|
||||
"dir/subdir",
|
||||
"dir/subdir/file",
|
||||
]:
|
||||
assert not System.is_symlink(fspath(tmp_dir / path))
|
||||
|
||||
|
||||
@@ -49,12 +77,27 @@ def test_collect(tmp_dir, scm, dvc, run_copy):
|
||||
scm.commit("Add buzz")
|
||||
|
||||
assert collect_outs("bar.dvc", with_deps=True) == {"foo", "bar"}
|
||||
|
||||
dvc.tree = GitTree(scm.repo, "new-branch")
|
||||
|
||||
assert collect_outs("buzz.dvc", with_deps=True) == {"foo", "bar", "buzz"}
|
||||
assert collect_outs("buzz.dvc", with_deps=False) == {"buzz"}
|
||||
|
||||
run_copy("foo", "foobar", name="copy-foo-foobar")
|
||||
assert collect_outs(":copy-foo-foobar") == {"foobar"}
|
||||
assert collect_outs(":copy-foo-foobar", with_deps=True) == {
|
||||
"foobar",
|
||||
"foo",
|
||||
}
|
||||
assert collect_outs("pipelines.yaml:copy-foo-foobar", recursive=True) == {
|
||||
"foobar"
|
||||
}
|
||||
|
||||
run_copy("foobar", "baz", name="copy-foobar-baz")
|
||||
assert collect_outs("pipelines.yaml") == {"foobar", "baz"}
|
||||
assert collect_outs("pipelines.yaml", with_deps=True) == {
|
||||
"foobar",
|
||||
"baz",
|
||||
"foo",
|
||||
}
|
||||
|
||||
|
||||
def test_stages(tmp_dir, dvc):
|
||||
def stages():
|
||||
|
||||
+18
-13
@@ -191,7 +191,7 @@ class TestReproWorkingDirectoryAsOutput(TestDvc):
|
||||
|
||||
# NOTE: os.walk() walks in a sorted order and we need dir2 subdirs to
|
||||
# be processed before dir1 to load error.dvc first.
|
||||
self.dvc.pipeline_stages = [
|
||||
self.dvc.stages = [
|
||||
nested_stage,
|
||||
Dvcfile(self.dvc, error_stage_path).stage,
|
||||
]
|
||||
@@ -225,9 +225,6 @@ class TestReproWorkingDirectoryAsOutput(TestDvc):
|
||||
self.fail("should not raise StagePathAsOutputError")
|
||||
|
||||
|
||||
# TODO: Test ^ for multistage
|
||||
|
||||
|
||||
class TestReproDepUnderDir(SingleStageRun, TestDvc):
|
||||
def test(self):
|
||||
stages = self.dvc.add(self.DATA_DIR)
|
||||
@@ -560,7 +557,13 @@ class TestReproLocked(TestReproChangedData):
|
||||
|
||||
def test_non_existing(self):
|
||||
with self.assertRaises(StageFileDoesNotExistError):
|
||||
self.dvc.lock_stage("non-existing-stage")
|
||||
self.dvc.lock_stage("Dvcfile")
|
||||
self.dvc.lock_stage("pipelines.yaml")
|
||||
self.dvc.lock_stage("pipelines.yaml:name")
|
||||
self.dvc.lock_stage("Dvcfile:name")
|
||||
self.dvc.lock_stage("stage.dvc")
|
||||
self.dvc.lock_stage("stage.dvc:name")
|
||||
self.dvc.lock_stage("not-existing-stage.json")
|
||||
|
||||
ret = main(["lock", "non-existing-stage"])
|
||||
self.assertNotEqual(ret, 0)
|
||||
@@ -808,7 +811,7 @@ class TestCmdReproChdir(TestDvc):
|
||||
self.assertTrue(filecmp.cmp(foo, bar, shallow=False))
|
||||
|
||||
|
||||
class TestReproExternalBase(TestDvc):
|
||||
class TestReproExternalBase(SingleStageRun, TestDvc):
|
||||
cache_type = None
|
||||
|
||||
@staticmethod
|
||||
@@ -923,13 +926,14 @@ class TestReproExternalBase(TestDvc):
|
||||
)
|
||||
self.assertEqual(self.dvc.status([import_remote_stage.path]), {})
|
||||
|
||||
cmd_stage = self.dvc.run(
|
||||
cmd_stage = self._run(
|
||||
outs=[out_bar_path],
|
||||
deps=[out_foo_path],
|
||||
cmd=self.cmd(foo_path, bar_path),
|
||||
name="external-base",
|
||||
)
|
||||
|
||||
self.assertEqual(self.dvc.status([cmd_stage.path]), {})
|
||||
self.assertEqual(self.dvc.status([cmd_stage.addressing]), {})
|
||||
self.assertEqual(self.dvc.status(), {})
|
||||
self.check_already_cached(cmd_stage)
|
||||
|
||||
@@ -945,19 +949,19 @@ class TestReproExternalBase(TestDvc):
|
||||
self.dvc.update(import_remote_stage.path)
|
||||
self.assertEqual(self.dvc.status([import_remote_stage.path]), {})
|
||||
|
||||
stages = self.dvc.reproduce(cmd_stage.path)
|
||||
stages = self.dvc.reproduce(cmd_stage.addressing)
|
||||
self.assertEqual(len(stages), 1)
|
||||
self.assertEqual(self.dvc.status([cmd_stage.path]), {})
|
||||
self.assertEqual(self.dvc.status([cmd_stage.addressing]), {})
|
||||
|
||||
self.assertEqual(self.dvc.status(), {})
|
||||
self.dvc.gc(workspace=True)
|
||||
self.assertEqual(self.dvc.status(), {})
|
||||
|
||||
self.dvc.remove(cmd_stage.path, outs_only=True)
|
||||
self.assertNotEqual(self.dvc.status([cmd_stage.path]), {})
|
||||
self.assertNotEqual(self.dvc.status([cmd_stage.addressing]), {})
|
||||
|
||||
self.dvc.checkout([cmd_stage.path], force=True)
|
||||
self.assertEqual(self.dvc.status([cmd_stage.path]), {})
|
||||
self.assertEqual(self.dvc.status([cmd_stage.addressing]), {})
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name == "nt", reason="temporarily disabled on windows")
|
||||
@@ -1196,10 +1200,11 @@ class TestReproExternalHTTP(TestReproExternalBase):
|
||||
with open("create-output.py", "w") as fd:
|
||||
fd.write(cmd)
|
||||
|
||||
run_stage = self.dvc.run(
|
||||
run_stage = self._run(
|
||||
deps=[run_dependency],
|
||||
outs=[run_output],
|
||||
cmd="python create-output.py",
|
||||
name="http_run",
|
||||
)
|
||||
self.assertTrue(run_stage is not None)
|
||||
|
||||
|
||||
@@ -3,9 +3,11 @@ from textwrap import dedent
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
from dvc.exceptions import CyclicGraphError
|
||||
from dvc.stage import PipelineStage
|
||||
from dvc.utils.stage import dump_stage_file
|
||||
from dvc.utils.stage import dump_stage_file, parse_stage
|
||||
|
||||
from tests.func import test_repro
|
||||
|
||||
from dvc.main import main
|
||||
@@ -157,6 +159,42 @@ class TestReproChangedDirDataMultiStage(
|
||||
pass
|
||||
|
||||
|
||||
class TestReproExternalS3MultiStage(
|
||||
MultiStageRun, test_repro.TestReproExternalS3
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
class TestReproExternalGSMultiStage(
|
||||
MultiStageRun, test_repro.TestReproExternalGS
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
class TestReproExternalHDFSMultiStage(
|
||||
MultiStageRun, test_repro.TestReproExternalHDFS
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
class TestReproExternalHTTPMultiStage(
|
||||
MultiStageRun, test_repro.TestReproExternalHTTP
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
class TestReproExternalLOCALMultiStage(
|
||||
MultiStageRun, test_repro.TestReproExternalLOCAL
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
class TestReproExternalSSHMultiStage(
|
||||
MultiStageRun, test_repro.TestReproExternalSSH
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
def test_non_existing_stage_name(tmp_dir, dvc, run_copy):
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -169,9 +207,6 @@ def test_non_existing_stage_name(tmp_dir, dvc, run_copy):
|
||||
assert main(["lock", ":copy-file1-file3"]) != 0
|
||||
|
||||
|
||||
# TODO: TestReproWorkingDirectoryAsOutput
|
||||
|
||||
|
||||
def test_downstream(tmp_dir, dvc):
|
||||
# The dependency graph should look like this:
|
||||
#
|
||||
@@ -204,17 +239,19 @@ def test_downstream(tmp_dir, dvc):
|
||||
# /
|
||||
# B
|
||||
#
|
||||
evaluation = dvc.reproduce("Dvcfile:B-gen", downstream=True, force=True)
|
||||
evaluation = dvc.reproduce(
|
||||
PIPELINE_FILE + ":B-gen", downstream=True, force=True
|
||||
)
|
||||
|
||||
assert len(evaluation) == 3
|
||||
assert (
|
||||
isinstance(evaluation[0], PipelineStage)
|
||||
and evaluation[0].relpath == "Dvcfile"
|
||||
and evaluation[0].relpath == PIPELINE_FILE
|
||||
and evaluation[0].name == "B-gen"
|
||||
)
|
||||
assert (
|
||||
isinstance(evaluation[1], PipelineStage)
|
||||
and evaluation[1].relpath == "Dvcfile"
|
||||
and evaluation[1].relpath == PIPELINE_FILE
|
||||
and evaluation[1].name == "D-gen"
|
||||
)
|
||||
assert (
|
||||
@@ -224,16 +261,19 @@ def test_downstream(tmp_dir, dvc):
|
||||
|
||||
|
||||
def test_repro_when_cmd_changes(tmp_dir, dvc, run_copy):
|
||||
from dvc.dvcfile import Dvcfile
|
||||
from dvc.dvcfile import PipelineFile
|
||||
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage = run_copy("foo", "bar", fname="copy-process.dvc", name="copy-file")
|
||||
target = "copy-process.dvc:copy-file"
|
||||
stage = run_copy("foo", "bar", name="copy-file")
|
||||
target = ":copy-file"
|
||||
assert not dvc.reproduce(target)
|
||||
|
||||
stage.cmd = " ".join(stage.cmd.split()) # change cmd spacing by two
|
||||
Dvcfile(dvc, "copy-process.dvc").dump_multistage_dvcfile(stage)
|
||||
PipelineFile(dvc, PIPELINE_FILE)._dump_pipeline_file(stage)
|
||||
|
||||
assert dvc.status([target]) == {
|
||||
PIPELINE_FILE + target: ["changed command"]
|
||||
}
|
||||
assert dvc.reproduce(target)[0] == stage
|
||||
|
||||
|
||||
@@ -246,10 +286,9 @@ def test_repro_when_new_deps_is_added_in_dvcfile(tmp_dir, dvc, run_copy):
|
||||
cmd="python copy.py {} {}".format("foo", "foobar"),
|
||||
outs=["foobar"],
|
||||
deps=["foo"],
|
||||
fname="copy-process.dvc",
|
||||
name="copy-file",
|
||||
)
|
||||
target = "copy-process.dvc:copy-file"
|
||||
target = PIPELINE_FILE + ":copy-file"
|
||||
assert not dvc.reproduce(target)
|
||||
|
||||
dvcfile = Dvcfile(dvc, stage.path)
|
||||
@@ -269,10 +308,9 @@ def test_repro_when_new_outs_is_added_in_dvcfile(tmp_dir, dvc):
|
||||
cmd="python copy.py {} {}".format("foo", "foobar"),
|
||||
outs=[], # scenario where user forgot to add
|
||||
deps=["foo"],
|
||||
fname="copy-process.dvc",
|
||||
name="copy-file",
|
||||
)
|
||||
target = "copy-process.dvc:copy-file"
|
||||
target = ":copy-file"
|
||||
assert not dvc.reproduce(target)
|
||||
|
||||
dvcfile = Dvcfile(dvc, stage.path)
|
||||
@@ -292,10 +330,9 @@ def test_repro_when_new_deps_is_moved(tmp_dir, dvc):
|
||||
cmd="python copy.py {} {}".format("foo", "foobar"),
|
||||
outs=["foobar"],
|
||||
deps=["foo"],
|
||||
fname="copy-process.dvc",
|
||||
name="copy-file",
|
||||
)
|
||||
target = "copy-process.dvc:copy-file"
|
||||
target = ":copy-file"
|
||||
assert not dvc.reproduce(target)
|
||||
|
||||
tmp_dir.gen("copy.py", COPY_SCRIPT_FORMAT.format("'bar'", "'foobar'"))
|
||||
@@ -317,7 +354,7 @@ def test_repro_when_new_out_overlaps_others_stage_outs(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"file1": "file1"}, "foo": "foo"})
|
||||
dvc.add("dir")
|
||||
dump_stage_file(
|
||||
"Dvcfile",
|
||||
PIPELINE_FILE,
|
||||
{
|
||||
"stages": {
|
||||
"run-copy": {
|
||||
@@ -338,7 +375,7 @@ def test_repro_when_new_deps_added_does_not_exist(tmp_dir, dvc):
|
||||
tmp_dir.gen("copy.py", COPY_SCRIPT)
|
||||
tmp_dir.gen("foo", "foo")
|
||||
dump_stage_file(
|
||||
"Dvcfile",
|
||||
PIPELINE_FILE,
|
||||
{
|
||||
"stages": {
|
||||
"run-copy": {
|
||||
@@ -359,7 +396,7 @@ def test_repro_when_new_outs_added_does_not_exist(tmp_dir, dvc):
|
||||
tmp_dir.gen("copy.py", COPY_SCRIPT)
|
||||
tmp_dir.gen("foo", "foo")
|
||||
dump_stage_file(
|
||||
"Dvcfile",
|
||||
PIPELINE_FILE,
|
||||
{
|
||||
"stages": {
|
||||
"run-copy": {
|
||||
@@ -378,7 +415,7 @@ def test_repro_when_lockfile_gets_deleted(tmp_dir, dvc):
|
||||
tmp_dir.gen("copy.py", COPY_SCRIPT)
|
||||
tmp_dir.gen("foo", "foo")
|
||||
dump_stage_file(
|
||||
"Dvcfile",
|
||||
PIPELINE_FILE,
|
||||
{
|
||||
"stages": {
|
||||
"run-copy": {
|
||||
@@ -390,20 +427,17 @@ def test_repro_when_lockfile_gets_deleted(tmp_dir, dvc):
|
||||
},
|
||||
)
|
||||
assert dvc.reproduce(":run-copy")
|
||||
assert os.path.exists("Dvcfile.lock")
|
||||
assert os.path.exists("foobar.dvc")
|
||||
assert os.path.exists(PIPELINE_LOCK)
|
||||
|
||||
assert not dvc.reproduce(":run-copy")
|
||||
os.unlink("Dvcfile.lock")
|
||||
os.unlink(PIPELINE_LOCK)
|
||||
stages = dvc.reproduce(":run-copy")
|
||||
assert (
|
||||
stages
|
||||
and stages[0].relpath == "Dvcfile"
|
||||
and stages[0].relpath == PIPELINE_FILE
|
||||
and stages[0].name == "run-copy"
|
||||
)
|
||||
|
||||
assert os.path.exists("foobar.dvc")
|
||||
|
||||
|
||||
def test_cyclic_graph_error(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
@@ -411,16 +445,13 @@ def test_cyclic_graph_error(tmp_dir, dvc, run_copy):
|
||||
run_copy("bar", "baz", name="copy-bar-baz")
|
||||
run_copy("baz", "foobar", name="copy-baz-foobar")
|
||||
|
||||
stage_dump = {
|
||||
"stages": {
|
||||
"copy-baz-foo": {
|
||||
"cmd": "echo baz > foo",
|
||||
"deps": ["baz"],
|
||||
"outs": ["foo"],
|
||||
}
|
||||
with open(PIPELINE_FILE, "r") as f:
|
||||
data = parse_stage(f.read(), PIPELINE_FILE)
|
||||
data["stages"]["copy-baz-foo"] = {
|
||||
"cmd": "echo baz > foo",
|
||||
"deps": ["baz"],
|
||||
"outs": ["foo"],
|
||||
}
|
||||
}
|
||||
dump_stage_file("cycle.dvc", stage_dump)
|
||||
|
||||
dump_stage_file(PIPELINE_FILE, data)
|
||||
with pytest.raises(CyclicGraphError):
|
||||
dvc.reproduce("cycle.dvc:copy-baz-foo")
|
||||
dvc.reproduce(":copy-baz-foo")
|
||||
|
||||
@@ -1,19 +1,20 @@
|
||||
import pytest
|
||||
import os
|
||||
|
||||
from dvc.stage.exceptions import InvalidStageName, DuplicateStageName
|
||||
|
||||
|
||||
def test_run_with_name(tmp_dir, dvc, run_copy):
|
||||
from dvc.stage import PipelineStage
|
||||
from dvc.dvcfile import DVC_FILE, DVC_FILE_SUFFIX
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
assert not os.path.exists(DVC_FILE)
|
||||
assert not os.path.exists(PIPELINE_FILE)
|
||||
stage = run_copy("foo", "bar", name="copy-foo-to-bar")
|
||||
assert isinstance(stage, PipelineStage)
|
||||
assert stage.name == "copy-foo-to-bar"
|
||||
assert os.path.exists(DVC_FILE)
|
||||
assert os.path.exists(DVC_FILE + ".lock")
|
||||
assert os.path.exists("foo" + DVC_FILE_SUFFIX)
|
||||
assert os.path.exists(PIPELINE_FILE)
|
||||
assert os.path.exists(PIPELINE_LOCK)
|
||||
|
||||
|
||||
def test_run_with_multistage_and_single_stage(tmp_dir, dvc, run_copy):
|
||||
@@ -32,40 +33,20 @@ def test_run_with_multistage_and_single_stage(tmp_dir, dvc, run_copy):
|
||||
|
||||
def test_run_multi_stage_repeat(tmp_dir, dvc, run_copy):
|
||||
from dvc.stage import PipelineStage
|
||||
from dvc.dvcfile import Dvcfile, DVC_FILE, DVC_FILE_SUFFIX
|
||||
from dvc.dvcfile import Dvcfile, PIPELINE_FILE
|
||||
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_copy("foo", "foo1", name="copy-foo-foo1")
|
||||
run_copy("foo1", "foo2", name="copy-foo1-foo2")
|
||||
run_copy("foo2", "foo3")
|
||||
|
||||
stages = list(Dvcfile(dvc, DVC_FILE).stages.values())
|
||||
stages = list(Dvcfile(dvc, PIPELINE_FILE).stages.values())
|
||||
assert len(stages) == 2
|
||||
assert all(isinstance(stage, PipelineStage) for stage in stages)
|
||||
assert set(stage.name for stage in stages) == {
|
||||
"copy-foo-foo1",
|
||||
"copy-foo1-foo2",
|
||||
}
|
||||
assert all(
|
||||
os.path.exists(file + DVC_FILE_SUFFIX)
|
||||
for file in ["foo1", "foo2", "foo3"]
|
||||
)
|
||||
|
||||
|
||||
def test_multi_stage_try_writing_on_single_stage_file(tmp_dir, dvc, run_copy):
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.dvcfile import MultiStageFileLoadError
|
||||
|
||||
tmp_dir.dvc_gen("foo")
|
||||
dvc.run(cmd="echo foo", deps=["foo"])
|
||||
|
||||
with pytest.raises(DvcException):
|
||||
run_copy("foo", "foo2", name="copy-foo1-foo2")
|
||||
|
||||
run_copy("foo", "foo2", name="copy-foo1-foo2", fname="DIFFERENT-FILE.dvc")
|
||||
|
||||
with pytest.raises(MultiStageFileLoadError):
|
||||
run_copy("foo2", "foo3", fname="DIFFERENT-FILE.dvc")
|
||||
|
||||
|
||||
def test_multi_stage_run_cached(tmp_dir, dvc, run_copy):
|
||||
@@ -78,8 +59,6 @@ def test_multi_stage_run_cached(tmp_dir, dvc, run_copy):
|
||||
|
||||
|
||||
def test_multistage_dump_on_non_cached_outputs(tmp_dir, dvc):
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX
|
||||
|
||||
tmp_dir.dvc_gen("foo")
|
||||
dvc.run(
|
||||
cmd="cp foo foo1",
|
||||
@@ -87,7 +66,6 @@ def test_multistage_dump_on_non_cached_outputs(tmp_dir, dvc):
|
||||
name="copy-foo1-foo2",
|
||||
outs_no_cache=["foo1"],
|
||||
)
|
||||
assert not os.path.exists("foo1" + DVC_FILE_SUFFIX)
|
||||
|
||||
|
||||
def test_multistage_with_wdir(tmp_dir, dvc):
|
||||
@@ -136,7 +114,7 @@ def test_graph(tmp_dir, dvc):
|
||||
|
||||
|
||||
def test_run_dump_on_multistage(tmp_dir, dvc):
|
||||
from dvc.dvcfile import Dvcfile
|
||||
from dvc.dvcfile import Dvcfile, PIPELINE_FILE
|
||||
|
||||
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||||
dvc.run(
|
||||
@@ -146,7 +124,7 @@ def test_run_dump_on_multistage(tmp_dir, dvc):
|
||||
outs=["foo1"],
|
||||
wdir="dir",
|
||||
)
|
||||
data, _ = Dvcfile(dvc, "Dvcfile")._load()
|
||||
data, _ = Dvcfile(dvc, PIPELINE_FILE)._load()
|
||||
assert data == {
|
||||
"stages": {
|
||||
"copy-foo-foo1": {
|
||||
@@ -166,7 +144,7 @@ def test_run_dump_on_multistage(tmp_dir, dvc):
|
||||
outs_persist=["foo2"],
|
||||
always_changed=True,
|
||||
)
|
||||
assert Dvcfile(dvc, "Dvcfile")._load()[0] == {
|
||||
assert Dvcfile(dvc, PIPELINE_FILE)._load()[0] == {
|
||||
"stages": {
|
||||
"copy-foo-foo2": {
|
||||
"cmd": "cp foo foo2",
|
||||
@@ -178,3 +156,16 @@ def test_run_dump_on_multistage(tmp_dir, dvc):
|
||||
**data["stages"],
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_run_with_invalid_stage_name(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
with pytest.raises(InvalidStageName):
|
||||
run_copy("foo", "bar", name="email@https://dvc.org")
|
||||
|
||||
|
||||
def test_run_already_exists(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy")
|
||||
with pytest.raises(DuplicateStageName):
|
||||
run_copy("bar", "foobar", name="copy")
|
||||
|
||||
+36
-18
@@ -7,7 +7,7 @@ from dvc.output.local import OutputLOCAL
|
||||
from dvc.remote.local import LocalRemote
|
||||
from dvc.repo import Repo
|
||||
from dvc.stage import Stage
|
||||
from dvc.dvcfile import Dvcfile
|
||||
from dvc.dvcfile import SingleStageFile
|
||||
from dvc.stage.exceptions import StageFileFormatError
|
||||
from dvc.utils.stage import dump_stage_file
|
||||
from dvc.utils.stage import load_stage_file
|
||||
@@ -16,40 +16,40 @@ from tests.basic_env import TestDvc
|
||||
|
||||
def test_cmd_obj():
|
||||
with pytest.raises(StageFileFormatError):
|
||||
Dvcfile.validate_single_stage({Stage.PARAM_CMD: {}})
|
||||
SingleStageFile.validate({Stage.PARAM_CMD: {}})
|
||||
|
||||
|
||||
def test_cmd_none():
|
||||
Dvcfile.validate_single_stage({Stage.PARAM_CMD: None})
|
||||
SingleStageFile.validate({Stage.PARAM_CMD: None})
|
||||
|
||||
|
||||
def test_no_cmd():
|
||||
Dvcfile.validate_single_stage({})
|
||||
SingleStageFile.validate({})
|
||||
|
||||
|
||||
def test_cmd_str():
|
||||
Dvcfile.validate_single_stage({Stage.PARAM_CMD: "cmd"})
|
||||
SingleStageFile.validate({Stage.PARAM_CMD: "cmd"})
|
||||
|
||||
|
||||
def test_object():
|
||||
with pytest.raises(StageFileFormatError):
|
||||
Dvcfile.validate_single_stage({Stage.PARAM_DEPS: {}})
|
||||
SingleStageFile.validate({Stage.PARAM_DEPS: {}})
|
||||
|
||||
with pytest.raises(StageFileFormatError):
|
||||
Dvcfile.validate_single_stage({Stage.PARAM_OUTS: {}})
|
||||
SingleStageFile.validate({Stage.PARAM_OUTS: {}})
|
||||
|
||||
|
||||
def test_none():
|
||||
Dvcfile.validate_single_stage({Stage.PARAM_DEPS: None})
|
||||
Dvcfile.validate_single_stage({Stage.PARAM_OUTS: None})
|
||||
SingleStageFile.validate({Stage.PARAM_DEPS: None})
|
||||
SingleStageFile.validate({Stage.PARAM_OUTS: None})
|
||||
|
||||
|
||||
def test_empty_list():
|
||||
d = {Stage.PARAM_DEPS: []}
|
||||
Dvcfile.validate_single_stage(d)
|
||||
SingleStageFile.validate(d)
|
||||
|
||||
d = {Stage.PARAM_OUTS: []}
|
||||
Dvcfile.validate_single_stage(d)
|
||||
SingleStageFile.validate(d)
|
||||
|
||||
|
||||
def test_list():
|
||||
@@ -59,12 +59,12 @@ def test_list():
|
||||
{OutputLOCAL.PARAM_PATH: "baz"},
|
||||
]
|
||||
d = {Stage.PARAM_DEPS: lst}
|
||||
Dvcfile.validate_single_stage(d)
|
||||
SingleStageFile.validate(d)
|
||||
|
||||
lst[0][OutputLOCAL.PARAM_CACHE] = True
|
||||
lst[1][OutputLOCAL.PARAM_CACHE] = False
|
||||
d = {Stage.PARAM_OUTS: lst}
|
||||
Dvcfile.validate_single_stage(d)
|
||||
SingleStageFile.validate(d)
|
||||
|
||||
|
||||
class TestReload(TestDvc):
|
||||
@@ -81,7 +81,7 @@ class TestReload(TestDvc):
|
||||
d[stage.PARAM_MD5] = md5
|
||||
dump_stage_file(stage.relpath, d)
|
||||
|
||||
dvcfile = Dvcfile(self.dvc, stage.relpath)
|
||||
dvcfile = SingleStageFile(self.dvc, stage.relpath)
|
||||
stage = dvcfile.stage
|
||||
|
||||
self.assertTrue(stage is not None)
|
||||
@@ -106,7 +106,7 @@ class TestDefaultWorkingDirectory(TestDvc):
|
||||
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
|
||||
|
||||
with self.dvc.lock, self.dvc.state:
|
||||
stage = Dvcfile(self.dvc, stage.relpath).stage
|
||||
stage = SingleStageFile(self.dvc, stage.relpath).stage
|
||||
self.assertFalse(stage.changed())
|
||||
|
||||
|
||||
@@ -157,8 +157,8 @@ def test_md5_ignores_comments(tmp_dir, dvc):
|
||||
with open(stage.path, "a") as f:
|
||||
f.write("# End comment\n")
|
||||
|
||||
new_stage = Dvcfile(dvc, stage.path).stage
|
||||
assert not new_stage.changed_md5()
|
||||
new_stage = SingleStageFile(dvc, stage.path).stage
|
||||
assert not new_stage.stage_changed()
|
||||
|
||||
|
||||
def test_meta_is_preserved(tmp_dir, dvc):
|
||||
@@ -170,7 +170,7 @@ def test_meta_is_preserved(tmp_dir, dvc):
|
||||
dump_stage_file(stage.path, data)
|
||||
|
||||
# Loading and dumping to test that it works and meta is retained
|
||||
dvcfile = Dvcfile(dvc, stage.path)
|
||||
dvcfile = SingleStageFile(dvc, stage.path)
|
||||
new_stage = dvcfile.stage
|
||||
dvcfile.dump(new_stage)
|
||||
|
||||
@@ -192,3 +192,21 @@ def test_parent_repo_collect_stages(tmp_dir, scm, dvc):
|
||||
|
||||
assert stages == []
|
||||
assert subrepo_stages != []
|
||||
|
||||
|
||||
def test_stage_addressing(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
stage1 = run_copy("foo", "bar")
|
||||
assert stage1.addressing == "bar.dvc"
|
||||
|
||||
stage2 = run_copy("bar", "baz", name="copy-bar-baz")
|
||||
assert stage2.addressing == "pipelines.yaml:copy-bar-baz"
|
||||
|
||||
folder = tmp_dir / "dir"
|
||||
folder.mkdir()
|
||||
with folder.chdir():
|
||||
assert stage1.addressing == os.path.relpath(stage1.path)
|
||||
assert (
|
||||
stage2.addressing
|
||||
== os.path.relpath(stage2.path) + ":" + stage2.name
|
||||
)
|
||||
|
||||
@@ -66,3 +66,30 @@ def test_status_before_and_after_dvc_init(tmp_dir, dvc, git_dir):
|
||||
"file ({})".format(fspath(git_dir)): "update available"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_status_on_pipeline_stages(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
stage = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
|
||||
stage.cmd = " ".join(stage.cmd.split())
|
||||
stage.dvcfile._dump_pipeline_file(stage)
|
||||
assert dvc.status() == {"pipelines.yaml:copy-foo-bar": ["changed command"]}
|
||||
|
||||
# delete outputs
|
||||
(tmp_dir / "bar").unlink()
|
||||
assert dvc.status() == {
|
||||
"pipelines.yaml:copy-foo-bar": [
|
||||
{"changed outs": {"bar": "deleted"}},
|
||||
"changed command",
|
||||
]
|
||||
}
|
||||
(tmp_dir / "foo").unlink()
|
||||
assert dvc.status() == {
|
||||
"foo.dvc": [{"changed outs": {"foo": "deleted"}}],
|
||||
"pipelines.yaml:copy-foo-bar": [
|
||||
{"changed deps": {"foo": "deleted"}},
|
||||
{"changed outs": {"bar": "deleted"}},
|
||||
"changed command",
|
||||
],
|
||||
}
|
||||
|
||||
@@ -52,7 +52,7 @@ def test_checksum_schema_fail(value):
|
||||
(
|
||||
False,
|
||||
(
|
||||
"Output 'path'(Stage stage.dvc) is missing version info. "
|
||||
"Output 'path'(stage: 'stage.dvc') is missing version info. "
|
||||
"Cache for it will not be collected. "
|
||||
"Use `dvc repro` to get your pipeline up to date."
|
||||
),
|
||||
@@ -60,19 +60,19 @@ def test_checksum_schema_fail(value):
|
||||
(
|
||||
True,
|
||||
(
|
||||
"Output 'path'(Stage stage.dvc) is missing version info. "
|
||||
"Output 'path'(stage: 'stage.dvc') is missing version info. "
|
||||
"Cache for it will not be collected. "
|
||||
"Use `dvc repro` to get your pipeline up to date.\n"
|
||||
"You can also use `dvc commit stage.dvc` to associate "
|
||||
"existing 'path' with 'stage.dvc'."
|
||||
"existing 'path' with stage: 'stage.dvc'."
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_get_used_cache(exists, expected_message, mocker, caplog):
|
||||
stage = mocker.MagicMock()
|
||||
mocker.patch.object(stage, "__str__", return_value="Stage stage.dvc")
|
||||
mocker.patch.object(stage, "relpath", "stage.dvc")
|
||||
mocker.patch.object(stage, "__str__", return_value="stage: 'stage.dvc'")
|
||||
mocker.patch.object(stage, "addressing", "stage.dvc")
|
||||
|
||||
output = OutputBase(stage, "path")
|
||||
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import pytest
|
||||
|
||||
from dvc.dvcfile import Dvcfile, PipelineFile, SingleStageFile
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path",
|
||||
[
|
||||
"pipelines.yaml",
|
||||
"pipelines.yml",
|
||||
"custom-pipelines.yml",
|
||||
"custom-pipelines.yaml",
|
||||
"../models/pipelines.yml",
|
||||
],
|
||||
)
|
||||
def test_pipelines_file(path):
|
||||
file_obj = Dvcfile(object(), path)
|
||||
assert isinstance(file_obj, PipelineFile)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path", ["Dvcfile", "stage.dvc", "../models/stage.dvc"]
|
||||
)
|
||||
def test_pipelines_single_stage_file(path):
|
||||
file_obj = Dvcfile(object(), path)
|
||||
assert isinstance(file_obj, SingleStageFile)
|
||||
+38
-31
@@ -1,27 +1,26 @@
|
||||
from dvc.stage import PipelineStage
|
||||
from dvc import lockfile, serialize
|
||||
import json
|
||||
from dvc.dvcfile import Lockfile, LockfileCorruptedError
|
||||
import yaml
|
||||
import pytest
|
||||
|
||||
|
||||
def test_stage_dump_no_outs_deps(tmp_dir, dvc):
|
||||
stage = PipelineStage(name="s1", repo=dvc, path="path", cmd="command")
|
||||
lockfile.dump(dvc, "path.lock", serialize.to_lockfile(stage))
|
||||
assert lockfile.load(dvc, "path.lock") == {
|
||||
"s1": {"cmd": "command", "deps": {}, "outs": {}}
|
||||
}
|
||||
lockfile = Lockfile(dvc, "path.lock")
|
||||
lockfile.dump(stage)
|
||||
assert lockfile.load() == {"s1": {"cmd": "command"}}
|
||||
|
||||
|
||||
def test_stage_dump_when_already_exists(tmp_dir, dvc):
|
||||
data = {"s1": {"cmd": "command", "deps": {}, "outs": {}}}
|
||||
data = {"s1": {"cmd": "command", "deps": [], "outs": []}}
|
||||
with open("path.lock", "w+") as f:
|
||||
json.dump(data, f)
|
||||
yaml.dump(data, f)
|
||||
stage = PipelineStage(name="s2", repo=dvc, path="path", cmd="command2")
|
||||
|
||||
lockfile.dump(dvc, "path.lock", serialize.to_lockfile(stage))
|
||||
assert lockfile.load(dvc, "path.lock") == {
|
||||
lockfile = Lockfile(dvc, "path.lock")
|
||||
lockfile.dump(stage)
|
||||
assert lockfile.load() == {
|
||||
**data,
|
||||
"s2": {"cmd": "command2", "deps": {}, "outs": {}},
|
||||
"s2": {"cmd": "command2"},
|
||||
}
|
||||
|
||||
|
||||
@@ -29,49 +28,57 @@ def test_stage_dump_with_deps_and_outs(tmp_dir, dvc):
|
||||
data = {
|
||||
"s1": {
|
||||
"cmd": "command",
|
||||
"deps": {"1.txt": "checksum"},
|
||||
"outs": {"2.txt": "checksum"},
|
||||
"deps": [{"md5": "1.txt", "path": "checksum"}],
|
||||
"outs": [{"md5": "2.txt", "path": "checksum"}],
|
||||
}
|
||||
}
|
||||
with open("path.lock", "w+") as f:
|
||||
json.dump(data, f)
|
||||
yaml.dump(data, f)
|
||||
|
||||
lockfile = Lockfile(dvc, "path.lock")
|
||||
stage = PipelineStage(name="s2", repo=dvc, path="path", cmd="command2")
|
||||
lockfile.dump(dvc, "path.lock", serialize.to_lockfile(stage))
|
||||
assert lockfile.load(dvc, "path.lock") == {
|
||||
lockfile.dump(stage)
|
||||
assert lockfile.load() == {
|
||||
**data,
|
||||
"s2": {"cmd": "command2", "deps": {}, "outs": {}},
|
||||
"s2": {"cmd": "command2"},
|
||||
}
|
||||
|
||||
|
||||
def test_stage_overwrites_if_already_exists(tmp_dir, dvc):
|
||||
lockfile = Lockfile(dvc, "path.lock",)
|
||||
stage = PipelineStage(name="s2", repo=dvc, path="path", cmd="command2")
|
||||
lockfile.dump(dvc, "path.lock", serialize.to_lockfile(stage))
|
||||
lockfile.dump(stage)
|
||||
stage = PipelineStage(name="s2", repo=dvc, path="path", cmd="command3")
|
||||
lockfile.dump(dvc, "path.lock", serialize.to_lockfile(stage))
|
||||
assert lockfile.load(dvc, "path.lock") == {
|
||||
"s2": {"cmd": "command3", "deps": {}, "outs": {}},
|
||||
lockfile.dump(stage)
|
||||
assert lockfile.load() == {
|
||||
"s2": {"cmd": "command3"},
|
||||
}
|
||||
|
||||
|
||||
def test_load_when_lockfile_does_not_exist(tmp_dir, dvc):
|
||||
assert {} == lockfile.load(dvc, "dvcfile.lock")
|
||||
assert {} == Lockfile(dvc, "pipelines.lock").load()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"corrupt_data",
|
||||
[
|
||||
{"s1": {"cmd": "command", "outs": {}}},
|
||||
{"s1": {"outs": {}}},
|
||||
{"s1": {"cmd": "command", "deps": {}}},
|
||||
{"s1": {"outs": []}},
|
||||
{"s1": {}},
|
||||
{"s1": {"cmd": "command", "outs": {"file": "checksum"}}},
|
||||
{"s1": {"cmd": "command", "deps": {"file": "checksum"}}},
|
||||
{
|
||||
"s1": {
|
||||
"cmd": "command",
|
||||
"outs": [
|
||||
{"md5": "checksum", "path": "path", "random": "value"}
|
||||
],
|
||||
}
|
||||
},
|
||||
{"s1": {"cmd": "command", "deps": [{"md5": "checksum"}]}},
|
||||
],
|
||||
)
|
||||
def test_load_when_lockfile_is_corrupted(tmp_dir, dvc, corrupt_data):
|
||||
with open("Dvcfile.lock", "w+") as f:
|
||||
json.dump(corrupt_data, f)
|
||||
with pytest.raises(lockfile.LockfileCorruptedError) as exc_info:
|
||||
lockfile.load(dvc, "Dvcfile.lock")
|
||||
yaml.dump(corrupt_data, f)
|
||||
lockfile = Lockfile(dvc, "Dvcfile.lock")
|
||||
with pytest.raises(LockfileCorruptedError) as exc_info:
|
||||
lockfile.load()
|
||||
assert "Dvcfile.lock" in str(exc_info.value)
|
||||
|
||||
@@ -4,11 +4,15 @@ import os
|
||||
import pytest
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.utils import file_md5, resolve_output
|
||||
from dvc.utils import fix_env
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils import to_chunks
|
||||
from dvc.utils import tmp_fname
|
||||
from dvc.utils import (
|
||||
file_md5,
|
||||
resolve_output,
|
||||
fix_env,
|
||||
relpath,
|
||||
to_chunks,
|
||||
tmp_fname,
|
||||
parse_target,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -128,3 +132,26 @@ def test_resolve_output(inp, out, is_dir, expected, mocker):
|
||||
mocker.patch("os.path.isdir", return_value=is_dir)
|
||||
result = resolve_output(inp, out)
|
||||
assert result == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"inp,out, default",
|
||||
[
|
||||
["pipelines.yaml", ("pipelines.yaml", None, None), None],
|
||||
["pipelines.yaml:name", ("pipelines.yaml", "name", None), None],
|
||||
[":name", ("pipelines.yaml", "name", None), None],
|
||||
["stage.dvc", ("stage.dvc", None, None), None],
|
||||
["pipelines.yaml:name@v1", ("pipelines.yaml", "name", "v1"), None],
|
||||
["../models/stage.dvc", ("../models/stage.dvc", None, None), "def"],
|
||||
[":name", ("default", "name", None), "default"],
|
||||
[":name@v2", ("default", "name", "v2"), "default"],
|
||||
],
|
||||
)
|
||||
def test_parse_target(inp, out, default):
|
||||
assert parse_target(inp, default) == out
|
||||
|
||||
|
||||
def test_hint_on_lockfile():
|
||||
with pytest.raises(Exception) as exc:
|
||||
assert parse_target("pipelines.lock:name@v223")
|
||||
assert "pipelines.yaml:name@v223" in str(exc.value)
|
||||
|
||||
Reference in New Issue
Block a user