Compare commits

..
8 Commits
Author SHA1 Message Date
Ruslan Kuprieiev 63972c3d12 dvc: bump to 1.0.0a6 2020-05-27 13:59:03 +03:00
Ruslan Kuprieiev 7027c631c5 dvcignore: fix bug in subrepo detection (#3880) 2020-05-27 12:13:53 +03:00
Saugat Pachhai e8f6cd7f5b run: raise error when command is not specified (#3850)
* run: raise error when command is not specified

* tests: adjust tests/func/plots/test_plots.py

* tests: adjust tests/func/plots/test_diff.py

* tests: add command to metrics/plots/params

* tests: fix test_remote.py

* tests: fix test_ls.py

* tests: fix test_repro* and test_run*

* tests: fix script in test_ls, was failing on Win
2020-05-27 12:13:26 +03:00
Peter Rowlands (변기호) 10f2bb93fa ignore: CleanTree should always check ignores, not just for walk() (#3876)
* tests: test that CleanTree works outside of walk()

* ignore: CleanTree should always check ignores, not just for walk()

* CleanTree: handle broken symlinks

* handle case for linking from shared local cache dirs

* Fix windows relpath/different drive issue
2020-05-27 12:07:38 +03:00
Saugat PachhaiandRuslan Kuprieiev 2fecc66cb9 addressing: allow stage name addressing without ":" (#3842)
* addressing: allow stage name addressing without colon

* repo: reorganize imports

* refactor/reorganize collect_granular

* add tests for stage addressing

* fix ds and cc issues

* fix unpack error in test

* tests: stage string representations

* exception: add error message

Also remove `cause` when throwing NoOutputOrStage, as
CheckoutErrorSuggestGit will start chaining all the messages
in cause of the exception.

* rename NoOutputOrStage NoOutputOrStageError

Co-authored-by: Ruslan Kuprieiev <[email protected]>
2020-05-26 03:51:51 +03:00
Ruslan Kuprieiev b47998463f run-cache: don't forget to set wdir (#3873)
Fixes #3848
2020-05-25 20:41:36 +03:00
Peter Rowlands (변기호) d53b740f35 state: always use working trees (#3871)
* tests: add test case for push/pull with --all-commits

* state: always use working tree
2020-05-25 15:47:30 +03:00
Saugat Pachhai 9198fa7459 log: reduce log, make it readable/easier (#3834)
* log: streamline/make it readable

* Remove 'run' from log message

* fix args passing in imports

* tests: fix test

* run: Add quotes around stage name

* run: add quotes for run-cache restore log

* address @dmitry's comments
2020-05-25 05:33:16 +03:00
49 changed files with 1187 additions and 407 deletions
+2 -4
View File
@@ -10,11 +10,10 @@ logger = logging.getLogger(__name__)
class CmdPipelineShow(CmdBase):
def _show(self, target, commands, outs, locked):
import networkx
from dvc import dvcfile
from dvc.utils import parse_target
path, name = parse_target(target)
stage = dvcfile.Dvcfile(self.repo, path).stages[name]
stage = self.repo.get_stage(path, name)
G = self.repo.graph
stages = networkx.dfs_postorder_nodes(G, stage)
if locked:
@@ -58,12 +57,11 @@ class CmdPipelineShow(CmdBase):
def _build_graph(self, target, commands=False, outs=False):
import networkx
from dvc import dvcfile
from dvc.repo.graph import get_pipeline
from dvc.utils import parse_target
path, name = parse_target(target)
target_stage = dvcfile.Dvcfile(self.repo, path).stages[name]
target_stage = self.repo.get_stage(path, name)
G = get_pipeline(self.repo.pipelines, target_stage)
nodes = set()
+16 -5
View File
@@ -182,12 +182,12 @@ class PipelineFile(FileMixin):
assert isinstance(stage, PipelineStage)
check_dvc_filename(self.path)
if not no_lock:
self._dump_lockfile(stage)
if update_pipeline and not stage.is_data_source:
self._dump_pipeline_file(stage)
if not no_lock:
self._dump_lockfile(stage)
def _dump_lockfile(self, stage):
self._lockfile.dump(stage)
@@ -197,6 +197,7 @@ class PipelineFile(FileMixin):
with open(self.path) as fd:
data = parse_stage_for_update(fd.read(), self.path)
else:
logger.info("Creating '%s'", self.relpath)
open(self.path, "w+").close()
data["stages"] = data.get("stages", {})
@@ -207,6 +208,9 @@ class PipelineFile(FileMixin):
else:
data["stages"].update(stage_data)
logger.info(
"Adding stage '%s' to '%s'", stage.name, self.relpath,
)
dump_stage_file(self.path, data)
self.repo.scm.track_file(relpath(self.path))
@@ -250,15 +254,22 @@ class Lockfile(FileMixin):
def dump(self, stage, **kwargs):
stage_data = serialize.to_lockfile(stage)
if not self.exists():
modified = True
logger.info("Generating lock file '%s'", self.relpath)
data = stage_data
open(self.path, "w+").close()
else:
with self.repo.tree.open(self.path, "r") as fd:
data = parse_stage_for_update(fd.read(), self.path)
modified = data.get(stage.name, {}) != stage_data.get(
stage.name, {}
)
if modified:
logger.info("Updating lock file '%s'", self.relpath)
data.update(stage_data)
dump_stage_file(self.path, data)
self.repo.scm.track_file(relpath(self.path))
if modified:
self.repo.scm.track_file(self.relpath)
class Dvcfile:
+12
View File
@@ -322,3 +322,15 @@ class RemoteCacheRequiredError(DvcException):
class IsADirectoryError(DvcException):
"""Raised when a file operation is requested on a directory."""
class NoOutputOrStageError(DvcException):
"""
Raised when the target is neither an output nor a stage name in dvc.yaml
"""
def __init__(self, target, file):
super().__init__(
f"'{target}' "
f"does not exist as an output or a stage name in '{file}'"
)
+71 -7
View File
@@ -5,7 +5,9 @@ from funcy import cached_property
from pathspec import PathSpec
from pathspec.patterns import GitWildMatchPattern
from dvc.path_info import PathInfo
from dvc.scm.tree import BaseTree
from dvc.utils import relpath
logger = logging.getLogger(__name__)
@@ -82,7 +84,7 @@ class DvcIgnoreRepo(DvcIgnore):
def is_dvc_repo(directory):
from dvc.repo import Repo
return os.path.isdir(os.path.join(directory, Repo.DVC_DIR))
return os.path.isdir(os.path.join(root, directory, Repo.DVC_DIR))
dirs = [d for d in dirs if not is_dvc_repo(d)]
@@ -125,19 +127,79 @@ class CleanTree(BaseTree):
return self.tree.tree_root
def open(self, path, mode="r", encoding="utf-8"):
return self.tree.open(path, mode, encoding)
if self.isfile(path):
return self.tree.open(path, mode, encoding)
raise FileNotFoundError
def exists(self, path):
return self.tree.exists(path)
if self.tree.exists(path) and self._parents_exist(path):
if self.tree.isdir(path):
return self._valid_dirname(path)
return self._valid_filename(path)
return False
def isdir(self, path):
return self.tree.isdir(path)
return (
self.tree.isdir(path)
and self._parents_exist(path)
and self._valid_dirname(path)
)
def _valid_dirname(self, path):
dirname, basename = os.path.split(os.path.normpath(path))
dirs, _ = self.dvcignore(os.path.abspath(dirname), [basename], [])
if dirs:
return True
return False
def isfile(self, path):
return self.tree.isfile(path)
return (
self.tree.isfile(path)
and self._parents_exist(path)
and self._valid_filename(path)
)
def _valid_filename(self, path):
dirname, basename = os.path.split(os.path.normpath(path))
_, files = self.dvcignore(os.path.abspath(dirname), [], [basename])
if files:
return True
return False
def isexec(self, path):
return self.tree.isexec(path)
return self.exists(path) and self.tree.isexec(path)
def _parents_exist(self, path):
from dvc.repo import Repo
path = PathInfo(path)
# if parent is tree_root or inside a .dvc dir we can skip this check
if path.parent == self.tree_root or Repo.DVC_DIR in path.parts:
return True
# if path is outside of tree, assume this is a local remote/local cache
# link/move operation where we do not need to filter ignores
path = relpath(path, self.tree_root)
if path.startswith("..") or (
os.name == "nt"
and not os.path.commonprefix(
[os.path.abspath(path), self.tree_root]
)
):
return True
# check if parent directories are in our ignores, starting from
# tree_root
for parent_dir in reversed(PathInfo(path).parents):
dirname, basename = os.path.split(parent_dir)
if basename == ".":
# parent_dir == tree_root
continue
dirs, _ = self.dvcignore(os.path.abspath(dirname), [basename], [])
if not dirs:
return False
return True
def walk(self, top, topdown=True):
for root, dirs, files in self.tree.walk(top, topdown):
@@ -148,4 +210,6 @@ class CleanTree(BaseTree):
yield root, dirs, files
def stat(self, path):
return self.tree.stat(path)
if self.exists(path):
return self.tree.stat(path)
raise FileNotFoundError
+3 -5
View File
@@ -245,17 +245,15 @@ class BaseOutput:
if self.metric or self.plot:
self.verify_metric()
if not self.IS_DEPENDENCY:
logger.info(
"Output '{}' doesn't use cache. Skipping saving.".format(
self
)
logger.debug(
"Output '%s' doesn't use cache. Skipping saving.", self
)
return
assert not self.IS_DEPENDENCY
if not self.changed():
logger.info(f"Output '{self}' didn't change. Skipping saving.")
logger.debug("Output '%s' didn't change. Skipping saving.", self)
return
self.info = self.save_info()
+95 -26
View File
@@ -1,3 +1,4 @@
import logging
import os
from contextlib import contextmanager
from functools import wraps
@@ -5,17 +6,25 @@ from functools import wraps
from funcy import cached_property, cat, first
from dvc.config import Config
from dvc.dvcfile import PIPELINE_FILE, Dvcfile, is_valid_filename
from dvc.exceptions import FileMissingError
from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
from dvc.exceptions import (
NoOutputOrStageError,
NotDvcRepoError,
OutputNotFoundError,
)
from dvc.ignore import CleanTree
from dvc.path_info import PathInfo
from dvc.repo.tree import RepoTree
from dvc.utils.fs import path_isin
from ..stage.exceptions import StageFileDoesNotExistError, StageNotFound
from ..utils import parse_target
from .graph import check_acyclic, get_pipeline, get_pipelines
logger = logging.getLogger(__name__)
def locked(f):
@wraps(f)
@@ -181,6 +190,25 @@ class Repo:
self.scm.ignore_list(flist)
def get_stage(self, path=None, name=None):
if not path:
path = PIPELINE_FILE
logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
dvcfile = Dvcfile(self, path)
return dvcfile.stages[name]
def get_stages(self, path=None, name=None):
if not path:
path = PIPELINE_FILE
logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
if name:
return [self.get_stage(path, name)]
dvcfile = Dvcfile(self, path)
return list(dvcfile.stages.values())
def check_modified_graph(self, new_stages):
"""Generate graph including the new stage to check for errors"""
# Building graph might be costly for the ones with many DVC-files,
@@ -204,10 +232,9 @@ class Repo:
stages = nx.dfs_postorder_nodes(graph)
return [stage for stage in stages if path_isin(stage.path, path)]
def collect(self, target, with_deps=False, recursive=False, graph=None):
import networkx as nx
from ..dvcfile import Dvcfile
def collect(
self, target=None, with_deps=False, recursive=False, graph=None
):
if not target:
return list(graph) if graph else self.stages
@@ -217,36 +244,81 @@ class Repo:
)
path, name = parse_target(target)
dvcfile = Dvcfile(self, path)
stages = list(dvcfile.stages.filter(name).values())
stages = self.get_stages(path, name)
if not with_deps:
return stages
res = set()
for stage in stages:
pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
res.update(nx.dfs_postorder_nodes(pipeline, stage))
res.update(self._collect_pipeline(stage, graph=graph))
return res
def collect_granular(self, target, *args, **kwargs):
from ..dvcfile import Dvcfile, is_valid_filename
def _collect_pipeline(self, stage, graph=None):
import networkx as nx
pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
return nx.dfs_postorder_nodes(pipeline, stage)
def _collect_from_default_dvcfile(self, target):
dvcfile = Dvcfile(self, PIPELINE_FILE)
if dvcfile.exists():
return dvcfile.stages.get(target)
def collect_granular(
self, target=None, with_deps=False, recursive=False, graph=None
):
"""
Priority is in the order of following in case of ambiguity:
- .dvc file or .yaml file
- dir if recursive and directory exists
- stage_name
- output file
"""
if not target:
return [(stage, None) for stage in self.stages]
file, name = parse_target(target)
if is_valid_filename(file) and not kwargs.get("with_deps"):
# Optimization: do not collect the graph for a specific .dvc target
stages = Dvcfile(self, file).stages.filter(name)
return [(stage, None) for stage in stages.values()]
stages = []
try:
(out,) = self.find_outs_by_path(file, strict=False)
filter_info = PathInfo(os.path.abspath(file))
return [(out.stage, filter_info)]
except OutputNotFoundError:
stages = self.collect(target, *args, **kwargs)
return [(stage, None) for stage in stages]
# Optimization: do not collect the graph for a specific target
if not file:
# parsing is ambiguous when it does not have a colon
# or if it's not a dvcfile, as it can be a stage name
# in `dvc.yaml` or, an output in a stage.
logger.debug(
"Checking if stage '%s' is in '%s'", target, PIPELINE_FILE
)
if not (recursive and os.path.isdir(target)):
stage = self._collect_from_default_dvcfile(target)
if stage:
stages = (
self._collect_pipeline(stage) if with_deps else [stage]
)
elif not with_deps and is_valid_filename(file):
stages = self.get_stages(file, name)
if not stages:
if not (recursive and os.path.isdir(target)):
try:
(out,) = self.find_outs_by_path(target, strict=False)
filter_info = PathInfo(os.path.abspath(target))
return [(out.stage, filter_info)]
except OutputNotFoundError:
pass
try:
stages = self.collect(target, with_deps, recursive, graph)
except StageFileDoesNotExistError as exc:
# collect() might try to use `target` as a stage name
# and throw error that dvc.yaml does not exist, whereas it
# should say that both stage name and file does not exist.
if file and is_valid_filename(file):
raise
raise NoOutputOrStageError(target, exc.file) from exc
except StageNotFound as exc:
raise NoOutputOrStageError(target, exc.file) from exc
return [(stage, None) for stage in stages]
def used_cache(
self,
@@ -443,16 +515,13 @@ class Repo:
return PlotTemplates(self.dvc_dir)
def _collect_stages(self):
from dvc.dvcfile import Dvcfile, is_valid_filename
stages = []
outs = set()
for root, dirs, files in self.tree.walk(self.root_dir):
for file_name in filter(is_valid_filename, files):
path = os.path.join(root, file_name)
stage_loader = Dvcfile(self, path).stages
stages.extend(stage_loader.values())
stages.extend(self.get_stages(path))
outs.update(
out.fspath
for stage in stages
+14 -8
View File
@@ -1,7 +1,11 @@
import logging
import os
from dvc.exceptions import CheckoutError, CheckoutErrorSuggestGit
from dvc.exceptions import (
CheckoutError,
CheckoutErrorSuggestGit,
NoOutputOrStageError,
)
from dvc.progress import Tqdm
from dvc.utils import relpath
@@ -18,11 +22,11 @@ def _get_unused_links(repo):
return repo.state.get_unused_links(used)
def _fspath_dir(path, root):
def _fspath_dir(path):
if not os.path.exists(str(path)):
return str(path)
path = relpath(path, root)
path = relpath(path)
return os.path.join(path, "") if os.path.isdir(path) else path
@@ -56,7 +60,7 @@ def _checkout(
targets = [None]
unused = _get_unused_links(self)
stats["deleted"] = [_fspath_dir(u, self.root_dir) for u in unused]
stats["deleted"] = [_fspath_dir(u) for u in unused]
self.state.remove_links(unused)
if isinstance(targets, str):
@@ -70,7 +74,11 @@ def _checkout(
target, with_deps=with_deps, recursive=recursive
)
)
except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
except (
StageFileDoesNotExistError,
StageFileBadNameError,
NoOutputOrStageError,
) as exc:
if not target:
raise
raise CheckoutErrorSuggestGit(target) from exc
@@ -87,9 +95,7 @@ def _checkout(
filter_info=filter_info,
)
for key, items in result.items():
stats[key].extend(
_fspath_dir(path, self.root_dir) for path in items
)
stats[key].extend(_fspath_dir(path) for path in items)
if stats.get("failed"):
raise CheckoutError(stats["failed"], stats)
+1
View File
@@ -46,3 +46,4 @@ def commit(self, target, with_deps=False, recursive=False, force=False):
stage.commit()
Dvcfile(self, stage.path).dump(stage)
return stages
+2 -4
View File
@@ -3,13 +3,11 @@ from . import locked
@locked
def lock(self, target, unlock=False):
from .. import dvcfile
from dvc.utils import parse_target
path, name = parse_target(target)
dvcfile = dvcfile.Dvcfile(self, path)
stage = dvcfile.stages[name]
stage = self.get_stage(path, name)
stage.locked = False if unlock else True
dvcfile.dump(stage, update_pipeline=True)
stage.dvcfile.dump(stage, update_pipeline=True)
return stage
+4 -5
View File
@@ -8,15 +8,14 @@ logger = logging.getLogger(__name__)
@locked
def remove(self, target, dvc_only=False):
from ..dvcfile import Dvcfile
from ..dvcfile import Dvcfile, is_valid_filename
path, name = parse_target(target)
dvcfile = Dvcfile(self, path)
stages = list(dvcfile.stages.filter(name).values())
stages = self.get_stages(path, name)
for stage in stages:
stage.remove_outs(force=True)
if not dvc_only:
dvcfile.remove()
if path and is_valid_filename(path) and not dvc_only:
Dvcfile(self, path).remove()
return stages
+6 -3
View File
@@ -61,7 +61,6 @@ def reproduce(
all_pipelines=False,
**kwargs
):
from ..dvcfile import Dvcfile
from dvc.utils import parse_target
if not target and not all_pipelines:
@@ -81,8 +80,7 @@ def reproduce(
if all_pipelines:
pipelines = active_pipelines
else:
dvcfile = Dvcfile(self, path)
stage = dvcfile.stages[name]
stage = self.get_stage(path, name)
pipelines = [get_pipeline(active_pipelines, stage)]
targets = []
@@ -164,7 +162,12 @@ def _reproduce_stages(
force_downstream = kwargs.pop("force_downstream", False)
result = []
# `ret` is used to add a cosmetic newline.
ret = []
for stage in pipeline:
if ret:
logger.info("")
try:
ret = _reproduce_stage(stage, **kwargs)
+4 -2
View File
@@ -45,6 +45,9 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
from dvc.stage import PipelineStage, Stage, create_stage
from dvc.dvcfile import Dvcfile, PIPELINE_FILE
if not kwargs.get("cmd"):
raise InvalidArgumentError("command is not specified")
stage_cls = PipelineStage
path = PIPELINE_FILE
stage_name = kwargs.get("name")
@@ -82,8 +85,7 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
raise OutputDuplicationError(exc.output, set(exc.stages) - {stage})
if no_exec:
for out in stage.outs:
out.ignore()
stage.ignore_outs()
else:
stage.run(
no_commit=kwargs.get("no_commit", False),
+24 -31
View File
@@ -77,7 +77,7 @@ def create_stage(cls, repo, path, **kwargs):
logger.warning("Build cache is ignored when persisting outputs.")
if not ignore_run_cache and stage.can_be_skipped:
logger.info("Stage is cached, skipping.")
logger.info("Stage is cached, skipping")
return None
return stage
@@ -144,14 +144,10 @@ class Stage(params.StageParams):
self._dvcfile = dvcfile
def __repr__(self):
return "Stage: '{path}'".format(
path=self.path_in_repo if self.path else "No path"
)
return f"Stage: '{self.addressing}'"
def __str__(self):
return "stage: '{path}'".format(
path=self.relpath if self.path else "No path"
)
return f"stage: '{self.addressing}'"
@property
def addressing(self):
@@ -159,7 +155,7 @@ class Stage(params.StageParams):
Useful for alternative presentations where we don't need
`Stage:` prefix.
"""
return self.relpath
return self.relpath if self.path else "No path"
def __hash__(self):
return hash(self.path_in_repo)
@@ -209,10 +205,11 @@ class Stage(params.StageParams):
return False
if self.is_callback:
logger.warning(
'{stage} is a "callback" stage '
logger.debug(
'%s is a "callback" stage '
"(has a command and no dependencies) and thus always "
"considered as changed.".format(stage=self)
"considered as changed.",
self,
)
return True
@@ -248,9 +245,9 @@ class Stage(params.StageParams):
return False
def changed_stage(self, warn=False):
def changed_stage(self):
changed = self.md5 != self.compute_md5()
if changed and warn:
if changed:
logger.debug(self._changed_stage_entry())
return changed
@@ -259,14 +256,12 @@ class Stage(params.StageParams):
is_changed = (
# Short-circuit order: stage md5 is fast,
# deps are expected to change
self.changed_stage(warn=True)
self.changed_stage()
or self.changed_deps()
or self.changed_outs()
)
if is_changed:
logger.info("%s changed.", self)
else:
logger.info("%s didn't change.", self)
logger.debug("%s changed.", self)
return is_changed
@rwlocked(write=["outs"])
@@ -294,8 +289,8 @@ class Stage(params.StageParams):
@rwlocked(read=["deps"], write=["outs"])
def reproduce(self, interactive=False, **kwargs):
if not (kwargs.get("force", False) or self.changed()):
logger.info("Stage '%s' didn't change, skipping", self.addressing)
return None
msg = (
@@ -379,6 +374,10 @@ class Stage(params.StageParams):
for out in self.outs:
out.save()
def ignore_outs(self):
for out in self.outs:
out.ignore()
@staticmethod
def _changed_entries(entries):
return [str(entry) for entry in entries if entry.changed_checksum()]
@@ -527,19 +526,13 @@ class PipelineStage(Stage):
def __hash__(self):
return hash((self.path_in_repo, self.name))
def __repr__(self):
return "Stage: '{path}:{name}'".format(
path=self.relpath if self.path else "No path", name=self.name
)
def __str__(self):
return "stage: '{path}:{name}'".format(
path=self.relpath if self.path else "No path", name=self.name
)
@property
def addressing(self):
return super().addressing + ":" + self.name
from dvc.dvcfile import PIPELINE_FILE
if self.path and self.relpath == PIPELINE_FILE:
return self.name
return f"{super().addressing}:{self.name}"
def reload(self):
return self.dvcfile.stages[self.name]
@@ -552,8 +545,8 @@ class PipelineStage(Stage):
if self.cmd_changed:
ret.append("changed command")
def changed_stage(self, warn=False):
if self.cmd_changed and warn:
def changed_stage(self):
if self.cmd_changed:
logger.debug(self._changed_stage_entry())
return self.cmd_changed
+3 -2
View File
@@ -77,7 +77,7 @@ class StageCache:
return None
def _create_stage(self, cache):
def _create_stage(self, cache, wdir=None):
from dvc.stage import create_stage, PipelineStage
params = []
@@ -98,6 +98,7 @@ class StageCache:
repo=self.repo,
path="dvc.yaml",
cmd=cache["cmd"],
wdir=wdir,
params=params,
deps=[dep["path"] for dep in cache.get("deps", [])],
outs=[out["path"] for out in cache["outs"]],
@@ -118,7 +119,7 @@ class StageCache:
# NOTE: using temporary stage to avoid accidentally modifying original
# stage and to workaround `commit/checkout` not working for uncached
# outputs.
cached_stage = self._create_stage(cache)
cached_stage = self._create_stage(cache, wdir=stage.wdir)
outs_no_cache = [
out.def_path for out in stage.outs if not out.use_cache
+5 -10
View File
@@ -15,15 +15,8 @@ class StageFileFormatError(DvcException):
class StageFileDoesNotExistError(DvcException):
def __init__(self, fname):
from dvc.dvcfile import DVC_FILE_SUFFIX, is_dvc_file
msg = f"'{fname}' does not exist."
sname = fname + DVC_FILE_SUFFIX
if is_dvc_file(sname):
msg += f" Do you mean '{sname}'?"
super().__init__(msg)
self.file = fname
super().__init__(f"'{self.file}' does not exist.")
class StageFileAlreadyExistsError(DvcException):
@@ -87,8 +80,10 @@ class MissingDataSource(DvcException):
class StageNotFound(KeyError, DvcException):
def __init__(self, file, name):
self.file = file.relpath
self.name = name
super().__init__(
f"Stage '{name}' not found inside '{file.relpath}' file"
f"Stage '{self.name}' not found inside '{self.file}' file"
)
+1 -5
View File
@@ -23,11 +23,7 @@ def sync_import(stage, dry=False, force=False):
if dry:
return
if (
not force
and not stage.changed_stage(warn=True)
and stage.already_cached()
):
if not force and not stage.changed_stage() and stage.already_cached():
stage.outs[0].checkout()
else:
stage.deps[0].download(stage.outs[0])
-13
View File
@@ -30,16 +30,6 @@ class StageLoader(Mapping):
self.stages_data = stages_data or {}
self.lockfile_data = lockfile_data or {}
def filter(self, item=None):
if not item:
return self
if item not in self:
raise StageNotFound(self.dvcfile, item)
data = {item: self.stages_data[item]} if item in self else {}
return self.__class__(self.dvcfile, data, self.lockfile_data)
@staticmethod
def fill_from_lock(stage, lock_data):
from .params import StageParams
@@ -224,9 +214,6 @@ class SingleStageLoader(Mapping):
self.stage_data = stage_data or {}
self.stage_text = stage_text
def filter(self, item=None):
return self
def __getitem__(self, item):
if item:
logger.warning(
+16 -4
View File
@@ -79,11 +79,14 @@ def cmd_run(stage, *args, **kwargs):
def _is_cached(stage):
return (
cached = (
not stage.is_callback
and not stage.always_changed
and stage.already_cached()
)
if cached:
logger.info("Stage '%s' is cached", stage.addressing)
return cached
def restored_from_cache(stage):
@@ -93,7 +96,10 @@ def restored_from_cache(stage):
return False
# restore stage from build cache
stage_cache.restore(stage)
return stage.outs_cached()
restored = stage.outs_cached()
if restored:
logger.info("Restored stage '%s' from run-cache", stage.addressing)
return restored
def run_stage(stage, dry=False, force=False, run_cache=False):
@@ -102,10 +108,16 @@ def run_stage(stage, dry=False, force=False, run_cache=False):
run_cache and restored_from_cache(stage)
)
if stage_cached:
logger.info("Stage is cached, skipping.")
logger.info("Skipping run, checking out outputs")
stage.checkout()
return
logger.info("Running command:\n\t%s", stage.cmd)
callback_str = "callback " if stage.is_callback else ""
logger.info(
"Running %s" "stage '%s' with command:",
callback_str,
stage.addressing,
)
logger.info("\t%s", stage.cmd)
if not dry:
cmd_run(stage)
+6 -6
View File
@@ -7,6 +7,7 @@ import sqlite3
from urllib.parse import urlencode, urlunparse
from dvc.exceptions import DvcException
from dvc.scm.tree import WorkingTree
from dvc.utils import current_timestamp, relpath, to_chunks
from dvc.utils.fs import get_inode, get_mtime_and_size, remove
@@ -91,6 +92,7 @@ class State: # pylint: disable=too-many-instance-attributes
def __init__(self, repo):
self.repo = repo
self.root_dir = repo.root_dir
self.tree = WorkingTree(self.root_dir)
state_config = repo.config.get("state", {})
self.row_limit = state_config.get("row_limit", self.STATE_ROW_LIMIT)
@@ -364,9 +366,7 @@ class State: # pylint: disable=too-many-instance-attributes
assert checksum is not None
assert os.path.exists(path_info)
actual_mtime, actual_size = get_mtime_and_size(
path_info, self.repo.tree
)
actual_mtime, actual_size = get_mtime_and_size(path_info, self.tree)
actual_inode = get_inode(path_info)
existing_record = self.get_state_record_for_inode(actual_inode)
@@ -397,7 +397,7 @@ class State: # pylint: disable=too-many-instance-attributes
if not os.path.exists(path):
return None
actual_mtime, actual_size = get_mtime_and_size(path, self.repo.tree)
actual_mtime, actual_size = get_mtime_and_size(path, self.tree)
actual_inode = get_inode(path)
existing_record = self.get_state_record_for_inode(actual_inode)
@@ -423,7 +423,7 @@ class State: # pylint: disable=too-many-instance-attributes
if not os.path.exists(path_info):
return
mtime, _ = get_mtime_and_size(path_info, self.repo.tree)
mtime, _ = get_mtime_and_size(path_info, self.tree)
inode = get_inode(path_info)
relative_path = relpath(path_info, self.root_dir)
@@ -450,7 +450,7 @@ class State: # pylint: disable=too-many-instance-attributes
continue
actual_inode = get_inode(path)
actual_mtime, _ = get_mtime_and_size(path, self.repo.tree)
actual_mtime, _ = get_mtime_and_size(path, self.tree)
if (inode, mtime) == (actual_inode, actual_mtime):
logger.debug("Removing '%s' as unused link.", path)
+12 -6
View File
@@ -395,7 +395,7 @@ def format_link(link):
def parse_target(target, default=None):
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK, is_valid_filename
from dvc.exceptions import DvcException
if not target:
@@ -409,13 +409,19 @@ def parse_target(target, default=None):
match.group("path"),
match.group("name"),
)
if path:
if os.path.basename(path) == PIPELINE_LOCK:
raise DvcException(
"Did you mean: `{}`?".format(
target.replace(".lock", ".yaml", 1)
)
)
if not name:
ret = (target, None)
return ret if is_valid_filename(target) else ret[::-1]
if not path:
path = default or PIPELINE_FILE
logger.debug("Assuming file to be '%s'", path)
if os.path.basename(path) == PIPELINE_LOCK:
raise DvcException(
"Did you mean: `{}`?".format(target.replace(".lock", ".yaml", 1))
)
return path, name
+1 -1
View File
@@ -6,7 +6,7 @@
import os
import subprocess
_BASE_VERSION = "1.0.0a5"
_BASE_VERSION = "1.0.0a6"
def _generate_version(base_version):
+1
View File
@@ -0,0 +1 @@
from tests.func.plots import run_copy_metrics # noqa: F401
+45 -37
View File
@@ -3,11 +3,10 @@ import json
import yaml
def test_metrics_diff_simple(tmp_dir, scm, dvc):
def test_metrics_diff_simple(tmp_dir, scm, dvc, run_copy_metrics):
def _gen(val):
tmp_dir.gen({"m.yaml": str(val)})
dvc.run(cmd="", metrics=["m.yaml"], single_stage=True)
dvc.scm.add(["m.yaml.dvc"])
tmp_dir.gen({"m_temp.yaml": str(val)})
run_copy_metrics("m_temp.yaml", "m.yaml", metrics=["m.yaml"])
dvc.scm.commit(str(val))
_gen(1)
@@ -19,13 +18,13 @@ def test_metrics_diff_simple(tmp_dir, scm, dvc):
assert dvc.metrics.diff(a_rev="HEAD~2") == expected
def test_metrics_diff_yaml(tmp_dir, scm, dvc):
def test_metrics_diff_yaml(tmp_dir, scm, dvc, run_copy_metrics):
def _gen(val):
metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}}
tmp_dir.gen({"m.yaml": yaml.dump(metrics)})
dvc.run(cmd="", metrics=["m.yaml"], single_stage=True)
dvc.scm.add(["m.yaml.dvc"])
dvc.scm.commit(str(val))
tmp_dir.gen({"m_temp.yaml": yaml.dump(metrics)})
run_copy_metrics(
"m_temp.yaml", "m.yaml", metrics=["m.yaml"], commit=str(val)
)
_gen(1)
_gen(2)
@@ -36,13 +35,13 @@ def test_metrics_diff_yaml(tmp_dir, scm, dvc):
assert dvc.metrics.diff(a_rev="HEAD~2") == expected
def test_metrics_diff_json(tmp_dir, scm, dvc):
def test_metrics_diff_json(tmp_dir, scm, dvc, run_copy_metrics):
def _gen(val):
metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}}
tmp_dir.gen({"m.json": json.dumps(metrics)})
dvc.run(cmd="", metrics=["m.json"], single_stage=True)
dvc.scm.add(["m.json.dvc"])
dvc.scm.commit(str(val))
tmp_dir.gen({"m_temp.json": json.dumps(metrics)})
run_copy_metrics(
"m_temp.json", "m.json", metrics=["m.json"], commit=str(val)
)
_gen(1)
_gen(2)
@@ -52,13 +51,13 @@ def test_metrics_diff_json(tmp_dir, scm, dvc):
assert dvc.metrics.diff(a_rev="HEAD~2") == expected
def test_metrics_diff_json_unchanged(tmp_dir, scm, dvc):
def test_metrics_diff_json_unchanged(tmp_dir, scm, dvc, run_copy_metrics):
def _gen(val):
metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}}
tmp_dir.gen({"m.json": json.dumps(metrics)})
dvc.run(cmd="", metrics=["m.json"], single_stage=True)
dvc.scm.add(["m.json.dvc"])
dvc.scm.commit(str(val))
tmp_dir.gen({"m_temp.json": json.dumps(metrics)})
run_copy_metrics(
"m_temp.json", "m.json", metrics=["m.json"], commit=str(val)
)
_gen(1)
_gen(2)
@@ -67,12 +66,15 @@ def test_metrics_diff_json_unchanged(tmp_dir, scm, dvc):
assert dvc.metrics.diff(a_rev="HEAD~2") == {}
def test_metrics_diff_broken_json(tmp_dir, scm, dvc):
def test_metrics_diff_broken_json(tmp_dir, scm, dvc, run_copy_metrics):
metrics = {"a": {"b": {"c": 1, "d": 1, "e": "3"}}}
tmp_dir.gen({"m.json": json.dumps(metrics)})
dvc.run(cmd="", metrics_no_cache=["m.json"], single_stage=True)
dvc.scm.add(["m.json.dvc", "m.json"])
dvc.scm.commit("add metrics")
tmp_dir.gen({"m_temp.json": json.dumps(metrics)})
run_copy_metrics(
"m_temp.json",
"m.json",
metrics_no_cache=["m.json"],
commit="add metrics",
)
(tmp_dir / "m.json").write_text(json.dumps(metrics) + "ma\nlformed\n")
@@ -89,10 +91,10 @@ def test_metrics_diff_no_metrics(tmp_dir, scm, dvc):
assert dvc.metrics.diff(a_rev="HEAD~1") == {}
def test_metrics_diff_new_metric(tmp_dir, scm, dvc):
def test_metrics_diff_new_metric(tmp_dir, scm, dvc, run_copy_metrics):
metrics = {"a": {"b": {"c": 1, "d": 1, "e": "3"}}}
tmp_dir.gen({"m.json": json.dumps(metrics)})
dvc.run(cmd="", metrics_no_cache=["m.json"], single_stage=True)
tmp_dir.gen({"m_temp.json": json.dumps(metrics)})
run_copy_metrics("m_temp.json", "m.json", metrics_no_cache=["m.json"])
assert dvc.metrics.diff() == {
"m.json": {
@@ -102,12 +104,15 @@ def test_metrics_diff_new_metric(tmp_dir, scm, dvc):
}
def test_metrics_diff_deleted_metric(tmp_dir, scm, dvc):
def test_metrics_diff_deleted_metric(tmp_dir, scm, dvc, run_copy_metrics):
metrics = {"a": {"b": {"c": 1, "d": 1, "e": "3"}}}
tmp_dir.gen({"m.json": json.dumps(metrics)})
dvc.run(cmd="", metrics_no_cache=["m.json"], single_stage=True)
dvc.scm.add(["m.json.dvc", "m.json"])
dvc.scm.commit("add metrics")
tmp_dir.gen({"m_temp.json": json.dumps(metrics)})
run_copy_metrics(
"m_temp.json",
"m.json",
metrics_no_cache=["m.json"],
commit="add metrics",
)
(tmp_dir / "m.json").unlink()
@@ -119,11 +124,14 @@ def test_metrics_diff_deleted_metric(tmp_dir, scm, dvc):
}
def test_metrics_diff_with_unchanged(tmp_dir, scm, dvc):
tmp_dir.gen("metrics.yaml", "foo: 1\nxyz: 10")
dvc.run(metrics_no_cache=["metrics.yaml"], single_stage=True)
scm.add(["metrics.yaml", "metrics.yaml.dvc"])
scm.commit("1")
def test_metrics_diff_with_unchanged(tmp_dir, scm, dvc, run_copy_metrics):
tmp_dir.gen("metrics_temp.yaml", "foo: 1\nxyz: 10")
run_copy_metrics(
"metrics_temp.yaml",
"metrics.yaml",
metrics_no_cache=["metrics.yaml"],
commit="1",
)
tmp_dir.scm_gen("metrics.yaml", "foo: 2\nxyz: 10", commit="2")
tmp_dir.scm_gen("metrics.yaml", "foo: 3\nxyz: 10", commit="3")
+25 -17
View File
@@ -8,36 +8,44 @@ def test_show_empty(dvc):
dvc.metrics.show()
def test_show_simple(tmp_dir, dvc):
tmp_dir.gen("metrics.yaml", "1.1")
dvc.run(metrics=["metrics.yaml"], single_stage=True)
def test_show_simple(tmp_dir, dvc, run_copy_metrics):
tmp_dir.gen("metrics_t.yaml", "1.1")
run_copy_metrics(
"metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"],
)
assert dvc.metrics.show() == {"": {"metrics.yaml": 1.1}}
def test_show(tmp_dir, dvc):
tmp_dir.gen("metrics.yaml", "foo: 1.1")
dvc.run(metrics=["metrics.yaml"], single_stage=True)
def test_show(tmp_dir, dvc, run_copy_metrics):
tmp_dir.gen("metrics_t.yaml", "foo: 1.1")
run_copy_metrics(
"metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"],
)
assert dvc.metrics.show() == {"": {"metrics.yaml": {"foo": 1.1}}}
def test_show_multiple(tmp_dir, dvc):
tmp_dir.gen("foo", "foo: 1\n")
tmp_dir.gen("baz", "baz: 2\n")
dvc.run(fname="foo.dvc", metrics=["foo"], single_stage=True)
dvc.run(fname="baz.dvc", metrics=["baz"], single_stage=True)
def test_show_multiple(tmp_dir, dvc, run_copy_metrics):
tmp_dir.gen("foo_temp", "foo: 1\n")
tmp_dir.gen("baz_temp", "baz: 2\n")
run_copy_metrics("foo_temp", "foo", fname="foo.dvc", metrics=["foo"])
run_copy_metrics("baz_temp", "baz", fname="baz.dvc", metrics=["baz"])
assert dvc.metrics.show() == {"": {"foo": {"foo": 1}, "baz": {"baz": 2}}}
def test_show_invalid_metric(tmp_dir, dvc):
tmp_dir.gen("metrics.yaml", "foo:\n- bar\n- baz\nxyz: string")
dvc.run(metrics=["metrics.yaml"], single_stage=True)
def test_show_invalid_metric(tmp_dir, dvc, run_copy_metrics):
tmp_dir.gen("metrics_temp.yaml", "foo:\n- bar\n- baz\nxyz: string")
run_copy_metrics(
"metrics_temp.yaml", "metrics.yaml", metrics=["metrics.yaml"]
)
with pytest.raises(NoMetricsError):
dvc.metrics.show()
def test_show_branch(tmp_dir, scm, dvc):
tmp_dir.gen("metrics.yaml", "foo: 1")
dvc.run(metrics_no_cache=["metrics.yaml"], single_stage=True)
def test_show_branch(tmp_dir, scm, dvc, run_copy_metrics):
tmp_dir.gen("metrics_temp.yaml", "foo: 1")
run_copy_metrics(
"metrics_temp.yaml", "metrics.yaml", metrics_no_cache=["metrics.yaml"]
)
scm.add(["metrics.yaml", "metrics.yaml.dvc"])
scm.commit("init")
+8 -8
View File
@@ -4,7 +4,7 @@ def test_diff_no_params(tmp_dir, scm, dvc):
def test_diff_no_changes(tmp_dir, scm, dvc):
tmp_dir.gen("params.yaml", "foo: bar")
dvc.run(params=["foo"], single_stage=True)
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
scm.add(["params.yaml", "Dvcfile"])
scm.commit("bar")
assert dvc.params.diff() == {}
@@ -12,7 +12,7 @@ def test_diff_no_changes(tmp_dir, scm, dvc):
def test_diff(tmp_dir, scm, dvc):
tmp_dir.gen("params.yaml", "foo: bar")
dvc.run(params=["foo"], single_stage=True)
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
scm.add(["params.yaml", "Dvcfile"])
scm.commit("bar")
@@ -26,7 +26,7 @@ def test_diff(tmp_dir, scm, dvc):
def test_diff_new(tmp_dir, scm, dvc):
tmp_dir.gen("params.yaml", "foo: bar")
dvc.run(params=["foo"], single_stage=True)
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
assert dvc.params.diff() == {
"params.yaml": {"foo": {"old": None, "new": "bar"}}
@@ -35,7 +35,7 @@ def test_diff_new(tmp_dir, scm, dvc):
def test_diff_deleted(tmp_dir, scm, dvc):
tmp_dir.gen("params.yaml", "foo: bar")
dvc.run(params=["foo"], single_stage=True)
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
scm.add(["params.yaml", "Dvcfile"])
scm.commit("bar")
@@ -48,7 +48,7 @@ def test_diff_deleted(tmp_dir, scm, dvc):
def test_diff_deleted_config(tmp_dir, scm, dvc):
tmp_dir.gen("params.yaml", "foo: bar")
dvc.run(params=["foo"], single_stage=True)
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
scm.add(["params.yaml", "Dvcfile"])
scm.commit("bar")
@@ -61,7 +61,7 @@ def test_diff_deleted_config(tmp_dir, scm, dvc):
def test_diff_list(tmp_dir, scm, dvc):
tmp_dir.gen("params.yaml", "foo:\n- bar\n- baz")
dvc.run(params=["foo"], single_stage=True)
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
scm.add(["params.yaml", "Dvcfile"])
scm.commit("foo")
@@ -76,7 +76,7 @@ def test_diff_list(tmp_dir, scm, dvc):
def test_diff_dict(tmp_dir, scm, dvc):
tmp_dir.gen("params.yaml", "foo:\n bar: baz")
dvc.run(params=["foo"], single_stage=True)
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
scm.add(["params.yaml", "Dvcfile"])
scm.commit("foo")
@@ -89,7 +89,7 @@ def test_diff_dict(tmp_dir, scm, dvc):
def test_diff_with_unchanged(tmp_dir, scm, dvc):
tmp_dir.gen("params.yaml", "foo: bar\nxyz: val")
dvc.run(params=["foo,xyz"], single_stage=True)
dvc.run(cmd="echo params.yaml", params=["foo,xyz"], single_stage=True)
scm.add(["params.yaml", "Dvcfile"])
scm.commit("bar")
+15 -5
View File
@@ -10,14 +10,24 @@ def test_show_empty(dvc):
def test_show(tmp_dir, dvc):
tmp_dir.gen("params.yaml", "foo: bar")
dvc.run(params=["foo"], single_stage=True)
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
assert dvc.params.show() == {"": {"params.yaml": {"foo": "bar"}}}
def test_show_multiple(tmp_dir, dvc):
tmp_dir.gen("params.yaml", "foo: bar\nbaz: qux\n")
dvc.run(fname="foo.dvc", params=["foo"], single_stage=True)
dvc.run(fname="baz.dvc", params=["baz"], single_stage=True)
dvc.run(
cmd="echo params.yaml",
fname="foo.dvc",
params=["foo"],
single_stage=True,
)
dvc.run(
cmd="echo params.yaml",
fname="baz.dvc",
params=["baz"],
single_stage=True,
)
assert dvc.params.show() == {
"": {"params.yaml": {"foo": "bar", "baz": "qux"}}
}
@@ -25,13 +35,13 @@ def test_show_multiple(tmp_dir, dvc):
def test_show_list(tmp_dir, dvc):
tmp_dir.gen("params.yaml", "foo:\n- bar\n- baz\n")
dvc.run(params=["foo"], single_stage=True)
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
assert dvc.params.show() == {"": {"params.yaml": {"foo": ["bar", "baz"]}}}
def test_show_branch(tmp_dir, scm, dvc):
tmp_dir.gen("params.yaml", "foo: bar")
dvc.run(params=["foo"], single_stage=True)
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
scm.add(["params.yaml", "Dvcfile"])
scm.commit("init")
+30
View File
@@ -0,0 +1,30 @@
import os
import pytest
@pytest.fixture
def run_copy_metrics(tmp_dir, run_copy):
def run(file1, file2, commit=None, tag=None, **kwargs):
stage = tmp_dir.dvc.run(
cmd=f"python copy.py {file1} {file2}",
deps=[file1],
single_stage=True,
**kwargs,
)
if hasattr(tmp_dir.dvc, "scm"):
files = [stage.path]
files += [
os.fspath(out.path_info)
for out in stage.outs
if not out.use_cache
]
tmp_dir.dvc.scm.add(files)
if commit:
tmp_dir.dvc.scm.commit(commit)
if tag:
tmp_dir.dvc.scm.tag(tag)
return stage
return run
+1
View File
@@ -0,0 +1 @@
from . import run_copy_metrics # noqa: F401
+20 -8
View File
@@ -1,20 +1,32 @@
import json
from .test_plots import PlotData, _run_with, _write_json
from .test_plots import PlotData, _write_json
def test_diff_dirty(tmp_dir, scm, dvc):
def test_diff_dirty(tmp_dir, scm, dvc, run_copy_metrics):
metric_1 = [{"y": 2}, {"y": 3}]
_write_json(tmp_dir, metric_1, "metric.json")
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init")
_write_json(tmp_dir, metric_1, "metric_t.json")
run_copy_metrics(
"metric_t.json",
"metric.json",
plots_no_cache=["metric.json"],
commit="init",
)
metric_2 = [{"y": 3}, {"y": 5}]
_write_json(tmp_dir, metric_2, "metric.json")
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="second")
_write_json(tmp_dir, metric_2, "metric_t.json")
run_copy_metrics(
"metric_t.json",
"metric.json",
plots_no_cache=["metric.json"],
commit="second",
)
metric_3 = [{"y": 5}, {"y": 6}]
_write_json(tmp_dir, metric_3, "metric.json")
_run_with(tmp_dir, plots_no_cache=["metric.json"])
_write_json(tmp_dir, metric_3, "metric_t.json")
run_copy_metrics(
"metric_t.json", "metric.json", plots_no_cache=["metric.json"]
)
plot_string = dvc.plots.diff(fields={"y"})["metric.json"]
+150 -76
View File
@@ -26,20 +26,6 @@ def _remove_whitespace(value):
return value.replace(" ", "").replace("\n", "")
def _run_with(tmp_dir, commit=None, tag=None, **kwargs):
stage = tmp_dir.dvc.run(single_stage=True, **kwargs)
if hasattr(tmp_dir.dvc, "scm"):
files = [stage.path]
files += [
os.fspath(out.path_info) for out in stage.outs if not out.use_cache
]
tmp_dir.dvc.scm.add(files)
if commit:
tmp_dir.dvc.scm.commit(commit)
if tag:
tmp_dir.dvc.scm.tag(tag)
def _write_csv(metric, filename, header=True):
with open(filename, "w", newline="") as csvobj:
if header:
@@ -59,11 +45,13 @@ def _write_json(tmp_dir, metric, filename):
tmp_dir.gen(filename, json.dumps(metric, sort_keys=True))
def test_plot_csv_one_column(tmp_dir, scm, dvc):
def test_plot_csv_one_column(tmp_dir, scm, dvc, run_copy_metrics):
# no header
metric = [{"val": 2}, {"val": 3}]
_write_csv(metric, "metric.csv", header=False)
_run_with(tmp_dir, plots_no_cache=["metric.csv"])
_write_csv(metric, "metric_t.csv", header=False)
run_copy_metrics(
"metric_t.csv", "metric.csv", plots_no_cache=["metric.csv"]
)
plot_string = dvc.plots.show(
csv_header=False,
@@ -84,13 +72,15 @@ def test_plot_csv_one_column(tmp_dir, scm, dvc):
assert plot_content["encoding"]["y"]["title"] == "y_title"
def test_plot_csv_multiple_columns(tmp_dir, scm, dvc):
def test_plot_csv_multiple_columns(tmp_dir, scm, dvc, run_copy_metrics):
metric = [
OrderedDict([("first_val", 100), ("second_val", 100), ("val", 2)]),
OrderedDict([("first_val", 200), ("second_val", 300), ("val", 3)]),
]
_write_csv(metric, "metric.csv")
_run_with(tmp_dir, plots_no_cache=["metric.csv"])
_write_csv(metric, "metric_t.csv")
run_copy_metrics(
"metric_t.csv", "metric.csv", plots_no_cache=["metric.csv"]
)
plot_string = dvc.plots.show()["metric.csv"]
@@ -115,13 +105,15 @@ def test_plot_csv_multiple_columns(tmp_dir, scm, dvc):
assert plot_content["encoding"]["y"]["field"] == "val"
def test_plot_csv_choose_axes(tmp_dir, scm, dvc):
def test_plot_csv_choose_axes(tmp_dir, scm, dvc, run_copy_metrics):
metric = [
OrderedDict([("first_val", 100), ("second_val", 100), ("val", 2)]),
OrderedDict([("first_val", 200), ("second_val", 300), ("val", 3)]),
]
_write_csv(metric, "metric.csv")
_run_with(tmp_dir, plots_no_cache=["metric.csv"])
_write_csv(metric, "metric_t.csv")
run_copy_metrics(
"metric_t.csv", "metric.csv", plots_no_cache=["metric.csv"]
)
plot_string = dvc.plots.show(x_field="first_val", y_field="second_val")[
"metric.csv"
@@ -146,10 +138,15 @@ def test_plot_csv_choose_axes(tmp_dir, scm, dvc):
assert plot_content["encoding"]["y"]["field"] == "second_val"
def test_plot_json_single_val(tmp_dir, scm, dvc):
def test_plot_json_single_val(tmp_dir, scm, dvc, run_copy_metrics):
metric = [{"val": 2}, {"val": 3}]
_write_json(tmp_dir, metric, "metric.json")
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="first run")
_write_json(tmp_dir, metric, "metric_t.json")
run_copy_metrics(
"metric_t.json",
"metric.json",
plots_no_cache=["metric.json"],
commit="first run",
)
plot_string = dvc.plots.show()["metric.json"]
@@ -162,13 +159,18 @@ def test_plot_json_single_val(tmp_dir, scm, dvc):
assert plot_json["encoding"]["y"]["field"] == "val"
def test_plot_json_multiple_val(tmp_dir, scm, dvc):
def test_plot_json_multiple_val(tmp_dir, scm, dvc, run_copy_metrics):
metric = [
{"first_val": 100, "val": 2},
{"first_val": 200, "val": 3},
]
_write_json(tmp_dir, metric, "metric.json")
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="first run")
_write_json(tmp_dir, metric, "metric_t.json")
run_copy_metrics(
"metric_t.json",
"metric.json",
plots_no_cache=["metric.json"],
commit="first run",
)
plot_string = dvc.plots.show()["metric.json"]
@@ -191,13 +193,18 @@ def test_plot_json_multiple_val(tmp_dir, scm, dvc):
assert plot_content["encoding"]["y"]["field"] == "val"
def test_plot_confusion(tmp_dir, dvc):
def test_plot_confusion(tmp_dir, dvc, run_copy_metrics):
confusion_matrix = [
{"predicted": "B", "actual": "A"},
{"predicted": "A", "actual": "A"},
]
_write_json(tmp_dir, confusion_matrix, "metric.json")
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="first run")
_write_json(tmp_dir, confusion_matrix, "metric_t.json")
run_copy_metrics(
"metric_t.json",
"metric.json",
plots_no_cache=["metric.json"],
commit="first run",
)
plot_string = dvc.plots.show(
template="confusion", x_field="predicted", y_field="actual",
@@ -212,21 +219,35 @@ def test_plot_confusion(tmp_dir, dvc):
assert plot_content["encoding"]["y"]["field"] == "actual"
def test_plot_multiple_revs_default(tmp_dir, scm, dvc):
def test_plot_multiple_revs_default(tmp_dir, scm, dvc, run_copy_metrics):
metric_1 = [{"y": 2}, {"y": 3}]
_write_json(tmp_dir, metric_1, "metric.json")
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init", tag="v1")
_write_json(tmp_dir, metric_1, "metric_t.json")
run_copy_metrics(
"metric_t.json",
"metric.json",
plots_no_cache=["metric.json"],
commit="init",
tag="v1",
)
metric_2 = [{"y": 3}, {"y": 5}]
_write_json(tmp_dir, metric_2, "metric.json")
_run_with(
tmp_dir, plots_no_cache=["metric.json"], commit="second", tag="v2"
_write_json(tmp_dir, metric_2, "metric_t.json")
run_copy_metrics(
"metric_t.json",
"metric.json",
plots_no_cache=["metric.json"],
commit="second",
tag="v2",
)
metric_3 = [{"y": 5}, {"y": 6}]
_write_json(tmp_dir, metric_3, "metric.json")
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="third")
_write_json(tmp_dir, metric_3, "metric_t.json")
run_copy_metrics(
"metric_t.json",
"metric.json",
plots_no_cache=["metric.json"],
commit="third",
)
plot_string = dvc.plots.show(fields={"y"}, revs=["HEAD", "v2", "v1"],)[
"metric.json"
]
@@ -244,22 +265,31 @@ def test_plot_multiple_revs_default(tmp_dir, scm, dvc):
assert plot_content["encoding"]["y"]["field"] == "y"
def test_plot_multiple_revs(tmp_dir, scm, dvc):
def test_plot_multiple_revs(tmp_dir, scm, dvc, run_copy_metrics):
shutil.copy(tmp_dir / ".dvc" / "plots" / "default.json", "template.json")
metric_1 = [{"y": 2}, {"y": 3}]
_write_json(tmp_dir, metric_1, "metric.json")
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init", tag="v1")
metric_2 = [{"y": 3}, {"y": 5}]
_write_json(tmp_dir, metric_2, "metric.json")
_run_with(
tmp_dir, plots_no_cache=["metric.json"], commit="second", tag="v2"
_write_json(tmp_dir, metric_1, "metric_t.json")
stage = run_copy_metrics(
"metric_t.json",
"metric.json",
plots_no_cache=["metric.json"],
commit="init",
tag="v1",
)
metric_2 = [{"y": 3}, {"y": 5}]
_write_json(tmp_dir, metric_2, "metric_t.json")
assert dvc.reproduce(stage.addressing) == [stage]
scm.add(["metric.json", stage.path])
scm.commit("second")
scm.tag("v2")
metric_3 = [{"y": 5}, {"y": 6}]
_write_json(tmp_dir, metric_3, "metric.json")
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="third")
_write_json(tmp_dir, metric_3, "metric_t.json")
assert dvc.reproduce(stage.addressing) == [stage]
scm.add(["metric.json", stage.path])
scm.commit("third")
plot_string = dvc.plots.show(
template="template.json", revs=["HEAD", "v2", "v1"],
@@ -278,14 +308,17 @@ def test_plot_multiple_revs(tmp_dir, scm, dvc):
assert plot_content["encoding"]["y"]["field"] == "y"
def test_plot_even_if_metric_missing(tmp_dir, scm, dvc, caplog):
def test_plot_even_if_metric_missing(
tmp_dir, scm, dvc, caplog, run_copy_metrics
):
tmp_dir.scm_gen("some_file", "content", commit="there is no metric")
scm.tag("v1")
metric = [{"y": 2}, {"y": 3}]
_write_json(tmp_dir, metric, "metric.json")
_run_with(
tmp_dir,
_write_json(tmp_dir, metric, "metric_t.json")
run_copy_metrics(
"metric_t.json",
"metric.json",
plots_no_cache=["metric.json"],
commit="there is metric",
tag="v2",
@@ -335,10 +368,16 @@ def custom_template(tmp_dir, dvc):
return custom_template
def test_custom_template(tmp_dir, scm, dvc, custom_template):
def test_custom_template(tmp_dir, scm, dvc, custom_template, run_copy_metrics):
metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
_write_json(tmp_dir, metric, "metric.json")
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init", tag="v1")
_write_json(tmp_dir, metric, "metric_t.json")
run_copy_metrics(
"metric_t.json",
"metric.json",
plots_no_cache=["metric.json"],
commit="init",
tag="v1",
)
plot_string = dvc.plots.show(
template=os.fspath(custom_template), x_field="a", y_field="b"
@@ -358,15 +397,21 @@ def _replace(path, src, dst):
def test_custom_template_with_specified_data(
tmp_dir, scm, dvc, custom_template
tmp_dir, scm, dvc, custom_template, run_copy_metrics
):
_replace(
custom_template, "DVC_METRIC_DATA", "DVC_METRIC_DATA,metric.json",
)
metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
_write_json(tmp_dir, metric, "metric.json")
_run_with(tmp_dir, outs_no_cache=["metric.json"], commit="init", tag="v1")
_write_json(tmp_dir, metric, "metric_t.json")
run_copy_metrics(
"metric_t.json",
"metric.json",
outs_no_cache=["metric.json"],
commit="init",
tag="v1",
)
plot_string = dvc.plots.show(
template=os.fspath(custom_template), x_field="a", y_field="b",
@@ -381,7 +426,9 @@ def test_custom_template_with_specified_data(
assert plot_content["encoding"]["y"]["field"] == "b"
def test_plot_override_specified_data_source(tmp_dir, scm, dvc):
def test_plot_override_specified_data_source(
tmp_dir, scm, dvc, run_copy_metrics
):
shutil.copy(
tmp_dir / ".dvc" / "plots" / "default.json",
tmp_dir / "newtemplate.json",
@@ -393,8 +440,14 @@ def test_plot_override_specified_data_source(tmp_dir, scm, dvc):
)
metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
_write_json(tmp_dir, metric, "metric2.json")
_run_with(tmp_dir, outs_no_cache=["metric2.json"], commit="init", tag="v1")
_write_json(tmp_dir, metric, "metric1.json")
run_copy_metrics(
"metric1.json",
"metric2.json",
outs_no_cache=["metric2.json"],
commit="init",
tag="v1",
)
plot_string = dvc.plots.show(
targets=["metric2.json"], template="newtemplate.json", x_field="a"
@@ -430,10 +483,18 @@ def test_plot_wrong_metric_type(tmp_dir, scm, dvc):
dvc.plots.show(targets=["metric.txt"])
def test_plot_choose_columns(tmp_dir, scm, dvc, custom_template):
def test_plot_choose_columns(
tmp_dir, scm, dvc, custom_template, run_copy_metrics
):
metric = [{"a": 1, "b": 2, "c": 3}, {"a": 2, "b": 3, "c": 4}]
_write_json(tmp_dir, metric, "metric.json")
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init", tag="v1")
_write_json(tmp_dir, metric, "metric_t.json")
run_copy_metrics(
"metric_t.json",
"metric.json",
plots_no_cache=["metric.json"],
commit="init",
tag="v1",
)
plot_string = dvc.plots.show(
template=os.fspath(custom_template),
@@ -451,10 +512,16 @@ def test_plot_choose_columns(tmp_dir, scm, dvc, custom_template):
assert plot_content["encoding"]["y"]["field"] == "c"
def test_plot_default_choose_column(tmp_dir, scm, dvc):
def test_plot_default_choose_column(tmp_dir, scm, dvc, run_copy_metrics):
metric = [{"a": 1, "b": 2, "c": 3}, {"a": 2, "b": 3, "c": 4}]
_write_json(tmp_dir, metric, "metric.json")
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init", tag="v1")
_write_json(tmp_dir, metric, "metric_t.json")
run_copy_metrics(
"metric_t.json",
"metric.json",
plots_no_cache=["metric.json"],
commit="init",
tag="v1",
)
plot_string = dvc.plots.show(fields={"b"})["metric.json"]
@@ -467,12 +534,14 @@ def test_plot_default_choose_column(tmp_dir, scm, dvc):
assert plot_content["encoding"]["y"]["field"] == "b"
def test_plot_yaml(tmp_dir, scm, dvc):
def test_plot_yaml(tmp_dir, scm, dvc, run_copy_metrics):
metric = [{"val": 2}, {"val": 3}]
with open("metric.yaml", "w") as fobj:
with open("metric_t.yaml", "w") as fobj:
yaml.dump(metric, fobj)
_run_with(tmp_dir, plots_no_cache=["metric.yaml"])
run_copy_metrics(
"metric_t.yaml", "metric.yaml", plots_no_cache=["metric.yaml"]
)
plot_string = dvc.plots.show()["metric.yaml"]
@@ -483,10 +552,15 @@ def test_plot_yaml(tmp_dir, scm, dvc):
]
def test_raise_on_wrong_field(tmp_dir, scm, dvc):
def test_raise_on_wrong_field(tmp_dir, scm, dvc, run_copy_metrics):
metric = [{"val": 2}, {"val": 3}]
_write_json(tmp_dir, metric, "metric.json")
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="first run")
_write_json(tmp_dir, metric, "metric_t.json")
run_copy_metrics(
"metric_t.json",
"metric.json",
plots_no_cache=["metric.json"],
commit="first run",
)
with pytest.raises(NoFieldInDataError):
dvc.plots.show("metric.json", x_field="no_val")
+60 -11
View File
@@ -7,22 +7,20 @@ import shutil
import pytest
from mock import patch
from dvc.dvcfile import DVC_FILE_SUFFIX, Dvcfile
from dvc.dvcfile import DVC_FILE_SUFFIX, PIPELINE_FILE, Dvcfile
from dvc.exceptions import (
CheckoutError,
CheckoutErrorSuggestGit,
ConfirmRemoveError,
DvcException,
NoOutputOrStageError,
)
from dvc.main import main
from dvc.remote import S3Remote
from dvc.remote.local import LocalRemote
from dvc.repo import Repo as DvcRepo
from dvc.stage import Stage
from dvc.stage.exceptions import (
StageFileBadNameError,
StageFileDoesNotExistError,
)
from dvc.stage.exceptions import StageFileDoesNotExistError
from dvc.system import System
from dvc.utils import relpath
from dvc.utils.fs import walk_files
@@ -403,17 +401,26 @@ class TestCheckoutSuggestGit(TestRepro):
def test(self):
try:
self.dvc.checkout(targets=["gitbranch"])
self.dvc.checkout(targets="gitbranch")
except DvcException as exc:
self.assertIsInstance(exc, CheckoutErrorSuggestGit)
self.assertIsInstance(exc.__cause__, StageFileDoesNotExistError)
self.assertIsInstance(exc.__cause__, NoOutputOrStageError)
self.assertIsInstance(
exc.__cause__.__cause__, StageFileDoesNotExistError
)
try:
self.dvc.checkout(targets=self.FOO)
except DvcException as exc:
self.assertIsInstance(exc, CheckoutErrorSuggestGit)
self.assertIsInstance(exc.__cause__, NoOutputOrStageError)
self.assertIsNone(exc.__cause__.__cause__)
try:
self.dvc.checkout(targets=[self.FOO])
self.dvc.checkout(targets="looks-like-dvcfile.dvc")
except DvcException as exc:
self.assertIsInstance(exc, CheckoutErrorSuggestGit)
self.assertIsInstance(exc.__cause__, StageFileBadNameError)
self.assertIsInstance(exc.__cause__, StageFileDoesNotExistError)
self.assertIsNone(exc.__cause__.__cause__)
@@ -770,9 +777,51 @@ def test_checkout_for_external_outputs(tmp_dir, dvc):
assert stats == {**empty_checkout, "modified": [str(file_path)]}
def test_checkouts_for_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
from dvc.dvcfile import PIPELINE_FILE
def test_checkouts_with_different_addressing(tmp_dir, dvc, run_copy):
tmp_dir.gen({"foo": "foo", "lorem": "lorem"})
run_copy("foo", "bar", name="copy-foo-bar")
run_copy("lorem", "ipsum", name="copy-lorem-ipsum")
(tmp_dir / "bar").unlink()
(tmp_dir / "ipsum").unlink()
assert set(dvc.checkout(PIPELINE_FILE)["added"]) == {"bar", "ipsum"}
(tmp_dir / "bar").unlink()
(tmp_dir / "ipsum").unlink()
assert set(dvc.checkout(":")["added"]) == {"bar", "ipsum"}
(tmp_dir / "bar").unlink()
assert dvc.checkout("copy-foo-bar")["added"] == ["bar"]
(tmp_dir / "bar").unlink()
assert dvc.checkout("dvc.yaml:copy-foo-bar")["added"] == ["bar"]
(tmp_dir / "bar").unlink()
assert dvc.checkout(":copy-foo-bar")["added"] == ["bar"]
(tmp_dir / "bar").unlink()
(tmp_dir / "data").mkdir()
with (tmp_dir / "data").chdir():
assert dvc.checkout(relpath(tmp_dir / "dvc.yaml") + ":copy-foo-bar")[
"added"
] == [relpath(tmp_dir / "bar")]
(tmp_dir / "bar").unlink()
assert dvc.checkout("bar")["added"] == ["bar"]
def test_checkouts_on_same_stage_name_and_output_name(tmp_dir, dvc, run_copy):
tmp_dir.gen("foo", "foo")
run_copy("foo", "bar", name="copy-foo-bar")
run_copy("foo", "copy-foo-bar", name="make_collision")
(tmp_dir / "bar").unlink()
(tmp_dir / "copy-foo-bar").unlink()
assert dvc.checkout("copy-foo-bar")["added"] == ["bar"]
assert dvc.checkout("./copy-foo-bar")["added"] == ["copy-foo-bar"]
def test_checkouts_for_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
tmp_dir.gen("foo", "foo")
stage1 = run_copy("foo", "bar", name="copy-foo-bar")
tmp_dir.gen("lorem", "lorem")
+14
View File
@@ -1,5 +1,6 @@
import pytest
from dvc.dvcfile import PIPELINE_FILE
from dvc.stage.exceptions import StageCommitError
from dvc.utils.stage import dump_stage_file, load_stage_file
@@ -81,3 +82,16 @@ def test_commit_no_exec(tmp_dir, dvc):
assert dvc.status(stage.path)
dvc.commit(stage.path, force=True)
assert dvc.status(stage.path) == {}
def test_commit_pipeline_stage(tmp_dir, dvc, run_copy):
tmp_dir.gen("foo", "foo")
stage = run_copy("foo", "bar", no_commit=True, name="copy-foo-bar")
assert dvc.status(stage.addressing)
assert dvc.commit(stage.addressing, force=True) == [stage]
assert not dvc.status(stage.addressing)
# just to confirm different variants work
assert dvc.commit(f":{stage.addressing}") == [stage]
assert dvc.commit(f"{PIPELINE_FILE}:{stage.addressing}") == [stage]
assert dvc.commit(PIPELINE_FILE) == [stage]
+41
View File
@@ -912,3 +912,44 @@ def test_pull_stats(tmp_dir, dvc, caplog, setup_remote):
with caplog.at_level(level=logging.INFO, logger="dvc"):
main(["pull"])
assert "Everything is up to date." in caplog.text
@pytest.mark.parametrize(
"key,expected", [("all_tags", 2), ("all_branches", 3), ("all_commits", 3)]
)
def test_push_pull_all(tmp_dir, scm, dvc, setup_remote, key, expected):
setup_remote(dvc)
tmp_dir.dvc_gen({"foo": "foo"}, commit="first")
scm.tag("v1")
dvc.remove("foo.dvc")
tmp_dir.dvc_gen({"bar": "bar"}, commit="second")
scm.tag("v2")
with tmp_dir.branch("branch", new=True):
dvc.remove("bar.dvc")
tmp_dir.dvc_gen({"baz": "baz"}, commit="branch")
assert dvc.push(**{key: True}) == expected
clean(["foo", "bar", "baz"], dvc)
assert dvc.pull(**{key: True})["fetched"] == expected
def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, setup_remote):
remote_path = setup_remote(dvc)
tmp_dir.dvc_gen("foo", "foo")
run_copy("foo", "bar", no_commit=True, name="copy-foo-bar")
dvc.push("copy-foo-bar")
assert len(recurse_list_dir(remote_path)) == 1
# pushing everything so as we can check pull/fetch only downloads
# from specified targets
dvc.push()
clean(["foo", "bar"], dvc)
dvc.pull("copy-foo-bar")
assert (tmp_dir / "bar").exists()
assert len(recurse_list_dir(dvc.cache.local.cache_dir)) == 1
clean(["bar"], dvc)
dvc.fetch("copy-foo-bar")
assert len(recurse_list_dir(dvc.cache.local.cache_dir)) == 1
+1 -38
View File
@@ -1,10 +1,7 @@
import pytest
from dvc.dvcfile import PIPELINE_FILE, Dvcfile
from dvc.stage.exceptions import (
StageFileDoesNotExistError,
StageNameUnspecified,
)
from dvc.stage.exceptions import StageFileDoesNotExistError
from dvc.stage.loader import StageNotFound
@@ -163,37 +160,3 @@ def test_stage_collection(tmp_dir, dvc):
single_stage=True,
)
assert {s for s in dvc.stages} == {stage1, stage3, stage2}
def test_stage_filter(tmp_dir, dvc, run_copy):
tmp_dir.gen("foo", "foo")
stage1 = run_copy("foo", "bar", name="copy-foo-bar")
stage2 = run_copy("bar", "bax", name="copy-bar-bax")
stage3 = run_copy("bax", "baz", name="copy-bax-baz")
stages = Dvcfile(dvc, PIPELINE_FILE).stages
assert set(stages.filter(None).values()) == {
stage1,
stage2,
stage3,
}
assert set(stages.filter("copy-bar-bax").values()) == {stage2}
assert stages.filter("copy-bar-bax").get("copy-bar-bax") == stage2
with pytest.raises(StageNameUnspecified):
stages.get(None)
with pytest.raises(StageNotFound):
assert stages["unknown"]
with pytest.raises(StageNotFound):
assert stages.filter("unknown")
def test_stage_filter_in_singlestage_file(tmp_dir, dvc, run_copy):
tmp_dir.gen("foo", "foo")
stage = run_copy("foo", "bar", single_stage=True)
dvcfile = Dvcfile(dvc, stage.path)
assert set(dvcfile.stages.filter(None).values()) == {stage}
assert dvcfile.stages.filter(None).get(None) == stage
assert dvcfile.stages.filter("copy-bar-bax").get("copy-bar-bax") == stage
assert dvcfile.stages.filter("copy-bar-bax").get(None) == stage
assert set(dvcfile.stages.filter("unknown").values()) == {stage}
+9 -6
View File
@@ -1,5 +1,6 @@
import os
import shutil
import textwrap
import pytest
@@ -29,17 +30,19 @@ def match_files(files, expected_files):
def create_dvc_pipeline(tmp_dir, dvc):
script = os.linesep.join(
[
"from pathlib import Path",
"Path({}).touch()".format(os.path.join("out", "file")),
]
script = textwrap.dedent(
"""\
import os, sys
f = sys.argv[1]
os.makedirs(os.path.dirname(f))
open(f, "w+").close()
"""
)
tmp_dir.scm_gen({"script.py": script}, commit="init")
tmp_dir.dvc_gen({"dep": "content"}, commit="init dvc")
dvc.run(
**{
"command": "python script.py",
"cmd": "python script.py {}".format(os.path.join("out", "file")),
"outs": [os.path.join("out", "file")],
"deps": ["dep"],
"fname": "out.dvc",
+8 -8
View File
@@ -296,20 +296,20 @@ def test_pipeline_list_show_multistage(tmp_dir, dvc, run_copy, caplog):
with caplog.at_level(logging.INFO, "dvc"):
command._show("foobar.dvc", False, False, False)
output = caplog.text.splitlines()
assert "dvc.yaml:copy-foo-bar" in output[0]
assert "copy-foo-bar" in output[0]
assert "foobar.dvc" in output[1]
caplog.clear()
with caplog.at_level(logging.INFO, "dvc"):
command._show("dvc.yaml:copy-foo-bar", False, False, False)
assert "dvc.yaml:copy-foo-bar" in caplog.text
command._show("copy-foo-bar", False, False, False)
assert "copy-foo-bar" in caplog.text
assert "foobar.dvc" not in caplog.text
command = CmdPipelineList([])
caplog.clear()
with caplog.at_level(logging.INFO, "dvc"):
command.run()
assert "dvc.yaml:copy-foo-bar" in caplog.text
assert "copy-foo-bar" in caplog.text
assert "foobar.dvc" in caplog.text
assert "1 pipelines in total"
@@ -320,13 +320,13 @@ def test_pipeline_ascii_multistage(tmp_dir, dvc, run_copy):
run_copy("bar", "foobar", single_stage=True)
command = CmdPipelineShow([])
nodes, edges, _ = command._build_graph("foobar.dvc")
assert set(nodes) == {"dvc.yaml:copy-foo-bar", "foobar.dvc"}
assert set(nodes) == {"copy-foo-bar", "foobar.dvc"}
assert set(edges) == {
("foobar.dvc", "dvc.yaml:copy-foo-bar"),
("foobar.dvc", "copy-foo-bar"),
}
nodes, *_ = command._build_graph("dvc.yaml:copy-foo-bar")
assert set(nodes) == {"dvc.yaml:copy-foo-bar"}
nodes, *_ = command._build_graph("copy-foo-bar")
assert set(nodes) == {"copy-foo-bar"}
def test_pipeline_multi_outputs_stages(dvc):
+5 -1
View File
@@ -239,7 +239,11 @@ def test_external_dir_resource_on_no_cache(tmp_dir, dvc, tmp_path_factory):
dvc.cache.local = None
with pytest.raises(RemoteCacheRequiredError):
dvc.run(deps=[os.fspath(external_dir)], single_stage=True)
dvc.run(
cmd="echo hello world",
deps=[os.fspath(external_dir)],
single_stage=True,
)
def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, setup_remote):
+251 -2
View File
@@ -1,13 +1,24 @@
import os
from operator import itemgetter
import pytest
from dvc.cache import Cache
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
from dvc.exceptions import NoOutputOrStageError
from dvc.path_info import PathInfo
from dvc.repo import Repo
from dvc.stage.exceptions import (
StageFileDoesNotExistError,
StageNameUnspecified,
StageNotFound,
)
from dvc.system import System
from dvc.utils import relpath
from dvc.utils.fs import remove
def test_destroy(tmp_dir, dvc, run_copy):
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
dvc.config["cache"]["type"] = ["symlink"]
dvc.cache = Cache(dvc)
@@ -88,6 +99,12 @@ def test_collect(tmp_dir, scm, dvc, run_copy):
assert collect_outs("dvc.yaml:copy-foo-foobar", recursive=True) == {
"foobar"
}
assert collect_outs("copy-foo-foobar") == {"foobar"}
assert collect_outs("copy-foo-foobar", with_deps=True) == {
"foobar",
"foo",
}
assert collect_outs("copy-foo-foobar", recursive=True) == {"foobar"}
run_copy("foobar", "baz", name="copy-foobar-baz")
assert collect_outs("dvc.yaml") == {"foobar", "baz"}
@@ -98,6 +115,26 @@ def test_collect(tmp_dir, scm, dvc, run_copy):
}
def test_collect_dir_recursive(tmp_dir, dvc, run_head):
tmp_dir.gen({"dir": {"foo": "foo"}})
(stage1,) = dvc.add("dir", recursive=True)
with (tmp_dir / "dir").chdir():
stage2 = run_head("foo", name="copy-foo-bar")
stage3 = run_head("foo-1", single_stage=True)
assert set(dvc.collect("dir", recursive=True)) == {stage1, stage2, stage3}
def test_collect_with_not_existing_output_or_stage_name(
tmp_dir, dvc, run_copy
):
with pytest.raises(StageFileDoesNotExistError):
dvc.collect("some_file")
tmp_dir.dvc_gen("foo", "foo")
run_copy("foo", "bar", name="copy-foo-bar")
with pytest.raises(StageNotFound):
dvc.collect("some_file")
def test_stages(tmp_dir, dvc):
def stages():
return {stage.relpath for stage in Repo(os.fspath(tmp_dir)).stages}
@@ -113,3 +150,215 @@ def test_stages(tmp_dir, dvc):
tmp_dir.gen(".dvcignore", "dir")
assert stages() == {"file.dvc"}
@pytest.fixture
def stages(tmp_dir, run_copy):
stage1, stage2 = tmp_dir.dvc_gen({"foo": "foo", "lorem": "lorem"})
return {
"foo-generate": stage1,
"lorem-generate": stage2,
"copy-foo-bar": run_copy("foo", "bar", single_stage=True),
"copy-bar-foobar": run_copy("bar", "foobar", name="copy-bar-foobar"),
"copy-lorem-ipsum": run_copy("lorem", "ipsum", name="lorem-ipsum"),
}
def test_collect_granular_with_no_target(tmp_dir, dvc, stages):
assert set(map(itemgetter(0), dvc.collect_granular())) == set(
stages.values()
)
assert list(map(itemgetter(1), dvc.collect_granular())) == [None] * len(
stages
)
def test_collect_granular_with_target(tmp_dir, dvc, stages):
assert dvc.collect_granular("bar.dvc") == [(stages["copy-foo-bar"], None)]
assert dvc.collect_granular(PIPELINE_FILE) == [
(stages["copy-bar-foobar"], None),
(stages["copy-lorem-ipsum"], None),
]
assert dvc.collect_granular(":") == [
(stages["copy-bar-foobar"], None),
(stages["copy-lorem-ipsum"], None),
]
assert dvc.collect_granular("copy-bar-foobar") == [
(stages["copy-bar-foobar"], None)
]
assert dvc.collect_granular(":copy-bar-foobar") == [
(stages["copy-bar-foobar"], None)
]
assert dvc.collect_granular("dvc.yaml:copy-bar-foobar") == [
(stages["copy-bar-foobar"], None)
]
with (tmp_dir / dvc.DVC_DIR).chdir():
assert dvc.collect_granular(
relpath(tmp_dir / PIPELINE_FILE) + ":copy-bar-foobar"
) == [(stages["copy-bar-foobar"], None)]
assert dvc.collect_granular("foobar") == [
(stages["copy-bar-foobar"], PathInfo(tmp_dir / "foobar"))
]
@pytest.mark.parametrize(
"target",
[
"not_existing.dvc",
"not_existing.dvc:stage_name",
"not_existing/dvc.yaml",
"not_existing/dvc.yaml:stage_name",
],
)
def test_collect_with_not_existing_dvcfile(tmp_dir, dvc, target):
with pytest.raises(StageFileDoesNotExistError):
dvc.collect_granular(target)
with pytest.raises(StageFileDoesNotExistError):
dvc.collect(target)
def test_collect_granular_with_not_existing_output_or_stage_name(tmp_dir, dvc):
with pytest.raises(NoOutputOrStageError):
dvc.collect_granular("some_file")
with pytest.raises(NoOutputOrStageError):
dvc.collect_granular("some_file", recursive=True)
def test_collect_granular_with_deps(tmp_dir, dvc, stages):
assert set(
map(itemgetter(0), dvc.collect_granular("bar.dvc", with_deps=True))
) == {stages["copy-foo-bar"], stages["foo-generate"]}
assert set(
map(
itemgetter(0),
dvc.collect_granular("copy-bar-foobar", with_deps=True),
)
) == {
stages["copy-bar-foobar"],
stages["copy-foo-bar"],
stages["foo-generate"],
}
assert set(
map(
itemgetter(0), dvc.collect_granular(PIPELINE_FILE, with_deps=True),
)
) == set(stages.values())
def test_collect_granular_same_output_name_stage_name(tmp_dir, dvc, run_copy):
(stage1,) = tmp_dir.dvc_gen("foo", "foo")
(stage2,) = tmp_dir.dvc_gen("copy-foo-bar", "copy-foo-bar")
stage3 = run_copy("foo", "bar", name="copy-foo-bar")
assert dvc.collect_granular("copy-foo-bar") == [(stage3, None)]
coll = dvc.collect_granular("copy-foo-bar", with_deps=True)
assert set(map(itemgetter(0), coll)) == {stage3, stage1}
assert list(map(itemgetter(1), coll)) == [None] * 2
assert dvc.collect_granular("./copy-foo-bar") == [
(stage2, PathInfo(tmp_dir / "copy-foo-bar"))
]
assert dvc.collect_granular("./copy-foo-bar", with_deps=True) == [
(stage2, PathInfo(tmp_dir / "copy-foo-bar"))
]
def test_collect_granular_priority_on_collision(tmp_dir, dvc, run_copy):
tmp_dir.gen({"dir": {"foo": "foo"}, "foo": "foo"})
(stage1,) = dvc.add("dir", recursive=True)
stage2 = run_copy("foo", "bar", name="dir")
assert dvc.collect_granular("dir") == [(stage2, None)]
assert dvc.collect_granular("dir", recursive=True) == [(stage1, None)]
remove(tmp_dir / "dir")
assert dvc.collect_granular("dir") == [(stage2, None)]
assert dvc.collect_granular("dir", recursive=True) == [(stage2, None)]
def test_collect_granular_collision_output_dir_stage_name(
tmp_dir, dvc, run_copy
):
stage1, *_ = tmp_dir.dvc_gen({"dir": {"foo": "foo"}, "foo": "foo"})
stage3 = run_copy("foo", "bar", name="dir")
assert dvc.collect_granular("dir") == [(stage3, None)]
assert not dvc.collect_granular("dir", recursive=True)
assert dvc.collect_granular("./dir") == [
(stage1, PathInfo(tmp_dir / "dir"))
]
def test_collect_granular_not_existing_stage_name(tmp_dir, dvc, run_copy):
tmp_dir.dvc_gen("foo", "foo")
(stage,) = tmp_dir.dvc_gen("copy-foo-bar", "copy-foo-bar")
run_copy("foo", "bar", name="copy-foo-bar")
assert dvc.collect_granular("copy-foo-bar.dvc:stage_name_not_needed") == [
(stage, None)
]
with pytest.raises(StageNotFound):
dvc.collect_granular("dvc.yaml:does-not-exist")
def test_get_stages(tmp_dir, dvc, run_copy):
with pytest.raises(StageFileDoesNotExistError):
dvc.get_stages()
tmp_dir.gen("foo", "foo")
stage1 = run_copy("foo", "bar", name="copy-foo-bar")
stage2 = run_copy("bar", "foobar", name="copy-bar-foobar")
assert set(dvc.get_stages()) == {stage1, stage2}
assert set(dvc.get_stages(path=PIPELINE_FILE)) == {stage1, stage2}
assert set(dvc.get_stages(name="copy-bar-foobar")) == {stage2}
assert set(dvc.get_stages(path=PIPELINE_FILE, name="copy-bar-foobar")) == {
stage2
}
with pytest.raises(StageFileDoesNotExistError):
dvc.get_stages(path=relpath(tmp_dir / ".." / PIPELINE_FILE))
with pytest.raises(StageNotFound):
dvc.get_stages(path=PIPELINE_FILE, name="copy")
def test_get_stages_old_dvcfile(tmp_dir, dvc):
(stage1,) = tmp_dir.dvc_gen("foo", "foo")
assert set(dvc.get_stages("foo.dvc")) == {stage1}
assert set(dvc.get_stages("foo.dvc", name="foo-generate")) == {stage1}
with pytest.raises(StageFileDoesNotExistError):
dvc.get_stages(path=relpath(tmp_dir / ".." / "foo.dvc"))
def test_get_stage(tmp_dir, dvc, run_copy):
tmp_dir.gen("foo", "foo")
stage1 = run_copy("foo", "bar", name="copy-foo-bar")
with pytest.raises(StageNameUnspecified):
dvc.get_stage()
with pytest.raises(StageNameUnspecified):
dvc.get_stage(path=PIPELINE_FILE)
assert dvc.get_stage(path=PIPELINE_FILE, name="copy-foo-bar") == stage1
assert dvc.get_stage(name="copy-foo-bar") == stage1
with pytest.raises(StageFileDoesNotExistError):
dvc.get_stage(path="something.yaml", name="name")
with pytest.raises(StageNotFound):
dvc.get_stage(name="random_name")
def test_get_stage_single_stage_dvcfile(tmp_dir, dvc):
(stage1,) = tmp_dir.dvc_gen("foo", "foo")
assert dvc.get_stage("foo.dvc") == stage1
assert dvc.get_stage("foo.dvc", name="jpt") == stage1
with pytest.raises(StageFileDoesNotExistError):
dvc.get_stage(path="bar.dvc", name="name")
+18 -4
View File
@@ -56,7 +56,7 @@ class SingleStageRun:
@staticmethod
def _get_stage_target(stage):
return stage.path
return stage.addressing
class TestRepro(SingleStageRun, TestDvc):
@@ -264,9 +264,11 @@ class TestReproDepDirWithOutputsUnderIt(SingleStageRun, TestDvc):
self.assertEqual(len(stages), 1)
self.assertTrue(stages[0] is not None)
deps = [self.DATA, self.DATA_SUB]
stage = self.dvc.run(
cmd="ls {}".format(" ".join(deps)),
fname="dvcfile2.dvc",
deps=[self.DATA, self.DATA_SUB],
deps=deps,
single_stage=True,
)
self.assertTrue(stage is not None)
@@ -405,7 +407,17 @@ class TestReproDryNoExec(TestDvc):
self.assertEqual(ret, 0)
ret = main(
["run", "--no-exec", "--single-stage", "-f", DVC_FILE] + deps
[
"run",
"--no-exec",
"--single-stage",
"-f",
DVC_FILE,
*deps,
"ls {}".format(
" ".join(dep for i, dep in enumerate(deps) if i % 2)
),
]
)
self.assertEqual(ret, 0)
@@ -1465,9 +1477,11 @@ def repro_dir(tmp_dir, dvc, run_copy):
stages["dir_file_copy"] = stage
last_stage = tmp_dir / "dir" / DVC_FILE
deps = [os.fspath(origin_copy_2), os.fspath(dir_file_copy)]
stage = dvc.run(
cmd="echo {}".format(" ".join(deps)),
fname=os.fspath(last_stage),
deps=[os.fspath(origin_copy_2), os.fspath(dir_file_copy)],
deps=deps,
single_stage=True,
)
assert stage is not None
+3 -5
View File
@@ -32,7 +32,7 @@ class MultiStageRun:
@staticmethod
def _get_stage_target(stage):
return stage.path + ":" + stage.name
return stage.addressing
class TestReproFailMultiStage(MultiStageRun, test_repro.TestReproFail):
@@ -312,15 +312,13 @@ def test_repro_when_cmd_changes(tmp_dir, dvc, run_copy):
tmp_dir.gen("foo", "foo")
stage = run_copy("foo", "bar", name="copy-file")
target = ":copy-file"
target = "copy-file"
assert not dvc.reproduce(target)
stage.cmd = " ".join(stage.cmd.split()) # change cmd spacing by two
PipelineFile(dvc, PIPELINE_FILE)._dump_pipeline_file(stage)
assert dvc.status([target]) == {
PIPELINE_FILE + target: ["changed command"]
}
assert dvc.status([target]) == {target: ["changed command"]}
assert dvc.reproduce(target)[0] == stage
+15
View File
@@ -3,6 +3,8 @@ import os
import pytest
import yaml
from dvc.exceptions import InvalidArgumentError
from dvc.repo import Repo
from dvc.stage.exceptions import DuplicateStageName, InvalidStageName
@@ -299,3 +301,16 @@ def test_run_params_no_exec(tmp_dir, dvc):
assert data["stages"]["read_params"]["params"] == [
{"params2.yaml": ["lists"]}
]
@pytest.mark.parametrize(
"kwargs",
[
{"outs": ["foo"], "deps": ["bar"]},
{"outs": ["foo"], "deps": ["bar"], "name": "copy-foo-bar"},
],
)
def test_run_without_cmd(kwargs):
with pytest.raises(InvalidArgumentError) as exc:
Repo().run(**kwargs)
assert "command is not specified" == str(exc.value)
+30 -21
View File
@@ -76,7 +76,7 @@ class TestRun(TestDvc):
class TestRunEmpty(TestDvc):
def test(self):
self.dvc.run(
cmd="",
cmd="echo hello world",
deps=[],
outs=[],
outs_no_cache=[],
@@ -91,7 +91,7 @@ class TestRunMissingDep(TestDvc):
with self.assertRaises(DependencyDoesNotExistError):
self.dvc.run(
cmd="",
cmd="command",
deps=["non-existing-dep"],
outs=[],
outs_no_cache=[],
@@ -118,7 +118,7 @@ class TestRunCircularDependency(TestDvc):
def test(self):
with self.assertRaises(CircularDependencyError):
self.dvc.run(
cmd="",
cmd="command",
deps=[self.FOO],
outs=[self.FOO],
fname="circular-dependency.dvc",
@@ -128,7 +128,7 @@ class TestRunCircularDependency(TestDvc):
def test_outs_no_cache(self):
with self.assertRaises(CircularDependencyError):
self.dvc.run(
cmd="",
cmd="command",
deps=[self.FOO],
outs_no_cache=[self.FOO],
fname="circular-dependency.dvc",
@@ -138,7 +138,7 @@ class TestRunCircularDependency(TestDvc):
def test_non_normalized_paths(self):
with self.assertRaises(CircularDependencyError):
self.dvc.run(
cmd="",
cmd="command",
deps=["./foo"],
outs=["foo"],
fname="circular-dependency.dvc",
@@ -173,7 +173,7 @@ class TestRunDuplicatedArguments(TestDvc):
def test(self):
with self.assertRaises(ArgumentDuplicationError):
self.dvc.run(
cmd="",
cmd="command",
deps=[],
outs=[self.FOO, self.FOO],
fname="circular-dependency.dvc",
@@ -183,7 +183,7 @@ class TestRunDuplicatedArguments(TestDvc):
def test_outs_no_cache(self):
with self.assertRaises(ArgumentDuplicationError):
self.dvc.run(
cmd="",
cmd="command",
outs=[self.FOO],
outs_no_cache=[self.FOO],
fname="circular-dependency.dvc",
@@ -193,7 +193,7 @@ class TestRunDuplicatedArguments(TestDvc):
def test_non_normalized_paths(self):
with self.assertRaises(ArgumentDuplicationError):
self.dvc.run(
cmd="",
cmd="command",
deps=[],
outs=["foo", "./foo"],
fname="circular-dependency.dvc",
@@ -204,22 +204,30 @@ class TestRunDuplicatedArguments(TestDvc):
class TestRunStageInsideOutput(TestDvc):
def test_cwd(self):
self.dvc.run(
cmd="", deps=[], outs=[self.DATA_DIR], single_stage=True,
cmd=f"mkdir {self.DATA_DIR}",
deps=[],
outs=[self.DATA_DIR],
single_stage=True,
)
with self.assertRaises(StagePathAsOutputError):
self.dvc.run(
cmd="",
cmd="command",
fname=os.path.join(self.DATA_DIR, "inside-cwd.dvc"),
single_stage=True,
)
def test_file_name(self):
self.dvc.run(cmd="", deps=[], outs=[self.DATA_DIR], single_stage=True)
self.dvc.run(
cmd=f"mkdir {self.DATA_DIR}",
deps=[],
outs=[self.DATA_DIR],
single_stage=True,
)
with self.assertRaises(StagePathAsOutputError):
self.dvc.run(
cmd="",
cmd="command",
outs=[self.FOO],
fname=os.path.join(self.DATA_DIR, "inside-cwd.dvc"),
single_stage=True,
@@ -229,30 +237,30 @@ class TestRunStageInsideOutput(TestDvc):
class TestRunBadCwd(TestDvc):
def test(self):
with self.assertRaises(StagePathOutsideError):
self.dvc.run(cmd="", wdir=self.mkdtemp(), single_stage=True)
self.dvc.run(cmd="command", wdir=self.mkdtemp(), single_stage=True)
def test_same_prefix(self):
with self.assertRaises(StagePathOutsideError):
path = f"{self._root_dir}-{uuid.uuid4()}"
os.mkdir(path)
self.dvc.run(cmd="", wdir=path, single_stage=True)
self.dvc.run(cmd="command", wdir=path, single_stage=True)
class TestRunBadWdir(TestDvc):
def test(self):
with self.assertRaises(StagePathOutsideError):
self.dvc.run(cmd="", wdir=self.mkdtemp(), single_stage=True)
self.dvc.run(cmd="command", wdir=self.mkdtemp(), single_stage=True)
def test_same_prefix(self):
with self.assertRaises(StagePathOutsideError):
path = f"{self._root_dir}-{uuid.uuid4()}"
os.mkdir(path)
self.dvc.run(cmd="", wdir=path, single_stage=True)
self.dvc.run(cmd="command", wdir=path, single_stage=True)
def test_not_found(self):
with self.assertRaises(StagePathNotFoundError):
path = os.path.join(self._root_dir, str(uuid.uuid4()))
self.dvc.run(cmd="", wdir=path, single_stage=True)
self.dvc.run(cmd="command", wdir=path, single_stage=True)
def test_not_dir(self):
with self.assertRaises(StagePathNotDirectoryError):
@@ -261,7 +269,7 @@ class TestRunBadWdir(TestDvc):
path = os.path.join(path, str(uuid.uuid4()))
open(path, "a").close()
self.dvc.run(
cmd="", wdir=path, single_stage=True,
cmd="command", wdir=path, single_stage=True,
)
@@ -269,7 +277,7 @@ class TestRunBadName(TestDvc):
def test(self):
with self.assertRaises(StagePathOutsideError):
self.dvc.run(
cmd="",
cmd="command",
fname=os.path.join(self.mkdtemp(), self.FOO + DVC_FILE_SUFFIX),
single_stage=True,
)
@@ -279,7 +287,7 @@ class TestRunBadName(TestDvc):
path = f"{self._root_dir}-{uuid.uuid4()}"
os.mkdir(path)
self.dvc.run(
cmd="",
cmd="command",
fname=os.path.join(path, self.FOO + DVC_FILE_SUFFIX),
single_stage=True,
)
@@ -288,7 +296,7 @@ class TestRunBadName(TestDvc):
with self.assertRaises(StagePathNotFoundError):
path = os.path.join(self._root_dir, str(uuid.uuid4()))
self.dvc.run(
cmd="",
cmd="command",
fname=os.path.join(path, self.FOO + DVC_FILE_SUFFIX),
single_stage=True,
)
@@ -574,6 +582,7 @@ class TestCmdRunOverwrite(TestDvc):
"out.dvc",
"-d",
self.BAR,
f"cat {self.BAR}",
]
)
self.assertEqual(ret, 0)
+37 -8
View File
@@ -8,7 +8,7 @@ from dvc.main import main
from dvc.output.local import LocalOutput
from dvc.remote.local import LocalRemote
from dvc.repo import Repo
from dvc.stage import Stage
from dvc.stage import PipelineStage, Stage
from dvc.stage.exceptions import StageFileFormatError
from dvc.utils.stage import dump_stage_file, load_stage_file
from tests.basic_env import TestDvc
@@ -182,33 +182,62 @@ def test_meta_is_preserved(tmp_dir, dvc):
def test_parent_repo_collect_stages(tmp_dir, scm, dvc):
tmp_dir.gen({"subdir": {}})
tmp_dir.gen({"deep": {"dir": {}}})
subrepo_dir = tmp_dir / "subdir"
deep_subrepo_dir = tmp_dir / "deep" / "dir"
with subrepo_dir.chdir():
subrepo = Repo.init(subdir=True)
subrepo_dir.gen("subrepo_file", "subrepo file content")
subrepo.add("subrepo_file")
with deep_subrepo_dir.chdir():
deep_subrepo = Repo.init(subdir=True)
deep_subrepo_dir.gen("subrepo_file", "subrepo file content")
deep_subrepo.add("subrepo_file")
stages = dvc.collect(None)
subrepo_stages = subrepo.collect(None)
deep_subrepo_stages = deep_subrepo.collect(None)
assert stages == []
assert subrepo_stages != []
assert deep_subrepo_stages != []
def test_stage_addressing(tmp_dir, dvc, run_copy):
def test_stage_strings_representation(tmp_dir, dvc, run_copy):
tmp_dir.dvc_gen("foo", "foo")
stage1 = run_copy("foo", "bar", single_stage=True)
assert stage1.addressing == "bar.dvc"
assert repr(stage1) == "Stage: 'bar.dvc'"
assert str(stage1) == "stage: 'bar.dvc'"
stage2 = run_copy("bar", "baz", name="copy-bar-baz")
assert stage2.addressing == "dvc.yaml:copy-bar-baz"
assert stage2.addressing == "copy-bar-baz"
assert repr(stage2) == "Stage: 'copy-bar-baz'"
assert str(stage2) == "stage: 'copy-bar-baz'"
folder = tmp_dir / "dir"
folder.mkdir()
with folder.chdir():
assert stage1.addressing == os.path.relpath(stage1.path)
assert (
stage2.addressing
== os.path.relpath(stage2.path) + ":" + stage2.name
)
rel_path = os.path.relpath(stage1.path)
assert stage1.addressing == rel_path
assert repr(stage1) == f"Stage: '{rel_path}'"
assert str(stage1) == f"stage: '{rel_path}'"
rel_path = os.path.relpath(stage2.path)
assert stage2.addressing == f"{rel_path}:{stage2.name}"
assert repr(stage2) == f"Stage: '{rel_path}:{stage2.name}'"
assert str(stage2) == f"stage: '{rel_path}:{stage2.name}'"
def test_stage_on_no_path_string_repr(tmp_dir, dvc):
s = Stage(dvc)
assert s.addressing == "No path"
assert repr(s) == "Stage: 'No path'"
assert str(s) == "stage: 'No path'"
p = PipelineStage(dvc, name="stage_name")
assert p.addressing == "No path:stage_name"
assert repr(p) == "Stage: 'No path:stage_name'"
assert str(p) == "stage: 'No path:stage_name'"
+3 -3
View File
@@ -73,12 +73,12 @@ def test_status_on_pipeline_stages(tmp_dir, dvc, run_copy):
stage.cmd = " ".join(stage.cmd.split())
stage.dvcfile._dump_pipeline_file(stage)
assert dvc.status() == {"dvc.yaml:copy-foo-bar": ["changed command"]}
assert dvc.status("copy-foo-bar") == {"copy-foo-bar": ["changed command"]}
# delete outputs
(tmp_dir / "bar").unlink()
assert dvc.status() == {
"dvc.yaml:copy-foo-bar": [
"copy-foo-bar": [
{"changed outs": {"bar": "deleted"}},
"changed command",
]
@@ -86,7 +86,7 @@ def test_status_on_pipeline_stages(tmp_dir, dvc, run_copy):
(tmp_dir / "foo").unlink()
assert dvc.status() == {
"foo.dvc": [{"changed outs": {"foo": "deleted"}}],
"dvc.yaml:copy-foo-bar": [
"copy-foo-bar": [
{"changed deps": {"foo": "deleted"}},
{"changed outs": {"bar": "deleted"}},
"changed command",
+21
View File
@@ -3,6 +3,7 @@ from os.path import join
from dvc.ignore import CleanTree
from dvc.path_info import PathInfo
from dvc.repo import Repo
from dvc.repo.tree import RepoTree
from dvc.scm import SCM
from dvc.scm.git import GitTree
@@ -220,3 +221,23 @@ def test_repotree_cache_save(tmp_dir, dvc, scm, erepo_dir, setup_remote):
cache.save(PathInfo(erepo_dir / "dir"), None, tree=tree)
for checksum in expected:
assert os.path.exists(cache.checksum_to_path_info(checksum))
def test_cleantree_subrepo(tmp_dir, dvc, scm, monkeypatch):
tmp_dir.gen({"subdir": {}})
subrepo_dir = tmp_dir / "subdir"
with subrepo_dir.chdir():
subrepo = Repo.init(subdir=True)
subrepo_dir.gen({"foo": "foo", "dir": {"bar": "bar"}})
assert isinstance(dvc.tree, CleanTree)
assert not dvc.tree.exists(subrepo_dir / "foo")
assert not dvc.tree.isfile(subrepo_dir / "foo")
assert not dvc.tree.exists(subrepo_dir / "dir")
assert not dvc.tree.isdir(subrepo_dir / "dir")
assert isinstance(subrepo.tree, CleanTree)
assert subrepo.tree.exists(subrepo_dir / "foo")
assert subrepo.tree.isfile(subrepo_dir / "foo")
assert subrepo.tree.exists(subrepo_dir / "dir")
assert subrepo.tree.isdir(subrepo_dir / "dir")
+15
View File
@@ -87,6 +87,21 @@ def test_collect_optimization(tmp_dir, dvc, mocker):
dvc.collect_granular(stage.path)
def test_collect_optimization_on_stage_name(tmp_dir, dvc, mocker, run_copy):
tmp_dir.dvc_gen("foo", "foo")
stage = run_copy("foo", "bar", name="copy-foo-bar")
# Forget cached stages and graph and error out on collection
dvc._reset()
mocker.patch(
"dvc.repo.Repo.stages",
property(raiser(Exception("Should not collect"))),
)
# Should read stage directly instead of collecting the whole graph
assert dvc.collect("copy-foo-bar") == [stage]
assert dvc.collect_granular("copy-foo-bar") == [(stage, None)]
def test_skip_graph_checks(tmp_dir, dvc, mocker, run_copy):
# See https://github.com/iterative/dvc/issues/2671 for more info
mock_collect_graph = mocker.patch("dvc.repo.Repo._collect_graph")
+54
View File
@@ -104,3 +104,57 @@ def test_stage_cache_params(tmp_dir, dvc, run_copy, mocker):
assert (tmp_dir / "out_no_cache").exists()
assert (tmp_dir / "out").read_text() == "out"
assert (tmp_dir / "out_no_cache").read_text() == "out_no_cache"
def test_stage_cache_wdir(tmp_dir, dvc, run_copy, mocker):
tmp_dir.gen("dep", "dep")
tmp_dir.gen(
"script.py",
(
'open("out", "w+").write("out"); '
'open("out_no_cache", "w+").write("out_no_cache")'
),
)
tmp_dir.gen({"wdir": {}})
stage = dvc.run(
cmd="python ../script.py",
deps=["../script.py", "../dep"],
outs=["out"],
outs_no_cache=["out_no_cache"],
single_stage=True,
wdir="wdir",
)
with dvc.lock, dvc.state:
stage.remove(remove_outs=True, force=True)
assert not (tmp_dir / "wdir" / "out").exists()
assert not (tmp_dir / "wdir" / "out_no_cache").exists()
assert not (tmp_dir / "wdir" / "out.dvc").exists()
cache_dir = os.path.join(
dvc.stage_cache.cache_dir,
"d2",
"d2b5da199f4da73a861027f5f76020a948794011db9704814fdb2a488ca93ec2",
)
cache_file = os.path.join(
cache_dir,
"65cc63ade5ab338541726b26185ebaf42331141ec3a670a7d6e8a227505afade",
)
assert os.path.isdir(cache_dir)
assert os.listdir(cache_dir) == [os.path.basename(cache_file)]
assert os.path.isfile(cache_file)
run_spy = mocker.patch("dvc.stage.run.cmd_run")
checkout_spy = mocker.spy(stage, "checkout")
with dvc.lock, dvc.state:
stage.run()
assert not run_spy.called
assert checkout_spy.call_count == 1
assert (tmp_dir / "wdir" / "out").exists()
assert (tmp_dir / "wdir" / "out_no_cache").exists()
assert (tmp_dir / "wdir" / "out").read_text() == "out"
assert (tmp_dir / "wdir" / "out_no_cache").read_text() == "out_no_cache"
+6 -2
View File
@@ -6,5 +6,9 @@ from dvc.stage.run import run_stage
def test_run_stage_dry(caplog):
with caplog.at_level(level=logging.INFO, logger="dvc"):
run_stage(Stage(None, cmd="mycmd arg1 arg2"), dry=True)
assert caplog.messages == ["Running command:\n\tmycmd arg1 arg2"]
stage = Stage(None, "stage.dvc", cmd="mycmd arg1 arg2")
run_stage(stage, dry=True)
assert caplog.messages == [
"Running callback stage 'stage.dvc' with command:",
"\t" + "mycmd arg1 arg2",
]
+3
View File
@@ -146,6 +146,9 @@ def test_resolve_output(inp, out, is_dir, expected, mocker):
["../models/stage.dvc", ("../models/stage.dvc", None), "def"],
[":name", ("default", "name"), "default"],
[":name", ("default", "name"), "default"],
["something.dvc:name", ("something.dvc", "name"), None],
["../something.dvc:name", ("../something.dvc", "name"), None],
["file", (None, "file"), None],
],
)
def test_parse_target(inp, out, default):