mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 23:10:14 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7504b68c12 | ||
|
|
86f3457dec | ||
|
|
219662597b | ||
|
|
536966f9d1 | ||
|
|
076eadceba | ||
|
|
d1e08b6157 | ||
|
|
edfa1741db | ||
|
|
9ee62ef8f0 | ||
|
|
0bb49d2549 | ||
|
|
516d82e411 | ||
|
|
ec312d8601 | ||
|
|
f504245f35 | ||
|
|
bb50fb8664 | ||
|
|
777a9f80ae | ||
|
|
02a52d038a | ||
|
|
7d0a06248d | ||
|
|
2d8c6043ca | ||
|
|
3f996048e8 | ||
|
|
8d9435e8ee | ||
|
|
889a17e2fc | ||
|
|
63972c3d12 | ||
|
|
7027c631c5 | ||
|
|
e8f6cd7f5b | ||
|
|
10f2bb93fa | ||
|
|
2fecc66cb9 | ||
|
|
b47998463f | ||
|
|
d53b740f35 | ||
|
|
9198fa7459 |
@@ -1,10 +1,19 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
|
||||
name: "\U0001F41B Bug Report"
|
||||
about: Create a bug report to help us improve DVC
|
||||
labels: bug, triage
|
||||
---
|
||||
|
||||
## Bug Report
|
||||
|
||||
**Please provide information about your setup**
|
||||
DVC version(i.e. `dvc --version`), Platform and method of installation (pip, homebrew, pkg Mac, exe (Windows), DEB(Linux), RPM(Linux))
|
||||
### Please provide information about your setup
|
||||
|
||||
**Output of `dvc version`:**
|
||||
|
||||
```console
|
||||
$ dvc version
|
||||
```
|
||||
|
||||
**Additional Information (if any):**
|
||||
|
||||
If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: "🤗 Need help?"
|
||||
url: https://dvc.org/chat
|
||||
about: If you have a question, ask us on Discord. Please join with this invite 👉
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: Feature request
|
||||
name: "\U0001F680 Feature Request"
|
||||
about: Suggest an idea for this project
|
||||
|
||||
labels: feature request, triage
|
||||
---
|
||||
|
||||
|
||||
+26
-7
@@ -27,18 +27,37 @@ repos:
|
||||
repo: https://github.com/lovesegfault/beautysh
|
||||
rev: master
|
||||
- hooks:
|
||||
- id: dvc-pre-commit
|
||||
language_version: python3
|
||||
- args:
|
||||
- git-hook
|
||||
- pre-commit
|
||||
entry: dvc
|
||||
id: dvc-pre-commit
|
||||
language: system
|
||||
name: DVC pre-commit
|
||||
stages:
|
||||
- commit
|
||||
- id: dvc-pre-push
|
||||
language_version: python3
|
||||
verbose: true
|
||||
require_serial: true
|
||||
- args:
|
||||
- git-hook
|
||||
- pre-push
|
||||
entry: dvc
|
||||
id: dvc-pre-push
|
||||
language: system
|
||||
name: DVC pre-push
|
||||
stages:
|
||||
- push
|
||||
require_serial: true
|
||||
- always_run: true
|
||||
args:
|
||||
- git-hook
|
||||
- post-checkout
|
||||
entry: dvc
|
||||
id: dvc-post-checkout
|
||||
language_version: python3
|
||||
language: system
|
||||
minimum_pre_commit_version: 2.2.0
|
||||
name: DVC post-checkout
|
||||
stages:
|
||||
- post-checkout
|
||||
repo: https://github.com/iterative/dvc
|
||||
rev: master
|
||||
require_serial: true
|
||||
repo: local
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
language: python
|
||||
language_version: python3
|
||||
name: DVC pre-commit
|
||||
require_serial: true
|
||||
stages:
|
||||
- commit
|
||||
verbose: true
|
||||
@@ -17,6 +18,7 @@
|
||||
language: python
|
||||
language_version: python3
|
||||
name: DVC pre-push
|
||||
require_serial: true
|
||||
stages:
|
||||
- push
|
||||
- always_run: true
|
||||
@@ -29,5 +31,6 @@
|
||||
language_version: python3
|
||||
minimum_pre_commit_version: 2.2.0
|
||||
name: DVC post-checkout
|
||||
require_serial: true
|
||||
stages:
|
||||
- post-checkout
|
||||
|
||||
+4
-4
@@ -10,9 +10,11 @@ from .command import (
|
||||
commit,
|
||||
config,
|
||||
daemon,
|
||||
dag,
|
||||
data_sync,
|
||||
destroy,
|
||||
diff,
|
||||
freeze,
|
||||
gc,
|
||||
get,
|
||||
get_url,
|
||||
@@ -21,12 +23,10 @@ from .command import (
|
||||
imp_url,
|
||||
init,
|
||||
install,
|
||||
lock,
|
||||
ls,
|
||||
metrics,
|
||||
move,
|
||||
params,
|
||||
pipeline,
|
||||
plots,
|
||||
remote,
|
||||
remove,
|
||||
@@ -66,8 +66,8 @@ COMMANDS = [
|
||||
install,
|
||||
root,
|
||||
ls,
|
||||
lock,
|
||||
pipeline,
|
||||
freeze,
|
||||
dag,
|
||||
daemon,
|
||||
commit,
|
||||
diff,
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _show_ascii(G):
|
||||
from dvc.repo.graph import get_pipelines
|
||||
from dvc.dagascii import draw
|
||||
|
||||
pipelines = get_pipelines(G)
|
||||
|
||||
ret = []
|
||||
for pipeline in pipelines:
|
||||
ret.append(draw(pipeline.nodes, pipeline.edges))
|
||||
|
||||
return "\n".join(ret)
|
||||
|
||||
|
||||
def _show_dot(G):
|
||||
import io
|
||||
from networkx.drawing.nx_pydot import write_dot
|
||||
|
||||
dot_file = io.StringIO()
|
||||
write_dot(G, dot_file)
|
||||
return dot_file.getvalue()
|
||||
|
||||
|
||||
def _build(G, target=None, full=False):
|
||||
import networkx as nx
|
||||
from dvc.repo.graph import get_pipeline, get_pipelines
|
||||
|
||||
if target:
|
||||
H = get_pipeline(get_pipelines(G), target)
|
||||
if not full:
|
||||
descendants = nx.descendants(G, target)
|
||||
descendants.add(target)
|
||||
H.remove_nodes_from(set(G.nodes()) - descendants)
|
||||
else:
|
||||
H = G
|
||||
|
||||
def _relabel(stage):
|
||||
return stage.addressing
|
||||
|
||||
return nx.relabel_nodes(H, _relabel, copy=False)
|
||||
|
||||
|
||||
class CmdDAG(CmdBase):
|
||||
def run(self):
|
||||
try:
|
||||
target = None
|
||||
if self.args.target:
|
||||
stages = self.repo.collect(self.args.target)
|
||||
if len(stages) > 1:
|
||||
logger.error(
|
||||
f"'{self.args.target}' contains more than one stage "
|
||||
"{stages}, please specify one stage"
|
||||
)
|
||||
return 1
|
||||
target = stages[0]
|
||||
|
||||
G = _build(self.repo.graph, target=target, full=self.args.full,)
|
||||
|
||||
if self.args.dot:
|
||||
logger.info(_show_dot(G))
|
||||
else:
|
||||
from dvc.utils.pager import pager
|
||||
|
||||
pager(_show_ascii(G))
|
||||
|
||||
return 0
|
||||
except DvcException:
|
||||
msg = "failed to show"
|
||||
if self.args.target:
|
||||
msg += f"a pipeline for '{target}'"
|
||||
else:
|
||||
msg += "pipelines"
|
||||
logger.exception(msg)
|
||||
return 1
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
DAG_HELP = "Visualize DVC project DAG."
|
||||
dag_parser = subparsers.add_parser(
|
||||
"dag",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(DAG_HELP, "dag"),
|
||||
help=DAG_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
dag_parser.add_argument(
|
||||
"--dot",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Print DAG with .dot format.",
|
||||
)
|
||||
dag_parser.add_argument(
|
||||
"--full",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"Show full DAG that the target belongs too, instead of "
|
||||
"showing DAG consisting only of ancestors."
|
||||
),
|
||||
)
|
||||
dag_parser.add_argument(
|
||||
"target",
|
||||
nargs="?",
|
||||
help="Stage or output to show pipeline for. Optional. "
|
||||
"(Finds all stages in the workspace by default.)",
|
||||
)
|
||||
dag_parser.set_defaults(func=CmdDAG)
|
||||
@@ -0,0 +1,57 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CmdFreezeBase(CmdBase):
|
||||
def _run(self, func, name):
|
||||
ret = 0
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
func(target)
|
||||
except DvcException:
|
||||
logger.exception(f"failed to {name} '{target}'")
|
||||
ret = 1
|
||||
return ret
|
||||
|
||||
|
||||
class CmdFreeze(CmdFreezeBase):
|
||||
def run(self):
|
||||
return self._run(self.repo.freeze, "freeze")
|
||||
|
||||
|
||||
class CmdUnfreeze(CmdFreezeBase):
|
||||
def run(self):
|
||||
return self._run(self.repo.unfreeze, "unfreeze")
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
FREEZE_HELP = "Freeze DVC-files."
|
||||
freeze_parser = subparsers.add_parser(
|
||||
"freeze",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(FREEZE_HELP, "freeze"),
|
||||
help=FREEZE_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
freeze_parser.add_argument(
|
||||
"targets", nargs="+", help="DVC-files to freeze."
|
||||
)
|
||||
freeze_parser.set_defaults(func=CmdFreeze)
|
||||
|
||||
UNFREEZE_HELP = "Unfreeze DVC-files."
|
||||
unfreeze_parser = subparsers.add_parser(
|
||||
"unfreeze",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(UNFREEZE_HELP, "unfreeze"),
|
||||
help=UNFREEZE_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
unfreeze_parser.add_argument(
|
||||
"targets", nargs="+", help="DVC-files to unfreeze."
|
||||
)
|
||||
unfreeze_parser.set_defaults(func=CmdUnfreeze)
|
||||
+2
-2
@@ -21,7 +21,7 @@ class CmdGC(CmdBase):
|
||||
|
||||
msg = "This will remove all cache except items used in "
|
||||
|
||||
msg += "the working tree"
|
||||
msg += "the workspace"
|
||||
if self.args.all_commits:
|
||||
msg += " and all git commits"
|
||||
elif self.args.all_branches and self.args.all_tags:
|
||||
@@ -77,7 +77,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"--workspace",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Keep data files used in the current workspace (working tree).",
|
||||
help="Keep data files used in the current workspace.",
|
||||
)
|
||||
gc_parser.add_argument(
|
||||
"-a",
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CmdLockBase(CmdBase):
|
||||
def _run(self, unlock):
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
self.repo.lock_stage(target, unlock=unlock)
|
||||
except DvcException:
|
||||
logger.exception(
|
||||
"failed to {}lock '{}'".format(
|
||||
"un" if unlock else "", target
|
||||
)
|
||||
)
|
||||
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
class CmdLock(CmdLockBase):
|
||||
def run(self):
|
||||
return self._run(False)
|
||||
|
||||
|
||||
class CmdUnlock(CmdLockBase):
|
||||
def run(self):
|
||||
return self._run(True)
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
LOCK_HELP = "Lock DVC-files."
|
||||
lock_parser = subparsers.add_parser(
|
||||
"lock",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(LOCK_HELP, "lock"),
|
||||
help=LOCK_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
lock_parser.add_argument("targets", nargs="+", help="DVC-files to lock.")
|
||||
lock_parser.set_defaults(func=CmdLock)
|
||||
|
||||
UNLOCK_HELP = "Unlock DVC-files."
|
||||
unlock_parser = subparsers.add_parser(
|
||||
"unlock",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(UNLOCK_HELP, "unlock"),
|
||||
help=UNLOCK_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
unlock_parser.add_argument(
|
||||
"targets", nargs="+", help="DVC-files to unlock."
|
||||
)
|
||||
unlock_parser.set_defaults(func=CmdUnlock)
|
||||
+1
-47
@@ -63,30 +63,6 @@ class CmdMetricsShow(CmdBase):
|
||||
return 0
|
||||
|
||||
|
||||
class CmdMetricsAdd(CmdBase):
|
||||
def run(self):
|
||||
try:
|
||||
self.repo.metrics.add(self.args.path)
|
||||
except DvcException:
|
||||
msg = f"failed to add metric file '{self.args.path}'"
|
||||
logger.exception(msg)
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
class CmdMetricsRemove(CmdBase):
|
||||
def run(self):
|
||||
try:
|
||||
self.repo.metrics.remove(self.args.path)
|
||||
except DvcException:
|
||||
msg = f"failed to remove metric file '{self.args.path}'"
|
||||
logger.exception(msg)
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def _show_diff(diff, markdown=False, no_path=False, old=False):
|
||||
from collections import OrderedDict
|
||||
|
||||
@@ -145,7 +121,7 @@ class CmdMetricsDiff(CmdBase):
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
METRICS_HELP = "Commands to add, manage, collect, and display metrics."
|
||||
METRICS_HELP = "Commands to display and compare metrics."
|
||||
|
||||
metrics_parser = subparsers.add_parser(
|
||||
"metrics",
|
||||
@@ -162,17 +138,6 @@ def add_parser(subparsers, parent_parser):
|
||||
|
||||
fix_subparsers(metrics_subparsers)
|
||||
|
||||
METRICS_ADD_HELP = "Mark a DVC-tracked file as a metric."
|
||||
metrics_add_parser = metrics_subparsers.add_parser(
|
||||
"add",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(METRICS_ADD_HELP, "metrics/add"),
|
||||
help=METRICS_ADD_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
metrics_add_parser.add_argument("path", help="Path to a metric file.")
|
||||
metrics_add_parser.set_defaults(func=CmdMetricsAdd)
|
||||
|
||||
METRICS_SHOW_HELP = "Print metrics, with optional formatting."
|
||||
metrics_show_parser = metrics_subparsers.add_parser(
|
||||
"show",
|
||||
@@ -291,14 +256,3 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Show old metric value.",
|
||||
)
|
||||
metrics_diff_parser.set_defaults(func=CmdMetricsDiff)
|
||||
|
||||
METRICS_REMOVE_HELP = "Remove metric mark on a DVC-tracked file."
|
||||
metrics_remove_parser = metrics_subparsers.add_parser(
|
||||
"remove",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(METRICS_REMOVE_HELP, "metrics/remove"),
|
||||
help=METRICS_REMOVE_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
metrics_remove_parser.add_argument("path", help="Path to a metric file.")
|
||||
metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
|
||||
|
||||
@@ -1,267 +0,0 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CmdPipelineShow(CmdBase):
|
||||
def _show(self, target, commands, outs, locked):
|
||||
import networkx
|
||||
from dvc import dvcfile
|
||||
from dvc.utils import parse_target
|
||||
|
||||
path, name = parse_target(target)
|
||||
stage = dvcfile.Dvcfile(self.repo, path).stages[name]
|
||||
G = self.repo.graph
|
||||
stages = networkx.dfs_postorder_nodes(G, stage)
|
||||
if locked:
|
||||
stages = [s for s in stages if s.locked]
|
||||
|
||||
for stage in stages:
|
||||
if commands:
|
||||
if stage.cmd is None:
|
||||
continue
|
||||
logger.info(stage.cmd)
|
||||
elif outs:
|
||||
for out in stage.outs:
|
||||
logger.info(str(out))
|
||||
else:
|
||||
logger.info(stage.addressing)
|
||||
|
||||
@staticmethod
|
||||
def _build_output_graph(G, target_stage):
|
||||
import networkx
|
||||
from itertools import product
|
||||
|
||||
nodes = {str(out) for out in target_stage.outs}
|
||||
edges = []
|
||||
|
||||
for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
|
||||
from_stage_deps = {dep.path_info.parts for dep in from_stage.deps}
|
||||
to_outs = {
|
||||
to_out
|
||||
for to_out in to_stage.outs
|
||||
if to_out.path_info.parts in from_stage_deps
|
||||
}
|
||||
from_outs = {
|
||||
from_out
|
||||
for from_out in from_stage.outs
|
||||
if str(from_out) in nodes
|
||||
}
|
||||
nodes |= {str(to_out) for to_out in to_outs}
|
||||
for from_out, to_out in product(from_outs, to_outs):
|
||||
edges.append((str(from_out), str(to_out)))
|
||||
return nodes, edges
|
||||
|
||||
def _build_graph(self, target, commands=False, outs=False):
|
||||
import networkx
|
||||
from dvc import dvcfile
|
||||
from dvc.repo.graph import get_pipeline
|
||||
from dvc.utils import parse_target
|
||||
|
||||
path, name = parse_target(target)
|
||||
target_stage = dvcfile.Dvcfile(self.repo, path).stages[name]
|
||||
G = get_pipeline(self.repo.pipelines, target_stage)
|
||||
|
||||
nodes = set()
|
||||
for stage in networkx.dfs_preorder_nodes(G, target_stage):
|
||||
if commands:
|
||||
if stage.cmd is None:
|
||||
continue
|
||||
nodes.add(stage.cmd)
|
||||
elif not outs:
|
||||
nodes.add(stage.addressing)
|
||||
|
||||
edges = []
|
||||
for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
|
||||
if commands:
|
||||
if to_stage.cmd is None:
|
||||
continue
|
||||
edges.append((from_stage.cmd, to_stage.cmd))
|
||||
elif not outs:
|
||||
edges.append((from_stage.addressing, to_stage.addressing))
|
||||
|
||||
if outs:
|
||||
nodes, edges = self._build_output_graph(G, target_stage)
|
||||
|
||||
return list(nodes), edges, networkx.is_tree(G)
|
||||
|
||||
def _show_ascii(self, target, commands, outs):
|
||||
from dvc.dagascii import draw
|
||||
|
||||
nodes, edges, _ = self._build_graph(target, commands, outs)
|
||||
|
||||
if not nodes:
|
||||
return
|
||||
|
||||
draw(nodes, edges)
|
||||
|
||||
def _show_dependencies_tree(self, target, commands, outs):
|
||||
from treelib import Tree
|
||||
|
||||
nodes, edges, is_tree = self._build_graph(target, commands, outs)
|
||||
if not nodes:
|
||||
return
|
||||
if not is_tree:
|
||||
raise DvcException(
|
||||
"DAG is not a tree, can not print it in tree-structure way, "
|
||||
"please use --ascii instead"
|
||||
)
|
||||
|
||||
tree = Tree()
|
||||
tree.create_node(target, target) # Root node
|
||||
observe_list = [target]
|
||||
while len(observe_list) > 0:
|
||||
current_root = observe_list[0]
|
||||
for edge in edges:
|
||||
if edge[0] == current_root:
|
||||
tree.create_node(edge[1], edge[1], parent=current_root)
|
||||
observe_list.append(edge[1])
|
||||
observe_list.pop(0)
|
||||
tree.show()
|
||||
|
||||
def _write_dot(self, target, commands, outs):
|
||||
import io
|
||||
import networkx
|
||||
from networkx.drawing.nx_pydot import write_dot
|
||||
|
||||
_, edges, _ = self._build_graph(target, commands, outs)
|
||||
edges = [edge[::-1] for edge in edges]
|
||||
|
||||
simple_g = networkx.DiGraph()
|
||||
simple_g.add_edges_from(edges)
|
||||
|
||||
dot_file = io.StringIO()
|
||||
write_dot(simple_g, dot_file)
|
||||
logger.info(dot_file.getvalue())
|
||||
|
||||
def run(self):
|
||||
from dvc.dvcfile import DVC_FILE
|
||||
|
||||
if not self.args.targets:
|
||||
self.args.targets = [DVC_FILE]
|
||||
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
if self.args.ascii:
|
||||
self._show_ascii(
|
||||
target, self.args.commands, self.args.outs
|
||||
)
|
||||
elif self.args.dot:
|
||||
self._write_dot(target, self.args.commands, self.args.outs)
|
||||
elif self.args.tree:
|
||||
self._show_dependencies_tree(
|
||||
target, self.args.commands, self.args.outs
|
||||
)
|
||||
else:
|
||||
self._show(
|
||||
target,
|
||||
self.args.commands,
|
||||
self.args.outs,
|
||||
self.args.locked,
|
||||
)
|
||||
except DvcException:
|
||||
msg = f"failed to show pipeline for '{target}'"
|
||||
logger.exception(msg)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
class CmdPipelineList(CmdBase):
|
||||
def run(self):
|
||||
pipelines = self.repo.pipelines
|
||||
for pipeline in pipelines:
|
||||
for stage in pipeline:
|
||||
logger.info(stage.addressing)
|
||||
if len(pipeline) != 0:
|
||||
logger.info("=" * 80)
|
||||
logger.info("{} pipelines total".format(len(pipelines)))
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
PIPELINE_HELP = "Manage pipelines."
|
||||
pipeline_parser = subparsers.add_parser(
|
||||
"pipeline",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(PIPELINE_HELP, "pipeline"),
|
||||
help=PIPELINE_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
|
||||
pipeline_subparsers = pipeline_parser.add_subparsers(
|
||||
dest="cmd",
|
||||
help="Use `dvc pipeline CMD --help` for command-specific help.",
|
||||
)
|
||||
|
||||
fix_subparsers(pipeline_subparsers)
|
||||
|
||||
PIPELINE_LIST_HELP = "List connected groups of stages (pipelines)."
|
||||
pipeline_list_parser = pipeline_subparsers.add_parser(
|
||||
"list",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(PIPELINE_LIST_HELP, "pipeline/list"),
|
||||
help=PIPELINE_LIST_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
pipeline_list_parser.set_defaults(func=CmdPipelineList)
|
||||
|
||||
PIPELINE_SHOW_HELP = "Show stages in a pipeline."
|
||||
pipeline_show_parser = pipeline_subparsers.add_parser(
|
||||
"show",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(PIPELINE_SHOW_HELP, "pipeline/show"),
|
||||
help=PIPELINE_SHOW_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group()
|
||||
pipeline_show_group.add_argument(
|
||||
"-c",
|
||||
"--commands",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Print commands instead of paths to DVC-files.",
|
||||
)
|
||||
pipeline_show_group.add_argument(
|
||||
"-o",
|
||||
"--outs",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Print output files instead of paths to DVC-files.",
|
||||
)
|
||||
pipeline_show_parser.add_argument(
|
||||
"-l",
|
||||
"--locked",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Print locked DVC stages",
|
||||
)
|
||||
pipeline_show_parser.add_argument(
|
||||
"--ascii",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Output DAG as ASCII.",
|
||||
)
|
||||
pipeline_show_parser.add_argument(
|
||||
"--dot",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Print DAG with .dot format.",
|
||||
)
|
||||
pipeline_show_parser.add_argument(
|
||||
"--tree",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Output DAG as Dependencies Tree.",
|
||||
)
|
||||
pipeline_show_parser.add_argument(
|
||||
"targets",
|
||||
nargs="*",
|
||||
help="DVC-files to show pipeline for. Optional. "
|
||||
"(Finds all DVC-files in the workspace by default.)",
|
||||
)
|
||||
pipeline_show_parser.set_defaults(func=CmdPipelineShow)
|
||||
+80
-86
@@ -4,6 +4,7 @@ import os
|
||||
|
||||
from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.schema import PLOT_PROPS
|
||||
from dvc.utils import format_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -32,23 +33,28 @@ class CmdPlots(CmdBase):
|
||||
def _func(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def _props(self):
|
||||
# Pass only props specified by user, to not shadow ones from plot def
|
||||
props = {p: getattr(self.args, p) for p in PLOT_PROPS}
|
||||
return {k: v for k, v in props.items() if v is not None}
|
||||
|
||||
def run(self):
|
||||
if self.args.show_vega:
|
||||
if not self.args.targets:
|
||||
logger.error("please specify a target for `--show-vega`")
|
||||
return 1
|
||||
if len(self.args.targets) > 1:
|
||||
logger.error(
|
||||
"you can only specify one target for `--show-vega`"
|
||||
)
|
||||
return 1
|
||||
|
||||
try:
|
||||
plots = self._func(
|
||||
targets=self.args.targets,
|
||||
template=self.args.template,
|
||||
x_field=self.args.x,
|
||||
y_field=self.args.y,
|
||||
csv_header=not self.args.no_csv_header,
|
||||
title=self.args.title,
|
||||
x_title=self.args.xlab,
|
||||
y_title=self.args.ylab,
|
||||
)
|
||||
plots = self._func(targets=self.args.targets, props=self._props())
|
||||
|
||||
if self.args.show_json:
|
||||
import json
|
||||
|
||||
logger.info(json.dumps(plots))
|
||||
if self.args.show_vega:
|
||||
target = self.args.targets[0]
|
||||
logger.info(plots[target])
|
||||
return 0
|
||||
|
||||
divs = [
|
||||
@@ -82,6 +88,14 @@ class CmdPlotsDiff(CmdPlots):
|
||||
return self.repo.plots.diff(*args, revs=self.args.revisions, **kwargs)
|
||||
|
||||
|
||||
class CmdPlotsModify(CmdPlots):
|
||||
def run(self):
|
||||
self.repo.plots.modify(
|
||||
self.args.target, props=self._props(), unset=self.args.unset,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
PLOTS_HELP = (
|
||||
"Generating plots for metrics stored in structured files "
|
||||
@@ -110,51 +124,13 @@ def add_parser(subparsers, parent_parser):
|
||||
help=SHOW_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"-t",
|
||||
"--template",
|
||||
nargs="?",
|
||||
default=None,
|
||||
help=(
|
||||
"Special JSON or HTML schema file to inject with the data. "
|
||||
"See {}".format(
|
||||
format_link("https://man.dvc.org/plots#plot-templates")
|
||||
)
|
||||
),
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"-o", "--out", default=None, help="Destination path to save plots to.",
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"-x", default=None, help="Field name for x axis."
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"-y", default=None, help="Field name for y axis."
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"--no-csv-header",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Required when CSV or TSV datafile does not have a header.",
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"--show-json",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show output in JSON format.",
|
||||
)
|
||||
plots_show_parser.add_argument("--title", default=None, help="Plot title.")
|
||||
plots_show_parser.add_argument(
|
||||
"--xlab", default=None, help="X axis title."
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"--ylab", default=None, help="Y axis title."
|
||||
)
|
||||
plots_show_parser.add_argument(
|
||||
"targets",
|
||||
nargs="*",
|
||||
help="Metrics files to visualize. Shows all plots by default.",
|
||||
help="Plots files to visualize. Shows all plots by default.",
|
||||
)
|
||||
_add_props_arguments(plots_show_parser)
|
||||
_add_output_arguments(plots_show_parser)
|
||||
plots_show_parser.set_defaults(func=CmdPlotsShow)
|
||||
|
||||
PLOTS_DIFF_HELP = (
|
||||
@@ -169,6 +145,37 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"--targets",
|
||||
nargs="*",
|
||||
help="Plots file to visualize. Shows all plots by default.",
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"revisions", nargs="*", default=None, help="Git commits to plot from",
|
||||
)
|
||||
_add_props_arguments(plots_diff_parser)
|
||||
_add_output_arguments(plots_diff_parser)
|
||||
plots_diff_parser.set_defaults(func=CmdPlotsDiff)
|
||||
|
||||
PLOTS_MODIFY_HELP = "Modify plot props associated with a target file."
|
||||
plots_modify_parser = plots_subparsers.add_parser(
|
||||
"modify",
|
||||
parents=[parent_parser],
|
||||
description=append_doc_link(PLOTS_MODIFY_HELP, "plots/modify"),
|
||||
help=PLOTS_MODIFY_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
plots_modify_parser.add_argument(
|
||||
"target", help="Plot file to set props to.",
|
||||
)
|
||||
_add_props_arguments(plots_modify_parser)
|
||||
plots_modify_parser.add_argument(
|
||||
"--unset", nargs="*", help="Props to unset.",
|
||||
)
|
||||
plots_modify_parser.set_defaults(func=CmdPlotsModify)
|
||||
|
||||
|
||||
def _add_props_arguments(parser):
|
||||
parser.add_argument(
|
||||
"-t",
|
||||
"--template",
|
||||
nargs="?",
|
||||
@@ -180,40 +187,27 @@ def add_parser(subparsers, parent_parser):
|
||||
)
|
||||
),
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"--targets",
|
||||
nargs="*",
|
||||
help="Metrics file to visualize. Shows all plots by default.",
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"-o", "--out", default=None, help="Destination path to save plots to.",
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"-x", default=None, help="Field name for x axis."
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"-y", default=None, help="Field name for y axis."
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
parser.add_argument("-x", default=None, help="Field name for x axis.")
|
||||
parser.add_argument("-y", default=None, help="Field name for y axis.")
|
||||
parser.add_argument(
|
||||
"--no-csv-header",
|
||||
action="store_true",
|
||||
default=False,
|
||||
action="store_false",
|
||||
dest="csv_header",
|
||||
default=None, # Use default None to distinguish when it's not used
|
||||
help="Provided CSV ot TSV datafile does not have a header.",
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"--show-json",
|
||||
parser.add_argument("--title", default=None, help="Plot title.")
|
||||
parser.add_argument("--xlab", default=None, help="X axis title.")
|
||||
parser.add_argument("--ylab", default=None, help="Y axis title.")
|
||||
|
||||
|
||||
def _add_output_arguments(parser):
|
||||
parser.add_argument(
|
||||
"-o", "--out", default=None, help="Destination path to save plots to.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--show-vega",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show output in JSON format.",
|
||||
help="Show output in VEGA format.",
|
||||
)
|
||||
plots_diff_parser.add_argument("--title", default=None, help="Plot title.")
|
||||
plots_diff_parser.add_argument(
|
||||
"--xlab", default=None, help="X axis title."
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"--ylab", default=None, help="Y axis title."
|
||||
)
|
||||
plots_diff_parser.add_argument(
|
||||
"revisions", nargs="*", default=None, help="Git commits to plot from",
|
||||
)
|
||||
plots_diff_parser.set_defaults(func=CmdPlotsDiff)
|
||||
|
||||
+4
-41
@@ -1,7 +1,6 @@
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
import dvc.prompt as prompt
|
||||
from dvc.command.base import CmdBase, append_doc_link
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
@@ -9,30 +8,10 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CmdRemove(CmdBase):
|
||||
def _is_dvc_only(self, target):
|
||||
if not self.args.purge:
|
||||
return True
|
||||
|
||||
if self.args.force:
|
||||
return False
|
||||
|
||||
msg = "Are you sure you want to remove '{}' with its outputs?".format(
|
||||
target
|
||||
)
|
||||
|
||||
if prompt.confirm(msg):
|
||||
return False
|
||||
|
||||
raise DvcException(
|
||||
"Cannot purge without a confirmation from the user."
|
||||
" Use `-f` to force."
|
||||
)
|
||||
|
||||
def run(self):
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
dvc_only = self._is_dvc_only(target)
|
||||
self.repo.remove(target, dvc_only=dvc_only)
|
||||
self.repo.remove(target, outs=self.args.outs)
|
||||
except DvcException:
|
||||
logger.exception(f"failed to remove '{target}'")
|
||||
return 1
|
||||
@@ -40,7 +19,7 @@ class CmdRemove(CmdBase):
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
REMOVE_HELP = "Remove DVC-tracked files or directories."
|
||||
REMOVE_HELP = "Remove stage entry and unprotect outputs"
|
||||
remove_parser = subparsers.add_parser(
|
||||
"remove",
|
||||
parents=[parent_parser],
|
||||
@@ -48,27 +27,11 @@ def add_parser(subparsers, parent_parser):
|
||||
help=REMOVE_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
remove_parser_group = remove_parser.add_mutually_exclusive_group()
|
||||
remove_parser_group.add_argument(
|
||||
"-o",
|
||||
remove_parser.add_argument(
|
||||
"--outs",
|
||||
action="store_true",
|
||||
default=True,
|
||||
help="Only remove DVC-file outputs. (Default)",
|
||||
)
|
||||
remove_parser_group.add_argument(
|
||||
"-p",
|
||||
"--purge",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Remove DVC-file and all its outputs.",
|
||||
)
|
||||
remove_parser.add_argument(
|
||||
"-f",
|
||||
"--force",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Force purge.",
|
||||
help="Remove outputs as well.",
|
||||
)
|
||||
remove_parser.add_argument(
|
||||
"targets", nargs="+", help="DVC-files to remove."
|
||||
|
||||
+3
-3
@@ -45,7 +45,7 @@ class CmdRun(CmdBase):
|
||||
fname=self.args.file,
|
||||
wdir=self.args.wdir,
|
||||
no_exec=self.args.no_exec,
|
||||
overwrite=self.args.overwrite_dvcfile,
|
||||
overwrite=self.args.overwrite,
|
||||
run_cache=not self.args.no_run_cache,
|
||||
no_commit=self.args.no_commit,
|
||||
outs_persist=self.args.outs_persist,
|
||||
@@ -177,10 +177,10 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Only create stage file without actually running it.",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--overwrite-dvcfile",
|
||||
"--overwrite",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Overwrite existing DVC-file without asking for confirmation.",
|
||||
help="Overwrite existing stage",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--no-run-cache",
|
||||
|
||||
+2
-3
@@ -96,10 +96,9 @@ class AsciiCanvas:
|
||||
|
||||
def draw(self):
|
||||
"""Draws ASCII canvas on the screen."""
|
||||
pager = find_pager()
|
||||
lines = map("".join, self.canvas)
|
||||
joined_lines = os.linesep.join(lines)
|
||||
pager(joined_lines)
|
||||
return joined_lines
|
||||
|
||||
def point(self, x, y, char):
|
||||
"""Create a point on ASCII canvas.
|
||||
@@ -316,4 +315,4 @@ def draw(vertexes, edges):
|
||||
int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data
|
||||
)
|
||||
|
||||
canvas.draw()
|
||||
return canvas.draw()
|
||||
|
||||
+60
-18
@@ -17,11 +17,7 @@ from dvc.stage.exceptions import (
|
||||
from dvc.stage.loader import SingleStageLoader, StageLoader
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.collections import apply_diff
|
||||
from dvc.utils.stage import (
|
||||
dump_stage_file,
|
||||
parse_stage,
|
||||
parse_stage_for_update,
|
||||
)
|
||||
from dvc.utils.yaml import dump_yaml, parse_yaml, parse_yaml_for_update
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -101,7 +97,7 @@ class FileMixin:
|
||||
|
||||
with self.repo.tree.open(self.path) as fd:
|
||||
stage_text = fd.read()
|
||||
d = parse_stage(stage_text, self.path)
|
||||
d = parse_yaml(stage_text, self.path)
|
||||
self.validate(d, self.relpath)
|
||||
return d, stage_text
|
||||
|
||||
@@ -149,8 +145,8 @@ class SingleStageFile(FileMixin):
|
||||
logger.debug(
|
||||
"Saving information to '{file}'.".format(file=relpath(self.path))
|
||||
)
|
||||
dump_stage_file(self.path, serialize.to_single_stage_file(stage))
|
||||
self.repo.scm.track_file(relpath(self.path))
|
||||
dump_yaml(self.path, serialize.to_single_stage_file(stage))
|
||||
self.repo.scm.track_file(self.relpath)
|
||||
|
||||
def remove_with_prompt(self, force=False):
|
||||
if not self.exists():
|
||||
@@ -165,6 +161,9 @@ class SingleStageFile(FileMixin):
|
||||
|
||||
self.remove()
|
||||
|
||||
def remove_stage(self, stage):
|
||||
self.remove()
|
||||
|
||||
|
||||
class PipelineFile(FileMixin):
|
||||
"""Abstraction for pipelines file, .yaml + .lock combined."""
|
||||
@@ -182,12 +181,12 @@ class PipelineFile(FileMixin):
|
||||
assert isinstance(stage, PipelineStage)
|
||||
check_dvc_filename(self.path)
|
||||
|
||||
if not no_lock:
|
||||
self._dump_lockfile(stage)
|
||||
|
||||
if update_pipeline and not stage.is_data_source:
|
||||
self._dump_pipeline_file(stage)
|
||||
|
||||
if not no_lock:
|
||||
self._dump_lockfile(stage)
|
||||
|
||||
def _dump_lockfile(self, stage):
|
||||
self._lockfile.dump(stage)
|
||||
|
||||
@@ -195,8 +194,9 @@ class PipelineFile(FileMixin):
|
||||
data = {}
|
||||
if self.exists():
|
||||
with open(self.path) as fd:
|
||||
data = parse_stage_for_update(fd.read(), self.path)
|
||||
data = parse_yaml_for_update(fd.read(), self.path)
|
||||
else:
|
||||
logger.info("Creating '%s'", self.relpath)
|
||||
open(self.path, "w+").close()
|
||||
|
||||
data["stages"] = data.get("stages", {})
|
||||
@@ -207,8 +207,11 @@ class PipelineFile(FileMixin):
|
||||
else:
|
||||
data["stages"].update(stage_data)
|
||||
|
||||
dump_stage_file(self.path, data)
|
||||
self.repo.scm.track_file(relpath(self.path))
|
||||
logger.info(
|
||||
"Adding stage '%s' to '%s'", stage.name, self.relpath,
|
||||
)
|
||||
dump_yaml(self.path, data)
|
||||
self.repo.scm.track_file(self.relpath)
|
||||
|
||||
@property
|
||||
def stage(self):
|
||||
@@ -230,6 +233,22 @@ class PipelineFile(FileMixin):
|
||||
super().remove()
|
||||
self._lockfile.remove()
|
||||
|
||||
def remove_stage(self, stage):
|
||||
self._lockfile.remove_stage(stage)
|
||||
if not self.exists():
|
||||
return
|
||||
|
||||
with open(self.path, "r") as f:
|
||||
d = parse_yaml_for_update(f.read(), self.path)
|
||||
|
||||
self.validate(d, self.path)
|
||||
if stage.name not in d.get("stages", {}):
|
||||
return
|
||||
|
||||
logger.debug("Removing '%s' from '%s'", stage.name, self.path)
|
||||
del d["stages"][stage.name]
|
||||
dump_yaml(self.path, d)
|
||||
|
||||
|
||||
class Lockfile(FileMixin):
|
||||
from dvc.schema import COMPILED_LOCKFILE_SCHEMA as SCHEMA
|
||||
@@ -238,7 +257,7 @@ class Lockfile(FileMixin):
|
||||
if not self.exists():
|
||||
return {}
|
||||
with self.repo.tree.open(self.path) as fd:
|
||||
data = parse_stage(fd.read(), self.path)
|
||||
data = parse_yaml(fd.read(), self.path)
|
||||
try:
|
||||
self.validate(data, fname=self.relpath)
|
||||
except StageFileFormatError:
|
||||
@@ -250,15 +269,38 @@ class Lockfile(FileMixin):
|
||||
def dump(self, stage, **kwargs):
|
||||
stage_data = serialize.to_lockfile(stage)
|
||||
if not self.exists():
|
||||
modified = True
|
||||
logger.info("Generating lock file '%s'", self.relpath)
|
||||
data = stage_data
|
||||
open(self.path, "w+").close()
|
||||
else:
|
||||
with self.repo.tree.open(self.path, "r") as fd:
|
||||
data = parse_stage_for_update(fd.read(), self.path)
|
||||
data = parse_yaml_for_update(fd.read(), self.path)
|
||||
modified = data.get(stage.name, {}) != stage_data.get(
|
||||
stage.name, {}
|
||||
)
|
||||
if modified:
|
||||
logger.info("Updating lock file '%s'", self.relpath)
|
||||
data.update(stage_data)
|
||||
dump_yaml(self.path, data)
|
||||
if modified:
|
||||
self.repo.scm.track_file(self.relpath)
|
||||
|
||||
dump_stage_file(self.path, data)
|
||||
self.repo.scm.track_file(relpath(self.path))
|
||||
def remove_stage(self, stage):
|
||||
if not self.exists():
|
||||
return
|
||||
|
||||
with open(self.path) as f:
|
||||
d = parse_yaml_for_update(f.read(), self.path)
|
||||
self.validate(d, self.path)
|
||||
|
||||
if stage.name not in d:
|
||||
return
|
||||
|
||||
logger.debug("Removing '%s' from '%s'", stage.name, self.path)
|
||||
del d[stage.name]
|
||||
|
||||
dump_yaml(self.path, d)
|
||||
|
||||
|
||||
class Dvcfile:
|
||||
|
||||
+16
-5
@@ -173,17 +173,16 @@ class BadMetricError(DvcException):
|
||||
class NoMetricsError(DvcException):
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
"no metric files in this repository. "
|
||||
"Use `dvc metrics add` to add a metric file to track."
|
||||
"no metric files in this repository. Use `-m/-M` options for "
|
||||
"`dvc run` to mark stage outputs as metrics."
|
||||
)
|
||||
|
||||
|
||||
class StageFileCorruptedError(DvcException):
|
||||
class YAMLFileCorruptedError(DvcException):
|
||||
def __init__(self, path):
|
||||
path = relpath(path)
|
||||
super().__init__(
|
||||
"unable to read DVC-file: {} "
|
||||
"YAML file structure is corrupted".format(path)
|
||||
f"unable to read: '{path}', YAML file structure is corrupted"
|
||||
)
|
||||
|
||||
|
||||
@@ -322,3 +321,15 @@ class RemoteCacheRequiredError(DvcException):
|
||||
|
||||
class IsADirectoryError(DvcException):
|
||||
"""Raised when a file operation is requested on a directory."""
|
||||
|
||||
|
||||
class NoOutputOrStageError(DvcException):
|
||||
"""
|
||||
Raised when the target is neither an output nor a stage name in dvc.yaml
|
||||
"""
|
||||
|
||||
def __init__(self, target, file):
|
||||
super().__init__(
|
||||
f"'{target}' "
|
||||
f"does not exist as an output or a stage name in '{file}'"
|
||||
)
|
||||
|
||||
+84
-12
@@ -5,7 +5,9 @@ from funcy import cached_property
|
||||
from pathspec import PathSpec
|
||||
from pathspec.patterns import GitWildMatchPattern
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.scm.tree import BaseTree
|
||||
from dvc.utils import relpath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -82,7 +84,7 @@ class DvcIgnoreRepo(DvcIgnore):
|
||||
def is_dvc_repo(directory):
|
||||
from dvc.repo import Repo
|
||||
|
||||
return os.path.isdir(os.path.join(directory, Repo.DVC_DIR))
|
||||
return os.path.isdir(os.path.join(root, directory, Repo.DVC_DIR))
|
||||
|
||||
dirs = [d for d in dirs if not is_dvc_repo(d)]
|
||||
|
||||
@@ -90,13 +92,14 @@ class DvcIgnoreRepo(DvcIgnore):
|
||||
|
||||
|
||||
class DvcIgnoreFilter:
|
||||
def __init__(self, tree):
|
||||
def __init__(self, tree, root_dir):
|
||||
self.tree = tree
|
||||
self.root_dir = root_dir
|
||||
self.ignores = {
|
||||
DvcIgnoreDirs([".git", ".hg", ".dvc"]),
|
||||
DvcIgnoreRepo(),
|
||||
}
|
||||
for root, dirs, files in self.tree.walk(self.tree.tree_root):
|
||||
for root, dirs, files in self.tree.walk(self.root_dir):
|
||||
self._update(root)
|
||||
dirs[:], files[:] = self(root, dirs, files)
|
||||
|
||||
@@ -113,31 +116,98 @@ class DvcIgnoreFilter:
|
||||
|
||||
|
||||
class CleanTree(BaseTree):
|
||||
def __init__(self, tree):
|
||||
def __init__(self, tree, tree_root=None):
|
||||
self.tree = tree
|
||||
if tree_root:
|
||||
self._tree_root = tree_root
|
||||
else:
|
||||
self._tree_root = self.tree.tree_root
|
||||
|
||||
@cached_property
|
||||
def dvcignore(self):
|
||||
return DvcIgnoreFilter(self.tree)
|
||||
return DvcIgnoreFilter(self.tree, self.tree_root)
|
||||
|
||||
@property
|
||||
def tree_root(self):
|
||||
return self.tree.tree_root
|
||||
return self._tree_root
|
||||
|
||||
def open(self, path, mode="r", encoding="utf-8"):
|
||||
return self.tree.open(path, mode, encoding)
|
||||
if self.isfile(path):
|
||||
return self.tree.open(path, mode, encoding)
|
||||
raise FileNotFoundError
|
||||
|
||||
def exists(self, path):
|
||||
return self.tree.exists(path)
|
||||
if self.tree.exists(path) and self._parents_exist(path):
|
||||
if self.tree.isdir(path):
|
||||
return self._valid_dirname(path)
|
||||
return self._valid_filename(path)
|
||||
return False
|
||||
|
||||
def isdir(self, path):
|
||||
return self.tree.isdir(path)
|
||||
return (
|
||||
self.tree.isdir(path)
|
||||
and self._parents_exist(path)
|
||||
and self._valid_dirname(path)
|
||||
)
|
||||
|
||||
def _valid_dirname(self, path):
|
||||
path = os.path.abspath(path)
|
||||
if path == self.tree_root:
|
||||
return True
|
||||
dirname, basename = os.path.split(path)
|
||||
dirs, _ = self.dvcignore(dirname, [basename], [])
|
||||
if dirs:
|
||||
return True
|
||||
return False
|
||||
|
||||
def isfile(self, path):
|
||||
return self.tree.isfile(path)
|
||||
return (
|
||||
self.tree.isfile(path)
|
||||
and self._parents_exist(path)
|
||||
and self._valid_filename(path)
|
||||
)
|
||||
|
||||
def _valid_filename(self, path):
|
||||
dirname, basename = os.path.split(os.path.normpath(path))
|
||||
_, files = self.dvcignore(os.path.abspath(dirname), [], [basename])
|
||||
if files:
|
||||
return True
|
||||
return False
|
||||
|
||||
def isexec(self, path):
|
||||
return self.tree.isexec(path)
|
||||
return self.exists(path) and self.tree.isexec(path)
|
||||
|
||||
def _parents_exist(self, path):
|
||||
from dvc.repo import Repo
|
||||
|
||||
path = PathInfo(path)
|
||||
|
||||
# if parent is tree_root or inside a .dvc dir we can skip this check
|
||||
if path.parent == self.tree_root or Repo.DVC_DIR in path.parts:
|
||||
return True
|
||||
|
||||
# if path is outside of tree, assume this is a local remote/local cache
|
||||
# link/move operation where we do not need to filter ignores
|
||||
path = relpath(path, self.tree_root)
|
||||
if path.startswith("..") or (
|
||||
os.name == "nt"
|
||||
and not os.path.commonprefix(
|
||||
[os.path.abspath(path), self.tree_root]
|
||||
)
|
||||
):
|
||||
return True
|
||||
|
||||
# check if parent directories are in our ignores, starting from
|
||||
# tree_root
|
||||
for parent_dir in reversed(PathInfo(path).parents):
|
||||
dirname, basename = os.path.split(parent_dir)
|
||||
if basename == ".":
|
||||
# parent_dir == tree_root
|
||||
continue
|
||||
dirs, _ = self.dvcignore(os.path.abspath(dirname), [basename], [])
|
||||
if not dirs:
|
||||
return False
|
||||
return True
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
for root, dirs, files in self.tree.walk(top, topdown):
|
||||
@@ -148,4 +218,6 @@ class CleanTree(BaseTree):
|
||||
yield root, dirs, files
|
||||
|
||||
def stat(self, path):
|
||||
return self.tree.stat(path)
|
||||
if self.exists(path):
|
||||
return self.tree.stat(path)
|
||||
raise FileNotFoundError
|
||||
|
||||
+9
-5
@@ -56,6 +56,12 @@ class BaseOutput:
|
||||
PARAM_METRIC_XPATH = "xpath"
|
||||
PARAM_PLOT = "plot"
|
||||
PARAM_PLOT_TEMPLATE = "template"
|
||||
PARAM_PLOT_X = "x"
|
||||
PARAM_PLOT_Y = "y"
|
||||
PARAM_PLOT_XLAB = "xlab"
|
||||
PARAM_PLOT_YLAB = "ylab"
|
||||
PARAM_PLOT_TITLE = "title"
|
||||
PARAM_PLOT_CSV_HEADER = "csv_header"
|
||||
PARAM_PERSIST = "persist"
|
||||
|
||||
METRIC_SCHEMA = Any(
|
||||
@@ -245,17 +251,15 @@ class BaseOutput:
|
||||
if self.metric or self.plot:
|
||||
self.verify_metric()
|
||||
if not self.IS_DEPENDENCY:
|
||||
logger.info(
|
||||
"Output '{}' doesn't use cache. Skipping saving.".format(
|
||||
self
|
||||
)
|
||||
logger.debug(
|
||||
"Output '%s' doesn't use cache. Skipping saving.", self
|
||||
)
|
||||
return
|
||||
|
||||
assert not self.IS_DEPENDENCY
|
||||
|
||||
if not self.changed():
|
||||
logger.info(f"Output '{self}' didn't change. Skipping saving.")
|
||||
logger.debug("Output '%s' didn't change. Skipping saving.", self)
|
||||
return
|
||||
|
||||
self.info = self.save_info()
|
||||
|
||||
@@ -340,7 +340,7 @@ class GDriveRemote(BaseRemote):
|
||||
"use the `gdrive_user_credentials_file` config\n"
|
||||
" option if you use multiple GDrive remotes with "
|
||||
"different email accounts.\n\nDetails".format(
|
||||
item_id, self.path_info, self._credentials_location
|
||||
item_id, self.path_info, self.credentials_location
|
||||
)
|
||||
) from exc
|
||||
raise
|
||||
|
||||
@@ -237,11 +237,15 @@ class SSHConnection:
|
||||
or channel.recv_ready()
|
||||
or channel.recv_stderr_ready()
|
||||
):
|
||||
import select
|
||||
import selectors
|
||||
|
||||
selector = selectors.DefaultSelector()
|
||||
selector.register(stdout.channel, selectors.EVENT_READ)
|
||||
|
||||
got_chunk = False
|
||||
readq, _, _ = select.select([stdout.channel], [], [], self.timeout)
|
||||
for c in readq:
|
||||
events = selector.select(self.timeout)
|
||||
for key, _ in events:
|
||||
c = key.fileobj
|
||||
if c.recv_ready():
|
||||
stdout_chunks.append(stdout.channel.recv(len(c.in_buffer)))
|
||||
got_chunk = True
|
||||
|
||||
+105
-29
@@ -1,3 +1,4 @@
|
||||
import logging
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
@@ -5,17 +6,26 @@ from functools import wraps
|
||||
from funcy import cached_property, cat, first
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.dvcfile import PIPELINE_FILE, Dvcfile, is_valid_filename
|
||||
from dvc.exceptions import FileMissingError
|
||||
from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
|
||||
from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
|
||||
from dvc.exceptions import (
|
||||
NoOutputOrStageError,
|
||||
NotDvcRepoError,
|
||||
OutputNotFoundError,
|
||||
)
|
||||
from dvc.ignore import CleanTree
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo.tree import RepoTree
|
||||
from dvc.scm.tree import is_working_tree
|
||||
from dvc.utils.fs import path_isin
|
||||
|
||||
from ..stage.exceptions import StageFileDoesNotExistError, StageNotFound
|
||||
from ..utils import parse_target
|
||||
from .graph import check_acyclic, get_pipeline, get_pipelines
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def locked(f):
|
||||
@wraps(f)
|
||||
@@ -38,7 +48,7 @@ class Repo:
|
||||
from dvc.repo.add import add
|
||||
from dvc.repo.remove import remove
|
||||
from dvc.repo.ls import ls
|
||||
from dvc.repo.lock import lock as lock_stage
|
||||
from dvc.repo.freeze import freeze, unfreeze
|
||||
from dvc.repo.move import move
|
||||
from dvc.repo.run import run
|
||||
from dvc.repo.imp import imp
|
||||
@@ -124,7 +134,13 @@ class Repo:
|
||||
|
||||
@tree.setter
|
||||
def tree(self, tree):
|
||||
self._tree = tree if isinstance(tree, CleanTree) else CleanTree(tree)
|
||||
if is_working_tree(tree) or tree.tree_root == self.root_dir:
|
||||
root = None
|
||||
else:
|
||||
root = self.root_dir
|
||||
self._tree = (
|
||||
tree if isinstance(tree, CleanTree) else CleanTree(tree, root)
|
||||
)
|
||||
# Our graph cache is no longer valid, as it was based on the previous
|
||||
# tree.
|
||||
self._reset()
|
||||
@@ -181,6 +197,25 @@ class Repo:
|
||||
|
||||
self.scm.ignore_list(flist)
|
||||
|
||||
def get_stage(self, path=None, name=None):
|
||||
if not path:
|
||||
path = PIPELINE_FILE
|
||||
logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
|
||||
|
||||
dvcfile = Dvcfile(self, path)
|
||||
return dvcfile.stages[name]
|
||||
|
||||
def get_stages(self, path=None, name=None):
|
||||
if not path:
|
||||
path = PIPELINE_FILE
|
||||
logger.debug("Assuming '%s' to be a stage inside '%s'", name, path)
|
||||
|
||||
if name:
|
||||
return [self.get_stage(path, name)]
|
||||
|
||||
dvcfile = Dvcfile(self, path)
|
||||
return list(dvcfile.stages.values())
|
||||
|
||||
def check_modified_graph(self, new_stages):
|
||||
"""Generate graph including the new stage to check for errors"""
|
||||
# Building graph might be costly for the ones with many DVC-files,
|
||||
@@ -204,10 +239,9 @@ class Repo:
|
||||
stages = nx.dfs_postorder_nodes(graph)
|
||||
return [stage for stage in stages if path_isin(stage.path, path)]
|
||||
|
||||
def collect(self, target, with_deps=False, recursive=False, graph=None):
|
||||
import networkx as nx
|
||||
from ..dvcfile import Dvcfile
|
||||
|
||||
def collect(
|
||||
self, target=None, with_deps=False, recursive=False, graph=None
|
||||
):
|
||||
if not target:
|
||||
return list(graph) if graph else self.stages
|
||||
|
||||
@@ -217,36 +251,81 @@ class Repo:
|
||||
)
|
||||
|
||||
path, name = parse_target(target)
|
||||
dvcfile = Dvcfile(self, path)
|
||||
stages = list(dvcfile.stages.filter(name).values())
|
||||
stages = self.get_stages(path, name)
|
||||
if not with_deps:
|
||||
return stages
|
||||
|
||||
res = set()
|
||||
for stage in stages:
|
||||
pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
|
||||
res.update(nx.dfs_postorder_nodes(pipeline, stage))
|
||||
res.update(self._collect_pipeline(stage, graph=graph))
|
||||
return res
|
||||
|
||||
def collect_granular(self, target, *args, **kwargs):
|
||||
from ..dvcfile import Dvcfile, is_valid_filename
|
||||
def _collect_pipeline(self, stage, graph=None):
|
||||
import networkx as nx
|
||||
|
||||
pipeline = get_pipeline(get_pipelines(graph or self.graph), stage)
|
||||
return nx.dfs_postorder_nodes(pipeline, stage)
|
||||
|
||||
def _collect_from_default_dvcfile(self, target):
|
||||
dvcfile = Dvcfile(self, PIPELINE_FILE)
|
||||
if dvcfile.exists():
|
||||
return dvcfile.stages.get(target)
|
||||
|
||||
def collect_granular(
|
||||
self, target=None, with_deps=False, recursive=False, graph=None
|
||||
):
|
||||
"""
|
||||
Priority is in the order of following in case of ambiguity:
|
||||
- .dvc file or .yaml file
|
||||
- dir if recursive and directory exists
|
||||
- stage_name
|
||||
- output file
|
||||
"""
|
||||
if not target:
|
||||
return [(stage, None) for stage in self.stages]
|
||||
|
||||
file, name = parse_target(target)
|
||||
if is_valid_filename(file) and not kwargs.get("with_deps"):
|
||||
# Optimization: do not collect the graph for a specific .dvc target
|
||||
stages = Dvcfile(self, file).stages.filter(name)
|
||||
return [(stage, None) for stage in stages.values()]
|
||||
stages = []
|
||||
|
||||
try:
|
||||
(out,) = self.find_outs_by_path(file, strict=False)
|
||||
filter_info = PathInfo(os.path.abspath(file))
|
||||
return [(out.stage, filter_info)]
|
||||
except OutputNotFoundError:
|
||||
stages = self.collect(target, *args, **kwargs)
|
||||
return [(stage, None) for stage in stages]
|
||||
# Optimization: do not collect the graph for a specific target
|
||||
if not file:
|
||||
# parsing is ambiguous when it does not have a colon
|
||||
# or if it's not a dvcfile, as it can be a stage name
|
||||
# in `dvc.yaml` or, an output in a stage.
|
||||
logger.debug(
|
||||
"Checking if stage '%s' is in '%s'", target, PIPELINE_FILE
|
||||
)
|
||||
if not (recursive and os.path.isdir(target)):
|
||||
stage = self._collect_from_default_dvcfile(target)
|
||||
if stage:
|
||||
stages = (
|
||||
self._collect_pipeline(stage) if with_deps else [stage]
|
||||
)
|
||||
elif not with_deps and is_valid_filename(file):
|
||||
stages = self.get_stages(file, name)
|
||||
|
||||
if not stages:
|
||||
if not (recursive and os.path.isdir(target)):
|
||||
try:
|
||||
(out,) = self.find_outs_by_path(target, strict=False)
|
||||
filter_info = PathInfo(os.path.abspath(target))
|
||||
return [(out.stage, filter_info)]
|
||||
except OutputNotFoundError:
|
||||
pass
|
||||
|
||||
try:
|
||||
stages = self.collect(target, with_deps, recursive, graph)
|
||||
except StageFileDoesNotExistError as exc:
|
||||
# collect() might try to use `target` as a stage name
|
||||
# and throw error that dvc.yaml does not exist, whereas it
|
||||
# should say that both stage name and file does not exist.
|
||||
if file and is_valid_filename(file):
|
||||
raise
|
||||
raise NoOutputOrStageError(target, exc.file) from exc
|
||||
except StageNotFound as exc:
|
||||
raise NoOutputOrStageError(target, exc.file) from exc
|
||||
|
||||
return [(stage, None) for stage in stages]
|
||||
|
||||
def used_cache(
|
||||
self,
|
||||
@@ -362,7 +441,7 @@ class Repo:
|
||||
stages = stages or self.stages
|
||||
outs = Trie() # Use trie to efficiently find overlapping outs and deps
|
||||
|
||||
for stage in filter(bool, stages):
|
||||
for stage in filter(bool, stages): # bug? not using it later
|
||||
for out in stage.outs:
|
||||
out_key = out.path_info.parts
|
||||
|
||||
@@ -443,16 +522,13 @@ class Repo:
|
||||
return PlotTemplates(self.dvc_dir)
|
||||
|
||||
def _collect_stages(self):
|
||||
from dvc.dvcfile import Dvcfile, is_valid_filename
|
||||
|
||||
stages = []
|
||||
outs = set()
|
||||
|
||||
for root, dirs, files in self.tree.walk(self.root_dir):
|
||||
for file_name in filter(is_valid_filename, files):
|
||||
path = os.path.join(root, file_name)
|
||||
stage_loader = Dvcfile(self, path).stages
|
||||
stages.extend(stage_loader.values())
|
||||
stages.extend(self.get_stages(path))
|
||||
outs.update(
|
||||
out.fspath
|
||||
for stage in stages
|
||||
|
||||
@@ -18,7 +18,7 @@ def brancher( # noqa: E302
|
||||
str: the display name for the currently selected tree, it could be:
|
||||
- a git revision identifier
|
||||
- empty string it there is no branches to iterate over
|
||||
- "Working Tree" if there are uncommitted changes in the SCM repo
|
||||
- "workspace" if there are uncommitted changes in the SCM repo
|
||||
"""
|
||||
if not any([revs, all_branches, all_tags, all_commits]):
|
||||
yield ""
|
||||
@@ -30,10 +30,10 @@ def brancher( # noqa: E302
|
||||
scm = self.scm
|
||||
|
||||
self.tree = WorkingTree(self.root_dir)
|
||||
yield "working tree"
|
||||
yield "workspace"
|
||||
|
||||
if revs and "working tree" in revs:
|
||||
revs.remove("working tree")
|
||||
if revs and "workspace" in revs:
|
||||
revs.remove("workspace")
|
||||
|
||||
if all_commits:
|
||||
revs = scm.list_all_commits()
|
||||
@@ -48,6 +48,9 @@ def brancher( # noqa: E302
|
||||
if revs:
|
||||
for sha, names in group_by(scm.resolve_rev, revs).items():
|
||||
self.tree = scm.get_tree(sha)
|
||||
yield ", ".join(names)
|
||||
# ignore revs that don't contain repo root
|
||||
# (i.e. revs from before a subdir=True repo was init'ed)
|
||||
if self.tree.exists(self.root_dir):
|
||||
yield ", ".join(names)
|
||||
finally:
|
||||
self.tree = saved_tree
|
||||
|
||||
+14
-8
@@ -1,7 +1,11 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
from dvc.exceptions import CheckoutError, CheckoutErrorSuggestGit
|
||||
from dvc.exceptions import (
|
||||
CheckoutError,
|
||||
CheckoutErrorSuggestGit,
|
||||
NoOutputOrStageError,
|
||||
)
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.utils import relpath
|
||||
|
||||
@@ -18,11 +22,11 @@ def _get_unused_links(repo):
|
||||
return repo.state.get_unused_links(used)
|
||||
|
||||
|
||||
def _fspath_dir(path, root):
|
||||
def _fspath_dir(path):
|
||||
if not os.path.exists(str(path)):
|
||||
return str(path)
|
||||
|
||||
path = relpath(path, root)
|
||||
path = relpath(path)
|
||||
return os.path.join(path, "") if os.path.isdir(path) else path
|
||||
|
||||
|
||||
@@ -56,7 +60,7 @@ def _checkout(
|
||||
targets = [None]
|
||||
unused = _get_unused_links(self)
|
||||
|
||||
stats["deleted"] = [_fspath_dir(u, self.root_dir) for u in unused]
|
||||
stats["deleted"] = [_fspath_dir(u) for u in unused]
|
||||
self.state.remove_links(unused)
|
||||
|
||||
if isinstance(targets, str):
|
||||
@@ -70,7 +74,11 @@ def _checkout(
|
||||
target, with_deps=with_deps, recursive=recursive
|
||||
)
|
||||
)
|
||||
except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
|
||||
except (
|
||||
StageFileDoesNotExistError,
|
||||
StageFileBadNameError,
|
||||
NoOutputOrStageError,
|
||||
) as exc:
|
||||
if not target:
|
||||
raise
|
||||
raise CheckoutErrorSuggestGit(target) from exc
|
||||
@@ -87,9 +95,7 @@ def _checkout(
|
||||
filter_info=filter_info,
|
||||
)
|
||||
for key, items in result.items():
|
||||
stats[key].extend(
|
||||
_fspath_dir(path, self.root_dir) for path in items
|
||||
)
|
||||
stats[key].extend(_fspath_dir(path) for path in items)
|
||||
|
||||
if stats.get("failed"):
|
||||
raise CheckoutError(stats["failed"], stats)
|
||||
|
||||
@@ -46,3 +46,4 @@ def commit(self, target, with_deps=False, recursive=False, force=False):
|
||||
stage.commit()
|
||||
|
||||
Dvcfile(self, stage.path).dump(stage)
|
||||
return stages
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ from dvc.scm.tree import is_working_tree
|
||||
@locked
|
||||
def diff(self, a_rev="HEAD", b_rev=None):
|
||||
"""
|
||||
By default, it compares the working tree with the last commit's tree.
|
||||
By default, it compares the workspace with the last commit's tree.
|
||||
|
||||
This implementation differs from `git diff` since DVC doesn't have
|
||||
the concept of `index`, but it keeps the same interface, thus,
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
from . import locked
|
||||
|
||||
|
||||
@locked
|
||||
def _set(repo, target, frozen):
|
||||
from dvc.utils import parse_target
|
||||
|
||||
path, name = parse_target(target)
|
||||
stage = repo.get_stage(path, name)
|
||||
stage.frozen = frozen
|
||||
stage.dvcfile.dump(stage, update_pipeline=True)
|
||||
|
||||
return stage
|
||||
|
||||
|
||||
def freeze(repo, target):
|
||||
return _set(repo, target, True)
|
||||
|
||||
|
||||
def unfreeze(repo, target):
|
||||
return _set(repo, target, False)
|
||||
+1
-1
@@ -3,4 +3,4 @@ def imp(self, url, path, out=None, rev=None):
|
||||
if rev is not None:
|
||||
erepo["rev"] = rev
|
||||
|
||||
return self.imp_url(path, out=out, erepo=erepo, locked=True)
|
||||
return self.imp_url(path, out=out, erepo=erepo, frozen=True)
|
||||
|
||||
+4
-4
@@ -5,12 +5,12 @@ from dvc.utils import relpath, resolve_output, resolve_paths
|
||||
from dvc.utils.fs import path_isin
|
||||
|
||||
from ..exceptions import OutputDuplicationError
|
||||
from . import locked as locked_repo
|
||||
from . import locked
|
||||
|
||||
|
||||
@locked_repo
|
||||
@locked
|
||||
@scm_context
|
||||
def imp_url(self, url, out=None, fname=None, erepo=None, locked=True):
|
||||
def imp_url(self, url, out=None, fname=None, erepo=None, frozen=True):
|
||||
from dvc.dvcfile import Dvcfile
|
||||
from dvc.stage import Stage, create_stage
|
||||
|
||||
@@ -48,7 +48,7 @@ def imp_url(self, url, out=None, fname=None, erepo=None, locked=True):
|
||||
|
||||
stage.run()
|
||||
|
||||
stage.locked = locked
|
||||
stage.frozen = frozen
|
||||
|
||||
dvcfile.dump(stage)
|
||||
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
from . import locked
|
||||
|
||||
|
||||
@locked
|
||||
def lock(self, target, unlock=False):
|
||||
from .. import dvcfile
|
||||
from dvc.utils import parse_target
|
||||
|
||||
path, name = parse_target(target)
|
||||
dvcfile = dvcfile.Dvcfile(self, path)
|
||||
stage = dvcfile.stages[name]
|
||||
stage.locked = False if unlock else True
|
||||
dvcfile.dump(stage, update_pipeline=True)
|
||||
|
||||
return stage
|
||||
@@ -2,21 +2,11 @@ class Metrics:
|
||||
def __init__(self, repo):
|
||||
self.repo = repo
|
||||
|
||||
def add(self, *args, **kwargs):
|
||||
from dvc.repo.metrics.add import add
|
||||
|
||||
return add(self.repo, *args, **kwargs)
|
||||
|
||||
def show(self, *args, **kwargs):
|
||||
from dvc.repo.metrics.show import show
|
||||
|
||||
return show(self.repo, *args, **kwargs)
|
||||
|
||||
def remove(self, *args, **kwargs):
|
||||
from dvc.repo.metrics.remove import remove
|
||||
|
||||
return remove(self.repo, *args, **kwargs)
|
||||
|
||||
def diff(self, *args, **kwargs):
|
||||
from .diff import diff
|
||||
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
from dvc.repo.metrics.modify import modify
|
||||
|
||||
|
||||
def add(repo, path):
|
||||
modify(repo, path)
|
||||
@@ -1,22 +0,0 @@
|
||||
from dvc.dvcfile import Dvcfile
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.repo import locked
|
||||
|
||||
|
||||
@locked
|
||||
def modify(repo, path, delete=False):
|
||||
outs = repo.find_outs_by_path(path)
|
||||
assert len(outs) == 1
|
||||
out = outs[0]
|
||||
|
||||
if out.scheme != "local":
|
||||
msg = "output '{}' scheme '{}' is not supported for metrics"
|
||||
raise DvcException(msg.format(out.path, out.path_info.scheme))
|
||||
|
||||
if delete:
|
||||
out.metric = None
|
||||
|
||||
out.verify_metric()
|
||||
|
||||
dvcfile = Dvcfile(repo, out.stage.path)
|
||||
dvcfile.dump(out.stage)
|
||||
@@ -1,5 +0,0 @@
|
||||
from dvc.repo.metrics.modify import modify
|
||||
|
||||
|
||||
def remove(repo, path):
|
||||
modify(repo, path, delete=True)
|
||||
@@ -104,13 +104,13 @@ def show(
|
||||
if not res:
|
||||
raise NoMetricsError()
|
||||
|
||||
# Hide working tree metrics if they are the same as in the active branch
|
||||
# Hide workspace metrics if they are the same as in the active branch
|
||||
try:
|
||||
active_branch = repo.scm.active_branch()
|
||||
except TypeError:
|
||||
pass # Detached head
|
||||
else:
|
||||
if res.get("working tree") == res.get(active_branch):
|
||||
res.pop("working tree", None)
|
||||
if res.get("workspace") == res.get(active_branch):
|
||||
res.pop("workspace", None)
|
||||
|
||||
return res
|
||||
|
||||
@@ -58,13 +58,13 @@ def show(repo, revs=None):
|
||||
if not res:
|
||||
raise NoParamsError("no parameter configs files in this repository")
|
||||
|
||||
# Hide working tree params if they are the same as in the active branch
|
||||
# Hide workspace params if they are the same as in the active branch
|
||||
try:
|
||||
active_branch = repo.scm.active_branch()
|
||||
except TypeError:
|
||||
pass # Detached head
|
||||
else:
|
||||
if res.get("working tree") == res.get(active_branch):
|
||||
res.pop("working tree", None)
|
||||
if res.get("workspace") == res.get(active_branch):
|
||||
res.pop("workspace", None)
|
||||
|
||||
return res
|
||||
|
||||
@@ -11,3 +11,25 @@ class Plots:
|
||||
from .diff import diff
|
||||
|
||||
return diff(self.repo, *args, **kwargs)
|
||||
|
||||
def modify(self, path, props=None, unset=None):
|
||||
from dvc.dvcfile import Dvcfile
|
||||
|
||||
(out,) = self.repo.find_outs_by_path(path)
|
||||
|
||||
# This out will become a plot unless it is one already
|
||||
if not isinstance(out.plot, dict):
|
||||
out.plot = {}
|
||||
|
||||
for field in unset or ():
|
||||
out.plot.pop(field, None)
|
||||
out.plot.update(props or {})
|
||||
|
||||
# Empty dict will move it to non-plots
|
||||
if not out.plot:
|
||||
out.plot = True
|
||||
|
||||
out.verify_metric()
|
||||
|
||||
dvcfile = Dvcfile(self.repo, out.stage.path)
|
||||
dvcfile.dump(out.stage, update_pipeline=True)
|
||||
|
||||
@@ -258,7 +258,7 @@ def _load_from_revisions(repo, datafile, revisions):
|
||||
exceptions = []
|
||||
|
||||
for rev in repo.brancher(revs=revisions):
|
||||
if rev == "working tree" and rev not in revisions:
|
||||
if rev == "workspace" and rev not in revisions:
|
||||
continue
|
||||
|
||||
try:
|
||||
|
||||
@@ -3,7 +3,7 @@ def _revisions(revs, is_dirty):
|
||||
if len(revisions) <= 1:
|
||||
if len(revisions) == 0 and is_dirty:
|
||||
revisions.append("HEAD")
|
||||
revisions.append("working tree")
|
||||
revisions.append("workspace")
|
||||
return revisions
|
||||
|
||||
|
||||
|
||||
+86
-90
@@ -1,13 +1,14 @@
|
||||
import copy
|
||||
import logging
|
||||
import os
|
||||
from collections import defaultdict
|
||||
|
||||
from funcy import first, last
|
||||
from funcy import first, last, project
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.repo import locked
|
||||
from dvc.schema import PLOT_PROPS
|
||||
|
||||
from .data import PlotData
|
||||
from .data import NoMetricInHistoryError, PlotData
|
||||
from .template import NoDataForTemplateError, Template
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -43,29 +44,24 @@ def _evaluate_templatepath(repo, template=None):
|
||||
|
||||
@locked
|
||||
def fill_template(
|
||||
repo,
|
||||
datafile,
|
||||
template_path,
|
||||
revisions,
|
||||
fields=None,
|
||||
path=None,
|
||||
csv_header=True,
|
||||
x_field=None,
|
||||
y_field=None,
|
||||
**kwargs,
|
||||
repo, datafile, template_path, revisions, props,
|
||||
):
|
||||
if x_field and fields:
|
||||
fields.add(x_field)
|
||||
# Copy things to not modify passed values
|
||||
props = props.copy()
|
||||
fields = copy.copy(props.get("fields"))
|
||||
|
||||
if y_field and fields:
|
||||
fields.add(y_field)
|
||||
if props.get("x") and fields:
|
||||
fields.add(props.get("x"))
|
||||
|
||||
if props.get("y") and fields:
|
||||
fields.add(props.get("y"))
|
||||
|
||||
template_datafiles, x_anchor, y_anchor = _parse_template(
|
||||
template_path, datafile
|
||||
)
|
||||
append_index = x_anchor and not x_field
|
||||
append_index = x_anchor and not props.get("x")
|
||||
if append_index:
|
||||
x_field = PlotData.INDEX_FIELD
|
||||
props["x"] = PlotData.INDEX_FIELD
|
||||
|
||||
template_data = {}
|
||||
for template_datafile in template_datafiles:
|
||||
@@ -76,13 +72,13 @@ def fill_template(
|
||||
for pd in plot_datas:
|
||||
rev_data_points = pd.to_datapoints(
|
||||
fields=fields,
|
||||
path=path,
|
||||
csv_header=csv_header,
|
||||
path=props.get("path"),
|
||||
csv_header=props.get("csv_header", True),
|
||||
append_index=append_index,
|
||||
)
|
||||
|
||||
if y_anchor and not y_field:
|
||||
y_field = _infer_y_field(rev_data_points, x_field)
|
||||
if y_anchor and not props.get("y"):
|
||||
props["y"] = _infer_y_field(rev_data_points, props.get("x"))
|
||||
tmp_data.extend(rev_data_points)
|
||||
|
||||
template_data[template_datafile] = tmp_data
|
||||
@@ -91,12 +87,7 @@ def fill_template(
|
||||
raise NoDataForTemplateError(template_path)
|
||||
|
||||
content = Template.fill(
|
||||
template_path,
|
||||
template_data,
|
||||
priority_datafile=datafile,
|
||||
x_field=x_field,
|
||||
y_field=y_field,
|
||||
**kwargs,
|
||||
template_path, template_data, priority_datafile=datafile, props=props,
|
||||
)
|
||||
|
||||
path = datafile or ",".join(template_datafiles)
|
||||
@@ -113,17 +104,17 @@ def _infer_y_field(rev_data_points, x_field):
|
||||
return y_field
|
||||
|
||||
|
||||
def _show(repo, datafile=None, template=None, revs=None, **kwargs):
|
||||
def _show(repo, datafile=None, revs=None, props=None):
|
||||
if revs is None:
|
||||
revs = ["working tree"]
|
||||
revs = ["workspace"]
|
||||
|
||||
if not datafile and not template:
|
||||
if not datafile and not props.get("template"):
|
||||
raise NoDataOrTemplateProvided()
|
||||
|
||||
template_path = _evaluate_templatepath(repo, template)
|
||||
template_path = _evaluate_templatepath(repo, props.get("template"))
|
||||
|
||||
plot_datafile, plot_content = fill_template(
|
||||
repo, datafile, template_path, revs, **kwargs
|
||||
repo, datafile, template_path, revs, props
|
||||
)
|
||||
|
||||
return plot_datafile, plot_content
|
||||
@@ -148,67 +139,72 @@ def _parse_template(template_path, priority_datafile):
|
||||
)
|
||||
|
||||
|
||||
def _collect_plots(repo):
|
||||
plots = defaultdict(set)
|
||||
def _collect_plots(repo, targets=None):
|
||||
from dvc.exceptions import OutputNotFoundError
|
||||
from contextlib import suppress
|
||||
|
||||
for stage in repo.stages:
|
||||
for out in stage.outs:
|
||||
if not out.plot:
|
||||
continue
|
||||
|
||||
if isinstance(out.plot, dict):
|
||||
template = out.plot[out.PARAM_PLOT_TEMPLATE]
|
||||
else:
|
||||
template = None
|
||||
|
||||
plots[str(out)].add(template)
|
||||
|
||||
return plots
|
||||
|
||||
|
||||
def show(repo, targets=None, template=None, revs=None, **kwargs) -> dict:
|
||||
if isinstance(targets, str):
|
||||
targets = [targets]
|
||||
def _targets_to_outs(targets):
|
||||
for t in targets:
|
||||
with suppress(OutputNotFoundError):
|
||||
(out,) = repo.find_outs_by_path(t)
|
||||
yield out
|
||||
|
||||
if targets:
|
||||
for target in targets:
|
||||
return {
|
||||
target: _show(
|
||||
repo,
|
||||
datafile=target,
|
||||
template=template,
|
||||
revs=revs,
|
||||
**kwargs,
|
||||
)[1]
|
||||
}
|
||||
|
||||
if not revs:
|
||||
plots = _collect_plots(repo)
|
||||
outs = _targets_to_outs(targets)
|
||||
else:
|
||||
plots = defaultdict(set)
|
||||
for rev in repo.brancher(revs=revs):
|
||||
for plot, templates in _collect_plots(repo).items():
|
||||
plots[plot].update(templates)
|
||||
outs = (out for stage in repo.stages for out in stage.outs if out.plot)
|
||||
|
||||
return {str(out): _plot_props(out) for out in outs}
|
||||
|
||||
|
||||
def _plot_props(out):
|
||||
if not out.plot:
|
||||
raise DvcException(
|
||||
f"'{out}' is not a plot. Use `dvc plots modify` to change that."
|
||||
)
|
||||
if isinstance(out.plot, list):
|
||||
raise DvcException("Multiple plots per data file not supported yet.")
|
||||
if isinstance(out.plot, bool):
|
||||
return {}
|
||||
|
||||
return project(out.plot, PLOT_PROPS)
|
||||
|
||||
|
||||
def show(repo, targets=None, revs=None, props=None) -> dict:
|
||||
if isinstance(targets, str):
|
||||
targets = [targets]
|
||||
if props is None:
|
||||
props = {}
|
||||
|
||||
# Collect plot data files with associated props
|
||||
plots = {}
|
||||
for rev in repo.brancher(revs=revs):
|
||||
if revs is not None and rev not in revs:
|
||||
continue
|
||||
|
||||
for datafile, file_props in _collect_plots(repo, targets).items():
|
||||
# props from command line overwrite plot props from out definition
|
||||
full_props = {**file_props, **props}
|
||||
|
||||
if datafile in plots:
|
||||
saved_rev, saved_props = plots[datafile]
|
||||
if saved_props != props:
|
||||
logger.warning(
|
||||
f"Inconsistent plot props for '{datafile}' in "
|
||||
f"'{saved_rev}' and '{rev}'. "
|
||||
f"Going to use ones from '{saved_rev}'"
|
||||
)
|
||||
else:
|
||||
plots[datafile] = rev, full_props
|
||||
|
||||
if not plots:
|
||||
datafile, plot = _show(
|
||||
repo, datafile=None, template=template, revs=revs, **kwargs
|
||||
)
|
||||
if targets:
|
||||
raise NoMetricInHistoryError(", ".join(targets))
|
||||
|
||||
datafile, plot = _show(repo, datafile=None, revs=revs, props=props)
|
||||
return {datafile: plot}
|
||||
|
||||
ret = {}
|
||||
for plot, templates in plots.items():
|
||||
tmplt = template
|
||||
if len(templates) == 1:
|
||||
tmplt = list(templates)[0]
|
||||
elif not template:
|
||||
raise TooManyTemplatesError(
|
||||
f"'{plot}' uses multiple templates '{templates}'. "
|
||||
"Use `-t|--template` to specify the template to use. "
|
||||
)
|
||||
|
||||
ret[plot] = _show(
|
||||
repo, datafile=plot, template=tmplt, revs=revs, **kwargs
|
||||
)[1]
|
||||
|
||||
return ret
|
||||
return {
|
||||
datafile: _show(repo, datafile=datafile, revs=revs, props=props)[1]
|
||||
for datafile, (_, props) in plots.items()
|
||||
}
|
||||
|
||||
+25
-41
@@ -86,29 +86,24 @@ class Template:
|
||||
|
||||
@staticmethod
|
||||
def fill(
|
||||
template_path,
|
||||
data,
|
||||
priority_datafile=None,
|
||||
x_field=None,
|
||||
y_field=None,
|
||||
title=None,
|
||||
x_title=None,
|
||||
y_title=None,
|
||||
template_path, data, priority_datafile=None, props=None,
|
||||
):
|
||||
props = props or {}
|
||||
|
||||
with open(template_path) as fobj:
|
||||
result_content = fobj.read()
|
||||
|
||||
if x_field:
|
||||
Template._check_field_exists(data, x_field)
|
||||
if y_field:
|
||||
Template._check_field_exists(data, y_field)
|
||||
if props.get("x"):
|
||||
Template._check_field_exists(data, props.get("x"))
|
||||
if props.get("y"):
|
||||
Template._check_field_exists(data, props.get("y"))
|
||||
|
||||
result_content = Template._replace_data_anchors(
|
||||
result_content, data, priority_datafile
|
||||
)
|
||||
|
||||
result_content = Template._replace_metadata_anchors(
|
||||
result_content, title, x_field, x_title, y_field, y_title
|
||||
result_content, props
|
||||
)
|
||||
|
||||
return result_content
|
||||
@@ -122,34 +117,23 @@ class Template:
|
||||
raise NoFieldInDataError(field)
|
||||
|
||||
@staticmethod
|
||||
def _replace_metadata_anchors(
|
||||
result_content, title, x_field, x_title, y_field, y_title
|
||||
):
|
||||
if Template.TITLE_ANCHOR in result_content:
|
||||
if title:
|
||||
result_content = result_content.replace(
|
||||
Template.TITLE_ANCHOR, title
|
||||
)
|
||||
else:
|
||||
result_content = result_content.replace(
|
||||
Template.TITLE_ANCHOR, ""
|
||||
)
|
||||
if Template.X_ANCHOR in result_content and x_field:
|
||||
result_content = result_content.replace(Template.X_ANCHOR, x_field)
|
||||
if Template.Y_ANCHOR in result_content and y_field:
|
||||
result_content = result_content.replace(Template.Y_ANCHOR, y_field)
|
||||
if Template.X_TITLE_ANCHOR in result_content:
|
||||
if not x_title and x_field:
|
||||
x_title = x_field
|
||||
result_content = result_content.replace(
|
||||
Template.X_TITLE_ANCHOR, x_title
|
||||
)
|
||||
if Template.Y_TITLE_ANCHOR in result_content:
|
||||
if not y_title and y_field:
|
||||
y_title = y_field
|
||||
result_content = result_content.replace(
|
||||
Template.Y_TITLE_ANCHOR, y_title
|
||||
)
|
||||
def _replace_metadata_anchors(result_content, props):
|
||||
props.setdefault("title", "")
|
||||
props.setdefault("xlab", props.get("x"))
|
||||
props.setdefault("ylab", props.get("y"))
|
||||
|
||||
replace_pairs = [
|
||||
(Template.TITLE_ANCHOR, "title"),
|
||||
(Template.X_ANCHOR, "x"),
|
||||
(Template.Y_ANCHOR, "y"),
|
||||
(Template.X_TITLE_ANCHOR, "xlab"),
|
||||
(Template.Y_TITLE_ANCHOR, "ylab"),
|
||||
]
|
||||
for anchor, key in replace_pairs:
|
||||
value = props.get(key)
|
||||
if anchor in result_content and value is not None:
|
||||
result_content = result_content.replace(anchor, value)
|
||||
|
||||
return result_content
|
||||
|
||||
@staticmethod
|
||||
|
||||
+4
-9
@@ -7,16 +7,11 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@locked
|
||||
def remove(self, target, dvc_only=False):
|
||||
from ..dvcfile import Dvcfile
|
||||
|
||||
def remove(self, target, outs=False):
|
||||
path, name = parse_target(target)
|
||||
dvcfile = Dvcfile(self, path)
|
||||
stages = list(dvcfile.stages.filter(name).values())
|
||||
for stage in stages:
|
||||
stage.remove_outs(force=True)
|
||||
stages = self.get_stages(path, name)
|
||||
|
||||
if not dvc_only:
|
||||
dvcfile.remove()
|
||||
for stage in stages:
|
||||
stage.remove(remove_outs=outs, force=outs)
|
||||
|
||||
return stages
|
||||
|
||||
@@ -10,9 +10,9 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _reproduce_stage(stage, **kwargs):
|
||||
if stage.locked and not stage.is_import:
|
||||
if stage.frozen and not stage.is_import:
|
||||
logger.warning(
|
||||
"{} is locked. Its dependencies are"
|
||||
"{} is frozen. Its dependencies are"
|
||||
" not going to be reproduced.".format(stage)
|
||||
)
|
||||
|
||||
@@ -34,7 +34,7 @@ def _get_active_graph(G):
|
||||
|
||||
active = G.copy()
|
||||
for stage in G:
|
||||
if not stage.locked:
|
||||
if not stage.frozen:
|
||||
continue
|
||||
active.remove_edges_from(G.out_edges(stage))
|
||||
for edge in G.out_edges(stage):
|
||||
@@ -61,7 +61,6 @@ def reproduce(
|
||||
all_pipelines=False,
|
||||
**kwargs
|
||||
):
|
||||
from ..dvcfile import Dvcfile
|
||||
from dvc.utils import parse_target
|
||||
|
||||
if not target and not all_pipelines:
|
||||
@@ -81,8 +80,7 @@ def reproduce(
|
||||
if all_pipelines:
|
||||
pipelines = active_pipelines
|
||||
else:
|
||||
dvcfile = Dvcfile(self, path)
|
||||
stage = dvcfile.stages[name]
|
||||
stage = self.get_stage(path, name)
|
||||
pipelines = [get_pipeline(active_pipelines, stage)]
|
||||
|
||||
targets = []
|
||||
@@ -164,7 +162,12 @@ def _reproduce_stages(
|
||||
|
||||
force_downstream = kwargs.pop("force_downstream", False)
|
||||
result = []
|
||||
# `ret` is used to add a cosmetic newline.
|
||||
ret = []
|
||||
for stage in pipeline:
|
||||
if ret:
|
||||
logger.info("")
|
||||
|
||||
try:
|
||||
ret = _reproduce_stage(stage, **kwargs)
|
||||
|
||||
|
||||
+14
-6
@@ -3,7 +3,11 @@ import os
|
||||
from funcy import concat, first
|
||||
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
from dvc.stage.exceptions import DuplicateStageName, InvalidStageName
|
||||
from dvc.stage.exceptions import (
|
||||
DuplicateStageName,
|
||||
InvalidStageName,
|
||||
StageFileAlreadyExistsError,
|
||||
)
|
||||
|
||||
from ..exceptions import OutputDuplicationError
|
||||
from . import locked
|
||||
@@ -45,6 +49,9 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
|
||||
from dvc.stage import PipelineStage, Stage, create_stage
|
||||
from dvc.dvcfile import Dvcfile, PIPELINE_FILE
|
||||
|
||||
if not kwargs.get("cmd"):
|
||||
raise InvalidArgumentError("command is not specified")
|
||||
|
||||
stage_cls = PipelineStage
|
||||
path = PIPELINE_FILE
|
||||
stage_name = kwargs.get("name")
|
||||
@@ -71,10 +78,12 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
|
||||
|
||||
dvcfile = Dvcfile(self, stage.path)
|
||||
if dvcfile.exists():
|
||||
if stage_name and stage_name in dvcfile.stages:
|
||||
if kwargs.get("overwrite", True):
|
||||
dvcfile.remove_stage(stage)
|
||||
elif stage_cls != PipelineStage:
|
||||
raise StageFileAlreadyExistsError(dvcfile.relpath)
|
||||
elif stage_name and stage_name in dvcfile.stages:
|
||||
raise DuplicateStageName(stage_name, dvcfile)
|
||||
if stage_cls != PipelineStage:
|
||||
dvcfile.remove_with_prompt(force=kwargs.get("overwrite", True))
|
||||
|
||||
try:
|
||||
self.check_modified_graph([stage])
|
||||
@@ -82,8 +91,7 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
|
||||
raise OutputDuplicationError(exc.output, set(exc.stages) - {stage})
|
||||
|
||||
if no_exec:
|
||||
for out in stage.outs:
|
||||
out.ignore()
|
||||
stage.ignore_outs()
|
||||
else:
|
||||
stage.run(
|
||||
no_commit=kwargs.get("no_commit", False),
|
||||
|
||||
+2
-2
@@ -14,9 +14,9 @@ def _joint_status(stages):
|
||||
status = {}
|
||||
|
||||
for stage in stages:
|
||||
if stage.locked and not stage.is_repo_import:
|
||||
if stage.frozen and not stage.is_repo_import:
|
||||
logger.warning(
|
||||
"{} is locked. Its dependencies are"
|
||||
"{} is frozen. Its dependencies are"
|
||||
" not going to be shown in the status output.".format(stage)
|
||||
)
|
||||
|
||||
|
||||
+17
-4
@@ -11,7 +11,8 @@ SINGLE_STAGE_SCHEMA = {
|
||||
StageParams.PARAM_WDIR: Any(str, None),
|
||||
StageParams.PARAM_DEPS: Any([dependency.SCHEMA], None),
|
||||
StageParams.PARAM_OUTS: Any([output.SCHEMA], None),
|
||||
StageParams.PARAM_LOCKED: bool,
|
||||
StageParams.PARAM_LOCKED: bool, # backard compatibility
|
||||
StageParams.PARAM_FROZEN: bool,
|
||||
StageParams.PARAM_META: object,
|
||||
StageParams.PARAM_ALWAYS_CHANGED: bool,
|
||||
}
|
||||
@@ -29,8 +30,20 @@ OUT_PSTAGE_DETAILED_SCHEMA = {
|
||||
str: {BaseOutput.PARAM_CACHE: bool, BaseOutput.PARAM_PERSIST: bool}
|
||||
}
|
||||
|
||||
PLOT_PSTAGE_SCHEMA = OUT_PSTAGE_DETAILED_SCHEMA.copy()
|
||||
PLOT_PSTAGE_SCHEMA[str][BaseOutput.PARAM_PLOT_TEMPLATE] = str
|
||||
PLOT_PROPS = {
|
||||
BaseOutput.PARAM_PLOT_TEMPLATE: str,
|
||||
BaseOutput.PARAM_PLOT_X: str,
|
||||
BaseOutput.PARAM_PLOT_Y: str,
|
||||
BaseOutput.PARAM_PLOT_XLAB: str,
|
||||
BaseOutput.PARAM_PLOT_YLAB: str,
|
||||
BaseOutput.PARAM_PLOT_TITLE: str,
|
||||
BaseOutput.PARAM_PLOT_CSV_HEADER: bool,
|
||||
}
|
||||
PLOT_PROPS_SCHEMA = {
|
||||
**OUT_PSTAGE_DETAILED_SCHEMA[str],
|
||||
**PLOT_PROPS,
|
||||
}
|
||||
PLOT_PSTAGE_SCHEMA = {str: Any(PLOT_PROPS_SCHEMA, [PLOT_PROPS_SCHEMA])}
|
||||
|
||||
PARAM_PSTAGE_NON_DEFAULT_SCHEMA = {str: [str]}
|
||||
|
||||
@@ -42,7 +55,7 @@ SINGLE_PIPELINE_STAGE_SCHEMA = {
|
||||
Optional(StageParams.PARAM_PARAMS): [
|
||||
Any(str, PARAM_PSTAGE_NON_DEFAULT_SCHEMA)
|
||||
],
|
||||
Optional(StageParams.PARAM_LOCKED): bool,
|
||||
Optional(StageParams.PARAM_FROZEN): bool,
|
||||
Optional(StageParams.PARAM_META): object,
|
||||
Optional(StageParams.PARAM_ALWAYS_CHANGED): bool,
|
||||
Optional(StageParams.PARAM_OUTS): [
|
||||
|
||||
+3
-3
@@ -10,7 +10,7 @@ from dvc.output import BaseOutput
|
||||
from dvc.stage.params import StageParams
|
||||
from dvc.stage.utils import resolve_wdir
|
||||
from dvc.utils.collections import apply_diff
|
||||
from dvc.utils.stage import parse_stage_for_update
|
||||
from dvc.utils.yaml import parse_yaml_for_update
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.stage import PipelineStage, Stage
|
||||
@@ -115,7 +115,7 @@ def to_pipeline_file(stage: "PipelineStage"):
|
||||
stage.PARAM_PLOTS,
|
||||
_get_outs([out for out in stage.outs if out.plot]),
|
||||
),
|
||||
(stage.PARAM_LOCKED, stage.locked),
|
||||
(stage.PARAM_FROZEN, stage.frozen),
|
||||
(stage.PARAM_ALWAYS_CHANGED, stage.always_changed),
|
||||
]
|
||||
return {
|
||||
@@ -165,7 +165,7 @@ def to_single_stage_file(stage: "Stage"):
|
||||
# - apply changes to a returned structure
|
||||
# - serialize it
|
||||
if stage._stage_text is not None:
|
||||
saved_state = parse_stage_for_update(stage._stage_text, stage.path)
|
||||
saved_state = parse_yaml_for_update(stage._stage_text, stage.path)
|
||||
# Stage doesn't work with meta in any way, so .dumpd() doesn't
|
||||
# have it. We simply copy it over.
|
||||
if "meta" in saved_state:
|
||||
|
||||
+37
-41
@@ -42,6 +42,7 @@ def loads_from(cls, repo, path, wdir, data):
|
||||
[
|
||||
Stage.PARAM_CMD,
|
||||
Stage.PARAM_LOCKED,
|
||||
Stage.PARAM_FROZEN,
|
||||
Stage.PARAM_ALWAYS_CHANGED,
|
||||
Stage.PARAM_MD5,
|
||||
"name",
|
||||
@@ -77,7 +78,7 @@ def create_stage(cls, repo, path, **kwargs):
|
||||
logger.warning("Build cache is ignored when persisting outputs.")
|
||||
|
||||
if not ignore_run_cache and stage.can_be_skipped:
|
||||
logger.info("Stage is cached, skipping.")
|
||||
logger.info("Stage is cached, skipping")
|
||||
return None
|
||||
|
||||
return stage
|
||||
@@ -93,7 +94,8 @@ class Stage(params.StageParams):
|
||||
deps=None,
|
||||
outs=None,
|
||||
md5=None,
|
||||
locked=False,
|
||||
locked=False, # backward compatibility
|
||||
frozen=False,
|
||||
always_changed=False,
|
||||
stage_text=None,
|
||||
dvcfile=None,
|
||||
@@ -110,7 +112,7 @@ class Stage(params.StageParams):
|
||||
self.outs = outs
|
||||
self.deps = deps
|
||||
self.md5 = md5
|
||||
self.locked = locked
|
||||
self.frozen = locked or frozen
|
||||
self.always_changed = always_changed
|
||||
self._stage_text = stage_text
|
||||
self._dvcfile = dvcfile
|
||||
@@ -144,14 +146,10 @@ class Stage(params.StageParams):
|
||||
self._dvcfile = dvcfile
|
||||
|
||||
def __repr__(self):
|
||||
return "Stage: '{path}'".format(
|
||||
path=self.path_in_repo if self.path else "No path"
|
||||
)
|
||||
return f"Stage: '{self.addressing}'"
|
||||
|
||||
def __str__(self):
|
||||
return "stage: '{path}'".format(
|
||||
path=self.relpath if self.path else "No path"
|
||||
)
|
||||
return f"stage: '{self.addressing}'"
|
||||
|
||||
@property
|
||||
def addressing(self):
|
||||
@@ -159,7 +157,7 @@ class Stage(params.StageParams):
|
||||
Useful for alternative presentations where we don't need
|
||||
`Stage:` prefix.
|
||||
"""
|
||||
return self.relpath
|
||||
return self.relpath if self.path else "No path"
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.path_in_repo)
|
||||
@@ -205,14 +203,15 @@ class Stage(params.StageParams):
|
||||
return isinstance(self.deps[0], dependency.RepoDependency)
|
||||
|
||||
def changed_deps(self):
|
||||
if self.locked:
|
||||
if self.frozen:
|
||||
return False
|
||||
|
||||
if self.is_callback:
|
||||
logger.warning(
|
||||
'{stage} is a "callback" stage '
|
||||
logger.debug(
|
||||
'%s is a "callback" stage '
|
||||
"(has a command and no dependencies) and thus always "
|
||||
"considered as changed.".format(stage=self)
|
||||
"considered as changed.",
|
||||
self,
|
||||
)
|
||||
return True
|
||||
|
||||
@@ -248,9 +247,9 @@ class Stage(params.StageParams):
|
||||
|
||||
return False
|
||||
|
||||
def changed_stage(self, warn=False):
|
||||
def changed_stage(self):
|
||||
changed = self.md5 != self.compute_md5()
|
||||
if changed and warn:
|
||||
if changed:
|
||||
logger.debug(self._changed_stage_entry())
|
||||
return changed
|
||||
|
||||
@@ -259,14 +258,12 @@ class Stage(params.StageParams):
|
||||
is_changed = (
|
||||
# Short-circuit order: stage md5 is fast,
|
||||
# deps are expected to change
|
||||
self.changed_stage(warn=True)
|
||||
self.changed_stage()
|
||||
or self.changed_deps()
|
||||
or self.changed_outs()
|
||||
)
|
||||
if is_changed:
|
||||
logger.info("%s changed.", self)
|
||||
else:
|
||||
logger.info("%s didn't change.", self)
|
||||
logger.debug("%s changed.", self)
|
||||
return is_changed
|
||||
|
||||
@rwlocked(write=["outs"])
|
||||
@@ -285,17 +282,18 @@ class Stage(params.StageParams):
|
||||
out.unprotect()
|
||||
|
||||
@rwlocked(write=["outs"])
|
||||
def remove(self, force=False, remove_outs=True):
|
||||
def remove(self, force=False, remove_outs=True, purge=True):
|
||||
if remove_outs:
|
||||
self.remove_outs(ignore_remove=True, force=force)
|
||||
else:
|
||||
self.unprotect_outs()
|
||||
self.dvcfile.remove()
|
||||
if purge:
|
||||
self.dvcfile.remove_stage(self)
|
||||
|
||||
@rwlocked(read=["deps"], write=["outs"])
|
||||
def reproduce(self, interactive=False, **kwargs):
|
||||
|
||||
if not (kwargs.get("force", False) or self.changed()):
|
||||
logger.info("Stage '%s' didn't change, skipping", self.addressing)
|
||||
return None
|
||||
|
||||
msg = (
|
||||
@@ -379,6 +377,10 @@ class Stage(params.StageParams):
|
||||
for out in self.outs:
|
||||
out.save()
|
||||
|
||||
def ignore_outs(self):
|
||||
for out in self.outs:
|
||||
out.ignore()
|
||||
|
||||
@staticmethod
|
||||
def _changed_entries(entries):
|
||||
return [str(entry) for entry in entries if entry.changed_checksum()]
|
||||
@@ -402,16 +404,16 @@ class Stage(params.StageParams):
|
||||
|
||||
@rwlocked(read=["deps"], write=["outs"])
|
||||
def run(self, dry=False, no_commit=False, force=False, run_cache=True):
|
||||
if (self.cmd or self.is_import) and not self.locked and not dry:
|
||||
if (self.cmd or self.is_import) and not self.frozen and not dry:
|
||||
self.remove_outs(ignore_remove=False, force=False)
|
||||
|
||||
if not self.locked and self.is_import:
|
||||
if not self.frozen and self.is_import:
|
||||
sync_import(self, dry, force)
|
||||
elif not self.locked and self.cmd:
|
||||
elif not self.frozen and self.cmd:
|
||||
run_stage(self, dry, force, run_cache)
|
||||
else:
|
||||
args = (
|
||||
("outputs", "locked ") if self.locked else ("data sources", "")
|
||||
("outputs", "frozen ") if self.frozen else ("data sources", "")
|
||||
)
|
||||
logger.info("Verifying %s in %s%s", *args, self)
|
||||
if not dry:
|
||||
@@ -453,7 +455,7 @@ class Stage(params.StageParams):
|
||||
ret = []
|
||||
show_import = self.is_repo_import and check_updates
|
||||
|
||||
if not self.locked or show_import:
|
||||
if not self.frozen or show_import:
|
||||
self._status_deps(ret)
|
||||
self._status_outs(ret)
|
||||
self._status_always_changed(ret)
|
||||
@@ -527,19 +529,13 @@ class PipelineStage(Stage):
|
||||
def __hash__(self):
|
||||
return hash((self.path_in_repo, self.name))
|
||||
|
||||
def __repr__(self):
|
||||
return "Stage: '{path}:{name}'".format(
|
||||
path=self.relpath if self.path else "No path", name=self.name
|
||||
)
|
||||
|
||||
def __str__(self):
|
||||
return "stage: '{path}:{name}'".format(
|
||||
path=self.relpath if self.path else "No path", name=self.name
|
||||
)
|
||||
|
||||
@property
|
||||
def addressing(self):
|
||||
return super().addressing + ":" + self.name
|
||||
from dvc.dvcfile import PIPELINE_FILE
|
||||
|
||||
if self.path and self.relpath == PIPELINE_FILE:
|
||||
return self.name
|
||||
return f"{super().addressing}:{self.name}"
|
||||
|
||||
def reload(self):
|
||||
return self.dvcfile.stages[self.name]
|
||||
@@ -552,8 +548,8 @@ class PipelineStage(Stage):
|
||||
if self.cmd_changed:
|
||||
ret.append("changed command")
|
||||
|
||||
def changed_stage(self, warn=False):
|
||||
if self.cmd_changed and warn:
|
||||
def changed_stage(self):
|
||||
if self.cmd_changed:
|
||||
logger.debug(self._changed_stage_entry())
|
||||
return self.cmd_changed
|
||||
|
||||
|
||||
+5
-4
@@ -11,7 +11,7 @@ from dvc.serialize import to_single_stage_lockfile
|
||||
from dvc.stage.loader import StageLoader
|
||||
from dvc.utils import dict_sha256, relpath
|
||||
from dvc.utils.fs import makedirs
|
||||
from dvc.utils.stage import dump_stage_file
|
||||
from dvc.utils.yaml import dump_yaml
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -77,7 +77,7 @@ class StageCache:
|
||||
|
||||
return None
|
||||
|
||||
def _create_stage(self, cache):
|
||||
def _create_stage(self, cache, wdir=None):
|
||||
from dvc.stage import create_stage, PipelineStage
|
||||
|
||||
params = []
|
||||
@@ -98,6 +98,7 @@ class StageCache:
|
||||
repo=self.repo,
|
||||
path="dvc.yaml",
|
||||
cmd=cache["cmd"],
|
||||
wdir=wdir,
|
||||
params=params,
|
||||
deps=[dep["path"] for dep in cache.get("deps", [])],
|
||||
outs=[out["path"] for out in cache["outs"]],
|
||||
@@ -118,7 +119,7 @@ class StageCache:
|
||||
# NOTE: using temporary stage to avoid accidentally modifying original
|
||||
# stage and to workaround `commit/checkout` not working for uncached
|
||||
# outputs.
|
||||
cached_stage = self._create_stage(cache)
|
||||
cached_stage = self._create_stage(cache, wdir=stage.wdir)
|
||||
|
||||
outs_no_cache = [
|
||||
out.def_path for out in stage.outs if not out.use_cache
|
||||
@@ -153,7 +154,7 @@ class StageCache:
|
||||
path = self._get_cache_path(cache_key, cache_value)
|
||||
dpath = os.path.dirname(path)
|
||||
makedirs(dpath, exist_ok=True)
|
||||
dump_stage_file(path, cache)
|
||||
dump_yaml(path, cache)
|
||||
|
||||
def is_cached(self, stage):
|
||||
return bool(self._load(stage))
|
||||
|
||||
+5
-10
@@ -15,15 +15,8 @@ class StageFileFormatError(DvcException):
|
||||
|
||||
class StageFileDoesNotExistError(DvcException):
|
||||
def __init__(self, fname):
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX, is_dvc_file
|
||||
|
||||
msg = f"'{fname}' does not exist."
|
||||
|
||||
sname = fname + DVC_FILE_SUFFIX
|
||||
if is_dvc_file(sname):
|
||||
msg += f" Do you mean '{sname}'?"
|
||||
|
||||
super().__init__(msg)
|
||||
self.file = fname
|
||||
super().__init__(f"'{self.file}' does not exist.")
|
||||
|
||||
|
||||
class StageFileAlreadyExistsError(DvcException):
|
||||
@@ -87,8 +80,10 @@ class MissingDataSource(DvcException):
|
||||
|
||||
class StageNotFound(KeyError, DvcException):
|
||||
def __init__(self, file, name):
|
||||
self.file = file.relpath
|
||||
self.name = name
|
||||
super().__init__(
|
||||
f"Stage '{name}' not found inside '{file.relpath}' file"
|
||||
f"Stage '{self.name}' not found inside '{self.file}' file"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -5,12 +5,12 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
def update_import(stage, rev=None):
|
||||
stage.deps[0].update(rev=rev)
|
||||
locked = stage.locked
|
||||
stage.locked = False
|
||||
frozen = stage.frozen
|
||||
stage.frozen = False
|
||||
try:
|
||||
stage.reproduce()
|
||||
finally:
|
||||
stage.locked = locked
|
||||
stage.frozen = frozen
|
||||
|
||||
|
||||
def sync_import(stage, dry=False, force=False):
|
||||
@@ -23,11 +23,7 @@ def sync_import(stage, dry=False, force=False):
|
||||
if dry:
|
||||
return
|
||||
|
||||
if (
|
||||
not force
|
||||
and not stage.changed_stage(warn=True)
|
||||
and stage.already_cached()
|
||||
):
|
||||
if not force and not stage.changed_stage() and stage.already_cached():
|
||||
stage.outs[0].checkout()
|
||||
else:
|
||||
stage.deps[0].download(stage.outs[0])
|
||||
|
||||
@@ -30,16 +30,6 @@ class StageLoader(Mapping):
|
||||
self.stages_data = stages_data or {}
|
||||
self.lockfile_data = lockfile_data or {}
|
||||
|
||||
def filter(self, item=None):
|
||||
if not item:
|
||||
return self
|
||||
|
||||
if item not in self:
|
||||
raise StageNotFound(self.dvcfile, item)
|
||||
|
||||
data = {item: self.stages_data[item]} if item in self else {}
|
||||
return self.__class__(self.dvcfile, data, self.lockfile_data)
|
||||
|
||||
@staticmethod
|
||||
def fill_from_lock(stage, lock_data):
|
||||
from .params import StageParams
|
||||
@@ -224,9 +214,6 @@ class SingleStageLoader(Mapping):
|
||||
self.stage_data = stage_data or {}
|
||||
self.stage_text = stage_text
|
||||
|
||||
def filter(self, item=None):
|
||||
return self
|
||||
|
||||
def __getitem__(self, item):
|
||||
if item:
|
||||
logger.warning(
|
||||
|
||||
+2
-1
@@ -4,7 +4,8 @@ class StageParams:
|
||||
PARAM_WDIR = "wdir"
|
||||
PARAM_DEPS = "deps"
|
||||
PARAM_OUTS = "outs"
|
||||
PARAM_LOCKED = "locked"
|
||||
PARAM_LOCKED = "locked" # backward compatibility
|
||||
PARAM_FROZEN = "frozen"
|
||||
PARAM_META = "meta"
|
||||
PARAM_ALWAYS_CHANGED = "always_changed"
|
||||
PARAM_PARAMS = "params"
|
||||
|
||||
+17
-4
@@ -79,11 +79,15 @@ def cmd_run(stage, *args, **kwargs):
|
||||
|
||||
|
||||
def _is_cached(stage):
|
||||
return (
|
||||
cached = (
|
||||
not stage.is_callback
|
||||
and not stage.always_changed
|
||||
and not stage.changed_stage()
|
||||
and stage.already_cached()
|
||||
)
|
||||
if cached:
|
||||
logger.info("Stage '%s' is cached", stage.addressing)
|
||||
return cached
|
||||
|
||||
|
||||
def restored_from_cache(stage):
|
||||
@@ -93,7 +97,10 @@ def restored_from_cache(stage):
|
||||
return False
|
||||
# restore stage from build cache
|
||||
stage_cache.restore(stage)
|
||||
return stage.outs_cached()
|
||||
restored = stage.outs_cached()
|
||||
if restored:
|
||||
logger.info("Restored stage '%s' from run-cache", stage.addressing)
|
||||
return restored
|
||||
|
||||
|
||||
def run_stage(stage, dry=False, force=False, run_cache=False):
|
||||
@@ -102,10 +109,16 @@ def run_stage(stage, dry=False, force=False, run_cache=False):
|
||||
run_cache and restored_from_cache(stage)
|
||||
)
|
||||
if stage_cached:
|
||||
logger.info("Stage is cached, skipping.")
|
||||
logger.info("Skipping run, checking out outputs")
|
||||
stage.checkout()
|
||||
return
|
||||
|
||||
logger.info("Running command:\n\t%s", stage.cmd)
|
||||
callback_str = "callback " if stage.is_callback else ""
|
||||
logger.info(
|
||||
"Running %s" "stage '%s' with command:",
|
||||
callback_str,
|
||||
stage.addressing,
|
||||
)
|
||||
logger.info("\t%s", stage.cmd)
|
||||
if not dry:
|
||||
cmd_run(stage)
|
||||
|
||||
+3
-2
@@ -136,7 +136,8 @@ def compute_md5(stage):
|
||||
return dict_md5(
|
||||
d,
|
||||
exclude=[
|
||||
stage.PARAM_LOCKED,
|
||||
stage.PARAM_LOCKED, # backward compatibility
|
||||
stage.PARAM_FROZEN,
|
||||
BaseOutput.PARAM_METRIC,
|
||||
BaseOutput.PARAM_PERSIST,
|
||||
],
|
||||
@@ -155,7 +156,7 @@ def get_dump(stage):
|
||||
stage.PARAM_MD5: stage.md5,
|
||||
stage.PARAM_CMD: stage.cmd,
|
||||
stage.PARAM_WDIR: resolve_wdir(stage.wdir, stage.path),
|
||||
stage.PARAM_LOCKED: stage.locked,
|
||||
stage.PARAM_FROZEN: stage.frozen,
|
||||
stage.PARAM_DEPS: [d.dumpd() for d in stage.deps],
|
||||
stage.PARAM_OUTS: [o.dumpd() for o in stage.outs],
|
||||
stage.PARAM_ALWAYS_CHANGED: stage.always_changed,
|
||||
|
||||
+6
-6
@@ -7,6 +7,7 @@ import sqlite3
|
||||
from urllib.parse import urlencode, urlunparse
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.scm.tree import WorkingTree
|
||||
from dvc.utils import current_timestamp, relpath, to_chunks
|
||||
from dvc.utils.fs import get_inode, get_mtime_and_size, remove
|
||||
|
||||
@@ -91,6 +92,7 @@ class State: # pylint: disable=too-many-instance-attributes
|
||||
def __init__(self, repo):
|
||||
self.repo = repo
|
||||
self.root_dir = repo.root_dir
|
||||
self.tree = WorkingTree(self.root_dir)
|
||||
|
||||
state_config = repo.config.get("state", {})
|
||||
self.row_limit = state_config.get("row_limit", self.STATE_ROW_LIMIT)
|
||||
@@ -364,9 +366,7 @@ class State: # pylint: disable=too-many-instance-attributes
|
||||
assert checksum is not None
|
||||
assert os.path.exists(path_info)
|
||||
|
||||
actual_mtime, actual_size = get_mtime_and_size(
|
||||
path_info, self.repo.tree
|
||||
)
|
||||
actual_mtime, actual_size = get_mtime_and_size(path_info, self.tree)
|
||||
actual_inode = get_inode(path_info)
|
||||
|
||||
existing_record = self.get_state_record_for_inode(actual_inode)
|
||||
@@ -397,7 +397,7 @@ class State: # pylint: disable=too-many-instance-attributes
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
|
||||
actual_mtime, actual_size = get_mtime_and_size(path, self.repo.tree)
|
||||
actual_mtime, actual_size = get_mtime_and_size(path, self.tree)
|
||||
actual_inode = get_inode(path)
|
||||
|
||||
existing_record = self.get_state_record_for_inode(actual_inode)
|
||||
@@ -423,7 +423,7 @@ class State: # pylint: disable=too-many-instance-attributes
|
||||
if not os.path.exists(path_info):
|
||||
return
|
||||
|
||||
mtime, _ = get_mtime_and_size(path_info, self.repo.tree)
|
||||
mtime, _ = get_mtime_and_size(path_info, self.tree)
|
||||
inode = get_inode(path_info)
|
||||
relative_path = relpath(path_info, self.root_dir)
|
||||
|
||||
@@ -450,7 +450,7 @@ class State: # pylint: disable=too-many-instance-attributes
|
||||
continue
|
||||
|
||||
actual_inode = get_inode(path)
|
||||
actual_mtime, _ = get_mtime_and_size(path, self.repo.tree)
|
||||
actual_mtime, _ = get_mtime_and_size(path, self.tree)
|
||||
|
||||
if (inode, mtime) == (actual_inode, actual_mtime):
|
||||
logger.debug("Removing '%s' as unused link.", path)
|
||||
|
||||
+12
-6
@@ -395,7 +395,7 @@ def format_link(link):
|
||||
|
||||
|
||||
def parse_target(target, default=None):
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK, is_valid_filename
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
if not target:
|
||||
@@ -409,13 +409,19 @@ def parse_target(target, default=None):
|
||||
match.group("path"),
|
||||
match.group("name"),
|
||||
)
|
||||
if path:
|
||||
if os.path.basename(path) == PIPELINE_LOCK:
|
||||
raise DvcException(
|
||||
"Did you mean: `{}`?".format(
|
||||
target.replace(".lock", ".yaml", 1)
|
||||
)
|
||||
)
|
||||
if not name:
|
||||
ret = (target, None)
|
||||
return ret if is_valid_filename(target) else ret[::-1]
|
||||
|
||||
if not path:
|
||||
path = default or PIPELINE_FILE
|
||||
logger.debug("Assuming file to be '%s'", path)
|
||||
|
||||
if os.path.basename(path) == PIPELINE_LOCK:
|
||||
raise DvcException(
|
||||
"Did you mean: `{}`?".format(target.replace(".lock", ".yaml", 1))
|
||||
)
|
||||
|
||||
return path, name
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Draws DAG in ASCII."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import pydoc
|
||||
import sys
|
||||
|
||||
from dvc.env import DVC_PAGER
|
||||
from dvc.utils import format_link
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
DEFAULT_PAGER = "less"
|
||||
DEFAULT_PAGER_FORMATTED = "{} --chop-long-lines --clear-screen".format(
|
||||
DEFAULT_PAGER
|
||||
)
|
||||
|
||||
|
||||
def make_pager(cmd):
|
||||
def _pager(text):
|
||||
return pydoc.tempfilepager(pydoc.plain(text), cmd)
|
||||
|
||||
return _pager
|
||||
|
||||
|
||||
def find_pager():
|
||||
if not sys.stdout.isatty():
|
||||
return pydoc.plainpager
|
||||
|
||||
env_pager = os.getenv(DVC_PAGER)
|
||||
if env_pager:
|
||||
return make_pager(env_pager)
|
||||
|
||||
if os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") == 0:
|
||||
return make_pager(DEFAULT_PAGER_FORMATTED)
|
||||
|
||||
logger.warning(
|
||||
"Unable to find `less` in the PATH. Check out "
|
||||
"{} for more info.".format(
|
||||
format_link("https://man.dvc.org/pipeline/show")
|
||||
)
|
||||
)
|
||||
return pydoc.plainpager
|
||||
|
||||
|
||||
def pager(text):
|
||||
find_pager()(text)
|
||||
@@ -4,7 +4,7 @@ import yaml
|
||||
from ruamel.yaml import YAML
|
||||
from ruamel.yaml.error import YAMLError
|
||||
|
||||
from dvc.exceptions import StageFileCorruptedError
|
||||
from dvc.exceptions import YAMLFileCorruptedError
|
||||
|
||||
try:
|
||||
from yaml import CSafeLoader as SafeLoader
|
||||
@@ -12,35 +12,35 @@ except ImportError:
|
||||
from yaml import SafeLoader
|
||||
|
||||
|
||||
def load_stage_file(path):
|
||||
def load_yaml(path):
|
||||
with open(path, encoding="utf-8") as fd:
|
||||
return parse_stage(fd.read(), path)
|
||||
return parse_yaml(fd.read(), path)
|
||||
|
||||
|
||||
def parse_stage(text, path):
|
||||
def parse_yaml(text, path):
|
||||
try:
|
||||
return yaml.load(text, Loader=SafeLoader) or {}
|
||||
except yaml.error.YAMLError as exc:
|
||||
raise StageFileCorruptedError(path) from exc
|
||||
raise YAMLFileCorruptedError(path) from exc
|
||||
|
||||
|
||||
def parse_stage_for_update(text, path):
|
||||
def parse_yaml_for_update(text, path):
|
||||
"""Parses text into Python structure.
|
||||
|
||||
Unlike `parse_stage()` this returns ordered dicts, values have special
|
||||
Unlike `parse_yaml()` this returns ordered dicts, values have special
|
||||
attributes to store comments and line breaks. This allows us to preserve
|
||||
all of those upon dump.
|
||||
|
||||
This one is, however, several times slower than simple `parse_stage()`.
|
||||
This one is, however, several times slower than simple `parse_yaml()`.
|
||||
"""
|
||||
try:
|
||||
yaml = YAML()
|
||||
return yaml.load(text) or {}
|
||||
except YAMLError as exc:
|
||||
raise StageFileCorruptedError(path) from exc
|
||||
raise YAMLFileCorruptedError(path) from exc
|
||||
|
||||
|
||||
def dump_stage_file(path, data):
|
||||
def dump_yaml(path, data):
|
||||
with open(path, "w", encoding="utf-8") as fd:
|
||||
yaml = YAML()
|
||||
yaml.default_flow_style = False
|
||||
+1
-1
@@ -6,7 +6,7 @@
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
_BASE_VERSION = "1.0.0a5"
|
||||
_BASE_VERSION = "1.0.0a7"
|
||||
|
||||
|
||||
def _generate_version(base_version):
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
set -x
|
||||
set -e
|
||||
|
||||
pip install Pygments collective.checkdocs pre-commit
|
||||
pip install Pygments collective.checkdocs pre-commit .
|
||||
|
||||
# stop the build if there are any readme formatting errors
|
||||
python setup.py checkdocs
|
||||
|
||||
+10
-17
@@ -4,9 +4,10 @@
|
||||
# - https://opensource.com/article/18/3/creating-bash-completion-script
|
||||
# - https://stackoverflow.com/questions/12933362
|
||||
|
||||
_dvc_commands='add cache checkout commit config destroy diff fetch get-url get gc \
|
||||
import-url import init install lock list metrics move params pipeline plots pull push \
|
||||
remote remove repro root run status unlock unprotect update version'
|
||||
_dvc_commands='add cache checkout commit config dag destroy diff fetch freeze \
|
||||
get-url get gc import-url import init install list metrics move params \
|
||||
plots pull push remote remove repro root run status unfreeze unprotect \
|
||||
update version'
|
||||
|
||||
_dvc_options='-h --help -V --version'
|
||||
_dvc_global_options='-h --help -q --quiet -v --verbose'
|
||||
@@ -20,6 +21,8 @@ _dvc_checkout_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_commit='-f --force -d --with-deps -R --recursive'
|
||||
_dvc_commit_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_config='-u --unset --local --system --global'
|
||||
_dvc_dag='--dot --full'
|
||||
_dvc_dag_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_destroy='-f --force'
|
||||
_dvc_diff='-t --show-json --show-hash --show-md'
|
||||
_dvc_fetch='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -R --recursive'
|
||||
@@ -33,26 +36,16 @@ _dvc_init='--no-scm -f --force'
|
||||
_dvc_install=''
|
||||
_dvc_list='-R --recursive --dvc-only --rev'
|
||||
_dvc_list_COMPGEN=_dvc_compgen_files
|
||||
_dvc_lock=''
|
||||
_dvc_lock_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_metrics='add diff modify remove show'
|
||||
_dvc_metrics_add='-t --type -x --xpath'
|
||||
_dvc_metrics_add_COMPGEN=_dvc_compgen_files
|
||||
_dvc_freeze=''
|
||||
_dvc_freeze_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_metrics='diff show'
|
||||
_dvc_metrics_diff='--targets -t --type -x --xpath -R --show-json --show-md --no-path --old'
|
||||
_dvc_metrics_modify='-t --type -x --xpath'
|
||||
_dvc_metrics_modify_COMPGEN=_dvc_compgen_files
|
||||
_dvc_metrics_remove=''
|
||||
_dvc_metrics_remove_COMPGEN=_dvc_compgen_files
|
||||
_dvc_metrics_show='-t --type -x --xpath -a --all-branches -T --all-tags -R --recursive'
|
||||
_dvc_metrics_show_COMPGEN=_dvc_compgen_files
|
||||
_dvc_move=''
|
||||
_dvc_move_COMPGEN=_dvc_compgen_files
|
||||
_dvc_params='diff'
|
||||
_dvc_params_diff='--all --show-json --show-md --no-path'
|
||||
_dvc_pipeline='list show'
|
||||
_dvc_pipeline_list=''
|
||||
_dvc_pipeline_show='-c --commands -o --outs --ascii --dot --tree -l --locked'
|
||||
_dvc_pipeline_show_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_plots='show diff'
|
||||
_dvc_plots_show='-t --template -o --out -x -y --show-json --no-csv-header --title --xlab --ylab'
|
||||
_dvc_plots_diff='-t --template --targets -o --out -x -y --show-json --no-csv-header --title --xlab --ylab'
|
||||
@@ -76,7 +69,7 @@ _dvc_run='--no-exec -f --file -d --deps -o --outs -O --outs-no-cache --outs-pers
|
||||
_dvc_run_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_status='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud'
|
||||
_dvc_status_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_unlock_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_unfreeze_COMPGEN=_dvc_compgen_DVCFiles
|
||||
_dvc_unprotect_COMPGEN=_dvc_compgen_files
|
||||
_dvc_update='--rev'
|
||||
_dvc_update_COMPGEN=_dvc_compgen_DVCFiles
|
||||
|
||||
+15
-13
@@ -17,6 +17,7 @@ _dvc_commands() {
|
||||
"checkout:Checkout data files from cache."
|
||||
"commit:Save changed data to cache and update DVC-files."
|
||||
"config:Get or set config settings."
|
||||
"dag:Visualize DVC project DAG."
|
||||
"destroy:Remove DVC-files, local DVC config and data cache."
|
||||
"diff:Show added, modified, or deleted data between commits in the DVC repository, or between a commit and the workspace."
|
||||
"fetch:Get files or directories tracked by DVC from remote storage into the cache."
|
||||
@@ -29,10 +30,9 @@ _dvc_commands() {
|
||||
"install:Install DVC git hooks into the repository."
|
||||
"list:List repository contents, including files and directories tracked by DVC and by Git."
|
||||
"lock:Lock DVC-file."
|
||||
"metrics:Commands to add, manage, collect and display metrics."
|
||||
"metrics:Commands to display and compare metrics."
|
||||
"move:Rename or move a DVC controlled data file or a directory."
|
||||
"params:Commands to display params."
|
||||
"pipeline:Manage pipelines."
|
||||
"pull:Pull data files from a DVC remote storage."
|
||||
"push:Push data files to a DVC remote storage."
|
||||
"plots:Generate plot for metrics structured as JSON, CSV or TSV."
|
||||
@@ -96,6 +96,12 @@ _dvc_config=(
|
||||
{-u,--unset}"[Unset option.]"
|
||||
)
|
||||
|
||||
_dvc_dag=(
|
||||
"--dot[Print DAG in DOT format.]"
|
||||
"--full[Show full DAG that the target belongs too, instead of showing DAG consisting only of ancestors.]"
|
||||
"1:Stage:"
|
||||
)
|
||||
|
||||
_dvc_destroy=(
|
||||
{-f,--force}"[Force destruction.]"
|
||||
)
|
||||
@@ -132,7 +138,7 @@ _dvc_get=(
|
||||
)
|
||||
|
||||
_dvc_gc=(
|
||||
{-w,--workspace}"[Keep data files used in the current workspace (working tree).]"
|
||||
{-w,--workspace}"[Keep data files used in the current workspace.]"
|
||||
{-a,--all-branches}"[Keep data files for the tips of all Git branches.]"
|
||||
"--all-commits[Keep data files for all Git commits.]"
|
||||
{-T,--all-tags}"[Keep data files for all Git tags.]"
|
||||
@@ -163,7 +169,7 @@ _dvc_init=(
|
||||
|
||||
_dvc_install=()
|
||||
|
||||
_dvc_lock=(
|
||||
_dvc_freeze=(
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
@@ -176,7 +182,7 @@ _dvc_list=(
|
||||
)
|
||||
|
||||
_dvc_metrics=(
|
||||
"1:Sub command:(add show diff modify remove)"
|
||||
"1:Sub command:(show diff)"
|
||||
)
|
||||
|
||||
_dvc_move=(
|
||||
@@ -188,10 +194,6 @@ _dvc_params=(
|
||||
"1:Sub command:(diff)"
|
||||
)
|
||||
|
||||
_dvc_pipeline=(
|
||||
"1:Sub command:(show list)"
|
||||
)
|
||||
|
||||
_dvc_pull=(
|
||||
{-j,--jobs}"[Number of jobs to run simultaneously.]:Number of jobs:"
|
||||
{-r,--remote}"[Remote repository to pull from.]:Remote repository:"
|
||||
@@ -278,7 +280,7 @@ _dvc_status=(
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
_dvc_unlock=(
|
||||
_dvc_unfreeze=(
|
||||
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
|
||||
)
|
||||
|
||||
@@ -305,9 +307,11 @@ case $words[1] in
|
||||
checkout) _arguments $_dvc_global_options $_dvc_checkout ;;
|
||||
commit) _arguments $_dvc_global_options $_dvc_commit ;;
|
||||
config) _arguments $_dvc_global_options $_dvc_config ;;
|
||||
dag) _arguments $_dvc_global_options $_dvc_dag ;;
|
||||
destroy) _arguments $_dvc_global_options $_dvc_destroy ;;
|
||||
diff) _arguments $_dvc_global_options $_dvc_diff ;;
|
||||
fetch) _arguments $_dvc_global_options $_dvc_fetch ;;
|
||||
freeze) _arguments $_dvc_global_options $_dvc_freeze ;;
|
||||
get-url) _arguments $_dvc_global_options $_dvc_geturl ;;
|
||||
get) _arguments $_dvc_global_options $_dvc_get ;;
|
||||
gc) _arguments $_dvc_global_options $_dvc_gc ;;
|
||||
@@ -315,12 +319,10 @@ case $words[1] in
|
||||
import) _arguments $_dvc_global_options $_dvc_import ;;
|
||||
init) _arguments $_dvc_global_options $_dvc_init ;;
|
||||
install) _arguments $_dvc_global_options $_dvc_install ;;
|
||||
lock) _arguments $_dvc_global_options $_dvc_lock ;;
|
||||
list) _arguments $_dvc_global_options $_dvc_list ;;
|
||||
metrics) _arguments $_dvc_global_options $_dvc_metrics ;;
|
||||
move) _arguments $_dvc_global_options $_dvc_move ;;
|
||||
params) _arguments $_dvc_global_options $_dvc_params ;;
|
||||
pipeline) _arguments $_dvc_global_options $_dvc_pipeline ;;
|
||||
pull) _arguments $_dvc_global_options $_dvc_pull ;;
|
||||
push) _arguments $_dvc_global_options $_dvc_push ;;
|
||||
plots) _arguments $_dvc_global_options $_dvc_plots ;;
|
||||
@@ -330,7 +332,7 @@ case $words[1] in
|
||||
root) _arguments $_dvc_global_options $_dvc_root ;;
|
||||
run) _arguments $_dvc_global_options $_dvc_run ;;
|
||||
status) _arguments $_dvc_global_options $_dvc_status ;;
|
||||
unlock) _arguments $_dvc_global_options $_dvc_unlock ;;
|
||||
unfreeze) _arguments $_dvc_global_options $_dvc_unfreeze ;;
|
||||
unprotect) _arguments $_dvc_global_options $_dvc_unprotect ;;
|
||||
update) _arguments $_dvc_global_options $_dvc_update ;;
|
||||
esac
|
||||
|
||||
@@ -12,6 +12,6 @@ select=B,C,E,F,W,T4,B9
|
||||
[isort]
|
||||
include_trailing_comma=true
|
||||
known_first_party=dvc,tests
|
||||
known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,mockssh,moto,nanotime,packaging,paramiko,pathspec,pytest,requests,ruamel,setuptools,shortuuid,tqdm,voluptuous,yaml,zc
|
||||
known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,mockssh,moto,nanotime,networkx,packaging,paramiko,pathspec,pytest,requests,ruamel,setuptools,shortuuid,tqdm,voluptuous,yaml,zc
|
||||
line_length=79
|
||||
multi_line_output=3
|
||||
|
||||
@@ -121,7 +121,7 @@ tests_requirements = [
|
||||
"flake8-docstrings",
|
||||
"pydocstyle<4.0",
|
||||
"jaraco.windows==3.9.2",
|
||||
"mock-ssh-server>=0.6.0",
|
||||
"mock-ssh-server@git+https://github.com/iterative/mock-ssh-server.git",
|
||||
"moto==1.3.14.dev464",
|
||||
"rangehttpserver==1.2.0",
|
||||
"beautifulsoup4==4.4.0",
|
||||
|
||||
+1
-1
@@ -34,7 +34,7 @@ user = "user"
|
||||
key_path = os.path.join(here, f"{user}.key")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@pytest.fixture(scope="session")
|
||||
def ssh_server():
|
||||
users = {user: key_path}
|
||||
with mockssh.Server(users) as s:
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
from tests.func.plots import run_copy_metrics # noqa: F401
|
||||
@@ -3,11 +3,10 @@ import json
|
||||
import yaml
|
||||
|
||||
|
||||
def test_metrics_diff_simple(tmp_dir, scm, dvc):
|
||||
def test_metrics_diff_simple(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
def _gen(val):
|
||||
tmp_dir.gen({"m.yaml": str(val)})
|
||||
dvc.run(cmd="", metrics=["m.yaml"], single_stage=True)
|
||||
dvc.scm.add(["m.yaml.dvc"])
|
||||
tmp_dir.gen({"m_temp.yaml": str(val)})
|
||||
run_copy_metrics("m_temp.yaml", "m.yaml", metrics=["m.yaml"])
|
||||
dvc.scm.commit(str(val))
|
||||
|
||||
_gen(1)
|
||||
@@ -19,13 +18,13 @@ def test_metrics_diff_simple(tmp_dir, scm, dvc):
|
||||
assert dvc.metrics.diff(a_rev="HEAD~2") == expected
|
||||
|
||||
|
||||
def test_metrics_diff_yaml(tmp_dir, scm, dvc):
|
||||
def test_metrics_diff_yaml(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
def _gen(val):
|
||||
metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}}
|
||||
tmp_dir.gen({"m.yaml": yaml.dump(metrics)})
|
||||
dvc.run(cmd="", metrics=["m.yaml"], single_stage=True)
|
||||
dvc.scm.add(["m.yaml.dvc"])
|
||||
dvc.scm.commit(str(val))
|
||||
tmp_dir.gen({"m_temp.yaml": yaml.dump(metrics)})
|
||||
run_copy_metrics(
|
||||
"m_temp.yaml", "m.yaml", metrics=["m.yaml"], commit=str(val)
|
||||
)
|
||||
|
||||
_gen(1)
|
||||
_gen(2)
|
||||
@@ -36,13 +35,13 @@ def test_metrics_diff_yaml(tmp_dir, scm, dvc):
|
||||
assert dvc.metrics.diff(a_rev="HEAD~2") == expected
|
||||
|
||||
|
||||
def test_metrics_diff_json(tmp_dir, scm, dvc):
|
||||
def test_metrics_diff_json(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
def _gen(val):
|
||||
metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}}
|
||||
tmp_dir.gen({"m.json": json.dumps(metrics)})
|
||||
dvc.run(cmd="", metrics=["m.json"], single_stage=True)
|
||||
dvc.scm.add(["m.json.dvc"])
|
||||
dvc.scm.commit(str(val))
|
||||
tmp_dir.gen({"m_temp.json": json.dumps(metrics)})
|
||||
run_copy_metrics(
|
||||
"m_temp.json", "m.json", metrics=["m.json"], commit=str(val)
|
||||
)
|
||||
|
||||
_gen(1)
|
||||
_gen(2)
|
||||
@@ -52,13 +51,13 @@ def test_metrics_diff_json(tmp_dir, scm, dvc):
|
||||
assert dvc.metrics.diff(a_rev="HEAD~2") == expected
|
||||
|
||||
|
||||
def test_metrics_diff_json_unchanged(tmp_dir, scm, dvc):
|
||||
def test_metrics_diff_json_unchanged(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
def _gen(val):
|
||||
metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}}
|
||||
tmp_dir.gen({"m.json": json.dumps(metrics)})
|
||||
dvc.run(cmd="", metrics=["m.json"], single_stage=True)
|
||||
dvc.scm.add(["m.json.dvc"])
|
||||
dvc.scm.commit(str(val))
|
||||
tmp_dir.gen({"m_temp.json": json.dumps(metrics)})
|
||||
run_copy_metrics(
|
||||
"m_temp.json", "m.json", metrics=["m.json"], commit=str(val)
|
||||
)
|
||||
|
||||
_gen(1)
|
||||
_gen(2)
|
||||
@@ -67,12 +66,15 @@ def test_metrics_diff_json_unchanged(tmp_dir, scm, dvc):
|
||||
assert dvc.metrics.diff(a_rev="HEAD~2") == {}
|
||||
|
||||
|
||||
def test_metrics_diff_broken_json(tmp_dir, scm, dvc):
|
||||
def test_metrics_diff_broken_json(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metrics = {"a": {"b": {"c": 1, "d": 1, "e": "3"}}}
|
||||
tmp_dir.gen({"m.json": json.dumps(metrics)})
|
||||
dvc.run(cmd="", metrics_no_cache=["m.json"], single_stage=True)
|
||||
dvc.scm.add(["m.json.dvc", "m.json"])
|
||||
dvc.scm.commit("add metrics")
|
||||
tmp_dir.gen({"m_temp.json": json.dumps(metrics)})
|
||||
run_copy_metrics(
|
||||
"m_temp.json",
|
||||
"m.json",
|
||||
metrics_no_cache=["m.json"],
|
||||
commit="add metrics",
|
||||
)
|
||||
|
||||
(tmp_dir / "m.json").write_text(json.dumps(metrics) + "ma\nlformed\n")
|
||||
|
||||
@@ -89,10 +91,10 @@ def test_metrics_diff_no_metrics(tmp_dir, scm, dvc):
|
||||
assert dvc.metrics.diff(a_rev="HEAD~1") == {}
|
||||
|
||||
|
||||
def test_metrics_diff_new_metric(tmp_dir, scm, dvc):
|
||||
def test_metrics_diff_new_metric(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metrics = {"a": {"b": {"c": 1, "d": 1, "e": "3"}}}
|
||||
tmp_dir.gen({"m.json": json.dumps(metrics)})
|
||||
dvc.run(cmd="", metrics_no_cache=["m.json"], single_stage=True)
|
||||
tmp_dir.gen({"m_temp.json": json.dumps(metrics)})
|
||||
run_copy_metrics("m_temp.json", "m.json", metrics_no_cache=["m.json"])
|
||||
|
||||
assert dvc.metrics.diff() == {
|
||||
"m.json": {
|
||||
@@ -102,12 +104,15 @@ def test_metrics_diff_new_metric(tmp_dir, scm, dvc):
|
||||
}
|
||||
|
||||
|
||||
def test_metrics_diff_deleted_metric(tmp_dir, scm, dvc):
|
||||
def test_metrics_diff_deleted_metric(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metrics = {"a": {"b": {"c": 1, "d": 1, "e": "3"}}}
|
||||
tmp_dir.gen({"m.json": json.dumps(metrics)})
|
||||
dvc.run(cmd="", metrics_no_cache=["m.json"], single_stage=True)
|
||||
dvc.scm.add(["m.json.dvc", "m.json"])
|
||||
dvc.scm.commit("add metrics")
|
||||
tmp_dir.gen({"m_temp.json": json.dumps(metrics)})
|
||||
run_copy_metrics(
|
||||
"m_temp.json",
|
||||
"m.json",
|
||||
metrics_no_cache=["m.json"],
|
||||
commit="add metrics",
|
||||
)
|
||||
|
||||
(tmp_dir / "m.json").unlink()
|
||||
|
||||
@@ -119,11 +124,14 @@ def test_metrics_diff_deleted_metric(tmp_dir, scm, dvc):
|
||||
}
|
||||
|
||||
|
||||
def test_metrics_diff_with_unchanged(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("metrics.yaml", "foo: 1\nxyz: 10")
|
||||
dvc.run(metrics_no_cache=["metrics.yaml"], single_stage=True)
|
||||
scm.add(["metrics.yaml", "metrics.yaml.dvc"])
|
||||
scm.commit("1")
|
||||
def test_metrics_diff_with_unchanged(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
tmp_dir.gen("metrics_temp.yaml", "foo: 1\nxyz: 10")
|
||||
run_copy_metrics(
|
||||
"metrics_temp.yaml",
|
||||
"metrics.yaml",
|
||||
metrics_no_cache=["metrics.yaml"],
|
||||
commit="1",
|
||||
)
|
||||
|
||||
tmp_dir.scm_gen("metrics.yaml", "foo: 2\nxyz: 10", commit="2")
|
||||
tmp_dir.scm_gen("metrics.yaml", "foo: 3\nxyz: 10", commit="3")
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.metrics.show import NoMetricsError
|
||||
|
||||
|
||||
@@ -8,36 +11,44 @@ def test_show_empty(dvc):
|
||||
dvc.metrics.show()
|
||||
|
||||
|
||||
def test_show_simple(tmp_dir, dvc):
|
||||
tmp_dir.gen("metrics.yaml", "1.1")
|
||||
dvc.run(metrics=["metrics.yaml"], single_stage=True)
|
||||
def test_show_simple(tmp_dir, dvc, run_copy_metrics):
|
||||
tmp_dir.gen("metrics_t.yaml", "1.1")
|
||||
run_copy_metrics(
|
||||
"metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"],
|
||||
)
|
||||
assert dvc.metrics.show() == {"": {"metrics.yaml": 1.1}}
|
||||
|
||||
|
||||
def test_show(tmp_dir, dvc):
|
||||
tmp_dir.gen("metrics.yaml", "foo: 1.1")
|
||||
dvc.run(metrics=["metrics.yaml"], single_stage=True)
|
||||
def test_show(tmp_dir, dvc, run_copy_metrics):
|
||||
tmp_dir.gen("metrics_t.yaml", "foo: 1.1")
|
||||
run_copy_metrics(
|
||||
"metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"],
|
||||
)
|
||||
assert dvc.metrics.show() == {"": {"metrics.yaml": {"foo": 1.1}}}
|
||||
|
||||
|
||||
def test_show_multiple(tmp_dir, dvc):
|
||||
tmp_dir.gen("foo", "foo: 1\n")
|
||||
tmp_dir.gen("baz", "baz: 2\n")
|
||||
dvc.run(fname="foo.dvc", metrics=["foo"], single_stage=True)
|
||||
dvc.run(fname="baz.dvc", metrics=["baz"], single_stage=True)
|
||||
def test_show_multiple(tmp_dir, dvc, run_copy_metrics):
|
||||
tmp_dir.gen("foo_temp", "foo: 1\n")
|
||||
tmp_dir.gen("baz_temp", "baz: 2\n")
|
||||
run_copy_metrics("foo_temp", "foo", fname="foo.dvc", metrics=["foo"])
|
||||
run_copy_metrics("baz_temp", "baz", fname="baz.dvc", metrics=["baz"])
|
||||
assert dvc.metrics.show() == {"": {"foo": {"foo": 1}, "baz": {"baz": 2}}}
|
||||
|
||||
|
||||
def test_show_invalid_metric(tmp_dir, dvc):
|
||||
tmp_dir.gen("metrics.yaml", "foo:\n- bar\n- baz\nxyz: string")
|
||||
dvc.run(metrics=["metrics.yaml"], single_stage=True)
|
||||
def test_show_invalid_metric(tmp_dir, dvc, run_copy_metrics):
|
||||
tmp_dir.gen("metrics_temp.yaml", "foo:\n- bar\n- baz\nxyz: string")
|
||||
run_copy_metrics(
|
||||
"metrics_temp.yaml", "metrics.yaml", metrics=["metrics.yaml"]
|
||||
)
|
||||
with pytest.raises(NoMetricsError):
|
||||
dvc.metrics.show()
|
||||
|
||||
|
||||
def test_show_branch(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("metrics.yaml", "foo: 1")
|
||||
dvc.run(metrics_no_cache=["metrics.yaml"], single_stage=True)
|
||||
def test_show_branch(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
tmp_dir.gen("metrics_temp.yaml", "foo: 1")
|
||||
run_copy_metrics(
|
||||
"metrics_temp.yaml", "metrics.yaml", metrics_no_cache=["metrics.yaml"]
|
||||
)
|
||||
scm.add(["metrics.yaml", "metrics.yaml.dvc"])
|
||||
scm.commit("init")
|
||||
|
||||
@@ -45,6 +56,40 @@ def test_show_branch(tmp_dir, scm, dvc):
|
||||
tmp_dir.scm_gen("metrics.yaml", "foo: 2", commit="branch")
|
||||
|
||||
assert dvc.metrics.show(revs=["branch"]) == {
|
||||
"working tree": {"metrics.yaml": {"foo": 1}},
|
||||
"workspace": {"metrics.yaml": {"foo": 1}},
|
||||
"branch": {"metrics.yaml": {"foo": 2}},
|
||||
}
|
||||
|
||||
|
||||
def test_show_subrepo_with_preexisting_tags(tmp_dir, scm):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
scm.add("foo")
|
||||
scm.commit("init")
|
||||
scm.tag("no-metrics")
|
||||
|
||||
tmp_dir.gen({"subdir": {}})
|
||||
subrepo_dir = tmp_dir / "subdir"
|
||||
with subrepo_dir.chdir():
|
||||
dvc = Repo.init(subdir=True)
|
||||
scm.commit("init dvc")
|
||||
|
||||
dvc.run(
|
||||
cmd="echo foo: 1 > metrics.yaml",
|
||||
metrics=["metrics.yaml"],
|
||||
single_stage=True,
|
||||
)
|
||||
|
||||
scm.add(
|
||||
[
|
||||
str(subrepo_dir / "metrics.yaml"),
|
||||
str(subrepo_dir / "metrics.yaml.dvc"),
|
||||
]
|
||||
)
|
||||
scm.commit("init metrics")
|
||||
scm.tag("v1")
|
||||
|
||||
expected_path = os.path.join("subdir", "metrics.yaml")
|
||||
assert dvc.metrics.show(all_tags=True) == {
|
||||
"workspace": {expected_path: {"foo": 1}},
|
||||
"v1": {expected_path: {"foo": 1}},
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ def test_diff_no_params(tmp_dir, scm, dvc):
|
||||
|
||||
def test_diff_no_changes(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo: bar")
|
||||
dvc.run(params=["foo"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
|
||||
scm.add(["params.yaml", "Dvcfile"])
|
||||
scm.commit("bar")
|
||||
assert dvc.params.diff() == {}
|
||||
@@ -12,7 +12,7 @@ def test_diff_no_changes(tmp_dir, scm, dvc):
|
||||
|
||||
def test_diff(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo: bar")
|
||||
dvc.run(params=["foo"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
|
||||
scm.add(["params.yaml", "Dvcfile"])
|
||||
scm.commit("bar")
|
||||
|
||||
@@ -26,7 +26,7 @@ def test_diff(tmp_dir, scm, dvc):
|
||||
|
||||
def test_diff_new(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo: bar")
|
||||
dvc.run(params=["foo"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
|
||||
|
||||
assert dvc.params.diff() == {
|
||||
"params.yaml": {"foo": {"old": None, "new": "bar"}}
|
||||
@@ -35,7 +35,7 @@ def test_diff_new(tmp_dir, scm, dvc):
|
||||
|
||||
def test_diff_deleted(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo: bar")
|
||||
dvc.run(params=["foo"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
|
||||
scm.add(["params.yaml", "Dvcfile"])
|
||||
scm.commit("bar")
|
||||
|
||||
@@ -48,7 +48,7 @@ def test_diff_deleted(tmp_dir, scm, dvc):
|
||||
|
||||
def test_diff_deleted_config(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo: bar")
|
||||
dvc.run(params=["foo"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
|
||||
scm.add(["params.yaml", "Dvcfile"])
|
||||
scm.commit("bar")
|
||||
|
||||
@@ -61,7 +61,7 @@ def test_diff_deleted_config(tmp_dir, scm, dvc):
|
||||
|
||||
def test_diff_list(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo:\n- bar\n- baz")
|
||||
dvc.run(params=["foo"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
|
||||
scm.add(["params.yaml", "Dvcfile"])
|
||||
scm.commit("foo")
|
||||
|
||||
@@ -76,7 +76,7 @@ def test_diff_list(tmp_dir, scm, dvc):
|
||||
|
||||
def test_diff_dict(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo:\n bar: baz")
|
||||
dvc.run(params=["foo"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
|
||||
scm.add(["params.yaml", "Dvcfile"])
|
||||
scm.commit("foo")
|
||||
|
||||
@@ -89,7 +89,7 @@ def test_diff_dict(tmp_dir, scm, dvc):
|
||||
|
||||
def test_diff_with_unchanged(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo: bar\nxyz: val")
|
||||
dvc.run(params=["foo,xyz"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo,xyz"], single_stage=True)
|
||||
scm.add(["params.yaml", "Dvcfile"])
|
||||
scm.commit("bar")
|
||||
|
||||
|
||||
@@ -10,14 +10,24 @@ def test_show_empty(dvc):
|
||||
|
||||
def test_show(tmp_dir, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo: bar")
|
||||
dvc.run(params=["foo"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
|
||||
assert dvc.params.show() == {"": {"params.yaml": {"foo": "bar"}}}
|
||||
|
||||
|
||||
def test_show_multiple(tmp_dir, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo: bar\nbaz: qux\n")
|
||||
dvc.run(fname="foo.dvc", params=["foo"], single_stage=True)
|
||||
dvc.run(fname="baz.dvc", params=["baz"], single_stage=True)
|
||||
dvc.run(
|
||||
cmd="echo params.yaml",
|
||||
fname="foo.dvc",
|
||||
params=["foo"],
|
||||
single_stage=True,
|
||||
)
|
||||
dvc.run(
|
||||
cmd="echo params.yaml",
|
||||
fname="baz.dvc",
|
||||
params=["baz"],
|
||||
single_stage=True,
|
||||
)
|
||||
assert dvc.params.show() == {
|
||||
"": {"params.yaml": {"foo": "bar", "baz": "qux"}}
|
||||
}
|
||||
@@ -25,13 +35,13 @@ def test_show_multiple(tmp_dir, dvc):
|
||||
|
||||
def test_show_list(tmp_dir, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo:\n- bar\n- baz\n")
|
||||
dvc.run(params=["foo"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
|
||||
assert dvc.params.show() == {"": {"params.yaml": {"foo": ["bar", "baz"]}}}
|
||||
|
||||
|
||||
def test_show_branch(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen("params.yaml", "foo: bar")
|
||||
dvc.run(params=["foo"], single_stage=True)
|
||||
dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
|
||||
scm.add(["params.yaml", "Dvcfile"])
|
||||
scm.commit("init")
|
||||
|
||||
@@ -39,7 +49,7 @@ def test_show_branch(tmp_dir, scm, dvc):
|
||||
tmp_dir.scm_gen("params.yaml", "foo: baz", commit="branch")
|
||||
|
||||
assert dvc.params.show(revs=["branch"]) == {
|
||||
"working tree": {"params.yaml": {"foo": "bar"}},
|
||||
"workspace": {"params.yaml": {"foo": "bar"}},
|
||||
"branch": {"params.yaml": {"foo": "baz"}},
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_copy_metrics(tmp_dir, run_copy):
|
||||
def run(file1, file2, commit=None, tag=None, **kwargs):
|
||||
stage = tmp_dir.dvc.run(
|
||||
cmd=f"python copy.py {file1} {file2}",
|
||||
deps=[file1],
|
||||
single_stage=True,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if hasattr(tmp_dir.dvc, "scm"):
|
||||
files = [stage.path]
|
||||
files += [
|
||||
os.fspath(out.path_info)
|
||||
for out in stage.outs
|
||||
if not out.use_cache
|
||||
]
|
||||
tmp_dir.dvc.scm.add(files)
|
||||
if commit:
|
||||
tmp_dir.dvc.scm.commit(commit)
|
||||
if tag:
|
||||
tmp_dir.dvc.scm.tag(tag)
|
||||
return stage
|
||||
|
||||
return run
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
from . import run_copy_metrics # noqa: F401
|
||||
@@ -1,27 +1,39 @@
|
||||
import json
|
||||
|
||||
from .test_plots import PlotData, _run_with, _write_json
|
||||
from .test_plots import PlotData, _write_json
|
||||
|
||||
|
||||
def test_diff_dirty(tmp_dir, scm, dvc):
|
||||
def test_diff_dirty(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metric_1 = [{"y": 2}, {"y": 3}]
|
||||
_write_json(tmp_dir, metric_1, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init")
|
||||
_write_json(tmp_dir, metric_1, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="init",
|
||||
)
|
||||
|
||||
metric_2 = [{"y": 3}, {"y": 5}]
|
||||
_write_json(tmp_dir, metric_2, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="second")
|
||||
_write_json(tmp_dir, metric_2, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="second",
|
||||
)
|
||||
|
||||
metric_3 = [{"y": 5}, {"y": 6}]
|
||||
_write_json(tmp_dir, metric_3, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"])
|
||||
_write_json(tmp_dir, metric_3, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json", "metric.json", plots_no_cache=["metric.json"]
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.diff(fields={"y"})["metric.json"]
|
||||
plot_string = dvc.plots.diff(props={"fields": {"y"}})["metric.json"]
|
||||
|
||||
plot_content = json.loads(plot_string)
|
||||
assert plot_content["data"]["values"] == [
|
||||
{"y": 5, PlotData.INDEX_FIELD: 0, "rev": "working tree"},
|
||||
{"y": 6, PlotData.INDEX_FIELD: 1, "rev": "working tree"},
|
||||
{"y": 5, PlotData.INDEX_FIELD: 0, "rev": "workspace"},
|
||||
{"y": 6, PlotData.INDEX_FIELD: 1, "rev": "workspace"},
|
||||
{"y": 3, PlotData.INDEX_FIELD: 0, "rev": "HEAD"},
|
||||
{"y": 5, PlotData.INDEX_FIELD: 1, "rev": "HEAD"},
|
||||
]
|
||||
|
||||
+250
-163
@@ -22,24 +22,6 @@ from dvc.repo.plots.template import (
|
||||
)
|
||||
|
||||
|
||||
def _remove_whitespace(value):
|
||||
return value.replace(" ", "").replace("\n", "")
|
||||
|
||||
|
||||
def _run_with(tmp_dir, commit=None, tag=None, **kwargs):
|
||||
stage = tmp_dir.dvc.run(single_stage=True, **kwargs)
|
||||
if hasattr(tmp_dir.dvc, "scm"):
|
||||
files = [stage.path]
|
||||
files += [
|
||||
os.fspath(out.path_info) for out in stage.outs if not out.use_cache
|
||||
]
|
||||
tmp_dir.dvc.scm.add(files)
|
||||
if commit:
|
||||
tmp_dir.dvc.scm.commit(commit)
|
||||
if tag:
|
||||
tmp_dir.dvc.scm.tag(tag)
|
||||
|
||||
|
||||
def _write_csv(metric, filename, header=True):
|
||||
with open(filename, "w", newline="") as csvobj:
|
||||
if header:
|
||||
@@ -59,24 +41,27 @@ def _write_json(tmp_dir, metric, filename):
|
||||
tmp_dir.gen(filename, json.dumps(metric, sort_keys=True))
|
||||
|
||||
|
||||
def test_plot_csv_one_column(tmp_dir, scm, dvc):
|
||||
def test_plot_csv_one_column(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
# no header
|
||||
metric = [{"val": 2}, {"val": 3}]
|
||||
_write_csv(metric, "metric.csv", header=False)
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.csv"])
|
||||
_write_csv(metric, "metric_t.csv", header=False)
|
||||
run_copy_metrics(
|
||||
"metric_t.csv", "metric.csv", plots_no_cache=["metric.csv"]
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show(
|
||||
csv_header=False,
|
||||
x_title="x_title",
|
||||
y_title="y_title",
|
||||
title="mytitle",
|
||||
)["metric.csv"]
|
||||
props = {
|
||||
"csv_header": False,
|
||||
"xlab": "x_title",
|
||||
"ylab": "y_title",
|
||||
"title": "mytitle",
|
||||
}
|
||||
plot_string = dvc.plots.show(props=props)["metric.csv"]
|
||||
|
||||
plot_content = json.loads(plot_string)
|
||||
assert plot_content["title"] == "mytitle"
|
||||
assert plot_content["data"]["values"] == [
|
||||
{"0": "2", PlotData.INDEX_FIELD: 0, "rev": "working tree"},
|
||||
{"0": "3", PlotData.INDEX_FIELD: 1, "rev": "working tree"},
|
||||
{"0": "2", PlotData.INDEX_FIELD: 0, "rev": "workspace"},
|
||||
{"0": "3", PlotData.INDEX_FIELD: 1, "rev": "workspace"},
|
||||
]
|
||||
assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
|
||||
assert plot_content["encoding"]["y"]["field"] == "0"
|
||||
@@ -84,13 +69,15 @@ def test_plot_csv_one_column(tmp_dir, scm, dvc):
|
||||
assert plot_content["encoding"]["y"]["title"] == "y_title"
|
||||
|
||||
|
||||
def test_plot_csv_multiple_columns(tmp_dir, scm, dvc):
|
||||
def test_plot_csv_multiple_columns(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metric = [
|
||||
OrderedDict([("first_val", 100), ("second_val", 100), ("val", 2)]),
|
||||
OrderedDict([("first_val", 200), ("second_val", 300), ("val", 3)]),
|
||||
]
|
||||
_write_csv(metric, "metric.csv")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.csv"])
|
||||
_write_csv(metric, "metric_t.csv")
|
||||
run_copy_metrics(
|
||||
"metric_t.csv", "metric.csv", plots_no_cache=["metric.csv"]
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show()["metric.csv"]
|
||||
|
||||
@@ -99,14 +86,14 @@ def test_plot_csv_multiple_columns(tmp_dir, scm, dvc):
|
||||
{
|
||||
"val": "2",
|
||||
PlotData.INDEX_FIELD: 0,
|
||||
"rev": "working tree",
|
||||
"rev": "workspace",
|
||||
"first_val": "100",
|
||||
"second_val": "100",
|
||||
},
|
||||
{
|
||||
"val": "3",
|
||||
PlotData.INDEX_FIELD: 1,
|
||||
"rev": "working tree",
|
||||
"rev": "workspace",
|
||||
"first_val": "200",
|
||||
"second_val": "300",
|
||||
},
|
||||
@@ -115,29 +102,30 @@ def test_plot_csv_multiple_columns(tmp_dir, scm, dvc):
|
||||
assert plot_content["encoding"]["y"]["field"] == "val"
|
||||
|
||||
|
||||
def test_plot_csv_choose_axes(tmp_dir, scm, dvc):
|
||||
def test_plot_csv_choose_axes(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metric = [
|
||||
OrderedDict([("first_val", 100), ("second_val", 100), ("val", 2)]),
|
||||
OrderedDict([("first_val", 200), ("second_val", 300), ("val", 3)]),
|
||||
]
|
||||
_write_csv(metric, "metric.csv")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.csv"])
|
||||
_write_csv(metric, "metric_t.csv")
|
||||
run_copy_metrics(
|
||||
"metric_t.csv", "metric.csv", plots_no_cache=["metric.csv"]
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show(x_field="first_val", y_field="second_val")[
|
||||
"metric.csv"
|
||||
]
|
||||
props = {"x": "first_val", "y": "second_val"}
|
||||
plot_string = dvc.plots.show(props=props)["metric.csv"]
|
||||
|
||||
plot_content = json.loads(plot_string)
|
||||
assert plot_content["data"]["values"] == [
|
||||
{
|
||||
"val": "2",
|
||||
"rev": "working tree",
|
||||
"rev": "workspace",
|
||||
"first_val": "100",
|
||||
"second_val": "100",
|
||||
},
|
||||
{
|
||||
"val": "3",
|
||||
"rev": "working tree",
|
||||
"rev": "workspace",
|
||||
"first_val": "200",
|
||||
"second_val": "300",
|
||||
},
|
||||
@@ -146,29 +134,39 @@ def test_plot_csv_choose_axes(tmp_dir, scm, dvc):
|
||||
assert plot_content["encoding"]["y"]["field"] == "second_val"
|
||||
|
||||
|
||||
def test_plot_json_single_val(tmp_dir, scm, dvc):
|
||||
def test_plot_json_single_val(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metric = [{"val": 2}, {"val": 3}]
|
||||
_write_json(tmp_dir, metric, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="first run")
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="first run",
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show()["metric.json"]
|
||||
|
||||
plot_json = json.loads(plot_string)
|
||||
assert plot_json["data"]["values"] == [
|
||||
{"val": 2, PlotData.INDEX_FIELD: 0, "rev": "working tree"},
|
||||
{"val": 3, PlotData.INDEX_FIELD: 1, "rev": "working tree"},
|
||||
{"val": 2, PlotData.INDEX_FIELD: 0, "rev": "workspace"},
|
||||
{"val": 3, PlotData.INDEX_FIELD: 1, "rev": "workspace"},
|
||||
]
|
||||
assert plot_json["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
|
||||
assert plot_json["encoding"]["y"]["field"] == "val"
|
||||
|
||||
|
||||
def test_plot_json_multiple_val(tmp_dir, scm, dvc):
|
||||
def test_plot_json_multiple_val(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metric = [
|
||||
{"first_val": 100, "val": 2},
|
||||
{"first_val": 200, "val": 3},
|
||||
]
|
||||
_write_json(tmp_dir, metric, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="first run")
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="first run",
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show()["metric.json"]
|
||||
|
||||
@@ -178,56 +176,118 @@ def test_plot_json_multiple_val(tmp_dir, scm, dvc):
|
||||
"val": 2,
|
||||
PlotData.INDEX_FIELD: 0,
|
||||
"first_val": 100,
|
||||
"rev": "working tree",
|
||||
"rev": "workspace",
|
||||
},
|
||||
{
|
||||
"val": 3,
|
||||
PlotData.INDEX_FIELD: 1,
|
||||
"first_val": 200,
|
||||
"rev": "working tree",
|
||||
"rev": "workspace",
|
||||
},
|
||||
]
|
||||
assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
|
||||
assert plot_content["encoding"]["y"]["field"] == "val"
|
||||
|
||||
|
||||
def test_plot_confusion(tmp_dir, dvc):
|
||||
def test_plot_confusion(tmp_dir, dvc, run_copy_metrics):
|
||||
confusion_matrix = [
|
||||
{"predicted": "B", "actual": "A"},
|
||||
{"predicted": "A", "actual": "A"},
|
||||
]
|
||||
_write_json(tmp_dir, confusion_matrix, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="first run")
|
||||
_write_json(tmp_dir, confusion_matrix, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="first run",
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show(
|
||||
template="confusion", x_field="predicted", y_field="actual",
|
||||
)["metric.json"]
|
||||
props = {"template": "confusion", "x": "predicted", "y": "actual"}
|
||||
plot_string = dvc.plots.show(props=props)["metric.json"]
|
||||
|
||||
plot_content = json.loads(plot_string)
|
||||
assert plot_content["data"]["values"] == [
|
||||
{"predicted": "B", "actual": "A", "rev": "working tree"},
|
||||
{"predicted": "A", "actual": "A", "rev": "working tree"},
|
||||
{"predicted": "B", "actual": "A", "rev": "workspace"},
|
||||
{"predicted": "A", "actual": "A", "rev": "workspace"},
|
||||
]
|
||||
assert plot_content["encoding"]["x"]["field"] == "predicted"
|
||||
assert plot_content["encoding"]["y"]["field"] == "actual"
|
||||
|
||||
|
||||
def test_plot_multiple_revs_default(tmp_dir, scm, dvc):
|
||||
def test_plot_multiple_revs_default(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metric_1 = [{"y": 2}, {"y": 3}]
|
||||
_write_json(tmp_dir, metric_1, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init", tag="v1")
|
||||
_write_json(tmp_dir, metric_1, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="init",
|
||||
tag="v1",
|
||||
)
|
||||
|
||||
metric_2 = [{"y": 3}, {"y": 5}]
|
||||
_write_json(tmp_dir, metric_2, "metric.json")
|
||||
_run_with(
|
||||
tmp_dir, plots_no_cache=["metric.json"], commit="second", tag="v2"
|
||||
_write_json(tmp_dir, metric_2, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="second",
|
||||
tag="v2",
|
||||
)
|
||||
|
||||
metric_3 = [{"y": 5}, {"y": 6}]
|
||||
_write_json(tmp_dir, metric_3, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="third")
|
||||
_write_json(tmp_dir, metric_3, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="third",
|
||||
)
|
||||
plot_string = dvc.plots.show(
|
||||
revs=["HEAD", "v2", "v1"], props={"fields": {"y"}}
|
||||
)["metric.json"]
|
||||
|
||||
plot_string = dvc.plots.show(fields={"y"}, revs=["HEAD", "v2", "v1"],)[
|
||||
plot_content = json.loads(plot_string)
|
||||
assert plot_content["data"]["values"] == [
|
||||
{"y": 5, PlotData.INDEX_FIELD: 0, "rev": "HEAD"},
|
||||
{"y": 6, PlotData.INDEX_FIELD: 1, "rev": "HEAD"},
|
||||
{"y": 3, PlotData.INDEX_FIELD: 0, "rev": "v2"},
|
||||
{"y": 5, PlotData.INDEX_FIELD: 1, "rev": "v2"},
|
||||
{"y": 2, PlotData.INDEX_FIELD: 0, "rev": "v1"},
|
||||
{"y": 3, PlotData.INDEX_FIELD: 1, "rev": "v1"},
|
||||
]
|
||||
assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
|
||||
assert plot_content["encoding"]["y"]["field"] == "y"
|
||||
|
||||
|
||||
def test_plot_multiple_revs(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
shutil.copy(tmp_dir / ".dvc" / "plots" / "default.json", "template.json")
|
||||
|
||||
metric_1 = [{"y": 2}, {"y": 3}]
|
||||
_write_json(tmp_dir, metric_1, "metric_t.json")
|
||||
stage = run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="init",
|
||||
tag="v1",
|
||||
)
|
||||
|
||||
metric_2 = [{"y": 3}, {"y": 5}]
|
||||
_write_json(tmp_dir, metric_2, "metric_t.json")
|
||||
assert dvc.reproduce(stage.addressing) == [stage]
|
||||
scm.add(["metric.json", stage.path])
|
||||
scm.commit("second")
|
||||
scm.tag("v2")
|
||||
|
||||
metric_3 = [{"y": 5}, {"y": 6}]
|
||||
_write_json(tmp_dir, metric_3, "metric_t.json")
|
||||
assert dvc.reproduce(stage.addressing) == [stage]
|
||||
scm.add(["metric.json", stage.path])
|
||||
scm.commit("third")
|
||||
|
||||
props = {"template": "template.json"}
|
||||
plot_string = dvc.plots.show(revs=["HEAD", "v2", "v1"], props=props)[
|
||||
"metric.json"
|
||||
]
|
||||
|
||||
@@ -244,48 +304,17 @@ def test_plot_multiple_revs_default(tmp_dir, scm, dvc):
|
||||
assert plot_content["encoding"]["y"]["field"] == "y"
|
||||
|
||||
|
||||
def test_plot_multiple_revs(tmp_dir, scm, dvc):
|
||||
shutil.copy(tmp_dir / ".dvc" / "plots" / "default.json", "template.json")
|
||||
|
||||
metric_1 = [{"y": 2}, {"y": 3}]
|
||||
_write_json(tmp_dir, metric_1, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init", tag="v1")
|
||||
|
||||
metric_2 = [{"y": 3}, {"y": 5}]
|
||||
_write_json(tmp_dir, metric_2, "metric.json")
|
||||
_run_with(
|
||||
tmp_dir, plots_no_cache=["metric.json"], commit="second", tag="v2"
|
||||
)
|
||||
|
||||
metric_3 = [{"y": 5}, {"y": 6}]
|
||||
_write_json(tmp_dir, metric_3, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="third")
|
||||
|
||||
plot_string = dvc.plots.show(
|
||||
template="template.json", revs=["HEAD", "v2", "v1"],
|
||||
)["metric.json"]
|
||||
|
||||
plot_content = json.loads(plot_string)
|
||||
assert plot_content["data"]["values"] == [
|
||||
{"y": 5, PlotData.INDEX_FIELD: 0, "rev": "HEAD"},
|
||||
{"y": 6, PlotData.INDEX_FIELD: 1, "rev": "HEAD"},
|
||||
{"y": 3, PlotData.INDEX_FIELD: 0, "rev": "v2"},
|
||||
{"y": 5, PlotData.INDEX_FIELD: 1, "rev": "v2"},
|
||||
{"y": 2, PlotData.INDEX_FIELD: 0, "rev": "v1"},
|
||||
{"y": 3, PlotData.INDEX_FIELD: 1, "rev": "v1"},
|
||||
]
|
||||
assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
|
||||
assert plot_content["encoding"]["y"]["field"] == "y"
|
||||
|
||||
|
||||
def test_plot_even_if_metric_missing(tmp_dir, scm, dvc, caplog):
|
||||
def test_plot_even_if_metric_missing(
|
||||
tmp_dir, scm, dvc, caplog, run_copy_metrics
|
||||
):
|
||||
tmp_dir.scm_gen("some_file", "content", commit="there is no metric")
|
||||
scm.tag("v1")
|
||||
|
||||
metric = [{"y": 2}, {"y": 3}]
|
||||
_write_json(tmp_dir, metric, "metric.json")
|
||||
_run_with(
|
||||
tmp_dir,
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="there is metric",
|
||||
tag="v2",
|
||||
@@ -335,19 +364,24 @@ def custom_template(tmp_dir, dvc):
|
||||
return custom_template
|
||||
|
||||
|
||||
def test_custom_template(tmp_dir, scm, dvc, custom_template):
|
||||
def test_custom_template(tmp_dir, scm, dvc, custom_template, run_copy_metrics):
|
||||
metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
_write_json(tmp_dir, metric, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init", tag="v1")
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="init",
|
||||
tag="v1",
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show(
|
||||
template=os.fspath(custom_template), x_field="a", y_field="b"
|
||||
)["metric.json"]
|
||||
props = {"template": os.fspath(custom_template), "x": "a", "y": "b"}
|
||||
plot_string = dvc.plots.show(props=props)["metric.json"]
|
||||
|
||||
plot_content = json.loads(plot_string)
|
||||
assert plot_content["data"]["values"] == [
|
||||
{"a": 1, "b": 2, "rev": "working tree"},
|
||||
{"a": 2, "b": 3, "rev": "working tree"},
|
||||
{"a": 1, "b": 2, "rev": "workspace"},
|
||||
{"a": 2, "b": 3, "rev": "workspace"},
|
||||
]
|
||||
assert plot_content["encoding"]["x"]["field"] == "a"
|
||||
assert plot_content["encoding"]["y"]["field"] == "b"
|
||||
@@ -358,30 +392,37 @@ def _replace(path, src, dst):
|
||||
|
||||
|
||||
def test_custom_template_with_specified_data(
|
||||
tmp_dir, scm, dvc, custom_template
|
||||
tmp_dir, scm, dvc, custom_template, run_copy_metrics
|
||||
):
|
||||
_replace(
|
||||
custom_template, "DVC_METRIC_DATA", "DVC_METRIC_DATA,metric.json",
|
||||
)
|
||||
|
||||
metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
_write_json(tmp_dir, metric, "metric.json")
|
||||
_run_with(tmp_dir, outs_no_cache=["metric.json"], commit="init", tag="v1")
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
outs_no_cache=["metric.json"],
|
||||
commit="init",
|
||||
tag="v1",
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show(
|
||||
template=os.fspath(custom_template), x_field="a", y_field="b",
|
||||
)["metric.json"]
|
||||
props = {"template": os.fspath(custom_template), "x": "a", "y": "b"}
|
||||
plot_string = dvc.plots.show(props=props)["metric.json"]
|
||||
|
||||
plot_content = json.loads(plot_string)
|
||||
assert plot_content["data"]["values"] == [
|
||||
{"a": 1, "b": 2, "rev": "working tree"},
|
||||
{"a": 2, "b": 3, "rev": "working tree"},
|
||||
{"a": 1, "b": 2, "rev": "workspace"},
|
||||
{"a": 2, "b": 3, "rev": "workspace"},
|
||||
]
|
||||
assert plot_content["encoding"]["x"]["field"] == "a"
|
||||
assert plot_content["encoding"]["y"]["field"] == "b"
|
||||
|
||||
|
||||
def test_plot_override_specified_data_source(tmp_dir, scm, dvc):
|
||||
def test_plot_override_specified_data_source(
|
||||
tmp_dir, scm, dvc, run_copy_metrics
|
||||
):
|
||||
shutil.copy(
|
||||
tmp_dir / ".dvc" / "plots" / "default.json",
|
||||
tmp_dir / "newtemplate.json",
|
||||
@@ -393,17 +434,24 @@ def test_plot_override_specified_data_source(tmp_dir, scm, dvc):
|
||||
)
|
||||
|
||||
metric = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
_write_json(tmp_dir, metric, "metric2.json")
|
||||
_run_with(tmp_dir, outs_no_cache=["metric2.json"], commit="init", tag="v1")
|
||||
_write_json(tmp_dir, metric, "metric1.json")
|
||||
run_copy_metrics(
|
||||
"metric1.json",
|
||||
"metric2.json",
|
||||
plots_no_cache=["metric2.json"],
|
||||
commit="init",
|
||||
tag="v1",
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show(
|
||||
targets=["metric2.json"], template="newtemplate.json", x_field="a"
|
||||
)["metric2.json"]
|
||||
props = {"template": "newtemplate.json", "x": "a"}
|
||||
plot_string = dvc.plots.show(targets=["metric2.json"], props=props)[
|
||||
"metric2.json"
|
||||
]
|
||||
|
||||
plot_content = json.loads(plot_string)
|
||||
assert plot_content["data"]["values"] == [
|
||||
{"a": 1, "b": 2, "rev": "working tree"},
|
||||
{"a": 2, "b": 3, "rev": "working tree"},
|
||||
{"a": 1, "b": 2, "rev": "workspace"},
|
||||
{"a": 2, "b": 3, "rev": "workspace"},
|
||||
]
|
||||
assert plot_content["encoding"]["x"]["field"] == "a"
|
||||
assert plot_content["encoding"]["y"]["field"] == "b"
|
||||
@@ -414,82 +462,121 @@ def test_should_raise_on_no_template_and_datafile(tmp_dir, dvc):
|
||||
dvc.plots.show()
|
||||
|
||||
|
||||
def test_should_raise_on_no_template(tmp_dir, dvc):
|
||||
def test_should_raise_on_no_template(tmp_dir, dvc, run_copy_metrics):
|
||||
metric = [{"val": 2}, {"val": 3}]
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="first run",
|
||||
)
|
||||
|
||||
with pytest.raises(TemplateNotFoundError):
|
||||
dvc.plots.show("metric.json", template="non_existing_template.json")
|
||||
props = {"template": "non_existing_template.json"}
|
||||
dvc.plots.show("metric.json", props=props)
|
||||
|
||||
|
||||
def test_plot_no_data(tmp_dir, dvc):
|
||||
with pytest.raises(NoDataForTemplateError):
|
||||
dvc.plots.show(template="default")
|
||||
dvc.plots.show(props={"template": "default"})
|
||||
|
||||
|
||||
def test_plot_wrong_metric_type(tmp_dir, scm, dvc):
|
||||
tmp_dir.scm_gen("metric.txt", "content", commit="initial")
|
||||
def test_plot_wrong_metric_type(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
tmp_dir.gen("metric_t.txt", "some text")
|
||||
run_copy_metrics(
|
||||
"metric_t.txt",
|
||||
"metric.txt",
|
||||
plots_no_cache=["metric.txt"],
|
||||
commit="add text metric",
|
||||
)
|
||||
|
||||
with pytest.raises(PlotMetricTypeError):
|
||||
dvc.plots.show(targets=["metric.txt"])
|
||||
|
||||
|
||||
def test_plot_choose_columns(tmp_dir, scm, dvc, custom_template):
|
||||
def test_plot_choose_columns(
|
||||
tmp_dir, scm, dvc, custom_template, run_copy_metrics
|
||||
):
|
||||
metric = [{"a": 1, "b": 2, "c": 3}, {"a": 2, "b": 3, "c": 4}]
|
||||
_write_json(tmp_dir, metric, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init", tag="v1")
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="init",
|
||||
tag="v1",
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show(
|
||||
template=os.fspath(custom_template),
|
||||
fields={"b", "c"},
|
||||
x_field="b",
|
||||
y_field="c",
|
||||
)["metric.json"]
|
||||
props = {
|
||||
"template": os.fspath(custom_template),
|
||||
"fields": {"b", "c"},
|
||||
"x": "b",
|
||||
"y": "c",
|
||||
}
|
||||
plot_string = dvc.plots.show(props=props)["metric.json"]
|
||||
|
||||
plot_content = json.loads(plot_string)
|
||||
assert plot_content["data"]["values"] == [
|
||||
{"b": 2, "c": 3, "rev": "working tree"},
|
||||
{"b": 3, "c": 4, "rev": "working tree"},
|
||||
{"b": 2, "c": 3, "rev": "workspace"},
|
||||
{"b": 3, "c": 4, "rev": "workspace"},
|
||||
]
|
||||
assert plot_content["encoding"]["x"]["field"] == "b"
|
||||
assert plot_content["encoding"]["y"]["field"] == "c"
|
||||
|
||||
|
||||
def test_plot_default_choose_column(tmp_dir, scm, dvc):
|
||||
def test_plot_default_choose_column(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metric = [{"a": 1, "b": 2, "c": 3}, {"a": 2, "b": 3, "c": 4}]
|
||||
_write_json(tmp_dir, metric, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="init", tag="v1")
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="init",
|
||||
tag="v1",
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show(fields={"b"})["metric.json"]
|
||||
plot_string = dvc.plots.show(props={"fields": {"b"}})["metric.json"]
|
||||
|
||||
plot_content = json.loads(plot_string)
|
||||
assert plot_content["data"]["values"] == [
|
||||
{PlotData.INDEX_FIELD: 0, "b": 2, "rev": "working tree"},
|
||||
{PlotData.INDEX_FIELD: 1, "b": 3, "rev": "working tree"},
|
||||
{PlotData.INDEX_FIELD: 0, "b": 2, "rev": "workspace"},
|
||||
{PlotData.INDEX_FIELD: 1, "b": 3, "rev": "workspace"},
|
||||
]
|
||||
assert plot_content["encoding"]["x"]["field"] == PlotData.INDEX_FIELD
|
||||
assert plot_content["encoding"]["y"]["field"] == "b"
|
||||
|
||||
|
||||
def test_plot_yaml(tmp_dir, scm, dvc):
|
||||
def test_plot_yaml(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metric = [{"val": 2}, {"val": 3}]
|
||||
with open("metric.yaml", "w") as fobj:
|
||||
with open("metric_t.yaml", "w") as fobj:
|
||||
yaml.dump(metric, fobj)
|
||||
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.yaml"])
|
||||
run_copy_metrics(
|
||||
"metric_t.yaml", "metric.yaml", plots_no_cache=["metric.yaml"]
|
||||
)
|
||||
|
||||
plot_string = dvc.plots.show()["metric.yaml"]
|
||||
|
||||
plot_content = json.loads(plot_string)
|
||||
assert plot_content["data"]["values"] == [
|
||||
{"val": 2, PlotData.INDEX_FIELD: 0, "rev": "working tree"},
|
||||
{"val": 3, PlotData.INDEX_FIELD: 1, "rev": "working tree"},
|
||||
{"val": 2, PlotData.INDEX_FIELD: 0, "rev": "workspace"},
|
||||
{"val": 3, PlotData.INDEX_FIELD: 1, "rev": "workspace"},
|
||||
]
|
||||
|
||||
|
||||
def test_raise_on_wrong_field(tmp_dir, scm, dvc):
|
||||
def test_raise_on_wrong_field(tmp_dir, scm, dvc, run_copy_metrics):
|
||||
metric = [{"val": 2}, {"val": 3}]
|
||||
_write_json(tmp_dir, metric, "metric.json")
|
||||
_run_with(tmp_dir, plots_no_cache=["metric.json"], commit="first run")
|
||||
_write_json(tmp_dir, metric, "metric_t.json")
|
||||
run_copy_metrics(
|
||||
"metric_t.json",
|
||||
"metric.json",
|
||||
plots_no_cache=["metric.json"],
|
||||
commit="first run",
|
||||
)
|
||||
|
||||
with pytest.raises(NoFieldInDataError):
|
||||
dvc.plots.show("metric.json", x_field="no_val")
|
||||
dvc.plots.show("metric.json", props={"x": "no_val"})
|
||||
|
||||
with pytest.raises(NoFieldInDataError):
|
||||
dvc.plots.show("metric.json", y_field="no_val")
|
||||
dvc.plots.show("metric.json", props={"y": "no_val"})
|
||||
|
||||
@@ -17,7 +17,7 @@ from dvc.exceptions import (
|
||||
OutputDuplicationError,
|
||||
OverlappingOutputPathsError,
|
||||
RecursiveAddingWhileUsingFilename,
|
||||
StageFileCorruptedError,
|
||||
YAMLFileCorruptedError,
|
||||
)
|
||||
from dvc.main import main
|
||||
from dvc.output.base import OutputAlreadyTrackedError, OutputIsStageFileError
|
||||
@@ -27,7 +27,7 @@ from dvc.stage import Stage
|
||||
from dvc.system import System
|
||||
from dvc.utils import LARGE_DIR_SIZE, file_md5, relpath
|
||||
from dvc.utils.fs import path_isin
|
||||
from dvc.utils.stage import load_stage_file
|
||||
from dvc.utils.yaml import load_yaml
|
||||
from tests.basic_env import TestDvc
|
||||
from tests.utils import get_gitignore_content
|
||||
|
||||
@@ -46,7 +46,7 @@ def test_add(tmp_dir, dvc):
|
||||
assert stage.outs[0].info["md5"] == md5
|
||||
assert stage.md5 is None
|
||||
|
||||
assert load_stage_file("foo.dvc") == {
|
||||
assert load_yaml("foo.dvc") == {
|
||||
"outs": [{"md5": "acbd18db4cc2f85cedef654fccc4a4d8", "path": "foo"}],
|
||||
}
|
||||
|
||||
@@ -220,14 +220,14 @@ class TestAddLocalRemoteFile(TestDvc):
|
||||
ret = main(["add", foo])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
d = load_stage_file("foo.dvc")
|
||||
d = load_yaml("foo.dvc")
|
||||
self.assertEqual(d["outs"][0]["path"], foo)
|
||||
|
||||
bar = os.path.join(cwd, self.BAR)
|
||||
ret = main(["add", bar])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
d = load_stage_file("bar.dvc")
|
||||
d = load_yaml("bar.dvc")
|
||||
self.assertEqual(d["outs"][0]["path"], self.BAR)
|
||||
|
||||
|
||||
@@ -371,7 +371,7 @@ class SymlinkAddTestBase(TestDvc):
|
||||
stage_file = self.data_file_name + DVC_FILE_SUFFIX
|
||||
self.assertTrue(os.path.exists(stage_file))
|
||||
|
||||
d = load_stage_file(stage_file)
|
||||
d = load_yaml(stage_file)
|
||||
relative_data_path = posixpath.join(
|
||||
self.link_name, self.data_file_name
|
||||
)
|
||||
@@ -436,8 +436,7 @@ class TestShouldThrowProperExceptionOnCorruptedStageFile(TestDvc):
|
||||
assert 1 == ret
|
||||
|
||||
expected_error = (
|
||||
"unable to read DVC-file: {} "
|
||||
"YAML file structure is corrupted".format(foo_stage)
|
||||
f"unable to read: '{foo_stage}', YAML file structure is corrupted"
|
||||
)
|
||||
|
||||
assert expected_error in self._caplog.text
|
||||
@@ -478,7 +477,7 @@ def test_failed_add_cleanup(tmp_dir, scm, dvc):
|
||||
dvc.add("foo")
|
||||
tmp_dir.gen("foo.dvc", "- broken\nyaml")
|
||||
|
||||
with pytest.raises(StageFileCorruptedError):
|
||||
with pytest.raises(YAMLFileCorruptedError):
|
||||
dvc.add("bar")
|
||||
|
||||
assert not os.path.exists("bar.dvc")
|
||||
|
||||
+64
-15
@@ -7,26 +7,24 @@ import shutil
|
||||
import pytest
|
||||
from mock import patch
|
||||
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX, Dvcfile
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX, PIPELINE_FILE, Dvcfile
|
||||
from dvc.exceptions import (
|
||||
CheckoutError,
|
||||
CheckoutErrorSuggestGit,
|
||||
ConfirmRemoveError,
|
||||
DvcException,
|
||||
NoOutputOrStageError,
|
||||
)
|
||||
from dvc.main import main
|
||||
from dvc.remote import S3Remote
|
||||
from dvc.remote.local import LocalRemote
|
||||
from dvc.repo import Repo as DvcRepo
|
||||
from dvc.stage import Stage
|
||||
from dvc.stage.exceptions import (
|
||||
StageFileBadNameError,
|
||||
StageFileDoesNotExistError,
|
||||
)
|
||||
from dvc.stage.exceptions import StageFileDoesNotExistError
|
||||
from dvc.system import System
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import walk_files
|
||||
from dvc.utils.stage import dump_stage_file, load_stage_file
|
||||
from dvc.utils.yaml import dump_yaml, load_yaml
|
||||
from tests.basic_env import TestDvc, TestDvcGit
|
||||
from tests.func.test_repro import TestRepro
|
||||
from tests.remotes import S3
|
||||
@@ -223,7 +221,7 @@ class TestCheckoutSelectiveRemove(CheckoutBase):
|
||||
self.assertEqual(0, ret)
|
||||
|
||||
stage_path = self.DATA_DIR + DVC_FILE_SUFFIX
|
||||
stage = load_stage_file(stage_path)
|
||||
stage = load_yaml(stage_path)
|
||||
staged_files = self.outs_info(stage)
|
||||
|
||||
# move instead of remove, to lock inode assigned to stage_files[0].path
|
||||
@@ -306,10 +304,10 @@ class TestGitIgnoreWhenCheckout(CheckoutBase):
|
||||
|
||||
class TestCheckoutMissingMd5InStageFile(TestRepro):
|
||||
def test(self):
|
||||
d = load_stage_file(self.file1_stage)
|
||||
d = load_yaml(self.file1_stage)
|
||||
del d[Stage.PARAM_OUTS][0][LocalRemote.PARAM_CHECKSUM]
|
||||
del d[Stage.PARAM_DEPS][0][LocalRemote.PARAM_CHECKSUM]
|
||||
dump_stage_file(self.file1_stage, d)
|
||||
dump_yaml(self.file1_stage, d)
|
||||
|
||||
with pytest.raises(CheckoutError):
|
||||
self.dvc.checkout(force=True)
|
||||
@@ -403,17 +401,26 @@ class TestCheckoutSuggestGit(TestRepro):
|
||||
def test(self):
|
||||
|
||||
try:
|
||||
self.dvc.checkout(targets=["gitbranch"])
|
||||
self.dvc.checkout(targets="gitbranch")
|
||||
except DvcException as exc:
|
||||
self.assertIsInstance(exc, CheckoutErrorSuggestGit)
|
||||
self.assertIsInstance(exc.__cause__, StageFileDoesNotExistError)
|
||||
self.assertIsInstance(exc.__cause__, NoOutputOrStageError)
|
||||
self.assertIsInstance(
|
||||
exc.__cause__.__cause__, StageFileDoesNotExistError
|
||||
)
|
||||
|
||||
try:
|
||||
self.dvc.checkout(targets=self.FOO)
|
||||
except DvcException as exc:
|
||||
self.assertIsInstance(exc, CheckoutErrorSuggestGit)
|
||||
self.assertIsInstance(exc.__cause__, NoOutputOrStageError)
|
||||
self.assertIsNone(exc.__cause__.__cause__)
|
||||
|
||||
try:
|
||||
self.dvc.checkout(targets=[self.FOO])
|
||||
self.dvc.checkout(targets="looks-like-dvcfile.dvc")
|
||||
except DvcException as exc:
|
||||
self.assertIsInstance(exc, CheckoutErrorSuggestGit)
|
||||
self.assertIsInstance(exc.__cause__, StageFileBadNameError)
|
||||
self.assertIsInstance(exc.__cause__, StageFileDoesNotExistError)
|
||||
self.assertIsNone(exc.__cause__.__cause__)
|
||||
|
||||
|
||||
@@ -770,9 +777,51 @@ def test_checkout_for_external_outputs(tmp_dir, dvc):
|
||||
assert stats == {**empty_checkout, "modified": [str(file_path)]}
|
||||
|
||||
|
||||
def test_checkouts_for_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
|
||||
from dvc.dvcfile import PIPELINE_FILE
|
||||
def test_checkouts_with_different_addressing(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen({"foo": "foo", "lorem": "lorem"})
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
run_copy("lorem", "ipsum", name="copy-lorem-ipsum")
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
(tmp_dir / "ipsum").unlink()
|
||||
assert set(dvc.checkout(PIPELINE_FILE)["added"]) == {"bar", "ipsum"}
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
(tmp_dir / "ipsum").unlink()
|
||||
assert set(dvc.checkout(":")["added"]) == {"bar", "ipsum"}
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
assert dvc.checkout("copy-foo-bar")["added"] == ["bar"]
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
assert dvc.checkout("dvc.yaml:copy-foo-bar")["added"] == ["bar"]
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
assert dvc.checkout(":copy-foo-bar")["added"] == ["bar"]
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
(tmp_dir / "data").mkdir()
|
||||
with (tmp_dir / "data").chdir():
|
||||
assert dvc.checkout(relpath(tmp_dir / "dvc.yaml") + ":copy-foo-bar")[
|
||||
"added"
|
||||
] == [relpath(tmp_dir / "bar")]
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
assert dvc.checkout("bar")["added"] == ["bar"]
|
||||
|
||||
|
||||
def test_checkouts_on_same_stage_name_and_output_name(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
run_copy("foo", "copy-foo-bar", name="make_collision")
|
||||
|
||||
(tmp_dir / "bar").unlink()
|
||||
(tmp_dir / "copy-foo-bar").unlink()
|
||||
assert dvc.checkout("copy-foo-bar")["added"] == ["bar"]
|
||||
assert dvc.checkout("./copy-foo-bar")["added"] == ["copy-foo-bar"]
|
||||
|
||||
|
||||
def test_checkouts_for_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage1 = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
tmp_dir.gen("lorem", "lorem")
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import pytest
|
||||
|
||||
from dvc.dvcfile import PIPELINE_FILE
|
||||
from dvc.stage.exceptions import StageCommitError
|
||||
from dvc.utils.stage import dump_stage_file, load_stage_file
|
||||
from dvc.utils.yaml import dump_yaml, load_yaml
|
||||
|
||||
|
||||
def test_commit_recursive(tmp_dir, dvc):
|
||||
@@ -62,15 +63,15 @@ def test_commit_changed_md5(tmp_dir, dvc):
|
||||
tmp_dir.gen({"file": "file content"})
|
||||
(stage,) = dvc.add("file", no_commit=True)
|
||||
|
||||
stage_file_content = load_stage_file(stage.path)
|
||||
stage_file_content = load_yaml(stage.path)
|
||||
stage_file_content["md5"] = "1111111111"
|
||||
dump_stage_file(stage.path, stage_file_content)
|
||||
dump_yaml(stage.path, stage_file_content)
|
||||
|
||||
with pytest.raises(StageCommitError):
|
||||
dvc.commit(stage.path)
|
||||
|
||||
dvc.commit(stage.path, force=True)
|
||||
assert "md5" not in load_stage_file(stage.path)
|
||||
assert "md5" not in load_yaml(stage.path)
|
||||
|
||||
|
||||
def test_commit_no_exec(tmp_dir, dvc):
|
||||
@@ -81,3 +82,16 @@ def test_commit_no_exec(tmp_dir, dvc):
|
||||
assert dvc.status(stage.path)
|
||||
dvc.commit(stage.path, force=True)
|
||||
assert dvc.status(stage.path) == {}
|
||||
|
||||
|
||||
def test_commit_pipeline_stage(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage = run_copy("foo", "bar", no_commit=True, name="copy-foo-bar")
|
||||
assert dvc.status(stage.addressing)
|
||||
assert dvc.commit(stage.addressing, force=True) == [stage]
|
||||
assert not dvc.status(stage.addressing)
|
||||
|
||||
# just to confirm different variants work
|
||||
assert dvc.commit(f":{stage.addressing}") == [stage]
|
||||
assert dvc.commit(f"{PIPELINE_FILE}:{stage.addressing}") == [stage]
|
||||
assert dvc.commit(PIPELINE_FILE) == [stage]
|
||||
|
||||
@@ -27,7 +27,7 @@ from dvc.remote.base import STATUS_DELETED, STATUS_NEW, STATUS_OK
|
||||
from dvc.stage.exceptions import StageNotFound
|
||||
from dvc.utils import file_md5
|
||||
from dvc.utils.fs import remove
|
||||
from dvc.utils.stage import dump_stage_file, load_stage_file
|
||||
from dvc.utils.yaml import dump_yaml, load_yaml
|
||||
from tests.basic_env import TestDvc
|
||||
from tests.remotes import (
|
||||
GCP,
|
||||
@@ -523,9 +523,9 @@ class TestWarnOnOutdatedStage(TestDvc):
|
||||
self.main(["push"])
|
||||
|
||||
stage_file_path = stage.relpath
|
||||
content = load_stage_file(stage_file_path)
|
||||
content = load_yaml(stage_file_path)
|
||||
del content["outs"][0]["md5"]
|
||||
dump_stage_file(stage_file_path, content)
|
||||
dump_yaml(stage_file_path, content)
|
||||
|
||||
with self._caplog.at_level(logging.WARNING, logger="dvc"):
|
||||
self._caplog.clear()
|
||||
@@ -912,3 +912,44 @@ def test_pull_stats(tmp_dir, dvc, caplog, setup_remote):
|
||||
with caplog.at_level(level=logging.INFO, logger="dvc"):
|
||||
main(["pull"])
|
||||
assert "Everything is up to date." in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"key,expected", [("all_tags", 2), ("all_branches", 3), ("all_commits", 3)]
|
||||
)
|
||||
def test_push_pull_all(tmp_dir, scm, dvc, setup_remote, key, expected):
|
||||
setup_remote(dvc)
|
||||
tmp_dir.dvc_gen({"foo": "foo"}, commit="first")
|
||||
scm.tag("v1")
|
||||
dvc.remove("foo.dvc")
|
||||
tmp_dir.dvc_gen({"bar": "bar"}, commit="second")
|
||||
scm.tag("v2")
|
||||
with tmp_dir.branch("branch", new=True):
|
||||
dvc.remove("bar.dvc")
|
||||
tmp_dir.dvc_gen({"baz": "baz"}, commit="branch")
|
||||
|
||||
assert dvc.push(**{key: True}) == expected
|
||||
|
||||
clean(["foo", "bar", "baz"], dvc)
|
||||
assert dvc.pull(**{key: True})["fetched"] == expected
|
||||
|
||||
|
||||
def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, setup_remote):
|
||||
remote_path = setup_remote(dvc)
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_copy("foo", "bar", no_commit=True, name="copy-foo-bar")
|
||||
|
||||
dvc.push("copy-foo-bar")
|
||||
assert len(recurse_list_dir(remote_path)) == 1
|
||||
# pushing everything so as we can check pull/fetch only downloads
|
||||
# from specified targets
|
||||
dvc.push()
|
||||
clean(["foo", "bar"], dvc)
|
||||
|
||||
dvc.pull("copy-foo-bar")
|
||||
assert (tmp_dir / "bar").exists()
|
||||
assert len(recurse_list_dir(dvc.cache.local.cache_dir)) == 1
|
||||
clean(["bar"], dvc)
|
||||
|
||||
dvc.fetch("copy-foo-bar")
|
||||
assert len(recurse_list_dir(dvc.cache.local.cache_dir)) == 1
|
||||
|
||||
+120
-27
@@ -1,11 +1,14 @@
|
||||
import textwrap
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.dvcfile import PIPELINE_FILE, Dvcfile
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK, Dvcfile
|
||||
from dvc.stage.exceptions import (
|
||||
StageFileDoesNotExistError,
|
||||
StageNameUnspecified,
|
||||
StageFileFormatError,
|
||||
)
|
||||
from dvc.stage.loader import StageNotFound
|
||||
from dvc.utils.yaml import dump_yaml
|
||||
|
||||
|
||||
def test_run_load_one_for_multistage(tmp_dir, dvc):
|
||||
@@ -165,35 +168,125 @@ def test_stage_collection(tmp_dir, dvc):
|
||||
assert {s for s in dvc.stages} == {stage1, stage3, stage2}
|
||||
|
||||
|
||||
def test_stage_filter(tmp_dir, dvc, run_copy):
|
||||
def test_remove_stage(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage1 = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
stage2 = run_copy("bar", "bax", name="copy-bar-bax")
|
||||
stage3 = run_copy("bax", "baz", name="copy-bax-baz")
|
||||
stages = Dvcfile(dvc, PIPELINE_FILE).stages
|
||||
assert set(stages.filter(None).values()) == {
|
||||
stage1,
|
||||
stage2,
|
||||
stage3,
|
||||
}
|
||||
assert set(stages.filter("copy-bar-bax").values()) == {stage2}
|
||||
assert stages.filter("copy-bar-bax").get("copy-bar-bax") == stage2
|
||||
with pytest.raises(StageNameUnspecified):
|
||||
stages.get(None)
|
||||
stage = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
stage2 = run_copy("bar", "foobar", name="copy-bar-foobar")
|
||||
|
||||
with pytest.raises(StageNotFound):
|
||||
assert stages["unknown"]
|
||||
dvc_file = Dvcfile(dvc, PIPELINE_FILE)
|
||||
assert dvc_file.exists()
|
||||
assert {"copy-bar-foobar", "copy-foo-bar"} == set(
|
||||
dvc_file._load()[0]["stages"].keys()
|
||||
)
|
||||
|
||||
with pytest.raises(StageNotFound):
|
||||
assert stages.filter("unknown")
|
||||
dvc_file.remove_stage(stage)
|
||||
|
||||
assert ["copy-bar-foobar"] == list(dvc_file._load()[0]["stages"].keys())
|
||||
|
||||
# sanity check
|
||||
stage2.reload()
|
||||
|
||||
# re-check to see if it fails if there's no stage entry
|
||||
dvc_file.remove_stage(stage)
|
||||
dvc_file.remove(force=True)
|
||||
# should not fail when there's no file at all.
|
||||
dvc_file.remove_stage(stage)
|
||||
|
||||
|
||||
def test_stage_filter_in_singlestage_file(tmp_dir, dvc, run_copy):
|
||||
def test_remove_stage_lockfile(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
stage2 = run_copy("bar", "foobar", name="copy-bar-foobar")
|
||||
|
||||
dvc_file = Dvcfile(dvc, PIPELINE_FILE)
|
||||
lock_file = dvc_file._lockfile
|
||||
assert dvc_file.exists()
|
||||
assert lock_file.exists()
|
||||
assert {"copy-bar-foobar", "copy-foo-bar"} == set(lock_file.load().keys())
|
||||
lock_file.remove_stage(stage)
|
||||
|
||||
assert ["copy-bar-foobar"] == list(lock_file.load().keys())
|
||||
|
||||
# sanity check
|
||||
stage2.reload()
|
||||
|
||||
# re-check to see if it fails if there's no stage entry
|
||||
lock_file.remove_stage(stage)
|
||||
lock_file.remove()
|
||||
# should not fail when there's no file at all.
|
||||
lock_file.remove_stage(stage)
|
||||
|
||||
|
||||
def test_remove_stage_dvcfiles(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage = run_copy("foo", "bar", single_stage=True)
|
||||
dvcfile = Dvcfile(dvc, stage.path)
|
||||
assert set(dvcfile.stages.filter(None).values()) == {stage}
|
||||
assert dvcfile.stages.filter(None).get(None) == stage
|
||||
assert dvcfile.stages.filter("copy-bar-bax").get("copy-bar-bax") == stage
|
||||
assert dvcfile.stages.filter("copy-bar-bax").get(None) == stage
|
||||
assert set(dvcfile.stages.filter("unknown").values()) == {stage}
|
||||
|
||||
dvc_file = Dvcfile(dvc, stage.path)
|
||||
assert dvc_file.exists()
|
||||
dvc_file.remove_stage(stage)
|
||||
assert not dvc_file.exists()
|
||||
|
||||
# re-check to see if it fails if there's no stage entry
|
||||
dvc_file.remove_stage(stage)
|
||||
dvc_file.remove(force=True)
|
||||
|
||||
# should not fail when there's no file at all.
|
||||
dvc_file.remove_stage(stage)
|
||||
|
||||
|
||||
def test_remove_stage_on_lockfile_format_error(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
dvc_file = Dvcfile(dvc, stage.path)
|
||||
lock_file = dvc_file._lockfile
|
||||
|
||||
data = dvc_file._load()[0]
|
||||
lock_data = lock_file.load()
|
||||
lock_data["gibberish"] = True
|
||||
data["gibberish"] = True
|
||||
dump_yaml(lock_file.relpath, lock_data)
|
||||
with pytest.raises(StageFileFormatError):
|
||||
dvc_file.remove_stage(stage)
|
||||
|
||||
lock_file.remove()
|
||||
dvc_file.dump(stage)
|
||||
|
||||
dump_yaml(dvc_file.relpath, data)
|
||||
with pytest.raises(StageFileFormatError):
|
||||
dvc_file.remove_stage(stage)
|
||||
|
||||
|
||||
def test_remove_stage_preserves_comment(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen(
|
||||
"dvc.yaml",
|
||||
textwrap.dedent(
|
||||
"""\
|
||||
stages:
|
||||
generate-foo:
|
||||
cmd: "echo foo > foo"
|
||||
# This copies 'foo' text to 'foo' file.
|
||||
outs:
|
||||
- foo
|
||||
copy-foo-bar:
|
||||
cmd: "python copy.py foo bar"
|
||||
deps:
|
||||
- foo
|
||||
outs:
|
||||
- bar"""
|
||||
),
|
||||
)
|
||||
|
||||
dvc.reproduce(PIPELINE_FILE)
|
||||
|
||||
dvc_file = Dvcfile(dvc, PIPELINE_FILE)
|
||||
|
||||
assert dvc_file.exists()
|
||||
assert (tmp_dir / PIPELINE_LOCK).exists()
|
||||
assert (tmp_dir / "foo").exists()
|
||||
assert (tmp_dir / "bar").exists()
|
||||
|
||||
dvc_file.remove_stage(dvc_file.stages["copy-foo-bar"])
|
||||
assert (
|
||||
"# This copies 'foo' text to 'foo' file."
|
||||
in (tmp_dir / PIPELINE_FILE).read_text()
|
||||
)
|
||||
|
||||
@@ -68,7 +68,7 @@ class TestGCBranchesTags(TestDvcGit):
|
||||
self.dvc.scm.tag("v1.0")
|
||||
|
||||
self.dvc.scm.checkout("test", create_new=True)
|
||||
self.dvc.remove(stages[0].relpath, dvc_only=True)
|
||||
self.dvc.remove(stages[0].relpath)
|
||||
with open(fname, "w+") as fobj:
|
||||
fobj.write("test")
|
||||
stages = self.dvc.add(fname)
|
||||
@@ -77,7 +77,7 @@ class TestGCBranchesTags(TestDvcGit):
|
||||
self.dvc.scm.commit("test")
|
||||
|
||||
self.dvc.scm.checkout("master")
|
||||
self.dvc.remove(stages[0].relpath, dvc_only=True)
|
||||
self.dvc.remove(stages[0].relpath)
|
||||
with open(fname, "w+") as fobj:
|
||||
fobj.write("trash")
|
||||
stages = self.dvc.add(fname)
|
||||
@@ -85,7 +85,7 @@ class TestGCBranchesTags(TestDvcGit):
|
||||
self.dvc.scm.add([".gitignore", stages[0].relpath])
|
||||
self.dvc.scm.commit("trash")
|
||||
|
||||
self.dvc.remove(stages[0].relpath, dvc_only=True)
|
||||
self.dvc.remove(stages[0].relpath)
|
||||
with open(fname, "w+") as fobj:
|
||||
fobj.write("master")
|
||||
stages = self.dvc.add(fname)
|
||||
|
||||
@@ -11,6 +11,7 @@ from dvc.ignore import (
|
||||
DvcIgnoreRepo,
|
||||
)
|
||||
from dvc.remote import LocalRemote
|
||||
from dvc.repo import Repo
|
||||
from dvc.scm.tree import WorkingTree
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import get_mtime_and_size
|
||||
@@ -151,3 +152,20 @@ def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory):
|
||||
remote = LocalRemote(dvc, {})
|
||||
result = {relpath(f, ext_dir) for f in remote.walk_files(ext_dir)}
|
||||
assert result == {"y.backup", "tmp"}
|
||||
|
||||
|
||||
def test_ignore_subrepo(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen({".dvcignore": "foo", "subdir": {"foo": "foo"}})
|
||||
scm.add([".dvcignore"])
|
||||
scm.commit("init parent dvcignore")
|
||||
|
||||
subrepo_dir = tmp_dir / "subdir"
|
||||
assert not dvc.tree.exists(subrepo_dir / "foo")
|
||||
|
||||
with subrepo_dir.chdir():
|
||||
subrepo = Repo.init(subdir=True)
|
||||
scm.add(str(subrepo_dir / "foo"))
|
||||
scm.commit("subrepo init")
|
||||
|
||||
for _ in subrepo.brancher(all_commits=True):
|
||||
assert subrepo.tree.exists(subrepo_dir / "foo")
|
||||
|
||||
@@ -6,7 +6,7 @@ import yaml
|
||||
from dvc.dvcfile import PIPELINE_LOCK
|
||||
from dvc.serialize import get_params_deps
|
||||
from dvc.utils.fs import remove
|
||||
from dvc.utils.stage import parse_stage_for_update
|
||||
from dvc.utils.yaml import parse_yaml_for_update
|
||||
from tests.func.test_run_multistage import supported_params
|
||||
|
||||
FS_STRUCTURE = {
|
||||
@@ -20,7 +20,7 @@ FS_STRUCTURE = {
|
||||
|
||||
def read_lock_file(file=PIPELINE_LOCK):
|
||||
with open(file) as f:
|
||||
data = parse_stage_for_update(f.read(), file)
|
||||
data = parse_yaml_for_update(f.read(), file)
|
||||
assert isinstance(data, OrderedDict)
|
||||
return data
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import os
|
||||
import shutil
|
||||
import textwrap
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -29,17 +30,19 @@ def match_files(files, expected_files):
|
||||
|
||||
|
||||
def create_dvc_pipeline(tmp_dir, dvc):
|
||||
script = os.linesep.join(
|
||||
[
|
||||
"from pathlib import Path",
|
||||
"Path({}).touch()".format(os.path.join("out", "file")),
|
||||
]
|
||||
script = textwrap.dedent(
|
||||
"""\
|
||||
import os, sys
|
||||
f = sys.argv[1]
|
||||
os.makedirs(os.path.dirname(f))
|
||||
open(f, "w+").close()
|
||||
"""
|
||||
)
|
||||
tmp_dir.scm_gen({"script.py": script}, commit="init")
|
||||
tmp_dir.dvc_gen({"dep": "content"}, commit="init dvc")
|
||||
dvc.run(
|
||||
**{
|
||||
"command": "python script.py",
|
||||
"cmd": "python script.py {}".format(os.path.join("out", "file")),
|
||||
"outs": [os.path.join("out", "file")],
|
||||
"deps": ["dep"],
|
||||
"fname": "out.dvc",
|
||||
|
||||
@@ -3,7 +3,7 @@ import os
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX
|
||||
from dvc.exceptions import DvcException, MoveNotDataSourceError
|
||||
from dvc.main import main
|
||||
from dvc.utils.stage import load_stage_file
|
||||
from dvc.utils.yaml import load_yaml
|
||||
from tests.basic_env import TestDvc, TestDvcGit
|
||||
from tests.func.test_repro import TestRepro
|
||||
from tests.utils import cd
|
||||
@@ -117,7 +117,7 @@ class TestMoveFileToDirectoryWithoutSpecifiedTargetName(TestDvc):
|
||||
self.assertTrue(os.path.exists(target_foo_path))
|
||||
self.assertTrue(os.path.exists(target_foo_stage_file_path))
|
||||
|
||||
new_stage = load_stage_file(target_foo_stage_file_path)
|
||||
new_stage = load_yaml(target_foo_stage_file_path)
|
||||
self.assertEqual(self.FOO, new_stage["outs"][0]["path"])
|
||||
|
||||
|
||||
@@ -170,7 +170,7 @@ class TestMoveFileBetweenDirectories(TestDvc):
|
||||
self.assertTrue(os.path.exists(new_data_path))
|
||||
self.assertTrue(os.path.exists(new_data_stage_file))
|
||||
|
||||
new_stage_file = load_stage_file(new_data_stage_file)
|
||||
new_stage_file = load_yaml(new_data_stage_file)
|
||||
self.assertEqual(
|
||||
os.path.basename(self.DATA), new_stage_file["outs"][0]["path"]
|
||||
)
|
||||
|
||||
@@ -1,354 +0,0 @@
|
||||
import logging
|
||||
|
||||
from dvc.command.pipeline import CmdPipelineList, CmdPipelineShow
|
||||
from dvc.main import main
|
||||
from tests.basic_env import TestDvc
|
||||
from tests.func.test_repro import TestRepro, TestReproChangedDeepData
|
||||
|
||||
|
||||
class TestPipelineShowSingle(TestDvc):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.stage = "foo.dvc"
|
||||
ret = main(["add", self.FOO])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test(self):
|
||||
ret = main(["pipeline", "show", self.stage])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_commands(self):
|
||||
ret = main(["pipeline", "show", self.stage, "--commands"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_outs(self):
|
||||
ret = main(["pipeline", "show", self.stage, "--outs"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_dot(self):
|
||||
ret = main(["pipeline", "show", "--dot", self.stage])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_tree(self):
|
||||
ret = main(["pipeline", "show", "--tree", self.stage])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_ascii_outs(self):
|
||||
ret = main(["pipeline", "show", "--ascii", self.stage, "--outs"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_dot_commands(self):
|
||||
ret = main(["pipeline", "show", "--dot", self.stage, "--commands"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_dot_outs(self):
|
||||
ret = main(["pipeline", "show", "--dot", self.stage, "--outs"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_not_dvc_file(self):
|
||||
ret = main(["pipeline", "show", self.FOO])
|
||||
self.assertNotEqual(ret, 0)
|
||||
|
||||
def test_non_existing(self):
|
||||
ret = main(["pipeline", "show", "non-existing"])
|
||||
self.assertNotEqual(ret, 0)
|
||||
|
||||
|
||||
def test_single_ascii(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen("foo", "foo content")
|
||||
assert main(["pipeline", "show", "--ascii", "foo.dvc"]) == 0
|
||||
|
||||
|
||||
def test_single_ascii_commands(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen("foo", "foo content")
|
||||
assert main(["pipeline", "show", "--ascii", "foo.dvc", "--commands"]) == 0
|
||||
|
||||
|
||||
class TestPipelineShow(TestRepro):
|
||||
def test(self):
|
||||
ret = main(["pipeline", "show", self.file1_stage])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_commands(self):
|
||||
ret = main(["pipeline", "show", self.file1_stage, "--commands"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_ascii(self):
|
||||
ret = main(["pipeline", "show", "--ascii", self.file1_stage])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_dot(self):
|
||||
ret = main(["pipeline", "show", "--dot", self.file1_stage])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_ascii_commands(self):
|
||||
ret = main(
|
||||
["pipeline", "show", "--ascii", self.file1_stage, "--commands"]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_ascii_outs(self):
|
||||
ret = main(["pipeline", "show", "--ascii", self.file1_stage, "--outs"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_dot_commands(self):
|
||||
ret = main(
|
||||
["pipeline", "show", "--dot", self.file1_stage, "--commands"]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
|
||||
def test_disconnected_stage(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen({"base": "base"})
|
||||
|
||||
dvc.add("base")
|
||||
dvc.run(
|
||||
deps=["base"],
|
||||
outs=["derived1"],
|
||||
cmd="echo derived1 > derived1",
|
||||
single_stage=True,
|
||||
)
|
||||
dvc.run(
|
||||
deps=["base"],
|
||||
outs=["derived2"],
|
||||
cmd="echo derived2 > derived2",
|
||||
single_stage=True,
|
||||
)
|
||||
final_stage = dvc.run(
|
||||
deps=["derived1"],
|
||||
outs=["final"],
|
||||
cmd="echo final > final",
|
||||
single_stage=True,
|
||||
)
|
||||
|
||||
command = CmdPipelineShow([])
|
||||
# Need to test __build_graph directly
|
||||
nodes, edges, is_tree = command._build_graph(
|
||||
final_stage.path, commands=False, outs=True
|
||||
)
|
||||
|
||||
assert set(nodes) == {"final", "derived1", "base"}
|
||||
assert edges == [("final", "derived1"), ("derived1", "base")]
|
||||
assert is_tree is True
|
||||
|
||||
|
||||
def test_print_locked_stages(tmp_dir, dvc, caplog):
|
||||
tmp_dir.dvc_gen({"foo": "foo content", "bar": "bar content"})
|
||||
dvc.lock_stage("foo.dvc")
|
||||
|
||||
caplog.clear()
|
||||
with caplog.at_level(logging.INFO, logger="dvc"):
|
||||
assert main(["pipeline", "show", "foo.dvc", "--locked"]) == 0
|
||||
|
||||
assert "foo.dvc" in caplog.text
|
||||
assert "bar.dvc" not in caplog.text
|
||||
|
||||
|
||||
def test_dot_outs(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo content")
|
||||
run_copy("foo", "file", single_stage=True)
|
||||
assert main(["pipeline", "show", "--dot", "file.dvc", "--outs"]) == 0
|
||||
|
||||
|
||||
class TestPipelineShowOuts(TestRepro):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
|
||||
def test_outs(self):
|
||||
ret = main(["pipeline", "show", self.file1_stage, "--outs"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
|
||||
class TestPipelineShowDeep(TestReproChangedDeepData):
|
||||
def test(self):
|
||||
ret = main(["pipeline", "show", self.file1_stage])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_commands(self):
|
||||
ret = main(["pipeline", "show", self.file1_stage, "--commands"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_outs(self):
|
||||
ret = main(["pipeline", "show", self.file1_stage, "--outs"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_ascii(self):
|
||||
ret = main(["pipeline", "show", "--ascii", self.file1_stage])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_dot(self):
|
||||
ret = main(["pipeline", "show", "--dot", self.file1_stage])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_ascii_commands(self):
|
||||
ret = main(
|
||||
["pipeline", "show", "--ascii", self.file1_stage, "--commands"]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_ascii_outs(self):
|
||||
ret = main(["pipeline", "show", "--ascii", self.file1_stage, "--outs"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_dot_commands(self):
|
||||
ret = main(
|
||||
["pipeline", "show", "--dot", self.file1_stage, "--commands"]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_dot_outs(self):
|
||||
ret = main(["pipeline", "show", "--dot", self.file1_stage, "--outs"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
|
||||
class TestPipelineListEmpty(TestDvc):
|
||||
def test(self):
|
||||
ret = main(["pipeline", "list"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
|
||||
class TestPipelineListSingle(TestPipelineShowDeep):
|
||||
def test(self):
|
||||
ret = main(["pipeline", "list"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
|
||||
class TestDvcRepoPipeline(TestDvc):
|
||||
def test_no_stages(self):
|
||||
pipelines = self.dvc.pipelines
|
||||
self.assertEqual(len(pipelines), 0)
|
||||
|
||||
def one_pipeline(self):
|
||||
self.dvc.add("foo")
|
||||
self.dvc.run(deps=["foo"], outs=["bar"], cmd="", single_stage=True)
|
||||
self.dvc.run(
|
||||
deps=["bar"], outs=["baz"], cmd="echo baz > baz", single_stage=True
|
||||
)
|
||||
pipelines = self.dvc.pipelines
|
||||
|
||||
self.assertEqual(len(pipelines), 1)
|
||||
self.assertEqual(pipelines[0].nodes, 3)
|
||||
self.assertEqual(pipelines[0].edges, 2)
|
||||
|
||||
def two_pipelines(self):
|
||||
self.dvc.add("foo")
|
||||
self.dvc.run(deps=["foo"], outs=["bar"], cmd="", single_stage=True)
|
||||
self.dvc.run(
|
||||
deps=["bar"], outs=["baz"], cmd="echo baz > baz", single_stage=True
|
||||
)
|
||||
|
||||
self.dvc.add("code.py")
|
||||
|
||||
pipelines = self.dvc.pipelines
|
||||
|
||||
self.assertEqual(len(pipelines), 2)
|
||||
self.assertEqual(pipelines[0].nodes, 3)
|
||||
self.assertEqual(pipelines[0].edges, 2)
|
||||
self.assertEqual(pipelines[1].nodes, 1)
|
||||
self.assertEqual(pipelines[1].edges, 0)
|
||||
|
||||
def locked_stage(self):
|
||||
self.dvc.add("foo")
|
||||
self.dvc.lock_stage("foo.dvc")
|
||||
|
||||
pipelines = self.dvc.pipelines
|
||||
self.assertEqual(len(pipelines), 0)
|
||||
|
||||
|
||||
def test_split_pipeline(tmp_dir, scm, dvc):
|
||||
tmp_dir.scm_gen("git_dep1", "git_dep1")
|
||||
tmp_dir.scm_gen("git_dep2", "git_dep2")
|
||||
|
||||
tmp_dir.dvc_gen("data", "source file content")
|
||||
dvc.run(
|
||||
deps=["git_dep1", "data"],
|
||||
outs=["data_train", "data_valid"],
|
||||
cmd="echo train >> data_train && echo valid >> data_valid",
|
||||
single_stage=True,
|
||||
)
|
||||
stage = dvc.run(
|
||||
deps=["git_dep2", "data_train", "data_valid"],
|
||||
outs=["result"],
|
||||
cmd="echo result >> result",
|
||||
single_stage=True,
|
||||
)
|
||||
|
||||
command = CmdPipelineShow([])
|
||||
nodes, edges, _ = command._build_graph(
|
||||
stage.path, commands=False, outs=True
|
||||
)
|
||||
assert set(nodes) == {"data", "data_train", "data_valid", "result"}
|
||||
assert set(edges) == {
|
||||
("result", "data_train"),
|
||||
("result", "data_valid"),
|
||||
("data_train", "data"),
|
||||
("data_valid", "data"),
|
||||
}
|
||||
|
||||
|
||||
def test_pipeline_list_show_multistage(tmp_dir, dvc, run_copy, caplog):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
run_copy("bar", "foobar", single_stage=True)
|
||||
command = CmdPipelineShow([])
|
||||
|
||||
caplog.clear()
|
||||
with caplog.at_level(logging.INFO, "dvc"):
|
||||
command._show("foobar.dvc", False, False, False)
|
||||
output = caplog.text.splitlines()
|
||||
assert "dvc.yaml:copy-foo-bar" in output[0]
|
||||
assert "foobar.dvc" in output[1]
|
||||
|
||||
caplog.clear()
|
||||
with caplog.at_level(logging.INFO, "dvc"):
|
||||
command._show("dvc.yaml:copy-foo-bar", False, False, False)
|
||||
assert "dvc.yaml:copy-foo-bar" in caplog.text
|
||||
assert "foobar.dvc" not in caplog.text
|
||||
|
||||
command = CmdPipelineList([])
|
||||
caplog.clear()
|
||||
with caplog.at_level(logging.INFO, "dvc"):
|
||||
command.run()
|
||||
assert "dvc.yaml:copy-foo-bar" in caplog.text
|
||||
assert "foobar.dvc" in caplog.text
|
||||
assert "1 pipelines in total"
|
||||
|
||||
|
||||
def test_pipeline_ascii_multistage(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
run_copy("bar", "foobar", single_stage=True)
|
||||
command = CmdPipelineShow([])
|
||||
nodes, edges, _ = command._build_graph("foobar.dvc")
|
||||
assert set(nodes) == {"dvc.yaml:copy-foo-bar", "foobar.dvc"}
|
||||
assert set(edges) == {
|
||||
("foobar.dvc", "dvc.yaml:copy-foo-bar"),
|
||||
}
|
||||
|
||||
nodes, *_ = command._build_graph("dvc.yaml:copy-foo-bar")
|
||||
assert set(nodes) == {"dvc.yaml:copy-foo-bar"}
|
||||
|
||||
|
||||
def test_pipeline_multi_outputs_stages(dvc):
|
||||
dvc.run(
|
||||
outs=["alice", "bob"],
|
||||
cmd="echo alice>alice && echo bob>bob",
|
||||
single_stage=True,
|
||||
)
|
||||
dvc.run(
|
||||
deps=["alice"],
|
||||
outs=["mary", "mike"],
|
||||
cmd="echo mary>mary && echo mike>mike",
|
||||
single_stage=True,
|
||||
)
|
||||
stage = dvc.run(
|
||||
deps=["mary"],
|
||||
outs=["carol"],
|
||||
cmd="echo carol>carol",
|
||||
single_stage=True,
|
||||
)
|
||||
|
||||
command = CmdPipelineShow([])
|
||||
nodes, edges, _ = command._build_graph(stage.path, outs=True)
|
||||
assert set(nodes) == {"alice", "mary", "carol"}
|
||||
assert set(edges) == {("carol", "mary"), ("mary", "alice")}
|
||||
@@ -239,7 +239,11 @@ def test_external_dir_resource_on_no_cache(tmp_dir, dvc, tmp_path_factory):
|
||||
|
||||
dvc.cache.local = None
|
||||
with pytest.raises(RemoteCacheRequiredError):
|
||||
dvc.run(deps=[os.fspath(external_dir)], single_stage=True)
|
||||
dvc.run(
|
||||
cmd="echo hello world",
|
||||
deps=[os.fspath(external_dir)],
|
||||
single_stage=True,
|
||||
)
|
||||
|
||||
|
||||
def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, setup_remote):
|
||||
|
||||
+42
-107
@@ -1,129 +1,64 @@
|
||||
import os
|
||||
|
||||
from mock import patch
|
||||
import pytest
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.main import main
|
||||
from dvc.stage import Stage
|
||||
from dvc.stage.exceptions import StageFileDoesNotExistError
|
||||
from dvc.system import System
|
||||
from dvc.utils.fs import remove
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestRemove(TestDvc):
|
||||
def test(self):
|
||||
stages = self.dvc.add(self.FOO)
|
||||
self.assertEqual(len(stages), 1)
|
||||
stage = stages[0]
|
||||
self.assertTrue(stage is not None)
|
||||
(stage_removed,) = self.dvc.remove(stage.path, dvc_only=True)
|
||||
@pytest.mark.parametrize("remove_outs", [True, False])
|
||||
def test_remove(tmp_dir, dvc, run_copy, remove_outs):
|
||||
(stage1,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
stage2 = run_copy("foo", "bar", single_stage=True)
|
||||
stage3 = run_copy("bar", "foobar", name="copy-bar-foobar")
|
||||
|
||||
self.assertIsInstance(stage_removed, Stage)
|
||||
self.assertEqual(stage.path, stage_removed.path)
|
||||
self.assertFalse(os.path.isfile(self.FOO))
|
||||
|
||||
(stage_removed,) = self.dvc.remove(stage.path)
|
||||
self.assertIsInstance(stage_removed, Stage)
|
||||
self.assertEqual(stage.path, stage_removed.path)
|
||||
self.assertFalse(os.path.isfile(self.FOO))
|
||||
self.assertFalse(os.path.exists(stage.path))
|
||||
for stage in [stage1, stage2, stage3]:
|
||||
dvc.remove(stage.addressing, outs=remove_outs)
|
||||
out_exists = (out.exists for out in stage.outs)
|
||||
assert stage not in dvc._collect_stages()
|
||||
if remove_outs:
|
||||
assert not any(out_exists)
|
||||
else:
|
||||
assert all(out_exists)
|
||||
|
||||
|
||||
class TestRemoveNonExistentFile(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(StageFileDoesNotExistError):
|
||||
self.dvc.remove("non_existent_dvc_file")
|
||||
def test_remove_non_existent_file(tmp_dir, dvc):
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
dvc.remove("non_existent_dvc_file.dvc")
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
dvc.remove("non_existent_stage_name")
|
||||
|
||||
|
||||
class TestRemoveBrokenSymlink(TestDvc):
|
||||
def test(self):
|
||||
ret = main(["config", "cache.type", "symlink"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
ret = main(["add", self.FOO])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
remove(self.dvc.cache.local.cache_dir)
|
||||
|
||||
self.assertTrue(System.is_symlink(self.FOO))
|
||||
|
||||
ret = main(["remove", self.FOO + ".dvc"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
self.assertFalse(os.path.lexists(self.FOO))
|
||||
|
||||
|
||||
class TestRemoveDirectory(TestDvc):
|
||||
def test(self):
|
||||
stages = self.dvc.add(self.DATA_DIR)
|
||||
self.assertEqual(len(stages), 1)
|
||||
stage_add = stages[0]
|
||||
self.assertTrue(stage_add is not None)
|
||||
(stage_removed,) = self.dvc.remove(stage_add.path)
|
||||
self.assertEqual(stage_add.path, stage_removed.path)
|
||||
self.assertFalse(os.path.exists(self.DATA_DIR))
|
||||
self.assertFalse(os.path.exists(stage_removed.path))
|
||||
|
||||
|
||||
class TestCmdRemove(TestDvc):
|
||||
def test(self):
|
||||
stages = self.dvc.add(self.FOO)
|
||||
self.assertEqual(len(stages), 1)
|
||||
stage = stages[0]
|
||||
self.assertTrue(stage is not None)
|
||||
ret = main(["remove", stage.path])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
ret = main(["remove", "non-existing-dvc-file"])
|
||||
self.assertNotEqual(ret, 0)
|
||||
|
||||
|
||||
class TestRemovePurge(TestDvc):
|
||||
def test(self):
|
||||
dvcfile = self.dvc.add(self.FOO)[0].path
|
||||
ret = main(["remove", "--purge", "--force", dvcfile])
|
||||
|
||||
self.assertEqual(ret, 0)
|
||||
self.assertFalse(os.path.exists(self.FOO))
|
||||
self.assertFalse(os.path.exists(dvcfile))
|
||||
|
||||
@patch("dvc.prompt.confirm", return_value=False)
|
||||
def test_force(self, mock_prompt):
|
||||
dvcfile = self.dvc.add(self.FOO)[0].path
|
||||
ret = main(["remove", "--purge", dvcfile])
|
||||
|
||||
mock_prompt.assert_called()
|
||||
self.assertEqual(ret, 1)
|
||||
self.assertRaises(DvcException)
|
||||
|
||||
|
||||
def test_remove_pipeline_outs(tmp_dir, dvc, run_copy):
|
||||
from dvc.dvcfile import PIPELINE_FILE
|
||||
|
||||
def test_remove_broken_symlink(tmp_dir, dvc):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
dvc.cache.local.cache_types = ["symlink"]
|
||||
|
||||
assert main(["remove", stage.path]) == 0
|
||||
assert not (tmp_dir / "bar").exists()
|
||||
assert (tmp_dir / PIPELINE_FILE).exists()
|
||||
(stage,) = dvc.add("foo")
|
||||
remove(dvc.cache.local.cache_dir)
|
||||
assert System.is_symlink("foo")
|
||||
|
||||
stage = run_copy("foo", "foobar", name="copy-foo-foobar")
|
||||
with pytest.raises(DvcException):
|
||||
dvc.remove(stage.addressing)
|
||||
assert os.path.lexists("foo")
|
||||
assert (tmp_dir / stage.relpath).exists()
|
||||
|
||||
dvc.remove(stage.addressing, outs=True)
|
||||
assert not os.path.lexists("foo")
|
||||
assert not (tmp_dir / stage.relpath).exists()
|
||||
|
||||
|
||||
def test_cmd_remove(tmp_dir, dvc):
|
||||
assert main(["remove", "non-existing-dvc-file"]) == 1
|
||||
|
||||
(stage,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
assert main(["remove", stage.addressing]) == 0
|
||||
assert not (tmp_dir / "foobar").exists()
|
||||
assert not (tmp_dir / stage.relpath).exists()
|
||||
assert (tmp_dir / "foo").exists()
|
||||
|
||||
stage = run_copy("foo", "baz", name="copy-foo-baz")
|
||||
assert main(["remove", "--purge", "-f", stage.addressing]) == 0
|
||||
assert not (tmp_dir / "baz").exists()
|
||||
assert (tmp_dir / "foo").exists()
|
||||
assert (tmp_dir / PIPELINE_FILE).exists()
|
||||
|
||||
dvc.reproduce(PIPELINE_FILE)
|
||||
assert main(["remove", PIPELINE_FILE]) == 0
|
||||
for file in ["bar", "foobar", "baz"]:
|
||||
assert not (tmp_dir / file).exists()
|
||||
|
||||
assert (tmp_dir / "foo").exists()
|
||||
assert (tmp_dir / PIPELINE_FILE).exists()
|
||||
(stage,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
assert main(["remove", stage.addressing, "--outs"]) == 0
|
||||
assert not (tmp_dir / stage.relpath).exists()
|
||||
assert not (tmp_dir / "foo").exists()
|
||||
|
||||
+251
-2
@@ -1,13 +1,24 @@
|
||||
import os
|
||||
from operator import itemgetter
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.cache import Cache
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
from dvc.exceptions import NoOutputOrStageError
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import Repo
|
||||
from dvc.stage.exceptions import (
|
||||
StageFileDoesNotExistError,
|
||||
StageNameUnspecified,
|
||||
StageNotFound,
|
||||
)
|
||||
from dvc.system import System
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
|
||||
def test_destroy(tmp_dir, dvc, run_copy):
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
|
||||
dvc.config["cache"]["type"] = ["symlink"]
|
||||
dvc.cache = Cache(dvc)
|
||||
|
||||
@@ -88,6 +99,12 @@ def test_collect(tmp_dir, scm, dvc, run_copy):
|
||||
assert collect_outs("dvc.yaml:copy-foo-foobar", recursive=True) == {
|
||||
"foobar"
|
||||
}
|
||||
assert collect_outs("copy-foo-foobar") == {"foobar"}
|
||||
assert collect_outs("copy-foo-foobar", with_deps=True) == {
|
||||
"foobar",
|
||||
"foo",
|
||||
}
|
||||
assert collect_outs("copy-foo-foobar", recursive=True) == {"foobar"}
|
||||
|
||||
run_copy("foobar", "baz", name="copy-foobar-baz")
|
||||
assert collect_outs("dvc.yaml") == {"foobar", "baz"}
|
||||
@@ -98,6 +115,26 @@ def test_collect(tmp_dir, scm, dvc, run_copy):
|
||||
}
|
||||
|
||||
|
||||
def test_collect_dir_recursive(tmp_dir, dvc, run_head):
|
||||
tmp_dir.gen({"dir": {"foo": "foo"}})
|
||||
(stage1,) = dvc.add("dir", recursive=True)
|
||||
with (tmp_dir / "dir").chdir():
|
||||
stage2 = run_head("foo", name="copy-foo-bar")
|
||||
stage3 = run_head("foo-1", single_stage=True)
|
||||
assert set(dvc.collect("dir", recursive=True)) == {stage1, stage2, stage3}
|
||||
|
||||
|
||||
def test_collect_with_not_existing_output_or_stage_name(
|
||||
tmp_dir, dvc, run_copy
|
||||
):
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
dvc.collect("some_file")
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
with pytest.raises(StageNotFound):
|
||||
dvc.collect("some_file")
|
||||
|
||||
|
||||
def test_stages(tmp_dir, dvc):
|
||||
def stages():
|
||||
return {stage.relpath for stage in Repo(os.fspath(tmp_dir)).stages}
|
||||
@@ -113,3 +150,215 @@ def test_stages(tmp_dir, dvc):
|
||||
tmp_dir.gen(".dvcignore", "dir")
|
||||
|
||||
assert stages() == {"file.dvc"}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stages(tmp_dir, run_copy):
|
||||
stage1, stage2 = tmp_dir.dvc_gen({"foo": "foo", "lorem": "lorem"})
|
||||
return {
|
||||
"foo-generate": stage1,
|
||||
"lorem-generate": stage2,
|
||||
"copy-foo-bar": run_copy("foo", "bar", single_stage=True),
|
||||
"copy-bar-foobar": run_copy("bar", "foobar", name="copy-bar-foobar"),
|
||||
"copy-lorem-ipsum": run_copy("lorem", "ipsum", name="lorem-ipsum"),
|
||||
}
|
||||
|
||||
|
||||
def test_collect_granular_with_no_target(tmp_dir, dvc, stages):
|
||||
assert set(map(itemgetter(0), dvc.collect_granular())) == set(
|
||||
stages.values()
|
||||
)
|
||||
assert list(map(itemgetter(1), dvc.collect_granular())) == [None] * len(
|
||||
stages
|
||||
)
|
||||
|
||||
|
||||
def test_collect_granular_with_target(tmp_dir, dvc, stages):
|
||||
assert dvc.collect_granular("bar.dvc") == [(stages["copy-foo-bar"], None)]
|
||||
assert dvc.collect_granular(PIPELINE_FILE) == [
|
||||
(stages["copy-bar-foobar"], None),
|
||||
(stages["copy-lorem-ipsum"], None),
|
||||
]
|
||||
assert dvc.collect_granular(":") == [
|
||||
(stages["copy-bar-foobar"], None),
|
||||
(stages["copy-lorem-ipsum"], None),
|
||||
]
|
||||
assert dvc.collect_granular("copy-bar-foobar") == [
|
||||
(stages["copy-bar-foobar"], None)
|
||||
]
|
||||
assert dvc.collect_granular(":copy-bar-foobar") == [
|
||||
(stages["copy-bar-foobar"], None)
|
||||
]
|
||||
assert dvc.collect_granular("dvc.yaml:copy-bar-foobar") == [
|
||||
(stages["copy-bar-foobar"], None)
|
||||
]
|
||||
|
||||
with (tmp_dir / dvc.DVC_DIR).chdir():
|
||||
assert dvc.collect_granular(
|
||||
relpath(tmp_dir / PIPELINE_FILE) + ":copy-bar-foobar"
|
||||
) == [(stages["copy-bar-foobar"], None)]
|
||||
|
||||
assert dvc.collect_granular("foobar") == [
|
||||
(stages["copy-bar-foobar"], PathInfo(tmp_dir / "foobar"))
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target",
|
||||
[
|
||||
"not_existing.dvc",
|
||||
"not_existing.dvc:stage_name",
|
||||
"not_existing/dvc.yaml",
|
||||
"not_existing/dvc.yaml:stage_name",
|
||||
],
|
||||
)
|
||||
def test_collect_with_not_existing_dvcfile(tmp_dir, dvc, target):
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
dvc.collect_granular(target)
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
dvc.collect(target)
|
||||
|
||||
|
||||
def test_collect_granular_with_not_existing_output_or_stage_name(tmp_dir, dvc):
|
||||
with pytest.raises(NoOutputOrStageError):
|
||||
dvc.collect_granular("some_file")
|
||||
with pytest.raises(NoOutputOrStageError):
|
||||
dvc.collect_granular("some_file", recursive=True)
|
||||
|
||||
|
||||
def test_collect_granular_with_deps(tmp_dir, dvc, stages):
|
||||
assert set(
|
||||
map(itemgetter(0), dvc.collect_granular("bar.dvc", with_deps=True))
|
||||
) == {stages["copy-foo-bar"], stages["foo-generate"]}
|
||||
assert set(
|
||||
map(
|
||||
itemgetter(0),
|
||||
dvc.collect_granular("copy-bar-foobar", with_deps=True),
|
||||
)
|
||||
) == {
|
||||
stages["copy-bar-foobar"],
|
||||
stages["copy-foo-bar"],
|
||||
stages["foo-generate"],
|
||||
}
|
||||
assert set(
|
||||
map(
|
||||
itemgetter(0), dvc.collect_granular(PIPELINE_FILE, with_deps=True),
|
||||
)
|
||||
) == set(stages.values())
|
||||
|
||||
|
||||
def test_collect_granular_same_output_name_stage_name(tmp_dir, dvc, run_copy):
|
||||
(stage1,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
(stage2,) = tmp_dir.dvc_gen("copy-foo-bar", "copy-foo-bar")
|
||||
stage3 = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
|
||||
assert dvc.collect_granular("copy-foo-bar") == [(stage3, None)]
|
||||
|
||||
coll = dvc.collect_granular("copy-foo-bar", with_deps=True)
|
||||
assert set(map(itemgetter(0), coll)) == {stage3, stage1}
|
||||
assert list(map(itemgetter(1), coll)) == [None] * 2
|
||||
|
||||
assert dvc.collect_granular("./copy-foo-bar") == [
|
||||
(stage2, PathInfo(tmp_dir / "copy-foo-bar"))
|
||||
]
|
||||
assert dvc.collect_granular("./copy-foo-bar", with_deps=True) == [
|
||||
(stage2, PathInfo(tmp_dir / "copy-foo-bar"))
|
||||
]
|
||||
|
||||
|
||||
def test_collect_granular_priority_on_collision(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen({"dir": {"foo": "foo"}, "foo": "foo"})
|
||||
(stage1,) = dvc.add("dir", recursive=True)
|
||||
stage2 = run_copy("foo", "bar", name="dir")
|
||||
|
||||
assert dvc.collect_granular("dir") == [(stage2, None)]
|
||||
assert dvc.collect_granular("dir", recursive=True) == [(stage1, None)]
|
||||
|
||||
remove(tmp_dir / "dir")
|
||||
|
||||
assert dvc.collect_granular("dir") == [(stage2, None)]
|
||||
assert dvc.collect_granular("dir", recursive=True) == [(stage2, None)]
|
||||
|
||||
|
||||
def test_collect_granular_collision_output_dir_stage_name(
|
||||
tmp_dir, dvc, run_copy
|
||||
):
|
||||
stage1, *_ = tmp_dir.dvc_gen({"dir": {"foo": "foo"}, "foo": "foo"})
|
||||
stage3 = run_copy("foo", "bar", name="dir")
|
||||
|
||||
assert dvc.collect_granular("dir") == [(stage3, None)]
|
||||
assert not dvc.collect_granular("dir", recursive=True)
|
||||
assert dvc.collect_granular("./dir") == [
|
||||
(stage1, PathInfo(tmp_dir / "dir"))
|
||||
]
|
||||
|
||||
|
||||
def test_collect_granular_not_existing_stage_name(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
(stage,) = tmp_dir.dvc_gen("copy-foo-bar", "copy-foo-bar")
|
||||
run_copy("foo", "bar", name="copy-foo-bar")
|
||||
|
||||
assert dvc.collect_granular("copy-foo-bar.dvc:stage_name_not_needed") == [
|
||||
(stage, None)
|
||||
]
|
||||
with pytest.raises(StageNotFound):
|
||||
dvc.collect_granular("dvc.yaml:does-not-exist")
|
||||
|
||||
|
||||
def test_get_stages(tmp_dir, dvc, run_copy):
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
dvc.get_stages()
|
||||
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage1 = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
stage2 = run_copy("bar", "foobar", name="copy-bar-foobar")
|
||||
|
||||
assert set(dvc.get_stages()) == {stage1, stage2}
|
||||
assert set(dvc.get_stages(path=PIPELINE_FILE)) == {stage1, stage2}
|
||||
assert set(dvc.get_stages(name="copy-bar-foobar")) == {stage2}
|
||||
assert set(dvc.get_stages(path=PIPELINE_FILE, name="copy-bar-foobar")) == {
|
||||
stage2
|
||||
}
|
||||
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
dvc.get_stages(path=relpath(tmp_dir / ".." / PIPELINE_FILE))
|
||||
|
||||
with pytest.raises(StageNotFound):
|
||||
dvc.get_stages(path=PIPELINE_FILE, name="copy")
|
||||
|
||||
|
||||
def test_get_stages_old_dvcfile(tmp_dir, dvc):
|
||||
(stage1,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
assert set(dvc.get_stages("foo.dvc")) == {stage1}
|
||||
assert set(dvc.get_stages("foo.dvc", name="foo-generate")) == {stage1}
|
||||
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
dvc.get_stages(path=relpath(tmp_dir / ".." / "foo.dvc"))
|
||||
|
||||
|
||||
def test_get_stage(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage1 = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
|
||||
with pytest.raises(StageNameUnspecified):
|
||||
dvc.get_stage()
|
||||
|
||||
with pytest.raises(StageNameUnspecified):
|
||||
dvc.get_stage(path=PIPELINE_FILE)
|
||||
|
||||
assert dvc.get_stage(path=PIPELINE_FILE, name="copy-foo-bar") == stage1
|
||||
assert dvc.get_stage(name="copy-foo-bar") == stage1
|
||||
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
dvc.get_stage(path="something.yaml", name="name")
|
||||
|
||||
with pytest.raises(StageNotFound):
|
||||
dvc.get_stage(name="random_name")
|
||||
|
||||
|
||||
def test_get_stage_single_stage_dvcfile(tmp_dir, dvc):
|
||||
(stage1,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
assert dvc.get_stage("foo.dvc") == stage1
|
||||
assert dvc.get_stage("foo.dvc", name="jpt") == stage1
|
||||
with pytest.raises(StageFileDoesNotExistError):
|
||||
dvc.get_stage(path="bar.dvc", name="name")
|
||||
|
||||
+85
-38
@@ -33,7 +33,7 @@ from dvc.stage.exceptions import StageFileDoesNotExistError
|
||||
from dvc.system import System
|
||||
from dvc.utils import file_md5, relpath
|
||||
from dvc.utils.fs import remove
|
||||
from dvc.utils.stage import dump_stage_file, load_stage_file
|
||||
from dvc.utils.yaml import dump_yaml, load_yaml
|
||||
from tests.basic_env import TestDvc
|
||||
from tests.remotes import (
|
||||
GCP,
|
||||
@@ -56,7 +56,7 @@ class SingleStageRun:
|
||||
|
||||
@staticmethod
|
||||
def _get_stage_target(stage):
|
||||
return stage.path
|
||||
return stage.addressing
|
||||
|
||||
|
||||
class TestRepro(SingleStageRun, TestDvc):
|
||||
@@ -108,7 +108,7 @@ class TestReproCyclicGraph(SingleStageRun, TestDvc):
|
||||
"deps": [{"path": "baz.txt"}],
|
||||
"outs": [{"path": self.FOO}],
|
||||
}
|
||||
dump_stage_file("cycle.dvc", stage_dump)
|
||||
dump_yaml("cycle.dvc", stage_dump)
|
||||
|
||||
with self.assertRaises(CyclicGraphError):
|
||||
self.dvc.reproduce("cycle.dvc")
|
||||
@@ -149,7 +149,7 @@ class TestReproWorkingDirectoryAsOutput(TestDvc):
|
||||
"cmd": f"echo something > {output}",
|
||||
"outs": [{"path": output}],
|
||||
}
|
||||
dump_stage_file(faulty_stage_path, stage_dump)
|
||||
dump_yaml(faulty_stage_path, stage_dump)
|
||||
|
||||
with self.assertRaises(StagePathAsOutputError):
|
||||
self.dvc.reproduce(faulty_stage_path)
|
||||
@@ -188,7 +188,7 @@ class TestReproWorkingDirectoryAsOutput(TestDvc):
|
||||
"cmd": f"echo something > {output}",
|
||||
"outs": [{"path": output}],
|
||||
}
|
||||
dump_stage_file(error_stage_path, stage_dump)
|
||||
dump_yaml(error_stage_path, stage_dump)
|
||||
|
||||
# NOTE: os.walk() walks in a sorted order and we need dir2 subdirs to
|
||||
# be processed before dir1 to load error.dvc first.
|
||||
@@ -220,7 +220,7 @@ class TestReproWorkingDirectoryAsOutput(TestDvc):
|
||||
stage = os.path.join("something-1", "a.dvc")
|
||||
|
||||
stage_dump = {"cmd": "echo a > a", "outs": [{"path": "a"}]}
|
||||
dump_stage_file(stage, stage_dump)
|
||||
dump_yaml(stage, stage_dump)
|
||||
|
||||
try:
|
||||
self.dvc.reproduce(stage)
|
||||
@@ -264,9 +264,11 @@ class TestReproDepDirWithOutputsUnderIt(SingleStageRun, TestDvc):
|
||||
self.assertEqual(len(stages), 1)
|
||||
self.assertTrue(stages[0] is not None)
|
||||
|
||||
deps = [self.DATA, self.DATA_SUB]
|
||||
stage = self.dvc.run(
|
||||
cmd="ls {}".format(" ".join(deps)),
|
||||
fname="dvcfile2.dvc",
|
||||
deps=[self.DATA, self.DATA_SUB],
|
||||
deps=deps,
|
||||
single_stage=True,
|
||||
)
|
||||
self.assertTrue(stage is not None)
|
||||
@@ -405,7 +407,17 @@ class TestReproDryNoExec(TestDvc):
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
ret = main(
|
||||
["run", "--no-exec", "--single-stage", "-f", DVC_FILE] + deps
|
||||
[
|
||||
"run",
|
||||
"--no-exec",
|
||||
"--single-stage",
|
||||
"-f",
|
||||
DVC_FILE,
|
||||
*deps,
|
||||
"ls {}".format(
|
||||
" ".join(dep for i, dep in enumerate(deps) if i % 2)
|
||||
),
|
||||
]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
@@ -542,7 +554,7 @@ class TestReproPipelines(SingleStageRun, TestDvc):
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
|
||||
class TestReproLocked(TestReproChangedData):
|
||||
class TestReproFrozen(TestReproChangedData):
|
||||
def test(self):
|
||||
file2 = "file2"
|
||||
file2_stage = self._run(
|
||||
@@ -555,12 +567,12 @@ class TestReproLocked(TestReproChangedData):
|
||||
|
||||
self.swap_foo_with_bar()
|
||||
|
||||
ret = main(["lock", self._get_stage_target(file2_stage)])
|
||||
ret = main(["freeze", self._get_stage_target(file2_stage)])
|
||||
self.assertEqual(ret, 0)
|
||||
stages = self.dvc.reproduce(self._get_stage_target(file2_stage))
|
||||
self.assertEqual(len(stages), 0)
|
||||
|
||||
ret = main(["unlock", self._get_stage_target(file2_stage)])
|
||||
ret = main(["unfreeze", self._get_stage_target(file2_stage)])
|
||||
self.assertEqual(ret, 0)
|
||||
stages = self.dvc.reproduce(self._get_stage_target(file2_stage))
|
||||
self.assertTrue(filecmp.cmp(self.file1, self.BAR, shallow=False))
|
||||
@@ -569,19 +581,19 @@ class TestReproLocked(TestReproChangedData):
|
||||
|
||||
def test_non_existing(self):
|
||||
with self.assertRaises(StageFileDoesNotExistError):
|
||||
self.dvc.lock_stage("Dvcfile")
|
||||
self.dvc.lock_stage("pipelines.yaml")
|
||||
self.dvc.lock_stage("pipelines.yaml:name")
|
||||
self.dvc.lock_stage("Dvcfile:name")
|
||||
self.dvc.lock_stage("stage.dvc")
|
||||
self.dvc.lock_stage("stage.dvc:name")
|
||||
self.dvc.lock_stage("not-existing-stage.json")
|
||||
self.dvc.freeze("Dvcfile")
|
||||
self.dvc.freeze("pipelines.yaml")
|
||||
self.dvc.freeze("pipelines.yaml:name")
|
||||
self.dvc.freeze("Dvcfile:name")
|
||||
self.dvc.freeze("stage.dvc")
|
||||
self.dvc.freeze("stage.dvc:name")
|
||||
self.dvc.freeze("not-existing-stage.json")
|
||||
|
||||
ret = main(["lock", "non-existing-stage"])
|
||||
ret = main(["freeze", "non-existing-stage"])
|
||||
self.assertNotEqual(ret, 0)
|
||||
|
||||
|
||||
class TestReproLockedCallback(SingleStageRun, TestDvc):
|
||||
class TestReproFrozenCallback(SingleStageRun, TestDvc):
|
||||
def test(self):
|
||||
file1 = "file1"
|
||||
file1_stage = file1 + ".dvc"
|
||||
@@ -598,26 +610,26 @@ class TestReproLockedCallback(SingleStageRun, TestDvc):
|
||||
stages = self.dvc.reproduce(self._get_stage_target(stage))
|
||||
self.assertEqual(len(stages), 1)
|
||||
|
||||
self.dvc.lock_stage(self._get_stage_target(stage))
|
||||
self.dvc.freeze(self._get_stage_target(stage))
|
||||
stages = self.dvc.reproduce(self._get_stage_target(stage))
|
||||
self.assertEqual(len(stages), 0)
|
||||
|
||||
self.dvc.lock_stage(self._get_stage_target(stage), unlock=True)
|
||||
self.dvc.unfreeze(self._get_stage_target(stage))
|
||||
stages = self.dvc.reproduce(self._get_stage_target(stage))
|
||||
self.assertEqual(len(stages), 1)
|
||||
|
||||
|
||||
class TestReproLockedUnchanged(TestRepro):
|
||||
class TestReproFrozenUnchanged(TestRepro):
|
||||
def test(self):
|
||||
"""
|
||||
Check that locking/unlocking doesn't affect stage state
|
||||
Check that freezing/unfreezing doesn't affect stage state
|
||||
"""
|
||||
target = self._get_stage_target(self.stage)
|
||||
self.dvc.lock_stage(target)
|
||||
self.dvc.freeze(target)
|
||||
stages = self.dvc.reproduce(target)
|
||||
self.assertEqual(len(stages), 0)
|
||||
|
||||
self.dvc.lock_stage(target, unlock=True)
|
||||
self.dvc.unfreeze(target)
|
||||
stages = self.dvc.reproduce(target)
|
||||
self.assertEqual(len(stages), 0)
|
||||
|
||||
@@ -644,11 +656,17 @@ class TestReproMetricsAddUnchanged(TestDvc):
|
||||
stages = self.dvc.reproduce(file1_stage)
|
||||
self.assertEqual(len(stages), 0)
|
||||
|
||||
self.dvc.metrics.add(file1)
|
||||
d = load_yaml(file1_stage)
|
||||
d["outs"][0]["metric"] = True
|
||||
dump_yaml(file1_stage, d)
|
||||
|
||||
stages = self.dvc.reproduce(file1_stage)
|
||||
self.assertEqual(len(stages), 0)
|
||||
|
||||
self.dvc.metrics.remove(file1)
|
||||
d = load_yaml(file1_stage)
|
||||
d["outs"][0]["metric"] = False
|
||||
dump_yaml(file1_stage, d)
|
||||
|
||||
stages = self.dvc.reproduce(file1_stage)
|
||||
self.assertEqual(len(stages), 0)
|
||||
|
||||
@@ -763,10 +781,10 @@ class TestReproChangedDirData(SingleStageRun, TestDvc):
|
||||
|
||||
class TestReproMissingMd5InStageFile(TestRepro):
|
||||
def test(self):
|
||||
d = load_stage_file(self.file1_stage)
|
||||
d = load_yaml(self.file1_stage)
|
||||
del d[Stage.PARAM_OUTS][0][LocalRemote.PARAM_CHECKSUM]
|
||||
del d[Stage.PARAM_DEPS][0][LocalRemote.PARAM_CHECKSUM]
|
||||
dump_stage_file(self.file1_stage, d)
|
||||
dump_yaml(self.file1_stage, d)
|
||||
|
||||
stages = self.dvc.reproduce(self.file1_stage)
|
||||
self.assertEqual(len(stages), 1)
|
||||
@@ -867,9 +885,9 @@ class TestReproExternalBase(SingleStageRun, TestDvc):
|
||||
with patch_download as mock_download:
|
||||
with patch_checkout as mock_checkout:
|
||||
with patch_run as mock_run:
|
||||
stage.locked = False
|
||||
stage.frozen = False
|
||||
stage.run()
|
||||
stage.locked = True
|
||||
stage.frozen = True
|
||||
|
||||
mock_run.assert_not_called()
|
||||
mock_download.assert_not_called()
|
||||
@@ -971,7 +989,8 @@ class TestReproExternalBase(SingleStageRun, TestDvc):
|
||||
self.dvc.gc(workspace=True)
|
||||
self.assertEqual(self.dvc.status(), {})
|
||||
|
||||
self.dvc.remove(cmd_stage.path, dvc_only=True)
|
||||
with self.dvc.lock:
|
||||
cmd_stage.remove_outs(force=True)
|
||||
self.assertNotEqual(self.dvc.status([cmd_stage.addressing]), {})
|
||||
|
||||
self.dvc.checkout([cmd_stage.path], force=True)
|
||||
@@ -1191,12 +1210,13 @@ class TestReproExternalHTTP(TestReproExternalBase):
|
||||
self.assertTrue(os.path.exists(import_output))
|
||||
self.assertTrue(filecmp.cmp(import_output, self.FOO, shallow=False))
|
||||
|
||||
self.dvc.remove("imported_file.dvc")
|
||||
self.dvc.remove("imported_file.dvc", outs=True)
|
||||
|
||||
with StaticFileServer(handler_class=ContentMD5Handler) as httpd:
|
||||
import_url = urljoin(self.get_remote(httpd.server_port), self.FOO)
|
||||
import_output = "imported_file"
|
||||
import_stage = self.dvc.imp_url(import_url, import_output)
|
||||
assert import_stage.repo == self.dvc
|
||||
|
||||
self.assertTrue(os.path.exists(import_output))
|
||||
self.assertTrue(filecmp.cmp(import_output, self.FOO, shallow=False))
|
||||
@@ -1213,7 +1233,7 @@ class TestReproExternalHTTP(TestReproExternalBase):
|
||||
self.assertEqual(ret1, 0)
|
||||
self.assertEqual(ret2, 0)
|
||||
|
||||
self.dvc = DvcRepo(".")
|
||||
self.dvc = import_stage.repo = DvcRepo(".")
|
||||
|
||||
run_dependency = urljoin(remote, self.BAR)
|
||||
run_output = "remote_file"
|
||||
@@ -1233,7 +1253,10 @@ class TestReproExternalHTTP(TestReproExternalBase):
|
||||
self.assertTrue(os.path.exists(run_output))
|
||||
|
||||
# Pull
|
||||
self.dvc.remove(import_stage.path, dvc_only=True)
|
||||
with self.dvc.lock:
|
||||
self.assertEqual(import_stage.repo.lock.is_locked, True)
|
||||
self.assertEqual(self.dvc.lock.is_locked, True)
|
||||
import_stage.remove_outs(force=True)
|
||||
self.assertFalse(os.path.exists(import_output))
|
||||
|
||||
shutil.move(self.local_cache, cache_id)
|
||||
@@ -1375,7 +1398,7 @@ class TestReproAlreadyCached(TestRepro):
|
||||
|
||||
with patch_download as mock_download:
|
||||
with patch_checkout as mock_checkout:
|
||||
assert main(["unlock", "bar.dvc"]) == 0
|
||||
assert main(["unfreeze", "bar.dvc"]) == 0
|
||||
ret = main(["repro", "--force", "bar.dvc"])
|
||||
self.assertEqual(ret, 0)
|
||||
self.assertEqual(mock_download.call_count, 1)
|
||||
@@ -1465,9 +1488,11 @@ def repro_dir(tmp_dir, dvc, run_copy):
|
||||
stages["dir_file_copy"] = stage
|
||||
|
||||
last_stage = tmp_dir / "dir" / DVC_FILE
|
||||
deps = [os.fspath(origin_copy_2), os.fspath(dir_file_copy)]
|
||||
stage = dvc.run(
|
||||
cmd="echo {}".format(" ".join(deps)),
|
||||
fname=os.fspath(last_stage),
|
||||
deps=[os.fspath(origin_copy_2), os.fspath(dir_file_copy)],
|
||||
deps=deps,
|
||||
single_stage=True,
|
||||
)
|
||||
assert stage is not None
|
||||
@@ -1751,3 +1776,25 @@ def test_ssh_dir_out(tmp_dir, dvc, ssh_server):
|
||||
|
||||
repo.reproduce("dir-out.dvc")
|
||||
repo.reproduce("dir-out.dvc", force=True)
|
||||
|
||||
|
||||
def test_repro_when_cmd_changes(tmp_dir, dvc, run_copy, mocker):
|
||||
from dvc.dvcfile import SingleStageFile
|
||||
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage = run_copy("foo", "bar", single_stage=True)
|
||||
assert not dvc.reproduce(stage.addressing)
|
||||
|
||||
from dvc.stage.run import cmd_run
|
||||
|
||||
m = mocker.patch("dvc.stage.run.cmd_run", wraps=cmd_run)
|
||||
|
||||
data = SingleStageFile(dvc, stage.path)._load()[0]
|
||||
data["cmd"] = " ".join(stage.cmd.split()) # change cmd spacing by two
|
||||
dump_yaml(stage.path, data)
|
||||
|
||||
assert dvc.status([stage.addressing]) == {
|
||||
stage.addressing: ["changed checksum"]
|
||||
}
|
||||
assert dvc.reproduce(stage.addressing)[0] == stage
|
||||
m.assert_called_once_with(stage)
|
||||
|
||||
@@ -10,7 +10,7 @@ from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
from dvc.exceptions import CyclicGraphError
|
||||
from dvc.main import main
|
||||
from dvc.stage import PipelineStage
|
||||
from dvc.utils.stage import dump_stage_file, parse_stage
|
||||
from dvc.utils.yaml import dump_yaml, parse_yaml
|
||||
from tests.func import test_repro
|
||||
|
||||
COPY_SCRIPT_FORMAT = dedent(
|
||||
@@ -32,7 +32,7 @@ class MultiStageRun:
|
||||
|
||||
@staticmethod
|
||||
def _get_stage_target(stage):
|
||||
return stage.path + ":" + stage.name
|
||||
return stage.addressing
|
||||
|
||||
|
||||
class TestReproFailMultiStage(MultiStageRun, test_repro.TestReproFail):
|
||||
@@ -101,18 +101,18 @@ class TestReproPipelinesMultiStage(
|
||||
pass
|
||||
|
||||
|
||||
class TestReproLockedMultiStage(MultiStageRun, test_repro.TestReproLocked):
|
||||
class TestReproFrozenMultiStage(MultiStageRun, test_repro.TestReproFrozen):
|
||||
pass
|
||||
|
||||
|
||||
class TestReproLockedCallbackMultiStage(
|
||||
MultiStageRun, test_repro.TestReproLockedCallback
|
||||
class TestReproFrozenCallbackMultiStage(
|
||||
MultiStageRun, test_repro.TestReproFrozenCallback
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
class TestReproLockedUnchangedMultiStage(
|
||||
MultiStageRun, test_repro.TestReproLockedUnchanged
|
||||
class TestReproFrozenUnchangedMultiStage(
|
||||
MultiStageRun, test_repro.TestReproFrozenUnchanged
|
||||
):
|
||||
pass
|
||||
|
||||
@@ -202,9 +202,9 @@ def test_non_existing_stage_name(tmp_dir, dvc, run_copy):
|
||||
run_copy("file1", "file2", name="copy-file1-file2")
|
||||
|
||||
with pytest.raises(DvcException):
|
||||
dvc.lock_stage(":copy-file1-file3")
|
||||
dvc.freeze(":copy-file1-file3")
|
||||
|
||||
assert main(["lock", ":copy-file1-file3"]) != 0
|
||||
assert main(["freeze", ":copy-file1-file3"]) != 0
|
||||
|
||||
|
||||
def test_downstream(tmp_dir, dvc):
|
||||
@@ -307,21 +307,23 @@ def test_downstream(tmp_dir, dvc):
|
||||
)
|
||||
|
||||
|
||||
def test_repro_when_cmd_changes(tmp_dir, dvc, run_copy):
|
||||
def test_repro_when_cmd_changes(tmp_dir, dvc, run_copy, mocker):
|
||||
from dvc.dvcfile import PipelineFile
|
||||
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage = run_copy("foo", "bar", name="copy-file")
|
||||
target = ":copy-file"
|
||||
target = "copy-file"
|
||||
assert not dvc.reproduce(target)
|
||||
|
||||
from dvc.stage.run import cmd_run
|
||||
|
||||
m = mocker.patch("dvc.stage.run.cmd_run", wraps=cmd_run)
|
||||
stage.cmd = " ".join(stage.cmd.split()) # change cmd spacing by two
|
||||
PipelineFile(dvc, PIPELINE_FILE)._dump_pipeline_file(stage)
|
||||
|
||||
assert dvc.status([target]) == {
|
||||
PIPELINE_FILE + target: ["changed command"]
|
||||
}
|
||||
assert dvc.status([target]) == {target: ["changed command"]}
|
||||
assert dvc.reproduce(target)[0] == stage
|
||||
m.assert_called_once_with(stage)
|
||||
|
||||
|
||||
def test_repro_when_new_deps_is_added_in_dvcfile(tmp_dir, dvc, run_copy):
|
||||
@@ -341,7 +343,7 @@ def test_repro_when_new_deps_is_added_in_dvcfile(tmp_dir, dvc, run_copy):
|
||||
dvcfile = Dvcfile(dvc, stage.path)
|
||||
data, _ = dvcfile._load()
|
||||
data["stages"]["copy-file"]["deps"] += ["copy.py"]
|
||||
dump_stage_file(stage.path, data)
|
||||
dump_yaml(stage.path, data)
|
||||
|
||||
assert dvc.reproduce(target)[0] == stage
|
||||
|
||||
@@ -363,7 +365,7 @@ def test_repro_when_new_outs_is_added_in_dvcfile(tmp_dir, dvc):
|
||||
dvcfile = Dvcfile(dvc, stage.path)
|
||||
data, _ = dvcfile._load()
|
||||
data["stages"]["copy-file"]["outs"] = ["foobar"]
|
||||
dump_stage_file(stage.path, data)
|
||||
dump_yaml(stage.path, data)
|
||||
|
||||
assert dvc.reproduce(target)[0] == stage
|
||||
|
||||
@@ -390,7 +392,7 @@ def test_repro_when_new_deps_is_moved(tmp_dir, dvc):
|
||||
dvcfile = Dvcfile(dvc, stage.path)
|
||||
data, _ = dvcfile._load()
|
||||
data["stages"]["copy-file"]["deps"] = ["bar"]
|
||||
dump_stage_file(stage.path, data)
|
||||
dump_yaml(stage.path, data)
|
||||
|
||||
assert dvc.reproduce(target)[0] == stage
|
||||
|
||||
@@ -400,7 +402,7 @@ def test_repro_when_new_out_overlaps_others_stage_outs(tmp_dir, dvc):
|
||||
|
||||
tmp_dir.gen({"dir": {"file1": "file1"}, "foo": "foo"})
|
||||
dvc.add("dir")
|
||||
dump_stage_file(
|
||||
dump_yaml(
|
||||
PIPELINE_FILE,
|
||||
{
|
||||
"stages": {
|
||||
@@ -421,7 +423,7 @@ def test_repro_when_new_deps_added_does_not_exist(tmp_dir, dvc):
|
||||
|
||||
tmp_dir.gen("copy.py", COPY_SCRIPT)
|
||||
tmp_dir.gen("foo", "foo")
|
||||
dump_stage_file(
|
||||
dump_yaml(
|
||||
PIPELINE_FILE,
|
||||
{
|
||||
"stages": {
|
||||
@@ -442,7 +444,7 @@ def test_repro_when_new_outs_added_does_not_exist(tmp_dir, dvc):
|
||||
|
||||
tmp_dir.gen("copy.py", COPY_SCRIPT)
|
||||
tmp_dir.gen("foo", "foo")
|
||||
dump_stage_file(
|
||||
dump_yaml(
|
||||
PIPELINE_FILE,
|
||||
{
|
||||
"stages": {
|
||||
@@ -461,7 +463,7 @@ def test_repro_when_new_outs_added_does_not_exist(tmp_dir, dvc):
|
||||
def test_repro_when_lockfile_gets_deleted(tmp_dir, dvc):
|
||||
tmp_dir.gen("copy.py", COPY_SCRIPT)
|
||||
tmp_dir.gen("foo", "foo")
|
||||
dump_stage_file(
|
||||
dump_yaml(
|
||||
PIPELINE_FILE,
|
||||
{
|
||||
"stages": {
|
||||
@@ -493,13 +495,13 @@ def test_cyclic_graph_error(tmp_dir, dvc, run_copy):
|
||||
run_copy("baz", "foobar", name="copy-baz-foobar")
|
||||
|
||||
with open(PIPELINE_FILE) as f:
|
||||
data = parse_stage(f.read(), PIPELINE_FILE)
|
||||
data = parse_yaml(f.read(), PIPELINE_FILE)
|
||||
data["stages"]["copy-baz-foo"] = {
|
||||
"cmd": "echo baz > foo",
|
||||
"deps": ["baz"],
|
||||
"outs": ["foo"],
|
||||
}
|
||||
dump_stage_file(PIPELINE_FILE, data)
|
||||
dump_yaml(PIPELINE_FILE, data)
|
||||
with pytest.raises(CyclicGraphError):
|
||||
dvc.reproduce(":copy-baz-foo")
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ import os
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
from dvc.repo import Repo
|
||||
from dvc.stage.exceptions import DuplicateStageName, InvalidStageName
|
||||
|
||||
|
||||
@@ -216,7 +218,8 @@ def test_run_already_exists(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
run_copy("foo", "bar", name="copy")
|
||||
with pytest.raises(DuplicateStageName):
|
||||
run_copy("bar", "foobar", name="copy")
|
||||
run_copy("bar", "foobar", name="copy", overwrite=False)
|
||||
run_copy("bar", "foobar", name="copy", overwrite=True)
|
||||
|
||||
|
||||
supported_params = {
|
||||
@@ -299,3 +302,16 @@ def test_run_params_no_exec(tmp_dir, dvc):
|
||||
assert data["stages"]["read_params"]["params"] == [
|
||||
{"params2.yaml": ["lists"]}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kwargs",
|
||||
[
|
||||
{"outs": ["foo"], "deps": ["bar"]},
|
||||
{"outs": ["foo"], "deps": ["bar"], "name": "copy-foo-bar"},
|
||||
],
|
||||
)
|
||||
def test_run_without_cmd(kwargs):
|
||||
with pytest.raises(InvalidArgumentError) as exc:
|
||||
Repo().run(**kwargs)
|
||||
assert "command is not specified" == str(exc.value)
|
||||
|
||||
@@ -31,7 +31,7 @@ from dvc.stage.exceptions import (
|
||||
)
|
||||
from dvc.system import System
|
||||
from dvc.utils import file_md5
|
||||
from dvc.utils.stage import load_stage_file
|
||||
from dvc.utils.yaml import load_yaml
|
||||
from tests.basic_env import TestDvc, TestDvcGit
|
||||
|
||||
|
||||
@@ -76,7 +76,7 @@ class TestRun(TestDvc):
|
||||
class TestRunEmpty(TestDvc):
|
||||
def test(self):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="echo hello world",
|
||||
deps=[],
|
||||
outs=[],
|
||||
outs_no_cache=[],
|
||||
@@ -91,7 +91,7 @@ class TestRunMissingDep(TestDvc):
|
||||
|
||||
with self.assertRaises(DependencyDoesNotExistError):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
deps=["non-existing-dep"],
|
||||
outs=[],
|
||||
outs_no_cache=[],
|
||||
@@ -118,7 +118,7 @@ class TestRunCircularDependency(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(CircularDependencyError):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
deps=[self.FOO],
|
||||
outs=[self.FOO],
|
||||
fname="circular-dependency.dvc",
|
||||
@@ -128,7 +128,7 @@ class TestRunCircularDependency(TestDvc):
|
||||
def test_outs_no_cache(self):
|
||||
with self.assertRaises(CircularDependencyError):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
deps=[self.FOO],
|
||||
outs_no_cache=[self.FOO],
|
||||
fname="circular-dependency.dvc",
|
||||
@@ -138,7 +138,7 @@ class TestRunCircularDependency(TestDvc):
|
||||
def test_non_normalized_paths(self):
|
||||
with self.assertRaises(CircularDependencyError):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
deps=["./foo"],
|
||||
outs=["foo"],
|
||||
fname="circular-dependency.dvc",
|
||||
@@ -173,7 +173,7 @@ class TestRunDuplicatedArguments(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(ArgumentDuplicationError):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
deps=[],
|
||||
outs=[self.FOO, self.FOO],
|
||||
fname="circular-dependency.dvc",
|
||||
@@ -183,7 +183,7 @@ class TestRunDuplicatedArguments(TestDvc):
|
||||
def test_outs_no_cache(self):
|
||||
with self.assertRaises(ArgumentDuplicationError):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
outs=[self.FOO],
|
||||
outs_no_cache=[self.FOO],
|
||||
fname="circular-dependency.dvc",
|
||||
@@ -193,7 +193,7 @@ class TestRunDuplicatedArguments(TestDvc):
|
||||
def test_non_normalized_paths(self):
|
||||
with self.assertRaises(ArgumentDuplicationError):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
deps=[],
|
||||
outs=["foo", "./foo"],
|
||||
fname="circular-dependency.dvc",
|
||||
@@ -204,22 +204,30 @@ class TestRunDuplicatedArguments(TestDvc):
|
||||
class TestRunStageInsideOutput(TestDvc):
|
||||
def test_cwd(self):
|
||||
self.dvc.run(
|
||||
cmd="", deps=[], outs=[self.DATA_DIR], single_stage=True,
|
||||
cmd=f"mkdir {self.DATA_DIR}",
|
||||
deps=[],
|
||||
outs=[self.DATA_DIR],
|
||||
single_stage=True,
|
||||
)
|
||||
|
||||
with self.assertRaises(StagePathAsOutputError):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
fname=os.path.join(self.DATA_DIR, "inside-cwd.dvc"),
|
||||
single_stage=True,
|
||||
)
|
||||
|
||||
def test_file_name(self):
|
||||
self.dvc.run(cmd="", deps=[], outs=[self.DATA_DIR], single_stage=True)
|
||||
self.dvc.run(
|
||||
cmd=f"mkdir {self.DATA_DIR}",
|
||||
deps=[],
|
||||
outs=[self.DATA_DIR],
|
||||
single_stage=True,
|
||||
)
|
||||
|
||||
with self.assertRaises(StagePathAsOutputError):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
outs=[self.FOO],
|
||||
fname=os.path.join(self.DATA_DIR, "inside-cwd.dvc"),
|
||||
single_stage=True,
|
||||
@@ -229,30 +237,30 @@ class TestRunStageInsideOutput(TestDvc):
|
||||
class TestRunBadCwd(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(StagePathOutsideError):
|
||||
self.dvc.run(cmd="", wdir=self.mkdtemp(), single_stage=True)
|
||||
self.dvc.run(cmd="command", wdir=self.mkdtemp(), single_stage=True)
|
||||
|
||||
def test_same_prefix(self):
|
||||
with self.assertRaises(StagePathOutsideError):
|
||||
path = f"{self._root_dir}-{uuid.uuid4()}"
|
||||
os.mkdir(path)
|
||||
self.dvc.run(cmd="", wdir=path, single_stage=True)
|
||||
self.dvc.run(cmd="command", wdir=path, single_stage=True)
|
||||
|
||||
|
||||
class TestRunBadWdir(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(StagePathOutsideError):
|
||||
self.dvc.run(cmd="", wdir=self.mkdtemp(), single_stage=True)
|
||||
self.dvc.run(cmd="command", wdir=self.mkdtemp(), single_stage=True)
|
||||
|
||||
def test_same_prefix(self):
|
||||
with self.assertRaises(StagePathOutsideError):
|
||||
path = f"{self._root_dir}-{uuid.uuid4()}"
|
||||
os.mkdir(path)
|
||||
self.dvc.run(cmd="", wdir=path, single_stage=True)
|
||||
self.dvc.run(cmd="command", wdir=path, single_stage=True)
|
||||
|
||||
def test_not_found(self):
|
||||
with self.assertRaises(StagePathNotFoundError):
|
||||
path = os.path.join(self._root_dir, str(uuid.uuid4()))
|
||||
self.dvc.run(cmd="", wdir=path, single_stage=True)
|
||||
self.dvc.run(cmd="command", wdir=path, single_stage=True)
|
||||
|
||||
def test_not_dir(self):
|
||||
with self.assertRaises(StagePathNotDirectoryError):
|
||||
@@ -261,7 +269,7 @@ class TestRunBadWdir(TestDvc):
|
||||
path = os.path.join(path, str(uuid.uuid4()))
|
||||
open(path, "a").close()
|
||||
self.dvc.run(
|
||||
cmd="", wdir=path, single_stage=True,
|
||||
cmd="command", wdir=path, single_stage=True,
|
||||
)
|
||||
|
||||
|
||||
@@ -269,7 +277,7 @@ class TestRunBadName(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(StagePathOutsideError):
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
fname=os.path.join(self.mkdtemp(), self.FOO + DVC_FILE_SUFFIX),
|
||||
single_stage=True,
|
||||
)
|
||||
@@ -279,7 +287,7 @@ class TestRunBadName(TestDvc):
|
||||
path = f"{self._root_dir}-{uuid.uuid4()}"
|
||||
os.mkdir(path)
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
fname=os.path.join(path, self.FOO + DVC_FILE_SUFFIX),
|
||||
single_stage=True,
|
||||
)
|
||||
@@ -288,7 +296,7 @@ class TestRunBadName(TestDvc):
|
||||
with self.assertRaises(StagePathNotFoundError):
|
||||
path = os.path.join(self._root_dir, str(uuid.uuid4()))
|
||||
self.dvc.run(
|
||||
cmd="",
|
||||
cmd="command",
|
||||
fname=os.path.join(path, self.FOO + DVC_FILE_SUFFIX),
|
||||
single_stage=True,
|
||||
)
|
||||
@@ -342,7 +350,7 @@ class TestRunUnprotectOutsCopy(TestDvc):
|
||||
ret = main(
|
||||
[
|
||||
"run",
|
||||
"--overwrite-dvcfile",
|
||||
"--overwrite",
|
||||
"--no-run-cache",
|
||||
"--single-stage",
|
||||
"-d",
|
||||
@@ -400,7 +408,7 @@ class TestRunUnprotectOutsSymlink(TestDvc):
|
||||
ret = main(
|
||||
[
|
||||
"run",
|
||||
"--overwrite-dvcfile",
|
||||
"--overwrite",
|
||||
"--no-run-cache",
|
||||
"--single-stage",
|
||||
"-d",
|
||||
@@ -459,7 +467,7 @@ class TestRunUnprotectOutsHardlink(TestDvc):
|
||||
ret = main(
|
||||
[
|
||||
"run",
|
||||
"--overwrite-dvcfile",
|
||||
"--overwrite",
|
||||
"--no-run-cache",
|
||||
"--single-stage",
|
||||
"-d",
|
||||
@@ -544,7 +552,7 @@ class TestCmdRunOverwrite(TestDvc):
|
||||
self.FOO,
|
||||
"-d",
|
||||
self.CODE,
|
||||
"--overwrite-dvcfile",
|
||||
"--overwrite",
|
||||
"--no-run-cache",
|
||||
"--single-stage",
|
||||
"-o",
|
||||
@@ -568,12 +576,13 @@ class TestCmdRunOverwrite(TestDvc):
|
||||
ret = main(
|
||||
[
|
||||
"run",
|
||||
"--overwrite-dvcfile",
|
||||
"--overwrite",
|
||||
"--single-stage",
|
||||
"-f",
|
||||
"out.dvc",
|
||||
"-d",
|
||||
self.BAR,
|
||||
f"cat {self.BAR}",
|
||||
]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
@@ -620,13 +629,13 @@ class TestCmdRunWorkingDirectory(TestDvc):
|
||||
wdir=".",
|
||||
single_stage=True,
|
||||
)
|
||||
d = load_stage_file(stage.relpath)
|
||||
d = load_yaml(stage.relpath)
|
||||
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
|
||||
|
||||
stage = self.dvc.run(
|
||||
cmd=f"echo test > {self.BAR}", outs=[self.BAR], single_stage=True,
|
||||
)
|
||||
d = load_stage_file(stage.relpath)
|
||||
d = load_yaml(stage.relpath)
|
||||
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
|
||||
|
||||
def test_fname_changes_path_and_wdir(self):
|
||||
@@ -646,7 +655,7 @@ class TestCmdRunWorkingDirectory(TestDvc):
|
||||
)
|
||||
|
||||
# Check that it is dumped properly (relative to fname)
|
||||
d = load_stage_file(stage.relpath)
|
||||
d = load_yaml(stage.relpath)
|
||||
self.assertEqual(d[Stage.PARAM_WDIR], "..")
|
||||
|
||||
|
||||
@@ -667,19 +676,19 @@ def test_rerun_deterministic_ignore_cache(tmp_dir, run_copy):
|
||||
|
||||
|
||||
def test_rerun_callback(dvc):
|
||||
def run_callback():
|
||||
def run_callback(overwrite=False):
|
||||
return dvc.run(
|
||||
cmd=("echo content > out"),
|
||||
cmd="echo content > out",
|
||||
outs=["out"],
|
||||
deps=[],
|
||||
overwrite=False,
|
||||
overwrite=overwrite,
|
||||
single_stage=True,
|
||||
)
|
||||
|
||||
assert run_callback() is not None
|
||||
|
||||
with mock.patch("dvc.prompt.confirm", return_value=True):
|
||||
with pytest.raises(StageFileAlreadyExistsError):
|
||||
assert run_callback() is not None
|
||||
assert run_callback(overwrite=True) is not None
|
||||
|
||||
|
||||
def test_rerun_changed_dep(tmp_dir, run_copy):
|
||||
@@ -689,6 +698,7 @@ def test_rerun_changed_dep(tmp_dir, run_copy):
|
||||
tmp_dir.gen("foo", "changed content")
|
||||
with pytest.raises(StageFileAlreadyExistsError):
|
||||
run_copy("foo", "out", overwrite=False, single_stage=True)
|
||||
assert run_copy("foo", "out", overwrite=True, single_stage=True)
|
||||
|
||||
|
||||
def test_rerun_changed_stage(tmp_dir, run_copy):
|
||||
@@ -765,7 +775,7 @@ class TestRunPersist(TestDvc):
|
||||
self.assertEqual(0, ret)
|
||||
|
||||
def stage_should_contain_persist_flag(self, stage_file):
|
||||
stage_file_content = load_stage_file(stage_file)
|
||||
stage_file_content = load_yaml(stage_file)
|
||||
self.assertEqual(
|
||||
True, stage_file_content["outs"][0][BaseOutput.PARAM_PERSIST]
|
||||
)
|
||||
@@ -779,7 +789,7 @@ class TestRunPersist(TestDvc):
|
||||
self.assertEqual(2, len(lines))
|
||||
|
||||
def should_remove_persistent_outs(self, file, stage_file):
|
||||
ret = main(["remove", stage_file])
|
||||
ret = main(["remove", stage_file, "--outs"])
|
||||
self.assertEqual(0, ret)
|
||||
|
||||
self.assertFalse(os.path.exists(file))
|
||||
@@ -860,7 +870,7 @@ class TestRerunWithSameOutputs(TestDvc):
|
||||
"run",
|
||||
self._outs_command,
|
||||
self.FOO,
|
||||
"--overwrite-dvcfile",
|
||||
"--overwrite",
|
||||
"--single-stage",
|
||||
f"echo {self.BAR_CONTENTS} >> {self.FOO}",
|
||||
]
|
||||
@@ -936,7 +946,7 @@ class TestPersistentOutput(TestDvc):
|
||||
|
||||
cmd = [
|
||||
"run",
|
||||
"--overwrite-dvcfile",
|
||||
"--overwrite",
|
||||
"--single-stage",
|
||||
"--deps",
|
||||
"immutable",
|
||||
|
||||
+74
-16
@@ -8,9 +8,9 @@ from dvc.main import main
|
||||
from dvc.output.local import LocalOutput
|
||||
from dvc.remote.local import LocalRemote
|
||||
from dvc.repo import Repo
|
||||
from dvc.stage import Stage
|
||||
from dvc.stage import PipelineStage, Stage
|
||||
from dvc.stage.exceptions import StageFileFormatError
|
||||
from dvc.utils.stage import dump_stage_file, load_stage_file
|
||||
from dvc.utils.yaml import dump_yaml, load_yaml
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
@@ -74,12 +74,12 @@ class TestReload(TestDvc):
|
||||
stage = stages[0]
|
||||
self.assertTrue(stage is not None)
|
||||
|
||||
d = load_stage_file(stage.relpath)
|
||||
d = load_yaml(stage.relpath)
|
||||
|
||||
# NOTE: checking that reloaded stage didn't change its checksum
|
||||
md5 = "11111111111111111111111111111111"
|
||||
d[stage.PARAM_MD5] = md5
|
||||
dump_stage_file(stage.relpath, d)
|
||||
dump_yaml(stage.relpath, d)
|
||||
|
||||
dvcfile = SingleStageFile(self.dvc, stage.relpath)
|
||||
stage = dvcfile.stage
|
||||
@@ -87,7 +87,7 @@ class TestReload(TestDvc):
|
||||
self.assertTrue(stage is not None)
|
||||
dvcfile.dump(stage)
|
||||
|
||||
d = load_stage_file(stage.relpath)
|
||||
d = load_yaml(stage.relpath)
|
||||
self.assertEqual(d[stage.PARAM_MD5], md5)
|
||||
|
||||
|
||||
@@ -103,7 +103,7 @@ class TestDefaultWorkingDirectory(TestDvc):
|
||||
d = stage.dumpd()
|
||||
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
|
||||
|
||||
d = load_stage_file(stage.relpath)
|
||||
d = load_yaml(stage.relpath)
|
||||
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
|
||||
|
||||
with self.dvc.lock, self.dvc.state:
|
||||
@@ -167,48 +167,106 @@ def test_meta_is_preserved(tmp_dir, dvc):
|
||||
(stage,) = tmp_dir.dvc_gen("foo", "foo content")
|
||||
|
||||
# Add meta to DVC-file
|
||||
data = load_stage_file(stage.path)
|
||||
data = load_yaml(stage.path)
|
||||
data["meta"] = {"custom_key": 42}
|
||||
dump_stage_file(stage.path, data)
|
||||
dump_yaml(stage.path, data)
|
||||
|
||||
# Loading and dumping to test that it works and meta is retained
|
||||
dvcfile = SingleStageFile(dvc, stage.path)
|
||||
new_stage = dvcfile.stage
|
||||
dvcfile.dump(new_stage)
|
||||
|
||||
new_data = load_stage_file(stage.path)
|
||||
new_data = load_yaml(stage.path)
|
||||
assert new_data["meta"] == data["meta"]
|
||||
|
||||
|
||||
def test_parent_repo_collect_stages(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen({"subdir": {}})
|
||||
tmp_dir.gen({"deep": {"dir": {}}})
|
||||
subrepo_dir = tmp_dir / "subdir"
|
||||
deep_subrepo_dir = tmp_dir / "deep" / "dir"
|
||||
|
||||
with subrepo_dir.chdir():
|
||||
subrepo = Repo.init(subdir=True)
|
||||
subrepo_dir.gen("subrepo_file", "subrepo file content")
|
||||
subrepo.add("subrepo_file")
|
||||
|
||||
with deep_subrepo_dir.chdir():
|
||||
deep_subrepo = Repo.init(subdir=True)
|
||||
deep_subrepo_dir.gen("subrepo_file", "subrepo file content")
|
||||
deep_subrepo.add("subrepo_file")
|
||||
|
||||
stages = dvc.collect(None)
|
||||
subrepo_stages = subrepo.collect(None)
|
||||
deep_subrepo_stages = deep_subrepo.collect(None)
|
||||
|
||||
assert stages == []
|
||||
assert subrepo_stages != []
|
||||
assert deep_subrepo_stages != []
|
||||
|
||||
|
||||
def test_stage_addressing(tmp_dir, dvc, run_copy):
|
||||
def test_stage_strings_representation(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
stage1 = run_copy("foo", "bar", single_stage=True)
|
||||
assert stage1.addressing == "bar.dvc"
|
||||
assert repr(stage1) == "Stage: 'bar.dvc'"
|
||||
assert str(stage1) == "stage: 'bar.dvc'"
|
||||
|
||||
stage2 = run_copy("bar", "baz", name="copy-bar-baz")
|
||||
assert stage2.addressing == "dvc.yaml:copy-bar-baz"
|
||||
assert stage2.addressing == "copy-bar-baz"
|
||||
assert repr(stage2) == "Stage: 'copy-bar-baz'"
|
||||
assert str(stage2) == "stage: 'copy-bar-baz'"
|
||||
|
||||
folder = tmp_dir / "dir"
|
||||
folder.mkdir()
|
||||
with folder.chdir():
|
||||
assert stage1.addressing == os.path.relpath(stage1.path)
|
||||
assert (
|
||||
stage2.addressing
|
||||
== os.path.relpath(stage2.path) + ":" + stage2.name
|
||||
)
|
||||
rel_path = os.path.relpath(stage1.path)
|
||||
assert stage1.addressing == rel_path
|
||||
assert repr(stage1) == f"Stage: '{rel_path}'"
|
||||
assert str(stage1) == f"stage: '{rel_path}'"
|
||||
|
||||
rel_path = os.path.relpath(stage2.path)
|
||||
assert stage2.addressing == f"{rel_path}:{stage2.name}"
|
||||
assert repr(stage2) == f"Stage: '{rel_path}:{stage2.name}'"
|
||||
assert str(stage2) == f"stage: '{rel_path}:{stage2.name}'"
|
||||
|
||||
|
||||
def test_stage_on_no_path_string_repr(tmp_dir, dvc):
|
||||
s = Stage(dvc)
|
||||
assert s.addressing == "No path"
|
||||
assert repr(s) == "Stage: 'No path'"
|
||||
assert str(s) == "stage: 'No path'"
|
||||
|
||||
p = PipelineStage(dvc, name="stage_name")
|
||||
assert p.addressing == "No path:stage_name"
|
||||
assert repr(p) == "Stage: 'No path:stage_name'"
|
||||
assert str(p) == "stage: 'No path:stage_name'"
|
||||
|
||||
|
||||
def test_stage_remove_pipeline_stage(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage = run_copy("foo", "bar", name="copy-foo-bar")
|
||||
run_copy("bar", "foobar", name="copy-bar-foobar")
|
||||
|
||||
dvc_file = stage.dvcfile
|
||||
with dvc.lock:
|
||||
stage.remove(purge=False)
|
||||
assert stage.name in dvc_file.stages
|
||||
|
||||
with dvc.lock:
|
||||
stage.remove()
|
||||
assert stage.name not in dvc_file.stages
|
||||
assert "copy-bar-foobar" in dvc_file.stages
|
||||
|
||||
|
||||
def test_stage_remove_pointer_stage(tmp_dir, dvc, run_copy):
|
||||
(stage,) = tmp_dir.dvc_gen("foo", "foo")
|
||||
|
||||
with dvc.lock:
|
||||
stage.remove(purge=False)
|
||||
assert not (tmp_dir / "foo").exists()
|
||||
assert (tmp_dir / stage.relpath).exists()
|
||||
|
||||
with dvc.lock:
|
||||
stage.remove()
|
||||
assert not (tmp_dir / stage.relpath).exists()
|
||||
|
||||
@@ -73,12 +73,12 @@ def test_status_on_pipeline_stages(tmp_dir, dvc, run_copy):
|
||||
|
||||
stage.cmd = " ".join(stage.cmd.split())
|
||||
stage.dvcfile._dump_pipeline_file(stage)
|
||||
assert dvc.status() == {"dvc.yaml:copy-foo-bar": ["changed command"]}
|
||||
assert dvc.status("copy-foo-bar") == {"copy-foo-bar": ["changed command"]}
|
||||
|
||||
# delete outputs
|
||||
(tmp_dir / "bar").unlink()
|
||||
assert dvc.status() == {
|
||||
"dvc.yaml:copy-foo-bar": [
|
||||
"copy-foo-bar": [
|
||||
{"changed outs": {"bar": "deleted"}},
|
||||
"changed command",
|
||||
]
|
||||
@@ -86,7 +86,7 @@ def test_status_on_pipeline_stages(tmp_dir, dvc, run_copy):
|
||||
(tmp_dir / "foo").unlink()
|
||||
assert dvc.status() == {
|
||||
"foo.dvc": [{"changed outs": {"foo": "deleted"}}],
|
||||
"dvc.yaml:copy-foo-bar": [
|
||||
"copy-foo-bar": [
|
||||
{"changed deps": {"foo": "deleted"}},
|
||||
{"changed outs": {"bar": "deleted"}},
|
||||
"changed command",
|
||||
|
||||
@@ -3,6 +3,7 @@ from os.path import join
|
||||
|
||||
from dvc.ignore import CleanTree
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.tree import RepoTree
|
||||
from dvc.scm import SCM
|
||||
from dvc.scm.git import GitTree
|
||||
@@ -220,3 +221,23 @@ def test_repotree_cache_save(tmp_dir, dvc, scm, erepo_dir, setup_remote):
|
||||
cache.save(PathInfo(erepo_dir / "dir"), None, tree=tree)
|
||||
for checksum in expected:
|
||||
assert os.path.exists(cache.checksum_to_path_info(checksum))
|
||||
|
||||
|
||||
def test_cleantree_subrepo(tmp_dir, dvc, scm, monkeypatch):
|
||||
tmp_dir.gen({"subdir": {}})
|
||||
subrepo_dir = tmp_dir / "subdir"
|
||||
with subrepo_dir.chdir():
|
||||
subrepo = Repo.init(subdir=True)
|
||||
subrepo_dir.gen({"foo": "foo", "dir": {"bar": "bar"}})
|
||||
|
||||
assert isinstance(dvc.tree, CleanTree)
|
||||
assert not dvc.tree.exists(subrepo_dir / "foo")
|
||||
assert not dvc.tree.isfile(subrepo_dir / "foo")
|
||||
assert not dvc.tree.exists(subrepo_dir / "dir")
|
||||
assert not dvc.tree.isdir(subrepo_dir / "dir")
|
||||
|
||||
assert isinstance(subrepo.tree, CleanTree)
|
||||
assert subrepo.tree.exists(subrepo_dir / "foo")
|
||||
assert subrepo.tree.isfile(subrepo_dir / "foo")
|
||||
assert subrepo.tree.exists(subrepo_dir / "dir")
|
||||
assert subrepo.tree.isdir(subrepo_dir / "dir")
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
import networkx as nx
|
||||
import pytest
|
||||
|
||||
from dvc.cli import parse_args
|
||||
from dvc.command.dag import CmdDAG, _build, _show_ascii, _show_dot
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fmt", [None, "--dot"])
|
||||
def test_dag(tmp_dir, dvc, mocker, fmt):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
|
||||
args = ["dag", "--full", "foo.dvc"]
|
||||
if fmt:
|
||||
args.append(fmt)
|
||||
cli_args = parse_args(args)
|
||||
assert cli_args.func == CmdDAG
|
||||
|
||||
cmd = cli_args.func(cli_args)
|
||||
|
||||
mocker.patch("dvc.command.dag._build", return_value=dvc.graph)
|
||||
|
||||
assert cmd.run() == 0
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def graph(tmp_dir, dvc):
|
||||
tmp_dir.dvc_gen("a", "a")
|
||||
tmp_dir.dvc_gen("b", "b")
|
||||
|
||||
dvc.run(
|
||||
no_exec=True, deps=["a", "c"], outs=["d", "e"], cmd="cmd1", name="1"
|
||||
)
|
||||
dvc.run(
|
||||
no_exec=True, deps=["b", "c"], outs=["f", "g"], cmd="cmd2", name="2"
|
||||
)
|
||||
dvc.run(
|
||||
no_exec=True,
|
||||
deps=["a", "b", "c"],
|
||||
outs=["h", "i"],
|
||||
cmd="cmd3",
|
||||
name="3",
|
||||
)
|
||||
dvc.run(no_exec=True, deps=["a", "h"], outs=["j"], cmd="cmd4", name="4")
|
||||
|
||||
return dvc.graph
|
||||
|
||||
|
||||
def test_build(graph):
|
||||
assert nx.is_isomorphic(_build(graph), graph)
|
||||
|
||||
|
||||
def test_build_target(graph):
|
||||
(stage,) = filter(
|
||||
lambda s: hasattr(s, "name") and s.name == "3", graph.nodes()
|
||||
)
|
||||
G = _build(graph, target=stage)
|
||||
assert set(G.nodes()) == {"3", "b.dvc", "a.dvc"}
|
||||
assert set(G.edges()) == {("3", "a.dvc"), ("3", "b.dvc")}
|
||||
|
||||
|
||||
def test_build_full(graph):
|
||||
(stage,) = filter(
|
||||
lambda s: hasattr(s, "name") and s.name == "3", graph.nodes()
|
||||
)
|
||||
G = _build(graph, target=stage, full=True)
|
||||
assert nx.is_isomorphic(G, graph)
|
||||
|
||||
|
||||
def test_show_ascii(graph):
|
||||
assert [line.rstrip() for line in _show_ascii(graph).splitlines()] == [
|
||||
" +----------------+ +----------------+", # noqa: E501
|
||||
" | stage: 'a.dvc' | | stage: 'b.dvc' |", # noqa: E501
|
||||
" *+----------------+**** +----------------+", # noqa: E501
|
||||
" ***** * ***** *** ***", # noqa: E501
|
||||
" **** * ***** ** **", # noqa: E501
|
||||
" *** * *** ** **", # noqa: E501
|
||||
"+------------+ ** +------------+ +------------+", # noqa: E501
|
||||
"| stage: '1' | ** | stage: '3' | | stage: '2' |", # noqa: E501
|
||||
"+------------+ *** +------------+ +------------+", # noqa: E501
|
||||
" ** ***",
|
||||
" ** **",
|
||||
" ** **",
|
||||
" +------------+",
|
||||
" | stage: '4' |",
|
||||
" +------------+",
|
||||
]
|
||||
|
||||
|
||||
def test_show_dot(graph):
|
||||
assert _show_dot(graph) == (
|
||||
"strict digraph {\n"
|
||||
"stage;\n"
|
||||
"stage;\n"
|
||||
"stage;\n"
|
||||
"stage;\n"
|
||||
"stage;\n"
|
||||
"stage;\n"
|
||||
"\"stage: '1'\" -> \"stage: 'a.dvc'\";\n"
|
||||
"\"stage: '2'\" -> \"stage: 'b.dvc'\";\n"
|
||||
"\"stage: '3'\" -> \"stage: 'a.dvc'\";\n"
|
||||
"\"stage: '3'\" -> \"stage: 'b.dvc'\";\n"
|
||||
"\"stage: '4'\" -> \"stage: 'a.dvc'\";\n"
|
||||
"\"stage: '4'\" -> \"stage: '3'\";\n"
|
||||
"}\n"
|
||||
)
|
||||
@@ -13,7 +13,7 @@ def test_metrics_diff(dvc, mocker):
|
||||
"template",
|
||||
"--targets",
|
||||
"datafile",
|
||||
"--show-json",
|
||||
"--show-vega",
|
||||
"-x",
|
||||
"x_field",
|
||||
"-y",
|
||||
@@ -32,21 +32,24 @@ def test_metrics_diff(dvc, mocker):
|
||||
assert cli_args.func == CmdPlotsDiff
|
||||
|
||||
cmd = cli_args.func(cli_args)
|
||||
m = mocker.patch("dvc.repo.plots.diff.diff", return_value={})
|
||||
m = mocker.patch(
|
||||
"dvc.repo.plots.diff.diff", return_value={"datafile": "filledtemplate"}
|
||||
)
|
||||
|
||||
assert cmd.run() == 0
|
||||
|
||||
m.assert_called_once_with(
|
||||
cmd.repo,
|
||||
targets=["datafile"],
|
||||
template="template",
|
||||
revs=["HEAD", "tag1", "tag2"],
|
||||
x_field="x_field",
|
||||
y_field="y_field",
|
||||
csv_header=True,
|
||||
title="my_title",
|
||||
x_title="x_title",
|
||||
y_title="y_title",
|
||||
props={
|
||||
"template": "template",
|
||||
"x": "x_field",
|
||||
"y": "y_field",
|
||||
"title": "my_title",
|
||||
"xlab": "x_title",
|
||||
"ylab": "y_title",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@@ -59,7 +62,7 @@ def test_metrics_show(dvc, mocker):
|
||||
"result.extension",
|
||||
"-t",
|
||||
"template",
|
||||
"--show-json",
|
||||
"--show-vega",
|
||||
"--no-csv-header",
|
||||
"datafile",
|
||||
]
|
||||
@@ -68,30 +71,34 @@ def test_metrics_show(dvc, mocker):
|
||||
|
||||
cmd = cli_args.func(cli_args)
|
||||
|
||||
m = mocker.patch("dvc.repo.plots.show.show", return_value={})
|
||||
m = mocker.patch(
|
||||
"dvc.repo.plots.show.show", return_value={"datafile": "filledtemplate"}
|
||||
)
|
||||
|
||||
assert cmd.run() == 0
|
||||
|
||||
m.assert_called_once_with(
|
||||
cmd.repo,
|
||||
targets=["datafile"],
|
||||
template="template",
|
||||
x_field=None,
|
||||
y_field=None,
|
||||
csv_header=False,
|
||||
title=None,
|
||||
x_title=None,
|
||||
y_title=None,
|
||||
props={"template": "template", "csv_header": False},
|
||||
)
|
||||
|
||||
|
||||
def test_plots_show_json(dvc, mocker, caplog):
|
||||
def test_plots_show_vega(dvc, mocker, caplog):
|
||||
cli_args = parse_args(
|
||||
["plots", "diff", "HEAD~10", "HEAD~1", "--show-json"]
|
||||
[
|
||||
"plots",
|
||||
"diff",
|
||||
"HEAD~10",
|
||||
"HEAD~1",
|
||||
"--show-vega",
|
||||
"--targets",
|
||||
"plots.csv",
|
||||
]
|
||||
)
|
||||
cmd = cli_args.func(cli_args)
|
||||
mocker.patch(
|
||||
"dvc.repo.plots.diff.diff", return_value={"plots.csv": "plothtml"}
|
||||
)
|
||||
assert cmd.run() == 0
|
||||
assert '{"plots.csv": "plothtml"}\n' in caplog.text
|
||||
assert "plothtml" in caplog.text
|
||||
|
||||
@@ -27,7 +27,7 @@ def test_run(mocker, dvc):
|
||||
"--wdir",
|
||||
"wdir",
|
||||
"--no-exec",
|
||||
"--overwrite-dvcfile",
|
||||
"--overwrite",
|
||||
"--no-run-cache",
|
||||
"--no-commit",
|
||||
"--outs-persist",
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user