Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-09 07:18:27 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix #3588 (#3770)

Browse Source 
* fix #3588

remove improper edges from output DAG.

* less code modification

* deepsource required

* deepsource required

* refactoring for reducing cognitive complexity.
This commit is contained in:
karajan1001
2020-05-09 21:00:46 +03:00
committed by GitHub
parent 6093a1fe97
commit 76b31ac282
2 changed files with 58 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/command/pipeline.py
+30 -9
View File
@@ -31,6 +31,31 @@ class CmdPipelineShow(CmdBase):
else:
logger.info(stage.addressing)
@staticmethod
def _build_output_graph(G, target_stage):
import networkx
from itertools import product
nodes = {str(out) for out in target_stage.outs}
edges = []
for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
from_stage_deps = {dep.path_info.parts for dep in from_stage.deps}
to_outs = {
to_out
for to_out in to_stage.outs
if to_out.path_info.parts in from_stage_deps
}
from_outs = {
from_out
for from_out in from_stage.outs
if str(from_out) in nodes
}
nodes |= {str(to_out) for to_out in to_outs}
for from_out, to_out in product(from_outs, to_outs):
edges.append((str(from_out), str(to_out)))
return nodes, edges
def _build_graph(self, target, commands=False, outs=False):
import networkx
from dvc import dvcfile
@@ -47,10 +72,7 @@ class CmdPipelineShow(CmdBase):
if stage.cmd is None:
continue
nodes.add(stage.cmd)
elif outs:
for out in stage.outs:
nodes.add(str(out))
else:
elif not outs:
nodes.add(stage.addressing)
edges = []
@@ -59,13 +81,12 @@ class CmdPipelineShow(CmdBase):
if to_stage.cmd is None:
continue
edges.append((from_stage.cmd, to_stage.cmd))
elif outs:
for from_out in from_stage.outs:
for to_out in to_stage.outs:
edges.append((str(from_out), str(to_out)))
else:
elif not outs:
edges.append((from_stage.addressing, to_stage.addressing))
if outs:
nodes, edges = self._build_output_graph(G, target_stage)
return list(nodes), edges, networkx.is_tree(G)
def _show_ascii(self, target, commands, outs):
tests/func/test_pipeline.py
+28 -3
View File
@@ -274,7 +274,7 @@ def test_split_pipeline(tmp_dir, scm, dvc):
)
command = CmdPipelineShow([])
nodes, edges, is_tree = command._build_graph(
nodes, edges, _ = command._build_graph(
stage.path, commands=False, outs=True
)
assert set(nodes) == {"data", "data_train", "data_valid", "result"}
@@ -319,11 +319,36 @@ def test_pipeline_ascii_multistage(tmp_dir, dvc, run_copy):
run_copy("foo", "bar", name="copy-foo-bar")
run_copy("bar", "foobar", single_stage=True)
command = CmdPipelineShow([])
nodes, edges, is_tree = command._build_graph("foobar.dvc")
nodes, edges, _ = command._build_graph("foobar.dvc")
assert set(nodes) == {"dvc.yaml:copy-foo-bar", "foobar.dvc"}
assert set(edges) == {
("foobar.dvc", "dvc.yaml:copy-foo-bar"),
}
nodes, edges, is_tree = command._build_graph("dvc.yaml:copy-foo-bar")
nodes, *_ = command._build_graph("dvc.yaml:copy-foo-bar")
assert set(nodes) == {"dvc.yaml:copy-foo-bar"}
def test_pipeline_multi_outputs_stages(dvc):
dvc.run(
outs=["alice", "bob"],
cmd="echo alice>alice && echo bob>bob",
single_stage=True,
)
dvc.run(
deps=["alice"],
outs=["mary", "mike"],
cmd="echo mary>mary && echo mike>mike",
single_stage=True,
)
stage = dvc.run(
deps=["mary"],
outs=["carol"],
cmd="echo carol>carol",
single_stage=True,
)
command = CmdPipelineShow([])
nodes, edges, _ = command._build_graph(stage.path, outs=True)
assert set(nodes) == {"alice", "mary", "carol"}
assert set(edges) == {("carol", "mary"), ("mary", "alice")}