From cea4dcda14e58127d6d64a6af904a23517c9e1f9 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 17 Sep 2017 23:35:56 -0700 Subject: [PATCH 1/5] Config cmd print error messages --- dvc/command/config.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dvc/command/config.py b/dvc/command/config.py index 178c6b553..9c6ed3461 100644 --- a/dvc/command/config.py +++ b/dvc/command/config.py @@ -5,6 +5,7 @@ from dvc.command.base import CmdBase, DvcLock from dvc.logger import Logger from dvc.config import Config + class CmdConfig(CmdBase): def __init__(self, settings): super(CmdConfig, self).__init__(settings) @@ -55,15 +56,18 @@ class CmdConfig(CmdBase): return 0 def check_opt(self): - _section, _opt = self.parsed_args.name.strip().split('.') + _section, _opt = self.parsed_args.name.strip().split('.', 1) add = (self.parsed_args.value != None and self.parsed_args.unset == False) section = self._get_key(self.configobj, _section, add) + if not section: + Logger.error('Invalid option name {}'.format(_section)) return 1 opt = self._get_key(self.configobj[section], _opt, add) if not opt: + Logger.error('Invalid option value: {}'.format(_opt)) return 1 self.section = section From cef6bab2501f46a5afa0a9c77809e317f9416b9b Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 17 Sep 2017 23:37:29 -0700 Subject: [PATCH 2/5] Parse scientific float format in target files --- dvc/git_wrapper.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dvc/git_wrapper.py b/dvc/git_wrapper.py index 5dc945e4e..9bbb0e2ba 100644 --- a/dvc/git_wrapper.py +++ b/dvc/git_wrapper.py @@ -91,6 +91,8 @@ class GitWrapperI(object): class GitWrapper(GitWrapperI): + FLOATS_FROM_STRING = re.compile(r'[-+]?(?:(?:\d*\.\d+)|(?:\d+\.?))(?:[Ee][+-]?\d+)?') + def __init__(self): super(GitWrapper, self).__init__() @@ -330,13 +332,13 @@ class GitWrapper(GitWrapperI): return None # Extract float from string. I.e. from 'AUC: 0.596182' - num = re.findall("\d+\.\d+", lines[0]) - if len(num) != 1: + nums = self.FLOATS_FROM_STRING.findall(lines[0]) + if len(nums) < 1: msg = '[dvc-git] Unable to parse metrics from \'{}\' file {}' Logger.warn(msg.format(lines[0], target)) return None - return float(num[0]) + return float(nums[0]) @staticmethod def get_merges_map(): From 2d8a7a69472c273742c799bb89a5ee6fbe1c21a5 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Mon, 18 Sep 2017 00:41:03 -0700 Subject: [PATCH 3/5] Fix show workflow issue: collapsed users commits --- dvc/graph/commit.py | 1 + tests/test_workflow.py | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/dvc/graph/commit.py b/dvc/graph/commit.py index 754c3bad3..a52071f98 100644 --- a/dvc/graph/commit.py +++ b/dvc/graph/commit.py @@ -36,6 +36,7 @@ class Commit(object): def add_collapsed_commit(self, commit): self._collapsed_commits.append(commit) + self._collapsed_commits += commit._collapsed_commits @property def text(self): diff --git a/tests/test_workflow.py b/tests/test_workflow.py index 9e664aa0c..a940c4fca 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -115,5 +115,32 @@ class TestWorkflow(TestCase): self.assertEqual(wf._commits['3']._target_metric, value) self.assertEqual(wf._commits['3'].branch_tips, branches) + self.assertTrue('4' in wf._commits) pass + def collapsed_all_commits_comments_test(self): + value = 0.812345 + branches = ['master', 'try_smth'] + not_metric_commit2 = Commit('2', '1', 'name1', 'today', 'change smth 2...', + True, branch_tips=branches) + metric_commit3 = Commit('3', '2', 'name3', 'today3', 'hello3', + True, value) + + wf = Workflow('', '') + wf.add_commit(self._commit1) + wf.add_commit(not_metric_commit2) + wf.add_commit(metric_commit3) + wf.add_commit(self._commit4) + + wf.collapse_commits(CollapseDvcReproCommitsStrategy(wf)) + wf.collapse_commits(CollapseNotMeticsCommitsStrategy(wf)) + self.assertEqual(len(wf._commits), 2) + + def is_collapsed_commit_presented(self, commit, collapsed_commit): + self.assertTrue('[' + collapsed_commit.hash + ']' in commit.text) + self.assertTrue(collapsed_commit._comment in commit.text) + + is_collapsed_commit_presented(self, wf._commits['3'], metric_commit3) + is_collapsed_commit_presented(self, wf._commits['3'], not_metric_commit2) + is_collapsed_commit_presented(self, wf._commits['3'], self._commit1) + pass From 80271928f093bbcb967fc9d622ecf26b2a8120eb Mon Sep 17 00:00:00 2001 From: Dmitry Date: Mon, 18 Sep 2017 00:48:24 -0700 Subject: [PATCH 4/5] Change target command --- functest/tutorial_branches.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/functest/tutorial_branches.sh b/functest/tutorial_branches.sh index 249e82a6a..ed66527d2 100755 --- a/functest/tutorial_branches.sh +++ b/functest/tutorial_branches.sh @@ -38,7 +38,9 @@ dvc run python code/train_model.py data/matrix-train.p 20170426 data/model.p dvc run python code/evaluate.py data/model.p data/matrix-test.p data/eval_auc.txt -dvc target data/eval_auc.txt +dvc config Global.Target data/evel_auc.txt +git commit -am 'Set Target' + cat data/eval_auc.txt # AUC: 0.645320 From f0eb59a722aa95164642ce6fe19f968f8d2e633f Mon Sep 17 00:00:00 2001 From: Dmitry Date: Mon, 18 Sep 2017 01:16:10 -0700 Subject: [PATCH 5/5] dvc show pipeline: small changes --- dvc/command/show_pipeline.py | 19 ++++++++++++------- dvc/graph/workflow.py | 4 ++-- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/dvc/command/show_pipeline.py b/dvc/command/show_pipeline.py index 8184e66cc..e8e168bf4 100644 --- a/dvc/command/show_pipeline.py +++ b/dvc/command/show_pipeline.py @@ -2,6 +2,7 @@ import os import networkx as nx from dvc.command.traverse import Traverse +from dvc.command.init import CmdInit from dvc.logger import Logger from dvc.state_file import StateFile @@ -12,11 +13,12 @@ class CmdShowPipeline(Traverse): self.g = nx.DiGraph() self.subs = [] - def draw(self, g, target, fname): + def draw(self, g, target, fname_suffix): + fname = 'pipeline_' + fname_suffix try: A = nx.nx_agraph.to_agraph(g) A.write(fname + '.dot') - A.draw(fname + '.svg', prog='dot') + A.draw(fname + '.jpeg', format='jpeg', prog='dot') except Exception as exc: Logger.error('Failed to draw dependency graph for {}: {}'.format(target, exc)) return 1 @@ -30,16 +32,16 @@ class CmdShowPipeline(Traverse): return self.g - def draw_targets(self): - if self.parsed_args.target == None: + def draw_targets(self, target): + if not target: target = self.settings.config.data_dir return self.draw(self.g, target, target) for t in self.parsed_args.target: - fname = os.path.basename(os.path.normpath(t)) + fname_suffix = os.path.basename(os.path.normpath(t)) s = self.find_sub(t) - ret = self.draw(s, t, fname) + ret = self.draw(s, t, fname_suffix) if ret != 0: return ret @@ -61,13 +63,16 @@ class CmdShowPipeline(Traverse): # when a bunch of data items were used independently. self.subs = nx.weakly_connected_component_subgraphs(self.g) - return self.draw_targets() + return self.draw_targets(saved_targets) def process_file(self, target): data_item = self._get_data_item(target) name = data_item.data.relative state = StateFile.load(data_item.state.relative, self.git) + if name == os.path.join(self.settings.config.data_dir, CmdInit.EMPTY_FILE_NAME): + return + self.g.add_node(name) for i in state.input_files: diff --git a/dvc/graph/workflow.py b/dvc/graph/workflow.py index 7f839b602..846b21fdf 100644 --- a/dvc/graph/workflow.py +++ b/dvc/graph/workflow.py @@ -124,10 +124,10 @@ class Workflow(object): g.add_edge(commit.hash, p) A = nx.nx_agraph.to_agraph(g) - fname = 'workflow.jpeg' + fname = 'workflow' A = A.to_undirected() A.write(fname + '.dot') - A.draw(fname, format='jpeg', prog='dot') + A.draw(fname + '.jpeg', format='jpeg', prog='dot') pass def modify_workflow(self, show_all_commits, show_dvc_commits):