Merge pull request #223 from dataversioncontrol/metric_to_state_file

Misc: config and show workflows

dvc/command/config.py

@@ -5,6 +5,7 @@ from dvc.command.base import CmdBase, DvcLock
 from dvc.logger import Logger
 from dvc.config import Config
 
+
 class CmdConfig(CmdBase):
     def __init__(self, settings):
         super(CmdConfig, self).__init__(settings)
@@ -55,15 +56,18 @@ class CmdConfig(CmdBase):
         return 0
 
     def check_opt(self):
-        _section, _opt = self.parsed_args.name.strip().split('.')
+        _section, _opt = self.parsed_args.name.strip().split('.', 1)
         add = (self.parsed_args.value != None and self.parsed_args.unset == False)
 
         section = self._get_key(self.configobj, _section, add)
+
         if not section:
+            Logger.error('Invalid option name {}'.format(_section))
             return 1
 
         opt = self._get_key(self.configobj[section], _opt, add)
         if not opt:
+            Logger.error('Invalid option value: {}'.format(_opt))
             return 1
 
         self.section = section

dvc/command/show_pipeline.py

@@ -2,6 +2,7 @@ import os
 import networkx as nx
 
 from dvc.command.traverse import Traverse
+from dvc.command.init import CmdInit
 from dvc.logger import Logger
 from dvc.state_file import StateFile
 
@@ -12,11 +13,12 @@ class CmdShowPipeline(Traverse):
         self.g = nx.DiGraph()
         self.subs = []
 
-    def draw(self, g, target, fname):
+    def draw(self, g, target, fname_suffix):
+        fname = 'pipeline_' + fname_suffix
         try:
             A = nx.nx_agraph.to_agraph(g)
             A.write(fname + '.dot')
-            A.draw(fname + '.svg', prog='dot')
+            A.draw(fname + '.jpeg', format='jpeg', prog='dot')
         except Exception as exc:
             Logger.error('Failed to draw dependency graph for {}: {}'.format(target, exc))
             return 1
@@ -30,16 +32,16 @@ class CmdShowPipeline(Traverse):
 
         return self.g
 
-    def draw_targets(self):
-        if self.parsed_args.target == None:
+    def draw_targets(self, target):
+        if not target:
             target = self.settings.config.data_dir
             return self.draw(self.g, target, target)
 
         for t in self.parsed_args.target:
-            fname = os.path.basename(os.path.normpath(t))
+            fname_suffix = os.path.basename(os.path.normpath(t))
             s = self.find_sub(t)
 
-            ret = self.draw(s, t, fname)
+            ret = self.draw(s, t, fname_suffix)
             if ret != 0:
                 return ret
 
@@ -61,13 +63,16 @@ class CmdShowPipeline(Traverse):
         # when a bunch of data items were used independently.
         self.subs = nx.weakly_connected_component_subgraphs(self.g)
 
-        return self.draw_targets()
+        return self.draw_targets(saved_targets)
 
     def process_file(self, target):
         data_item = self._get_data_item(target)
         name = data_item.data.relative
         state = StateFile.load(data_item.state.relative, self.git)
 
+        if name == os.path.join(self.settings.config.data_dir, CmdInit.EMPTY_FILE_NAME):
+            return
+
         self.g.add_node(name)
 
         for i in state.input_files:

dvc/git_wrapper.py

@@ -91,6 +91,8 @@ class GitWrapperI(object):
 
 
 class GitWrapper(GitWrapperI):
+    FLOATS_FROM_STRING = re.compile(r'[-+]?(?:(?:\d*\.\d+)|(?:\d+\.?))(?:[Ee][+-]?\d+)?')
+
     def __init__(self):
         super(GitWrapper, self).__init__()
 
@@ -330,13 +332,13 @@ class GitWrapper(GitWrapperI):
             return None
 
         # Extract float from string. I.e. from 'AUC: 0.596182'
-        num = re.findall("\d+\.\d+", lines[0])
-        if len(num) != 1:
+        nums = self.FLOATS_FROM_STRING.findall(lines[0])
+        if len(nums) < 1:
             msg = '[dvc-git] Unable to parse metrics from \'{}\' file {}'
             Logger.warn(msg.format(lines[0], target))
             return None
 
-        return float(num[0])
+        return float(nums[0])
 
     @staticmethod
     def get_merges_map():

dvc/graph/commit.py

@@ -36,6 +36,7 @@ class Commit(object):
 
     def add_collapsed_commit(self, commit):
         self._collapsed_commits.append(commit)
+        self._collapsed_commits += commit._collapsed_commits
 
     @property
     def text(self):

dvc/graph/workflow.py

@@ -124,10 +124,10 @@ class Workflow(object):
                 g.add_edge(commit.hash, p)
 
         A = nx.nx_agraph.to_agraph(g)
-        fname = 'workflow.jpeg'
+        fname = 'workflow'
         A = A.to_undirected()
         A.write(fname + '.dot')
-        A.draw(fname, format='jpeg', prog='dot')
+        A.draw(fname + '.jpeg', format='jpeg', prog='dot')
         pass
 
     def modify_workflow(self, show_all_commits, show_dvc_commits):

functest/tutorial_branches.sh

@@ -38,7 +38,9 @@ dvc run python code/train_model.py data/matrix-train.p 20170426 data/model.p
 
 dvc run python code/evaluate.py data/model.p data/matrix-test.p data/eval_auc.txt
 
-dvc target data/eval_auc.txt
+dvc config Global.Target data/evel_auc.txt
+git commit -am 'Set Target'
+
 cat data/eval_auc.txt
 # AUC: 0.645320
 

tests/test_workflow.py

@@ -115,5 +115,32 @@ class TestWorkflow(TestCase):
 
         self.assertEqual(wf._commits['3']._target_metric, value)
         self.assertEqual(wf._commits['3'].branch_tips, branches)
+        self.assertTrue('4' in wf._commits)
         pass
 
+    def collapsed_all_commits_comments_test(self):
+        value = 0.812345
+        branches = ['master', 'try_smth']
+        not_metric_commit2 = Commit('2', '1', 'name1', 'today', 'change smth 2...',
+                                    True, branch_tips=branches)
+        metric_commit3 = Commit('3', '2', 'name3', 'today3', 'hello3',
+                                True, value)
+
+        wf = Workflow('', '')
+        wf.add_commit(self._commit1)
+        wf.add_commit(not_metric_commit2)
+        wf.add_commit(metric_commit3)
+        wf.add_commit(self._commit4)
+
+        wf.collapse_commits(CollapseDvcReproCommitsStrategy(wf))
+        wf.collapse_commits(CollapseNotMeticsCommitsStrategy(wf))
+        self.assertEqual(len(wf._commits), 2)
+
+        def is_collapsed_commit_presented(self, commit, collapsed_commit):
+            self.assertTrue('[' + collapsed_commit.hash + ']' in commit.text)
+            self.assertTrue(collapsed_commit._comment in commit.text)
+
+        is_collapsed_commit_presented(self, wf._commits['3'], metric_commit3)
+        is_collapsed_commit_presented(self, wf._commits['3'], not_metric_commit2)
+        is_collapsed_commit_presented(self, wf._commits['3'], self._commit1)
+        pass