diff --git a/dvc/command/show.py b/dvc/command/show.py index 1ec8842db..5bc5ac70b 100644 --- a/dvc/command/show.py +++ b/dvc/command/show.py @@ -7,6 +7,7 @@ from dvc.command.init import CmdInit from dvc.logger import Logger from dvc.state_file import StateFile + class CmdShow(Traverse): def __init__(self, settings): super(CmdShow, self).__init__(settings, "collect", do_not_start_from_root=False) @@ -49,7 +50,26 @@ class CmdShow(Traverse): def run(self): if self.parsed_args.workflow: # Workflow - wf = self.git.get_all_commits('data/eval_auc.txt') + target = self.settings.parsed_args.target + if not target: + target = self.settings.config.target_file + + if not os.path.exists(target): + Logger.warn(u'Target is not defined: use empty target') + target = '' + else: + target = open(target).read() + Logger.debug(u'Set show workflow target as {}'.format(target)) + + if type(target) is list: + if len(target) != 1: + msg = 'Only one target is expected for show workflow command. Target: {}' + Logger.error(msg.format(target)) + return 1 + + target = target[0] + + wf = self.git.get_all_commits(target, self.settings) wf.build_graph() return diff --git a/dvc/git_wrapper.py b/dvc/git_wrapper.py index 754a317bc..5b93c9f07 100644 --- a/dvc/git_wrapper.py +++ b/dvc/git_wrapper.py @@ -3,6 +3,7 @@ import os from dvc.logger import Logger from dvc.config import Config from dvc.executor import Executor, ExecutorError +from dvc.path.data_item import DataItemError from dvc.system import System from dvc.workflow import Workflow, Commit @@ -231,8 +232,7 @@ class GitWrapper(GitWrapperI): LOG_SEPARATOR = '|' LOG_FORMAT = ['%h', '%p', '%an', '%ai', '%s'] - @staticmethod - def get_all_commits(target): + def get_all_commits(self, target, settings): # git log --all --abbrev=7 --pretty=format:"%h|%p|%an|%ai|%s" try: merges_map = GitWrapper.get_merges_map() @@ -249,18 +249,59 @@ class GitWrapper(GitWrapperI): hash, parent_hash, name, date, comment = items wf.add_commit(Commit(hash, parent_hash, name, date, comment, - GitWrapper.is_target(hash, target))) + *self.is_target(hash, target, settings))) return wf except ExecutorError: raise - @staticmethod - def is_target(hash, target): - # git show --pretty="" --name-only b6fa39b01 + def is_target(self, hash, target, settings): git_cmd = ['git', 'show', '--pretty=', '--name-only', hash] files = set(Executor.exec_cmd_only_success(git_cmd).split('\n')) - return target in files + + if target in files: + try: + settings.path_factory.data_item(target) + except DataItemError as ex: + Logger.warn('Target file {} is not data item: {}'.format(target, ex)) + return True, None + + try: + cmd_symlink_data = ['git', 'show', '{}:{}'.format(hash, target)] + symlink_content = Executor.exec_cmd_only_success(cmd_symlink_data).split('\n') + except ExecutorError as ex: + msg = '[dvc-git] Cannot obtain content of target symbolic file {} with hash {}: {}' + Logger.warn(msg.format(target, hash, ex)) + return True, None + + if not symlink_content or len(symlink_content) != 1: + msg = '[dvc-git] Target symbolic file {} with hash {} has wrong format' + Logger.warn(msg.format(target, hash)) + return True, None + + return True, self.target_metric_from_git_history(hash, symlink_content[0], target, settings) + + return False, None + + def target_metric_from_git_history(self, hash, symlink_content, target, settings): + cache_rel_to_data = os.path.relpath(settings.config.cache_dir, settings.config.data_dir) + common_prefix = os.path.commonprefix([symlink_content, cache_rel_to_data]) + cache_file_name = symlink_content[len(common_prefix):] + if cache_file_name[0] == os.path.sep: + cache_file_name = cache_file_name[1:] + + file_name = os.path.join(settings.config.cache_dir, cache_file_name) + full_file_name = os.path.join(self.git_dir_abs, file_name) + + if os.path.exists(full_file_name): + lines = open(full_file_name).readlines(2) + if len(lines) != 1: + msg = '[dvc-git] Target file {} with hash {} has wrong format: {} lines were obtained, 1 expected.' + Logger.warn(msg.format(target, hash, len(lines))) + else: + return float(lines[0]) + + return None @staticmethod def get_merges_map(): diff --git a/dvc/workflow.py b/dvc/workflow.py index fc0edbd51..bb3cb7548 100644 --- a/dvc/workflow.py +++ b/dvc/workflow.py @@ -13,13 +13,15 @@ class Commit(object): DVC_REPRO_PREFIX = 'DVC repro' COLLAPSED_TEXT = DVC_REPRO_PREFIX + '\n<< collapsed commits >>' - def __init__(self, hash, parents, name, date, comment, is_target): + def __init__(self, hash, parents, name, date, comment, is_target=False, target_metric=None): self._hash = hash self._parent_hashes = set(parents.split()) self._name = name self._date = date self._comment = comment self._is_target = is_target + self._target_metric = target_metric + self._is_collapsed = False @property @@ -38,9 +40,14 @@ class Commit(object): @property def text(self): + metric_text = '' + if self._is_target and self._target_metric: + metric_text = '\nTarget metric: {}'.format(self._target_metric) + if self._is_collapsed: - return self.COLLAPSED_TEXT - return self._comment[:self.TEXT_LIMIT] + '\n' + self.hash + return self.COLLAPSED_TEXT + metric_text + + return self._comment[:self.TEXT_LIMIT] + '\n' + self.hash + metric_text @property def is_repro(self): @@ -49,6 +56,18 @@ class Commit(object): def make_colapsed(self): self._is_collapsed = True + @property + def has_target_metric(self): + return self._is_target and self._target_metric is not None + + @property + def target_metric(self): + return self._target_metric + + def set_target_metric(self, value): + self._target_metric = value + self._is_target = True + class Workflow(object): def __init__(self, target, merges_map, root='', no_repro_commits=True): @@ -133,6 +152,11 @@ class Workflow(object): child_commit_hashes = self._edges.get(commit.hash) if child_commit_hashes and all(self._commits[hash].is_repro for hash in child_commit_hashes): + if commit.has_target_metric: + for hash in child_commit_hashes: + if not self._commits[hash].has_target_metric: + self._commits[hash].set_target_metric(commit.target_metric) + for hash in child_commit_hashes: self._commits[hash].add_parents(commit.parent_hashes) self._commits[hash].remove_parent(commit.hash) diff --git a/tests/test_workflow.py b/tests/test_workflow.py index 35402bda9..a8ec64ae3 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -5,19 +5,19 @@ from dvc.workflow import Workflow, Commit class TestWorkflow(TestCase): def setUp(self): - self._commit4 = Commit('4', '3', 'name1', 'today', 'comment4', False) - self._commit3 = Commit('3', '2', 'name1', 'today', 'DVC repro-run ...', False) - self._commit2 = Commit('2', '1', 'name1', 'today', 'DVC repro-run ...', False) - self._commit1 = Commit('1', '', 'name1', 'today', 'comment1', False) + self._commit4 = Commit('4', '3', 'name1', 'today', 'comment4') + self._commit3 = Commit('3', '2', 'name1', 'today', 'DVC repro-run ...') + self._commit2 = Commit('2', '1', 'name1', 'today', 'DVC repro-run ...') + self._commit1 = Commit('1', '', 'name1', 'today', 'comment1') - def test_commits(self): + def commits_basic_test(self): self.assertFalse(self._commit1.is_repro) self.assertTrue(self._commit2.is_repro) self.assertTrue(self._commit3.is_repro) self.assertFalse(self._commit4.is_repro) pass - def test_workflow_basic(self): + def workflow_basic_test(self): wf = Workflow('', '') wf.add_commit(self._commit1) wf.add_commit(self._commit2) @@ -32,7 +32,7 @@ class TestWorkflow(TestCase): self.assertEqual(wf._commits['4'].text, self._commit4._comment + '\n' + self._commit4.hash) pass - def test_collapse(self): + def collapse_test(self): wf = Workflow('', '') wf.add_commit(self._commit1) wf.add_commit(self._commit2) @@ -50,7 +50,7 @@ class TestWorkflow(TestCase): self.assertFalse('2' in wf._back_edges) pass - def test_collapse_at_dead_end(self): + def collapse_at_dead_end_test(self): wf = Workflow('', '') wf.add_commit(self._commit1) wf.add_commit(self._commit2) @@ -64,3 +64,19 @@ class TestWorkflow(TestCase): self.assertEqual(wf._commits[self._commit3.hash].text, Commit.COLLAPSED_TEXT) self.assertTrue('2' not in wf._commits) pass + + def collapse_target_metric_commit_test(self): + value = 0.812345 + metric_commit3 = Commit('2', '1', 'name1', 'today', 'DVC repro-run ...', True, value) + + wf = Workflow('', '') + wf.add_commit(self._commit1) + wf.add_commit(metric_commit3) + wf.add_commit(self._commit3) + + self.assertEqual(len(wf._commits), 3) + wf.collapse_repro_commits() + self.assertEqual(len(wf._commits), 2) + + self.assertEqual(wf._commits['3']._target_metric, value) + pass \ No newline at end of file