mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
Show target metric on the graph
This commit is contained in:
+21
-1
@@ -7,6 +7,7 @@ from dvc.command.init import CmdInit
|
||||
from dvc.logger import Logger
|
||||
from dvc.state_file import StateFile
|
||||
|
||||
|
||||
class CmdShow(Traverse):
|
||||
def __init__(self, settings):
|
||||
super(CmdShow, self).__init__(settings, "collect", do_not_start_from_root=False)
|
||||
@@ -49,7 +50,26 @@ class CmdShow(Traverse):
|
||||
def run(self):
|
||||
if self.parsed_args.workflow:
|
||||
# Workflow
|
||||
wf = self.git.get_all_commits('data/eval_auc.txt')
|
||||
target = self.settings.parsed_args.target
|
||||
if not target:
|
||||
target = self.settings.config.target_file
|
||||
|
||||
if not os.path.exists(target):
|
||||
Logger.warn(u'Target is not defined: use empty target')
|
||||
target = ''
|
||||
else:
|
||||
target = open(target).read()
|
||||
Logger.debug(u'Set show workflow target as {}'.format(target))
|
||||
|
||||
if type(target) is list:
|
||||
if len(target) != 1:
|
||||
msg = 'Only one target is expected for show workflow command. Target: {}'
|
||||
Logger.error(msg.format(target))
|
||||
return 1
|
||||
|
||||
target = target[0]
|
||||
|
||||
wf = self.git.get_all_commits(target, self.settings)
|
||||
wf.build_graph()
|
||||
return
|
||||
|
||||
|
||||
+48
-7
@@ -3,6 +3,7 @@ import os
|
||||
from dvc.logger import Logger
|
||||
from dvc.config import Config
|
||||
from dvc.executor import Executor, ExecutorError
|
||||
from dvc.path.data_item import DataItemError
|
||||
from dvc.system import System
|
||||
from dvc.workflow import Workflow, Commit
|
||||
|
||||
@@ -231,8 +232,7 @@ class GitWrapper(GitWrapperI):
|
||||
LOG_SEPARATOR = '|'
|
||||
LOG_FORMAT = ['%h', '%p', '%an', '%ai', '%s']
|
||||
|
||||
@staticmethod
|
||||
def get_all_commits(target):
|
||||
def get_all_commits(self, target, settings):
|
||||
# git log --all --abbrev=7 --pretty=format:"%h|%p|%an|%ai|%s"
|
||||
try:
|
||||
merges_map = GitWrapper.get_merges_map()
|
||||
@@ -249,18 +249,59 @@ class GitWrapper(GitWrapperI):
|
||||
hash, parent_hash, name, date, comment = items
|
||||
|
||||
wf.add_commit(Commit(hash, parent_hash, name, date, comment,
|
||||
GitWrapper.is_target(hash, target)))
|
||||
*self.is_target(hash, target, settings)))
|
||||
|
||||
return wf
|
||||
except ExecutorError:
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def is_target(hash, target):
|
||||
# git show --pretty="" --name-only b6fa39b01
|
||||
def is_target(self, hash, target, settings):
|
||||
git_cmd = ['git', 'show', '--pretty=', '--name-only', hash]
|
||||
files = set(Executor.exec_cmd_only_success(git_cmd).split('\n'))
|
||||
return target in files
|
||||
|
||||
if target in files:
|
||||
try:
|
||||
settings.path_factory.data_item(target)
|
||||
except DataItemError as ex:
|
||||
Logger.warn('Target file {} is not data item: {}'.format(target, ex))
|
||||
return True, None
|
||||
|
||||
try:
|
||||
cmd_symlink_data = ['git', 'show', '{}:{}'.format(hash, target)]
|
||||
symlink_content = Executor.exec_cmd_only_success(cmd_symlink_data).split('\n')
|
||||
except ExecutorError as ex:
|
||||
msg = '[dvc-git] Cannot obtain content of target symbolic file {} with hash {}: {}'
|
||||
Logger.warn(msg.format(target, hash, ex))
|
||||
return True, None
|
||||
|
||||
if not symlink_content or len(symlink_content) != 1:
|
||||
msg = '[dvc-git] Target symbolic file {} with hash {} has wrong format'
|
||||
Logger.warn(msg.format(target, hash))
|
||||
return True, None
|
||||
|
||||
return True, self.target_metric_from_git_history(hash, symlink_content[0], target, settings)
|
||||
|
||||
return False, None
|
||||
|
||||
def target_metric_from_git_history(self, hash, symlink_content, target, settings):
|
||||
cache_rel_to_data = os.path.relpath(settings.config.cache_dir, settings.config.data_dir)
|
||||
common_prefix = os.path.commonprefix([symlink_content, cache_rel_to_data])
|
||||
cache_file_name = symlink_content[len(common_prefix):]
|
||||
if cache_file_name[0] == os.path.sep:
|
||||
cache_file_name = cache_file_name[1:]
|
||||
|
||||
file_name = os.path.join(settings.config.cache_dir, cache_file_name)
|
||||
full_file_name = os.path.join(self.git_dir_abs, file_name)
|
||||
|
||||
if os.path.exists(full_file_name):
|
||||
lines = open(full_file_name).readlines(2)
|
||||
if len(lines) != 1:
|
||||
msg = '[dvc-git] Target file {} with hash {} has wrong format: {} lines were obtained, 1 expected.'
|
||||
Logger.warn(msg.format(target, hash, len(lines)))
|
||||
else:
|
||||
return float(lines[0])
|
||||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_merges_map():
|
||||
|
||||
+27
-3
@@ -13,13 +13,15 @@ class Commit(object):
|
||||
DVC_REPRO_PREFIX = 'DVC repro'
|
||||
COLLAPSED_TEXT = DVC_REPRO_PREFIX + '\n<< collapsed commits >>'
|
||||
|
||||
def __init__(self, hash, parents, name, date, comment, is_target):
|
||||
def __init__(self, hash, parents, name, date, comment, is_target=False, target_metric=None):
|
||||
self._hash = hash
|
||||
self._parent_hashes = set(parents.split())
|
||||
self._name = name
|
||||
self._date = date
|
||||
self._comment = comment
|
||||
self._is_target = is_target
|
||||
self._target_metric = target_metric
|
||||
|
||||
self._is_collapsed = False
|
||||
|
||||
@property
|
||||
@@ -38,9 +40,14 @@ class Commit(object):
|
||||
|
||||
@property
|
||||
def text(self):
|
||||
metric_text = ''
|
||||
if self._is_target and self._target_metric:
|
||||
metric_text = '\nTarget metric: {}'.format(self._target_metric)
|
||||
|
||||
if self._is_collapsed:
|
||||
return self.COLLAPSED_TEXT
|
||||
return self._comment[:self.TEXT_LIMIT] + '\n' + self.hash
|
||||
return self.COLLAPSED_TEXT + metric_text
|
||||
|
||||
return self._comment[:self.TEXT_LIMIT] + '\n' + self.hash + metric_text
|
||||
|
||||
@property
|
||||
def is_repro(self):
|
||||
@@ -49,6 +56,18 @@ class Commit(object):
|
||||
def make_colapsed(self):
|
||||
self._is_collapsed = True
|
||||
|
||||
@property
|
||||
def has_target_metric(self):
|
||||
return self._is_target and self._target_metric is not None
|
||||
|
||||
@property
|
||||
def target_metric(self):
|
||||
return self._target_metric
|
||||
|
||||
def set_target_metric(self, value):
|
||||
self._target_metric = value
|
||||
self._is_target = True
|
||||
|
||||
|
||||
class Workflow(object):
|
||||
def __init__(self, target, merges_map, root='', no_repro_commits=True):
|
||||
@@ -133,6 +152,11 @@ class Workflow(object):
|
||||
child_commit_hashes = self._edges.get(commit.hash)
|
||||
|
||||
if child_commit_hashes and all(self._commits[hash].is_repro for hash in child_commit_hashes):
|
||||
if commit.has_target_metric:
|
||||
for hash in child_commit_hashes:
|
||||
if not self._commits[hash].has_target_metric:
|
||||
self._commits[hash].set_target_metric(commit.target_metric)
|
||||
|
||||
for hash in child_commit_hashes:
|
||||
self._commits[hash].add_parents(commit.parent_hashes)
|
||||
self._commits[hash].remove_parent(commit.hash)
|
||||
|
||||
+24
-8
@@ -5,19 +5,19 @@ from dvc.workflow import Workflow, Commit
|
||||
|
||||
class TestWorkflow(TestCase):
|
||||
def setUp(self):
|
||||
self._commit4 = Commit('4', '3', 'name1', 'today', 'comment4', False)
|
||||
self._commit3 = Commit('3', '2', 'name1', 'today', 'DVC repro-run ...', False)
|
||||
self._commit2 = Commit('2', '1', 'name1', 'today', 'DVC repro-run ...', False)
|
||||
self._commit1 = Commit('1', '', 'name1', 'today', 'comment1', False)
|
||||
self._commit4 = Commit('4', '3', 'name1', 'today', 'comment4')
|
||||
self._commit3 = Commit('3', '2', 'name1', 'today', 'DVC repro-run ...')
|
||||
self._commit2 = Commit('2', '1', 'name1', 'today', 'DVC repro-run ...')
|
||||
self._commit1 = Commit('1', '', 'name1', 'today', 'comment1')
|
||||
|
||||
def test_commits(self):
|
||||
def commits_basic_test(self):
|
||||
self.assertFalse(self._commit1.is_repro)
|
||||
self.assertTrue(self._commit2.is_repro)
|
||||
self.assertTrue(self._commit3.is_repro)
|
||||
self.assertFalse(self._commit4.is_repro)
|
||||
pass
|
||||
|
||||
def test_workflow_basic(self):
|
||||
def workflow_basic_test(self):
|
||||
wf = Workflow('', '')
|
||||
wf.add_commit(self._commit1)
|
||||
wf.add_commit(self._commit2)
|
||||
@@ -32,7 +32,7 @@ class TestWorkflow(TestCase):
|
||||
self.assertEqual(wf._commits['4'].text, self._commit4._comment + '\n' + self._commit4.hash)
|
||||
pass
|
||||
|
||||
def test_collapse(self):
|
||||
def collapse_test(self):
|
||||
wf = Workflow('', '')
|
||||
wf.add_commit(self._commit1)
|
||||
wf.add_commit(self._commit2)
|
||||
@@ -50,7 +50,7 @@ class TestWorkflow(TestCase):
|
||||
self.assertFalse('2' in wf._back_edges)
|
||||
pass
|
||||
|
||||
def test_collapse_at_dead_end(self):
|
||||
def collapse_at_dead_end_test(self):
|
||||
wf = Workflow('', '')
|
||||
wf.add_commit(self._commit1)
|
||||
wf.add_commit(self._commit2)
|
||||
@@ -64,3 +64,19 @@ class TestWorkflow(TestCase):
|
||||
self.assertEqual(wf._commits[self._commit3.hash].text, Commit.COLLAPSED_TEXT)
|
||||
self.assertTrue('2' not in wf._commits)
|
||||
pass
|
||||
|
||||
def collapse_target_metric_commit_test(self):
|
||||
value = 0.812345
|
||||
metric_commit3 = Commit('2', '1', 'name1', 'today', 'DVC repro-run ...', True, value)
|
||||
|
||||
wf = Workflow('', '')
|
||||
wf.add_commit(self._commit1)
|
||||
wf.add_commit(metric_commit3)
|
||||
wf.add_commit(self._commit3)
|
||||
|
||||
self.assertEqual(len(wf._commits), 3)
|
||||
wf.collapse_repro_commits()
|
||||
self.assertEqual(len(wf._commits), 2)
|
||||
|
||||
self.assertEqual(wf._commits['3']._target_metric, value)
|
||||
pass
|
||||
Reference in New Issue
Block a user