Merge pull request #224 from dataversioncontrol/metric_to_state_file2

Extract target metric to state files
This commit is contained in:
Dmitry Petrov
2017-09-18 05:15:29 -07:00
committed by GitHub
14 changed files with 152 additions and 101 deletions
+7 -1
View File
@@ -15,7 +15,6 @@ from dvc.command.show_workflow import CmdShowWorkflow
from dvc.command.instance_create import CmdInstanceCreate
from dvc.command.config import CmdConfig
from dvc.command.show_pipeline import CmdShowPipeline
from dvc.config import Config
from dvc import VERSION
@@ -399,6 +398,13 @@ def parse_args(argv=None):
action='store_true',
default=False,
help='Show all commits')
workflow_parser.add_argument(
'-m',
'--max-commits',
metavar='M',
type=int,
default=4,
help='Max commits per graph vertex. 4 by default.')
workflow_parser.set_defaults(func=CmdShowWorkflow)
+1 -1
View File
@@ -137,7 +137,7 @@ class CmdImportFile(CmdBase):
System.symlink(data_item.symlink_file, data_item.data.relative)
state_file = StateFile(StateFile.COMMAND_IMPORT_FILE,
data_item.state.relative,
data_item,
self.settings,
argv=[input, output],
input_files=[],
+2 -3
View File
@@ -131,17 +131,16 @@ ProjectName =
empty_data_path = os.path.join(self.parsed_args.data_dir, self.EMPTY_FILE_NAME)
cache_file_suffix = self.EMPTY_FILE_NAME + '_' + self.EMPTY_FILE_CHECKSUM
empty_cache_path = os.path.join(Config.CONFIG_DIR, Config.CACHE_DIR, cache_file_suffix)
empty_state_path = os.path.join(Config.CONFIG_DIR, Config.STATE_DIR, self.EMPTY_FILE_NAME + '.state')
open(empty_cache_path, 'w').close()
System.symlink(os.path.join('..', empty_cache_path), empty_data_path)
StateFile(StateFile.COMMAND_EMPTY_FILE,
empty_state_path,
self.settings.path_factory.data_item(empty_data_path),
self.settings,
input_files=[],
output_files=[],
lock=False).save()
lock=False).save(is_update_target_metrics=False)
pass
def modify_gitignore(self, config_dir_name, cache_dir_name):
+1 -1
View File
@@ -18,7 +18,7 @@ class CmdLock(CmdBase):
for file in files:
try:
data_item = self.settings.path_factory.existing_data_item(file)
state = StateFile.load(data_item.state.relative, self.settings)
state = StateFile.load(data_item, self.settings)
if state.locked and target:
Logger.warn('Data item {} is already locked'.format(data_item.data.relative))
+2 -2
View File
@@ -123,7 +123,7 @@ class ReproChange(object):
raise ReproError('data item {} is not symlink'.format(data_item.data.relative))
try:
self._state = StateFile.load(data_item.state.relative, self.git)
self._state = StateFile.load(data_item, self.git)
except Exception as ex:
raise ReproError('Error: state file "{}" cannot be loaded: {}'.
format(data_item.state.relative, ex))
@@ -257,7 +257,7 @@ class ReproChange(object):
return result
def is_repro_required(self, changed_files, data_item):
state_file = StateFile.load(data_item.state.relative, self._settings)
state_file = StateFile.load(data_item, self._settings)
if state_file.locked:
Logger.debug(u'Repro is not required for locked data item {}'.format(data_item.data.relative))
return False
+1 -1
View File
@@ -123,7 +123,7 @@ class CmdRun(CmdBase):
Logger.debug('Create state file "{}"'.format(data_item.state.relative))
state_file = StateFile(StateFile.COMMAND_RUN,
data_item.state.relative,
data_item,
self.settings,
input_files_dvc,
output_files_dvc,
+1 -1
View File
@@ -68,7 +68,7 @@ class CmdShowPipeline(Traverse):
def process_file(self, target):
data_item = self._get_data_item(target)
name = data_item.data.relative
state = StateFile.load(data_item.state.relative, self.git)
state = StateFile.load(data_item, self.git)
if name == os.path.join(self.settings.config.data_dir, CmdInit.EMPTY_FILE_NAME):
return
+3 -1
View File
@@ -15,7 +15,9 @@ class CmdShowWorkflow(CmdBase):
Logger.debug(u'Set show workflow target as {}'.format(target))
wf = self.git.get_all_commits(target, self.settings)
wf.build_graph(self.settings.parsed_args.dvc_commits, self.settings.parsed_args.all_commits)
wf.build_graph(self.settings.parsed_args.dvc_commits,
self.settings.parsed_args.all_commits,
self.settings.parsed_args.max_commits)
return 0
@property
+24 -49
View File
@@ -2,15 +2,22 @@ import os
import re
from collections import defaultdict
from dvc.exceptions import DvcException
from dvc.logger import Logger
from dvc.config import Config
from dvc.executor import Executor, ExecutorError
from dvc.path.data_item import DataItemError
from dvc.state_file import StateFile
from dvc.system import System
from dvc.graph.workflow import Workflow
from dvc.graph.commit import Commit
class GitWrapperError(DvcException):
def __init__(self, msg):
DvcException.__init__(self, msg)
class GitWrapperI(object):
COMMIT_LEN = 7
@@ -222,6 +229,8 @@ class GitWrapper(GitWrapperI):
@staticmethod
def get_target_commit(fname):
Logger.debug('[dvc-git] Get target commit. Command: '
'git log -1 --pretty=format:"%h" {}'.format(fname))
try:
commit = Executor.exec_cmd_only_success(['git',
'log',
@@ -267,7 +276,7 @@ class GitWrapper(GitWrapperI):
hash, parent_hash, name, date, comment = items
commit = Commit(hash, parent_hash, name, date, comment,
*self.is_target(hash, target, settings),
*self.was_target_changed(hash, target, settings),
branch_tips=branches_multimap.get(hash))
wf.add_commit(commit)
@@ -275,70 +284,36 @@ class GitWrapper(GitWrapperI):
except ExecutorError:
raise
def is_target(self, hash, target, settings):
def was_target_changed(self, hash, target, settings):
git_cmd = ['git', 'show', '--pretty=', '--name-only', hash]
files = set(Executor.exec_cmd_only_success(git_cmd).split('\n'))
changed_files = set(Executor.exec_cmd_only_success(git_cmd).split('\n'))
if target in files:
symlink_content = self._get_symlink_content(hash, target, settings)
if symlink_content is not None:
metric = self.target_metric_from_git_history(hash, symlink_content, target, settings)
else:
metric = None
if target not in changed_files:
return False, None
return True, metric
metric = self._read_metric_from_state_file(hash, target, settings)
if metric is None:
return False, None
return False, None
return True, metric
def _get_symlink_content(self, hash, target, settings):
def _read_metric_from_state_file(self, hash, target, settings):
try:
settings.path_factory.data_item(target)
data_item = settings.path_factory.data_item(target)
except DataItemError as ex:
Logger.warn('Target file {} is not data item: {}'.format(target, ex))
return None
try:
cmd_symlink_data = ['git', 'show', '{}:{}'.format(hash, target)]
symlink_content = Executor.exec_cmd_only_success(cmd_symlink_data).split('\n')
cmd_corresponded_state_file = ['git', 'show', '{}:{}'.format(hash, data_item.state.relative)]
state_file_content = Executor.exec_cmd_only_success(cmd_corresponded_state_file)
except ExecutorError as ex:
msg = '[dvc-git] Cannot obtain content of target symbolic file {} with hash {}: {}'
Logger.warn(msg.format(target, hash, ex))
return None
if not symlink_content or len(symlink_content) != 1:
msg = '[dvc-git] Target symbolic file {} with hash {} has wrong format'
Logger.warn(msg.format(target, hash))
return None
return symlink_content[0]
def target_metric_from_git_history(self, hash, symlink_content, target, settings):
cache_rel_to_data = os.path.relpath(settings.config.cache_dir, settings.config.data_dir)
common_prefix = os.path.commonprefix([symlink_content, cache_rel_to_data])
cache_file_name = symlink_content[len(common_prefix):]
if cache_file_name[0] == os.path.sep:
cache_file_name = cache_file_name[1:]
file_name = os.path.join(settings.config.cache_dir, cache_file_name)
full_file_name = os.path.join(self.git_dir_abs, file_name)
if not os.path.exists(full_file_name):
return None
lines = open(full_file_name).readlines(2)
if len(lines) != 1:
msg = '[dvc-git] Target file {} with hash {} has wrong format: {} lines were obtained, 1 expected.'
Logger.warn(msg.format(target, hash, len(lines)))
return None
# Extract float from string. I.e. from 'AUC: 0.596182'
nums = self.FLOATS_FROM_STRING.findall(lines[0])
if len(nums) < 1:
msg = '[dvc-git] Unable to parse metrics from \'{}\' file {}'
Logger.warn(msg.format(lines[0], target))
return None
return float(nums[0])
state_file = StateFile.loads(state_file_content, settings)
return state_file.single_target_metric
@staticmethod
def get_merges_map():
+10 -7
View File
@@ -38,13 +38,12 @@ class Commit(object):
self._collapsed_commits.append(commit)
self._collapsed_commits += commit._collapsed_commits
@property
def text(self):
def text(self, max_commits=100):
branch_text = ''
if self._branch_tips:
branch_text = 'BRANCH TIPS: {}\n'.format(', '.join(self._branch_tips))
return self._text_metrics_line() + branch_text + self._comments_text
return self._text_metrics_line() + branch_text + self._comments_text(max_commits)
@staticmethod
def _text_comment(commit):
@@ -61,8 +60,7 @@ class Commit(object):
else:
return comment[:Commit.TEXT_LIMIT-3] + '...'
@property
def _comments_text(self):
def _comments_text(self, max_commits=100):
res = []
if not self.is_repro:
res.append(self._text_comment(self))
@@ -71,11 +69,16 @@ class Commit(object):
if not commit.is_repro:
text = self._text_comment(commit)
res.append(text)
if len(res) == 0:
return self._text_comment(self)
was_collapsed = len(res) > max_commits
res = res[:max_commits]
max_len = max(map(lambda x: len(x), res))
res_extended_len = map(lambda x: x.ljust(max_len), res)
if was_collapsed:
res_extended_len.append('<<Collapsed commits>>')
return '\n'.join(res_extended_len)
def _text_metrics_line(self):
+2 -2
View File
@@ -104,7 +104,7 @@ class Workflow(object):
return []
return [self._commits[h] for h in self._edges[hash]]
def build_graph(self, show_dvc_commits, show_all_commits):
def build_graph(self, show_dvc_commits, show_all_commits, max_commits):
self.modify_workflow(show_all_commits, show_dvc_commits)
g = nx.DiGraph(name='DVC Workflow', directed=False)
@@ -114,7 +114,7 @@ class Workflow(object):
g.add_node(hash,
attr_dict={
'label': commit.text,
'label': commit.text(max_commits),
'color': self.node_color(commit)
}
)
+85 -21
View File
@@ -3,6 +3,7 @@ import ntpath
import sys
import json
import time
import re
from dvc.exceptions import DvcException
from dvc.path.data_item import NotInDataDirError
@@ -18,6 +19,8 @@ class StateFile(object):
MAGIC = 'DVC-State'
VERSION = '0.1'
FLOATS_FROM_STRING = re.compile(r'[-+]?(?:(?:\d*\.\d+)|(?:\d+\.?))(?:[Ee][+-]?\d+)?')
DVC_PYTHON_FILE_NAME = 'dvc.py'
DVC_COMMAND = 'dvc'
@@ -39,10 +42,12 @@ class StateFile(object):
PARAM_STDOUT = "Stdout"
PARAM_STDERR = "Stderr"
PARAM_SHELL = "Shell"
PARAM_TARGET_METRICS = 'TargetMetrics'
TARGET_METRICS_SINGLE_METRIC = 'SingleMetric'
def __init__(self,
command,
file,
data_item,
settings,
input_files,
output_files,
@@ -53,8 +58,10 @@ class StateFile(object):
stderr=None,
created_at=time.strftime('%Y-%m-%d %H:%M:%S %z'),
cwd=None,
shell=False):
self.file = file
shell=False,
target_metrics={}):
self.data_item = data_item
self.settings = settings
self.input_files = input_files
self.output_files = output_files
@@ -77,7 +84,51 @@ class StateFile(object):
self.cwd = cwd
else:
self.cwd = self.get_dvc_path()
pass
self.target_metrics = target_metrics
@property
def file(self):
return self.data_item.state.relative
@property
def single_target_metric(self):
res = self.target_metrics.get(self.TARGET_METRICS_SINGLE_METRIC)
if res is None:
return None
return float(res)
def update_target_metrics(self):
if not self.data_item:
raise StateFileError('Unable to get target metric: data item is not defined')
data_file = self.data_item.data.relative
if not os.path.exists(data_file):
raise StateFileError('Unable to get target metric: data item does not exist')
target_metric = StateFile.try_parse_target_metric(data_file)
if target_metric:
self.target_metrics[StateFile.TARGET_METRICS_SINGLE_METRIC] = target_metric
@staticmethod
def parse_target_metric(file_name):
lines = open(file_name).readlines(2)
if len(lines) != 1:
raise StateFileError('file contains more then one line')
# Extract float from string. I.e. from 'AUC: 0.596182'
nums = StateFile.FLOATS_FROM_STRING.findall(lines[0])
if len(nums) < 1:
raise StateFileError("Unable to parse metrics from the first line")
return float(nums[0])
@staticmethod
def try_parse_target_metric(file_name):
try:
return StateFile.parse_target_metric(file_name)
except StateFileError:
return None
@property
def is_import_file(self):
@@ -123,27 +174,39 @@ class StateFile(object):
return StateFile.encode_paths([p])[0]
@staticmethod
def load(filename, settings):
with open(filename, 'r') as fd:
data = json.load(fd)
return StateFile(StateFile.decode_path(data.get(StateFile.PARAM_COMMAND)),
filename,
def load_json(json, settings):
return StateFile(StateFile.decode_path(json.get(StateFile.PARAM_COMMAND)),
None,
settings,
StateFile.decode_paths(data.get(StateFile.PARAM_INPUT_FILES, [])),
StateFile.decode_paths(data.get(StateFile.PARAM_OUTPUT_FILES, [])),
StateFile.decode_paths(data.get(StateFile.PARAM_CODE_DEPENDENCIES, [])),
data.get(StateFile.PARAM_LOCKED, False),
StateFile.decode_paths(data.get(StateFile.PARAM_ARGV)),
StateFile.decode_path(data.get(StateFile.PARAM_STDOUT)),
StateFile.decode_path(data.get(StateFile.PARAM_STDERR)),
data.get(StateFile.PARAM_CREATED_AT),
StateFile.decode_path(data.get(StateFile.PARAM_CWD)),
data.get(StateFile.PARAM_SHELL, False))
StateFile.decode_paths(json.get(StateFile.PARAM_INPUT_FILES, [])),
StateFile.decode_paths(json.get(StateFile.PARAM_OUTPUT_FILES, [])),
StateFile.decode_paths(json.get(StateFile.PARAM_CODE_DEPENDENCIES, [])),
json.get(StateFile.PARAM_LOCKED, False),
StateFile.decode_paths(json.get(StateFile.PARAM_ARGV)),
StateFile.decode_path(json.get(StateFile.PARAM_STDOUT)),
StateFile.decode_path(json.get(StateFile.PARAM_STDERR)),
json.get(StateFile.PARAM_CREATED_AT),
StateFile.decode_path(json.get(StateFile.PARAM_CWD)),
json.get(StateFile.PARAM_SHELL, False),
json.get(StateFile.PARAM_TARGET_METRICS, {}))
def save(self):
@staticmethod
def load(data_item, settings):
with open(data_item.state.relative, 'r') as fd:
data = json.load(fd)
return StateFile.load_json(data, settings)
@staticmethod
def loads(content, settings):
data = json.loads(content)
return StateFile.load_json(data, settings)
def save(self, is_update_target_metrics=True):
argv = self._argv_paths_normalization(self._argv)
if is_update_target_metrics:
self.update_target_metrics()
res = {
self.PARAM_COMMAND: self.encode_path(self.command),
self.PARAM_TYPE: self.MAGIC,
@@ -157,6 +220,7 @@ class StateFile(object):
self.PARAM_STDOUT: self.encode_path(self.stdout),
self.PARAM_STDERR: self.encode_path(self.stderr),
self.PARAM_SHELL: self.shell,
self.PARAM_TARGET_METRICS: self.target_metrics
}
if self.locked:
+3 -1
View File
@@ -38,7 +38,7 @@ dvc run python code/train_model.py data/matrix-train.p 20170426 data/model.p
dvc run python code/evaluate.py data/model.p data/matrix-test.p data/eval_auc.txt
dvc config Global.Target data/evel_auc.txt
dvc config Global.Target data/eval_auc.txt
git commit -am 'Set Target'
cat data/eval_auc.txt
@@ -53,6 +53,8 @@ cat data/eval_auc.txt
# 0.596182
vi code/train_model.py # estimators=500
git commit -am 'estimators=500'
vi code/train_model.py # n_jobs=6
git commit -am 'n_jobs=6'
dvc repro
cat data/eval_auc.txt
# 0.619262
+10 -10
View File
@@ -35,10 +35,10 @@ class TestWorkflow(TestCase):
self.assertEqual(len(wf._commits), 4)
self.assertEqual(wf._commits['1'].text, self._showed_text1)
self.assertEqual(wf._commits['2'].text, self._showed_text2)
self.assertEqual(wf._commits['3'].text, self._showed_text3)
self.assertEqual(wf._commits['4'].text, self._showed_text4)
self.assertEqual(wf._commits['1'].text(), self._showed_text1)
self.assertEqual(wf._commits['2'].text(), self._showed_text2)
self.assertEqual(wf._commits['3'].text(), self._showed_text3)
self.assertEqual(wf._commits['4'].text(), self._showed_text4)
pass
def collapse_test(self):
@@ -51,8 +51,8 @@ class TestWorkflow(TestCase):
wf.collapse_commits(CollapseDvcReproCommitsStrategy(wf))
self.assertEqual(len(wf._commits), 3)
self.assertEqual(wf._commits[self._commit1.hash].text, self._showed_text1)
self.assertEqual(wf._commits[self._commit3.hash].text, self._showed_text3_collapsed)
self.assertEqual(wf._commits[self._commit1.hash].text(), self._showed_text1)
self.assertEqual(wf._commits[self._commit3.hash].text(), self._showed_text3_collapsed)
self.assertTrue('2' not in wf._commits)
self.assertFalse('2' in wf._edges)
@@ -69,8 +69,8 @@ class TestWorkflow(TestCase):
wf.collapse_commits(CollapseDvcReproCommitsStrategy(wf))
self.assertEqual(len(wf._commits), 2)
self.assertEqual(wf._commits[self._commit1.hash].text, self._showed_text1)
self.assertEqual(wf._commits[self._commit3.hash].text, self._showed_text3)
self.assertEqual(wf._commits[self._commit1.hash].text(), self._showed_text1)
self.assertEqual(wf._commits[self._commit3.hash].text(), self._showed_text3)
self.assertTrue('2' not in wf._commits)
pass
@@ -137,8 +137,8 @@ class TestWorkflow(TestCase):
self.assertEqual(len(wf._commits), 2)
def is_collapsed_commit_presented(self, commit, collapsed_commit):
self.assertTrue('[' + collapsed_commit.hash + ']' in commit.text)
self.assertTrue(collapsed_commit._comment in commit.text)
self.assertTrue('[' + collapsed_commit.hash + ']' in commit.text())
self.assertTrue(collapsed_commit._comment in commit.text())
is_collapsed_commit_presented(self, wf._commits['3'], metric_commit3)
is_collapsed_commit_presented(self, wf._commits['3'], not_metric_commit2)