diff --git a/dvc/cli.py b/dvc/cli.py index d76907aad..cf96ab7a9 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -15,7 +15,6 @@ from dvc.command.show_workflow import CmdShowWorkflow from dvc.command.instance_create import CmdInstanceCreate from dvc.command.config import CmdConfig from dvc.command.show_pipeline import CmdShowPipeline -from dvc.config import Config from dvc import VERSION @@ -399,6 +398,13 @@ def parse_args(argv=None): action='store_true', default=False, help='Show all commits') + workflow_parser.add_argument( + '-m', + '--max-commits', + metavar='M', + type=int, + default=4, + help='Max commits per graph vertex. 4 by default.') workflow_parser.set_defaults(func=CmdShowWorkflow) diff --git a/dvc/command/import_file.py b/dvc/command/import_file.py index e820cdada..55e552633 100644 --- a/dvc/command/import_file.py +++ b/dvc/command/import_file.py @@ -137,7 +137,7 @@ class CmdImportFile(CmdBase): System.symlink(data_item.symlink_file, data_item.data.relative) state_file = StateFile(StateFile.COMMAND_IMPORT_FILE, - data_item.state.relative, + data_item, self.settings, argv=[input, output], input_files=[], diff --git a/dvc/command/init.py b/dvc/command/init.py index f1a8043c1..96cfc3fd8 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -131,17 +131,16 @@ ProjectName = empty_data_path = os.path.join(self.parsed_args.data_dir, self.EMPTY_FILE_NAME) cache_file_suffix = self.EMPTY_FILE_NAME + '_' + self.EMPTY_FILE_CHECKSUM empty_cache_path = os.path.join(Config.CONFIG_DIR, Config.CACHE_DIR, cache_file_suffix) - empty_state_path = os.path.join(Config.CONFIG_DIR, Config.STATE_DIR, self.EMPTY_FILE_NAME + '.state') open(empty_cache_path, 'w').close() System.symlink(os.path.join('..', empty_cache_path), empty_data_path) StateFile(StateFile.COMMAND_EMPTY_FILE, - empty_state_path, + self.settings.path_factory.data_item(empty_data_path), self.settings, input_files=[], output_files=[], - lock=False).save() + lock=False).save(is_update_target_metrics=False) pass def modify_gitignore(self, config_dir_name, cache_dir_name): diff --git a/dvc/command/lock.py b/dvc/command/lock.py index b2dc9e0be..9b8f8a674 100644 --- a/dvc/command/lock.py +++ b/dvc/command/lock.py @@ -18,7 +18,7 @@ class CmdLock(CmdBase): for file in files: try: data_item = self.settings.path_factory.existing_data_item(file) - state = StateFile.load(data_item.state.relative, self.settings) + state = StateFile.load(data_item, self.settings) if state.locked and target: Logger.warn('Data item {} is already locked'.format(data_item.data.relative)) diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 6f9662cbb..e96f79dfb 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -123,7 +123,7 @@ class ReproChange(object): raise ReproError('data item {} is not symlink'.format(data_item.data.relative)) try: - self._state = StateFile.load(data_item.state.relative, self.git) + self._state = StateFile.load(data_item, self.git) except Exception as ex: raise ReproError('Error: state file "{}" cannot be loaded: {}'. format(data_item.state.relative, ex)) @@ -257,7 +257,7 @@ class ReproChange(object): return result def is_repro_required(self, changed_files, data_item): - state_file = StateFile.load(data_item.state.relative, self._settings) + state_file = StateFile.load(data_item, self._settings) if state_file.locked: Logger.debug(u'Repro is not required for locked data item {}'.format(data_item.data.relative)) return False diff --git a/dvc/command/run.py b/dvc/command/run.py index 489345e49..662d4499c 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -123,7 +123,7 @@ class CmdRun(CmdBase): Logger.debug('Create state file "{}"'.format(data_item.state.relative)) state_file = StateFile(StateFile.COMMAND_RUN, - data_item.state.relative, + data_item, self.settings, input_files_dvc, output_files_dvc, diff --git a/dvc/command/show_pipeline.py b/dvc/command/show_pipeline.py index e8e168bf4..55f0c6e03 100644 --- a/dvc/command/show_pipeline.py +++ b/dvc/command/show_pipeline.py @@ -68,7 +68,7 @@ class CmdShowPipeline(Traverse): def process_file(self, target): data_item = self._get_data_item(target) name = data_item.data.relative - state = StateFile.load(data_item.state.relative, self.git) + state = StateFile.load(data_item, self.git) if name == os.path.join(self.settings.config.data_dir, CmdInit.EMPTY_FILE_NAME): return diff --git a/dvc/command/show_workflow.py b/dvc/command/show_workflow.py index 617242995..80322d99c 100644 --- a/dvc/command/show_workflow.py +++ b/dvc/command/show_workflow.py @@ -15,7 +15,9 @@ class CmdShowWorkflow(CmdBase): Logger.debug(u'Set show workflow target as {}'.format(target)) wf = self.git.get_all_commits(target, self.settings) - wf.build_graph(self.settings.parsed_args.dvc_commits, self.settings.parsed_args.all_commits) + wf.build_graph(self.settings.parsed_args.dvc_commits, + self.settings.parsed_args.all_commits, + self.settings.parsed_args.max_commits) return 0 @property diff --git a/dvc/git_wrapper.py b/dvc/git_wrapper.py index 9bbb0e2ba..8ffe20425 100644 --- a/dvc/git_wrapper.py +++ b/dvc/git_wrapper.py @@ -2,15 +2,22 @@ import os import re from collections import defaultdict +from dvc.exceptions import DvcException from dvc.logger import Logger from dvc.config import Config from dvc.executor import Executor, ExecutorError from dvc.path.data_item import DataItemError +from dvc.state_file import StateFile from dvc.system import System from dvc.graph.workflow import Workflow from dvc.graph.commit import Commit +class GitWrapperError(DvcException): + def __init__(self, msg): + DvcException.__init__(self, msg) + + class GitWrapperI(object): COMMIT_LEN = 7 @@ -222,6 +229,8 @@ class GitWrapper(GitWrapperI): @staticmethod def get_target_commit(fname): + Logger.debug('[dvc-git] Get target commit. Command: ' + 'git log -1 --pretty=format:"%h" {}'.format(fname)) try: commit = Executor.exec_cmd_only_success(['git', 'log', @@ -267,7 +276,7 @@ class GitWrapper(GitWrapperI): hash, parent_hash, name, date, comment = items commit = Commit(hash, parent_hash, name, date, comment, - *self.is_target(hash, target, settings), + *self.was_target_changed(hash, target, settings), branch_tips=branches_multimap.get(hash)) wf.add_commit(commit) @@ -275,70 +284,36 @@ class GitWrapper(GitWrapperI): except ExecutorError: raise - def is_target(self, hash, target, settings): + def was_target_changed(self, hash, target, settings): git_cmd = ['git', 'show', '--pretty=', '--name-only', hash] - files = set(Executor.exec_cmd_only_success(git_cmd).split('\n')) + changed_files = set(Executor.exec_cmd_only_success(git_cmd).split('\n')) - if target in files: - symlink_content = self._get_symlink_content(hash, target, settings) - if symlink_content is not None: - metric = self.target_metric_from_git_history(hash, symlink_content, target, settings) - else: - metric = None + if target not in changed_files: + return False, None - return True, metric + metric = self._read_metric_from_state_file(hash, target, settings) + if metric is None: + return False, None - return False, None + return True, metric - def _get_symlink_content(self, hash, target, settings): + def _read_metric_from_state_file(self, hash, target, settings): try: - settings.path_factory.data_item(target) + data_item = settings.path_factory.data_item(target) except DataItemError as ex: Logger.warn('Target file {} is not data item: {}'.format(target, ex)) return None try: - cmd_symlink_data = ['git', 'show', '{}:{}'.format(hash, target)] - symlink_content = Executor.exec_cmd_only_success(cmd_symlink_data).split('\n') + cmd_corresponded_state_file = ['git', 'show', '{}:{}'.format(hash, data_item.state.relative)] + state_file_content = Executor.exec_cmd_only_success(cmd_corresponded_state_file) except ExecutorError as ex: msg = '[dvc-git] Cannot obtain content of target symbolic file {} with hash {}: {}' Logger.warn(msg.format(target, hash, ex)) return None - if not symlink_content or len(symlink_content) != 1: - msg = '[dvc-git] Target symbolic file {} with hash {} has wrong format' - Logger.warn(msg.format(target, hash)) - return None - - return symlink_content[0] - - def target_metric_from_git_history(self, hash, symlink_content, target, settings): - cache_rel_to_data = os.path.relpath(settings.config.cache_dir, settings.config.data_dir) - common_prefix = os.path.commonprefix([symlink_content, cache_rel_to_data]) - cache_file_name = symlink_content[len(common_prefix):] - if cache_file_name[0] == os.path.sep: - cache_file_name = cache_file_name[1:] - - file_name = os.path.join(settings.config.cache_dir, cache_file_name) - full_file_name = os.path.join(self.git_dir_abs, file_name) - - if not os.path.exists(full_file_name): - return None - - lines = open(full_file_name).readlines(2) - if len(lines) != 1: - msg = '[dvc-git] Target file {} with hash {} has wrong format: {} lines were obtained, 1 expected.' - Logger.warn(msg.format(target, hash, len(lines))) - return None - - # Extract float from string. I.e. from 'AUC: 0.596182' - nums = self.FLOATS_FROM_STRING.findall(lines[0]) - if len(nums) < 1: - msg = '[dvc-git] Unable to parse metrics from \'{}\' file {}' - Logger.warn(msg.format(lines[0], target)) - return None - - return float(nums[0]) + state_file = StateFile.loads(state_file_content, settings) + return state_file.single_target_metric @staticmethod def get_merges_map(): diff --git a/dvc/graph/commit.py b/dvc/graph/commit.py index a52071f98..e008468f4 100644 --- a/dvc/graph/commit.py +++ b/dvc/graph/commit.py @@ -38,13 +38,12 @@ class Commit(object): self._collapsed_commits.append(commit) self._collapsed_commits += commit._collapsed_commits - @property - def text(self): + def text(self, max_commits=100): branch_text = '' if self._branch_tips: branch_text = 'BRANCH TIPS: {}\n'.format(', '.join(self._branch_tips)) - return self._text_metrics_line() + branch_text + self._comments_text + return self._text_metrics_line() + branch_text + self._comments_text(max_commits) @staticmethod def _text_comment(commit): @@ -61,8 +60,7 @@ class Commit(object): else: return comment[:Commit.TEXT_LIMIT-3] + '...' - @property - def _comments_text(self): + def _comments_text(self, max_commits=100): res = [] if not self.is_repro: res.append(self._text_comment(self)) @@ -71,11 +69,16 @@ class Commit(object): if not commit.is_repro: text = self._text_comment(commit) res.append(text) - if len(res) == 0: - return self._text_comment(self) + + was_collapsed = len(res) > max_commits + res = res[:max_commits] max_len = max(map(lambda x: len(x), res)) res_extended_len = map(lambda x: x.ljust(max_len), res) + + if was_collapsed: + res_extended_len.append('<>') + return '\n'.join(res_extended_len) def _text_metrics_line(self): diff --git a/dvc/graph/workflow.py b/dvc/graph/workflow.py index 846b21fdf..56a0c68f7 100644 --- a/dvc/graph/workflow.py +++ b/dvc/graph/workflow.py @@ -104,7 +104,7 @@ class Workflow(object): return [] return [self._commits[h] for h in self._edges[hash]] - def build_graph(self, show_dvc_commits, show_all_commits): + def build_graph(self, show_dvc_commits, show_all_commits, max_commits): self.modify_workflow(show_all_commits, show_dvc_commits) g = nx.DiGraph(name='DVC Workflow', directed=False) @@ -114,7 +114,7 @@ class Workflow(object): g.add_node(hash, attr_dict={ - 'label': commit.text, + 'label': commit.text(max_commits), 'color': self.node_color(commit) } ) diff --git a/dvc/state_file.py b/dvc/state_file.py index 39a6b5be2..0c26458dd 100644 --- a/dvc/state_file.py +++ b/dvc/state_file.py @@ -3,6 +3,7 @@ import ntpath import sys import json import time +import re from dvc.exceptions import DvcException from dvc.path.data_item import NotInDataDirError @@ -18,6 +19,8 @@ class StateFile(object): MAGIC = 'DVC-State' VERSION = '0.1' + FLOATS_FROM_STRING = re.compile(r'[-+]?(?:(?:\d*\.\d+)|(?:\d+\.?))(?:[Ee][+-]?\d+)?') + DVC_PYTHON_FILE_NAME = 'dvc.py' DVC_COMMAND = 'dvc' @@ -39,10 +42,12 @@ class StateFile(object): PARAM_STDOUT = "Stdout" PARAM_STDERR = "Stderr" PARAM_SHELL = "Shell" + PARAM_TARGET_METRICS = 'TargetMetrics' + TARGET_METRICS_SINGLE_METRIC = 'SingleMetric' def __init__(self, command, - file, + data_item, settings, input_files, output_files, @@ -53,8 +58,10 @@ class StateFile(object): stderr=None, created_at=time.strftime('%Y-%m-%d %H:%M:%S %z'), cwd=None, - shell=False): - self.file = file + shell=False, + target_metrics={}): + self.data_item = data_item + self.settings = settings self.input_files = input_files self.output_files = output_files @@ -77,7 +84,51 @@ class StateFile(object): self.cwd = cwd else: self.cwd = self.get_dvc_path() - pass + + self.target_metrics = target_metrics + + @property + def file(self): + return self.data_item.state.relative + + @property + def single_target_metric(self): + res = self.target_metrics.get(self.TARGET_METRICS_SINGLE_METRIC) + if res is None: + return None + return float(res) + + def update_target_metrics(self): + if not self.data_item: + raise StateFileError('Unable to get target metric: data item is not defined') + + data_file = self.data_item.data.relative + if not os.path.exists(data_file): + raise StateFileError('Unable to get target metric: data item does not exist') + + target_metric = StateFile.try_parse_target_metric(data_file) + if target_metric: + self.target_metrics[StateFile.TARGET_METRICS_SINGLE_METRIC] = target_metric + + @staticmethod + def parse_target_metric(file_name): + lines = open(file_name).readlines(2) + if len(lines) != 1: + raise StateFileError('file contains more then one line') + + # Extract float from string. I.e. from 'AUC: 0.596182' + nums = StateFile.FLOATS_FROM_STRING.findall(lines[0]) + if len(nums) < 1: + raise StateFileError("Unable to parse metrics from the first line") + + return float(nums[0]) + + @staticmethod + def try_parse_target_metric(file_name): + try: + return StateFile.parse_target_metric(file_name) + except StateFileError: + return None @property def is_import_file(self): @@ -123,27 +174,39 @@ class StateFile(object): return StateFile.encode_paths([p])[0] @staticmethod - def load(filename, settings): - with open(filename, 'r') as fd: - data = json.load(fd) - - return StateFile(StateFile.decode_path(data.get(StateFile.PARAM_COMMAND)), - filename, + def load_json(json, settings): + return StateFile(StateFile.decode_path(json.get(StateFile.PARAM_COMMAND)), + None, settings, - StateFile.decode_paths(data.get(StateFile.PARAM_INPUT_FILES, [])), - StateFile.decode_paths(data.get(StateFile.PARAM_OUTPUT_FILES, [])), - StateFile.decode_paths(data.get(StateFile.PARAM_CODE_DEPENDENCIES, [])), - data.get(StateFile.PARAM_LOCKED, False), - StateFile.decode_paths(data.get(StateFile.PARAM_ARGV)), - StateFile.decode_path(data.get(StateFile.PARAM_STDOUT)), - StateFile.decode_path(data.get(StateFile.PARAM_STDERR)), - data.get(StateFile.PARAM_CREATED_AT), - StateFile.decode_path(data.get(StateFile.PARAM_CWD)), - data.get(StateFile.PARAM_SHELL, False)) + StateFile.decode_paths(json.get(StateFile.PARAM_INPUT_FILES, [])), + StateFile.decode_paths(json.get(StateFile.PARAM_OUTPUT_FILES, [])), + StateFile.decode_paths(json.get(StateFile.PARAM_CODE_DEPENDENCIES, [])), + json.get(StateFile.PARAM_LOCKED, False), + StateFile.decode_paths(json.get(StateFile.PARAM_ARGV)), + StateFile.decode_path(json.get(StateFile.PARAM_STDOUT)), + StateFile.decode_path(json.get(StateFile.PARAM_STDERR)), + json.get(StateFile.PARAM_CREATED_AT), + StateFile.decode_path(json.get(StateFile.PARAM_CWD)), + json.get(StateFile.PARAM_SHELL, False), + json.get(StateFile.PARAM_TARGET_METRICS, {})) - def save(self): + @staticmethod + def load(data_item, settings): + with open(data_item.state.relative, 'r') as fd: + data = json.load(fd) + return StateFile.load_json(data, settings) + + @staticmethod + def loads(content, settings): + data = json.loads(content) + return StateFile.load_json(data, settings) + + def save(self, is_update_target_metrics=True): argv = self._argv_paths_normalization(self._argv) + if is_update_target_metrics: + self.update_target_metrics() + res = { self.PARAM_COMMAND: self.encode_path(self.command), self.PARAM_TYPE: self.MAGIC, @@ -157,6 +220,7 @@ class StateFile(object): self.PARAM_STDOUT: self.encode_path(self.stdout), self.PARAM_STDERR: self.encode_path(self.stderr), self.PARAM_SHELL: self.shell, + self.PARAM_TARGET_METRICS: self.target_metrics } if self.locked: diff --git a/functest/tutorial_branches.sh b/functest/tutorial_branches.sh index ed66527d2..fb3e22949 100755 --- a/functest/tutorial_branches.sh +++ b/functest/tutorial_branches.sh @@ -38,7 +38,7 @@ dvc run python code/train_model.py data/matrix-train.p 20170426 data/model.p dvc run python code/evaluate.py data/model.p data/matrix-test.p data/eval_auc.txt -dvc config Global.Target data/evel_auc.txt +dvc config Global.Target data/eval_auc.txt git commit -am 'Set Target' cat data/eval_auc.txt @@ -53,6 +53,8 @@ cat data/eval_auc.txt # 0.596182 vi code/train_model.py # estimators=500 git commit -am 'estimators=500' +vi code/train_model.py # n_jobs=6 +git commit -am 'n_jobs=6' dvc repro cat data/eval_auc.txt # 0.619262 diff --git a/tests/test_workflow.py b/tests/test_workflow.py index a940c4fca..50ca6c453 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -35,10 +35,10 @@ class TestWorkflow(TestCase): self.assertEqual(len(wf._commits), 4) - self.assertEqual(wf._commits['1'].text, self._showed_text1) - self.assertEqual(wf._commits['2'].text, self._showed_text2) - self.assertEqual(wf._commits['3'].text, self._showed_text3) - self.assertEqual(wf._commits['4'].text, self._showed_text4) + self.assertEqual(wf._commits['1'].text(), self._showed_text1) + self.assertEqual(wf._commits['2'].text(), self._showed_text2) + self.assertEqual(wf._commits['3'].text(), self._showed_text3) + self.assertEqual(wf._commits['4'].text(), self._showed_text4) pass def collapse_test(self): @@ -51,8 +51,8 @@ class TestWorkflow(TestCase): wf.collapse_commits(CollapseDvcReproCommitsStrategy(wf)) self.assertEqual(len(wf._commits), 3) - self.assertEqual(wf._commits[self._commit1.hash].text, self._showed_text1) - self.assertEqual(wf._commits[self._commit3.hash].text, self._showed_text3_collapsed) + self.assertEqual(wf._commits[self._commit1.hash].text(), self._showed_text1) + self.assertEqual(wf._commits[self._commit3.hash].text(), self._showed_text3_collapsed) self.assertTrue('2' not in wf._commits) self.assertFalse('2' in wf._edges) @@ -69,8 +69,8 @@ class TestWorkflow(TestCase): wf.collapse_commits(CollapseDvcReproCommitsStrategy(wf)) self.assertEqual(len(wf._commits), 2) - self.assertEqual(wf._commits[self._commit1.hash].text, self._showed_text1) - self.assertEqual(wf._commits[self._commit3.hash].text, self._showed_text3) + self.assertEqual(wf._commits[self._commit1.hash].text(), self._showed_text1) + self.assertEqual(wf._commits[self._commit3.hash].text(), self._showed_text3) self.assertTrue('2' not in wf._commits) pass @@ -137,8 +137,8 @@ class TestWorkflow(TestCase): self.assertEqual(len(wf._commits), 2) def is_collapsed_commit_presented(self, commit, collapsed_commit): - self.assertTrue('[' + collapsed_commit.hash + ']' in commit.text) - self.assertTrue(collapsed_commit._comment in commit.text) + self.assertTrue('[' + collapsed_commit.hash + ']' in commit.text()) + self.assertTrue(collapsed_commit._comment in commit.text()) is_collapsed_commit_presented(self, wf._commits['3'], metric_commit3) is_collapsed_commit_presented(self, wf._commits['3'], not_metric_commit2)