From 9436430040d4b06cba56a9fb89364e701ff60e3d Mon Sep 17 00:00:00 2001 From: Dmitry Date: Mon, 18 Sep 2017 04:47:39 -0700 Subject: [PATCH] Read metrics from state file --- dvc/command/init.py | 2 +- dvc/git_wrapper.py | 79 +++++++++---------------------------- dvc/state_file.py | 96 +++++++++++++++++++++++++++++++++++---------- 3 files changed, 94 insertions(+), 83 deletions(-) diff --git a/dvc/command/init.py b/dvc/command/init.py index bf8ee12bb..96cfc3fd8 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -140,7 +140,7 @@ ProjectName = self.settings, input_files=[], output_files=[], - lock=False).save() + lock=False).save(is_update_target_metrics=False) pass def modify_gitignore(self, config_dir_name, cache_dir_name): diff --git a/dvc/git_wrapper.py b/dvc/git_wrapper.py index f7c886bc1..8ffe20425 100644 --- a/dvc/git_wrapper.py +++ b/dvc/git_wrapper.py @@ -7,6 +7,7 @@ from dvc.logger import Logger from dvc.config import Config from dvc.executor import Executor, ExecutorError from dvc.path.data_item import DataItemError +from dvc.state_file import StateFile from dvc.system import System from dvc.graph.workflow import Workflow from dvc.graph.commit import Commit @@ -228,6 +229,8 @@ class GitWrapper(GitWrapperI): @staticmethod def get_target_commit(fname): + Logger.debug('[dvc-git] Get target commit. Command: ' + 'git log -1 --pretty=format:"%h" {}'.format(fname)) try: commit = Executor.exec_cmd_only_success(['git', 'log', @@ -273,7 +276,7 @@ class GitWrapper(GitWrapperI): hash, parent_hash, name, date, comment = items commit = Commit(hash, parent_hash, name, date, comment, - *self.is_target(hash, target, settings), + *self.was_target_changed(hash, target, settings), branch_tips=branches_multimap.get(hash)) wf.add_commit(commit) @@ -281,82 +284,36 @@ class GitWrapper(GitWrapperI): except ExecutorError: raise - def is_target(self, hash, target, settings): + def was_target_changed(self, hash, target, settings): git_cmd = ['git', 'show', '--pretty=', '--name-only', hash] - files = set(Executor.exec_cmd_only_success(git_cmd).split('\n')) + changed_files = set(Executor.exec_cmd_only_success(git_cmd).split('\n')) - if target in files: - symlink_content = self._get_symlink_content(hash, target, settings) - if symlink_content is not None: - metric = self.target_metric_from_git_history(hash, symlink_content, target, settings) - else: - metric = None + if target not in changed_files: + return False, None - return True, metric + metric = self._read_metric_from_state_file(hash, target, settings) + if metric is None: + return False, None - return False, None + return True, metric - def _get_symlink_content(self, hash, target, settings): + def _read_metric_from_state_file(self, hash, target, settings): try: - settings.path_factory.data_item(target) + data_item = settings.path_factory.data_item(target) except DataItemError as ex: Logger.warn('Target file {} is not data item: {}'.format(target, ex)) return None try: - cmd_symlink_data = ['git', 'show', '{}:{}'.format(hash, target)] - symlink_content = Executor.exec_cmd_only_success(cmd_symlink_data).split('\n') + cmd_corresponded_state_file = ['git', 'show', '{}:{}'.format(hash, data_item.state.relative)] + state_file_content = Executor.exec_cmd_only_success(cmd_corresponded_state_file) except ExecutorError as ex: msg = '[dvc-git] Cannot obtain content of target symbolic file {} with hash {}: {}' Logger.warn(msg.format(target, hash, ex)) return None - if not symlink_content or len(symlink_content) != 1: - msg = '[dvc-git] Target symbolic file {} with hash {} has wrong format' - Logger.warn(msg.format(target, hash)) - return None - - return symlink_content[0] - - def target_metric_from_git_history(self, hash, symlink_content, target, settings): - cache_rel_to_data = os.path.relpath(settings.config.cache_dir, settings.config.data_dir) - common_prefix = os.path.commonprefix([symlink_content, cache_rel_to_data]) - cache_file_name = symlink_content[len(common_prefix):] - if cache_file_name[0] == os.path.sep: - cache_file_name = cache_file_name[1:] - - file_name = os.path.join(settings.config.cache_dir, cache_file_name) - full_file_name = os.path.join(self.git_dir_abs, file_name) - - if not os.path.exists(full_file_name): - return None - - try: - return self.parse_target_metric(full_file_name) - except GitWrapperError as ex: - msg = '[dvc-git] Unable to parse target file {} from Git commit {}: {}' - Logger.warn(msg.format(target, hash, ex)) - return None - - @staticmethod - def parse_target_metric(file_name): - lines = open(file_name).readlines(2) - if len(lines) != 1: - raise GitWrapperError('file contains more then one line') - - # Extract float from string. I.e. from 'AUC: 0.596182' - nums = GitWrapper.FLOATS_FROM_STRING.findall(lines[0]) - if len(nums) < 1: - raise GitWrapperError("Unable to parse metrics from the first line") - - return float(nums[0]) - - @staticmethod - def try_parse_target_metric(file_name): - try: - return GitWrapper.parse_target_metric(file_name) - except GitWrapperError: - return None + state_file = StateFile.loads(state_file_content, settings) + return state_file.single_target_metric @staticmethod def get_merges_map(): diff --git a/dvc/state_file.py b/dvc/state_file.py index 53fa24526..0c26458dd 100644 --- a/dvc/state_file.py +++ b/dvc/state_file.py @@ -3,9 +3,9 @@ import ntpath import sys import json import time +import re from dvc.exceptions import DvcException -from dvc.git_wrapper import GitWrapper from dvc.path.data_item import NotInDataDirError from dvc.system import System @@ -19,6 +19,8 @@ class StateFile(object): MAGIC = 'DVC-State' VERSION = '0.1' + FLOATS_FROM_STRING = re.compile(r'[-+]?(?:(?:\d*\.\d+)|(?:\d+\.?))(?:[Ee][+-]?\d+)?') + DVC_PYTHON_FILE_NAME = 'dvc.py' DVC_COMMAND = 'dvc' @@ -40,8 +42,8 @@ class StateFile(object): PARAM_STDOUT = "Stdout" PARAM_STDERR = "Stderr" PARAM_SHELL = "Shell" - PARAM_TARGET_METRICS = 'target_metrics' - TARGET_METRICS_SINGLE_METRIC = 'single_metric' + PARAM_TARGET_METRICS = 'TargetMetrics' + TARGET_METRICS_SINGLE_METRIC = 'SingleMetric' def __init__(self, command, @@ -56,9 +58,10 @@ class StateFile(object): stderr=None, created_at=time.strftime('%Y-%m-%d %H:%M:%S %z'), cwd=None, - shell=False): + shell=False, + target_metrics={}): self.data_item = data_item - self.file = self.data_item.state.relative + self.settings = settings self.input_files = input_files self.output_files = output_files @@ -82,12 +85,51 @@ class StateFile(object): else: self.cwd = self.get_dvc_path() - self.target_metrics = {} + self.target_metrics = target_metrics - target_metric = GitWrapper.try_parse_target_metric(self.data_item.data.relative) + @property + def file(self): + return self.data_item.state.relative + + @property + def single_target_metric(self): + res = self.target_metrics.get(self.TARGET_METRICS_SINGLE_METRIC) + if res is None: + return None + return float(res) + + def update_target_metrics(self): + if not self.data_item: + raise StateFileError('Unable to get target metric: data item is not defined') + + data_file = self.data_item.data.relative + if not os.path.exists(data_file): + raise StateFileError('Unable to get target metric: data item does not exist') + + target_metric = StateFile.try_parse_target_metric(data_file) if target_metric: self.target_metrics[StateFile.TARGET_METRICS_SINGLE_METRIC] = target_metric + @staticmethod + def parse_target_metric(file_name): + lines = open(file_name).readlines(2) + if len(lines) != 1: + raise StateFileError('file contains more then one line') + + # Extract float from string. I.e. from 'AUC: 0.596182' + nums = StateFile.FLOATS_FROM_STRING.findall(lines[0]) + if len(nums) < 1: + raise StateFileError("Unable to parse metrics from the first line") + + return float(nums[0]) + + @staticmethod + def try_parse_target_metric(file_name): + try: + return StateFile.parse_target_metric(file_name) + except StateFileError: + return None + @property def is_import_file(self): return self.command == self.COMMAND_IMPORT_FILE @@ -131,28 +173,40 @@ class StateFile(object): def encode_path(p): return StateFile.encode_paths([p])[0] + @staticmethod + def load_json(json, settings): + return StateFile(StateFile.decode_path(json.get(StateFile.PARAM_COMMAND)), + None, + settings, + StateFile.decode_paths(json.get(StateFile.PARAM_INPUT_FILES, [])), + StateFile.decode_paths(json.get(StateFile.PARAM_OUTPUT_FILES, [])), + StateFile.decode_paths(json.get(StateFile.PARAM_CODE_DEPENDENCIES, [])), + json.get(StateFile.PARAM_LOCKED, False), + StateFile.decode_paths(json.get(StateFile.PARAM_ARGV)), + StateFile.decode_path(json.get(StateFile.PARAM_STDOUT)), + StateFile.decode_path(json.get(StateFile.PARAM_STDERR)), + json.get(StateFile.PARAM_CREATED_AT), + StateFile.decode_path(json.get(StateFile.PARAM_CWD)), + json.get(StateFile.PARAM_SHELL, False), + json.get(StateFile.PARAM_TARGET_METRICS, {})) + @staticmethod def load(data_item, settings): with open(data_item.state.relative, 'r') as fd: data = json.load(fd) + return StateFile.load_json(data, settings) - return StateFile(StateFile.decode_path(data.get(StateFile.PARAM_COMMAND)), - data_item, - settings, - StateFile.decode_paths(data.get(StateFile.PARAM_INPUT_FILES, [])), - StateFile.decode_paths(data.get(StateFile.PARAM_OUTPUT_FILES, [])), - StateFile.decode_paths(data.get(StateFile.PARAM_CODE_DEPENDENCIES, [])), - data.get(StateFile.PARAM_LOCKED, False), - StateFile.decode_paths(data.get(StateFile.PARAM_ARGV)), - StateFile.decode_path(data.get(StateFile.PARAM_STDOUT)), - StateFile.decode_path(data.get(StateFile.PARAM_STDERR)), - data.get(StateFile.PARAM_CREATED_AT), - StateFile.decode_path(data.get(StateFile.PARAM_CWD)), - data.get(StateFile.PARAM_SHELL, False)) + @staticmethod + def loads(content, settings): + data = json.loads(content) + return StateFile.load_json(data, settings) - def save(self): + def save(self, is_update_target_metrics=True): argv = self._argv_paths_normalization(self._argv) + if is_update_target_metrics: + self.update_target_metrics() + res = { self.PARAM_COMMAND: self.encode_path(self.command), self.PARAM_TYPE: self.MAGIC,