From 9f21c6a51c2863f3e605eb5771f86c134cb37802 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 12 Nov 2017 22:36:02 -0800 Subject: [PATCH 1/6] dvc checkout witout git checkout. Close #292 --- dvc/command/checkout.py | 26 ++-- dvc/command/repro.py | 2 +- dvc/command/repro.py__ | 319 ++++++++++++++++++++++++++++++++++++++++ dvc/git_wrapper.py | 14 ++ dvc/path/factory.py | 19 ++- 5 files changed, 359 insertions(+), 21 deletions(-) create mode 100644 dvc/command/repro.py__ diff --git a/dvc/command/checkout.py b/dvc/command/checkout.py index 5e9fa3ab9..8baa65dd3 100644 --- a/dvc/command/checkout.py +++ b/dvc/command/checkout.py @@ -35,16 +35,12 @@ class CmdCheckout(CmdBase): def run(self): with DvcLock(self.is_locker, self.git): - curr_commit = self.git.curr_branch_or_commit - prev_items = [] try: - self.git.checkout_previous() - prev_items = self.settings.path_factory.all_existing_data_items() + states = self.git.state_files_for_previous_commit() + prev_items = self.settings.path_factory.data_items_from_states(states, existing=False) except Exception as ex: - Logger.error(u'Unable to get '.format(ex)) + Logger.error(u'Unable to get data files from previous commit'.format(ex)) return 1 - finally: - self.git.checkout(curr_commit) curr_items = self.settings.path_factory.all_existing_data_items() self.checkout(curr_items) @@ -55,10 +51,14 @@ class CmdCheckout(CmdBase): @staticmethod def remove_files(removed_items_set): for item in removed_items_set: - Logger.info(u'Remove \'{}\''.format(item.data.relative)) - os.remove(item.data.relative) - dir = os.path.dirname(item.data.relative) - if not os.listdir(dir): - Logger.info(u'Remove directory \'{}\''.format(dir)) - os.removedirs(dir) + path = item.data.relative + if not os.path.exists(path): + print(u'Remove \'{}\' - file does not exist'.format(path)) + else: + Logger.info(u'Remove \'{}\''.format(path)) + os.remove(path) + dir = os.path.dirname(path) + if not os.listdir(dir): + Logger.info(u'Remove directory \'{}\''.format(dir)) + os.removedirs(dir) pass diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 990cbb953..ff64210b3 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -156,7 +156,7 @@ class ReproChange(object): Logger.debug('Removing output file {} before reproduction.'.format(output_dvc)) try: - data_item = self.cmd_obj.settings.path_factory.existing_data_item_from_dvc_path(output_dvc) + data_item = self.cmd_obj.settings.path_factory.data_item_from_dvc_path(output_dvc) os.remove(data_item.data.relative) except Exception as ex: msg = 'Data item {} cannot be removed before reproduction: {}' diff --git a/dvc/command/repro.py__ b/dvc/command/repro.py__ new file mode 100644 index 000000000..990cbb953 --- /dev/null +++ b/dvc/command/repro.py__ @@ -0,0 +1,319 @@ +import os +import copy + +from dvc.command.base import DvcLock +from dvc.command.import_file import CmdImportFile +from dvc.command.run import CmdRun +from dvc.logger import Logger +from dvc.exceptions import DvcException +from dvc.path.data_item import DataDirError +from dvc.state_file import StateFile +from dvc.system import System + + +class ReproError(DvcException): + def __init__(self, msg): + DvcException.__init__(self, 'Run error: {}'.format(msg)) + + +class CmdRepro(CmdRun): + def __init__(self, settings): + super(CmdRepro, self).__init__(settings) + + self._code = [] + pass + + @property + def code_dependencies(self): + return self._code + + @property + def lock(self): + return True + + @property + def declaration_input_data_items(self): + return [] + + @property + def declaration_output_data_items(self): + return [] + + def run(self): + with DvcLock(self.is_locker, self.git): + recursive = not self.parsed_args.single_item + targets = [] + + if self.parsed_args.target: + targets += self.parsed_args.target + else: + target = self.settings.config._config['Global'].get('Target', None) + if not target or len(target) == 0: + Logger.error('Reproduction target is not defined. ' + + 'Specify data file or set target by ' + + '`dvc config global.target target` command.') + return 1 + targets += [target] + + return self.repro_target(targets, recursive, self.parsed_args.force) + pass + + def repro_target(self, target, recursive, force): + if not self.no_git_actions and not self.git.is_ready_to_go(): + return 1 + + data_item_list, external_files_names = self.settings.path_factory.to_data_items(target) + if external_files_names: + Logger.error('Files from outside of the repo could not be reproduced: {}'. + format(' '.join(external_files_names))) + return 1 + + if self.repro_data_items(data_item_list, recursive, force): + return 0 + return 1 + + def repro_data_items(self, data_item_list, recursive, force): + error = False + changed = False + + for data_item in data_item_list: + try: + target_commit = self.git.get_target_commit(data_item.state.relative) + if target_commit is None: + msg = 'Data item "{}" cannot be reproduced: cannot obtain commit hashsum' + Logger.warn(msg.format(data_item.data.relative)) + continue + + globally_changed_files = self.git.get_changed_files(target_commit) + Logger.debug('Repro globally changed files: {}'.format(', '.join(globally_changed_files))) + changed_files = set() + change = ReproChange(data_item, self, globally_changed_files, recursive, force) + if change.reproduce(changed_files): + changed = True + Logger.info(u'Data item "{}" was reproduced.'.format( + data_item.data.relative + )) + else: + Logger.info(u'Reproduction is not required for data item "{}".'.format( + data_item.data.relative + )) + except ReproError as err: + Logger.error('Error in reproducing data item {}: {}'.format( + data_item.data.relative, str(err) + )) + error = True + break + + if error and not self.no_git_actions: + Logger.error('Errors occurred. One or more repro cmd was not successful.') + self.not_committed_changes_warning() + + return changed and not error + + +class ReproChange(object): + def __init__(self, data_item, cmd_obj, globally_changed_files, recursive, force): + self._data_item = data_item + self.git = cmd_obj.git + self._cmd_obj = cmd_obj + self._globally_changed_files = globally_changed_files + self._recursive = recursive + self._force = force + + try: + self._state = StateFile.load(data_item, self.git) + except Exception as ex: + raise ReproError('Error: state file "{}" cannot be loaded: {}'. + format(data_item.state.relative, ex)) + + if not self.state.argv: + raise ReproError('Error: parameter {} is not defined in state file "{}"'. + format(StateFile.PARAM_ARGV, data_item.state.relative)) + if len(self.state.argv) < 1: + raise ReproError('Error: reproducible cmd in state file "{}" is too short'. + format(self.state.file)) + + self._settings = copy.copy(self._cmd_obj.settings) + self._settings.set_args(self.state.argv) + pass + + def is_cache_exists(self): + path = System.realpath(self._data_item.data.relative) + return os.path.exists(path) + + @property + def cmd_obj(self): + return self._cmd_obj + + @property + def state(self): + return self._state + + def reproduce_data_item(self, changed_files): + Logger.debug('Reproducing data item {}.'.format(self._data_item.data.dvc)) + + for output_dvc in self._state.output_files: + Logger.debug('Removing output file {} before reproduction.'.format(output_dvc)) + + try: + data_item = self.cmd_obj.settings.path_factory.existing_data_item_from_dvc_path(output_dvc) + os.remove(data_item.data.relative) + except Exception as ex: + msg = 'Data item {} cannot be removed before reproduction: {}' + Logger.error(msg.format(output_dvc, ex)) + + changed_files.add(output_dvc) + + if self.state.is_import_file: + Logger.debug('Reproducing data item {}. Re-import cmd: {}'.format( + self._data_item.data.relative, ' '.join(self.state.argv))) + + if len(self.state.argv) != 2: + msg = 'Data item "{}" cannot be re-imported because of arguments number {} is incorrect. Argv: {}' + raise ReproError(msg.format(self._data_item.data.relative, len(self.state.argv), self.state.argv)) + + input = self.state.argv[0] + output = self.state.argv[1] + + cmd = CmdImportFile(self._settings) + cmd.set_git_action(True) + cmd.set_locker(False) + + Logger.info(u'Reproducing import command: {}'.format(output)) + if cmd.import_and_commit_if_needed(input, output, lock=True, + check_if_ready=False, + is_repro=True) != 0: + raise ReproError('Import command reproduction failed') + return True + elif self.state.is_run: + cmd = CmdRun(self._settings) + cmd.set_git_action(True) + cmd.set_locker(False) + + Logger.info('Reproducing run command for data item {}. Args: {}'.format( + self._data_item.data.relative, ' '.join(self.state.argv))) + + data_items_from_args, not_data_items_from_args = self.cmd_obj.argv_files_by_type(self.state.argv) + if cmd.run_and_commit_if_needed(self.state.argv, + data_items_from_args, + not_data_items_from_args, + self.state.stdout, + self.state.stderr, + self.state.shell, + [], + [], + [], + False, + check_if_ready=False, + is_repro=True) != 0: + raise ReproError('Run command reproduction failed') + return True + else: + # Ignore EMPTY_FILE command + pass + pass + + def reproduce(self, changed_files): + Logger.debug('Reproduce data item {}. recursive={}, force={}'.format( + self._data_item.data.relative, self._recursive, self._force)) + + if self.state.locked: + Logger.debug('Data item {} is not reproducible'.format(self._data_item.data.relative)) + return False + + if self.is_repro_required(changed_files, self._data_item): + if self._data_item.data.dvc not in changed_files: + Logger.debug('Data item {} is going to be reproduced'.format(self._data_item.data.relative)) + self.reproduce_data_item(changed_files) + changed_files.add(self._data_item.data.dvc) + return True + else: + msg = 'Data item {} is not going to be reproduced because it is already reproduced' + Logger.debug(msg.format(self._data_item.data.relative)) + else: + Logger.debug('Data item {} is up to date'.format(self._data_item.data.relative)) + return False + pass + + def were_dependencies_changed(self, changed_files, data_item_dvc): + result = False + + for data_item in self.dependencies: + change = ReproChange(data_item, self._cmd_obj, self._globally_changed_files, self._recursive, self._force) + if change.reproduce(changed_files): + result = True + Logger.debug(u'Repro data item {}: dependency {} was changed'.format( + data_item_dvc, data_item.data.dvc)) + elif data_item.state.dvc in self._globally_changed_files: + dep_state = StateFile.load(data_item, self._settings) + if dep_state.command != StateFile.COMMAND_IMPORT_FILE: + msg = u'Repro data item {}: dependency {} was not changed but the state file was changed' + Logger.debug(msg.format(data_item_dvc, data_item.data.dvc)) + result = True + else: + Logger.debug(u'Repro data item {}: skip dependency state file change {} due to data source command') + else: + msg = u'Repro data item {}: dependency {} was not changed' + Logger.debug(msg.format(data_item_dvc, data_item.data.dvc)) + + return result + + def is_repro_required(self, changed_files, data_item): + state_file = StateFile.load(data_item, self._settings) + if state_file.locked: + Logger.debug(u'Repro is not required for locked data item {}'.format(data_item.data.relative)) + return False + + is_dependency_check_required = self._recursive + + if not is_dependency_check_required and not self.is_cache_exists(): + is_dependency_check_required = True + Logger.info(u'Reproduction {}. Force dependency check since cache file is missing.'.format( + self._data_item.data.relative)) + + if is_dependency_check_required: + if self.were_dependencies_changed(changed_files, data_item.data.dvc): + self.log_repro_reason(u'input dependencies were changed') + return True + + if self._force: + self.log_repro_reason(u'it was forced') + return True + + if not self.is_cache_exists(): + self.log_repro_reason(u'cache file is missing.') + return True + + if self.were_sources_changed(self._globally_changed_files): + self.log_repro_reason(u'sources were changed') + return True + + return False + + def log_repro_reason(self, reason): + msg = u'Repro is required for data item {} because of {}' + Logger.debug(msg.format(self._data_item.data.relative, reason)) + + def were_sources_changed(self, globally_changed_files): + were_sources_changed = self.git.were_files_changed( + self.state.code_dependencies, + self._settings.path_factory, + globally_changed_files + ) + return were_sources_changed + + @property + def dependencies(self): + dependency_data_items = [] + for input_file in self.state.input_files: + try: + data_item = self._settings.path_factory.data_item(input_file) + except DataDirError: + raise ReproError(u'The dependency file "{}" is not a data item'.format(input_file)) + except Exception as ex: + raise ReproError(u'Unable to reproduced the dependency file "{}": {}'.format( + input_file, ex)) + + dependency_data_items.append(data_item) + + return dependency_data_items diff --git a/dvc/git_wrapper.py b/dvc/git_wrapper.py index 29e590dc3..c1d27fbb9 100644 --- a/dvc/git_wrapper.py +++ b/dvc/git_wrapper.py @@ -423,3 +423,17 @@ class GitWrapper(GitWrapperI): for (hash, branch) in items: result[hash].append(branch) return result + + @staticmethod + def git_prev_commit(): + cmd = 'git check-ref-format --branch @{-1}' + return Executor.exec_cmd_only_success(cmd.split()).strip() + + @staticmethod + def state_files_for_previous_commit(): + commit = GitWrapper.git_prev_commit() + Logger.debug(u'[dvc-git] Previous commit {}'.format(commit)) + + git_cmd = u'git ls-tree --name-only -r {}'.format(commit) + lines = Executor.exec_cmd_only_success(git_cmd.split()).split('\n') + return filter(lambda s: s.startswith('.dvc/state'), map(unicode.split, lines)) diff --git a/dvc/path/factory.py b/dvc/path/factory.py index f79ddd0f6..e1cdbd7cd 100644 --- a/dvc/path/factory.py +++ b/dvc/path/factory.py @@ -21,9 +21,12 @@ class PathFactory(object): def stated_data_item(self, state, data_file): return StatedDataItem(state, data_file, self._git, self._config) - def existing_data_item_from_dvc_path(self, dvc_path): + def data_item_from_dvc_path(self, dvc_path, existing=True): path = Path.from_dvc_path(dvc_path, self._git) - return self.existing_data_item(path.relative) + if existing: + return self.existing_data_item(path.relative) + else: + return self.data_item(path.relative) def is_data_item(self, fname): data = os.path.relpath(os.path.realpath(fname), self._git.git_dir_abs) @@ -50,7 +53,6 @@ class PathFactory(object): return result, externally_created_files def all_existing_data_items(self, subdir='.'): - items = [] states = [] for root, dirs, files in os.walk(os.path.join(self._config.state_dir, subdir)): @@ -63,8 +65,11 @@ class PathFactory(object): states.append(path) - for state in states: - data = os.path.relpath(state, self._config.state_dir)[:-len(DataItem.STATE_FILE_SUFFIX)] - items.append(self.existing_data_item_from_dvc_path(data)) + return self.data_items_from_states(states) - return items + def data_items_from_states(self, states, existing=True): + return map(lambda s: self.data_item_from_dvc_path(self.state_path_to_dvc_path(s), existing), + states) + + def state_path_to_dvc_path(self, state): + return os.path.relpath(state, self._config.state_dir)[:-len(DataItem.STATE_FILE_SUFFIX)] From 410a05c93711f471217e6d8edaaf515895da70df Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 12 Nov 2017 22:37:35 -0800 Subject: [PATCH 2/6] rm repro.py__ --- dvc/command/repro.py__ | 319 ----------------------------------------- 1 file changed, 319 deletions(-) delete mode 100644 dvc/command/repro.py__ diff --git a/dvc/command/repro.py__ b/dvc/command/repro.py__ deleted file mode 100644 index 990cbb953..000000000 --- a/dvc/command/repro.py__ +++ /dev/null @@ -1,319 +0,0 @@ -import os -import copy - -from dvc.command.base import DvcLock -from dvc.command.import_file import CmdImportFile -from dvc.command.run import CmdRun -from dvc.logger import Logger -from dvc.exceptions import DvcException -from dvc.path.data_item import DataDirError -from dvc.state_file import StateFile -from dvc.system import System - - -class ReproError(DvcException): - def __init__(self, msg): - DvcException.__init__(self, 'Run error: {}'.format(msg)) - - -class CmdRepro(CmdRun): - def __init__(self, settings): - super(CmdRepro, self).__init__(settings) - - self._code = [] - pass - - @property - def code_dependencies(self): - return self._code - - @property - def lock(self): - return True - - @property - def declaration_input_data_items(self): - return [] - - @property - def declaration_output_data_items(self): - return [] - - def run(self): - with DvcLock(self.is_locker, self.git): - recursive = not self.parsed_args.single_item - targets = [] - - if self.parsed_args.target: - targets += self.parsed_args.target - else: - target = self.settings.config._config['Global'].get('Target', None) - if not target or len(target) == 0: - Logger.error('Reproduction target is not defined. ' + - 'Specify data file or set target by ' + - '`dvc config global.target target` command.') - return 1 - targets += [target] - - return self.repro_target(targets, recursive, self.parsed_args.force) - pass - - def repro_target(self, target, recursive, force): - if not self.no_git_actions and not self.git.is_ready_to_go(): - return 1 - - data_item_list, external_files_names = self.settings.path_factory.to_data_items(target) - if external_files_names: - Logger.error('Files from outside of the repo could not be reproduced: {}'. - format(' '.join(external_files_names))) - return 1 - - if self.repro_data_items(data_item_list, recursive, force): - return 0 - return 1 - - def repro_data_items(self, data_item_list, recursive, force): - error = False - changed = False - - for data_item in data_item_list: - try: - target_commit = self.git.get_target_commit(data_item.state.relative) - if target_commit is None: - msg = 'Data item "{}" cannot be reproduced: cannot obtain commit hashsum' - Logger.warn(msg.format(data_item.data.relative)) - continue - - globally_changed_files = self.git.get_changed_files(target_commit) - Logger.debug('Repro globally changed files: {}'.format(', '.join(globally_changed_files))) - changed_files = set() - change = ReproChange(data_item, self, globally_changed_files, recursive, force) - if change.reproduce(changed_files): - changed = True - Logger.info(u'Data item "{}" was reproduced.'.format( - data_item.data.relative - )) - else: - Logger.info(u'Reproduction is not required for data item "{}".'.format( - data_item.data.relative - )) - except ReproError as err: - Logger.error('Error in reproducing data item {}: {}'.format( - data_item.data.relative, str(err) - )) - error = True - break - - if error and not self.no_git_actions: - Logger.error('Errors occurred. One or more repro cmd was not successful.') - self.not_committed_changes_warning() - - return changed and not error - - -class ReproChange(object): - def __init__(self, data_item, cmd_obj, globally_changed_files, recursive, force): - self._data_item = data_item - self.git = cmd_obj.git - self._cmd_obj = cmd_obj - self._globally_changed_files = globally_changed_files - self._recursive = recursive - self._force = force - - try: - self._state = StateFile.load(data_item, self.git) - except Exception as ex: - raise ReproError('Error: state file "{}" cannot be loaded: {}'. - format(data_item.state.relative, ex)) - - if not self.state.argv: - raise ReproError('Error: parameter {} is not defined in state file "{}"'. - format(StateFile.PARAM_ARGV, data_item.state.relative)) - if len(self.state.argv) < 1: - raise ReproError('Error: reproducible cmd in state file "{}" is too short'. - format(self.state.file)) - - self._settings = copy.copy(self._cmd_obj.settings) - self._settings.set_args(self.state.argv) - pass - - def is_cache_exists(self): - path = System.realpath(self._data_item.data.relative) - return os.path.exists(path) - - @property - def cmd_obj(self): - return self._cmd_obj - - @property - def state(self): - return self._state - - def reproduce_data_item(self, changed_files): - Logger.debug('Reproducing data item {}.'.format(self._data_item.data.dvc)) - - for output_dvc in self._state.output_files: - Logger.debug('Removing output file {} before reproduction.'.format(output_dvc)) - - try: - data_item = self.cmd_obj.settings.path_factory.existing_data_item_from_dvc_path(output_dvc) - os.remove(data_item.data.relative) - except Exception as ex: - msg = 'Data item {} cannot be removed before reproduction: {}' - Logger.error(msg.format(output_dvc, ex)) - - changed_files.add(output_dvc) - - if self.state.is_import_file: - Logger.debug('Reproducing data item {}. Re-import cmd: {}'.format( - self._data_item.data.relative, ' '.join(self.state.argv))) - - if len(self.state.argv) != 2: - msg = 'Data item "{}" cannot be re-imported because of arguments number {} is incorrect. Argv: {}' - raise ReproError(msg.format(self._data_item.data.relative, len(self.state.argv), self.state.argv)) - - input = self.state.argv[0] - output = self.state.argv[1] - - cmd = CmdImportFile(self._settings) - cmd.set_git_action(True) - cmd.set_locker(False) - - Logger.info(u'Reproducing import command: {}'.format(output)) - if cmd.import_and_commit_if_needed(input, output, lock=True, - check_if_ready=False, - is_repro=True) != 0: - raise ReproError('Import command reproduction failed') - return True - elif self.state.is_run: - cmd = CmdRun(self._settings) - cmd.set_git_action(True) - cmd.set_locker(False) - - Logger.info('Reproducing run command for data item {}. Args: {}'.format( - self._data_item.data.relative, ' '.join(self.state.argv))) - - data_items_from_args, not_data_items_from_args = self.cmd_obj.argv_files_by_type(self.state.argv) - if cmd.run_and_commit_if_needed(self.state.argv, - data_items_from_args, - not_data_items_from_args, - self.state.stdout, - self.state.stderr, - self.state.shell, - [], - [], - [], - False, - check_if_ready=False, - is_repro=True) != 0: - raise ReproError('Run command reproduction failed') - return True - else: - # Ignore EMPTY_FILE command - pass - pass - - def reproduce(self, changed_files): - Logger.debug('Reproduce data item {}. recursive={}, force={}'.format( - self._data_item.data.relative, self._recursive, self._force)) - - if self.state.locked: - Logger.debug('Data item {} is not reproducible'.format(self._data_item.data.relative)) - return False - - if self.is_repro_required(changed_files, self._data_item): - if self._data_item.data.dvc not in changed_files: - Logger.debug('Data item {} is going to be reproduced'.format(self._data_item.data.relative)) - self.reproduce_data_item(changed_files) - changed_files.add(self._data_item.data.dvc) - return True - else: - msg = 'Data item {} is not going to be reproduced because it is already reproduced' - Logger.debug(msg.format(self._data_item.data.relative)) - else: - Logger.debug('Data item {} is up to date'.format(self._data_item.data.relative)) - return False - pass - - def were_dependencies_changed(self, changed_files, data_item_dvc): - result = False - - for data_item in self.dependencies: - change = ReproChange(data_item, self._cmd_obj, self._globally_changed_files, self._recursive, self._force) - if change.reproduce(changed_files): - result = True - Logger.debug(u'Repro data item {}: dependency {} was changed'.format( - data_item_dvc, data_item.data.dvc)) - elif data_item.state.dvc in self._globally_changed_files: - dep_state = StateFile.load(data_item, self._settings) - if dep_state.command != StateFile.COMMAND_IMPORT_FILE: - msg = u'Repro data item {}: dependency {} was not changed but the state file was changed' - Logger.debug(msg.format(data_item_dvc, data_item.data.dvc)) - result = True - else: - Logger.debug(u'Repro data item {}: skip dependency state file change {} due to data source command') - else: - msg = u'Repro data item {}: dependency {} was not changed' - Logger.debug(msg.format(data_item_dvc, data_item.data.dvc)) - - return result - - def is_repro_required(self, changed_files, data_item): - state_file = StateFile.load(data_item, self._settings) - if state_file.locked: - Logger.debug(u'Repro is not required for locked data item {}'.format(data_item.data.relative)) - return False - - is_dependency_check_required = self._recursive - - if not is_dependency_check_required and not self.is_cache_exists(): - is_dependency_check_required = True - Logger.info(u'Reproduction {}. Force dependency check since cache file is missing.'.format( - self._data_item.data.relative)) - - if is_dependency_check_required: - if self.were_dependencies_changed(changed_files, data_item.data.dvc): - self.log_repro_reason(u'input dependencies were changed') - return True - - if self._force: - self.log_repro_reason(u'it was forced') - return True - - if not self.is_cache_exists(): - self.log_repro_reason(u'cache file is missing.') - return True - - if self.were_sources_changed(self._globally_changed_files): - self.log_repro_reason(u'sources were changed') - return True - - return False - - def log_repro_reason(self, reason): - msg = u'Repro is required for data item {} because of {}' - Logger.debug(msg.format(self._data_item.data.relative, reason)) - - def were_sources_changed(self, globally_changed_files): - were_sources_changed = self.git.were_files_changed( - self.state.code_dependencies, - self._settings.path_factory, - globally_changed_files - ) - return were_sources_changed - - @property - def dependencies(self): - dependency_data_items = [] - for input_file in self.state.input_files: - try: - data_item = self._settings.path_factory.data_item(input_file) - except DataDirError: - raise ReproError(u'The dependency file "{}" is not a data item'.format(input_file)) - except Exception as ex: - raise ReproError(u'Unable to reproduced the dependency file "{}": {}'.format( - input_file, ex)) - - dependency_data_items.append(data_item) - - return dependency_data_items From 62e8b751ba22cfe06c10d2d1c2520b3a7bce1ab8 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 12 Nov 2017 22:41:46 -0800 Subject: [PATCH 3/6] Rename STATE_DIR to STATE_DIR_NAME --- dvc/command/init.py | 4 ++-- dvc/config.py | 8 ++++---- tests/basic_env.py | 10 +++++----- tests/test_cmd_run.py | 10 +++++----- tests/test_data_file_obj.py | 20 ++++++++++---------- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/dvc/command/init.py b/dvc/command/init.py index 0af442928..b5e3b5e40 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -85,8 +85,8 @@ ProjectName = return 1 config_dir_path = self.get_not_existing_path(Config.CONFIG_DIR) - cache_dir_path = self.get_not_existing_path(Config.CONFIG_DIR, Config.CACHE_DIR) - state_dir_path = self.get_not_existing_path(Config.CONFIG_DIR, Config.STATE_DIR) + cache_dir_path = self.get_not_existing_path(Config.CONFIG_DIR, Config.CACHE_DIR_NAME) + state_dir_path = self.get_not_existing_path(Config.CONFIG_DIR, Config.STATE_DIR_NAME) conf_file_name = self.get_not_existing_conf_file_name() diff --git a/dvc/config.py b/dvc/config.py index cff3b41e0..3f301a79b 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -19,8 +19,8 @@ class ConfigI(object): CONFIG_DIR = '.dvc' TARGET_FILE_DEFAULT = 'target' CONFIG = 'config' - STATE_DIR = 'state' - CACHE_DIR = 'cache' + STATE_DIR_NAME = 'state' + CACHE_DIR_NAME = 'cache' def __init__(self, cloud=None, conf_parser=None): self._conf_parser = None @@ -35,12 +35,12 @@ class ConfigI(object): @property def cache_dir(self): """ Directory with cached data files """ - return os.path.join(self.CONFIG_DIR, self.CACHE_DIR) + return os.path.join(self.CONFIG_DIR, self.CACHE_DIR_NAME) @property def state_dir(self): """ Directory with state files """ - return os.path.join(self.CONFIG_DIR, self.STATE_DIR) + return os.path.join(self.CONFIG_DIR, self.STATE_DIR_NAME) @property def cloud(self): diff --git a/tests/basic_env.py b/tests/basic_env.py index ebfdab9fd..1d01f1c0d 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -34,8 +34,8 @@ class BasicEnvironment(TestCase): os.makedirs(self._test_git_dir) data_dir = os.path.join(self._test_git_dir, 'data') - cache_dir = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR) - state_dir = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, 'data') + cache_dir = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME) + state_dir = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data') os.makedirs(data_dir) os.makedirs(cache_dir) @@ -103,13 +103,13 @@ class DirHierarchyEnvironment(BasicEnvironment): def crate_data_item(self, data_file, cache_file=True, content='random text'): file_result = os.path.join('data', data_file) - state_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, file_result + DataItem.STATE_FILE_SUFFIX) + state_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, file_result + DataItem.STATE_FILE_SUFFIX) dummy_md5 = os.path.basename(data_file) + DataItem.CACHE_FILE_SEP + self._commit self.create_content_file(state_result, '{"Md5" : "' + dummy_md5 + '", "Command" : "run", "Cwd" : "dir"}') if cache_file: - cache_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, dummy_md5) + cache_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, dummy_md5) self.create_content_file(cache_result, content) System.hardlink(cache_result, file_result) @@ -121,4 +121,4 @@ class DirHierarchyEnvironment(BasicEnvironment): def create_dirs(self, dir): os.makedirs(os.path.join('data', dir)) - os.makedirs(os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, 'data', dir)) + os.makedirs(os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', dir)) diff --git a/tests/test_cmd_run.py b/tests/test_cmd_run.py index 8c2772390..cda6a5a9f 100644 --- a/tests/test_cmd_run.py +++ b/tests/test_cmd_run.py @@ -23,8 +23,8 @@ class RunBasicTest(TestCase): os.chdir(self.test_dir) os.mkdir('data') os.mkdir(ConfigI.CONFIG_DIR) - os.mkdir(os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR)) - os.mkdir(os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR)) + os.mkdir(os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME)) + os.mkdir(os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME)) self.init_git_repo() self.git = GitWrapper() @@ -37,7 +37,7 @@ class RunBasicTest(TestCase): def init_git_repo(self): Executor.exec_cmd_only_success(['git', 'init']) - msg = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR) + msg = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME) msg += '\n' msg += os.path.join(ConfigI.CONFIG_DIR, '.' + ConfigI.CONFIG + '.lock') self.create_file('.gitignore', msg) @@ -72,8 +72,8 @@ class RunTwoFilesBase(RunBasicTest): stdout=self.file_name1, stderr=self.file_name2) - self.state_file_name1 = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, self.file_name1 + DataItem.STATE_FILE_SUFFIX) - self.state_file_name2 = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, self.file_name2 + DataItem.STATE_FILE_SUFFIX) + self.state_file_name1 = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, self.file_name1 + DataItem.STATE_FILE_SUFFIX) + self.state_file_name2 = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, self.file_name2 + DataItem.STATE_FILE_SUFFIX) self.state_file1 = None self.state_file2 = None diff --git a/tests/test_data_file_obj.py b/tests/test_data_file_obj.py index cc7ea821a..1b8c748b6 100644 --- a/tests/test_data_file_obj.py +++ b/tests/test_data_file_obj.py @@ -16,7 +16,7 @@ class TestDataFileObjBasic(BasicEnvironment): config = ConfigI() file = os.path.join('data', 'file.txt') - self.cache_file_full_path = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, 'file.txt_ad45ba8') + self.cache_file_full_path = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'file.txt_ad45ba8') self._data_path = DataItem(file, git, config, cache_file=self.cache_file_full_path) pass @@ -28,7 +28,7 @@ class TestDataFileObjBasic(BasicEnvironment): self.assertEqual(self._data_path.cache.abs, self.cache_file_full_path) def test_state_file(self): - state_file_full_path = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, 'data', 'file.txt.state') + state_file_full_path = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', 'file.txt.state') self.assertEqual(self._data_path.state.abs, state_file_full_path) @@ -40,7 +40,7 @@ class TestDataItemWithGivenCache(BasicEnvironment): self._config = ConfigI('data') self._file = os.path.join('data', 'file.txt') - self._cache = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, 'file.txt_abcd') + self._cache = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'file.txt_abcd') self._data_item = DataItem(self._file, self._git, self._config, self._cache) pass @@ -58,7 +58,7 @@ class TestDeepDataItemWithGivenCache(BasicEnvironment): self._config = ConfigI('data') self._file = os.path.join('data', 'dir1', 'file.txt') - self._cache = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, 'dir1', 'file.txt_abcd') + self._cache = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'dir1', 'file.txt_abcd') self._data_item = DataItem(self._file, self._git, self._config, self._cache) pass @@ -78,8 +78,8 @@ class TestPathFactory(BasicEnvironment): self.data_file = os.path.join('data', 'file.txt') dummy_md5 = 'fsymlinc.txt' - self.cache_file = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, dummy_md5) - self.state_file = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, 'data', 'file.txt' + DataItem.STATE_FILE_SUFFIX) + self.cache_file = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, dummy_md5) + self.state_file = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', 'file.txt' + DataItem.STATE_FILE_SUFFIX) with open(self.cache_file, 'w+') as fd: fd.write('some text') @@ -134,7 +134,7 @@ class TestDataPathInDataDir(BasicEnvironment): self.path_factory = PathFactory(git, config) deep_path = os.path.join(self.data_dir, 'dir1', 'd2', 'file.txt') - self.cache_file = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, 'dir1', 'd2', 'file.txt_eeeff8f') + self.cache_file = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'dir1', 'd2', 'file.txt_eeeff8f') self.data_path = self.path_factory.data_item(deep_path, cache_file=self.cache_file) pass @@ -146,7 +146,7 @@ class TestDataPathInDataDir(BasicEnvironment): self.assertEqual(self.data_path.cache.abs, self.cache_file) def test_state_file(self): - target = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, 'da', 'dir1', 'd2', 'file.txt.state') + target = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'da', 'dir1', 'd2', 'file.txt.state') self.assertEqual(self.data_path.state.abs, target) @@ -162,7 +162,7 @@ class TestDataFileObjLongPath(BasicEnvironment): self._config = ConfigI('data') self.path_factory = PathFactory(self._git, self._config) - self.cache_file = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, 'file1.txt_123ed8') + self.cache_file = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'file1.txt_123ed8') self.data_path = self.path_factory.data_item(os.path.join('..', '..', 'data', 'file1.txt'), cache_file=self.cache_file) pass @@ -174,7 +174,7 @@ class TestDataFileObjLongPath(BasicEnvironment): self.assertEqual(self.data_path.cache.abs, self.cache_file) def test_state_file(self): - target = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, 'data', 'file1.txt.state') + target = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', 'file1.txt.state') self.assertEqual(self.data_path.state.abs, target) From 8d6c1347800d6919cda43fa5a63eb58e56bac2db Mon Sep 17 00:00:00 2001 From: Dmitry Date: Mon, 13 Nov 2017 00:38:44 -0800 Subject: [PATCH 4/6] Extract cache_dir\state_dir as a CongigI class variable --- dvc/command/gc.py | 6 ++++-- dvc/command/import_file.py | 4 ++-- dvc/command/merge.py | 5 +++-- dvc/command/run.py | 2 +- dvc/config.py | 21 ++++++++++++--------- dvc/path/data_item.py | 14 +++++++------- dvc/path/factory.py | 8 +++++--- 7 files changed, 34 insertions(+), 26 deletions(-) diff --git a/dvc/command/gc.py b/dvc/command/gc.py index 88142e264..ed39afc6a 100644 --- a/dvc/command/gc.py +++ b/dvc/command/gc.py @@ -4,6 +4,8 @@ from dvc.command.base import DvcLock from dvc.command.traverse import Traverse from dvc.logger import Logger from dvc.path.data_item import DataItemError +from dvc.config import ConfigI + class CmdGC(Traverse): def __init__(self, settings): @@ -16,8 +18,8 @@ class CmdGC(Traverse): Logger.error('Failed to collect used cache') return 1 - for cache in os.listdir(self.settings.config.cache_dir): - fname = os.path.join(self.settings.config.cache_dir, cache) + for cache in os.listdir(ConfigI.CACHE_DIR): + fname = os.path.join(ConfigI.CACHE_DIR, cache) if fname in self.clist: continue os.remove(fname) diff --git a/dvc/command/import_file.py b/dvc/command/import_file.py index 9bcb3d304..ff9e53126 100644 --- a/dvc/command/import_file.py +++ b/dvc/command/import_file.py @@ -107,8 +107,8 @@ class CmdImportFile(CmdBase): raise ImportFileError('Output file directory "{}" does not exists'.format( os.path.dirname(data_item.data.relative))) - if not os.path.exists(data_item.cache_dir): - os.makedirs(data_item.cache_dir) + if not os.path.exists(data_item.cache_dir_abs): + os.makedirs(data_item.cache_dir_abs) data_targets.append((input, data_item)) diff --git a/dvc/command/merge.py b/dvc/command/merge.py index 35795bd0d..ed30906e9 100644 --- a/dvc/command/merge.py +++ b/dvc/command/merge.py @@ -2,6 +2,7 @@ import os import json from dvc.command.base import CmdBase, DvcLock +from dvc.config import ConfigI from dvc.logger import Logger from dvc.state_file import StateFile from dvc.path.data_item import DataItem @@ -20,8 +21,8 @@ class CmdMerge(CmdBase): dlist = [] flist = self.git.get_last_merge_changed_files() for fname in flist: - if fname.startswith(self.settings.config.state_dir) and fname.endswith(DataItem.STATE_FILE_SUFFIX): - data = os.path.relpath(fname, self.settings.config.state_dir)[:-len(DataItem.STATE_FILE_SUFFIX)] + if fname.startswith(ConfigI.STATE_DIR) and fname.endswith(DataItem.STATE_FILE_SUFFIX): + data = os.path.relpath(fname, ConfigI.STATE_DIR)[:-len(DataItem.STATE_FILE_SUFFIX)] dlist.append(data) return dlist diff --git a/dvc/command/run.py b/dvc/command/run.py index 498d1b2a5..30c081277 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -117,7 +117,7 @@ class CmdRun(CmdBase): result = [] for data_item in repo_change.changed_data_items: Logger.debug('Move output file "{}" to cache dir "{}" and create a hardlink'.format( - data_item.data.relative, data_item.cache_dir)) + data_item.data.relative, data_item.cache_dir_abs)) data_item.move_data_to_cache() Logger.debug('Create state file "{}"'.format(data_item.state.relative)) diff --git a/dvc/config.py b/dvc/config.py index 3f301a79b..af9b97ca7 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -22,6 +22,9 @@ class ConfigI(object): STATE_DIR_NAME = 'state' CACHE_DIR_NAME = 'cache' + CACHE_DIR = os.path.join(CONFIG_DIR, CACHE_DIR_NAME) + STATE_DIR = os.path.join(CONFIG_DIR, STATE_DIR_NAME) + def __init__(self, cloud=None, conf_parser=None): self._conf_parser = None self._cloud = None @@ -32,15 +35,15 @@ class ConfigI(object): self._cloud = cloud self._conf_parser = conf_parser - @property - def cache_dir(self): - """ Directory with cached data files """ - return os.path.join(self.CONFIG_DIR, self.CACHE_DIR_NAME) - - @property - def state_dir(self): - """ Directory with state files """ - return os.path.join(self.CONFIG_DIR, self.STATE_DIR_NAME) + # @classmethod + # def CACHE_DIR(self): + # """ Directory with cached data files """ + # return os.path.join(self.CONFIG_DIR, self.CACHE_DIR_NAME) + # + # @classmethod + # def STATE_DIR(self): + # """ Directory with state files """ + # return os.path.join(self.CONFIG_DIR, self.STATE_DIR_NAME) @property def cloud(self): diff --git a/dvc/path/data_item.py b/dvc/path/data_item.py index ccfba0c2c..e1e9df32d 100644 --- a/dvc/path/data_item.py +++ b/dvc/path/data_item.py @@ -1,7 +1,7 @@ import os import stat -import shutil +from dvc.config import ConfigI from dvc.path.path import Path from dvc.exceptions import DvcException from dvc.system import System @@ -71,17 +71,17 @@ class DataItem(object): @cached_property def state(self): - state_dir = os.path.join(self._git.git_dir_abs, self._config.state_dir) + state_dir = os.path.join(self._git.git_dir_abs, ConfigI.STATE_DIR) state_file = os.path.join(state_dir, self.data.dvc + self.STATE_FILE_SUFFIX) return Path(state_file, self._git) @cached_property - def cache_dir(self): - return os.path.join(self._git.git_dir_abs, self._config.cache_dir) + def cache_dir_abs(self): + return os.path.join(self._git.git_dir_abs, ConfigI.CACHE_DIR) @cached_property def cache(self): - cache_dir = self.cache_dir + cache_dir = self.cache_dir_abs if self._cache_file: file_name = os.path.relpath(os.path.realpath(self._cache_file), cache_dir) @@ -94,11 +94,11 @@ class DataItem(object): @cached_property def state_dir_abs(self): - return os.path.join(self._git.git_dir_abs, self._config.state_dir) + return os.path.join(self._git.git_dir_abs, ConfigI.STATE_DIR) def move_data_to_cache(self): md5 = file_md5(self.data.relative)[0] - self._cache_file = os.path.join(self.cache_dir, md5) + self._cache_file = os.path.join(self.cache_dir_abs, md5) self._git.modify_gitignore([self.data.relative]) if not os.path.isfile(self.cache.relative): System.hardlink(self.data.relative, self.cache.relative) diff --git a/dvc/path/factory.py b/dvc/path/factory.py index e1cdbd7cd..2f7ef5a36 100644 --- a/dvc/path/factory.py +++ b/dvc/path/factory.py @@ -1,5 +1,6 @@ import os +from dvc.config import ConfigI from dvc.path.data_item import DataItem, DataItemError, DataDirError from dvc.path.path import Path from dvc.path.stated_data_item import StatedDataItem @@ -30,7 +31,7 @@ class PathFactory(object): def is_data_item(self, fname): data = os.path.relpath(os.path.realpath(fname), self._git.git_dir_abs) - state = os.path.join(self._config.state_dir, data + DataItem.STATE_FILE_SUFFIX) + state = os.path.join(ConfigI.STATE_DIR, data + DataItem.STATE_FILE_SUFFIX) return os.path.isfile(state) def existing_data_item(self, fname): @@ -55,7 +56,7 @@ class PathFactory(object): def all_existing_data_items(self, subdir='.'): states = [] - for root, dirs, files in os.walk(os.path.join(self._config.state_dir, subdir)): + for root, dirs, files in os.walk(os.path.join(ConfigI.STATE_DIR, subdir)): for fname in files: path = os.path.join(root, fname) @@ -71,5 +72,6 @@ class PathFactory(object): return map(lambda s: self.data_item_from_dvc_path(self.state_path_to_dvc_path(s), existing), states) + @staticmethod def state_path_to_dvc_path(self, state): - return os.path.relpath(state, self._config.state_dir)[:-len(DataItem.STATE_FILE_SUFFIX)] + return os.path.relpath(state, ConfigI.STATE_DIR)[:-len(DataItem.STATE_FILE_SUFFIX)] From 524901d4ad78a3b5f9a819237eff550a89cee98b Mon Sep 17 00:00:00 2001 From: Dmitry Date: Mon, 13 Nov 2017 00:51:54 -0800 Subject: [PATCH 5/6] Bug fixing --- dvc/command/checkout.py | 5 +++-- dvc/git_wrapper.py | 6 +++--- dvc/path/factory.py | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/dvc/command/checkout.py b/dvc/command/checkout.py index 8baa65dd3..eb51a824b 100644 --- a/dvc/command/checkout.py +++ b/dvc/command/checkout.py @@ -39,7 +39,7 @@ class CmdCheckout(CmdBase): states = self.git.state_files_for_previous_commit() prev_items = self.settings.path_factory.data_items_from_states(states, existing=False) except Exception as ex: - Logger.error(u'Unable to get data files from previous commit'.format(ex)) + Logger.error(u'Unable to get data files from previous commit: {}'.format(ex)) return 1 curr_items = self.settings.path_factory.all_existing_data_items() @@ -53,7 +53,8 @@ class CmdCheckout(CmdBase): for item in removed_items_set: path = item.data.relative if not os.path.exists(path): - print(u'Remove \'{}\' - file does not exist'.format(path)) + print(u'Remove \'{}\'.'.format(path)) + Logger.warn(u'File \'{}\' does not exist'.format(path)) else: Logger.info(u'Remove \'{}\''.format(path)) os.remove(path) diff --git a/dvc/git_wrapper.py b/dvc/git_wrapper.py index c1d27fbb9..857b77a96 100644 --- a/dvc/git_wrapper.py +++ b/dvc/git_wrapper.py @@ -4,7 +4,7 @@ from collections import defaultdict from dvc.exceptions import DvcException from dvc.logger import Logger -from dvc.config import Config +from dvc.config import Config, ConfigI from dvc.executor import Executor, ExecutorError from dvc.path.data_item import DataItemError from dvc.state_file import StateFile @@ -432,8 +432,8 @@ class GitWrapper(GitWrapperI): @staticmethod def state_files_for_previous_commit(): commit = GitWrapper.git_prev_commit() - Logger.debug(u'[dvc-git] Previous commit {}'.format(commit)) + Logger.debug(u'[dvc-git] Previous commit "{}"'.format(commit)) git_cmd = u'git ls-tree --name-only -r {}'.format(commit) lines = Executor.exec_cmd_only_success(git_cmd.split()).split('\n') - return filter(lambda s: s.startswith('.dvc/state'), map(unicode.split, lines)) + return filter(lambda s: s.startswith(ConfigI.STATE_DIR), lines) diff --git a/dvc/path/factory.py b/dvc/path/factory.py index 2f7ef5a36..4f7d8a591 100644 --- a/dvc/path/factory.py +++ b/dvc/path/factory.py @@ -73,5 +73,5 @@ class PathFactory(object): states) @staticmethod - def state_path_to_dvc_path(self, state): + def state_path_to_dvc_path(state): return os.path.relpath(state, ConfigI.STATE_DIR)[:-len(DataItem.STATE_FILE_SUFFIX)] From b4bf6ffbd79e3faa43f813dd6c93f44b9fa81bc9 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Mon, 13 Nov 2017 01:51:48 -0800 Subject: [PATCH 6/6] No commit history issue --- dvc/git_wrapper.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dvc/git_wrapper.py b/dvc/git_wrapper.py index 857b77a96..05ac13b9f 100644 --- a/dvc/git_wrapper.py +++ b/dvc/git_wrapper.py @@ -427,13 +427,20 @@ class GitWrapper(GitWrapperI): @staticmethod def git_prev_commit(): cmd = 'git check-ref-format --branch @{-1}' - return Executor.exec_cmd_only_success(cmd.split()).strip() + code, out, err = Executor.exec_cmd(cmd.split()) + if code != 0: + return '' + return out @staticmethod def state_files_for_previous_commit(): commit = GitWrapper.git_prev_commit() Logger.debug(u'[dvc-git] Previous commit "{}"'.format(commit)) + if not commit: + Logger.debug(u'[dvc-git] Previous commit does not exist') + return [] + git_cmd = u'git ls-tree --name-only -r {}'.format(commit) lines = Executor.exec_cmd_only_success(git_cmd.split()).split('\n') return filter(lambda s: s.startswith(ConfigI.STATE_DIR), lines)