diff --git a/dvc/command/repro.py b/dvc/command/repro.py index e96f79dfb..5fcca4e23 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -78,15 +78,8 @@ class CmdRepro(CmdRun): for data_item in data_item_list: try: - target_commit = self.git.get_target_commit(data_item.data.relative) - if target_commit is None: - msg = 'Data item "{}" cannot be reproduced: cannot obtain commit hashsum' - Logger.warn(msg.format(data_item.data.relative)) - continue - - globally_changed_files = self.git.get_changed_files(target_commit) changed_files = set() - change = ReproChange(data_item, self, globally_changed_files, recursive, force) + change = ReproChange(data_item, self, None, recursive, force) if change.reproduce(changed_files): changed = True Logger.info(u'Data item "{}" was reproduced.'.format( @@ -115,7 +108,6 @@ class ReproChange(object): self._data_item = data_item self.git = cmd_obj.git self._cmd_obj = cmd_obj - self._globally_changed_files = globally_changed_files self._recursive = recursive self._force = force @@ -135,10 +127,27 @@ class ReproChange(object): raise ReproError('Error: reproducible cmd in state file "{}" is too short'. format(self.state.file)) + self._globally_changed_files = self.get_globally_changed_files(globally_changed_files) + self._settings = copy.copy(self._cmd_obj.settings) self._settings.set_args(self.state.argv) pass + def get_globally_changed_files(self, globally_changed_files): + if globally_changed_files != None: + return globally_changed_files + + target_commit = self.state.target_commit + if target_commit is None: + # Backward compatibility + target_commit = self.git.get_target_commit(data_item.data.relative) + + if target_commit is None: + msg = 'Data item "{}" cannot be reproduced: cannot obtain commit hashsum' + raise ReproError(msg.format(data_item.data.relative)) + + return self.git.get_changed_files(target_commit) + def is_cache_exists(self): path = System.realpath(self._data_item.data.relative) return os.path.exists(path) diff --git a/dvc/command/run.py b/dvc/command/run.py index 662d4499c..334ce2c3e 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -59,19 +59,30 @@ class CmdRun(CmdBase): if check_if_ready and not self.no_git_actions and not self.git.is_ready_to_go(): return 1 - self.run_command(command_args, - data_items_from_args, - not_data_items_from_args, - stdout, - stderr, - shell, - output_data_items, - input_data_items, - code_dependencies, - lock) + state_files = self.run_command(command_args, + data_items_from_args, + not_data_items_from_args, + stdout, + stderr, + shell, + output_data_items, + input_data_items, + code_dependencies, + lock) + return self.add_commit(state_files, command_args, is_repro) + + def add_commit(self, state_files, command_args, is_repro): + """ Commit and update TargetCommit in state files """ cmd_name = 'repro-run' if is_repro else 'run' - return self.commit_if_needed('DVC {}: {}'.format(cmd_name, ' '.join(command_args))) + ret = self.commit_if_needed('DVC {}: {}'.format(cmd_name, ' '.join(command_args))) + if ret != 0: + return ret + + for state in state_files: + state.add_target_commit(self.git.curr_commit) + + return self.commit_if_needed('DVC add target commit to state files') def run_command(self, cmd_args, data_items_from_args, not_data_items_from_args, stdout=None, stderr=None, shell=False, diff --git a/dvc/state_file.py b/dvc/state_file.py index 0c26458dd..9f01859a2 100644 --- a/dvc/state_file.py +++ b/dvc/state_file.py @@ -44,6 +44,7 @@ class StateFile(object): PARAM_SHELL = "Shell" PARAM_TARGET_METRICS = 'TargetMetrics' TARGET_METRICS_SINGLE_METRIC = 'SingleMetric' + PARAM_TARGET_COMMIT = 'TargetCommit' def __init__(self, command, @@ -59,7 +60,8 @@ class StateFile(object): created_at=time.strftime('%Y-%m-%d %H:%M:%S %z'), cwd=None, shell=False, - target_metrics={}): + target_metrics={}, + target_commit=None): self.data_item = data_item self.settings = settings @@ -68,6 +70,7 @@ class StateFile(object): self.locked = lock self.code_dependencies = code_dependencies self.shell = shell + self.target_commit = target_commit if command not in self.ACCEPTED_COMMANDS: raise StateFileError('Args error: unknown command %s' % command) @@ -188,7 +191,8 @@ class StateFile(object): json.get(StateFile.PARAM_CREATED_AT), StateFile.decode_path(json.get(StateFile.PARAM_CWD)), json.get(StateFile.PARAM_SHELL, False), - json.get(StateFile.PARAM_TARGET_METRICS, {})) + json.get(StateFile.PARAM_TARGET_METRICS, {}), + json.get(StateFile.PARAM_TARGET_COMMIT, None)) @staticmethod def load(data_item, settings): @@ -201,6 +205,13 @@ class StateFile(object): data = json.loads(content) return StateFile.load_json(data, settings) + def add_target_commit(self, commit): + with open(self.file, 'r') as state_file: + state = json.load(state_file) + state[self.PARAM_TARGET_COMMIT] = commit + with open(self.file, 'w') as state_file: + json.dump(state, state_file, indent=2) + def save(self, is_update_target_metrics=True): argv = self._argv_paths_normalization(self._argv)