diff --git a/dvc/command/base.py b/dvc/command/base.py index eb66dc57c..179b3b858 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -108,3 +108,7 @@ class CmdBase(object): file_name = os.path.relpath(cache_prefix_file_name, self.config.cache_dir) dvc_file_path_trim = file_name.replace(os.sep, '/').strip('/') return self.config.aws_storage_prefix + '/' + dvc_file_path_trim + + @staticmethod + def warning_dvc_is_busy(): + Logger.warn('Cannot perform the cmd since DVC is busy and locked. Please retry the cmd later.') diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 04fa3692d..91e5ac6a3 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -30,7 +30,7 @@ class CmdRepro(CmdRun): def define_args(self, parser): self.set_skip_git_actions(parser) - parser.add_argument('target', metavar='', help='Data file to reproduce', nargs='*') + parser.add_argument('target', metavar='', help='Data item to reproduce', nargs='*') pass @@ -50,7 +50,7 @@ class CmdRepro(CmdRun): lock = fasteners.InterProcessLock(self.git.lock_file) gotten = lock.acquire(timeout=5) if not gotten: - Logger.printing('Cannot perform the cmd since DVC is busy and locked. Please retry the cmd later.') + self.warning_dvc_is_busy() return 1 try: @@ -79,15 +79,15 @@ class CmdRepro(CmdRun): repro_change = ReproChange(data_item, self) if repro_change.reproduce(): changed = True - Logger.info(u'Data file "{}" was reproduced.'.format( + Logger.info(u'Data item "{}" was reproduced.'.format( data_item.data.relative )) else: - Logger.info(u'Reproduction is not required for data file "{}".'.format( + Logger.info(u'Reproduction is not required for data item "{}".'.format( data_item.data.relative )) except ReproError as err: - Logger.error('Error in reproducing data file {}: {}'.format( + Logger.error('Error in reproducing data item {}: {}'.format( data_item.data.relative, str(err) )) error = True @@ -121,48 +121,60 @@ class ReproChange(object): self._repro_argv = argv pass + @property + def cmd_obj(self): + return self._cmd_obj + + @property + def state(self): + return self._state + def reproduce_data_file(self): - Logger.debug('Reproducing data file "{}". Removing the file...'.format( - self._data_item.data.relative)) + Logger.debug('Reproducing data item "{}". Removing the file...'.format( + self._data_item.data.dvc)) os.remove(self._data_item.data.relative) - Logger.debug('Reproducing data file "{}". Re-runs cmd: {}'.format( + Logger.debug('Reproducing data item "{}". Re-runs cmd: {}'.format( self._data_item.data.relative, ' '.join(self._repro_argv))) - data_items_from_args = self._cmd_obj._data_items_from_args(self._repro_argv) - return self._cmd_obj.run_command(self._repro_argv, data_items_from_args) + data_items_from_args = self.cmd_obj.data_items_from_args(self._repro_argv) + return self.cmd_obj.run_command(self._repro_argv, + data_items_from_args, + self.state.stdout, + self.state.stderr) def reproduce(self, force=False): - print('+++++++++++++++ reproduce', self._data_item.data.dvc) + Logger.debug('Reproduce data item {} with dependencies: {}'.format( + self._data_item.data.dvc, + ', '.join([x.data.dvc for x in self.dependencies]))) + were_input_files_changed = False - if not self._state.is_reproducible: - Logger.debug('Data file "{}" is not reproducible'.format(self._data_item.data.relative)) + if not self.state.is_reproducible: + Logger.debug('Data item "{}" is not reproducible'.format(self._data_item.data.relative)) return False - print('+++++++++++++++ reproduce dependencies: '.format(self.get_dependencies())) - - for data_item in self.get_dependencies(): + for data_item in self.dependencies: change = ReproChange(data_item, self._cmd_obj) if change.reproduce(force): were_input_files_changed = True was_source_code_changed = self.git.were_files_changed(self._data_item.data.relative, - self._state.code_dependencies) + self.state.code_dependencies) + if was_source_code_changed: + Logger.debug('Source code was changed for "{}"'.format(self._data_item.data.dvc)) if not force and not was_source_code_changed and not were_input_files_changed: - Logger.debug('Data file "{}" is up to date'.format( + Logger.debug('Data item "{}" is up to date'.format( self._data_item.data.relative)) return False return self.reproduce_data_file() - def get_dependencies(self): - - print('+++++++++++++== INPUT FILES: {}'.format(self._state.input_files)) - + @property + def dependencies(self): dependency_data_items = [] - for input_file in self._state.input_files: + for input_file in self.state.input_files: try: data_item = self._cmd_obj.path_factory.data_item(input_file) except NotInDataDirError: diff --git a/dvc/command/run.py b/dvc/command/run.py index bce131f5e..acce034fb 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -1,17 +1,17 @@ import os import sys -import shutil + import fasteners -from dvc.path.data_item import NotInDataDirError -from dvc.git_wrapper import GitWrapper from dvc.command.base import CmdBase -from dvc.logger import Logger from dvc.exceptions import DvcException +from dvc.git_wrapper import GitWrapper +from dvc.logger import Logger +from dvc.path.data_item import NotInDataDirError from dvc.repository_change import RepositoryChange from dvc.state_file import StateFile -from dvc.utils import run from dvc.utils import cached_property +from dvc.utils import run class RunError(DvcException): @@ -61,7 +61,7 @@ class CmdRun(CmdBase): lock = fasteners.InterProcessLock(self.git.lock_file) gotten = lock.acquire(timeout=5) if not gotten: - Logger.printing('Cannot perform the cmd since DVC is busy and locked. Please retry the cmd later.') + self.warning_dvc_is_busy() return 1 try: @@ -69,7 +69,7 @@ class CmdRun(CmdBase): return 1 self.run_command(self._args_unkn, - self._data_items_from_args(self._args_unkn), + self.data_items_from_args(self._args_unkn), self.args.stdout, self.args.stderr) return self.commit_if_needed('DVC run: {}'.format(' '.join(sys.argv))) @@ -79,6 +79,12 @@ class CmdRun(CmdBase): return 1 def run_command(self, argv, data_items_from_args, stdout=None, stderr=None): + Logger.debug('Run command: {}. Data items from args: {}. stdout={}, stderr={}'.format( + ' '.join(argv), + ', '.join([x.data.dvc for x in data_items_from_args]), + stdout, + stderr)) + repo_change = RepositoryChange(argv, stdout, stderr, self.git, self.config, self.path_factory) if not self.skip_git_actions and not self._validate_file_states(repo_change): @@ -106,12 +112,19 @@ class CmdRun(CmdBase): output_files_dvc, code_dependencies_dvc, argv=argv, - is_reproducible=self.is_reproducible) + is_reproducible=self.is_reproducible, + stdout=self._stdout_to_dvc(stdout), + stderr=self._stdout_to_dvc(stderr)) state_file.save() result.append(state_file) return result + def _stdout_to_dvc(self, stdout): + if stdout in {None, '-'}: + return stdout + return self.path_factory.data_item(stdout).data.dvc + @staticmethod def remove_new_files(repo_change): for data_item in repo_change.new_data_items: @@ -132,7 +145,7 @@ class CmdRun(CmdBase): return not error - def _data_items_from_args(self, argv): + def data_items_from_args(self, argv): result = [] for arg in argv: diff --git a/dvc/repository_change.py b/dvc/repository_change.py index d688ee25e..b00122fa9 100644 --- a/dvc/repository_change.py +++ b/dvc/repository_change.py @@ -3,6 +3,7 @@ import os from dvc.exceptions import DvcException from dvc.git_wrapper import GitWrapper from dvc.executor import Executor +from dvc.logger import Logger from dvc.path.data_item import NotInDataDirError from dvc.path.stated_data_item import StatedDataItem from dvc.utils import cached_property @@ -21,6 +22,10 @@ class RepositoryChange(object): self.config = config self.path_factory = path_factory + Logger.debug(u'[Repository change] Exec command: {}. stdout={}, stderr={}'.format( + u' '.join(args), + stdout, + stderr)) Executor.exec_cmd_only_success(args, stdout, stderr) self._stated_data_items = [] @@ -55,6 +60,9 @@ class RepositoryChange(object): try: item = self.path_factory.stated_data_item(state, file) self._stated_data_items.append(item) + Logger.debug('[Repository change] Add status: {} {}'.format( + item.status, + item.data.dvc)) except NotInDataDirError: self._externally_created_files.append(file) pass diff --git a/dvc/state_file.py b/dvc/state_file.py index 7e2d6cad2..6c64160a7 100644 --- a/dvc/state_file.py +++ b/dvc/state_file.py @@ -25,11 +25,15 @@ class StateFile(object): PARAM_OUTPUT_FILES = 'OutputFiles' PARAM_CODE_DEPENDENCIES = 'CodeDependencies' PARAM_NOT_REPRODUCIBLE = 'NotReproducible' + PARAM_STDOUT = "Stdout" + PARAM_STDERR = "Stderr" def __init__(self, file, git, input_files, output_files, code_dependencies=[], is_reproducible=True, argv=sys.argv, + stdout=None, + stderr=None, norm_argv=None, created_at=time.strftime('%Y-%m-%d %H:%M:%S %z'), cwd=None): @@ -46,6 +50,9 @@ class StateFile(object): else: self.norm_argv = self.normalized_args() + self.stdout = stdout + self.stderr = stderr + self.created_at = created_at if cwd: @@ -66,6 +73,8 @@ class StateFile(object): data.get(StateFile.PARAM_CODE_DEPENDENCIES, []), not data.get(StateFile.PARAM_NOT_REPRODUCIBLE, False), data.get(StateFile.PARAM_ARGV), + data.get(StateFile.PARAM_STDOUT), + data.get(StateFile.PARAM_STDERR), data.get(StateFile.PARAM_NORM_ARGV), data.get(StateFile.PARAM_CREATED_AT), data.get(StateFile.PARAM_CWD)) @@ -80,7 +89,9 @@ class StateFile(object): self.PARAM_CREATED_AT: self.created_at, self.PARAM_INPUT_FILES: self.input_files, self.PARAM_OUTPUT_FILES: self.output_files, - self.PARAM_CODE_DEPENDENCIES: self.code_dependencies + self.PARAM_CODE_DEPENDENCIES: self.code_dependencies, + self.PARAM_STDOUT: self.stdout, + self.PARAM_STDERR: self.stderr } if not self.is_reproducible: @@ -98,14 +109,14 @@ class StateFile(object): result = [] if len(self.argv) > 0: - cmd = self.argv [0] + cmd = self.argv[0] pos = cmd.rfind(os.sep) if pos >= 0: cmd = cmd[pos+1:] result.append(cmd) - for arg in self.argv [1:]: - if os.path.isfile(arg): + for arg in self.argv[1:]: + if os.path.isfile(arg): # CHANGE to data items path = os.path.abspath(arg) dvc_path = os.path.relpath(path, self.git.git_dir_abs) result.append(dvc_path) diff --git a/tests/test_repro.py b/tests/test_repro.py index 65986d53a..dd63447ce 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -60,12 +60,24 @@ from tests.test_cmd_run import RunBasicTest class ReproBasicEnv(RunBasicTest): def setUp(self): super(ReproBasicEnv, self).setUp() + self.file_name1 = 'data/file1' - CmdRun(args=['printf', 'Hello\nWorld!', '--not-repro', '--stdout', self.file_name1], - parse_config=False, - config_obj=self.config, - git_obj=self.git - ).run() + self.file1_code_file = 'file1.py' + self.create_file_and_commit(self.file1_code_file, 'An awesome code...') + cmd_file1 = CmdRun(args=[ + 'printf', + 'Hello\nMary', + '--not-repro', + '--stdout', + self.file_name1, + '--code', + self.file1_code_file + ], + parse_config=False, + config_obj=self.config, + git_obj=self.git) + self.assertEqual(cmd_file1.code_dependencies, [self.file1_code_file]) + cmd_file1.run() self.file_name11 = 'data/file11' CmdRun(args=['head', '-n', '1', self.file_name1, '--stdout', self.file_name11], @@ -81,54 +93,73 @@ class ReproBasicEnv(RunBasicTest): git_obj=self.git ).run() + self.file_res_code_file = 'code_res.py' + self.create_file_and_commit(self.file_res_code_file, 'Another piece of code') self.file_name_res = 'data/file_res' - CmdRun(args=['cat', self.file_name11, self.file_name2, '--stdout', self.file_name_res], - parse_config=False, - config_obj=self.config, - git_obj=self.git - ).run() + cmd_res = CmdRun(args=[ + 'cat', + self.file_name11, + self.file_name2, + '--stdout', + self.file_name_res, + '--code', + self.file_res_code_file + ], + parse_config=False, + config_obj=self.config, + git_obj=self.git) + self.assertEqual(cmd_res.code_dependencies, [self.file_res_code_file]) + cmd_res.run() self.assertEqual(open(self.file_name_res).read(), 'Hello\nBobby') -# -# class ReproRemovedFileTest(ReproBasicEnv): -# def test(self): -# # CmdDataRemove(args=[self.file_name_res, '--keep-in-cloud'], -# # config_obj=self.config, -# # git_obj=self.git -# # ).run() -# -# # CmdRun(args=['printf', 'Goodbye\nWorld!', '--stdout', self.file_name1], -# # config_obj=self.config, -# # git_obj=self.git -# # ).run() -# -# CmdRepro(args=[self.file_name_res], -# config_obj=self.config, -# git_obj=self.git -# ).run() -# -# self.assertEqual(open(self.file_name_res).read(), 'Hello\nBobby') + def create_file_and_commit(self, file_name, content='Any', message='Just a commit'): + self.create_file(file_name, content) + self.commit_file(file_name, message) + + @staticmethod + def commit_file(file_name, message='Just a commit'): + Executor.exec_cmd_only_success(['git', 'add', file_name]) + Executor.exec_cmd_only_success(['git', 'commit', '-m', message]) + + def modify_file_and_commit(self, filename, content_to_add=' '): + fd = open(filename, 'a') + fd.write(content_to_add) + fd.close() + self.commit_file(filename) -class ReproChangedDependency(ReproBasicEnv): +class ReproCodeDependencyTest(ReproBasicEnv): + def test(self): - - self.assertEqual(open(self.file_name_res).read(), 'Hello\nBobby') - - CmdDataRemove(args=[self.file_name1, '--keep-in-cloud'], - config_obj=self.config, - git_obj=self.git - ).run() - - CmdRun(args=['printf', 'Goodbye\nWorld!', '--stdout', self.file_name1], - config_obj=self.config, - git_obj=self.git - ).run() + self.modify_file_and_commit(self.file_res_code_file) CmdRepro(args=[self.file_name_res], config_obj=self.config, git_obj=self.git ).run() - # self.assertEqual(open(self.file_name_res).read(), 'Goodbye\nBobby') + self.assertEqual(open(self.file_name_res).read(), 'Hello\nBobby') + + +# class ReproChangedDependency(ReproBasicEnv): +# def test(self): +# +# self.assertEqual(open(self.file_name_res).read(), 'Hello\nBobby') +# +# CmdDataRemove(args=[self.file_name1, '--keep-in-cloud'], +# config_obj=self.config, +# git_obj=self.git +# ).run() +# +# CmdRun(args=['printf', 'Goodbye\nWorld!', '--stdout', self.file_name1], +# config_obj=self.config, +# git_obj=self.git +# ).run() +# +# CmdRepro(args=[self.file_name_res], +# config_obj=self.config, +# git_obj=self.git +# ).run() +# +# # self.assertEqual(open(self.file_name_res).read(), 'Goodbye\nBobby')