diff --git a/dvc/command/base.py b/dvc/command/base.py index 9cce781d6..ae06ba78d 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -49,23 +49,21 @@ class CmdBase(object): def define_args(self, parser): pass - def set_skip_git_actions(self, parser): - parser.add_argument('--skip-git-actions', '-s', action='store_true', default=False, - help='Skip all git actions including reproducibility check and commits') - # parser.add_argument('--no-lock', '-L', action='store_true', default=False, - # help='Do not set DVC locker') + def set_no_git_actions(self, parser): + parser.add_argument('--no-git-actions', '-G', action='store_true', default=False, + help='Skip all git actions including reproducibility check and commits.') def set_lock_action(self, parser): - parser.add_argument('-l', '--lock', action='store_true', default=False, - help='Lock data item - disable reproduction. ' + \ + parser.add_argument('--lock', '-l', action='store_true', default=False, + help='Lock data item - disable reproduction. ' + 'It can be enabled by `dvc lock` command or by forcing reproduction.') @property - def skip_git_actions(self): - return self.parsed_args.skip_git_actions + def no_git_actions(self): + return self.parsed_args.no_git_actions def set_git_action(self, value): - self.parsed_args.skip_git_actions = not value + self.parsed_args.no_git_actions = not value @property def is_locker(self): @@ -77,7 +75,7 @@ class CmdBase(object): self.parsed_args.no_lock = value def commit_if_needed(self, message, error=False): - if error or self.skip_git_actions: + if error or self.no_git_actions: self.not_committed_changes_warning() return 1 else: diff --git a/dvc/command/import_bulk.py b/dvc/command/import_bulk.py index e70c40999..e81acbc04 100644 --- a/dvc/command/import_bulk.py +++ b/dvc/command/import_bulk.py @@ -12,7 +12,7 @@ class CmdImportBulk(CmdBase): super(CmdImportBulk, self).__init__(settings) def define_args(self, parser): - self.set_skip_git_actions(parser) + self.set_no_git_actions(parser) self.set_lock_action(parser) parser.add_argument('input', @@ -33,11 +33,11 @@ class CmdImportBulk(CmdBase): return 1 try: - if not self.skip_git_actions and not self.git.is_ready_to_go(): + if not self.no_git_actions and not self.git.is_ready_to_go(): return 1 cmd = CmdImportFile(self.settings) - cmd.set_git_action(not self.skip_git_actions) + cmd.set_git_action(not self.no_git_actions) cmd.set_locker(False) output = self.parsed_args.output diff --git a/dvc/command/import_file.py b/dvc/command/import_file.py index f60ff1bc3..c3e24772d 100644 --- a/dvc/command/import_file.py +++ b/dvc/command/import_file.py @@ -23,7 +23,7 @@ class CmdImportFile(CmdBase): super(CmdImportFile, self).__init__(settings) def define_args(self, parser): - self.set_skip_git_actions(parser) + self.set_no_git_actions(parser) self.add_string_arg(parser, 'input', 'Input file') self.add_string_arg(parser, 'output', 'Output file') @@ -50,7 +50,7 @@ class CmdImportFile(CmdBase): pass def import_and_commit_if_needed(self, input, output, lock=False, check_if_ready=True): - if check_if_ready and not self.skip_git_actions and not self.git.is_ready_to_go(): + if check_if_ready and not self.no_git_actions and not self.git.is_ready_to_go(): return 1 self.import_file(input, output, lock) @@ -93,14 +93,13 @@ class CmdImportFile(CmdBase): Logger.debug('Creating symlink {} --> {}'.format(data_item.symlink_file, data_item.data.relative)) System.symlink(data_item.symlink_file, data_item.data.relative) - # import_file_argv = [StateFile.DVC_PYTHON_FILE_NAME, StateFile.COMMAND_IMPORT_FILE, input, output] state_file = StateFile(StateFile.COMMAND_IMPORT_FILE, data_item.state.relative, self.settings, argv=[input, output], input_files=[], output_files=[output], - is_reproducible=lock) + lock=lock) state_file.save() Logger.debug('State file "{}" was created'.format(data_item.state.relative)) pass diff --git a/dvc/command/init.py b/dvc/command/init.py index 87570cf8b..fa9da1cf4 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -60,11 +60,11 @@ ProjectName = super(CmdInit, self).__init__(settings) def define_args(self, parser): - self.set_skip_git_actions(parser) + self.set_no_git_actions(parser) - self.add_string_arg(parser, '--data-dir', 'NeatLynx data directory', 'data') - self.add_string_arg(parser, '--cache-dir', 'NeatLynx cache directory', '.cache') - self.add_string_arg(parser, '--state-dir', 'NeatLynx state directory', '.state') + self.add_string_arg(parser, '--data-dir', 'Data directory', 'data') + self.add_string_arg(parser, '--cache-dir', 'Cache directory', '.cache') + self.add_string_arg(parser, '--state-dir', 'State directory', '.state') pass def get_not_existing_dir(self, dir): @@ -80,7 +80,7 @@ ProjectName = return file_name def run(self): - if not self.skip_git_actions and not self.git.is_ready_to_go(): + if not self.no_git_actions and not self.git.is_ready_to_go(): return 1 if os.path.realpath(os.path.curdir) != self.settings.git.git_dir_abs: @@ -140,7 +140,7 @@ ProjectName = self.settings, input_files=[], output_files=[], - is_reproducible=False).save() + lock=False).save() pass def modify_gitignore(self, cache_dir_name): diff --git a/dvc/command/lock.py b/dvc/command/lock.py index f47d66fcf..b3fdc017d 100644 --- a/dvc/command/lock.py +++ b/dvc/command/lock.py @@ -10,7 +10,7 @@ class CmdLock(CmdBase): super(CmdLock, self).__init__(settings) def define_args(self, parser): - self.set_skip_git_actions(parser) + self.set_no_git_actions(parser) parser.add_argument('-u', '--unlock', action='store_true', default=False, help='Unlock data item - enable reproduction.') @@ -41,12 +41,12 @@ class CmdLock(CmdBase): data_item = self.settings.path_factory.existing_data_item(file) state = StateFile.load(data_item.state.relative, self.settings) - if state.lock and target: + if state.locked and target: Logger.warn('Data item {} is already locked'.format(data_item.data.relative)) - elif not state.lock and not target: + elif not state.locked and not target: Logger.warn('Data item {} is already unlocked'.format(data_item.data.relative)) else: - state.lock = target + state.locked = target Logger.debug('Saving status file for data item {}'.format(data_item.data.relative)) state.save() Logger.info('Data item {} was {}ed'.format(data_item.data.relative, cmd)) diff --git a/dvc/command/remove.py b/dvc/command/remove.py index be6f886ca..f53f999f5 100644 --- a/dvc/command/remove.py +++ b/dvc/command/remove.py @@ -19,7 +19,7 @@ class CmdDataRemove(CmdBase): super(CmdDataRemove, self).__init__(settings) def define_args(self, parser): - self.set_skip_git_actions(parser) + self.set_no_git_actions(parser) parser.add_argument('target', metavar='', help='Target to remove - file or directory', nargs='*') parser.add_argument('-r', '--recursive', action='store_true', help='Remove directory recursively') @@ -47,7 +47,7 @@ class CmdDataRemove(CmdBase): return 0 def remove_all_targets(self): - if not self.skip_git_actions and not self.git.is_ready_to_go(): + if not self.no_git_actions and not self.git.is_ready_to_go(): return False error = False diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 0508526a3..f30feab52 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -29,13 +29,13 @@ class CmdRepro(CmdRun): return self._code def define_args(self, parser): - self.set_skip_git_actions(parser) + self.set_no_git_actions(parser) parser.add_argument('target', metavar='', help='Data items to reproduce.', nargs='*') parser.add_argument('-f', '--force', action='store_true', default=False, - help='Reproduce even if dependencies were not changed') - parser.add_argument('-r', '--recursive', action='store_true', default=False, - help='Recursively reproduce all changed dependencies') + help='Reproduce even if dependencies were not changed.') + parser.add_argument('-s', '--single-item', action='store_true', default=False, + help='Reproduce only single data item without recursive dependencies check.') pass @property @@ -58,15 +58,15 @@ class CmdRepro(CmdRun): self.warning_dvc_is_busy() return 1 try: - return self.repro_target(self.parsed_args.target, self.parsed_args.recursive, self.parsed_args.force) + recursive = not self.parsed_args.single_item + return self.repro_target(self.parsed_args.target, recursive, self.parsed_args.force) finally: if self.is_locker: lock.release() - pass def repro_target(self, target, recursive, force): - if not self.skip_git_actions and not self.git.is_ready_to_go(): + if not self.no_git_actions and not self.git.is_ready_to_go(): return 1 data_item_list, external_files_names = self.settings.path_factory.to_data_items(target) @@ -77,7 +77,7 @@ class CmdRepro(CmdRun): if self.repro_data_items(data_item_list, recursive, force): return 0 - pass + return 1 def repro_data_items(self, data_item_list, recursive, force): error = False @@ -110,7 +110,7 @@ class CmdRepro(CmdRun): error = True break - if error and not self.skip_git_actions: + if error and not self.no_git_actions: Logger.error('Errors occurred. One or more repro cmd was not successful.') self.not_committed_changes_warning() @@ -141,6 +141,9 @@ class ReproChange(object): if len(self.state.argv) < 1: raise ReproError('Error: reproducible cmd in state file "{}" is too short'. format(self.state.file)) + + self._settings = copy.copy(self._cmd_obj.settings) + self._settings.set_args(self.state.argv) pass def is_cache_exists(self): @@ -168,9 +171,6 @@ class ReproChange(object): msg = 'Data item {} cannot be removed before reproduction: {}' Logger.error(msg.format(output_dvc, ex)) - settings = copy.copy(self._cmd_obj.settings) - settings.set_args(self.state.argv) - if self.state.is_import_file: Logger.debug('Reproducing data item {}. Re-import cmd: {}'.format( self._data_item.data.relative, ' '.join(self.state.argv))) @@ -182,16 +182,16 @@ class ReproChange(object): input = self.state.argv[0] output = self.state.argv[1] - cmd = CmdImportFile(settings) + cmd = CmdImportFile(self._settings) cmd.set_git_action(True) cmd.set_locker(False) Logger.info(u'Reproducing import command: {}'.format(output)) - if cmd.import_and_commit_if_needed(input, output, is_reproducible=True, check_if_ready=False) != 0: + if cmd.import_and_commit_if_needed(input, output, lock=True, check_if_ready=False) != 0: raise ReproError('Import command reproduction failed') return True elif self.state.is_run: - cmd = CmdRun(settings) + cmd = CmdRun(self._settings) cmd.set_git_action(True) cmd.set_locker(False) @@ -217,16 +217,15 @@ class ReproChange(object): Logger.debug('Reproduce data item {}. recursive={}, force={}'.format( self._data_item.data.relative, self._recursive, self._force)) - if not self.state.lock: + if self.state.locked: Logger.debug('Data item {} is not reproducible'.format(self._data_item.data.relative)) return False - data_item_dvc = self._data_item.data.dvc - if self.is_repro_required(changed_files, data_item_dvc): - if data_item_dvc not in changed_files: + if self.is_repro_required(changed_files, self._data_item): + if self._data_item.data.dvc not in changed_files: Logger.debug('Data item {} is going to be reproduced'.format(self._data_item.data.relative)) self.reproduce_data_item() - changed_files.add(data_item_dvc) + changed_files.add(self._data_item.data.dvc) return True else: msg = 'Data item {} is not going to be reproduced because it is already reproduced' @@ -255,7 +254,12 @@ class ReproChange(object): return result - def is_repro_required(self, changed_files, data_item_dvc): + def is_repro_required(self, changed_files, data_item): + state_file = StateFile.load(data_item.state.relative, self._settings) + if state_file.locked: + Logger.debug(u'Repro is not required for locked data item {}'.format(data_item.data.relative)) + return False + is_dependency_check_required = self._recursive if not is_dependency_check_required and not self.is_cache_exists(): @@ -264,7 +268,7 @@ class ReproChange(object): self._data_item.data.relative)) if is_dependency_check_required: - if self.were_dependencies_changed(changed_files, data_item_dvc): + if self.were_dependencies_changed(changed_files, data_item.data.dvc): self.log_repro_reason(u'input dependencies were changed') return True @@ -289,7 +293,7 @@ class ReproChange(object): def were_sources_changed(self, globally_changed_files): were_sources_changed = self.git.were_files_changed( self.state.code_dependencies, - self.cmd_obj.settings.path_factory, + self._settings.path_factory, globally_changed_files ) return were_sources_changed @@ -299,7 +303,7 @@ class ReproChange(object): dependency_data_items = [] for input_file in self.state.input_files: try: - data_item = self._cmd_obj.settings.path_factory.data_item(input_file) + data_item = self._settings.path_factory.data_item(input_file) except NotInDataDirError: raise ReproError(u'The dependency file "{}" is not a data item'.format(input_file)) except Exception as ex: diff --git a/dvc/command/run.py b/dvc/command/run.py index 5d06f47ba..cb9685ae1 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -22,25 +22,23 @@ class CmdRun(CmdBase): super(CmdRun, self).__init__(settings) def define_args(self, parser): - self.set_skip_git_actions(parser) + self.set_no_git_actions(parser) self.set_lock_action(parser) - parser.add_argument('--not-repro', help='Not reproducible', - action='store_false', default=False) - parser.add_argument('--stdout', help='output std output to a file') - parser.add_argument('--stderr', help='output std error to a file') + parser.add_argument('--stdout', help='Output std output to a file.') + parser.add_argument('--stderr', help='Output std error to a file.') parser.add_argument('--input', '-i', action='append', - help='Declare input data items for reproducible cmd') + help='Declare input data items for reproducible cmd.') parser.add_argument('--output', '-o', action='append', - help='Declare output data items for reproducible cmd') + help='Declare output data items for reproducible cmd.') parser.add_argument('--code', '-c', action='append', - help='Code dependencies which produce the output') + help='Code dependencies which produce the output.') parser.add_argument('--shell', help='Shell command', action='store_true', default=False) pass @property def lock(self): - return not self.parsed_args.lock + return self.parsed_args.lock @property def code_dependencies(self): @@ -76,7 +74,7 @@ class CmdRun(CmdBase): def run_and_commit_if_needed(self, command_args, data_items_from_args, not_data_items_from_args, stdout, stderr, shell, check_if_ready=True): - if check_if_ready and not self.skip_git_actions and not self.git.is_ready_to_go(): + if check_if_ready and not self.no_git_actions and not self.git.is_ready_to_go(): return 1 self.run_command(command_args, @@ -99,7 +97,7 @@ class CmdRun(CmdBase): repo_change = RepositoryChange(cmd_args, self.settings, stdout, stderr, shell=shell) - if not self.skip_git_actions and not self._validate_file_states(repo_change): + if not self.no_git_actions and not self._validate_file_states(repo_change): self.remove_new_files(repo_change) raise RunError('Errors occurred.') diff --git a/dvc/command/test.py b/dvc/command/test.py index 80d90a075..49b46e097 100644 --- a/dvc/command/test.py +++ b/dvc/command/test.py @@ -19,7 +19,7 @@ class CmdTest(CmdBase): super(CmdTest, self).__init__(settings) def define_args(self, parser): - self.set_skip_git_actions(parser) + self.set_no_git_actions(parser) def run(self): diff --git a/dvc/state_file.py b/dvc/state_file.py index 83b767364..009e93930 100644 --- a/dvc/state_file.py +++ b/dvc/state_file.py @@ -34,7 +34,7 @@ class StateFile(object): PARAM_INPUT_FILES = 'InputFiles' PARAM_OUTPUT_FILES = 'OutputFiles' PARAM_CODE_DEPENDENCIES = 'CodeDependencies' - PARAM_LOCK = 'Lock' + PARAM_LOCKED = 'Locked' PARAM_STDOUT = "Stdout" PARAM_STDERR = "Stderr" PARAM_SHELL = "Shell" @@ -57,7 +57,7 @@ class StateFile(object): self.settings = settings self.input_files = input_files self.output_files = output_files - self.lock = lock + self.locked = lock self.code_dependencies = code_dependencies self.shell = shell @@ -101,7 +101,7 @@ class StateFile(object): data.get(StateFile.PARAM_INPUT_FILES, []), data.get(StateFile.PARAM_OUTPUT_FILES, []), data.get(StateFile.PARAM_CODE_DEPENDENCIES, []), - data.get(StateFile.PARAM_LOCK, False), + data.get(StateFile.PARAM_LOCKED, False), data.get(StateFile.PARAM_ARGV), data.get(StateFile.PARAM_STDOUT), data.get(StateFile.PARAM_STDERR), @@ -127,8 +127,8 @@ class StateFile(object): self.PARAM_SHELL: self.shell } - if self.lock: - res[self.PARAM_LOCK] = True + if self.locked: + res[self.PARAM_LOCKED] = True file_dir = os.path.dirname(self.file) if file_dir != '' and not os.path.isdir(file_dir): diff --git a/tests/test_cmd_data_remove.py b/tests/test_cmd_data_remove.py index 64276f39f..1827523a8 100644 --- a/tests/test_cmd_data_remove.py +++ b/tests/test_cmd_data_remove.py @@ -139,7 +139,7 @@ class TestRemoveEndToEnd(DirHierarchyEnvironment): cmd.parsed_args.target = [dir11_full, self.file5] cmd.parsed_args.recursive = True - cmd.parsed_args.skip_git_actions = True + cmd.parsed_args.no_git_actions = True self.assertTrue(os.path.exists(dir11_full)) self.assertTrue(os.path.exists(self.file5)) @@ -163,7 +163,7 @@ class TestRemoveEndToEnd(DirHierarchyEnvironment): cmd.parsed_args.target = [dir11_full, self.file5] cmd.parsed_args.recursive = True - cmd.parsed_args.skip_git_actions = True + cmd.parsed_args.no_git_actions = True self.assertTrue(os.path.exists(dir11_full)) self.assertTrue(os.path.exists(self.file5)) @@ -193,7 +193,7 @@ class TestRemoveEndToEndWithAnError(DirHierarchyEnvironment): cmd.parsed_args.target = [dir11_full, self.file5, file6_full] cmd.parsed_args.recursive = True - cmd.parsed_args.skip_git_actions = True + cmd.parsed_args.no_git_actions = True self.assertTrue(os.path.exists(dir11_full)) self.assertTrue(os.path.exists(self.file5)) diff --git a/tests/test_repro.py b/tests/test_repro.py index 2a363678a..6483286fe 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -4,6 +4,7 @@ from dvc.command.remove import CmdDataRemove from dvc.command.repro import CmdRepro from dvc.command.run import CmdRun from dvc.executor import Executor +from dvc.logger import Logger from tests.test_cmd_run import RunBasicTest @@ -86,7 +87,7 @@ class ReproChangedDependency(ReproBasicEnv): def test(self): self.recreate_file1() - self.settings._args = [self.file_name11, '-r'] + self.settings._args = [self.file_name11] CmdRepro(self.settings).run() self.assertEqual(open(self.file_name11).read(), 'Goodbye\n') @@ -107,7 +108,7 @@ class ReproChangedDeepDependency(ReproChangedDependency): def test(self): self.recreate_file1() - self.settings._args = [self.file_name_res, '-r'] + self.settings._args = [self.file_name_res] CmdRepro(self.settings).run() self.assertEqual(open(self.file_name_res).read().strip(), 'Goodbye\nBobby')