From 223380d2999c034e5e95a829fed26c492ee5be23 Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Sat, 29 Apr 2017 15:12:31 -0700 Subject: [PATCH] New command: lock --- dvc/command/base.py | 9 ++++-- dvc/command/import_bulk.py | 7 ++--- dvc/command/import_file.py | 10 +++---- dvc/command/lock.py | 61 ++++++++++++++++++++++++++++++++++++++ dvc/command/repro.py | 4 +-- dvc/command/run.py | 7 +++-- dvc/runtime.py | 2 +- dvc/state_file.py | 17 +++++------ 8 files changed, 91 insertions(+), 26 deletions(-) create mode 100644 dvc/command/lock.py diff --git a/dvc/command/base.py b/dvc/command/base.py index bd873ac97..9cce781d6 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -52,8 +52,13 @@ class CmdBase(object): def set_skip_git_actions(self, parser): parser.add_argument('--skip-git-actions', '-s', action='store_true', default=False, help='Skip all git actions including reproducibility check and commits') - parser.add_argument('--no-lock', '-L', action='store_true', default=False, - help='Do not set DVC locker') + # parser.add_argument('--no-lock', '-L', action='store_true', default=False, + # help='Do not set DVC locker') + + def set_lock_action(self, parser): + parser.add_argument('-l', '--lock', action='store_true', default=False, + help='Lock data item - disable reproduction. ' + \ + 'It can be enabled by `dvc lock` command or by forcing reproduction.') @property def skip_git_actions(self): diff --git a/dvc/command/import_bulk.py b/dvc/command/import_bulk.py index c989b0f76..e70c40999 100644 --- a/dvc/command/import_bulk.py +++ b/dvc/command/import_bulk.py @@ -13,6 +13,7 @@ class CmdImportBulk(CmdBase): def define_args(self, parser): self.set_skip_git_actions(parser) + self.set_lock_action(parser) parser.add_argument('input', metavar='', @@ -21,8 +22,6 @@ class CmdImportBulk(CmdBase): self.add_string_arg(parser, 'output', 'Output file') - parser.add_argument('-r', '--is-reproducible', action='store_false', default=True, - help='Is data file reproducible') pass def run(self): @@ -44,7 +43,7 @@ class CmdImportBulk(CmdBase): output = self.parsed_args.output for input in self.parsed_args.input: if not os.path.isdir(input): - cmd.import_and_commit_if_needed(input, output, self.parsed_args.is_reproducible) + cmd.import_and_commit_if_needed(input, output, self.parsed_args.lock) else: input_dir = os.path.basename(input) for root, dirs, files in os.walk(input): @@ -58,7 +57,7 @@ class CmdImportBulk(CmdBase): if not os.path.exists(out_dir): os.mkdir(out_dir) - cmd.import_and_commit_if_needed(filename, out, self.parsed_args.is_reproducible) + cmd.import_and_commit_if_needed(filename, out, self.parsed_args.lock) pass finally: if self.is_locker: diff --git a/dvc/command/import_file.py b/dvc/command/import_file.py index 67718b8cf..f60ff1bc3 100644 --- a/dvc/command/import_file.py +++ b/dvc/command/import_file.py @@ -43,22 +43,22 @@ class CmdImportFile(CmdBase): try: return self.import_and_commit_if_needed(self.parsed_args.input, self.parsed_args.output, - self.parsed_args.is_reproducible) + self.parsed_args.lock) finally: if self.is_locker: lock.release() pass - def import_and_commit_if_needed(self, input, output, is_reproducible=True, check_if_ready=True): + def import_and_commit_if_needed(self, input, output, lock=False, check_if_ready=True): if check_if_ready and not self.skip_git_actions and not self.git.is_ready_to_go(): return 1 - self.import_file(input, output, is_reproducible) + self.import_file(input, output, lock) message = 'DVC import file: {} {}'.format(input, output) return self.commit_if_needed(message) - def import_file(self, input, output, is_reproducible=True): + def import_file(self, input, output, lock=False): if not CmdImportFile.is_url(input): if not os.path.exists(input): raise ImportFileError('Input file "{}" does not exist'.format(input)) @@ -100,7 +100,7 @@ class CmdImportFile(CmdBase): argv=[input, output], input_files=[], output_files=[output], - is_reproducible=is_reproducible) + is_reproducible=lock) state_file.save() Logger.debug('State file "{}" was created'.format(data_item.state.relative)) pass diff --git a/dvc/command/lock.py b/dvc/command/lock.py new file mode 100644 index 000000000..f47d66fcf --- /dev/null +++ b/dvc/command/lock.py @@ -0,0 +1,61 @@ +import fasteners +from dvc.command.base import CmdBase +from dvc.logger import Logger +from dvc.runtime import Runtime +from dvc.state_file import StateFile + + +class CmdLock(CmdBase): + def __init__(self, settings): + super(CmdLock, self).__init__(settings) + + def define_args(self, parser): + self.set_skip_git_actions(parser) + + parser.add_argument('-u', '--unlock', action='store_true', default=False, + help='Unlock data item - enable reproduction.') + + parser.add_argument('files', metavar='', help='Data items to lock or unlock.', nargs='*') + pass + + def run(self): + if self.is_locker: + lock = fasteners.InterProcessLock(self.git.lock_file) + gotten = lock.acquire(timeout=5) + if not gotten: + self.warning_dvc_is_busy() + return 1 + + try: + + return self.lock_files(self.parsed_args.files, not self.parsed_args.unlock) + finally: + if self.is_locker: + lock.release() + + def lock_files(self, files, target): + cmd = 'lock' if target else 'unlock' + + for file in files: + try: + data_item = self.settings.path_factory.existing_data_item(file) + state = StateFile.load(data_item.state.relative, self.settings) + + if state.lock and target: + Logger.warn('Data item {} is already locked'.format(data_item.data.relative)) + elif not state.lock and not target: + Logger.warn('Data item {} is already unlocked'.format(data_item.data.relative)) + else: + state.lock = target + Logger.debug('Saving status file for data item {}'.format(data_item.data.relative)) + state.save() + Logger.info('Data item {} was {}ed'.format(data_item.data.relative, cmd)) + except Exception as ex: + Logger.error('Unable to {} {}: {}'.format(cmd, file, ex)) + raise + + return 0 + + +if __name__ == '__main__': + Runtime.run(CmdLock, False) diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 84aefa32e..0508526a3 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -39,7 +39,7 @@ class CmdRepro(CmdRun): pass @property - def is_reproducible(self): + def lock(self): return True @property @@ -217,7 +217,7 @@ class ReproChange(object): Logger.debug('Reproduce data item {}. recursive={}, force={}'.format( self._data_item.data.relative, self._recursive, self._force)) - if not self.state.is_reproducible: + if not self.state.lock: Logger.debug('Data item {} is not reproducible'.format(self._data_item.data.relative)) return False diff --git a/dvc/command/run.py b/dvc/command/run.py index abeb964ab..5d06f47ba 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -23,6 +23,7 @@ class CmdRun(CmdBase): def define_args(self, parser): self.set_skip_git_actions(parser) + self.set_lock_action(parser) parser.add_argument('--not-repro', help='Not reproducible', action='store_false', default=False) @@ -38,8 +39,8 @@ class CmdRun(CmdBase): pass @property - def is_reproducible(self): - return not self.parsed_args.not_repro + def lock(self): + return not self.parsed_args.lock @property def code_dependencies(self): @@ -125,7 +126,7 @@ class CmdRun(CmdBase): output_files_dvc, code_dependencies_dvc, argv=cmd_args, - is_reproducible=self.is_reproducible, + lock=self.lock, stdout=self._stdout_to_dvc(stdout), stderr=self._stdout_to_dvc(stderr), shell=shell) diff --git a/dvc/runtime.py b/dvc/runtime.py index 5e543aca7..47ea3d010 100644 --- a/dvc/runtime.py +++ b/dvc/runtime.py @@ -17,7 +17,7 @@ class Runtime(object): return System.realpath(os.path.join(git_dir, Runtime.CONFIG)) @staticmethod - def run(cmd_class, parse_config=True, args_start_loc=1): + def run(cmd_class, parse_config=True, args_start_loc=2): """ Arguments: diff --git a/dvc/state_file.py b/dvc/state_file.py index 3cef69266..83b767364 100644 --- a/dvc/state_file.py +++ b/dvc/state_file.py @@ -34,7 +34,7 @@ class StateFile(object): PARAM_INPUT_FILES = 'InputFiles' PARAM_OUTPUT_FILES = 'OutputFiles' PARAM_CODE_DEPENDENCIES = 'CodeDependencies' - PARAM_NOT_REPRODUCIBLE = 'NotReproducible' + PARAM_LOCK = 'Lock' PARAM_STDOUT = "Stdout" PARAM_STDERR = "Stderr" PARAM_SHELL = "Shell" @@ -46,7 +46,7 @@ class StateFile(object): input_files, output_files, code_dependencies=[], - is_reproducible=True, + lock=False, argv=sys.argv, stdout=None, stderr=None, @@ -57,7 +57,7 @@ class StateFile(object): self.settings = settings self.input_files = input_files self.output_files = output_files - self.is_reproducible = is_reproducible + self.lock = lock self.code_dependencies = code_dependencies self.shell = shell @@ -91,17 +91,17 @@ class StateFile(object): return self._argv @staticmethod - def load(filename, git): + def load(filename, settings): with open(filename, 'r') as fd: data = json.load(fd) return StateFile(data.get(StateFile.PARAM_COMMAND), filename, - git, + settings, data.get(StateFile.PARAM_INPUT_FILES, []), data.get(StateFile.PARAM_OUTPUT_FILES, []), data.get(StateFile.PARAM_CODE_DEPENDENCIES, []), - not data.get(StateFile.PARAM_NOT_REPRODUCIBLE, False), + data.get(StateFile.PARAM_LOCK, False), data.get(StateFile.PARAM_ARGV), data.get(StateFile.PARAM_STDOUT), data.get(StateFile.PARAM_STDERR), @@ -110,7 +110,6 @@ class StateFile(object): data.get(StateFile.PARAM_SHELL, False)) def save(self): - # cmd, argv = self.process_args(self._argv) argv = self._argv_paths_normalization(self._argv) res = { @@ -128,8 +127,8 @@ class StateFile(object): self.PARAM_SHELL: self.shell } - if not self.is_reproducible: - res[self.PARAM_NOT_REPRODUCIBLE] = True + if self.lock: + res[self.PARAM_LOCK] = True file_dir = os.path.dirname(self.file) if file_dir != '' and not os.path.isdir(file_dir):