Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-01 23:08:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

New command: lock

Browse Source
This commit is contained in:
Dmitry Petrov
2017-04-29 15:12:31 -07:00
parent b9a7c737e1
commit 223380d299
8 changed files with 91 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/command/base.py
+7 -2
View File
@@ -52,8 +52,13 @@ class CmdBase(object):
def set_skip_git_actions(self, parser):
parser.add_argument('--skip-git-actions', '-s', action='store_true', default=False,
help='Skip all git actions including reproducibility check and commits')
parser.add_argument('--no-lock', '-L', action='store_true', default=False,
help='Do not set DVC locker')
# parser.add_argument('--no-lock', '-L', action='store_true', default=False,
# help='Do not set DVC locker')
def set_lock_action(self, parser):
parser.add_argument('-l', '--lock', action='store_true', default=False,
help='Lock data item - disable reproduction. ' + \
'It can be enabled by `dvc lock` command or by forcing reproduction.')
@property
def skip_git_actions(self):
dvc/command/import_bulk.py
+3 -4
View File
@@ -13,6 +13,7 @@ class CmdImportBulk(CmdBase):
def define_args(self, parser):
self.set_skip_git_actions(parser)
self.set_lock_action(parser)
parser.add_argument('input',
metavar='',
@@ -21,8 +22,6 @@ class CmdImportBulk(CmdBase):
self.add_string_arg(parser, 'output', 'Output file')
parser.add_argument('-r', '--is-reproducible', action='store_false', default=True,
help='Is data file reproducible')
pass
def run(self):
@@ -44,7 +43,7 @@ class CmdImportBulk(CmdBase):
output = self.parsed_args.output
for input in self.parsed_args.input:
if not os.path.isdir(input):
cmd.import_and_commit_if_needed(input, output, self.parsed_args.is_reproducible)
cmd.import_and_commit_if_needed(input, output, self.parsed_args.lock)
else:
input_dir = os.path.basename(input)
for root, dirs, files in os.walk(input):
@@ -58,7 +57,7 @@ class CmdImportBulk(CmdBase):
if not os.path.exists(out_dir):
os.mkdir(out_dir)
cmd.import_and_commit_if_needed(filename, out, self.parsed_args.is_reproducible)
cmd.import_and_commit_if_needed(filename, out, self.parsed_args.lock)
pass
finally:
if self.is_locker:
dvc/command/import_file.py
+5 -5
View File
@@ -43,22 +43,22 @@ class CmdImportFile(CmdBase):
try:
return self.import_and_commit_if_needed(self.parsed_args.input,
self.parsed_args.output,
self.parsed_args.is_reproducible)
self.parsed_args.lock)
finally:
if self.is_locker:
lock.release()
pass
def import_and_commit_if_needed(self, input, output, is_reproducible=True, check_if_ready=True):
def import_and_commit_if_needed(self, input, output, lock=False, check_if_ready=True):
if check_if_ready and not self.skip_git_actions and not self.git.is_ready_to_go():
return 1
self.import_file(input, output, is_reproducible)
self.import_file(input, output, lock)
message = 'DVC import file: {} {}'.format(input, output)
return self.commit_if_needed(message)
def import_file(self, input, output, is_reproducible=True):
def import_file(self, input, output, lock=False):
if not CmdImportFile.is_url(input):
if not os.path.exists(input):
raise ImportFileError('Input file "{}" does not exist'.format(input))
@@ -100,7 +100,7 @@ class CmdImportFile(CmdBase):
argv=[input, output],
input_files=[],
output_files=[output],
is_reproducible=is_reproducible)
is_reproducible=lock)
state_file.save()
Logger.debug('State file "{}" was created'.format(data_item.state.relative))
pass
dvc/command/lock.py
+61
View File
@@ -0,0 +1,61 @@
import fasteners
from dvc.command.base import CmdBase
from dvc.logger import Logger
from dvc.runtime import Runtime
from dvc.state_file import StateFile
class CmdLock(CmdBase):
def __init__(self, settings):
super(CmdLock, self).__init__(settings)
def define_args(self, parser):
self.set_skip_git_actions(parser)
parser.add_argument('-u', '--unlock', action='store_true', default=False,
help='Unlock data item - enable reproduction.')
parser.add_argument('files', metavar='', help='Data items to lock or unlock.', nargs='*')
pass
def run(self):
if self.is_locker:
lock = fasteners.InterProcessLock(self.git.lock_file)
gotten = lock.acquire(timeout=5)
if not gotten:
self.warning_dvc_is_busy()
return 1
try:
return self.lock_files(self.parsed_args.files, not self.parsed_args.unlock)
finally:
if self.is_locker:
lock.release()
def lock_files(self, files, target):
cmd = 'lock' if target else 'unlock'
for file in files:
try:
data_item = self.settings.path_factory.existing_data_item(file)
state = StateFile.load(data_item.state.relative, self.settings)
if state.lock and target:
Logger.warn('Data item {} is already locked'.format(data_item.data.relative))
elif not state.lock and not target:
Logger.warn('Data item {} is already unlocked'.format(data_item.data.relative))
else:
state.lock = target
Logger.debug('Saving status file for data item {}'.format(data_item.data.relative))
state.save()
Logger.info('Data item {} was {}ed'.format(data_item.data.relative, cmd))
except Exception as ex:
Logger.error('Unable to {} {}: {}'.format(cmd, file, ex))
raise
return 0
if __name__ == '__main__':
Runtime.run(CmdLock, False)
dvc/command/repro.py
+2 -2
View File
@@ -39,7 +39,7 @@ class CmdRepro(CmdRun):
pass
@property
def is_reproducible(self):
def lock(self):
return True
@property
@@ -217,7 +217,7 @@ class ReproChange(object):
Logger.debug('Reproduce data item {}. recursive={}, force={}'.format(
self._data_item.data.relative, self._recursive, self._force))
if not self.state.is_reproducible:
if not self.state.lock:
Logger.debug('Data item {} is not reproducible'.format(self._data_item.data.relative))
return False
dvc/command/run.py
+4 -3
View File
@@ -23,6 +23,7 @@ class CmdRun(CmdBase):
def define_args(self, parser):
self.set_skip_git_actions(parser)
self.set_lock_action(parser)
parser.add_argument('--not-repro', help='Not reproducible',
action='store_false', default=False)
@@ -38,8 +39,8 @@ class CmdRun(CmdBase):
pass
@property
def is_reproducible(self):
return not self.parsed_args.not_repro
def lock(self):
return not self.parsed_args.lock
@property
def code_dependencies(self):
@@ -125,7 +126,7 @@ class CmdRun(CmdBase):
output_files_dvc,
code_dependencies_dvc,
argv=cmd_args,
is_reproducible=self.is_reproducible,
lock=self.lock,
stdout=self._stdout_to_dvc(stdout),
stderr=self._stdout_to_dvc(stderr),
shell=shell)
dvc/runtime.py
+1 -1
View File
@@ -17,7 +17,7 @@ class Runtime(object):
return System.realpath(os.path.join(git_dir, Runtime.CONFIG))
@staticmethod
def run(cmd_class, parse_config=True, args_start_loc=1):
def run(cmd_class, parse_config=True, args_start_loc=2):
"""
Arguments:
dvc/state_file.py
+8 -9
View File
@@ -34,7 +34,7 @@ class StateFile(object):
PARAM_INPUT_FILES = 'InputFiles'
PARAM_OUTPUT_FILES = 'OutputFiles'
PARAM_CODE_DEPENDENCIES = 'CodeDependencies'
PARAM_NOT_REPRODUCIBLE = 'NotReproducible'
PARAM_LOCK = 'Lock'
PARAM_STDOUT = "Stdout"
PARAM_STDERR = "Stderr"
PARAM_SHELL = "Shell"
@@ -46,7 +46,7 @@ class StateFile(object):
input_files,
output_files,
code_dependencies=[],
is_reproducible=True,
lock=False,
argv=sys.argv,
stdout=None,
stderr=None,
@@ -57,7 +57,7 @@ class StateFile(object):
self.settings = settings
self.input_files = input_files
self.output_files = output_files
self.is_reproducible = is_reproducible
self.lock = lock
self.code_dependencies = code_dependencies
self.shell = shell
@@ -91,17 +91,17 @@ class StateFile(object):
return self._argv
@staticmethod
def load(filename, git):
def load(filename, settings):
with open(filename, 'r') as fd:
data = json.load(fd)
return StateFile(data.get(StateFile.PARAM_COMMAND),
filename,
git,
settings,
data.get(StateFile.PARAM_INPUT_FILES, []),
data.get(StateFile.PARAM_OUTPUT_FILES, []),
data.get(StateFile.PARAM_CODE_DEPENDENCIES, []),
not data.get(StateFile.PARAM_NOT_REPRODUCIBLE, False),
data.get(StateFile.PARAM_LOCK, False),
data.get(StateFile.PARAM_ARGV),
data.get(StateFile.PARAM_STDOUT),
data.get(StateFile.PARAM_STDERR),
@@ -110,7 +110,6 @@ class StateFile(object):
data.get(StateFile.PARAM_SHELL, False))
def save(self):
# cmd, argv = self.process_args(self._argv)
argv = self._argv_paths_normalization(self._argv)
res = {
@@ -128,8 +127,8 @@ class StateFile(object):
self.PARAM_SHELL: self.shell
}
if not self.is_reproducible:
res[self.PARAM_NOT_REPRODUCIBLE] = True
if self.lock:
res[self.PARAM_LOCK] = True
file_dir = os.path.dirname(self.file)
if file_dir != '' and not os.path.isdir(file_dir):