Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-01 23:08:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

Lock cmd: small changes

Browse Source
This commit is contained in:
Dmitry Petrov
2017-04-29 18:01:24 -07:00
parent 6ec0025d79
commit f6ded2fa72
12 changed files with 76 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/command/base.py
+9 -11
View File
@@ -49,23 +49,21 @@ class CmdBase(object):
def define_args(self, parser):
pass
def set_skip_git_actions(self, parser):
parser.add_argument('--skip-git-actions', '-s', action='store_true', default=False,
help='Skip all git actions including reproducibility check and commits')
# parser.add_argument('--no-lock', '-L', action='store_true', default=False,
# help='Do not set DVC locker')
def set_no_git_actions(self, parser):
parser.add_argument('--no-git-actions', '-G', action='store_true', default=False,
help='Skip all git actions including reproducibility check and commits.')
def set_lock_action(self, parser):
parser.add_argument('-l', '--lock', action='store_true', default=False,
help='Lock data item - disable reproduction. ' + \
parser.add_argument('--lock', '-l', action='store_true', default=False,
help='Lock data item - disable reproduction. ' +
'It can be enabled by `dvc lock` command or by forcing reproduction.')
@property
def skip_git_actions(self):
return self.parsed_args.skip_git_actions
def no_git_actions(self):
return self.parsed_args.no_git_actions
def set_git_action(self, value):
self.parsed_args.skip_git_actions = not value
self.parsed_args.no_git_actions = not value
@property
def is_locker(self):
@@ -77,7 +75,7 @@ class CmdBase(object):
self.parsed_args.no_lock = value
def commit_if_needed(self, message, error=False):
if error or self.skip_git_actions:
if error or self.no_git_actions:
self.not_committed_changes_warning()
return 1
else:
dvc/command/import_bulk.py
+3 -3
View File
@@ -12,7 +12,7 @@ class CmdImportBulk(CmdBase):
super(CmdImportBulk, self).__init__(settings)
def define_args(self, parser):
self.set_skip_git_actions(parser)
self.set_no_git_actions(parser)
self.set_lock_action(parser)
parser.add_argument('input',
@@ -33,11 +33,11 @@ class CmdImportBulk(CmdBase):
return 1
try:
if not self.skip_git_actions and not self.git.is_ready_to_go():
if not self.no_git_actions and not self.git.is_ready_to_go():
return 1
cmd = CmdImportFile(self.settings)
cmd.set_git_action(not self.skip_git_actions)
cmd.set_git_action(not self.no_git_actions)
cmd.set_locker(False)
output = self.parsed_args.output
dvc/command/import_file.py
+3 -4
View File
@@ -23,7 +23,7 @@ class CmdImportFile(CmdBase):
super(CmdImportFile, self).__init__(settings)
def define_args(self, parser):
self.set_skip_git_actions(parser)
self.set_no_git_actions(parser)
self.add_string_arg(parser, 'input', 'Input file')
self.add_string_arg(parser, 'output', 'Output file')
@@ -50,7 +50,7 @@ class CmdImportFile(CmdBase):
pass
def import_and_commit_if_needed(self, input, output, lock=False, check_if_ready=True):
if check_if_ready and not self.skip_git_actions and not self.git.is_ready_to_go():
if check_if_ready and not self.no_git_actions and not self.git.is_ready_to_go():
return 1
self.import_file(input, output, lock)
@@ -93,14 +93,13 @@ class CmdImportFile(CmdBase):
Logger.debug('Creating symlink {} --> {}'.format(data_item.symlink_file, data_item.data.relative))
System.symlink(data_item.symlink_file, data_item.data.relative)
# import_file_argv = [StateFile.DVC_PYTHON_FILE_NAME, StateFile.COMMAND_IMPORT_FILE, input, output]
state_file = StateFile(StateFile.COMMAND_IMPORT_FILE,
data_item.state.relative,
self.settings,
argv=[input, output],
input_files=[],
output_files=[output],
is_reproducible=lock)
lock=lock)
state_file.save()
Logger.debug('State file "{}" was created'.format(data_item.state.relative))
pass
dvc/command/init.py
+6 -6
View File
@@ -60,11 +60,11 @@ ProjectName =
super(CmdInit, self).__init__(settings)
def define_args(self, parser):
self.set_skip_git_actions(parser)
self.set_no_git_actions(parser)
self.add_string_arg(parser, '--data-dir', 'NeatLynx data directory', 'data')
self.add_string_arg(parser, '--cache-dir', 'NeatLynx cache directory', '.cache')
self.add_string_arg(parser, '--state-dir', 'NeatLynx state directory', '.state')
self.add_string_arg(parser, '--data-dir', 'Data directory', 'data')
self.add_string_arg(parser, '--cache-dir', 'Cache directory', '.cache')
self.add_string_arg(parser, '--state-dir', 'State directory', '.state')
pass
def get_not_existing_dir(self, dir):
@@ -80,7 +80,7 @@ ProjectName =
return file_name
def run(self):
if not self.skip_git_actions and not self.git.is_ready_to_go():
if not self.no_git_actions and not self.git.is_ready_to_go():
return 1
if os.path.realpath(os.path.curdir) != self.settings.git.git_dir_abs:
@@ -140,7 +140,7 @@ ProjectName =
self.settings,
input_files=[],
output_files=[],
is_reproducible=False).save()
lock=False).save()
pass
def modify_gitignore(self, cache_dir_name):
dvc/command/lock.py
+4 -4
View File
@@ -10,7 +10,7 @@ class CmdLock(CmdBase):
super(CmdLock, self).__init__(settings)
def define_args(self, parser):
self.set_skip_git_actions(parser)
self.set_no_git_actions(parser)
parser.add_argument('-u', '--unlock', action='store_true', default=False,
help='Unlock data item - enable reproduction.')
@@ -41,12 +41,12 @@ class CmdLock(CmdBase):
data_item = self.settings.path_factory.existing_data_item(file)
state = StateFile.load(data_item.state.relative, self.settings)
if state.lock and target:
if state.locked and target:
Logger.warn('Data item {} is already locked'.format(data_item.data.relative))
elif not state.lock and not target:
elif not state.locked and not target:
Logger.warn('Data item {} is already unlocked'.format(data_item.data.relative))
else:
state.lock = target
state.locked = target
Logger.debug('Saving status file for data item {}'.format(data_item.data.relative))
state.save()
Logger.info('Data item {} was {}ed'.format(data_item.data.relative, cmd))
dvc/command/remove.py
+2 -2
View File
@@ -19,7 +19,7 @@ class CmdDataRemove(CmdBase):
super(CmdDataRemove, self).__init__(settings)
def define_args(self, parser):
self.set_skip_git_actions(parser)
self.set_no_git_actions(parser)
parser.add_argument('target', metavar='', help='Target to remove - file or directory', nargs='*')
parser.add_argument('-r', '--recursive', action='store_true', help='Remove directory recursively')
@@ -47,7 +47,7 @@ class CmdDataRemove(CmdBase):
return 0
def remove_all_targets(self):
if not self.skip_git_actions and not self.git.is_ready_to_go():
if not self.no_git_actions and not self.git.is_ready_to_go():
return False
error = False
dvc/command/repro.py
+28 -24
View File
@@ -29,13 +29,13 @@ class CmdRepro(CmdRun):
return self._code
def define_args(self, parser):
self.set_skip_git_actions(parser)
self.set_no_git_actions(parser)
parser.add_argument('target', metavar='', help='Data items to reproduce.', nargs='*')
parser.add_argument('-f', '--force', action='store_true', default=False,
help='Reproduce even if dependencies were not changed')
parser.add_argument('-r', '--recursive', action='store_true', default=False,
help='Recursively reproduce all changed dependencies')
help='Reproduce even if dependencies were not changed.')
parser.add_argument('-s', '--single-item', action='store_true', default=False,
help='Reproduce only single data item without recursive dependencies check.')
pass
@property
@@ -58,15 +58,15 @@ class CmdRepro(CmdRun):
self.warning_dvc_is_busy()
return 1
try:
return self.repro_target(self.parsed_args.target, self.parsed_args.recursive, self.parsed_args.force)
recursive = not self.parsed_args.single_item
return self.repro_target(self.parsed_args.target, recursive, self.parsed_args.force)
finally:
if self.is_locker:
lock.release()
pass
def repro_target(self, target, recursive, force):
if not self.skip_git_actions and not self.git.is_ready_to_go():
if not self.no_git_actions and not self.git.is_ready_to_go():
return 1
data_item_list, external_files_names = self.settings.path_factory.to_data_items(target)
@@ -77,7 +77,7 @@ class CmdRepro(CmdRun):
if self.repro_data_items(data_item_list, recursive, force):
return 0
pass
return 1
def repro_data_items(self, data_item_list, recursive, force):
error = False
@@ -110,7 +110,7 @@ class CmdRepro(CmdRun):
error = True
break
if error and not self.skip_git_actions:
if error and not self.no_git_actions:
Logger.error('Errors occurred. One or more repro cmd was not successful.')
self.not_committed_changes_warning()
@@ -141,6 +141,9 @@ class ReproChange(object):
if len(self.state.argv) < 1:
raise ReproError('Error: reproducible cmd in state file "{}" is too short'.
format(self.state.file))
self._settings = copy.copy(self._cmd_obj.settings)
self._settings.set_args(self.state.argv)
pass
def is_cache_exists(self):
@@ -168,9 +171,6 @@ class ReproChange(object):
msg = 'Data item {} cannot be removed before reproduction: {}'
Logger.error(msg.format(output_dvc, ex))
settings = copy.copy(self._cmd_obj.settings)
settings.set_args(self.state.argv)
if self.state.is_import_file:
Logger.debug('Reproducing data item {}. Re-import cmd: {}'.format(
self._data_item.data.relative, ' '.join(self.state.argv)))
@@ -182,16 +182,16 @@ class ReproChange(object):
input = self.state.argv[0]
output = self.state.argv[1]
cmd = CmdImportFile(settings)
cmd = CmdImportFile(self._settings)
cmd.set_git_action(True)
cmd.set_locker(False)
Logger.info(u'Reproducing import command: {}'.format(output))
if cmd.import_and_commit_if_needed(input, output, is_reproducible=True, check_if_ready=False) != 0:
if cmd.import_and_commit_if_needed(input, output, lock=True, check_if_ready=False) != 0:
raise ReproError('Import command reproduction failed')
return True
elif self.state.is_run:
cmd = CmdRun(settings)
cmd = CmdRun(self._settings)
cmd.set_git_action(True)
cmd.set_locker(False)
@@ -217,16 +217,15 @@ class ReproChange(object):
Logger.debug('Reproduce data item {}. recursive={}, force={}'.format(
self._data_item.data.relative, self._recursive, self._force))
if not self.state.lock:
if self.state.locked:
Logger.debug('Data item {} is not reproducible'.format(self._data_item.data.relative))
return False
data_item_dvc = self._data_item.data.dvc
if self.is_repro_required(changed_files, data_item_dvc):
if data_item_dvc not in changed_files:
if self.is_repro_required(changed_files, self._data_item):
if self._data_item.data.dvc not in changed_files:
Logger.debug('Data item {} is going to be reproduced'.format(self._data_item.data.relative))
self.reproduce_data_item()
changed_files.add(data_item_dvc)
changed_files.add(self._data_item.data.dvc)
return True
else:
msg = 'Data item {} is not going to be reproduced because it is already reproduced'
@@ -255,7 +254,12 @@ class ReproChange(object):
return result
def is_repro_required(self, changed_files, data_item_dvc):
def is_repro_required(self, changed_files, data_item):
state_file = StateFile.load(data_item.state.relative, self._settings)
if state_file.locked:
Logger.debug(u'Repro is not required for locked data item {}'.format(data_item.data.relative))
return False
is_dependency_check_required = self._recursive
if not is_dependency_check_required and not self.is_cache_exists():
@@ -264,7 +268,7 @@ class ReproChange(object):
self._data_item.data.relative))
if is_dependency_check_required:
if self.were_dependencies_changed(changed_files, data_item_dvc):
if self.were_dependencies_changed(changed_files, data_item.data.dvc):
self.log_repro_reason(u'input dependencies were changed')
return True
@@ -289,7 +293,7 @@ class ReproChange(object):
def were_sources_changed(self, globally_changed_files):
were_sources_changed = self.git.were_files_changed(
self.state.code_dependencies,
self.cmd_obj.settings.path_factory,
self._settings.path_factory,
globally_changed_files
)
return were_sources_changed
@@ -299,7 +303,7 @@ class ReproChange(object):
dependency_data_items = []
for input_file in self.state.input_files:
try:
data_item = self._cmd_obj.settings.path_factory.data_item(input_file)
data_item = self._settings.path_factory.data_item(input_file)
except NotInDataDirError:
raise ReproError(u'The dependency file "{}" is not a data item'.format(input_file))
except Exception as ex:
dvc/command/run.py
+9 -11
View File
@@ -22,25 +22,23 @@ class CmdRun(CmdBase):
super(CmdRun, self).__init__(settings)
def define_args(self, parser):
self.set_skip_git_actions(parser)
self.set_no_git_actions(parser)
self.set_lock_action(parser)
parser.add_argument('--not-repro', help='Not reproducible',
action='store_false', default=False)
parser.add_argument('--stdout', help='output std output to a file')
parser.add_argument('--stderr', help='output std error to a file')
parser.add_argument('--stdout', help='Output std output to a file.')
parser.add_argument('--stderr', help='Output std error to a file.')
parser.add_argument('--input', '-i', action='append',
help='Declare input data items for reproducible cmd')
help='Declare input data items for reproducible cmd.')
parser.add_argument('--output', '-o', action='append',
help='Declare output data items for reproducible cmd')
help='Declare output data items for reproducible cmd.')
parser.add_argument('--code', '-c', action='append',
help='Code dependencies which produce the output')
help='Code dependencies which produce the output.')
parser.add_argument('--shell', help='Shell command', action='store_true', default=False)
pass
@property
def lock(self):
return not self.parsed_args.lock
return self.parsed_args.lock
@property
def code_dependencies(self):
@@ -76,7 +74,7 @@ class CmdRun(CmdBase):
def run_and_commit_if_needed(self, command_args, data_items_from_args, not_data_items_from_args,
stdout, stderr, shell, check_if_ready=True):
if check_if_ready and not self.skip_git_actions and not self.git.is_ready_to_go():
if check_if_ready and not self.no_git_actions and not self.git.is_ready_to_go():
return 1
self.run_command(command_args,
@@ -99,7 +97,7 @@ class CmdRun(CmdBase):
repo_change = RepositoryChange(cmd_args, self.settings, stdout, stderr, shell=shell)
if not self.skip_git_actions and not self._validate_file_states(repo_change):
if not self.no_git_actions and not self._validate_file_states(repo_change):
self.remove_new_files(repo_change)
raise RunError('Errors occurred.')
dvc/command/test.py
+1 -1
View File
@@ -19,7 +19,7 @@ class CmdTest(CmdBase):
super(CmdTest, self).__init__(settings)
def define_args(self, parser):
self.set_skip_git_actions(parser)
self.set_no_git_actions(parser)
def run(self):
dvc/state_file.py
+5 -5
View File
@@ -34,7 +34,7 @@ class StateFile(object):
PARAM_INPUT_FILES = 'InputFiles'
PARAM_OUTPUT_FILES = 'OutputFiles'
PARAM_CODE_DEPENDENCIES = 'CodeDependencies'
PARAM_LOCK = 'Lock'
PARAM_LOCKED = 'Locked'
PARAM_STDOUT = "Stdout"
PARAM_STDERR = "Stderr"
PARAM_SHELL = "Shell"
@@ -57,7 +57,7 @@ class StateFile(object):
self.settings = settings
self.input_files = input_files
self.output_files = output_files
self.lock = lock
self.locked = lock
self.code_dependencies = code_dependencies
self.shell = shell
@@ -101,7 +101,7 @@ class StateFile(object):
data.get(StateFile.PARAM_INPUT_FILES, []),
data.get(StateFile.PARAM_OUTPUT_FILES, []),
data.get(StateFile.PARAM_CODE_DEPENDENCIES, []),
data.get(StateFile.PARAM_LOCK, False),
data.get(StateFile.PARAM_LOCKED, False),
data.get(StateFile.PARAM_ARGV),
data.get(StateFile.PARAM_STDOUT),
data.get(StateFile.PARAM_STDERR),
@@ -127,8 +127,8 @@ class StateFile(object):
self.PARAM_SHELL: self.shell
}
if self.lock:
res[self.PARAM_LOCK] = True
if self.locked:
res[self.PARAM_LOCKED] = True
file_dir = os.path.dirname(self.file)
if file_dir != '' and not os.path.isdir(file_dir):
tests/test_cmd_data_remove.py
+3 -3
View File
@@ -139,7 +139,7 @@ class TestRemoveEndToEnd(DirHierarchyEnvironment):
cmd.parsed_args.target = [dir11_full, self.file5]
cmd.parsed_args.recursive = True
cmd.parsed_args.skip_git_actions = True
cmd.parsed_args.no_git_actions = True
self.assertTrue(os.path.exists(dir11_full))
self.assertTrue(os.path.exists(self.file5))
@@ -163,7 +163,7 @@ class TestRemoveEndToEnd(DirHierarchyEnvironment):
cmd.parsed_args.target = [dir11_full, self.file5]
cmd.parsed_args.recursive = True
cmd.parsed_args.skip_git_actions = True
cmd.parsed_args.no_git_actions = True
self.assertTrue(os.path.exists(dir11_full))
self.assertTrue(os.path.exists(self.file5))
@@ -193,7 +193,7 @@ class TestRemoveEndToEndWithAnError(DirHierarchyEnvironment):
cmd.parsed_args.target = [dir11_full, self.file5, file6_full]
cmd.parsed_args.recursive = True
cmd.parsed_args.skip_git_actions = True
cmd.parsed_args.no_git_actions = True
self.assertTrue(os.path.exists(dir11_full))
self.assertTrue(os.path.exists(self.file5))
tests/test_repro.py
+3 -2
View File
@@ -4,6 +4,7 @@ from dvc.command.remove import CmdDataRemove
from dvc.command.repro import CmdRepro
from dvc.command.run import CmdRun
from dvc.executor import Executor
from dvc.logger import Logger
from tests.test_cmd_run import RunBasicTest
@@ -86,7 +87,7 @@ class ReproChangedDependency(ReproBasicEnv):
def test(self):
self.recreate_file1()
self.settings._args = [self.file_name11, '-r']
self.settings._args = [self.file_name11]
CmdRepro(self.settings).run()
self.assertEqual(open(self.file_name11).read(), 'Goodbye\n')
@@ -107,7 +108,7 @@ class ReproChangedDeepDependency(ReproChangedDependency):
def test(self):
self.recreate_file1()
self.settings._args = [self.file_name_res, '-r']
self.settings._args = [self.file_name_res]
CmdRepro(self.settings).run()
self.assertEqual(open(self.file_name_res).read().strip(), 'Goodbye\nBobby')