Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Finalize ReproCodeDependencyTest

Browse Source
This commit is contained in:
Dmitry Petrov
2017-04-03 15:02:49 -07:00
parent dccc15c0d6
commit 84218deccb
6 changed files with 158 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/command/base.py
+4
View File
@@ -108,3 +108,7 @@ class CmdBase(object):
file_name = os.path.relpath(cache_prefix_file_name, self.config.cache_dir)
dvc_file_path_trim = file_name.replace(os.sep, '/').strip('/')
return self.config.aws_storage_prefix + '/' + dvc_file_path_trim
@staticmethod
def warning_dvc_is_busy():
Logger.warn('Cannot perform the cmd since DVC is busy and locked. Please retry the cmd later.')
dvc/command/repro.py
+35 -23
View File
@@ -30,7 +30,7 @@ class CmdRepro(CmdRun):
def define_args(self, parser):
self.set_skip_git_actions(parser)
parser.add_argument('target', metavar='', help='Data file to reproduce', nargs='*')
parser.add_argument('target', metavar='', help='Data item to reproduce', nargs='*')
pass
@@ -50,7 +50,7 @@ class CmdRepro(CmdRun):
lock = fasteners.InterProcessLock(self.git.lock_file)
gotten = lock.acquire(timeout=5)
if not gotten:
Logger.printing('Cannot perform the cmd since DVC is busy and locked. Please retry the cmd later.')
self.warning_dvc_is_busy()
return 1
try:
@@ -79,15 +79,15 @@ class CmdRepro(CmdRun):
repro_change = ReproChange(data_item, self)
if repro_change.reproduce():
changed = True
Logger.info(u'Data file "{}" was reproduced.'.format(
Logger.info(u'Data item "{}" was reproduced.'.format(
data_item.data.relative
))
else:
Logger.info(u'Reproduction is not required for data file "{}".'.format(
Logger.info(u'Reproduction is not required for data item "{}".'.format(
data_item.data.relative
))
except ReproError as err:
Logger.error('Error in reproducing data file {}: {}'.format(
Logger.error('Error in reproducing data item {}: {}'.format(
data_item.data.relative, str(err)
))
error = True
@@ -121,48 +121,60 @@ class ReproChange(object):
self._repro_argv = argv
pass
@property
def cmd_obj(self):
return self._cmd_obj
@property
def state(self):
return self._state
def reproduce_data_file(self):
Logger.debug('Reproducing data file "{}". Removing the file...'.format(
self._data_item.data.relative))
Logger.debug('Reproducing data item "{}". Removing the file...'.format(
self._data_item.data.dvc))
os.remove(self._data_item.data.relative)
Logger.debug('Reproducing data file "{}". Re-runs cmd: {}'.format(
Logger.debug('Reproducing data item "{}". Re-runs cmd: {}'.format(
self._data_item.data.relative, ' '.join(self._repro_argv)))
data_items_from_args = self._cmd_obj._data_items_from_args(self._repro_argv)
return self._cmd_obj.run_command(self._repro_argv, data_items_from_args)
data_items_from_args = self.cmd_obj.data_items_from_args(self._repro_argv)
return self.cmd_obj.run_command(self._repro_argv,
data_items_from_args,
self.state.stdout,
self.state.stderr)
def reproduce(self, force=False):
print('+++++++++++++++ reproduce', self._data_item.data.dvc)
Logger.debug('Reproduce data item {} with dependencies: {}'.format(
self._data_item.data.dvc,
', '.join([x.data.dvc for x in self.dependencies])))
were_input_files_changed = False
if not self._state.is_reproducible:
Logger.debug('Data file "{}" is not reproducible'.format(self._data_item.data.relative))
if not self.state.is_reproducible:
Logger.debug('Data item "{}" is not reproducible'.format(self._data_item.data.relative))
return False
print('+++++++++++++++ reproduce dependencies: '.format(self.get_dependencies()))
for data_item in self.get_dependencies():
for data_item in self.dependencies:
change = ReproChange(data_item, self._cmd_obj)
if change.reproduce(force):
were_input_files_changed = True
was_source_code_changed = self.git.were_files_changed(self._data_item.data.relative,
self._state.code_dependencies)
self.state.code_dependencies)
if was_source_code_changed:
Logger.debug('Source code was changed for "{}"'.format(self._data_item.data.dvc))
if not force and not was_source_code_changed and not were_input_files_changed:
Logger.debug('Data file "{}" is up to date'.format(
Logger.debug('Data item "{}" is up to date'.format(
self._data_item.data.relative))
return False
return self.reproduce_data_file()
def get_dependencies(self):
print('+++++++++++++== INPUT FILES: {}'.format(self._state.input_files))
@property
def dependencies(self):
dependency_data_items = []
for input_file in self._state.input_files:
for input_file in self.state.input_files:
try:
data_item = self._cmd_obj.path_factory.data_item(input_file)
except NotInDataDirError:
dvc/command/run.py
+22 -9
View File
@@ -1,17 +1,17 @@
import os
import sys
import shutil
import fasteners
from dvc.path.data_item import NotInDataDirError
from dvc.git_wrapper import GitWrapper
from dvc.command.base import CmdBase
from dvc.logger import Logger
from dvc.exceptions import DvcException
from dvc.git_wrapper import GitWrapper
from dvc.logger import Logger
from dvc.path.data_item import NotInDataDirError
from dvc.repository_change import RepositoryChange
from dvc.state_file import StateFile
from dvc.utils import run
from dvc.utils import cached_property
from dvc.utils import run
class RunError(DvcException):
@@ -61,7 +61,7 @@ class CmdRun(CmdBase):
lock = fasteners.InterProcessLock(self.git.lock_file)
gotten = lock.acquire(timeout=5)
if not gotten:
Logger.printing('Cannot perform the cmd since DVC is busy and locked. Please retry the cmd later.')
self.warning_dvc_is_busy()
return 1
try:
@@ -69,7 +69,7 @@ class CmdRun(CmdBase):
return 1
self.run_command(self._args_unkn,
self._data_items_from_args(self._args_unkn),
self.data_items_from_args(self._args_unkn),
self.args.stdout,
self.args.stderr)
return self.commit_if_needed('DVC run: {}'.format(' '.join(sys.argv)))
@@ -79,6 +79,12 @@ class CmdRun(CmdBase):
return 1
def run_command(self, argv, data_items_from_args, stdout=None, stderr=None):
Logger.debug('Run command: {}. Data items from args: {}. stdout={}, stderr={}'.format(
' '.join(argv),
', '.join([x.data.dvc for x in data_items_from_args]),
stdout,
stderr))
repo_change = RepositoryChange(argv, stdout, stderr, self.git, self.config, self.path_factory)
if not self.skip_git_actions and not self._validate_file_states(repo_change):
@@ -106,12 +112,19 @@ class CmdRun(CmdBase):
output_files_dvc,
code_dependencies_dvc,
argv=argv,
is_reproducible=self.is_reproducible)
is_reproducible=self.is_reproducible,
stdout=self._stdout_to_dvc(stdout),
stderr=self._stdout_to_dvc(stderr))
state_file.save()
result.append(state_file)
return result
def _stdout_to_dvc(self, stdout):
if stdout in {None, '-'}:
return stdout
return self.path_factory.data_item(stdout).data.dvc
@staticmethod
def remove_new_files(repo_change):
for data_item in repo_change.new_data_items:
@@ -132,7 +145,7 @@ class CmdRun(CmdBase):
return not error
def _data_items_from_args(self, argv):
def data_items_from_args(self, argv):
result = []
for arg in argv:
dvc/repository_change.py
+8
View File
@@ -3,6 +3,7 @@ import os
from dvc.exceptions import DvcException
from dvc.git_wrapper import GitWrapper
from dvc.executor import Executor
from dvc.logger import Logger
from dvc.path.data_item import NotInDataDirError
from dvc.path.stated_data_item import StatedDataItem
from dvc.utils import cached_property
@@ -21,6 +22,10 @@ class RepositoryChange(object):
self.config = config
self.path_factory = path_factory
Logger.debug(u'[Repository change] Exec command: {}. stdout={}, stderr={}'.format(
u' '.join(args),
stdout,
stderr))
Executor.exec_cmd_only_success(args, stdout, stderr)
self._stated_data_items = []
@@ -55,6 +60,9 @@ class RepositoryChange(object):
try:
item = self.path_factory.stated_data_item(state, file)
self._stated_data_items.append(item)
Logger.debug('[Repository change] Add status: {} {}'.format(
item.status,
item.data.dvc))
except NotInDataDirError:
self._externally_created_files.append(file)
pass
dvc/state_file.py
+15 -4
View File
@@ -25,11 +25,15 @@ class StateFile(object):
PARAM_OUTPUT_FILES = 'OutputFiles'
PARAM_CODE_DEPENDENCIES = 'CodeDependencies'
PARAM_NOT_REPRODUCIBLE = 'NotReproducible'
PARAM_STDOUT = "Stdout"
PARAM_STDERR = "Stderr"
def __init__(self, file, git, input_files, output_files,
code_dependencies=[],
is_reproducible=True,
argv=sys.argv,
stdout=None,
stderr=None,
norm_argv=None,
created_at=time.strftime('%Y-%m-%d %H:%M:%S %z'),
cwd=None):
@@ -46,6 +50,9 @@ class StateFile(object):
else:
self.norm_argv = self.normalized_args()
self.stdout = stdout
self.stderr = stderr
self.created_at = created_at
if cwd:
@@ -66,6 +73,8 @@ class StateFile(object):
data.get(StateFile.PARAM_CODE_DEPENDENCIES, []),
not data.get(StateFile.PARAM_NOT_REPRODUCIBLE, False),
data.get(StateFile.PARAM_ARGV),
data.get(StateFile.PARAM_STDOUT),
data.get(StateFile.PARAM_STDERR),
data.get(StateFile.PARAM_NORM_ARGV),
data.get(StateFile.PARAM_CREATED_AT),
data.get(StateFile.PARAM_CWD))
@@ -80,7 +89,9 @@ class StateFile(object):
self.PARAM_CREATED_AT: self.created_at,
self.PARAM_INPUT_FILES: self.input_files,
self.PARAM_OUTPUT_FILES: self.output_files,
self.PARAM_CODE_DEPENDENCIES: self.code_dependencies
self.PARAM_CODE_DEPENDENCIES: self.code_dependencies,
self.PARAM_STDOUT: self.stdout,
self.PARAM_STDERR: self.stderr
}
if not self.is_reproducible:
@@ -98,14 +109,14 @@ class StateFile(object):
result = []
if len(self.argv) > 0:
cmd = self.argv [0]
cmd = self.argv[0]
pos = cmd.rfind(os.sep)
if pos >= 0:
cmd = cmd[pos+1:]
result.append(cmd)
for arg in self.argv [1:]:
if os.path.isfile(arg):
for arg in self.argv[1:]:
if os.path.isfile(arg): # CHANGE to data items
path = os.path.abspath(arg)
dvc_path = os.path.relpath(path, self.git.git_dir_abs)
result.append(dvc_path)
tests/test_repro.py
+74 -43
View File
@@ -60,12 +60,24 @@ from tests.test_cmd_run import RunBasicTest
class ReproBasicEnv(RunBasicTest):
def setUp(self):
super(ReproBasicEnv, self).setUp()
self.file_name1 = 'data/file1'
CmdRun(args=['printf', 'Hello\nWorld!', '--not-repro', '--stdout', self.file_name1],
parse_config=False,
config_obj=self.config,
git_obj=self.git
).run()
self.file1_code_file = 'file1.py'
self.create_file_and_commit(self.file1_code_file, 'An awesome code...')
cmd_file1 = CmdRun(args=[
'printf',
'Hello\nMary',
'--not-repro',
'--stdout',
self.file_name1,
'--code',
self.file1_code_file
],
parse_config=False,
config_obj=self.config,
git_obj=self.git)
self.assertEqual(cmd_file1.code_dependencies, [self.file1_code_file])
cmd_file1.run()
self.file_name11 = 'data/file11'
CmdRun(args=['head', '-n', '1', self.file_name1, '--stdout', self.file_name11],
@@ -81,54 +93,73 @@ class ReproBasicEnv(RunBasicTest):
git_obj=self.git
).run()
self.file_res_code_file = 'code_res.py'
self.create_file_and_commit(self.file_res_code_file, 'Another piece of code')
self.file_name_res = 'data/file_res'
CmdRun(args=['cat', self.file_name11, self.file_name2, '--stdout', self.file_name_res],
parse_config=False,
config_obj=self.config,
git_obj=self.git
).run()
cmd_res = CmdRun(args=[
'cat',
self.file_name11,
self.file_name2,
'--stdout',
self.file_name_res,
'--code',
self.file_res_code_file
],
parse_config=False,
config_obj=self.config,
git_obj=self.git)
self.assertEqual(cmd_res.code_dependencies, [self.file_res_code_file])
cmd_res.run()
self.assertEqual(open(self.file_name_res).read(), 'Hello\nBobby')
#
# class ReproRemovedFileTest(ReproBasicEnv):
# def test(self):
# # CmdDataRemove(args=[self.file_name_res, '--keep-in-cloud'],
# # config_obj=self.config,
# # git_obj=self.git
# # ).run()
#
# # CmdRun(args=['printf', 'Goodbye\nWorld!', '--stdout', self.file_name1],
# # config_obj=self.config,
# # git_obj=self.git
# # ).run()
#
# CmdRepro(args=[self.file_name_res],
# config_obj=self.config,
# git_obj=self.git
# ).run()
#
# self.assertEqual(open(self.file_name_res).read(), 'Hello\nBobby')
def create_file_and_commit(self, file_name, content='Any', message='Just a commit'):
self.create_file(file_name, content)
self.commit_file(file_name, message)
@staticmethod
def commit_file(file_name, message='Just a commit'):
Executor.exec_cmd_only_success(['git', 'add', file_name])
Executor.exec_cmd_only_success(['git', 'commit', '-m', message])
def modify_file_and_commit(self, filename, content_to_add=' '):
fd = open(filename, 'a')
fd.write(content_to_add)
fd.close()
self.commit_file(filename)
class ReproChangedDependency(ReproBasicEnv):
class ReproCodeDependencyTest(ReproBasicEnv):
def test(self):
self.assertEqual(open(self.file_name_res).read(), 'Hello\nBobby')
CmdDataRemove(args=[self.file_name1, '--keep-in-cloud'],
config_obj=self.config,
git_obj=self.git
).run()
CmdRun(args=['printf', 'Goodbye\nWorld!', '--stdout', self.file_name1],
config_obj=self.config,
git_obj=self.git
).run()
self.modify_file_and_commit(self.file_res_code_file)
CmdRepro(args=[self.file_name_res],
config_obj=self.config,
git_obj=self.git
).run()
# self.assertEqual(open(self.file_name_res).read(), 'Goodbye\nBobby')
self.assertEqual(open(self.file_name_res).read(), 'Hello\nBobby')
# class ReproChangedDependency(ReproBasicEnv):
# def test(self):
#
# self.assertEqual(open(self.file_name_res).read(), 'Hello\nBobby')
#
# CmdDataRemove(args=[self.file_name1, '--keep-in-cloud'],
# config_obj=self.config,
# git_obj=self.git
# ).run()
#
# CmdRun(args=['printf', 'Goodbye\nWorld!', '--stdout', self.file_name1],
# config_obj=self.config,
# git_obj=self.git
# ).run()
#
# CmdRepro(args=[self.file_name_res],
# config_obj=self.config,
# git_obj=self.git
# ).run()
#
# # self.assertEqual(open(self.file_name_res).read(), 'Goodbye\nBobby')