Refactoring: extract Settings from BaseCmd; move path_factory to Settings

This commit is contained in:
Dmitry Petrov
2017-04-05 23:08:40 -07:00
parent c3c2116220
commit 5f3c53603f
18 changed files with 190 additions and 216 deletions
+14 -43
View File
@@ -1,53 +1,28 @@
import argparse
import os
import sys
from dvc.git_wrapper import GitWrapper
from dvc.config import Config, ConfigError
from dvc.config import ConfigError
from dvc.logger import Logger
from dvc.path.factory import PathFactory
from dvc.utils import cached_property
class CmdBase(object):
CONFIG = 'dvc.conf'
def __init__(self, args=None, parse_config=True, git_obj=None, config_obj=None):
if git_obj:
self._git = git_obj
else:
self._git = GitWrapper()
if config_obj:
self._config = config_obj
else:
self._config = None
if parse_config:
self._config = Config(os.path.realpath(os.path.join(self.git.git_dir, self.CONFIG)))
if args is None:
self._args = sys.argv[1:]
else:
self._args = args
self._parsed_args = None
self._dvc_home = None
def __init__(self, settings):
self._settings = settings
parser = argparse.ArgumentParser()
self.define_args(parser)
self._parsed_args, self._command_args = parser.parse_known_args(args=self._args)
self._dvc_home = os.environ.get('DVC_HOME')
if not self.lnx_home:
raise ConfigError('DVC_HOME environment variable is not defined')
if not os.path.exists(self._dvc_home):
raise ConfigError("DVC_HOME directory doesn't exists")
self._parsed_args, self._command_args = parser.parse_known_args(args=self.args)
pass
@property
def settings(self):
return self._settings
@property
def args(self):
return self._args
return self._settings.args
@property
def parsed_args(self):
@@ -57,24 +32,20 @@ class CmdBase(object):
def command_args(self):
return self._command_args
@cached_property
def path_factory(self):
return PathFactory(self.git, self.config)
@property
def config(self):
return self._config
return self._settings.config
def cache_file_aws_key(self, file):
return '{}/{}'.format(self._config.aws_storage_prefix, file).strip('/')
return '{}/{}'.format(self.config.aws_storage_prefix, file).strip('/')
@property
def lnx_home(self):
return self._dvc_home
def dvc_home(self):
return self._settings.dvc_home
@property
def git(self):
return self._git
return self._settings.git
def define_args(self, parser):
pass
+5 -6
View File
@@ -8,8 +8,8 @@ from dvc.command.base import CmdBase
from dvc.command.data_sync import sizeof_fmt
from dvc.logger import Logger
from dvc.exceptions import DvcException
from dvc.runtime import Runtime
from dvc.state_file import StateFile
from dvc.utils import run
class DataImportError(DvcException):
@@ -18,9 +18,8 @@ class DataImportError(DvcException):
class CmdDataImport(CmdBase):
def __init__(self):
CmdBase.__init__(self)
pass
def __init__(self, settings):
super(CmdDataImport, self).__init__(settings)
def define_args(self, parser):
self.set_skip_git_actions(parser)
@@ -67,7 +66,7 @@ class CmdDataImport(CmdBase):
if os.path.isdir(output):
output = os.path.join(output, os.path.basename(input))
data_item = self.path_factory.data_item(output)
data_item = self.settings.path_factory.data_item(output)
if os.path.exists(data_item.data.relative):
raise DataImportError('Output file "{}" already exists'.format(data_item.data.relative))
@@ -135,4 +134,4 @@ class CmdDataImport(CmdBase):
return
if __name__ == '__main__':
run(CmdDataImport())
Runtime.run(CmdDataImport)
+4 -4
View File
@@ -6,7 +6,7 @@ from boto.s3.connection import S3Connection
from dvc.command.base import CmdBase
from dvc.logger import Logger
from dvc.exceptions import DvcException
from dvc.utils import run
from dvc.runtime import Runtime
class DataSyncError(DvcException):
@@ -35,8 +35,8 @@ def file_md5(fname):
class CmdDataSync(CmdBase):
def __init__(self):
CmdBase.__init__(self)
def __init__(self, settings):
super(CmdDataSync, self).__init__(settings)
conn = S3Connection(self.config.aws_access_key_id, self.config.aws_secret_access_key)
@@ -114,4 +114,4 @@ class CmdDataSync(CmdBase):
if __name__ == '__main__':
run(CmdDataSync())
Runtime.run(CmdDataSync)
+4 -5
View File
@@ -5,7 +5,7 @@ from dvc.command.base import CmdBase
from dvc.logger import Logger
from dvc.config import Config
from dvc.exceptions import DvcException
from dvc.utils import run
from dvc.runtime import Runtime
class InitError(DvcException):
@@ -44,9 +44,8 @@ KeyName = dvc-key
SecurityGroup = dvc-group'''
def __init__(self):
CmdBase.__init__(self, parse_config=False)
pass
def __init__(self, settings):
super(CmdInit, self).__init__(settings)
def define_args(self, parser):
self.set_skip_git_actions(parser)
@@ -110,4 +109,4 @@ SecurityGroup = dvc-group'''
Logger.info('Directory {} was added to .gitignore file'.format(cache_dir_name))
if __name__ == '__main__':
run(CmdInit())
Runtime.run(CmdInit, False)
+6 -7
View File
@@ -6,7 +6,7 @@ from boto.s3.connection import S3Connection
from dvc.command.base import CmdBase
from dvc.exceptions import DvcException
from dvc.logger import Logger
from dvc.utils import run
from dvc.runtime import Runtime
class DataRemoveError(DvcException):
@@ -15,9 +15,8 @@ class DataRemoveError(DvcException):
class CmdDataRemove(CmdBase):
def __init__(self, args=None, parse_config=True, git_obj=None, config_obj=None):
super(CmdDataRemove, self).__init__(args, parse_config, git_obj, config_obj)
pass
def __init__(self, settings):
super(CmdDataRemove, self).__init__(settings)
def define_args(self, parser):
self.set_skip_git_actions(parser)
@@ -71,7 +70,7 @@ class CmdDataRemove(CmdBase):
if not self.parsed_args.recursive:
raise DataRemoveError('[Cmd-Remove] Directory "%s" cannot be removed. Use --recurcive flag.' % target)
data_item = self.path_factory.data_item(target)
data_item = self.settings.path_factory.data_item(target)
if data_item.data_dvc_short == '':
raise DataRemoveError('[Cmd-Remove] Data directory "%s" cannot be removed' % target)
@@ -89,7 +88,7 @@ class CmdDataRemove(CmdBase):
# it raises exception if not a symlink is provided
Logger.debug(u'[Cmd-Remove] Remove file {}.'.format(target))
data_item = self.path_factory.existing_data_item(target)
data_item = self.settings.path_factory.existing_data_item(target)
self._remove_cache_file(data_item)
self._remove_state_file(data_item)
@@ -154,4 +153,4 @@ class CmdDataRemove(CmdBase):
if __name__ == '__main__':
run(CmdDataRemove())
Runtime.run(CmdDataRemove)
+7 -8
View File
@@ -1,13 +1,12 @@
import os
import fasteners
from dvc.command.base import CmdBase
from dvc.command.run import CmdRun
from dvc.logger import Logger
from dvc.exceptions import DvcException
from dvc.path.data_item import NotInDataDirError
from dvc.runtime import Runtime
from dvc.state_file import StateFile
from dvc.utils import run
class ReproError(DvcException):
@@ -16,8 +15,8 @@ class ReproError(DvcException):
class CmdRepro(CmdRun):
def __init__(self, args=None,parse_config=True, git_obj=None, config_obj=None):
CmdBase.__init__(self, args, parse_config, git_obj, config_obj)
def __init__(self, settings):
super(CmdRun, self).__init__(settings)
self._code = []
pass
@@ -56,7 +55,7 @@ class CmdRepro(CmdRun):
if not self.skip_git_actions and not self.git.is_ready_to_go():
return 1
data_item_list, external_files_names = self.path_factory.to_data_items(self.parsed_args.target)
data_item_list, external_files_names = self.settings.path_factory.to_data_items(self.parsed_args.target)
if external_files_names:
Logger.error('Files from outside of the data directory "{}" could not be reproduced: {}'.
format(self.config.data_dir, ' '.join(external_files_names)))
@@ -160,7 +159,7 @@ class ReproChange(object):
was_source_code_changed = self.git.were_files_changed(self._data_item.data.relative,
self.state.code_dependencies + self.state.input_files,
self.cmd_obj.path_factory)
self.cmd_obj.settings.path_factory)
if was_source_code_changed:
Logger.debug('Source code was changed for "{}"'.format(self._data_item.data.dvc))
@@ -176,7 +175,7 @@ class ReproChange(object):
dependency_data_items = []
for input_file in self.state.input_files:
try:
data_item = self._cmd_obj.path_factory.data_item(input_file)
data_item = self._cmd_obj.settings.path_factory.data_item(input_file)
except NotInDataDirError:
raise ReproError(u'The dependency file "{}" is not a data item'.format(input_file))
except Exception as ex:
@@ -189,4 +188,4 @@ class ReproChange(object):
if __name__ == '__main__':
run(CmdRepro())
Runtime.run(CmdRepro)
+12 -13
View File
@@ -7,9 +7,9 @@ from dvc.git_wrapper import GitWrapper
from dvc.logger import Logger
from dvc.path.data_item import NotInDataDirError
from dvc.repository_change import RepositoryChange
from dvc.runtime import Runtime
from dvc.state_file import StateFile
from dvc.utils import cached_property
from dvc.utils import run
class RunError(DvcException):
@@ -18,9 +18,8 @@ class RunError(DvcException):
class CmdRun(CmdBase):
def __init__(self, args=None, parse_config=True, git_obj=None, config_obj=None):
super(CmdRun, self).__init__(args, parse_config, git_obj, config_obj)
pass
def __init__(self, settings):
super(CmdRun, self).__init__(settings)
def define_args(self, parser):
self.set_skip_git_actions(parser)
@@ -76,14 +75,14 @@ class CmdRun(CmdBase):
return 1
def run_command(self, argv, data_items_from_args, stdout=None, stderr=None):
Logger.debug('Run command: {}. Data items from args: {}. stdout={}, stderr={}'.format(
' '.join(argv),
def run_command(self, cmd_args, data_items_from_args, stdout=None, stderr=None):
Logger.debug('Run command with args: {}. Data items from args: {}. stdout={}, stderr={}'.format(
' '.join(cmd_args),
', '.join([x.data.dvc for x in data_items_from_args]),
stdout,
stderr))
repo_change = RepositoryChange(argv, stdout, stderr, self.git, self.config, self.path_factory)
repo_change = RepositoryChange(cmd_args, self.settings, stdout, stderr)
if not self.skip_git_actions and not self._validate_file_states(repo_change):
self.remove_new_files(repo_change)
@@ -109,7 +108,7 @@ class CmdRun(CmdBase):
input_files_dvc,
output_files_dvc,
code_dependencies_dvc,
argv=argv,
argv=cmd_args,
is_reproducible=self.is_reproducible,
stdout=self._stdout_to_dvc(stdout),
stderr=self._stdout_to_dvc(stderr))
@@ -121,7 +120,7 @@ class CmdRun(CmdBase):
def _stdout_to_dvc(self, stdout):
if stdout in {None, '-'}:
return stdout
return self.path_factory.data_item(stdout).data.dvc
return self.settings.path_factory.data_item(stdout).data.dvc
@staticmethod
def remove_new_files(repo_change):
@@ -149,7 +148,7 @@ class CmdRun(CmdBase):
for arg in argv:
try:
if os.path.isfile(arg):
data_item = self.path_factory.data_item(arg)
data_item = self.settings.path_factory.data_item(arg)
result.append(data_item)
except NotInDataDirError:
pass
@@ -160,7 +159,7 @@ class CmdRun(CmdBase):
if not files:
return []
data_items, external = self.path_factory.to_data_items(files)
data_items, external = self.settings.path_factory.to_data_items(files)
if external:
raise RunError('{} should point to data items from data dir: {}'.format(
param_text, ', '.join(external))
@@ -169,4 +168,4 @@ class CmdRun(CmdBase):
if __name__ == '__main__':
run(CmdRun())
Runtime.run(CmdRun)
+4 -21
View File
@@ -10,7 +10,7 @@ class ConfigError(DvcException):
class ConfigI(object):
def __init__(self, data_dir, cache_dir, state_dir):
def __init__(self, data_dir=None, cache_dir=None, state_dir=None):
self._data_dir = data_dir
self._cache_dir = cache_dir
self._state_dir = state_dir
@@ -44,32 +44,15 @@ class Config(ConfigI):
raise ConfigError('Config file "{}" does not exist'.format(conf_file))
self._config.read(conf_file)
# self._global = self._config['Global']
#
# self._data_dir = self._global['DataDir']
# self._cache_dir = self._global['CacheDir']
# self._state_dir = self._global['StateDir']
super(Config, self).__init__(self._config['Global']['DataDir'],
self._config['Global']['CacheDir'],
self._config['Global']['StateDir'])
pass
# def get_data_file_obj(self):
# return DataFileObj(self.config)
@property
def file(self):
return self._conf_file
@property
def data_dir(self):
return self._config['Global']['DataDir']
@property
def cache_dir(self):
return self._config['Global']['CacheDir']
@property
def state_dir(self):
return self._config['Global']['StateDir']
@property
def aws_access_key_id(self):
return self._config['AWS']['AccessKeyId']
+6 -12
View File
@@ -17,25 +17,19 @@ class RepositoryChangeError(DvcException):
class RepositoryChange(object):
"""Pre-condition: git repository has no changes"""
def __init__(self, args, stdout, stderr, git, config, path_factory):
self.git = git
self.config = config
self.path_factory = path_factory
def __init__(self, cmd_args, settings, stdout, stderr):
self._settings = settings
Logger.debug(u'[Repository change] Exec command: {}. stdout={}, stderr={}'.format(
u' '.join(args),
u' '.join(cmd_args),
stdout,
stderr))
Executor.exec_cmd_only_success(args, stdout, stderr)
Executor.exec_cmd_only_success(cmd_args, stdout, stderr)
self._stated_data_items = []
self._externally_created_files = []
self._init_file_states()
@staticmethod
def exec_cmd(args, stdout, stderr, git, path_factory):
return RepositoryChange(args, stdout, stderr, git)
@cached_property
def removed_data_items(self):
return [x for x in self._stated_data_items if x.is_removed]
@@ -58,7 +52,7 @@ class RepositoryChange(object):
def _add_stated_data_item(self, state, file):
try:
item = self.path_factory.stated_data_item(state, file)
item = self._settings.path_factory.stated_data_item(state, file)
self._stated_data_items.append(item)
Logger.debug('[Repository change] Add status: {} {}'.format(
item.status,
@@ -71,7 +65,7 @@ class RepositoryChange(object):
statuses = GitWrapper.git_file_statuses()
for status, file in statuses:
file_path = os.path.join(self.git.git_dir_abs, file)
file_path = os.path.join(self._settings.git.git_dir_abs, file)
if not os.path.isdir(file_path):
self._add_stated_data_item(status, file_path)
+34
View File
@@ -0,0 +1,34 @@
import os
import sys
from dvc.config import Config, ConfigI
from dvc.exceptions import DvcException
from dvc.git_wrapper import GitWrapper
from dvc.logger import Logger
from dvc.settings import Settings
class Runtime(object):
CONFIG = 'dvc.conf'
@staticmethod
def conf_file_path(git_dir):
return os.path.realpath(os.path.join(git_dir, Runtime.CONFIG))
@staticmethod
def run(cmd_class, parse_config=True):
try:
runtime_git = GitWrapper()
if parse_config:
runtime_config = Config(Runtime.conf_file_path(runtime_git.git_dir))
else:
runtime_config = ConfigI()
args = sys.argv[1:]
instance = cmd_class(Settings(args, runtime_git, runtime_config))
sys.exit(instance.run())
except DvcException as e:
Logger.error(e)
sys.exit(1)
+44
View File
@@ -0,0 +1,44 @@
import os
from dvc.exceptions import DvcException
from dvc.path.factory import PathFactory
class SettingsError(DvcException):
def __init__(self, msg):
DvcException.__init__(self, msg)
class Settings(object):
def __init__(self, args, git, config):
self._args = args
self._git = git
self._config = config
self._path_factory = PathFactory(git, config)
self._dvc_home = os.environ.get('DVC_HOME')
if not self._dvc_home:
raise SettingsError('DVC_HOME environment variable is not defined')
if not os.path.exists(self._dvc_home):
raise SettingsError("DVC_HOME directory doesn't exists")
pass
@property
def args(self):
return self._args
@property
def git(self):
return self._git
@property
def config(self):
return self._config
@property
def dvc_home(self):
return self._dvc_home
@property
def path_factory(self):
return self._path_factory
-12
View File
@@ -1,6 +1,3 @@
from dvc.exceptions import DvcException
from dvc.logger import Logger
def cached_property(f):
def get(self):
@@ -15,12 +12,3 @@ def cached_property(f):
return x
return property(get)
def run(cmd):
import sys
try:
sys.exit(cmd.run())
except DvcException as e:
Logger.error(e)
sys.exit(1)
+3 -5
View File
@@ -3,9 +3,10 @@ import shutil
from unittest import TestCase
from dvc.config import ConfigI
from dvc.git_wrapper import GitWrapperI, GitWrapper
from dvc.git_wrapper import GitWrapperI
from dvc.path.data_item import DataItem
from dvc.path.factory import PathFactory
from dvc.settings import Settings
class BasicEnvironment(TestCase):
@@ -33,8 +34,6 @@ class BasicEnvironment(TestCase):
os.makedirs(data_dir)
os.chdir(self._curr_dir)
# GitWrapper.exec_cmd(['git', 'init'])
# self._commit = GitWrapper().curr_commit
def tearDown(self):
if self._old_curr_dir_abs:
@@ -69,6 +68,7 @@ class DirHierarchyEnvironment(BasicEnvironment):
self._git = GitWrapperI(git_dir=self._test_git_dir, commit=self._commit)
self._config = ConfigI('data', 'cache', 'state')
self.path_factory = PathFactory(self._git, self._config)
self.settings = Settings([], self._git, self._config)
self.dir1 = os.path.join('dir1')
self.dir11 = os.path.join('dir1/dir11')
@@ -91,7 +91,6 @@ class DirHierarchyEnvironment(BasicEnvironment):
@staticmethod
def create_content_file(file, content='some test'):
# print(file)
fd = open(file, 'w+')
fd.write(content)
fd.close()
@@ -108,7 +107,6 @@ class DirHierarchyEnvironment(BasicEnvironment):
relevant_dir = self.relevant_dir(data_file)
os.symlink(os.path.join(relevant_dir, cache_result), file_result)
# print(file_result)
else:
cache_result = None
self.create_content_file(file_result, content)
+10 -10
View File
@@ -10,7 +10,7 @@ class TestCmdDataRemove(DirHierarchyEnvironment):
DirHierarchyEnvironment.init_environment(self)
def test_file_by_file_removal(self):
cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config)
cmd = CmdDataRemove(self.settings)
cmd.parsed_args.keep_in_cloud = True
dir1_dvc_name = os.path.join('data', self.dir1)
@@ -20,7 +20,7 @@ class TestCmdDataRemove(DirHierarchyEnvironment):
self.assertFalse(os.path.exists(dir1_dvc_name))
def test_remove_deep_dir(self):
cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config)
cmd = CmdDataRemove(self.settings)
cmd.parsed_args.keep_in_cloud = True
cmd.parsed_args.recursive = True
@@ -31,7 +31,7 @@ class TestCmdDataRemove(DirHierarchyEnvironment):
self.assertFalse(os.path.exists(dir1_dvc_name))
def test_not_recursive_removal(self):
cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config)
cmd = CmdDataRemove(self.settings)
cmd.parsed_args.keep_in_cloud = True
cmd.parsed_args.recursive = False
@@ -43,7 +43,7 @@ class TestCmdDataRemove(DirHierarchyEnvironment):
pass
def test_data_dir_removal(self):
cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config)
cmd = CmdDataRemove(self.settings)
cmd.parsed_args.keep_in_cloud = True
cmd.parsed_args.recursive = True
@@ -60,7 +60,7 @@ class TestRemoveDataItem(DirHierarchyEnvironment):
DirHierarchyEnvironment.init_environment(self)
def test_remove_data_instance(self):
cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config)
cmd = CmdDataRemove(self.settings)
cmd.parsed_args.keep_in_cloud = True
self.assertTrue(os.path.isfile(self.file1))
@@ -73,7 +73,7 @@ class TestRemoveDataItem(DirHierarchyEnvironment):
self.assertFalse(os.path.exists(self.state1))
def test_keep_cache(self):
cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config)
cmd = CmdDataRemove(self.settings)
cmd.parsed_args.keep_in_cloud = True
cmd.parsed_args.keep_in_cache = True
@@ -87,7 +87,7 @@ class TestRemoveDataItem(DirHierarchyEnvironment):
self.assertFalse(os.path.exists(self.state1))
def test_remove_data_instance_without_cache(self):
cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config)
cmd = CmdDataRemove(self.settings)
cmd.parsed_args.keep_in_cloud = True
self.assertTrue(os.path.isfile(self.file6))
@@ -103,7 +103,7 @@ class TestRemoveEndToEnd(DirHierarchyEnvironment):
DirHierarchyEnvironment.init_environment(self)
def test_end_to_end_with_an_error(self):
cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config)
cmd = CmdDataRemove(self.settings)
cmd.parsed_args.keep_in_cloud = True
dir11_full = os.path.join('data', self.dir11)
@@ -130,7 +130,7 @@ class TestRemoveEndToEnd(DirHierarchyEnvironment):
self.assertTrue(os.path.exists(self.file6))
def test_end_to_end_with_no_error(self):
cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config)
cmd = CmdDataRemove(self.settings)
cmd.parsed_args.keep_in_cloud = True
dir11_full = os.path.join('data', self.dir11)
@@ -154,7 +154,7 @@ class TestRemoveEndToEnd(DirHierarchyEnvironment):
self.assertTrue(os.path.exists(self.file1))
def test_run(self):
cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config)
cmd = CmdDataRemove(self.settings)
cmd.parsed_args.keep_in_cloud = True
dir11_full = os.path.join('data', self.dir11)
+7 -2
View File
@@ -9,6 +9,7 @@ from dvc.executor import Executor
from dvc.path.data_item import DataItem
from dvc.path.factory import PathFactory
from dvc.git_wrapper import GitWrapper
from dvc.settings import Settings
class RunBasicTest(TestCase):
@@ -28,6 +29,8 @@ class RunBasicTest(TestCase):
self.config = ConfigI('data', 'cache', 'state')
self.path_factory = PathFactory(self.git, self.config)
self.settings = Settings([], self.git, self.config)
pass
def init_git_repo(self):
@@ -49,7 +52,8 @@ class RunBasicTest(TestCase):
class TestRunOutsideData(RunBasicTest):
def test(self):
cmd_run = CmdRun(config_obj=self.config)
self.settings._args = []
cmd_run = CmdRun(self.settings)
with self.assertRaises(RunError):
cmd_run.run_command(['touch', 'file1', 'file2'], [])
pass
@@ -59,7 +63,8 @@ class RunTwoFilesBase(RunBasicTest):
def setUp(self):
super(RunTwoFilesBase, self).setUp()
cmd_run = CmdRun(config_obj=self.config, git_obj=self.git)
self.settings._args = []
cmd_run = CmdRun(self.settings)
self.input_param_file = 'data/extra_input'
cmd_run.parsed_args.input = [self.input_param_file]
-2
View File
@@ -152,8 +152,6 @@ class TestDataFileObjLongPath(BasicEnvironment):
self._config = ConfigI('data', 'cache', 'state')
self.path_factory = PathFactory(self._git, self._config)
# self._dobj = DataFileObj(os.path.join('..', '..', 'data', 'file1.txt'),
# self._git, self._config)
self.data_path = self.path_factory.data_item(os.path.join('..', '..', 'data', 'file1.txt'))
pass
+8 -12
View File
@@ -8,6 +8,7 @@ from dvc.config import ConfigI
from dvc.path.factory import PathFactory
from dvc.git_wrapper import GitWrapperI
from dvc.repository_change import RepositoryChange
from dvc.settings import Settings
class BasicTestRepositoryChange(TestCase):
@@ -26,6 +27,7 @@ class BasicTestRepositoryChange(TestCase):
self.git = GitWrapperI(self.test_dir)
self.config = ConfigI('data', 'cache', 'state')
self.path_factory = PathFactory(self.git, self.config)
self.settings = Settings([], self.git, self.config)
def tearDown(self):
shutil.rmtree(self.test_dir, ignore_errors=True)
@@ -56,11 +58,9 @@ class TestRepositoryChange(BasicTestRepositoryChange):
os.remove(file_to_remove)
change = RepositoryChange(['ls', '-la'],
self.settings,
file_created_by_run,
file_to_modify,
self.git,
self.config,
self.path_factory)
file_to_modify)
expected_new = [file_created_by_run, file_created_before_run]
new_file_abs = [x.data.dvc for x in change.new_data_items]
@@ -86,11 +86,9 @@ class TestRepositoryChangeDeepInDirectory(BasicTestRepositoryChange):
self.create_file(deep_file)
change = RepositoryChange(['ls', '-la'],
self.settings,
deep_file,
None,
self.git,
self.config,
self.path_factory)
None)
self.assertEqual([x.data.dvc for x in change.new_data_items], [deep_file])
@@ -101,11 +99,9 @@ class TestRepositoryChangeExternallyCreated(BasicTestRepositoryChange):
self.create_file(not_in_data_dir_file)
change = RepositoryChange(['ls', '-la'],
self.settings,
not_in_data_dir_file,
None,
self.git,
self.config,
self.path_factory)
None)
self.assertEqual([os.path.realpath(not_in_data_dir_file)],
change.externally_created_files)
+22 -54
View File
@@ -2,6 +2,7 @@ from dvc.command.remove import CmdDataRemove
from dvc.command.repro import ReproChange, CmdRepro
from dvc.command.run import CmdRun, RunError
from dvc.executor import Executor
from dvc.settings import Settings
from tests.test_cmd_run import RunBasicTest
@@ -12,50 +13,26 @@ class ReproBasicEnv(RunBasicTest):
self.file_name1 = 'data/file1'
self.file1_code_file = 'file1.py'
self.create_file_and_commit(self.file1_code_file, 'An awesome code...')
cmd_file1 = CmdRun(args=[
'printf',
'Hello\nMary',
'--not-repro',
'--stdout',
self.file_name1,
'--code',
self.file1_code_file
],
parse_config=False,
config_obj=self.config,
git_obj=self.git)
self.settings._args = ['printf', 'Hello\nMary', '--not-repro',
'--stdout', self.file_name1, '--code', self.file1_code_file]
cmd_file1 = CmdRun(self.settings)
self.assertEqual(cmd_file1.code_dependencies, [self.file1_code_file])
cmd_file1.run()
self.file_name11 = 'data/file11'
CmdRun(args=['head', '-n', '1', self.file_name1, '--stdout', self.file_name11],
parse_config=False,
config_obj=self.config,
git_obj=self.git
).run()
self.settings._args = ['head', '-n', '1', self.file_name1, '--stdout', self.file_name11]
CmdRun(self.settings).run()
self.file_name2 = 'data/file2'
CmdRun(args=['printf', 'Bobby', '--not-repro', '--stdout', self.file_name2],
parse_config=False,
config_obj=self.config,
git_obj=self.git
).run()
self.settings._args = ['printf', 'Bobby', '--not-repro', '--stdout', self.file_name2]
CmdRun(self.settings).run()
self.file_res_code_file = 'code_res.py'
self.create_file_and_commit(self.file_res_code_file, 'Another piece of code')
self.file_name_res = 'data/file_res'
cmd_res = CmdRun(args=[
'cat',
self.file_name11,
self.file_name2,
'--stdout',
self.file_name_res,
'--code',
self.file_res_code_file
],
parse_config=False,
config_obj=self.config,
git_obj=self.git)
self.settings._args = ['cat', self.file_name11, self.file_name2, '--stdout', self.file_name_res,
'--code', self.file_res_code_file]
cmd_res = CmdRun(self.settings)
self.assertEqual(cmd_res.code_dependencies, [self.file_res_code_file])
cmd_res.run()
@@ -81,10 +58,8 @@ class ReproCodeDependencyTest(ReproBasicEnv):
def test(self):
self.modify_file_and_commit(self.file_res_code_file)
CmdRepro(args=[self.file_name_res],
config_obj=self.config,
git_obj=self.git
).run()
self.settings._args = [self.file_name_res]
CmdRepro(self.settings).run()
self.assertEqual(open(self.file_name_res).read(), 'Hello\nBobby')
@@ -93,31 +68,24 @@ class ReproChangedDependency(ReproBasicEnv):
def test(self):
self.recreate_file1()
CmdRepro(args=[self.file_name11],
config_obj=self.config,
git_obj=self.git
).run()
self.settings._args = [self.file_name11]
CmdRepro(self.settings).run()
self.assertEqual(open(self.file_name11).read(), 'Goodbye\n')
def recreate_file1(self):
CmdDataRemove(args=[self.file_name1, '--keep-in-cloud'], # ???
config_obj=self.config,
git_obj=self.git
).run()
CmdRun(args=['printf', 'Goodbye\nJack', '--stdout', self.file_name1],
config_obj=self.config,
git_obj=self.git
).run()
self.settings._args = [self.file_name1, '--keep-in-cloud']
CmdDataRemove(self.settings).run()
self.settings._args = ['printf', 'Goodbye\nJack', '--stdout', self.file_name1]
CmdRun(self.settings).run()
class ReproChangedDeepDependency(ReproChangedDependency):
def test(self):
self.recreate_file1()
CmdRepro(args=[self.file_name_res],
config_obj=self.config,
git_obj=self.git
).run()
self.settings._args = [self.file_name_res]
CmdRepro(self.settings).run()
self.assertEqual(open(self.file_name_res).read(), 'Goodbye\nBobby')