From 5f3c53603f5cfc11189500d5e6fd141a63b2d66e Mon Sep 17 00:00:00 2001 From: Dmitry Petrov Date: Wed, 5 Apr 2017 23:08:40 -0700 Subject: [PATCH] Refactoring: extract Settings from BaseCmd; move path_factory to Settings --- dvc/command/base.py | 57 ++++++------------------- dvc/command/data_import.py | 11 +++-- dvc/command/data_sync.py | 8 ++-- dvc/command/init.py | 9 ++-- dvc/command/remove.py | 13 +++--- dvc/command/repro.py | 15 +++---- dvc/command/run.py | 25 ++++++----- dvc/config.py | 25 ++--------- dvc/repository_change.py | 18 +++----- dvc/runtime.py | 34 +++++++++++++++ dvc/settings.py | 44 +++++++++++++++++++ dvc/utils.py | 12 ------ tests/basic_env.py | 8 ++-- tests/test_cmd_data_remove.py | 20 ++++----- tests/test_cmd_run.py | 9 +++- tests/test_data_file_obj.py | 2 - tests/test_repository_change.py | 20 ++++----- tests/test_repro.py | 76 ++++++++++----------------------- 18 files changed, 190 insertions(+), 216 deletions(-) create mode 100644 dvc/runtime.py create mode 100644 dvc/settings.py diff --git a/dvc/command/base.py b/dvc/command/base.py index 2f6773798..b79de01b1 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -1,53 +1,28 @@ import argparse import os -import sys -from dvc.git_wrapper import GitWrapper -from dvc.config import Config, ConfigError +from dvc.config import ConfigError from dvc.logger import Logger from dvc.path.factory import PathFactory from dvc.utils import cached_property class CmdBase(object): - CONFIG = 'dvc.conf' - - def __init__(self, args=None, parse_config=True, git_obj=None, config_obj=None): - if git_obj: - self._git = git_obj - else: - self._git = GitWrapper() - - if config_obj: - self._config = config_obj - else: - self._config = None - if parse_config: - self._config = Config(os.path.realpath(os.path.join(self.git.git_dir, self.CONFIG))) - - if args is None: - self._args = sys.argv[1:] - else: - self._args = args - - self._parsed_args = None - self._dvc_home = None + def __init__(self, settings): + self._settings = settings parser = argparse.ArgumentParser() self.define_args(parser) - self._parsed_args, self._command_args = parser.parse_known_args(args=self._args) - - self._dvc_home = os.environ.get('DVC_HOME') - - if not self.lnx_home: - raise ConfigError('DVC_HOME environment variable is not defined') - if not os.path.exists(self._dvc_home): - raise ConfigError("DVC_HOME directory doesn't exists") + self._parsed_args, self._command_args = parser.parse_known_args(args=self.args) pass + @property + def settings(self): + return self._settings + @property def args(self): - return self._args + return self._settings.args @property def parsed_args(self): @@ -57,24 +32,20 @@ class CmdBase(object): def command_args(self): return self._command_args - @cached_property - def path_factory(self): - return PathFactory(self.git, self.config) - @property def config(self): - return self._config + return self._settings.config def cache_file_aws_key(self, file): - return '{}/{}'.format(self._config.aws_storage_prefix, file).strip('/') + return '{}/{}'.format(self.config.aws_storage_prefix, file).strip('/') @property - def lnx_home(self): - return self._dvc_home + def dvc_home(self): + return self._settings.dvc_home @property def git(self): - return self._git + return self._settings.git def define_args(self, parser): pass diff --git a/dvc/command/data_import.py b/dvc/command/data_import.py index f72b49401..b4d161351 100644 --- a/dvc/command/data_import.py +++ b/dvc/command/data_import.py @@ -8,8 +8,8 @@ from dvc.command.base import CmdBase from dvc.command.data_sync import sizeof_fmt from dvc.logger import Logger from dvc.exceptions import DvcException +from dvc.runtime import Runtime from dvc.state_file import StateFile -from dvc.utils import run class DataImportError(DvcException): @@ -18,9 +18,8 @@ class DataImportError(DvcException): class CmdDataImport(CmdBase): - def __init__(self): - CmdBase.__init__(self) - pass + def __init__(self, settings): + super(CmdDataImport, self).__init__(settings) def define_args(self, parser): self.set_skip_git_actions(parser) @@ -67,7 +66,7 @@ class CmdDataImport(CmdBase): if os.path.isdir(output): output = os.path.join(output, os.path.basename(input)) - data_item = self.path_factory.data_item(output) + data_item = self.settings.path_factory.data_item(output) if os.path.exists(data_item.data.relative): raise DataImportError('Output file "{}" already exists'.format(data_item.data.relative)) @@ -135,4 +134,4 @@ class CmdDataImport(CmdBase): return if __name__ == '__main__': - run(CmdDataImport()) + Runtime.run(CmdDataImport) diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index 41f0f4f38..abdaaf923 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -6,7 +6,7 @@ from boto.s3.connection import S3Connection from dvc.command.base import CmdBase from dvc.logger import Logger from dvc.exceptions import DvcException -from dvc.utils import run +from dvc.runtime import Runtime class DataSyncError(DvcException): @@ -35,8 +35,8 @@ def file_md5(fname): class CmdDataSync(CmdBase): - def __init__(self): - CmdBase.__init__(self) + def __init__(self, settings): + super(CmdDataSync, self).__init__(settings) conn = S3Connection(self.config.aws_access_key_id, self.config.aws_secret_access_key) @@ -114,4 +114,4 @@ class CmdDataSync(CmdBase): if __name__ == '__main__': - run(CmdDataSync()) + Runtime.run(CmdDataSync) diff --git a/dvc/command/init.py b/dvc/command/init.py index 893f09352..e646a32cd 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -5,7 +5,7 @@ from dvc.command.base import CmdBase from dvc.logger import Logger from dvc.config import Config from dvc.exceptions import DvcException -from dvc.utils import run +from dvc.runtime import Runtime class InitError(DvcException): @@ -44,9 +44,8 @@ KeyName = dvc-key SecurityGroup = dvc-group''' - def __init__(self): - CmdBase.__init__(self, parse_config=False) - pass + def __init__(self, settings): + super(CmdInit, self).__init__(settings) def define_args(self, parser): self.set_skip_git_actions(parser) @@ -110,4 +109,4 @@ SecurityGroup = dvc-group''' Logger.info('Directory {} was added to .gitignore file'.format(cache_dir_name)) if __name__ == '__main__': - run(CmdInit()) + Runtime.run(CmdInit, False) diff --git a/dvc/command/remove.py b/dvc/command/remove.py index 1908ac49b..aa7b9785d 100644 --- a/dvc/command/remove.py +++ b/dvc/command/remove.py @@ -6,7 +6,7 @@ from boto.s3.connection import S3Connection from dvc.command.base import CmdBase from dvc.exceptions import DvcException from dvc.logger import Logger -from dvc.utils import run +from dvc.runtime import Runtime class DataRemoveError(DvcException): @@ -15,9 +15,8 @@ class DataRemoveError(DvcException): class CmdDataRemove(CmdBase): - def __init__(self, args=None, parse_config=True, git_obj=None, config_obj=None): - super(CmdDataRemove, self).__init__(args, parse_config, git_obj, config_obj) - pass + def __init__(self, settings): + super(CmdDataRemove, self).__init__(settings) def define_args(self, parser): self.set_skip_git_actions(parser) @@ -71,7 +70,7 @@ class CmdDataRemove(CmdBase): if not self.parsed_args.recursive: raise DataRemoveError('[Cmd-Remove] Directory "%s" cannot be removed. Use --recurcive flag.' % target) - data_item = self.path_factory.data_item(target) + data_item = self.settings.path_factory.data_item(target) if data_item.data_dvc_short == '': raise DataRemoveError('[Cmd-Remove] Data directory "%s" cannot be removed' % target) @@ -89,7 +88,7 @@ class CmdDataRemove(CmdBase): # it raises exception if not a symlink is provided Logger.debug(u'[Cmd-Remove] Remove file {}.'.format(target)) - data_item = self.path_factory.existing_data_item(target) + data_item = self.settings.path_factory.existing_data_item(target) self._remove_cache_file(data_item) self._remove_state_file(data_item) @@ -154,4 +153,4 @@ class CmdDataRemove(CmdBase): if __name__ == '__main__': - run(CmdDataRemove()) + Runtime.run(CmdDataRemove) diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 1034aecc9..9e497dde0 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -1,13 +1,12 @@ import os import fasteners -from dvc.command.base import CmdBase from dvc.command.run import CmdRun from dvc.logger import Logger from dvc.exceptions import DvcException from dvc.path.data_item import NotInDataDirError +from dvc.runtime import Runtime from dvc.state_file import StateFile -from dvc.utils import run class ReproError(DvcException): @@ -16,8 +15,8 @@ class ReproError(DvcException): class CmdRepro(CmdRun): - def __init__(self, args=None,parse_config=True, git_obj=None, config_obj=None): - CmdBase.__init__(self, args, parse_config, git_obj, config_obj) + def __init__(self, settings): + super(CmdRun, self).__init__(settings) self._code = [] pass @@ -56,7 +55,7 @@ class CmdRepro(CmdRun): if not self.skip_git_actions and not self.git.is_ready_to_go(): return 1 - data_item_list, external_files_names = self.path_factory.to_data_items(self.parsed_args.target) + data_item_list, external_files_names = self.settings.path_factory.to_data_items(self.parsed_args.target) if external_files_names: Logger.error('Files from outside of the data directory "{}" could not be reproduced: {}'. format(self.config.data_dir, ' '.join(external_files_names))) @@ -160,7 +159,7 @@ class ReproChange(object): was_source_code_changed = self.git.were_files_changed(self._data_item.data.relative, self.state.code_dependencies + self.state.input_files, - self.cmd_obj.path_factory) + self.cmd_obj.settings.path_factory) if was_source_code_changed: Logger.debug('Source code was changed for "{}"'.format(self._data_item.data.dvc)) @@ -176,7 +175,7 @@ class ReproChange(object): dependency_data_items = [] for input_file in self.state.input_files: try: - data_item = self._cmd_obj.path_factory.data_item(input_file) + data_item = self._cmd_obj.settings.path_factory.data_item(input_file) except NotInDataDirError: raise ReproError(u'The dependency file "{}" is not a data item'.format(input_file)) except Exception as ex: @@ -189,4 +188,4 @@ class ReproChange(object): if __name__ == '__main__': - run(CmdRepro()) + Runtime.run(CmdRepro) diff --git a/dvc/command/run.py b/dvc/command/run.py index e1cfe095f..bc8618b0e 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -7,9 +7,9 @@ from dvc.git_wrapper import GitWrapper from dvc.logger import Logger from dvc.path.data_item import NotInDataDirError from dvc.repository_change import RepositoryChange +from dvc.runtime import Runtime from dvc.state_file import StateFile from dvc.utils import cached_property -from dvc.utils import run class RunError(DvcException): @@ -18,9 +18,8 @@ class RunError(DvcException): class CmdRun(CmdBase): - def __init__(self, args=None, parse_config=True, git_obj=None, config_obj=None): - super(CmdRun, self).__init__(args, parse_config, git_obj, config_obj) - pass + def __init__(self, settings): + super(CmdRun, self).__init__(settings) def define_args(self, parser): self.set_skip_git_actions(parser) @@ -76,14 +75,14 @@ class CmdRun(CmdBase): return 1 - def run_command(self, argv, data_items_from_args, stdout=None, stderr=None): - Logger.debug('Run command: {}. Data items from args: {}. stdout={}, stderr={}'.format( - ' '.join(argv), + def run_command(self, cmd_args, data_items_from_args, stdout=None, stderr=None): + Logger.debug('Run command with args: {}. Data items from args: {}. stdout={}, stderr={}'.format( + ' '.join(cmd_args), ', '.join([x.data.dvc for x in data_items_from_args]), stdout, stderr)) - repo_change = RepositoryChange(argv, stdout, stderr, self.git, self.config, self.path_factory) + repo_change = RepositoryChange(cmd_args, self.settings, stdout, stderr) if not self.skip_git_actions and not self._validate_file_states(repo_change): self.remove_new_files(repo_change) @@ -109,7 +108,7 @@ class CmdRun(CmdBase): input_files_dvc, output_files_dvc, code_dependencies_dvc, - argv=argv, + argv=cmd_args, is_reproducible=self.is_reproducible, stdout=self._stdout_to_dvc(stdout), stderr=self._stdout_to_dvc(stderr)) @@ -121,7 +120,7 @@ class CmdRun(CmdBase): def _stdout_to_dvc(self, stdout): if stdout in {None, '-'}: return stdout - return self.path_factory.data_item(stdout).data.dvc + return self.settings.path_factory.data_item(stdout).data.dvc @staticmethod def remove_new_files(repo_change): @@ -149,7 +148,7 @@ class CmdRun(CmdBase): for arg in argv: try: if os.path.isfile(arg): - data_item = self.path_factory.data_item(arg) + data_item = self.settings.path_factory.data_item(arg) result.append(data_item) except NotInDataDirError: pass @@ -160,7 +159,7 @@ class CmdRun(CmdBase): if not files: return [] - data_items, external = self.path_factory.to_data_items(files) + data_items, external = self.settings.path_factory.to_data_items(files) if external: raise RunError('{} should point to data items from data dir: {}'.format( param_text, ', '.join(external)) @@ -169,4 +168,4 @@ class CmdRun(CmdBase): if __name__ == '__main__': - run(CmdRun()) + Runtime.run(CmdRun) diff --git a/dvc/config.py b/dvc/config.py index 397a6fa17..29649afab 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -10,7 +10,7 @@ class ConfigError(DvcException): class ConfigI(object): - def __init__(self, data_dir, cache_dir, state_dir): + def __init__(self, data_dir=None, cache_dir=None, state_dir=None): self._data_dir = data_dir self._cache_dir = cache_dir self._state_dir = state_dir @@ -44,32 +44,15 @@ class Config(ConfigI): raise ConfigError('Config file "{}" does not exist'.format(conf_file)) self._config.read(conf_file) - # self._global = self._config['Global'] - # - # self._data_dir = self._global['DataDir'] - # self._cache_dir = self._global['CacheDir'] - # self._state_dir = self._global['StateDir'] + super(Config, self).__init__(self._config['Global']['DataDir'], + self._config['Global']['CacheDir'], + self._config['Global']['StateDir']) pass - # def get_data_file_obj(self): - # return DataFileObj(self.config) - @property def file(self): return self._conf_file - @property - def data_dir(self): - return self._config['Global']['DataDir'] - - @property - def cache_dir(self): - return self._config['Global']['CacheDir'] - - @property - def state_dir(self): - return self._config['Global']['StateDir'] - @property def aws_access_key_id(self): return self._config['AWS']['AccessKeyId'] diff --git a/dvc/repository_change.py b/dvc/repository_change.py index b00122fa9..a1428969c 100644 --- a/dvc/repository_change.py +++ b/dvc/repository_change.py @@ -17,25 +17,19 @@ class RepositoryChangeError(DvcException): class RepositoryChange(object): """Pre-condition: git repository has no changes""" - def __init__(self, args, stdout, stderr, git, config, path_factory): - self.git = git - self.config = config - self.path_factory = path_factory + def __init__(self, cmd_args, settings, stdout, stderr): + self._settings = settings Logger.debug(u'[Repository change] Exec command: {}. stdout={}, stderr={}'.format( - u' '.join(args), + u' '.join(cmd_args), stdout, stderr)) - Executor.exec_cmd_only_success(args, stdout, stderr) + Executor.exec_cmd_only_success(cmd_args, stdout, stderr) self._stated_data_items = [] self._externally_created_files = [] self._init_file_states() - @staticmethod - def exec_cmd(args, stdout, stderr, git, path_factory): - return RepositoryChange(args, stdout, stderr, git) - @cached_property def removed_data_items(self): return [x for x in self._stated_data_items if x.is_removed] @@ -58,7 +52,7 @@ class RepositoryChange(object): def _add_stated_data_item(self, state, file): try: - item = self.path_factory.stated_data_item(state, file) + item = self._settings.path_factory.stated_data_item(state, file) self._stated_data_items.append(item) Logger.debug('[Repository change] Add status: {} {}'.format( item.status, @@ -71,7 +65,7 @@ class RepositoryChange(object): statuses = GitWrapper.git_file_statuses() for status, file in statuses: - file_path = os.path.join(self.git.git_dir_abs, file) + file_path = os.path.join(self._settings.git.git_dir_abs, file) if not os.path.isdir(file_path): self._add_stated_data_item(status, file_path) diff --git a/dvc/runtime.py b/dvc/runtime.py new file mode 100644 index 000000000..3c04557a1 --- /dev/null +++ b/dvc/runtime.py @@ -0,0 +1,34 @@ +import os +import sys + +from dvc.config import Config, ConfigI +from dvc.exceptions import DvcException +from dvc.git_wrapper import GitWrapper +from dvc.logger import Logger +from dvc.settings import Settings + + +class Runtime(object): + CONFIG = 'dvc.conf' + + @staticmethod + def conf_file_path(git_dir): + return os.path.realpath(os.path.join(git_dir, Runtime.CONFIG)) + + @staticmethod + def run(cmd_class, parse_config=True): + try: + runtime_git = GitWrapper() + + if parse_config: + runtime_config = Config(Runtime.conf_file_path(runtime_git.git_dir)) + else: + runtime_config = ConfigI() + + args = sys.argv[1:] + + instance = cmd_class(Settings(args, runtime_git, runtime_config)) + sys.exit(instance.run()) + except DvcException as e: + Logger.error(e) + sys.exit(1) diff --git a/dvc/settings.py b/dvc/settings.py new file mode 100644 index 000000000..7233b0379 --- /dev/null +++ b/dvc/settings.py @@ -0,0 +1,44 @@ +import os + +from dvc.exceptions import DvcException +from dvc.path.factory import PathFactory + + +class SettingsError(DvcException): + def __init__(self, msg): + DvcException.__init__(self, msg) + + +class Settings(object): + def __init__(self, args, git, config): + self._args = args + self._git = git + self._config = config + self._path_factory = PathFactory(git, config) + + self._dvc_home = os.environ.get('DVC_HOME') + if not self._dvc_home: + raise SettingsError('DVC_HOME environment variable is not defined') + if not os.path.exists(self._dvc_home): + raise SettingsError("DVC_HOME directory doesn't exists") + pass + + @property + def args(self): + return self._args + + @property + def git(self): + return self._git + + @property + def config(self): + return self._config + + @property + def dvc_home(self): + return self._dvc_home + + @property + def path_factory(self): + return self._path_factory \ No newline at end of file diff --git a/dvc/utils.py b/dvc/utils.py index a49ce173f..4528a937d 100644 --- a/dvc/utils.py +++ b/dvc/utils.py @@ -1,6 +1,3 @@ -from dvc.exceptions import DvcException -from dvc.logger import Logger - def cached_property(f): def get(self): @@ -15,12 +12,3 @@ def cached_property(f): return x return property(get) - - -def run(cmd): - import sys - try: - sys.exit(cmd.run()) - except DvcException as e: - Logger.error(e) - sys.exit(1) \ No newline at end of file diff --git a/tests/basic_env.py b/tests/basic_env.py index bb258ac69..9895a38a2 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -3,9 +3,10 @@ import shutil from unittest import TestCase from dvc.config import ConfigI -from dvc.git_wrapper import GitWrapperI, GitWrapper +from dvc.git_wrapper import GitWrapperI from dvc.path.data_item import DataItem from dvc.path.factory import PathFactory +from dvc.settings import Settings class BasicEnvironment(TestCase): @@ -33,8 +34,6 @@ class BasicEnvironment(TestCase): os.makedirs(data_dir) os.chdir(self._curr_dir) - # GitWrapper.exec_cmd(['git', 'init']) - # self._commit = GitWrapper().curr_commit def tearDown(self): if self._old_curr_dir_abs: @@ -69,6 +68,7 @@ class DirHierarchyEnvironment(BasicEnvironment): self._git = GitWrapperI(git_dir=self._test_git_dir, commit=self._commit) self._config = ConfigI('data', 'cache', 'state') self.path_factory = PathFactory(self._git, self._config) + self.settings = Settings([], self._git, self._config) self.dir1 = os.path.join('dir1') self.dir11 = os.path.join('dir1/dir11') @@ -91,7 +91,6 @@ class DirHierarchyEnvironment(BasicEnvironment): @staticmethod def create_content_file(file, content='some test'): - # print(file) fd = open(file, 'w+') fd.write(content) fd.close() @@ -108,7 +107,6 @@ class DirHierarchyEnvironment(BasicEnvironment): relevant_dir = self.relevant_dir(data_file) os.symlink(os.path.join(relevant_dir, cache_result), file_result) - # print(file_result) else: cache_result = None self.create_content_file(file_result, content) diff --git a/tests/test_cmd_data_remove.py b/tests/test_cmd_data_remove.py index ca55e94cf..1a8f91c0b 100644 --- a/tests/test_cmd_data_remove.py +++ b/tests/test_cmd_data_remove.py @@ -10,7 +10,7 @@ class TestCmdDataRemove(DirHierarchyEnvironment): DirHierarchyEnvironment.init_environment(self) def test_file_by_file_removal(self): - cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config) + cmd = CmdDataRemove(self.settings) cmd.parsed_args.keep_in_cloud = True dir1_dvc_name = os.path.join('data', self.dir1) @@ -20,7 +20,7 @@ class TestCmdDataRemove(DirHierarchyEnvironment): self.assertFalse(os.path.exists(dir1_dvc_name)) def test_remove_deep_dir(self): - cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config) + cmd = CmdDataRemove(self.settings) cmd.parsed_args.keep_in_cloud = True cmd.parsed_args.recursive = True @@ -31,7 +31,7 @@ class TestCmdDataRemove(DirHierarchyEnvironment): self.assertFalse(os.path.exists(dir1_dvc_name)) def test_not_recursive_removal(self): - cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config) + cmd = CmdDataRemove(self.settings) cmd.parsed_args.keep_in_cloud = True cmd.parsed_args.recursive = False @@ -43,7 +43,7 @@ class TestCmdDataRemove(DirHierarchyEnvironment): pass def test_data_dir_removal(self): - cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config) + cmd = CmdDataRemove(self.settings) cmd.parsed_args.keep_in_cloud = True cmd.parsed_args.recursive = True @@ -60,7 +60,7 @@ class TestRemoveDataItem(DirHierarchyEnvironment): DirHierarchyEnvironment.init_environment(self) def test_remove_data_instance(self): - cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config) + cmd = CmdDataRemove(self.settings) cmd.parsed_args.keep_in_cloud = True self.assertTrue(os.path.isfile(self.file1)) @@ -73,7 +73,7 @@ class TestRemoveDataItem(DirHierarchyEnvironment): self.assertFalse(os.path.exists(self.state1)) def test_keep_cache(self): - cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config) + cmd = CmdDataRemove(self.settings) cmd.parsed_args.keep_in_cloud = True cmd.parsed_args.keep_in_cache = True @@ -87,7 +87,7 @@ class TestRemoveDataItem(DirHierarchyEnvironment): self.assertFalse(os.path.exists(self.state1)) def test_remove_data_instance_without_cache(self): - cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config) + cmd = CmdDataRemove(self.settings) cmd.parsed_args.keep_in_cloud = True self.assertTrue(os.path.isfile(self.file6)) @@ -103,7 +103,7 @@ class TestRemoveEndToEnd(DirHierarchyEnvironment): DirHierarchyEnvironment.init_environment(self) def test_end_to_end_with_an_error(self): - cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config) + cmd = CmdDataRemove(self.settings) cmd.parsed_args.keep_in_cloud = True dir11_full = os.path.join('data', self.dir11) @@ -130,7 +130,7 @@ class TestRemoveEndToEnd(DirHierarchyEnvironment): self.assertTrue(os.path.exists(self.file6)) def test_end_to_end_with_no_error(self): - cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config) + cmd = CmdDataRemove(self.settings) cmd.parsed_args.keep_in_cloud = True dir11_full = os.path.join('data', self.dir11) @@ -154,7 +154,7 @@ class TestRemoveEndToEnd(DirHierarchyEnvironment): self.assertTrue(os.path.exists(self.file1)) def test_run(self): - cmd = CmdDataRemove(parse_config=False, git_obj=self._git, config_obj=self._config) + cmd = CmdDataRemove(self.settings) cmd.parsed_args.keep_in_cloud = True dir11_full = os.path.join('data', self.dir11) diff --git a/tests/test_cmd_run.py b/tests/test_cmd_run.py index b183279e5..748b6e745 100644 --- a/tests/test_cmd_run.py +++ b/tests/test_cmd_run.py @@ -9,6 +9,7 @@ from dvc.executor import Executor from dvc.path.data_item import DataItem from dvc.path.factory import PathFactory from dvc.git_wrapper import GitWrapper +from dvc.settings import Settings class RunBasicTest(TestCase): @@ -28,6 +29,8 @@ class RunBasicTest(TestCase): self.config = ConfigI('data', 'cache', 'state') self.path_factory = PathFactory(self.git, self.config) + + self.settings = Settings([], self.git, self.config) pass def init_git_repo(self): @@ -49,7 +52,8 @@ class RunBasicTest(TestCase): class TestRunOutsideData(RunBasicTest): def test(self): - cmd_run = CmdRun(config_obj=self.config) + self.settings._args = [] + cmd_run = CmdRun(self.settings) with self.assertRaises(RunError): cmd_run.run_command(['touch', 'file1', 'file2'], []) pass @@ -59,7 +63,8 @@ class RunTwoFilesBase(RunBasicTest): def setUp(self): super(RunTwoFilesBase, self).setUp() - cmd_run = CmdRun(config_obj=self.config, git_obj=self.git) + self.settings._args = [] + cmd_run = CmdRun(self.settings) self.input_param_file = 'data/extra_input' cmd_run.parsed_args.input = [self.input_param_file] diff --git a/tests/test_data_file_obj.py b/tests/test_data_file_obj.py index e8c799feb..c8e083381 100644 --- a/tests/test_data_file_obj.py +++ b/tests/test_data_file_obj.py @@ -152,8 +152,6 @@ class TestDataFileObjLongPath(BasicEnvironment): self._config = ConfigI('data', 'cache', 'state') self.path_factory = PathFactory(self._git, self._config) - # self._dobj = DataFileObj(os.path.join('..', '..', 'data', 'file1.txt'), - # self._git, self._config) self.data_path = self.path_factory.data_item(os.path.join('..', '..', 'data', 'file1.txt')) pass diff --git a/tests/test_repository_change.py b/tests/test_repository_change.py index 43803391c..46209a4b6 100644 --- a/tests/test_repository_change.py +++ b/tests/test_repository_change.py @@ -8,6 +8,7 @@ from dvc.config import ConfigI from dvc.path.factory import PathFactory from dvc.git_wrapper import GitWrapperI from dvc.repository_change import RepositoryChange +from dvc.settings import Settings class BasicTestRepositoryChange(TestCase): @@ -26,6 +27,7 @@ class BasicTestRepositoryChange(TestCase): self.git = GitWrapperI(self.test_dir) self.config = ConfigI('data', 'cache', 'state') self.path_factory = PathFactory(self.git, self.config) + self.settings = Settings([], self.git, self.config) def tearDown(self): shutil.rmtree(self.test_dir, ignore_errors=True) @@ -56,11 +58,9 @@ class TestRepositoryChange(BasicTestRepositoryChange): os.remove(file_to_remove) change = RepositoryChange(['ls', '-la'], + self.settings, file_created_by_run, - file_to_modify, - self.git, - self.config, - self.path_factory) + file_to_modify) expected_new = [file_created_by_run, file_created_before_run] new_file_abs = [x.data.dvc for x in change.new_data_items] @@ -86,11 +86,9 @@ class TestRepositoryChangeDeepInDirectory(BasicTestRepositoryChange): self.create_file(deep_file) change = RepositoryChange(['ls', '-la'], + self.settings, deep_file, - None, - self.git, - self.config, - self.path_factory) + None) self.assertEqual([x.data.dvc for x in change.new_data_items], [deep_file]) @@ -101,11 +99,9 @@ class TestRepositoryChangeExternallyCreated(BasicTestRepositoryChange): self.create_file(not_in_data_dir_file) change = RepositoryChange(['ls', '-la'], + self.settings, not_in_data_dir_file, - None, - self.git, - self.config, - self.path_factory) + None) self.assertEqual([os.path.realpath(not_in_data_dir_file)], change.externally_created_files) diff --git a/tests/test_repro.py b/tests/test_repro.py index d9c1a1965..bc05a9740 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -2,6 +2,7 @@ from dvc.command.remove import CmdDataRemove from dvc.command.repro import ReproChange, CmdRepro from dvc.command.run import CmdRun, RunError from dvc.executor import Executor +from dvc.settings import Settings from tests.test_cmd_run import RunBasicTest @@ -12,50 +13,26 @@ class ReproBasicEnv(RunBasicTest): self.file_name1 = 'data/file1' self.file1_code_file = 'file1.py' self.create_file_and_commit(self.file1_code_file, 'An awesome code...') - cmd_file1 = CmdRun(args=[ - 'printf', - 'Hello\nMary', - '--not-repro', - '--stdout', - self.file_name1, - '--code', - self.file1_code_file - ], - parse_config=False, - config_obj=self.config, - git_obj=self.git) + self.settings._args = ['printf', 'Hello\nMary', '--not-repro', + '--stdout', self.file_name1, '--code', self.file1_code_file] + cmd_file1 = CmdRun(self.settings) self.assertEqual(cmd_file1.code_dependencies, [self.file1_code_file]) cmd_file1.run() self.file_name11 = 'data/file11' - CmdRun(args=['head', '-n', '1', self.file_name1, '--stdout', self.file_name11], - parse_config=False, - config_obj=self.config, - git_obj=self.git - ).run() + self.settings._args = ['head', '-n', '1', self.file_name1, '--stdout', self.file_name11] + CmdRun(self.settings).run() self.file_name2 = 'data/file2' - CmdRun(args=['printf', 'Bobby', '--not-repro', '--stdout', self.file_name2], - parse_config=False, - config_obj=self.config, - git_obj=self.git - ).run() + self.settings._args = ['printf', 'Bobby', '--not-repro', '--stdout', self.file_name2] + CmdRun(self.settings).run() self.file_res_code_file = 'code_res.py' self.create_file_and_commit(self.file_res_code_file, 'Another piece of code') self.file_name_res = 'data/file_res' - cmd_res = CmdRun(args=[ - 'cat', - self.file_name11, - self.file_name2, - '--stdout', - self.file_name_res, - '--code', - self.file_res_code_file - ], - parse_config=False, - config_obj=self.config, - git_obj=self.git) + self.settings._args = ['cat', self.file_name11, self.file_name2, '--stdout', self.file_name_res, + '--code', self.file_res_code_file] + cmd_res = CmdRun(self.settings) self.assertEqual(cmd_res.code_dependencies, [self.file_res_code_file]) cmd_res.run() @@ -81,10 +58,8 @@ class ReproCodeDependencyTest(ReproBasicEnv): def test(self): self.modify_file_and_commit(self.file_res_code_file) - CmdRepro(args=[self.file_name_res], - config_obj=self.config, - git_obj=self.git - ).run() + self.settings._args = [self.file_name_res] + CmdRepro(self.settings).run() self.assertEqual(open(self.file_name_res).read(), 'Hello\nBobby') @@ -93,31 +68,24 @@ class ReproChangedDependency(ReproBasicEnv): def test(self): self.recreate_file1() - CmdRepro(args=[self.file_name11], - config_obj=self.config, - git_obj=self.git - ).run() + self.settings._args = [self.file_name11] + CmdRepro(self.settings).run() self.assertEqual(open(self.file_name11).read(), 'Goodbye\n') def recreate_file1(self): - CmdDataRemove(args=[self.file_name1, '--keep-in-cloud'], # ??? - config_obj=self.config, - git_obj=self.git - ).run() - CmdRun(args=['printf', 'Goodbye\nJack', '--stdout', self.file_name1], - config_obj=self.config, - git_obj=self.git - ).run() + self.settings._args = [self.file_name1, '--keep-in-cloud'] + CmdDataRemove(self.settings).run() + + self.settings._args = ['printf', 'Goodbye\nJack', '--stdout', self.file_name1] + CmdRun(self.settings).run() class ReproChangedDeepDependency(ReproChangedDependency): def test(self): self.recreate_file1() - CmdRepro(args=[self.file_name_res], - config_obj=self.config, - git_obj=self.git - ).run() + self.settings._args = [self.file_name_res] + CmdRepro(self.settings).run() self.assertEqual(open(self.file_name_res).read(), 'Goodbye\nBobby')