diff --git a/dvc/config.py b/dvc/config.py index 6e11bf917..25efa3022 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -36,14 +36,21 @@ class ConfigI(object): class Config(ConfigI): CONFIG = 'dvc.conf' - - def __init__(self, conf_file): + def __init__(self, conf_file, conf_pseudo_file=None): + """ + Params: + conf_file (String): configuration file + conf_pseudo_file (String): for unit testing, something that supports readline; supersedes conf_file + """ self._conf_file = conf_file self._config = configparser.SafeConfigParser() - if not os.path.isfile(conf_file): - raise ConfigError('Config file "{}" does not exist'.format(conf_file)) - self._config.read(conf_file) + if conf_pseudo_file is not None: + self._config.readfp(conf_pseudo_file) + else: + if not os.path.isfile(conf_file): + raise ConfigError('Config file "{}" does not exist'.format(conf_file)) + self._config.read(conf_file) level = self._config['Global']['LogLevel'] Logger.set_level(level) @@ -167,10 +174,11 @@ class Config(ConfigI): cc = configparser.SafeConfigParser() threw = False try: - read = cc.read(path) - except: + # use readfp(open( ... to aid mocking. + cc.readfp(open(path, 'r')) + except Exception as e: threw = True - if not threw and len(read) > 0 and read[0] == path and 'default' in cc.keys(): + if not threw and 'default' in cc.keys(): access_key = cc['default'].get('aws_access_key_id', None) secret = cc['default'].get('aws_secret_access_key', None) @@ -194,7 +202,7 @@ class Config(ConfigI): errors = [] for key in ['Cloud']: if key.lower() not in self._config['Global'].keys() or len(self._config['Global'][key]) < 1: - errors.append('Please set %s in section Global in config file %s' % (key, self.CONFIG)) + errors.append('Please set %s in section Global in config file %s' % (key, self.file)) # now that a cloud is chosen, can check StoragePath sp = self.storage_path diff --git a/requirements.txt b/requirements.txt index 7d38fb5c0..65bb20b7d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ decorator==4.0.11 dill==0.2.6 enum34==1.1.6 fasteners==0.14.1 +funcsigs==1.0.2 future==0.16.0 futures==3.0.5 gapic-google-cloud-datastore-v1==0.15.3 @@ -48,6 +49,7 @@ httplib2==0.10.3 ipython==5.3.0 ipython-genutils==0.2.0 macholib==1.8 +mock==2.0.0 modulegraph==0.14 monotonic==1.3 nose==1.3.7 @@ -55,6 +57,7 @@ oauth2client==3.0.0 packaging==16.8 pathlib==1.0.1 pathlib2==2.2.1 +pbr==3.0.0 pexpect==4.2.1 pickleshare==0.7.4 ply==3.8 diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 000000000..6e5ce1749 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,129 @@ +import io +import os +import shutil +import StringIO +import traceback +from unittest import TestCase +import mock + + +from dvc.path.path import Path +from dvc.system import System +from dvc.config import Config + + +class TestConfigTest(TestCase): + def setUp(self): + pass + + def storage_path_hierarchy_test(self): + """ StoragePath should be read first from global and then from cloud section """ + + c = ("[Global]", + "LogLevel =", + "DataDir = ", + "CacheDir = ", + "StateDir = ", + "Cloud = amazon", + "StoragePath = globalsb/global_storage_path", + "[AWS]", + "StoragePath = awssb/aws_storage_path", + "CredentialPath =", + "[GC]", + "StoragePath = googlesb/google_storage_path" + ) + s = StringIO.StringIO('\n'.join(c)) + conf = Config(s, conf_pseudo_file = s) + + self.assertEqual(conf.cloud, 'amazon') + self.assertEqual(conf.storage_bucket, 'globalsb') + self.assertEqual(conf.storage_prefix, 'global_storage_path') + + c = ("[Global]", + "LogLevel =", + "DataDir = ", + "CacheDir = ", + "StateDir = ", + "Cloud = amazon", + "[AWS]", + "StoragePath = awssb/aws_storage_path", + "CredentialPath =", + "[GC]", + "StoragePath = googlesb/google_storage_path" + ) + s = StringIO.StringIO('\n'.join(c)) + conf = Config(s, conf_pseudo_file=s) + + self.assertEqual(conf.cloud, 'amazon') + self.assertEqual(conf.storage_bucket, 'awssb') + self.assertEqual(conf.storage_prefix, 'aws_storage_path') + + + def mocked_open_aws_default_credentials(klass, file, asdf): + """ mocked open for ~/.aws/credentials """ + if file == os.path.expanduser('~/.aws/credentials'): + creds0 = ("[default]", + "aws_access_key_id = default_access_id", + "aws_secret_access_key = default_sekret") + return io.BytesIO('\n'.join(creds0)) + + creds1 = ("[default]", + "aws_access_key_id = override_access_id", + "aws_secret_access_key = override_sekret") + + return io.BytesIO('\n'.join(creds1)) + + def aws_credentials_default_test(self): + """ in absence of [AWS] -> CredentialPath, aws creds should be read from ~/.aws/credentials """ + + default_path = os.path.expanduser('~/.aws/credentials') + c = ("[Global]", + "LogLevel =", + "DataDir = ", + "CacheDir = ", + "StateDir = ", + "Cloud = amazon", + "[AWS]", + "StoragePath = awssb/aws_storage_path", + "CredentialPath =", + ) + s = StringIO.StringIO('\n'.join(c)) + + patcher = mock.patch('__builtin__.open', side_effect=self.mocked_open_aws_default_credentials) + patcher.start() + conf = Config(s, conf_pseudo_file=s) + aws_creds = conf.get_aws_credentials() + patcher.stop() + + self.assertEqual(aws_creds[0], 'default_access_id') + self.assertEqual(aws_creds[1], 'default_sekret') + + + def mocked_isfile(file): + """ tell os some_credential_path exists """ + return file == 'some_credential_path' + + @mock.patch('dvc.config.os.path.isfile', side_effect = mocked_isfile) + def aws_credentials_specified_test(self, isfile_function): + """ in presence of [AWS] -> CredentialPath, use those credentials """ + + c = ("[Global]", + "LogLevel =", + "DataDir = ", + "CacheDir = ", + "StateDir = ", + "Cloud = amazon", + "[AWS]", + "StoragePath = awssb/aws_storage_path", + "CredentialPath = some_credential_path", + ) + s = StringIO.StringIO('\n'.join(c)) + + patcher = mock.patch('__builtin__.open', side_effect=self.mocked_open_aws_default_credentials) + patcher.start() + conf = Config(s, conf_pseudo_file=s) + aws_creds = conf.get_aws_credentials() + patcher.stop() + + self.assertEqual(aws_creds[0], 'override_access_id') + self.assertEqual(aws_creds[1], 'override_sekret')