Merge pull request #529 from efiop/master

config and cloud fixes

dvc/cli.py

@@ -130,7 +130,7 @@ def parse_args(argv=None):
     parent_cache_parser.add_argument('-j',
                         '--jobs',
                         type=int,
-                        default=cpu_count(),
+                        default=8*cpu_count(),
                         help='Number of jobs to run simultaneously.')
     parent_cache_parser.add_argument(
                         'targets',

dvc/cloud/aws.py

@@ -267,12 +267,14 @@ class DataCloudAWS(DataCloudBase):
                     continue
 
                 fobj.seek(offset)
+                name = os.path.relpath(fname, self._cloud_settings.cache.cache_dir)
+                cb = create_cb(name, offset, source_size)
                 multipart.upload_part_from_file(fp=fobj,
                                                 replace=False,
                                                 size=size,
                                                 num_cb=100,
                                                 part_num=part_num,
-                                                cb=create_cb(fname, offset, source_size))
+                                                cb=cb)
 
         if len(multipart.get_all_parts()) != chunk_count:
             raise Exception("Couldn't upload all file parts")

dvc/cloud/data_cloud.py

@@ -5,6 +5,7 @@ from dvc.logger import Logger
 from dvc.exceptions import DvcException
 from dvc.config import ConfigError
 from dvc.utils import map_progress
+from dvc.config import Config
 
 from dvc.cloud.aws import DataCloudAWS
 from dvc.cloud.gcp import DataCloudGCP
@@ -29,7 +30,7 @@ class DataCloud(object):
     def __init__(self, cache, config):
         self._config = config
 
-        cloud_type = self._config['Global'].get('Cloud', '').strip().upper()
+        cloud_type = self._config[Config.SECTION_CORE].get('Cloud', '').strip().upper()
         if cloud_type not in self.CLOUD_MAP.keys():
             raise ConfigError('Wrong cloud type %s specified' % cloud_type)
 
@@ -54,7 +55,7 @@ class DataCloud(object):
             cloud_config = None
         else:
             cloud_config = config[cloud_type]
-        global_storage_path = config['Global'].get('StoragePath', None)
+        global_storage_path = config[Config.SECTION_CORE].get('StoragePath', None)
         cloud_settings = CloudSettings(cache, global_storage_path, cloud_config)
         return cloud_settings
 
@@ -70,8 +71,9 @@ class DataCloud(object):
             (F, issues) if bad
         """
         key = 'Cloud'
-        if key.lower() not in [k.lower() for k in self._config['Global'].keys()] or len(self._config['Global'][key]) < 1:
-            raise ConfigError('Please set %s in section Global in config file' % key)
+        core = self._config[Config.SECTION_CORE]
+        if key.lower() not in [k.lower() for k in core.keys()] or len(core[key]) < 1:
+            raise ConfigError('Please set {} in section {} in config file'.format(key, Config.SECTION_CORE))
 
         # now that a cloud is chosen, can check StoragePath
         storage_path = self._cloud.storage_path

dvc/config.py

@@ -15,9 +15,9 @@ class ConfigError(DvcException):
 
 class Config(object):
     CONFIG = 'config'
-    SECTION_GLOBAL = 'Global'
+    SECTION_CORE = 'Core'
     CONFIG_TEMPLATE = '''
-[Global]
+[Core]
 # Supported clouds: AWS, GCP
 Cloud = AWS
 
@@ -47,8 +47,8 @@ ProjectName =
         except configparser.Error as ex:
             raise ConfigError(ex.message)
 
-        if not self._config.has_section(self.SECTION_GLOBAL):
-            raise ConfigError(u'section \'{}\' was not found'.format(self.SECTION_GLOBAL))
+        if not self._config.has_section(self.SECTION_CORE):
+            raise ConfigError(u'section \'{}\' was not found'.format(self.SECTION_CORE))
 
     @staticmethod
     def init(dvc_dir):

dvc/logger.py

@@ -27,7 +27,7 @@ class Logger(object):
 
     def __init__(self, config=None):
         if config:
-            level = config[Config.SECTION_GLOBAL].get('LogLevel', None)
+            level = config[Config.SECTION_CORE].get('LogLevel', None)
             Logger.set_level(level)
 
     @staticmethod

tests/test_config.py

@@ -33,7 +33,7 @@ class TestConfigTest(TestCase):
     def storage_path_hierarchy_test(self):
         """ StoragePath should be read first from global and then from cloud section """
 
-        c = ("[Global]",
+        c = ("[Core]",
              "LogLevel =",
              "DataDir = ",
              "CacheDir = ",
@@ -53,7 +53,7 @@ class TestConfigTest(TestCase):
         self.assertEqual(cloud._cloud.storage_bucket, 'globalsb')
         self.assertEqual(cloud._cloud.storage_prefix, 'global_storage_path')
 
-        c = ("[Global]",
+        c = ("[Core]",
              "LogLevel =",
              "DataDir = ",
              "CacheDir = ",
@@ -90,7 +90,7 @@ class TestConfigTest(TestCase):
         """ in absence of [AWS] -> CredentialPath, aws creds should be read from ~/.aws/credentials """
 
         default_path = os.path.expanduser('~/.aws/credentials')
-        c = ("[Global]",
+        c = ("[Core]",
              "LogLevel =",
              "DataDir = ",
              "CacheDir = ",
@@ -121,7 +121,7 @@ class TestConfigTest(TestCase):
     def aws_credentials_specified_tes_t_DISABLED(self, isfile_function):
         """ in presence of [AWS] -> CredentialPath, use those credentials """
 
-        c = ("[Global]",
+        c = ("[Core]",
              "LogLevel =",
              "DataDir = ",
              "CacheDir = ",

tests/test_data_cloud.py

@@ -4,7 +4,7 @@ import uuid
 import shutil
 
 from dvc.main import main
-from dvc.config import ConfigError
+from dvc.config import Config, ConfigError
 from dvc.cloud.data_cloud import DataCloud, DataCloudAWS, DataCloudLOCAL, DataCloudGCP
 from dvc.cloud.base import STATUS_UNKNOWN, STATUS_OK, STATUS_MODIFIED, STATUS_NEW, STATUS_DELETED
 from dvc.cloud.instance_manager import CloudSettings
@@ -15,14 +15,14 @@ from tests.basic_env import TestDvc
 class TestDataCloud(TestDvc):
     def test(self):
         for k,v in DataCloud.CLOUD_MAP.items():
-            config = {'Global': {'Cloud': k},
+            config = {Config.SECTION_CORE: {'Cloud': k},
                       k: {'StoragePath': 'a/b',
                           'ProjectName': 'name'}}
             cloud = DataCloud(self.dvc.cache, config)
             self.assertIsInstance(cloud._cloud, v)
 
         with self.assertRaises(ConfigError) as cx:
-            config = {'Global': {'Cloud': 'not_supported_type'}}
+            config = {Config.SECTION_CORE: {'Cloud': 'not_supported_type'}}
             DataCloud(self.dvc.cache, config)
 
 
@@ -194,7 +194,7 @@ class TestDataCloudLocalCli(TestDvc):
         dname = 'cloud'
         os.mkdir(dname)
 
-        self.main(['config', 'global.cloud', 'LOCAL'])
+        self.main(['config', 'Core.cloud', 'LOCAL'])
         self.main(['config', 'LOCAL.StoragePath', dname])
 
         stage = self.dvc.add(self.FOO)

tests/test_logger.py

@@ -1,6 +1,7 @@
 import colorama
 import logging
 
+from dvc.config import Config
 from dvc.logger import Logger
 
 from tests.basic_env import TestDvc
@@ -8,11 +9,11 @@ from tests.basic_env import TestDvc
 
 class TestLogger(TestDvc):
     def test_config(self):
-        config1 = {'Global': {'LogLevel': 'debug'}}
+        config1 = {Config.SECTION_CORE: {'LogLevel': 'debug'}}
         Logger(config1)
         self.assertEqual(Logger.logger().getEffectiveLevel(), logging.DEBUG)
 
-        config2 = {'Global': {'LogLevel': 'error'}}
+        config2 = {Config.SECTION_CORE: {'LogLevel': 'error'}}
         Logger(config2)
         self.assertEqual(Logger.logger().getEffectiveLevel(), logging.ERROR)