From fa433092222c7b4846d78ac6ebf1e087bbbbc555 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 8 Mar 2018 09:04:28 +0200 Subject: [PATCH 1/3] config: rename 'Global' section to 'Core' Signed-off-by: Ruslan Kuprieiev --- dvc/cloud/data_cloud.py | 10 ++++++---- dvc/config.py | 8 ++++---- dvc/logger.py | 2 +- tests/test_config.py | 8 ++++---- tests/test_data_cloud.py | 8 ++++---- tests/test_logger.py | 5 +++-- 6 files changed, 22 insertions(+), 19 deletions(-) diff --git a/dvc/cloud/data_cloud.py b/dvc/cloud/data_cloud.py index f5cc84711..3080da018 100644 --- a/dvc/cloud/data_cloud.py +++ b/dvc/cloud/data_cloud.py @@ -5,6 +5,7 @@ from dvc.logger import Logger from dvc.exceptions import DvcException from dvc.config import ConfigError from dvc.utils import map_progress +from dvc.config import Config from dvc.cloud.aws import DataCloudAWS from dvc.cloud.gcp import DataCloudGCP @@ -29,7 +30,7 @@ class DataCloud(object): def __init__(self, cache, config): self._config = config - cloud_type = self._config['Global'].get('Cloud', '').strip().upper() + cloud_type = self._config[Config.SECTION_CORE].get('Cloud', '').strip().upper() if cloud_type not in self.CLOUD_MAP.keys(): raise ConfigError('Wrong cloud type %s specified' % cloud_type) @@ -54,7 +55,7 @@ class DataCloud(object): cloud_config = None else: cloud_config = config[cloud_type] - global_storage_path = config['Global'].get('StoragePath', None) + global_storage_path = config[Config.SECTION_CORE].get('StoragePath', None) cloud_settings = CloudSettings(cache, global_storage_path, cloud_config) return cloud_settings @@ -70,8 +71,9 @@ class DataCloud(object): (F, issues) if bad """ key = 'Cloud' - if key.lower() not in [k.lower() for k in self._config['Global'].keys()] or len(self._config['Global'][key]) < 1: - raise ConfigError('Please set %s in section Global in config file' % key) + core = self._config[Config.SECTION_CORE] + if key.lower() not in [k.lower() for k in core.keys()] or len(core[key]) < 1: + raise ConfigError('Please set {} in section {} in config file'.format(key, Config.SECTION_CORE)) # now that a cloud is chosen, can check StoragePath storage_path = self._cloud.storage_path diff --git a/dvc/config.py b/dvc/config.py index a8ea854bb..4e3e901a9 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -15,9 +15,9 @@ class ConfigError(DvcException): class Config(object): CONFIG = 'config' - SECTION_GLOBAL = 'Global' + SECTION_CORE = 'Core' CONFIG_TEMPLATE = ''' -[Global] +[Core] # Supported clouds: AWS, GCP Cloud = AWS @@ -47,8 +47,8 @@ ProjectName = except configparser.Error as ex: raise ConfigError(ex.message) - if not self._config.has_section(self.SECTION_GLOBAL): - raise ConfigError(u'section \'{}\' was not found'.format(self.SECTION_GLOBAL)) + if not self._config.has_section(self.SECTION_CORE): + raise ConfigError(u'section \'{}\' was not found'.format(self.SECTION_CORE)) @staticmethod def init(dvc_dir): diff --git a/dvc/logger.py b/dvc/logger.py index 95a1b35ac..d4a7bf990 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -27,7 +27,7 @@ class Logger(object): def __init__(self, config=None): if config: - level = config[Config.SECTION_GLOBAL].get('LogLevel', None) + level = config[Config.SECTION_CORE].get('LogLevel', None) Logger.set_level(level) @staticmethod diff --git a/tests/test_config.py b/tests/test_config.py index 96c9ae60d..1fb84aed4 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -33,7 +33,7 @@ class TestConfigTest(TestCase): def storage_path_hierarchy_test(self): """ StoragePath should be read first from global and then from cloud section """ - c = ("[Global]", + c = ("[Core]", "LogLevel =", "DataDir = ", "CacheDir = ", @@ -53,7 +53,7 @@ class TestConfigTest(TestCase): self.assertEqual(cloud._cloud.storage_bucket, 'globalsb') self.assertEqual(cloud._cloud.storage_prefix, 'global_storage_path') - c = ("[Global]", + c = ("[Core]", "LogLevel =", "DataDir = ", "CacheDir = ", @@ -90,7 +90,7 @@ class TestConfigTest(TestCase): """ in absence of [AWS] -> CredentialPath, aws creds should be read from ~/.aws/credentials """ default_path = os.path.expanduser('~/.aws/credentials') - c = ("[Global]", + c = ("[Core]", "LogLevel =", "DataDir = ", "CacheDir = ", @@ -121,7 +121,7 @@ class TestConfigTest(TestCase): def aws_credentials_specified_tes_t_DISABLED(self, isfile_function): """ in presence of [AWS] -> CredentialPath, use those credentials """ - c = ("[Global]", + c = ("[Core]", "LogLevel =", "DataDir = ", "CacheDir = ", diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index 510080a62..9ce221615 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -4,7 +4,7 @@ import uuid import shutil from dvc.main import main -from dvc.config import ConfigError +from dvc.config import Config, ConfigError from dvc.cloud.data_cloud import DataCloud, DataCloudAWS, DataCloudLOCAL, DataCloudGCP from dvc.cloud.base import STATUS_UNKNOWN, STATUS_OK, STATUS_MODIFIED, STATUS_NEW, STATUS_DELETED from dvc.cloud.instance_manager import CloudSettings @@ -15,14 +15,14 @@ from tests.basic_env import TestDvc class TestDataCloud(TestDvc): def test(self): for k,v in DataCloud.CLOUD_MAP.items(): - config = {'Global': {'Cloud': k}, + config = {Config.SECTION_CORE: {'Cloud': k}, k: {'StoragePath': 'a/b', 'ProjectName': 'name'}} cloud = DataCloud(self.dvc.cache, config) self.assertIsInstance(cloud._cloud, v) with self.assertRaises(ConfigError) as cx: - config = {'Global': {'Cloud': 'not_supported_type'}} + config = {Config.SECTION_CORE: {'Cloud': 'not_supported_type'}} DataCloud(self.dvc.cache, config) @@ -194,7 +194,7 @@ class TestDataCloudLocalCli(TestDvc): dname = 'cloud' os.mkdir(dname) - self.main(['config', 'global.cloud', 'LOCAL']) + self.main(['config', 'Core.cloud', 'LOCAL']) self.main(['config', 'LOCAL.StoragePath', dname]) stage = self.dvc.add(self.FOO) diff --git a/tests/test_logger.py b/tests/test_logger.py index 3c6e04d1d..cdb5a1919 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -1,6 +1,7 @@ import colorama import logging +from dvc.config import Config from dvc.logger import Logger from tests.basic_env import TestDvc @@ -8,11 +9,11 @@ from tests.basic_env import TestDvc class TestLogger(TestDvc): def test_config(self): - config1 = {'Global': {'LogLevel': 'debug'}} + config1 = {Config.SECTION_CORE: {'LogLevel': 'debug'}} Logger(config1) self.assertEqual(Logger.logger().getEffectiveLevel(), logging.DEBUG) - config2 = {'Global': {'LogLevel': 'error'}} + config2 = {Config.SECTION_CORE: {'LogLevel': 'error'}} Logger(config2) self.assertEqual(Logger.logger().getEffectiveLevel(), logging.ERROR) From ff77d9dc469db854090601f4bf1cf9e5ab9a962f Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 8 Mar 2018 09:51:24 +0200 Subject: [PATCH 2/3] dvc: calculate n jobs as 8*n_cpu Signed-off-by: Ruslan Kuprieiev --- dvc/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/cli.py b/dvc/cli.py index f0a1556fb..f275f788b 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -130,7 +130,7 @@ def parse_args(argv=None): parent_cache_parser.add_argument('-j', '--jobs', type=int, - default=cpu_count(), + default=8*cpu_count(), help='Number of jobs to run simultaneously.') parent_cache_parser.add_argument( 'targets', From 4bc62cbf66fc67aa49e6dcdf1c6c775b47d4d733 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 8 Mar 2018 12:44:44 +0200 Subject: [PATCH 3/3] aws: don't forget to use relpath in progress callback Fixes #519 Signed-off-by: Ruslan Kuprieiev --- dvc/cloud/aws.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dvc/cloud/aws.py b/dvc/cloud/aws.py index 7e354f140..eb1e024b7 100644 --- a/dvc/cloud/aws.py +++ b/dvc/cloud/aws.py @@ -267,12 +267,14 @@ class DataCloudAWS(DataCloudBase): continue fobj.seek(offset) + name = os.path.relpath(fname, self._cloud_settings.cache.cache_dir) + cb = create_cb(name, offset, source_size) multipart.upload_part_from_file(fp=fobj, replace=False, size=size, num_cb=100, part_num=part_num, - cb=create_cb(fname, offset, source_size)) + cb=cb) if len(multipart.get_all_parts()) != chunk_count: raise Exception("Couldn't upload all file parts")