Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-09 23:10:24 +01:00
Code Issues Packages Projects Releases Wiki Activity

dvc: misc changes for DataCloud, 'data sync' and 'remove'

Browse Source 
This patch introduces DataCloud class that used config to
determine proper cloud class(DataCloudAWS and DataCloudGCP
are awailable) to use in "data_sync" and "remove" commands.

This patch focuses on introducing the common class itself
saving original code and definetly needs some RFC later.

It also adds simple DataCloudLOCAL class that uses directory
as a StoragePath and cp as a sync method. Which makes testing
so much easier and covers local scenarios.

And it also introduces ability to run data sync in parallel
by collecting files first and then running sync in a number
of threads.

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2017-05-28 22:28:58 +03:00
parent adcd9038ce
commit a01ef3197c
7 changed files with 417 additions and 402 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_config.py
+18 -16
View File
@@ -10,7 +10,7 @@ import mock
from dvc.path.path import Path
from dvc.system import System
from dvc.config import Config
from dvc.command.data_cloud import DataCloud
class TestConfigTest(TestCase):
def setUp(self):
@@ -24,20 +24,20 @@ class TestConfigTest(TestCase):
"DataDir = ",
"CacheDir = ",
"StateDir = ",
"Cloud = amazon",
"Cloud = aws",
"StoragePath = globalsb/global_storage_path",
"[AWS]",
"StoragePath = awssb/aws_storage_path",
"CredentialPath =",
"[GC]",
"[GCP]",
"StoragePath = googlesb/google_storage_path"
)
s = StringIO.StringIO('\n'.join(c))
conf = Config(s, conf_pseudo_file = s)
self.assertEqual(conf.cloud, 'amazon')
self.assertEqual(conf.storage_bucket, 'globalsb')
self.assertEqual(conf.storage_prefix, 'global_storage_path')
cloud = DataCloud(conf)
self.assertEqual(cloud.typ, 'aws')
self.assertEqual(cloud._cloud.storage_bucket, 'globalsb')
self.assertEqual(cloud._cloud.storage_prefix, 'global_storage_path')
c = ("[Global]",
"LogLevel =",
@@ -48,15 +48,15 @@ class TestConfigTest(TestCase):
"[AWS]",
"StoragePath = awssb/aws_storage_path",
"CredentialPath =",
"[GC]",
"[GCP]",
"StoragePath = googlesb/google_storage_path"
)
s = StringIO.StringIO('\n'.join(c))
conf = Config(s, conf_pseudo_file=s)
self.assertEqual(conf.cloud, 'aws')
self.assertEqual(conf.storage_bucket, 'awssb')
self.assertEqual(conf.storage_prefix, 'aws_storage_path')
cloud = DataCloud(conf)
self.assertEqual(cloud.typ, 'aws')
self.assertEqual(cloud._cloud.storage_bucket, 'awssb')
self.assertEqual(cloud._cloud.storage_prefix, 'aws_storage_path')
def mocked_open_aws_default_credentials(klass, file, asdf):
@@ -82,7 +82,7 @@ class TestConfigTest(TestCase):
"DataDir = ",
"CacheDir = ",
"StateDir = ",
"Cloud = amazon",
"Cloud = aws",
"[AWS]",
"StoragePath = awssb/aws_storage_path",
"CredentialPath =",
@@ -92,7 +92,8 @@ class TestConfigTest(TestCase):
patcher = mock.patch('__builtin__.open', side_effect=self.mocked_open_aws_default_credentials)
patcher.start()
conf = Config(s, conf_pseudo_file=s)
aws_creds = conf.get_aws_credentials()
cloud = DataCloud(conf)
aws_creds = cloud._cloud.get_aws_credentials()
patcher.stop()
self.assertEqual(aws_creds[0], 'default_access_id')
@@ -112,7 +113,7 @@ class TestConfigTest(TestCase):
"DataDir = ",
"CacheDir = ",
"StateDir = ",
"Cloud = amazon",
"Cloud = aws",
"[AWS]",
"StoragePath = awssb/aws_storage_path",
"CredentialPath = some_credential_path",
@@ -122,7 +123,8 @@ class TestConfigTest(TestCase):
patcher = mock.patch('__builtin__.open', side_effect=self.mocked_open_aws_default_credentials)
patcher.start()
conf = Config(s, conf_pseudo_file=s)
aws_creds = conf.get_aws_credentials()
cloud = DataCloud(conf)
aws_creds = cloud._cloud.get_aws_credentials()
patcher.stop()
self.assertEqual(aws_creds[0], 'override_access_id')