test: add initial tests for LOCAL and AWS data clouds

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2018-01-07 22:33:26 +02:00
parent eaee02e619
commit 1c89bb50c5
2 changed files with 129 additions and 22 deletions
+22 -22
View File
@@ -192,38 +192,38 @@ class DataCloudLOCAL(DataCloudBase):
"""
Driver for local storage.
"""
def push(self, item):
Logger.debug('sync to cloud ' + item.cache.dvc + " " + self.storage_path)
copyfile(item.cache.dvc, self.storage_path)
return item
def push(self, path):
Logger.debug('sync to cloud ' + path + " " + self.storage_path)
copyfile(path, self.storage_path)
return path
def _import(self, bucket, i, item):
tmp_file = self.tmp_file(item.data.dvc)
def _import(self, bucket, i, path):
inp = os.path.join(self.storage_path, i)
tmp_file = self.tmp_file(path)
try:
copyfile(i, tmp_file)
copyfile(inp, tmp_file)
except Exception as exc:
Logger.error('Failed to copy "{}": {}'.format(i, exc))
return None
os.rename(tmp_file, item.data.dvc)
item.move_data_to_cache()
os.rename(tmp_file, path)
return item
return path
def pull(self, item):
Logger.debug('sync from cloud ' + self.storage_path + " " + item.cache.dvc)
return self._import(None, self.storage_path, item)
def pull(self, path):
Logger.debug('sync from cloud ' + path)
return self._import(None, path, path)
def remove(self, item):
Logger.debug('rm from cloud ' + item.cache.dvc)
os.remove(item.cache.dvc)
def remove(self, path):
Logger.debug('rm from cloud ' + path)
os.remove(path)
def import_data(self, path, item):
Logger.debug('import from cloud ' + path + " " + item.data.dvc)
return self._import(None, path, item)
def import_data(self, path, out):
Logger.debug('import from cloud ' + path + " " + out)
return self._import(None, path, out)
def _status(self, data_item):
local = data_item.cache.relative
def _status(self, path):
local = path
remote = '{}/{}'.format(self.storage_path, os.path.basename(local))
remote_exists = os.path.exists(remote)
@@ -814,7 +814,7 @@ class DataCloud(object):
(F, issues) if bad
"""
key = 'Cloud'
if key.lower() not in self._config['Global'].keys() or len(self._config['Global'][key]) < 1:
if key.lower() not in [k.lower() for k in self._config['Global'].keys()] or len(self._config['Global'][key]) < 1:
raise ConfigError('Please set %s in section Global in config file' % key)
# now that a cloud is chosen, can check StoragePath
+107
View File
@@ -0,0 +1,107 @@
import os
from dvc.config import ConfigError
from dvc.data_cloud import DataCloud, DataCloudAWS, DataCloudLOCAL
from dvc.data_cloud import STATUS_UNKNOWN, STATUS_OK, STATUS_MODIFIED, STATUS_NEW, STATUS_DELETED
from dvc.cloud.instance_manager import CloudSettings
from tests.basic_env import TestDvc
class TestDataCloud(TestDvc):
def test(self):
for k,v in DataCloud.CLOUD_MAP.items():
config = {'Global': {'Cloud': k},
k: {'StoragePath': 'a/b',
'ProjectName': 'name'}}
cloud = DataCloud(config)
self.assertIsInstance(cloud._cloud, v)
with self.assertRaises(ConfigError) as cx:
config = {'Global': {'Cloud': 'not_supported_type'}}
DataCloud(config)
class TestDataCloudBase(TestDvc):
def _setup_cloud(self):
self.cloud = None
def _test_cloud(self):
self._setup_cloud()
# Check status
status = self.cloud.status(self.FOO)
self.assertEqual(status, STATUS_NEW)
# Push and check status
self.cloud.push(self.FOO)
self.assertTrue(os.path.exists(self.FOO))
self.assertTrue(os.path.isfile(self.FOO))
status = self.cloud.status(self.FOO)
self.assertEqual(status, STATUS_OK)
# Modify and check status
with open(self.FOO, 'a') as fd:
fd.write('addon')
status = self.cloud.status(self.FOO)
self.assertEqual(status, STATUS_MODIFIED)
# Remove and check status
os.unlink(self.FOO)
status = self.cloud.status(self.FOO)
self.assertEqual(status, STATUS_DELETED)
# Pull and check status
self.cloud.pull(self.FOO)
self.assertTrue(os.path.exists(self.FOO))
self.assertTrue(os.path.isfile(self.FOO))
with open(self.FOO, 'r') as fd:
self.assertEqual(fd.read(), self.FOO_CONTENTS)
status = self.cloud.status(self.FOO)
self.assertEqual(status, STATUS_OK)
class TestDataCloudAWS(TestDataCloudBase):
TEST_REPO_S3 = 'dvc-test/myrepo'
TEST_REPO_REGION = 'us-east-2'
def _should_test(self):
if os.getenv("TRAVIS_PULL_REQUEST") == "false" and \
os.getenv("TRAVIS_SECURE_ENV_VARS") == "true":
return True
return False
def _setup_cloud(self):
if not self._should_test():
return
# Cleanup
os.system('aws s3 rm --recursive "s3://{}/"'.format(self.TEST_REPO_S3))
# Setup cloud
config = {'StoragePath': self.TEST_REPO_S3,
'Region': self.TEST_REPO_REGION}
cloud_settings = CloudSettings(None, config)
self.cloud = DataCloudAWS(cloud_settings)
def test(self):
if os.getenv("TRAVIS_PULL_REQUEST") == "false" and \
os.getenv("TRAVIS_SECURE_ENV_VARS") == "true":
self._test_cloud()
class TestDataCloudLOCAL(TestDataCloudBase):
def _setup_cloud(self):
self.dname = 'cloud'
os.mkdir(self.dname)
config = {'StoragePath': self.dname}
cloud_settings = CloudSettings(None, config)
self.cloud = DataCloudLOCAL(cloud_settings)
def test(self):
self._test_cloud()
self.assertTrue(os.path.isdir(self.dname))
path = os.path.join(self.dname, self.FOO)
self.assertTrue(os.path.isfile(path))