mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 23:11:44 +01:00
test: add initial tests for LOCAL and AWS data clouds
Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
+22
-22
@@ -192,38 +192,38 @@ class DataCloudLOCAL(DataCloudBase):
|
||||
"""
|
||||
Driver for local storage.
|
||||
"""
|
||||
def push(self, item):
|
||||
Logger.debug('sync to cloud ' + item.cache.dvc + " " + self.storage_path)
|
||||
copyfile(item.cache.dvc, self.storage_path)
|
||||
return item
|
||||
def push(self, path):
|
||||
Logger.debug('sync to cloud ' + path + " " + self.storage_path)
|
||||
copyfile(path, self.storage_path)
|
||||
return path
|
||||
|
||||
def _import(self, bucket, i, item):
|
||||
tmp_file = self.tmp_file(item.data.dvc)
|
||||
def _import(self, bucket, i, path):
|
||||
inp = os.path.join(self.storage_path, i)
|
||||
tmp_file = self.tmp_file(path)
|
||||
try:
|
||||
copyfile(i, tmp_file)
|
||||
copyfile(inp, tmp_file)
|
||||
except Exception as exc:
|
||||
Logger.error('Failed to copy "{}": {}'.format(i, exc))
|
||||
return None
|
||||
|
||||
os.rename(tmp_file, item.data.dvc)
|
||||
item.move_data_to_cache()
|
||||
os.rename(tmp_file, path)
|
||||
|
||||
return item
|
||||
return path
|
||||
|
||||
def pull(self, item):
|
||||
Logger.debug('sync from cloud ' + self.storage_path + " " + item.cache.dvc)
|
||||
return self._import(None, self.storage_path, item)
|
||||
def pull(self, path):
|
||||
Logger.debug('sync from cloud ' + path)
|
||||
return self._import(None, path, path)
|
||||
|
||||
def remove(self, item):
|
||||
Logger.debug('rm from cloud ' + item.cache.dvc)
|
||||
os.remove(item.cache.dvc)
|
||||
def remove(self, path):
|
||||
Logger.debug('rm from cloud ' + path)
|
||||
os.remove(path)
|
||||
|
||||
def import_data(self, path, item):
|
||||
Logger.debug('import from cloud ' + path + " " + item.data.dvc)
|
||||
return self._import(None, path, item)
|
||||
def import_data(self, path, out):
|
||||
Logger.debug('import from cloud ' + path + " " + out)
|
||||
return self._import(None, path, out)
|
||||
|
||||
def _status(self, data_item):
|
||||
local = data_item.cache.relative
|
||||
def _status(self, path):
|
||||
local = path
|
||||
remote = '{}/{}'.format(self.storage_path, os.path.basename(local))
|
||||
|
||||
remote_exists = os.path.exists(remote)
|
||||
@@ -814,7 +814,7 @@ class DataCloud(object):
|
||||
(F, issues) if bad
|
||||
"""
|
||||
key = 'Cloud'
|
||||
if key.lower() not in self._config['Global'].keys() or len(self._config['Global'][key]) < 1:
|
||||
if key.lower() not in [k.lower() for k in self._config['Global'].keys()] or len(self._config['Global'][key]) < 1:
|
||||
raise ConfigError('Please set %s in section Global in config file' % key)
|
||||
|
||||
# now that a cloud is chosen, can check StoragePath
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
import os
|
||||
|
||||
from dvc.config import ConfigError
|
||||
from dvc.data_cloud import DataCloud, DataCloudAWS, DataCloudLOCAL
|
||||
from dvc.data_cloud import STATUS_UNKNOWN, STATUS_OK, STATUS_MODIFIED, STATUS_NEW, STATUS_DELETED
|
||||
from dvc.cloud.instance_manager import CloudSettings
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
class TestDataCloud(TestDvc):
|
||||
def test(self):
|
||||
for k,v in DataCloud.CLOUD_MAP.items():
|
||||
config = {'Global': {'Cloud': k},
|
||||
k: {'StoragePath': 'a/b',
|
||||
'ProjectName': 'name'}}
|
||||
cloud = DataCloud(config)
|
||||
self.assertIsInstance(cloud._cloud, v)
|
||||
|
||||
with self.assertRaises(ConfigError) as cx:
|
||||
config = {'Global': {'Cloud': 'not_supported_type'}}
|
||||
DataCloud(config)
|
||||
|
||||
|
||||
class TestDataCloudBase(TestDvc):
|
||||
def _setup_cloud(self):
|
||||
self.cloud = None
|
||||
|
||||
def _test_cloud(self):
|
||||
self._setup_cloud()
|
||||
|
||||
# Check status
|
||||
status = self.cloud.status(self.FOO)
|
||||
self.assertEqual(status, STATUS_NEW)
|
||||
|
||||
# Push and check status
|
||||
self.cloud.push(self.FOO)
|
||||
self.assertTrue(os.path.exists(self.FOO))
|
||||
self.assertTrue(os.path.isfile(self.FOO))
|
||||
|
||||
status = self.cloud.status(self.FOO)
|
||||
self.assertEqual(status, STATUS_OK)
|
||||
|
||||
# Modify and check status
|
||||
with open(self.FOO, 'a') as fd:
|
||||
fd.write('addon')
|
||||
status = self.cloud.status(self.FOO)
|
||||
self.assertEqual(status, STATUS_MODIFIED)
|
||||
|
||||
# Remove and check status
|
||||
os.unlink(self.FOO)
|
||||
status = self.cloud.status(self.FOO)
|
||||
self.assertEqual(status, STATUS_DELETED)
|
||||
|
||||
# Pull and check status
|
||||
self.cloud.pull(self.FOO)
|
||||
self.assertTrue(os.path.exists(self.FOO))
|
||||
self.assertTrue(os.path.isfile(self.FOO))
|
||||
with open(self.FOO, 'r') as fd:
|
||||
self.assertEqual(fd.read(), self.FOO_CONTENTS)
|
||||
|
||||
status = self.cloud.status(self.FOO)
|
||||
self.assertEqual(status, STATUS_OK)
|
||||
|
||||
|
||||
class TestDataCloudAWS(TestDataCloudBase):
|
||||
TEST_REPO_S3 = 'dvc-test/myrepo'
|
||||
TEST_REPO_REGION = 'us-east-2'
|
||||
|
||||
def _should_test(self):
|
||||
if os.getenv("TRAVIS_PULL_REQUEST") == "false" and \
|
||||
os.getenv("TRAVIS_SECURE_ENV_VARS") == "true":
|
||||
return True
|
||||
return False
|
||||
|
||||
def _setup_cloud(self):
|
||||
if not self._should_test():
|
||||
return
|
||||
|
||||
# Cleanup
|
||||
os.system('aws s3 rm --recursive "s3://{}/"'.format(self.TEST_REPO_S3))
|
||||
|
||||
# Setup cloud
|
||||
config = {'StoragePath': self.TEST_REPO_S3,
|
||||
'Region': self.TEST_REPO_REGION}
|
||||
cloud_settings = CloudSettings(None, config)
|
||||
self.cloud = DataCloudAWS(cloud_settings)
|
||||
|
||||
def test(self):
|
||||
if os.getenv("TRAVIS_PULL_REQUEST") == "false" and \
|
||||
os.getenv("TRAVIS_SECURE_ENV_VARS") == "true":
|
||||
self._test_cloud()
|
||||
|
||||
|
||||
class TestDataCloudLOCAL(TestDataCloudBase):
|
||||
def _setup_cloud(self):
|
||||
self.dname = 'cloud'
|
||||
os.mkdir(self.dname)
|
||||
config = {'StoragePath': self.dname}
|
||||
cloud_settings = CloudSettings(None, config)
|
||||
self.cloud = DataCloudLOCAL(cloud_settings)
|
||||
|
||||
def test(self):
|
||||
self._test_cloud()
|
||||
self.assertTrue(os.path.isdir(self.dname))
|
||||
path = os.path.join(self.dname, self.FOO)
|
||||
self.assertTrue(os.path.isfile(path))
|
||||
Reference in New Issue
Block a user