From 1c89bb50c5babaa30b2048de2d49f3eb60af7923 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 7 Jan 2018 22:32:46 +0200 Subject: [PATCH] test: add initial tests for LOCAL and AWS data clouds Signed-off-by: Ruslan Kuprieiev --- dvc/data_cloud.py | 44 ++++++++-------- tests/test_data_cloud.py | 107 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 22 deletions(-) create mode 100644 tests/test_data_cloud.py diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 7ae5244f9..42b10bf65 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -192,38 +192,38 @@ class DataCloudLOCAL(DataCloudBase): """ Driver for local storage. """ - def push(self, item): - Logger.debug('sync to cloud ' + item.cache.dvc + " " + self.storage_path) - copyfile(item.cache.dvc, self.storage_path) - return item + def push(self, path): + Logger.debug('sync to cloud ' + path + " " + self.storage_path) + copyfile(path, self.storage_path) + return path - def _import(self, bucket, i, item): - tmp_file = self.tmp_file(item.data.dvc) + def _import(self, bucket, i, path): + inp = os.path.join(self.storage_path, i) + tmp_file = self.tmp_file(path) try: - copyfile(i, tmp_file) + copyfile(inp, tmp_file) except Exception as exc: Logger.error('Failed to copy "{}": {}'.format(i, exc)) return None - os.rename(tmp_file, item.data.dvc) - item.move_data_to_cache() + os.rename(tmp_file, path) - return item + return path - def pull(self, item): - Logger.debug('sync from cloud ' + self.storage_path + " " + item.cache.dvc) - return self._import(None, self.storage_path, item) + def pull(self, path): + Logger.debug('sync from cloud ' + path) + return self._import(None, path, path) - def remove(self, item): - Logger.debug('rm from cloud ' + item.cache.dvc) - os.remove(item.cache.dvc) + def remove(self, path): + Logger.debug('rm from cloud ' + path) + os.remove(path) - def import_data(self, path, item): - Logger.debug('import from cloud ' + path + " " + item.data.dvc) - return self._import(None, path, item) + def import_data(self, path, out): + Logger.debug('import from cloud ' + path + " " + out) + return self._import(None, path, out) - def _status(self, data_item): - local = data_item.cache.relative + def _status(self, path): + local = path remote = '{}/{}'.format(self.storage_path, os.path.basename(local)) remote_exists = os.path.exists(remote) @@ -814,7 +814,7 @@ class DataCloud(object): (F, issues) if bad """ key = 'Cloud' - if key.lower() not in self._config['Global'].keys() or len(self._config['Global'][key]) < 1: + if key.lower() not in [k.lower() for k in self._config['Global'].keys()] or len(self._config['Global'][key]) < 1: raise ConfigError('Please set %s in section Global in config file' % key) # now that a cloud is chosen, can check StoragePath diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py new file mode 100644 index 000000000..dbc292031 --- /dev/null +++ b/tests/test_data_cloud.py @@ -0,0 +1,107 @@ +import os + +from dvc.config import ConfigError +from dvc.data_cloud import DataCloud, DataCloudAWS, DataCloudLOCAL +from dvc.data_cloud import STATUS_UNKNOWN, STATUS_OK, STATUS_MODIFIED, STATUS_NEW, STATUS_DELETED +from dvc.cloud.instance_manager import CloudSettings + +from tests.basic_env import TestDvc + + +class TestDataCloud(TestDvc): + def test(self): + for k,v in DataCloud.CLOUD_MAP.items(): + config = {'Global': {'Cloud': k}, + k: {'StoragePath': 'a/b', + 'ProjectName': 'name'}} + cloud = DataCloud(config) + self.assertIsInstance(cloud._cloud, v) + + with self.assertRaises(ConfigError) as cx: + config = {'Global': {'Cloud': 'not_supported_type'}} + DataCloud(config) + + +class TestDataCloudBase(TestDvc): + def _setup_cloud(self): + self.cloud = None + + def _test_cloud(self): + self._setup_cloud() + + # Check status + status = self.cloud.status(self.FOO) + self.assertEqual(status, STATUS_NEW) + + # Push and check status + self.cloud.push(self.FOO) + self.assertTrue(os.path.exists(self.FOO)) + self.assertTrue(os.path.isfile(self.FOO)) + + status = self.cloud.status(self.FOO) + self.assertEqual(status, STATUS_OK) + + # Modify and check status + with open(self.FOO, 'a') as fd: + fd.write('addon') + status = self.cloud.status(self.FOO) + self.assertEqual(status, STATUS_MODIFIED) + + # Remove and check status + os.unlink(self.FOO) + status = self.cloud.status(self.FOO) + self.assertEqual(status, STATUS_DELETED) + + # Pull and check status + self.cloud.pull(self.FOO) + self.assertTrue(os.path.exists(self.FOO)) + self.assertTrue(os.path.isfile(self.FOO)) + with open(self.FOO, 'r') as fd: + self.assertEqual(fd.read(), self.FOO_CONTENTS) + + status = self.cloud.status(self.FOO) + self.assertEqual(status, STATUS_OK) + + +class TestDataCloudAWS(TestDataCloudBase): + TEST_REPO_S3 = 'dvc-test/myrepo' + TEST_REPO_REGION = 'us-east-2' + + def _should_test(self): + if os.getenv("TRAVIS_PULL_REQUEST") == "false" and \ + os.getenv("TRAVIS_SECURE_ENV_VARS") == "true": + return True + return False + + def _setup_cloud(self): + if not self._should_test(): + return + + # Cleanup + os.system('aws s3 rm --recursive "s3://{}/"'.format(self.TEST_REPO_S3)) + + # Setup cloud + config = {'StoragePath': self.TEST_REPO_S3, + 'Region': self.TEST_REPO_REGION} + cloud_settings = CloudSettings(None, config) + self.cloud = DataCloudAWS(cloud_settings) + + def test(self): + if os.getenv("TRAVIS_PULL_REQUEST") == "false" and \ + os.getenv("TRAVIS_SECURE_ENV_VARS") == "true": + self._test_cloud() + + +class TestDataCloudLOCAL(TestDataCloudBase): + def _setup_cloud(self): + self.dname = 'cloud' + os.mkdir(self.dname) + config = {'StoragePath': self.dname} + cloud_settings = CloudSettings(None, config) + self.cloud = DataCloudLOCAL(cloud_settings) + + def test(self): + self._test_cloud() + self.assertTrue(os.path.isdir(self.dname)) + path = os.path.join(self.dname, self.FOO) + self.assertTrue(os.path.isfile(path))