dvc: add get_url()/open() API

As a part of this:
- DataCloud._cloud was refactored into `.get_remote()`
- data stopped using internal _cloud
- added a parametrization for remote tests
This commit is contained in:
Alexander Schepanovski
2019-05-27 17:56:31 +07:00
parent eac6adada6
commit ca93dbcf13
6 changed files with 147 additions and 36 deletions
+93
View File
@@ -0,0 +1,93 @@
import pytest
import shutil
from dvc import api
from dvc.main import main
from .test_data_cloud import (
_should_test_aws,
get_aws_url,
_should_test_gcp,
get_gcp_url,
_should_test_azure,
get_azure_url,
_should_test_oss,
get_oss_url,
_should_test_ssh,
get_ssh_url,
_should_test_hdfs,
get_hdfs_url,
)
# NOTE: staticmethod is only needed in Python 2
class S3:
should_test = staticmethod(_should_test_aws)
get_url = staticmethod(get_aws_url)
class GCP:
should_test = staticmethod(_should_test_gcp)
get_url = staticmethod(get_gcp_url)
class Azure:
should_test = staticmethod(_should_test_azure)
get_url = staticmethod(get_azure_url)
class OSS:
should_test = staticmethod(_should_test_oss)
get_url = staticmethod(get_oss_url)
class SSH:
should_test = staticmethod(_should_test_ssh)
get_url = staticmethod(get_ssh_url)
class HDFS:
should_test = staticmethod(_should_test_hdfs)
get_url = staticmethod(get_hdfs_url)
remote_params = [S3, GCP, Azure, OSS, SSH, HDFS]
@pytest.fixture
def remote(request):
if not request.param.should_test():
raise pytest.skip()
return request.param
def pytest_generate_tests(metafunc):
if "remote" in metafunc.fixturenames:
metafunc.parametrize("remote", remote_params, indirect=True)
def run_dvc(*argv):
assert main(argv) == 0
def test_get_url(repo_dir, dvc_repo, remote):
remote_url = remote.get_url()
run_dvc("remote", "add", "-d", "upstream", remote_url)
dvc_repo.add(repo_dir.FOO)
assert api.get_url(repo_dir.FOO) == "%s/%s" % (
remote_url,
"ac/bd18db4cc2f85cedef654fccc4a4d8",
)
def test_open(repo_dir, dvc_repo, remote):
run_dvc("remote", "add", "-d", "upstream", remote.get_url())
dvc_repo.add(repo_dir.FOO)
run_dvc("push")
# Remove cache to force download
shutil.rmtree(dvc_repo.cache.local.cache_dir)
with api.open(repo_dir.FOO) as fd:
assert fd.read() == repo_dir.FOO_CONTENTS
+9 -9
View File
@@ -164,7 +164,7 @@ def get_local_url():
def get_ssh_url():
return "ssh://{}@127.0.0.1:{}".format(
return "ssh://{}@127.0.0.1:22{}".format(
getpass.getuser(), get_local_storagepath()
)
@@ -236,7 +236,7 @@ def get_oss_url():
class TestDataCloud(TestDvc):
def _test_cloud(self, config, cl):
cloud = DataCloud(self.dvc, config=config)
self.assertIsInstance(cloud._cloud, cl)
self.assertIsInstance(cloud.get_remote(), cl)
def test(self):
config = copy.deepcopy(TEST_CONFIG)
@@ -287,7 +287,7 @@ class TestDataCloudBase(TestDvc):
config[TEST_SECTION][Config.SECTION_REMOTE_KEY_FILE] = keyfile
self.cloud = DataCloud(self.dvc, config)
self.assertIsInstance(self.cloud._cloud, self._get_cloud_class())
self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
def _test_cloud(self):
self._setup_cloud()
@@ -396,7 +396,7 @@ class TestRemoteGS(TestDataCloudBase):
] = TEST_GCP_CREDS_FILE
self.cloud = DataCloud(self.dvc, config)
self.assertIsInstance(self.cloud._cloud, self._get_cloud_class())
self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
def _get_url(self):
return get_gcp_url()
@@ -760,9 +760,8 @@ class TestRecursiveSyncOperations(TestDataCloudBase):
return RemoteLOCAL
def _prepare_repo(self):
self.main(
["remote", "add", "-d", TEST_REMOTE, self.cloud._cloud.cache_dir]
)
remote = self.cloud.get_remote()
self.main(["remote", "add", "-d", TEST_REMOTE, remote.cache_dir])
self.dvc.add(self.DATA)
self.dvc.add(self.DATA_SUB)
@@ -801,8 +800,9 @@ class TestRecursiveSyncOperations(TestDataCloudBase):
self.assertTrue(os.path.exists(local_cache_data_sub_path))
def _test_recursive_push(self, data_md5, data_sub_md5):
cloud_data_path = self.cloud._cloud.get(data_md5)
cloud_data_sub_path = self.cloud._cloud.get(data_sub_md5)
remote = self.cloud.get_remote()
cloud_data_path = remote.get(data_md5)
cloud_data_sub_path = remote.get(data_sub_md5)
self.assertFalse(os.path.exists(cloud_data_path))
self.assertFalse(os.path.exists(cloud_data_sub_path))