mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 23:11:44 +01:00
dvc: add get_url()/open() API
As a part of this: - DataCloud._cloud was refactored into `.get_remote()` - data stopped using internal _cloud - added a parametrization for remote tests
This commit is contained in:
@@ -0,0 +1,93 @@
|
||||
import pytest
|
||||
import shutil
|
||||
|
||||
from dvc import api
|
||||
from dvc.main import main
|
||||
from .test_data_cloud import (
|
||||
_should_test_aws,
|
||||
get_aws_url,
|
||||
_should_test_gcp,
|
||||
get_gcp_url,
|
||||
_should_test_azure,
|
||||
get_azure_url,
|
||||
_should_test_oss,
|
||||
get_oss_url,
|
||||
_should_test_ssh,
|
||||
get_ssh_url,
|
||||
_should_test_hdfs,
|
||||
get_hdfs_url,
|
||||
)
|
||||
|
||||
|
||||
# NOTE: staticmethod is only needed in Python 2
|
||||
class S3:
|
||||
should_test = staticmethod(_should_test_aws)
|
||||
get_url = staticmethod(get_aws_url)
|
||||
|
||||
|
||||
class GCP:
|
||||
should_test = staticmethod(_should_test_gcp)
|
||||
get_url = staticmethod(get_gcp_url)
|
||||
|
||||
|
||||
class Azure:
|
||||
should_test = staticmethod(_should_test_azure)
|
||||
get_url = staticmethod(get_azure_url)
|
||||
|
||||
|
||||
class OSS:
|
||||
should_test = staticmethod(_should_test_oss)
|
||||
get_url = staticmethod(get_oss_url)
|
||||
|
||||
|
||||
class SSH:
|
||||
should_test = staticmethod(_should_test_ssh)
|
||||
get_url = staticmethod(get_ssh_url)
|
||||
|
||||
|
||||
class HDFS:
|
||||
should_test = staticmethod(_should_test_hdfs)
|
||||
get_url = staticmethod(get_hdfs_url)
|
||||
|
||||
|
||||
remote_params = [S3, GCP, Azure, OSS, SSH, HDFS]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def remote(request):
|
||||
if not request.param.should_test():
|
||||
raise pytest.skip()
|
||||
return request.param
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
if "remote" in metafunc.fixturenames:
|
||||
metafunc.parametrize("remote", remote_params, indirect=True)
|
||||
|
||||
|
||||
def run_dvc(*argv):
|
||||
assert main(argv) == 0
|
||||
|
||||
|
||||
def test_get_url(repo_dir, dvc_repo, remote):
|
||||
remote_url = remote.get_url()
|
||||
|
||||
run_dvc("remote", "add", "-d", "upstream", remote_url)
|
||||
dvc_repo.add(repo_dir.FOO)
|
||||
|
||||
assert api.get_url(repo_dir.FOO) == "%s/%s" % (
|
||||
remote_url,
|
||||
"ac/bd18db4cc2f85cedef654fccc4a4d8",
|
||||
)
|
||||
|
||||
|
||||
def test_open(repo_dir, dvc_repo, remote):
|
||||
run_dvc("remote", "add", "-d", "upstream", remote.get_url())
|
||||
dvc_repo.add(repo_dir.FOO)
|
||||
run_dvc("push")
|
||||
|
||||
# Remove cache to force download
|
||||
shutil.rmtree(dvc_repo.cache.local.cache_dir)
|
||||
|
||||
with api.open(repo_dir.FOO) as fd:
|
||||
assert fd.read() == repo_dir.FOO_CONTENTS
|
||||
@@ -164,7 +164,7 @@ def get_local_url():
|
||||
|
||||
|
||||
def get_ssh_url():
|
||||
return "ssh://{}@127.0.0.1:{}".format(
|
||||
return "ssh://{}@127.0.0.1:22{}".format(
|
||||
getpass.getuser(), get_local_storagepath()
|
||||
)
|
||||
|
||||
@@ -236,7 +236,7 @@ def get_oss_url():
|
||||
class TestDataCloud(TestDvc):
|
||||
def _test_cloud(self, config, cl):
|
||||
cloud = DataCloud(self.dvc, config=config)
|
||||
self.assertIsInstance(cloud._cloud, cl)
|
||||
self.assertIsInstance(cloud.get_remote(), cl)
|
||||
|
||||
def test(self):
|
||||
config = copy.deepcopy(TEST_CONFIG)
|
||||
@@ -287,7 +287,7 @@ class TestDataCloudBase(TestDvc):
|
||||
config[TEST_SECTION][Config.SECTION_REMOTE_KEY_FILE] = keyfile
|
||||
self.cloud = DataCloud(self.dvc, config)
|
||||
|
||||
self.assertIsInstance(self.cloud._cloud, self._get_cloud_class())
|
||||
self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
|
||||
|
||||
def _test_cloud(self):
|
||||
self._setup_cloud()
|
||||
@@ -396,7 +396,7 @@ class TestRemoteGS(TestDataCloudBase):
|
||||
] = TEST_GCP_CREDS_FILE
|
||||
self.cloud = DataCloud(self.dvc, config)
|
||||
|
||||
self.assertIsInstance(self.cloud._cloud, self._get_cloud_class())
|
||||
self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
|
||||
|
||||
def _get_url(self):
|
||||
return get_gcp_url()
|
||||
@@ -760,9 +760,8 @@ class TestRecursiveSyncOperations(TestDataCloudBase):
|
||||
return RemoteLOCAL
|
||||
|
||||
def _prepare_repo(self):
|
||||
self.main(
|
||||
["remote", "add", "-d", TEST_REMOTE, self.cloud._cloud.cache_dir]
|
||||
)
|
||||
remote = self.cloud.get_remote()
|
||||
self.main(["remote", "add", "-d", TEST_REMOTE, remote.cache_dir])
|
||||
|
||||
self.dvc.add(self.DATA)
|
||||
self.dvc.add(self.DATA_SUB)
|
||||
@@ -801,8 +800,9 @@ class TestRecursiveSyncOperations(TestDataCloudBase):
|
||||
self.assertTrue(os.path.exists(local_cache_data_sub_path))
|
||||
|
||||
def _test_recursive_push(self, data_md5, data_sub_md5):
|
||||
cloud_data_path = self.cloud._cloud.get(data_md5)
|
||||
cloud_data_sub_path = self.cloud._cloud.get(data_sub_md5)
|
||||
remote = self.cloud.get_remote()
|
||||
cloud_data_path = remote.get(data_md5)
|
||||
cloud_data_sub_path = remote.get(data_sub_md5)
|
||||
|
||||
self.assertFalse(os.path.exists(cloud_data_path))
|
||||
self.assertFalse(os.path.exists(cloud_data_sub_path))
|
||||
|
||||
Reference in New Issue
Block a user