mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
Batuhan Taskaya
11b0581536
* import-url: support importing empty directories on object storage services Empty directories are basically files with a trailing slash at the end of their names, with empty contents on object service providers (such as azure, google storage). This patch implements the `is_empty()` method for the `FSSpecWrapper` to determining whether a path is empty (directory or a file) and also tweaks `.isdir()` in order to see empty directories. The base class is only used by azure and gcsfs for now, but if we go on the route of migrating normal providers (like proper filesystems, not just object storages), we might need to do these tweaks on a subclass (like `ObjectStorageFSSpecWrapper` etc), but I don't think it is neccesary as of now. The `is_empty` method is already there, and also used in the importing part to warn users that they are importing something empty (https://github.com/iterative/dvc/blob/e385e59a098cd661ea1ea4257c9a5735c5c32bc6/dvc/output/base.py#L252-L254) so this is not a new method. Just for the reference, as far as I can locate there is no variant of this method in the fsspec. * tests: revert custom mkdir for cloud storages * tests: allign assingments * add some comments
106 lines
2.5 KiB
Python
106 lines
2.5 KiB
Python
import locale
|
|
import os
|
|
import uuid
|
|
|
|
import pytest
|
|
from funcy import cached_property
|
|
|
|
from dvc.path_info import CloudURLInfo
|
|
from dvc.utils import env2bool
|
|
|
|
from .base import Base
|
|
|
|
TEST_GCP_REPO_BUCKET = os.environ.get("DVC_TEST_GCP_REPO_BUCKET", "dvc-test")
|
|
|
|
TEST_GCP_CREDS_FILE = os.path.abspath(
|
|
os.environ.get(
|
|
"GOOGLE_APPLICATION_CREDENTIALS",
|
|
os.path.join("scripts", "ci", "gcp-creds.json"),
|
|
)
|
|
)
|
|
# Ensure that absolute path is used
|
|
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = TEST_GCP_CREDS_FILE
|
|
|
|
|
|
class GCP(Base, CloudURLInfo):
|
|
|
|
IS_OBJECT_STORAGE = True
|
|
|
|
@staticmethod
|
|
def should_test():
|
|
do_test = env2bool("DVC_TEST_GCP", undefined=None)
|
|
if do_test is not None:
|
|
return do_test
|
|
|
|
if not os.path.exists(TEST_GCP_CREDS_FILE):
|
|
return False
|
|
|
|
return True
|
|
|
|
@cached_property
|
|
def config(self):
|
|
return {
|
|
"url": self.url,
|
|
"credentialpath": TEST_GCP_CREDS_FILE,
|
|
}
|
|
|
|
@staticmethod
|
|
def _get_storagepath():
|
|
return TEST_GCP_REPO_BUCKET + "/" + str(uuid.uuid4())
|
|
|
|
@staticmethod
|
|
def get_url():
|
|
return "gs://" + GCP._get_storagepath()
|
|
|
|
@property
|
|
def _gc(self):
|
|
from google.cloud.storage import Client
|
|
|
|
return Client.from_service_account_json(TEST_GCP_CREDS_FILE)
|
|
|
|
@property
|
|
def _bucket(self):
|
|
return self._gc.bucket(self.bucket)
|
|
|
|
@property
|
|
def _blob(self):
|
|
return self._bucket.blob(self.path)
|
|
|
|
def is_file(self):
|
|
if self.path.endswith("/"):
|
|
return False
|
|
|
|
return self._blob.exists()
|
|
|
|
def is_dir(self):
|
|
dir_path = self / ""
|
|
return bool(list(self._bucket.list_blobs(prefix=dir_path.path)))
|
|
|
|
def exists(self):
|
|
return self.is_file() or self.is_dir()
|
|
|
|
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
|
|
assert mode == 0o777
|
|
assert parents
|
|
|
|
def write_bytes(self, contents):
|
|
assert isinstance(contents, bytes)
|
|
self._blob.upload_from_string(contents)
|
|
|
|
def read_bytes(self):
|
|
return self._blob.download_as_string()
|
|
|
|
def read_text(self, encoding=None, errors=None):
|
|
if not encoding:
|
|
encoding = locale.getpreferredencoding(False)
|
|
assert errors is None
|
|
return self.read_bytes().decode(encoding)
|
|
|
|
|
|
@pytest.fixture
|
|
def gs(test_config):
|
|
test_config.requires("gs")
|
|
if not GCP.should_test():
|
|
pytest.skip("no gs")
|
|
yield GCP(GCP.get_url())
|