mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
Batuhan Taskaya
11b0581536
* import-url: support importing empty directories on object storage services Empty directories are basically files with a trailing slash at the end of their names, with empty contents on object service providers (such as azure, google storage). This patch implements the `is_empty()` method for the `FSSpecWrapper` to determining whether a path is empty (directory or a file) and also tweaks `.isdir()` in order to see empty directories. The base class is only used by azure and gcsfs for now, but if we go on the route of migrating normal providers (like proper filesystems, not just object storages), we might need to do these tweaks on a subclass (like `ObjectStorageFSSpecWrapper` etc), but I don't think it is neccesary as of now. The `is_empty` method is already there, and also used in the importing part to warn users that they are importing something empty (https://github.com/iterative/dvc/blob/e385e59a098cd661ea1ea4257c9a5735c5c32bc6/dvc/output/base.py#L252-L254) so this is not a new method. Just for the reference, as far as I can locate there is no variant of this method in the fsspec. * tests: revert custom mkdir for cloud storages * tests: allign assingments * add some comments
114 lines
2.6 KiB
Python
114 lines
2.6 KiB
Python
import locale
|
|
import os
|
|
import uuid
|
|
|
|
import pytest
|
|
from funcy import cached_property
|
|
from moto import mock_s3
|
|
|
|
from dvc.path_info import CloudURLInfo
|
|
from dvc.utils import env2bool
|
|
|
|
from .base import Base
|
|
|
|
TEST_AWS_REPO_BUCKET = os.environ.get("DVC_TEST_AWS_REPO_BUCKET", "dvc-temp")
|
|
|
|
|
|
class S3(Base, CloudURLInfo):
|
|
|
|
IS_OBJECT_STORAGE = True
|
|
|
|
@staticmethod
|
|
def should_test():
|
|
do_test = env2bool("DVC_TEST_AWS", undefined=None)
|
|
if do_test is not None:
|
|
return do_test
|
|
|
|
if os.getenv("AWS_ACCESS_KEY_ID") and os.getenv(
|
|
"AWS_SECRET_ACCESS_KEY"
|
|
):
|
|
return True
|
|
|
|
return False
|
|
|
|
@staticmethod
|
|
def _get_storagepath():
|
|
return (
|
|
TEST_AWS_REPO_BUCKET
|
|
+ "/"
|
|
+ "dvc_test_caches"
|
|
+ "/"
|
|
+ str(uuid.uuid4())
|
|
)
|
|
|
|
@staticmethod
|
|
def get_url():
|
|
return "s3://" + S3._get_storagepath()
|
|
|
|
@cached_property
|
|
def _s3(self):
|
|
import boto3
|
|
|
|
return boto3.client("s3")
|
|
|
|
def is_file(self):
|
|
from botocore.exceptions import ClientError
|
|
|
|
if self.path.endswith("/"):
|
|
return False
|
|
|
|
try:
|
|
self._s3.head_object(Bucket=self.bucket, Key=self.path)
|
|
except ClientError as exc:
|
|
if exc.response["Error"]["Code"] != "404":
|
|
raise
|
|
return False
|
|
|
|
return True
|
|
|
|
def is_dir(self):
|
|
path = (self / "").path
|
|
resp = self._s3.list_objects(Bucket=self.bucket, Prefix=path)
|
|
return bool(resp.get("Contents"))
|
|
|
|
def exists(self):
|
|
return self.is_file() or self.is_dir()
|
|
|
|
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
|
|
assert mode == 0o777
|
|
assert parents
|
|
|
|
def write_bytes(self, contents):
|
|
self._s3.put_object(
|
|
Bucket=self.bucket, Key=self.path, Body=contents,
|
|
)
|
|
|
|
def read_bytes(self):
|
|
data = self._s3.get_object(Bucket=self.bucket, Key=self.path)
|
|
return data["Body"].read()
|
|
|
|
def read_text(self, encoding=None, errors=None):
|
|
if not encoding:
|
|
encoding = locale.getpreferredencoding(False)
|
|
assert errors is None
|
|
return self.read_bytes().decode(encoding)
|
|
|
|
|
|
@pytest.fixture
|
|
def s3(test_config):
|
|
test_config.requires("s3")
|
|
with mock_s3():
|
|
import boto3
|
|
|
|
boto3.client("s3").create_bucket(Bucket=TEST_AWS_REPO_BUCKET)
|
|
|
|
yield S3(S3.get_url())
|
|
|
|
|
|
@pytest.fixture
|
|
def real_s3(test_config):
|
|
test_config.requires("s3")
|
|
if not S3.should_test():
|
|
pytest.skip("no real s3")
|
|
yield S3(S3.get_url())
|