mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-04 15:28:27 +01:00
Batuhan Taskaya
11b0581536
* import-url: support importing empty directories on object storage services Empty directories are basically files with a trailing slash at the end of their names, with empty contents on object service providers (such as azure, google storage). This patch implements the `is_empty()` method for the `FSSpecWrapper` to determining whether a path is empty (directory or a file) and also tweaks `.isdir()` in order to see empty directories. The base class is only used by azure and gcsfs for now, but if we go on the route of migrating normal providers (like proper filesystems, not just object storages), we might need to do these tweaks on a subclass (like `ObjectStorageFSSpecWrapper` etc), but I don't think it is neccesary as of now. The `is_empty` method is already there, and also used in the importing part to warn users that they are importing something empty (https://github.com/iterative/dvc/blob/e385e59a098cd661ea1ea4257c9a5735c5c32bc6/dvc/output/base.py#L252-L254) so this is not a new method. Just for the reference, as far as I can locate there is no variant of this method in the fsspec. * tests: revert custom mkdir for cloud storages * tests: allign assingments * add some comments
77 lines
1.9 KiB
Python
77 lines
1.9 KiB
Python
import locale
|
|
import pathlib
|
|
|
|
from funcy import cached_property
|
|
|
|
from dvc.path_info import URLInfo
|
|
|
|
|
|
class Base(URLInfo):
|
|
|
|
IS_OBJECT_STORAGE = False
|
|
|
|
def is_file(self):
|
|
raise NotImplementedError
|
|
|
|
def is_dir(self):
|
|
raise NotImplementedError
|
|
|
|
def exists(self):
|
|
raise NotImplementedError
|
|
|
|
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
|
|
raise NotImplementedError
|
|
|
|
def write_text(self, contents, encoding=None, errors=None):
|
|
if not encoding:
|
|
encoding = locale.getpreferredencoding(False)
|
|
assert errors is None
|
|
self.write_bytes(contents.encode(encoding))
|
|
|
|
def write_bytes(self, contents):
|
|
raise NotImplementedError
|
|
|
|
def read_text(self, encoding=None, errors=None):
|
|
raise NotImplementedError
|
|
|
|
def read_bytes(self):
|
|
raise NotImplementedError
|
|
|
|
def _gen(self, struct, prefix=None):
|
|
for name, contents in struct.items():
|
|
path = (prefix or self) / name
|
|
|
|
if isinstance(contents, dict):
|
|
if not contents:
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
else:
|
|
self._gen(contents, prefix=path)
|
|
else:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
if isinstance(contents, bytes):
|
|
path.write_bytes(contents)
|
|
else:
|
|
path.write_text(contents, encoding="utf-8")
|
|
|
|
def gen(self, struct, text=""):
|
|
if isinstance(struct, (str, bytes, pathlib.PurePath)):
|
|
struct = {struct: text}
|
|
|
|
self._gen(struct)
|
|
return struct.keys()
|
|
|
|
def close(self):
|
|
pass
|
|
|
|
@staticmethod
|
|
def should_test():
|
|
return True
|
|
|
|
@staticmethod
|
|
def get_url():
|
|
raise NotImplementedError
|
|
|
|
@cached_property
|
|
def config(self):
|
|
return {"url": self.url}
|