mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-10 15:28:34 +01:00
64 lines
1.8 KiB
Python
64 lines
1.8 KiB
Python
import locale
|
|
import uuid
|
|
from contextlib import contextmanager
|
|
|
|
import pytest
|
|
|
|
from dvc.path_info import CloudURLInfo
|
|
|
|
from .base import Base
|
|
|
|
|
|
class WebHDFS(Base, CloudURLInfo): # pylint: disable=abstract-method
|
|
@contextmanager
|
|
def _webhdfs(self):
|
|
from hdfs import InsecureClient
|
|
|
|
client = InsecureClient(
|
|
f"http://{self.host}:{self.port}", self.user, root="/"
|
|
)
|
|
yield client
|
|
|
|
def is_file(self):
|
|
with self._webhdfs() as _hdfs:
|
|
return _hdfs.status(self.path)["type"] == "FILE"
|
|
|
|
def is_dir(self):
|
|
with self._webhdfs() as _hdfs:
|
|
return _hdfs.status(self.path)["type"] == "DIRECTORY"
|
|
|
|
def exists(self):
|
|
with self._webhdfs() as _hdfs:
|
|
return _hdfs.status(self.path, strict=False) is not None
|
|
|
|
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
|
|
assert mode == 0o777
|
|
assert parents
|
|
|
|
with self._webhdfs() as _hdfs:
|
|
# NOTE: hdfs.makekdirs always creates parents
|
|
_hdfs.makedirs(self.path, permission=mode)
|
|
|
|
def write_bytes(self, contents):
|
|
with self._webhdfs() as _hdfs:
|
|
with _hdfs.write(self.path, overwrite=True) as writer:
|
|
writer.write(contents)
|
|
|
|
def read_bytes(self):
|
|
with self._webhdfs() as _hdfs:
|
|
with _hdfs.read(self.path) as reader:
|
|
return reader.read()
|
|
|
|
def read_text(self, encoding=None, errors=None):
|
|
if not encoding:
|
|
encoding = locale.getpreferredencoding(False)
|
|
assert errors is None
|
|
return self.read_bytes().decode(encoding)
|
|
|
|
|
|
@pytest.fixture
|
|
def webhdfs(hdfs_server):
|
|
port = hdfs_server["webhdfs"]
|
|
url = f"webhdfs://127.0.0.1:{port}/{uuid.uuid4()}"
|
|
yield WebHDFS(url)
|