Files
dvc/dvc/tree/webhdfs.py
T
Batuhan TaskayaandRuslan Kuprieiev a5e19ecd6a tree: Implement upload_fobj protocol to all trees (#5307)
* tree: Implement upload_fobj protocol to azure

* tree: Implement upload_fobj protocol to google storage

* tree: Implement upload_fobj protocol to google drive

* tree: Implement upload_fobj protocol to hdfs

* tree: Implement upload_fobj protocol to oss (alibaba cloud)

* tree: Implement upload_fobj protocol to webhdfs (also some cleanup)

* tree: Implement upload_fobj protocol to http/https

* Add _generate_download_url() to the HTTP (for tests)

* tree: Implement a temporary upload_fobj protocol to webdav/webdavs

* Unify progress bars

* Lazily evaluate the file size, for http uploads

* Remove copy_fobj protocol and _transfer_file_as_whole

* show progress bar on cache transfer

* Simplify s3 upload

* Remove CHUNK_SIZE (except for some special cases)

* remove concurrency restrictions

* Update dvc/tree/oss.py

Co-authored-by: Ruslan Kuprieiev <[email protected]>

Co-authored-by: Ruslan Kuprieiev <[email protected]>
2021-01-26 13:57:51 +02:00

174 lines
5.1 KiB
Python

import logging
import os
import posixpath
import shutil
import threading
from contextlib import contextmanager
from urllib.parse import urlparse
from funcy import cached_property, wrap_prop
from dvc.hash_info import HashInfo
from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.scheme import Schemes
from .base import BaseTree
logger = logging.getLogger(__name__)
def update_pbar(pbar, total):
"""Update pbar to accept the two arguments passed by hdfs"""
def update(_, bytes_transfered):
if bytes_transfered == -1:
pbar.update_to(total)
return
pbar.update_to(bytes_transfered)
return update
class WebHDFSTree(BaseTree):
scheme = Schemes.WEBHDFS
PATH_CLS = CloudURLInfo
REQUIRES = {"hdfs": "hdfs"}
PARAM_CHECKSUM = "checksum"
TRAVERSE_PREFIX_LEN = 2
def __init__(self, repo, config):
super().__init__(repo, config)
self.path_info = None
url = config.get("url")
if not url:
return
parsed = urlparse(url)
user = parsed.username or config.get("user")
self.path_info = self.PATH_CLS.from_parts(
scheme="webhdfs",
host=parsed.hostname,
user=user,
port=parsed.port,
path=parsed.path,
)
self.hdfscli_config = config.get("hdfscli_config")
self.token = config.get("webhdfs_token")
self.alias = config.get("webhdfs_alias")
@wrap_prop(threading.Lock())
@cached_property
def hdfs_client(self):
import hdfs
logger.debug("URL: %s", self.path_info)
logger.debug("HDFSConfig: %s", self.hdfscli_config)
try:
return hdfs.config.Config(self.hdfscli_config).get_client(
self.alias
)
except hdfs.util.HdfsError as exc:
exc_msg = str(exc)
errors = (
"No alias specified",
"Invalid configuration file",
f"Alias {self.alias} not found",
)
if not any(err in exc_msg for err in errors):
raise
http_url = f"http://{self.path_info.host}:{self.path_info.port}"
if self.token is not None:
client = hdfs.TokenClient(http_url, token=self.token, root="/")
else:
client = hdfs.InsecureClient(
http_url, user=self.path_info.user, root="/"
)
return client
@contextmanager
def open(self, path_info, mode="r", encoding=None, **kwargs):
assert mode in {"r", "rt", "rb"}
with self.hdfs_client.read(
path_info.path, encoding=encoding
) as reader:
yield reader
def walk_files(self, path_info, **kwargs):
if not self.exists(path_info):
return
root = path_info.path
for path, _, fnames in self.hdfs_client.walk(root):
for fname in fnames:
yield path_info.replace(path=posixpath.join(path, fname))
def remove(self, path_info):
if path_info.scheme != self.scheme:
raise NotImplementedError
self.hdfs_client.delete(path_info.path)
def exists(self, path_info, use_dvcignore=True):
assert not isinstance(path_info, list)
assert path_info.scheme == "webhdfs"
status = self.hdfs_client.status(path_info.path, strict=False)
return status is not None
def getsize(self, path_info):
return self.hdfs_client.status(path_info.path)["length"]
def get_file_hash(self, path_info):
checksum = self.hdfs_client.checksum(path_info.path)
hash_info = HashInfo(self.PARAM_CHECKSUM, checksum["bytes"])
hash_info.size = self.hdfs_client.status(path_info.path)["length"]
return hash_info
def copy(self, from_info, to_info, **_kwargs):
with self.hdfs_client.read(from_info.path) as reader:
with self.hdfs_client.write(to_info.path) as writer:
shutil.copyfileobj(reader, writer)
def move(self, from_info, to_info):
self.hdfs_client.makedirs(to_info.parent.path)
self.hdfs_client.rename(from_info.path, to_info.path)
def _upload_fobj(self, fobj, to_info):
with self.hdfs_client.write(to_info.path) as fdest:
shutil.copyfileobj(fobj, fdest)
def _upload(
self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
):
total = os.path.getsize(from_file)
with Tqdm(
desc=name, total=total, disable=no_progress_bar, bytes=True
) as pbar:
self.hdfs_client.upload(
to_info.path,
from_file,
overwrite=True,
progress=update_pbar(pbar, total),
)
def _download(
self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
):
total = self.getsize(from_info)
with Tqdm(
desc=name, total=total, disable=no_progress_bar, bytes=True
) as pbar:
self.hdfs_client.download(
from_info.path, to_file, progress=update_pbar(pbar, total)
)