tree: stop using self.path_info (#5403)

* webhdfs: don't use self.path_info

* azure: don't use self.path_info

* webdav: don't use self.path_info

* oss: don't use self.path_info

* http: don't use self.path_info

* gdrive: don't use self.path_info
This commit is contained in:
Ruslan Kuprieiev
2021-02-03 20:50:30 +02:00
committed by GitHub
parent 2c81bf1748
commit b87e76aa84
6 changed files with 35 additions and 45 deletions
+5 -7
View File
@@ -31,10 +31,12 @@ class AzureTree(BaseTree):
url = config.get("url", "azure://")
self.path_info = self.PATH_CLS(url)
self.bucket = self.path_info.bucket
if not self.path_info.bucket:
if not self.bucket:
container = self._az_config.get("storage", "container_name", None)
self.path_info = self.PATH_CLS(f"azure://{container}")
self.bucket = self.path_info.bucket
self._conn_str = config.get(
"connection_string"
@@ -73,8 +75,6 @@ class AzureTree(BaseTree):
)
from azure.storage.blob import BlobServiceClient
logger.debug(f"URL {self.path_info}")
if self._conn_str:
logger.debug(f"Using connection string '{self._conn_str}'")
blob_service = BlobServiceClient.from_connection_string(
@@ -86,10 +86,8 @@ class AzureTree(BaseTree):
self._account_url, credential=self._credential
)
logger.debug(f"Container name {self.path_info.bucket}")
container_client = blob_service.get_container_client(
self.path_info.bucket
)
logger.debug(f"Container name {self.bucket}")
container_client = blob_service.get_container_client(self.bucket)
try: # verify that container exists
container_client.get_container_properties()
+2 -1
View File
@@ -113,6 +113,7 @@ class GDriveTree(BaseTree):
)
self._bucket = self.path_info.bucket
self._path = self.path_info.path
self._trash_only = config.get("gdrive_trash_only")
self._use_service_account = config.get("gdrive_use_service_account")
self._service_account_email = config.get(
@@ -284,7 +285,7 @@ class GDriveTree(BaseTree):
),
}
self._cache_path_id(self.path_info.path, cache["root_id"], cache)
self._cache_path_id(self._path, cache["root_id"], cache)
for item in self._gdrive_list(
"'{}' in parents and trashed=false".format(cache["root_id"])
+6 -11
View File
@@ -42,9 +42,8 @@ class HTTPTree(BaseTree): # pylint:disable=abstract-method
url = config.get("url")
if url:
self.path_info = self.PATH_CLS(url)
user = config.get("user", None)
if user:
self.path_info.user = user
self.user = config.get("user", None)
self.host = self.path_info.host
else:
self.path_info = None
@@ -56,20 +55,16 @@ class HTTPTree(BaseTree): # pylint:disable=abstract-method
self.ssl_verify = config.get("ssl_verify", True)
self.method = config.get("method", "POST")
def _auth_method(self, path_info=None):
def _auth_method(self):
from requests.auth import HTTPBasicAuth, HTTPDigestAuth
if path_info is None:
path_info = self.path_info
if self.auth:
if self.ask_password and self.password is None:
host, user = path_info.host, path_info.user
self.password = ask_password(host, user)
self.password = ask_password(self.host, self.user)
if self.auth == "basic":
return HTTPBasicAuth(path_info.user, self.password)
return HTTPBasicAuth(self.user, self.password)
if self.auth == "digest":
return HTTPDigestAuth(path_info.user, self.password)
return HTTPDigestAuth(self.user, self.password)
if self.auth == "custom" and self.custom_auth_header:
self.headers.update({self.custom_auth_header: self.password})
return None
+3 -3
View File
@@ -45,6 +45,7 @@ class OSSTree(BaseTree): # pylint:disable=abstract-method
url = config.get("url")
self.path_info = self.PATH_CLS(url) if url else None
self.bucket = self.path_info.bucket
self.endpoint = config.get("oss_endpoint") or os.getenv("OSS_ENDPOINT")
self.key_id = (
@@ -64,12 +65,11 @@ class OSSTree(BaseTree): # pylint:disable=abstract-method
def oss_service(self):
import oss2
logger.debug(f"URL: {self.path_info}")
logger.debug(f"key id: {self.key_id}")
logger.debug(f"key secret: {self.key_secret}")
auth = oss2.Auth(self.key_id, self.key_secret)
bucket = oss2.Bucket(auth, self.endpoint, self.path_info.bucket)
bucket = oss2.Bucket(auth, self.endpoint, self.bucket)
# Ensure bucket exists
try:
@@ -84,7 +84,7 @@ class OSSTree(BaseTree): # pylint:disable=abstract-method
return bucket
def _generate_download_url(self, path_info, expires=3600):
assert path_info.bucket == self.path_info.bucket
assert path_info.bucket == self.bucket
return self.oss_service.sign_url("GET", path_info.path, expires)
+10 -11
View File
@@ -82,24 +82,23 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method
if self.user is None and self.path_info.user is not None:
self.user = self.path_info.user
# Construct hostname from path_info by stripping path
http_info = HTTPURLInfo(self.path_info.url)
self.hostname = http_info.replace(path="").url
# Webdav client
@wrap_prop(threading.Lock())
@cached_property
def _client(self):
from webdav3.client import Client
# Construct hostname from path_info by stripping path
http_info = HTTPURLInfo(self.path_info.url)
hostname = http_info.replace(path="").url
# Set password or ask for it
if self.ask_password and self.password is None and self.token is None:
host, user = self.path_info.host, self.path_info.user
self.password = ask_password(host, user)
self.password = ask_password(self.hostname, self.user)
# Setup webdav client options dictionary
options = {
"webdav_hostname": hostname,
"webdav_hostname": self.hostname,
"webdav_login": self.user,
"webdav_password": self.password,
"webdav_token": self.token,
@@ -114,12 +113,12 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method
# Check whether client options are valid
if not client.valid():
raise ConfigError(
f"Configuration for WebDAV {hostname} is invalid."
f"Configuration for WebDAV {self.hostname} is invalid."
)
# Check whether connection is valid (root should always exist)
if not client.check(self.path_info.path):
raise WebDAVConnectionError(hostname)
if not client.check("/"):
raise WebDAVConnectionError(self.hostname)
return client
@@ -198,7 +197,7 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method
# Creates directories
def makedirs(self, path_info):
# Terminate recursion
if path_info.path == self.path_info.path or self.exists(path_info):
if path_info.path == "/" or self.exists(path_info):
return
# Recursively descent to root
+9 -12
View File
@@ -45,16 +45,13 @@ class WebHDFSTree(BaseTree):
if not url:
return
parsed = urlparse(url)
user = parsed.username or config.get("user")
self.path_info = self.PATH_CLS(url)
self.path_info = self.PATH_CLS.from_parts(
scheme="webhdfs",
host=parsed.hostname,
user=user,
port=parsed.port,
path=parsed.path,
)
parsed = urlparse(url)
self.host = parsed.hostname
self.user = parsed.username or config.get("user")
self.port = parsed.port
self.hdfscli_config = config.get("hdfscli_config")
self.token = config.get("webhdfs_token")
@@ -65,7 +62,6 @@ class WebHDFSTree(BaseTree):
def hdfs_client(self):
import hdfs
logger.debug("URL: %s", self.path_info)
logger.debug("HDFSConfig: %s", self.hdfscli_config)
try:
@@ -82,13 +78,14 @@ class WebHDFSTree(BaseTree):
if not any(err in exc_msg for err in errors):
raise
http_url = f"http://{self.path_info.host}:{self.path_info.port}"
http_url = f"http://{self.host}:{self.port}"
logger.debug("URL: %s", http_url)
if self.token is not None:
client = hdfs.TokenClient(http_url, token=self.token, root="/")
else:
client = hdfs.InsecureClient(
http_url, user=self.path_info.user, root="/"
http_url, user=self.user, root="/"
)
return client