From b87e76aa844926a15244066fac4032c66fba22d1 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 3 Feb 2021 20:50:30 +0200 Subject: [PATCH] tree: stop using self.path_info (#5403) * webhdfs: don't use self.path_info * azure: don't use self.path_info * webdav: don't use self.path_info * oss: don't use self.path_info * http: don't use self.path_info * gdrive: don't use self.path_info --- dvc/tree/azure.py | 12 +++++------- dvc/tree/gdrive.py | 3 ++- dvc/tree/http.py | 17 ++++++----------- dvc/tree/oss.py | 6 +++--- dvc/tree/webdav.py | 21 ++++++++++----------- dvc/tree/webhdfs.py | 21 +++++++++------------ 6 files changed, 35 insertions(+), 45 deletions(-) diff --git a/dvc/tree/azure.py b/dvc/tree/azure.py index b80ad74c3..de4f6bca6 100644 --- a/dvc/tree/azure.py +++ b/dvc/tree/azure.py @@ -31,10 +31,12 @@ class AzureTree(BaseTree): url = config.get("url", "azure://") self.path_info = self.PATH_CLS(url) + self.bucket = self.path_info.bucket - if not self.path_info.bucket: + if not self.bucket: container = self._az_config.get("storage", "container_name", None) self.path_info = self.PATH_CLS(f"azure://{container}") + self.bucket = self.path_info.bucket self._conn_str = config.get( "connection_string" @@ -73,8 +75,6 @@ class AzureTree(BaseTree): ) from azure.storage.blob import BlobServiceClient - logger.debug(f"URL {self.path_info}") - if self._conn_str: logger.debug(f"Using connection string '{self._conn_str}'") blob_service = BlobServiceClient.from_connection_string( @@ -86,10 +86,8 @@ class AzureTree(BaseTree): self._account_url, credential=self._credential ) - logger.debug(f"Container name {self.path_info.bucket}") - container_client = blob_service.get_container_client( - self.path_info.bucket - ) + logger.debug(f"Container name {self.bucket}") + container_client = blob_service.get_container_client(self.bucket) try: # verify that container exists container_client.get_container_properties() diff --git a/dvc/tree/gdrive.py b/dvc/tree/gdrive.py index 05d37db5d..fa70dffb9 100644 --- a/dvc/tree/gdrive.py +++ b/dvc/tree/gdrive.py @@ -113,6 +113,7 @@ class GDriveTree(BaseTree): ) self._bucket = self.path_info.bucket + self._path = self.path_info.path self._trash_only = config.get("gdrive_trash_only") self._use_service_account = config.get("gdrive_use_service_account") self._service_account_email = config.get( @@ -284,7 +285,7 @@ class GDriveTree(BaseTree): ), } - self._cache_path_id(self.path_info.path, cache["root_id"], cache) + self._cache_path_id(self._path, cache["root_id"], cache) for item in self._gdrive_list( "'{}' in parents and trashed=false".format(cache["root_id"]) diff --git a/dvc/tree/http.py b/dvc/tree/http.py index f3b762291..a9afb2ea9 100644 --- a/dvc/tree/http.py +++ b/dvc/tree/http.py @@ -42,9 +42,8 @@ class HTTPTree(BaseTree): # pylint:disable=abstract-method url = config.get("url") if url: self.path_info = self.PATH_CLS(url) - user = config.get("user", None) - if user: - self.path_info.user = user + self.user = config.get("user", None) + self.host = self.path_info.host else: self.path_info = None @@ -56,20 +55,16 @@ class HTTPTree(BaseTree): # pylint:disable=abstract-method self.ssl_verify = config.get("ssl_verify", True) self.method = config.get("method", "POST") - def _auth_method(self, path_info=None): + def _auth_method(self): from requests.auth import HTTPBasicAuth, HTTPDigestAuth - if path_info is None: - path_info = self.path_info - if self.auth: if self.ask_password and self.password is None: - host, user = path_info.host, path_info.user - self.password = ask_password(host, user) + self.password = ask_password(self.host, self.user) if self.auth == "basic": - return HTTPBasicAuth(path_info.user, self.password) + return HTTPBasicAuth(self.user, self.password) if self.auth == "digest": - return HTTPDigestAuth(path_info.user, self.password) + return HTTPDigestAuth(self.user, self.password) if self.auth == "custom" and self.custom_auth_header: self.headers.update({self.custom_auth_header: self.password}) return None diff --git a/dvc/tree/oss.py b/dvc/tree/oss.py index 302e4447b..619333edc 100644 --- a/dvc/tree/oss.py +++ b/dvc/tree/oss.py @@ -45,6 +45,7 @@ class OSSTree(BaseTree): # pylint:disable=abstract-method url = config.get("url") self.path_info = self.PATH_CLS(url) if url else None + self.bucket = self.path_info.bucket self.endpoint = config.get("oss_endpoint") or os.getenv("OSS_ENDPOINT") self.key_id = ( @@ -64,12 +65,11 @@ class OSSTree(BaseTree): # pylint:disable=abstract-method def oss_service(self): import oss2 - logger.debug(f"URL: {self.path_info}") logger.debug(f"key id: {self.key_id}") logger.debug(f"key secret: {self.key_secret}") auth = oss2.Auth(self.key_id, self.key_secret) - bucket = oss2.Bucket(auth, self.endpoint, self.path_info.bucket) + bucket = oss2.Bucket(auth, self.endpoint, self.bucket) # Ensure bucket exists try: @@ -84,7 +84,7 @@ class OSSTree(BaseTree): # pylint:disable=abstract-method return bucket def _generate_download_url(self, path_info, expires=3600): - assert path_info.bucket == self.path_info.bucket + assert path_info.bucket == self.bucket return self.oss_service.sign_url("GET", path_info.path, expires) diff --git a/dvc/tree/webdav.py b/dvc/tree/webdav.py index 7becd5bc9..4c149d7e8 100644 --- a/dvc/tree/webdav.py +++ b/dvc/tree/webdav.py @@ -82,24 +82,23 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method if self.user is None and self.path_info.user is not None: self.user = self.path_info.user + # Construct hostname from path_info by stripping path + http_info = HTTPURLInfo(self.path_info.url) + self.hostname = http_info.replace(path="").url + # Webdav client @wrap_prop(threading.Lock()) @cached_property def _client(self): from webdav3.client import Client - # Construct hostname from path_info by stripping path - http_info = HTTPURLInfo(self.path_info.url) - hostname = http_info.replace(path="").url - # Set password or ask for it if self.ask_password and self.password is None and self.token is None: - host, user = self.path_info.host, self.path_info.user - self.password = ask_password(host, user) + self.password = ask_password(self.hostname, self.user) # Setup webdav client options dictionary options = { - "webdav_hostname": hostname, + "webdav_hostname": self.hostname, "webdav_login": self.user, "webdav_password": self.password, "webdav_token": self.token, @@ -114,12 +113,12 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method # Check whether client options are valid if not client.valid(): raise ConfigError( - f"Configuration for WebDAV {hostname} is invalid." + f"Configuration for WebDAV {self.hostname} is invalid." ) # Check whether connection is valid (root should always exist) - if not client.check(self.path_info.path): - raise WebDAVConnectionError(hostname) + if not client.check("/"): + raise WebDAVConnectionError(self.hostname) return client @@ -198,7 +197,7 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method # Creates directories def makedirs(self, path_info): # Terminate recursion - if path_info.path == self.path_info.path or self.exists(path_info): + if path_info.path == "/" or self.exists(path_info): return # Recursively descent to root diff --git a/dvc/tree/webhdfs.py b/dvc/tree/webhdfs.py index cfb651309..64b391f20 100644 --- a/dvc/tree/webhdfs.py +++ b/dvc/tree/webhdfs.py @@ -45,16 +45,13 @@ class WebHDFSTree(BaseTree): if not url: return - parsed = urlparse(url) - user = parsed.username or config.get("user") + self.path_info = self.PATH_CLS(url) - self.path_info = self.PATH_CLS.from_parts( - scheme="webhdfs", - host=parsed.hostname, - user=user, - port=parsed.port, - path=parsed.path, - ) + parsed = urlparse(url) + + self.host = parsed.hostname + self.user = parsed.username or config.get("user") + self.port = parsed.port self.hdfscli_config = config.get("hdfscli_config") self.token = config.get("webhdfs_token") @@ -65,7 +62,6 @@ class WebHDFSTree(BaseTree): def hdfs_client(self): import hdfs - logger.debug("URL: %s", self.path_info) logger.debug("HDFSConfig: %s", self.hdfscli_config) try: @@ -82,13 +78,14 @@ class WebHDFSTree(BaseTree): if not any(err in exc_msg for err in errors): raise - http_url = f"http://{self.path_info.host}:{self.path_info.port}" + http_url = f"http://{self.host}:{self.port}" + logger.debug("URL: %s", http_url) if self.token is not None: client = hdfs.TokenClient(http_url, token=self.token, root="/") else: client = hdfs.InsecureClient( - http_url, user=self.path_info.user, root="/" + http_url, user=self.user, root="/" ) return client