mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
tree: stop using self.path_info (#5403)
* webhdfs: don't use self.path_info * azure: don't use self.path_info * webdav: don't use self.path_info * oss: don't use self.path_info * http: don't use self.path_info * gdrive: don't use self.path_info
This commit is contained in:
+5
-7
@@ -31,10 +31,12 @@ class AzureTree(BaseTree):
|
||||
|
||||
url = config.get("url", "azure://")
|
||||
self.path_info = self.PATH_CLS(url)
|
||||
self.bucket = self.path_info.bucket
|
||||
|
||||
if not self.path_info.bucket:
|
||||
if not self.bucket:
|
||||
container = self._az_config.get("storage", "container_name", None)
|
||||
self.path_info = self.PATH_CLS(f"azure://{container}")
|
||||
self.bucket = self.path_info.bucket
|
||||
|
||||
self._conn_str = config.get(
|
||||
"connection_string"
|
||||
@@ -73,8 +75,6 @@ class AzureTree(BaseTree):
|
||||
)
|
||||
from azure.storage.blob import BlobServiceClient
|
||||
|
||||
logger.debug(f"URL {self.path_info}")
|
||||
|
||||
if self._conn_str:
|
||||
logger.debug(f"Using connection string '{self._conn_str}'")
|
||||
blob_service = BlobServiceClient.from_connection_string(
|
||||
@@ -86,10 +86,8 @@ class AzureTree(BaseTree):
|
||||
self._account_url, credential=self._credential
|
||||
)
|
||||
|
||||
logger.debug(f"Container name {self.path_info.bucket}")
|
||||
container_client = blob_service.get_container_client(
|
||||
self.path_info.bucket
|
||||
)
|
||||
logger.debug(f"Container name {self.bucket}")
|
||||
container_client = blob_service.get_container_client(self.bucket)
|
||||
|
||||
try: # verify that container exists
|
||||
container_client.get_container_properties()
|
||||
|
||||
+2
-1
@@ -113,6 +113,7 @@ class GDriveTree(BaseTree):
|
||||
)
|
||||
|
||||
self._bucket = self.path_info.bucket
|
||||
self._path = self.path_info.path
|
||||
self._trash_only = config.get("gdrive_trash_only")
|
||||
self._use_service_account = config.get("gdrive_use_service_account")
|
||||
self._service_account_email = config.get(
|
||||
@@ -284,7 +285,7 @@ class GDriveTree(BaseTree):
|
||||
),
|
||||
}
|
||||
|
||||
self._cache_path_id(self.path_info.path, cache["root_id"], cache)
|
||||
self._cache_path_id(self._path, cache["root_id"], cache)
|
||||
|
||||
for item in self._gdrive_list(
|
||||
"'{}' in parents and trashed=false".format(cache["root_id"])
|
||||
|
||||
+6
-11
@@ -42,9 +42,8 @@ class HTTPTree(BaseTree): # pylint:disable=abstract-method
|
||||
url = config.get("url")
|
||||
if url:
|
||||
self.path_info = self.PATH_CLS(url)
|
||||
user = config.get("user", None)
|
||||
if user:
|
||||
self.path_info.user = user
|
||||
self.user = config.get("user", None)
|
||||
self.host = self.path_info.host
|
||||
else:
|
||||
self.path_info = None
|
||||
|
||||
@@ -56,20 +55,16 @@ class HTTPTree(BaseTree): # pylint:disable=abstract-method
|
||||
self.ssl_verify = config.get("ssl_verify", True)
|
||||
self.method = config.get("method", "POST")
|
||||
|
||||
def _auth_method(self, path_info=None):
|
||||
def _auth_method(self):
|
||||
from requests.auth import HTTPBasicAuth, HTTPDigestAuth
|
||||
|
||||
if path_info is None:
|
||||
path_info = self.path_info
|
||||
|
||||
if self.auth:
|
||||
if self.ask_password and self.password is None:
|
||||
host, user = path_info.host, path_info.user
|
||||
self.password = ask_password(host, user)
|
||||
self.password = ask_password(self.host, self.user)
|
||||
if self.auth == "basic":
|
||||
return HTTPBasicAuth(path_info.user, self.password)
|
||||
return HTTPBasicAuth(self.user, self.password)
|
||||
if self.auth == "digest":
|
||||
return HTTPDigestAuth(path_info.user, self.password)
|
||||
return HTTPDigestAuth(self.user, self.password)
|
||||
if self.auth == "custom" and self.custom_auth_header:
|
||||
self.headers.update({self.custom_auth_header: self.password})
|
||||
return None
|
||||
|
||||
+3
-3
@@ -45,6 +45,7 @@ class OSSTree(BaseTree): # pylint:disable=abstract-method
|
||||
url = config.get("url")
|
||||
self.path_info = self.PATH_CLS(url) if url else None
|
||||
|
||||
self.bucket = self.path_info.bucket
|
||||
self.endpoint = config.get("oss_endpoint") or os.getenv("OSS_ENDPOINT")
|
||||
|
||||
self.key_id = (
|
||||
@@ -64,12 +65,11 @@ class OSSTree(BaseTree): # pylint:disable=abstract-method
|
||||
def oss_service(self):
|
||||
import oss2
|
||||
|
||||
logger.debug(f"URL: {self.path_info}")
|
||||
logger.debug(f"key id: {self.key_id}")
|
||||
logger.debug(f"key secret: {self.key_secret}")
|
||||
|
||||
auth = oss2.Auth(self.key_id, self.key_secret)
|
||||
bucket = oss2.Bucket(auth, self.endpoint, self.path_info.bucket)
|
||||
bucket = oss2.Bucket(auth, self.endpoint, self.bucket)
|
||||
|
||||
# Ensure bucket exists
|
||||
try:
|
||||
@@ -84,7 +84,7 @@ class OSSTree(BaseTree): # pylint:disable=abstract-method
|
||||
return bucket
|
||||
|
||||
def _generate_download_url(self, path_info, expires=3600):
|
||||
assert path_info.bucket == self.path_info.bucket
|
||||
assert path_info.bucket == self.bucket
|
||||
|
||||
return self.oss_service.sign_url("GET", path_info.path, expires)
|
||||
|
||||
|
||||
+10
-11
@@ -82,24 +82,23 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method
|
||||
if self.user is None and self.path_info.user is not None:
|
||||
self.user = self.path_info.user
|
||||
|
||||
# Construct hostname from path_info by stripping path
|
||||
http_info = HTTPURLInfo(self.path_info.url)
|
||||
self.hostname = http_info.replace(path="").url
|
||||
|
||||
# Webdav client
|
||||
@wrap_prop(threading.Lock())
|
||||
@cached_property
|
||||
def _client(self):
|
||||
from webdav3.client import Client
|
||||
|
||||
# Construct hostname from path_info by stripping path
|
||||
http_info = HTTPURLInfo(self.path_info.url)
|
||||
hostname = http_info.replace(path="").url
|
||||
|
||||
# Set password or ask for it
|
||||
if self.ask_password and self.password is None and self.token is None:
|
||||
host, user = self.path_info.host, self.path_info.user
|
||||
self.password = ask_password(host, user)
|
||||
self.password = ask_password(self.hostname, self.user)
|
||||
|
||||
# Setup webdav client options dictionary
|
||||
options = {
|
||||
"webdav_hostname": hostname,
|
||||
"webdav_hostname": self.hostname,
|
||||
"webdav_login": self.user,
|
||||
"webdav_password": self.password,
|
||||
"webdav_token": self.token,
|
||||
@@ -114,12 +113,12 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method
|
||||
# Check whether client options are valid
|
||||
if not client.valid():
|
||||
raise ConfigError(
|
||||
f"Configuration for WebDAV {hostname} is invalid."
|
||||
f"Configuration for WebDAV {self.hostname} is invalid."
|
||||
)
|
||||
|
||||
# Check whether connection is valid (root should always exist)
|
||||
if not client.check(self.path_info.path):
|
||||
raise WebDAVConnectionError(hostname)
|
||||
if not client.check("/"):
|
||||
raise WebDAVConnectionError(self.hostname)
|
||||
|
||||
return client
|
||||
|
||||
@@ -198,7 +197,7 @@ class WebDAVTree(BaseTree): # pylint:disable=abstract-method
|
||||
# Creates directories
|
||||
def makedirs(self, path_info):
|
||||
# Terminate recursion
|
||||
if path_info.path == self.path_info.path or self.exists(path_info):
|
||||
if path_info.path == "/" or self.exists(path_info):
|
||||
return
|
||||
|
||||
# Recursively descent to root
|
||||
|
||||
+9
-12
@@ -45,16 +45,13 @@ class WebHDFSTree(BaseTree):
|
||||
if not url:
|
||||
return
|
||||
|
||||
parsed = urlparse(url)
|
||||
user = parsed.username or config.get("user")
|
||||
self.path_info = self.PATH_CLS(url)
|
||||
|
||||
self.path_info = self.PATH_CLS.from_parts(
|
||||
scheme="webhdfs",
|
||||
host=parsed.hostname,
|
||||
user=user,
|
||||
port=parsed.port,
|
||||
path=parsed.path,
|
||||
)
|
||||
parsed = urlparse(url)
|
||||
|
||||
self.host = parsed.hostname
|
||||
self.user = parsed.username or config.get("user")
|
||||
self.port = parsed.port
|
||||
|
||||
self.hdfscli_config = config.get("hdfscli_config")
|
||||
self.token = config.get("webhdfs_token")
|
||||
@@ -65,7 +62,6 @@ class WebHDFSTree(BaseTree):
|
||||
def hdfs_client(self):
|
||||
import hdfs
|
||||
|
||||
logger.debug("URL: %s", self.path_info)
|
||||
logger.debug("HDFSConfig: %s", self.hdfscli_config)
|
||||
|
||||
try:
|
||||
@@ -82,13 +78,14 @@ class WebHDFSTree(BaseTree):
|
||||
if not any(err in exc_msg for err in errors):
|
||||
raise
|
||||
|
||||
http_url = f"http://{self.path_info.host}:{self.path_info.port}"
|
||||
http_url = f"http://{self.host}:{self.port}"
|
||||
logger.debug("URL: %s", http_url)
|
||||
|
||||
if self.token is not None:
|
||||
client = hdfs.TokenClient(http_url, token=self.token, root="/")
|
||||
else:
|
||||
client = hdfs.InsecureClient(
|
||||
http_url, user=self.path_info.user, root="/"
|
||||
http_url, user=self.user, root="/"
|
||||
)
|
||||
|
||||
return client
|
||||
|
||||
Reference in New Issue
Block a user