mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 23:11:44 +01:00
remote: protect all remote client/session creation code with locks
With @cached_property last write wins, so all threads will use the same session. However, several might be created concurrently wasting resources and even potentially asking something from the user. This change guarantees only one client instance will be created for a remote.
This commit is contained in:
+4
-3
@@ -4,10 +4,10 @@ from __future__ import unicode_literals
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
from datetime import datetime, timedelta
|
||||
import threading
|
||||
|
||||
from funcy import cached_property
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.path_info import CloudURLInfo
|
||||
@@ -64,6 +64,7 @@ class RemoteAZURE(RemoteBASE):
|
||||
else self.path_cls.from_parts(scheme=self.scheme, netloc=bucket)
|
||||
)
|
||||
|
||||
@wrap_prop(threading.Lock())
|
||||
@cached_property
|
||||
def blob_service(self):
|
||||
from azure.storage.blob import BlockBlobService
|
||||
|
||||
+3
-1
@@ -5,8 +5,9 @@ from datetime import timedelta
|
||||
from functools import wraps
|
||||
import io
|
||||
import os.path
|
||||
import threading
|
||||
|
||||
from funcy import cached_property
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.exceptions import DvcException
|
||||
@@ -91,6 +92,7 @@ class RemoteGS(RemoteBASE):
|
||||
self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
|
||||
self.credentialpath = config.get(Config.SECTION_GCP_CREDENTIALPATH)
|
||||
|
||||
@wrap_prop(threading.Lock())
|
||||
@cached_property
|
||||
def gs(self):
|
||||
from google.cloud.storage import Client
|
||||
|
||||
+3
-1
@@ -1,8 +1,9 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import logging
|
||||
import threading
|
||||
|
||||
from funcy import cached_property
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.config import ConfigError
|
||||
@@ -81,6 +82,7 @@ class RemoteHTTP(RemoteBASE):
|
||||
|
||||
return etag
|
||||
|
||||
@wrap_prop(threading.Lock())
|
||||
@cached_property
|
||||
def _session(self):
|
||||
import requests
|
||||
|
||||
+22
-20
@@ -3,6 +3,9 @@ from __future__ import unicode_literals
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.path_info import CloudURLInfo
|
||||
@@ -61,30 +64,29 @@ class RemoteOSS(RemoteBASE):
|
||||
or "defaultSecret"
|
||||
)
|
||||
|
||||
self._bucket = None
|
||||
|
||||
@property
|
||||
@wrap_prop(threading.Lock())
|
||||
@cached_property
|
||||
def oss_service(self):
|
||||
import oss2
|
||||
|
||||
if self._bucket is None:
|
||||
logger.debug("URL {}".format(self.path_info))
|
||||
logger.debug("key id {}".format(self.key_id))
|
||||
logger.debug("key secret {}".format(self.key_secret))
|
||||
auth = oss2.Auth(self.key_id, self.key_secret)
|
||||
self._bucket = oss2.Bucket(
|
||||
auth, self.endpoint, self.path_info.bucket
|
||||
logger.debug("URL {}".format(self.path_info))
|
||||
logger.debug("key id {}".format(self.key_id))
|
||||
logger.debug("key secret {}".format(self.key_secret))
|
||||
|
||||
auth = oss2.Auth(self.key_id, self.key_secret)
|
||||
bucket = oss2.Bucket(auth, self.endpoint, self.path_info.bucket)
|
||||
|
||||
# Ensure bucket exists
|
||||
try:
|
||||
bucket.get_bucket_info()
|
||||
except oss2.exceptions.NoSuchBucket:
|
||||
bucket.create_bucket(
|
||||
oss2.BUCKET_ACL_PUBLIC_READ,
|
||||
oss2.models.BucketCreateConfig(
|
||||
oss2.BUCKET_STORAGE_CLASS_STANDARD
|
||||
),
|
||||
)
|
||||
try: # verify that bucket exists
|
||||
self._bucket.get_bucket_info()
|
||||
except oss2.exceptions.NoSuchBucket:
|
||||
self._bucket.create_bucket(
|
||||
oss2.BUCKET_ACL_PUBLIC_READ,
|
||||
oss2.models.BucketCreateConfig(
|
||||
oss2.BUCKET_STORAGE_CLASS_STANDARD
|
||||
),
|
||||
)
|
||||
return self._bucket
|
||||
return bucket
|
||||
|
||||
def remove(self, path_info):
|
||||
if path_info.scheme != self.scheme:
|
||||
|
||||
+3
-1
@@ -1,7 +1,8 @@
|
||||
from collections import deque
|
||||
from contextlib import contextmanager
|
||||
import threading
|
||||
|
||||
from funcy import memoize
|
||||
from funcy import memoize, wrap_with
|
||||
|
||||
|
||||
@contextmanager
|
||||
@@ -17,6 +18,7 @@ def get_connection(conn_func, *args, **kwargs):
|
||||
pool.release(conn)
|
||||
|
||||
|
||||
@wrap_with(threading.Lock())
|
||||
@memoize
|
||||
def get_pool(conn_func, *args, **kwargs):
|
||||
return Pool(conn_func, *args, **kwargs)
|
||||
|
||||
+3
-1
@@ -3,8 +3,9 @@ from __future__ import unicode_literals
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
|
||||
from funcy import cached_property
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.config import Config
|
||||
from dvc.exceptions import DvcException
|
||||
@@ -56,6 +57,7 @@ class RemoteS3(RemoteBASE):
|
||||
if shared_creds:
|
||||
os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
|
||||
|
||||
@wrap_prop(threading.Lock())
|
||||
@cached_property
|
||||
def s3(self):
|
||||
import boto3
|
||||
|
||||
Reference in New Issue
Block a user