Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-13 23:10:15 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1306 from mroutis/fix-1041

Browse Source 
s3: Use default boto3 session
This commit is contained in:
Ruslan Kuprieiev
2018-11-08 12:06:00 +02:00
committed by GitHub
parent e6e72febf9 4ee615e5b4
commit 1922d1189d
2 changed files with 28 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/config.py
+6 -6
View File
@@ -117,9 +117,9 @@ class Config(object):
SECTION_AWS_SCHEMA = {
SECTION_AWS_STORAGEPATH: str,
Optional(SECTION_AWS_REGION): str,
Optional(SECTION_AWS_PROFILE, default='default'): str,
Optional(SECTION_AWS_CREDENTIALPATH, default=''): str,
Optional(SECTION_AWS_ENDPOINT_URL, default=None): str,
Optional(SECTION_AWS_PROFILE): str,
Optional(SECTION_AWS_CREDENTIALPATH): str,
Optional(SECTION_AWS_ENDPOINT_URL): str,
}
# backward compatibility
@@ -150,9 +150,9 @@ class Config(object):
SECTION_REMOTE_SCHEMA = {
SECTION_REMOTE_URL: And(supported_url, error="Unsupported URL"),
Optional(SECTION_AWS_REGION): str,
Optional(SECTION_AWS_PROFILE, default='default'): str,
Optional(SECTION_AWS_CREDENTIALPATH, default=''): str,
Optional(SECTION_AWS_ENDPOINT_URL, default=None): str,
Optional(SECTION_AWS_PROFILE): str,
Optional(SECTION_AWS_CREDENTIALPATH): str,
Optional(SECTION_AWS_ENDPOINT_URL): str,
Optional(SECTION_GCP_PROJECTNAME): str,
Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
Optional(SECTION_CACHE_PROTECTED,
dvc/remote/s3.py
+22 -34
View File
@@ -40,29 +40,29 @@ class RemoteS3(RemoteBase):
PARAM_ETAG = 'etag'
def __init__(self, project, config):
import configobj
self.project = project
storagepath = 's3://' + config.get(Config.SECTION_AWS_STORAGEPATH,
'').lstrip('/')
storagepath = 's3://{}'.format(
config.get(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/')
)
self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
self.region = config.get(Config.SECTION_AWS_REGION, None)
self.profile = os.getenv('AWS_PROFILE',
config.get(Config.SECTION_AWS_PROFILE,
'default'))
self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL, None)
credentialpath = config.get(Config.SECTION_AWS_CREDENTIALPATH, None)
if credentialpath:
creds_conf = configobj.ConfigObj(credentialpath)
creds = creds_conf.get(self.profile, {})
else:
creds = self._get_aws_config_from_env()
self.region = (
os.environ.get('AWS_DEFAULT_REGION') or
config.get(Config.SECTION_AWS_REGION)
)
self.creds = creds
self.region = creds.get('region', self.region)
self.aws_access_key_id = creds.get('aws_access_key_id', None)
self.aws_secret_access_key = creds.get('aws_secret_access_key', None)
self.profile = (
os.environ.get('AWS_PROFILE') or
config.get(Config.SECTION_AWS_PROFILE)
)
self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
if shared_creds:
os.environ.setdefault('AWS_SHARED_CREDENTIALS_FILE', shared_creds)
@staticmethod
def compat_config(config):
@@ -81,22 +81,10 @@ class RemoteS3(RemoteBase):
@property
def s3(self):
if not self.creds:
session = boto3.session.Session(profile_name=self.profile)
else:
session = boto3.session.Session(
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
region_name=self.region)
return session.client('s3', endpoint_url=self.endpoint_url)
session = boto3.session.Session(profile_name=self.profile,
region_name=self.region)
def _get_aws_config_from_env(self):
creds = {
'region': os.getenv('AWS_DEFAULT_REGION'),
'aws_access_key_id': os.getenv('AWS_ACCESS_KEY_ID'),
'aws_secret_access_key': os.getenv('AWS_SECRET_ACCESS_KEY'),
}
return {k: v for k, v in creds.items() if v is not None}
return session.client('s3', endpoint_url=self.endpoint_url)
def get_etag(self, bucket, key):
try: