Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

s3: Use default boto3 session

Browse Source 
Fixes #1041
This commit is contained in:
Mr. Outis
2018-11-07 18:19:53 -06:00
parent b6842fb089
commit c06eca5709
2 changed files with 21 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/config.py
+6 -6
View File
@@ -117,9 +117,9 @@ class Config(object):
SECTION_AWS_SCHEMA = {
SECTION_AWS_STORAGEPATH: str,
Optional(SECTION_AWS_REGION): str,
Optional(SECTION_AWS_PROFILE, default='default'): str,
Optional(SECTION_AWS_CREDENTIALPATH, default=''): str,
Optional(SECTION_AWS_ENDPOINT_URL, default=None): str,
Optional(SECTION_AWS_PROFILE): str,
Optional(SECTION_AWS_CREDENTIALPATH): str,
Optional(SECTION_AWS_ENDPOINT_URL): str,
}
# backward compatibility
@@ -150,9 +150,9 @@ class Config(object):
SECTION_REMOTE_SCHEMA = {
SECTION_REMOTE_URL: And(supported_url, error="Unsupported URL"),
Optional(SECTION_AWS_REGION): str,
Optional(SECTION_AWS_PROFILE, default='default'): str,
Optional(SECTION_AWS_CREDENTIALPATH, default=''): str,
Optional(SECTION_AWS_ENDPOINT_URL, default=None): str,
Optional(SECTION_AWS_PROFILE): str,
Optional(SECTION_AWS_CREDENTIALPATH): str,
Optional(SECTION_AWS_ENDPOINT_URL): str,
Optional(SECTION_GCP_PROJECTNAME): str,
Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
Optional(SECTION_CACHE_PROTECTED,
dvc/remote/s3.py
+15 -34
View File
@@ -40,29 +40,22 @@ class RemoteS3(RemoteBase):
PARAM_ETAG = 'etag'
def __init__(self, project, config):
import configobj
self.project = project
storagepath = 's3://' + config.get(Config.SECTION_AWS_STORAGEPATH,
'').lstrip('/')
storagepath = 's3://{}'.format(
config.get(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/')
)
self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
self.region = config.get(Config.SECTION_AWS_REGION, None)
self.profile = os.getenv('AWS_PROFILE',
config.get(Config.SECTION_AWS_PROFILE,
'default'))
self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL, None)
self.region = config.get(Config.SECTION_AWS_REGION)
self.profile = config.get(Config.SECTION_AWS_PROFILE)
self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
# backward compatibility
credentialpath = config.get(Config.SECTION_AWS_CREDENTIALPATH)
credentialpath = config.get(Config.SECTION_AWS_CREDENTIALPATH, None)
if credentialpath:
creds_conf = configobj.ConfigObj(credentialpath)
creds = creds_conf.get(self.profile, {})
else:
creds = self._get_aws_config_from_env()
self.creds = creds
self.region = creds.get('region', self.region)
self.aws_access_key_id = creds.get('aws_access_key_id', None)
self.aws_secret_access_key = creds.get('aws_secret_access_key', None)
os.environ['AWS_SHARED_CREDENTIALS_FILE'] = credentialpath
@staticmethod
def compat_config(config):
@@ -81,22 +74,10 @@ class RemoteS3(RemoteBase):
@property
def s3(self):
if not self.creds:
session = boto3.session.Session(profile_name=self.profile)
else:
session = boto3.session.Session(
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
region_name=self.region)
return session.client('s3', endpoint_url=self.endpoint_url)
session = boto3.session.Session(profile_name=self.profile,
region_name=self.region)
def _get_aws_config_from_env(self):
creds = {
'region': os.getenv('AWS_DEFAULT_REGION'),
'aws_access_key_id': os.getenv('AWS_ACCESS_KEY_ID'),
'aws_secret_access_key': os.getenv('AWS_SECRET_ACCESS_KEY'),
}
return {k: v for k, v in creds.items() if v is not None}
return session.client('s3', endpoint_url=self.endpoint_url)
def get_etag(self, bucket, key):
try: