mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 07:18:27 +01:00
cloud: fixup for #404
Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
@@ -0,0 +1,91 @@
|
||||
import os
|
||||
import configparser
|
||||
|
||||
from dvc.utils import cached_property
|
||||
from dvc.logger import Logger
|
||||
|
||||
|
||||
class AWSCredentials(object):
|
||||
def __init__(self, cloud_config):
|
||||
self._conf_credpath = cloud_config.get('CredentialPath', None)
|
||||
self._conf_credsect = cloud_config.get('CredentialSection', 'default')
|
||||
|
||||
@property
|
||||
def access_key_id(self):
|
||||
if self.creds:
|
||||
return self.creds[0]
|
||||
return None
|
||||
|
||||
@property
|
||||
def secret_access_key(self):
|
||||
if self.creds:
|
||||
return self.creds[1]
|
||||
return None
|
||||
|
||||
@cached_property
|
||||
def creds(self):
|
||||
return self._get_credentials()
|
||||
|
||||
def _get_credentials(self):
|
||||
""" gets aws credentials, looking in various places
|
||||
|
||||
Params:
|
||||
|
||||
Searches:
|
||||
1 any override in dvc.conf [AWS] CredentialPath;
|
||||
2 ~/.aws/credentials
|
||||
|
||||
|
||||
Returns:
|
||||
if successfully found, (access_key_id, secret)
|
||||
None otherwise
|
||||
"""
|
||||
|
||||
# FIX: It won't work in Windows.
|
||||
default_path = os.path.expanduser('~/.aws/credentials')
|
||||
default_sect = 'default'
|
||||
default_cred_location = (default_path, default_sect)
|
||||
|
||||
cred_locations = self._credential_paths(default_cred_location)
|
||||
for cred_location in cred_locations:
|
||||
try:
|
||||
path = cred_location[0]
|
||||
section = cred_location[1]
|
||||
|
||||
cc = configparser.SafeConfigParser()
|
||||
|
||||
# use readfp(open( ... to aid mocking.
|
||||
cc.readfp(open(path, 'r'))
|
||||
|
||||
if section in cc.keys():
|
||||
access_key = cc[section].get('aws_access_key_id', None)
|
||||
secret = cc[section].get('aws_secret_access_key', None)
|
||||
|
||||
if access_key is not None and secret is not None:
|
||||
return (access_key, secret)
|
||||
else:
|
||||
Logger.warn('Unable to find section {} in AWS credential file {}'.format(section, path))
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
def _credential_paths(self, default_cred_location):
|
||||
results = []
|
||||
if self._conf_credpath is not None and len(self._conf_credpath) > 0:
|
||||
credpath = os.path.expanduser(self._conf_credpath)
|
||||
if os.path.isfile(credpath):
|
||||
results.append((credpath, self._conf_credsect))
|
||||
else:
|
||||
msg = 'AWS CredentialPath {} not found; falling back to default file {} and section {}'
|
||||
Logger.warn(msg.format(credpath, default_cred_location[0], default_cred_location[1]))
|
||||
results.append(default_cred_location)
|
||||
else:
|
||||
results.append(default_cred_location)
|
||||
return results
|
||||
|
||||
def sanity_check(self):
|
||||
creds = self._get_credentials()
|
||||
if creds is None:
|
||||
Logger.info("can't find aws credetials, assuming envirment variables or iam role")
|
||||
# self._aws_creds = creds
|
||||
@@ -0,0 +1,4 @@
|
||||
class CloudSettings(object):
|
||||
def __init__(self, global_storage_path, cloud_config):
|
||||
self.cloud_config = cloud_config
|
||||
self.global_storage_path = global_storage_path
|
||||
+3
-94
@@ -29,13 +29,15 @@ except ImportError:
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import dvc
|
||||
from dvc.cloud.instance_manager import CloudSettings
|
||||
from dvc.logger import Logger
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.config import ConfigError
|
||||
from dvc.progress import progress
|
||||
from dvc.utils import copyfile
|
||||
from dvc.cloud.credentials_aws import AWSCredentials
|
||||
from dvc.system import System
|
||||
from dvc.utils import map_progress, cached_property
|
||||
from dvc.utils import map_progress
|
||||
|
||||
STATUS_UNKNOWN = 0
|
||||
STATUS_OK = 1
|
||||
@@ -51,99 +53,6 @@ STATUS_MAP = {
|
||||
(False, True, None) : STATUS_DELETED,
|
||||
}
|
||||
|
||||
|
||||
class CloudSettings(object):
|
||||
def __init__(self, global_storage_path, cloud_config):
|
||||
self.cloud_config = cloud_config
|
||||
self.global_storage_path = global_storage_path
|
||||
|
||||
|
||||
class AWSCredentials(object):
|
||||
def __init__(self, cloud_config):
|
||||
self._conf_credpath = cloud_config.get('CredentialPath', None)
|
||||
self._conf_credsect = cloud_config.get('CredentialSection', 'default')
|
||||
|
||||
@property
|
||||
def access_key_id(self):
|
||||
if self.creds:
|
||||
return self.creds[0]
|
||||
return None
|
||||
|
||||
@property
|
||||
def secret_access_key(self):
|
||||
if self.creds:
|
||||
return self.creds[1]
|
||||
return None
|
||||
|
||||
@cached_property
|
||||
def creds(self):
|
||||
return self._get_credentials()
|
||||
|
||||
def _get_credentials(self):
|
||||
""" gets aws credentials, looking in various places
|
||||
|
||||
Params:
|
||||
|
||||
Searches:
|
||||
1 any override in dvc.conf [AWS] CredentialPath;
|
||||
2 ~/.aws/credentials
|
||||
|
||||
|
||||
Returns:
|
||||
if successfully found, (access_key_id, secret)
|
||||
None otherwise
|
||||
"""
|
||||
|
||||
# FIX: It won't work in Windows.
|
||||
default_path = os.path.expanduser('~/.aws/credentials')
|
||||
default_sect = 'default'
|
||||
default_cred_location = (default_path, default_sect)
|
||||
|
||||
cred_locations = self._credential_paths(default_cred_location)
|
||||
for cred_location in cred_locations:
|
||||
try:
|
||||
path = cred_location[0]
|
||||
section = cred_location[1]
|
||||
|
||||
cc = configparser.SafeConfigParser()
|
||||
|
||||
# use readfp(open( ... to aid mocking.
|
||||
cc.readfp(open(path, 'r'))
|
||||
|
||||
if section in cc.keys():
|
||||
access_key = cc[section].get('aws_access_key_id', None)
|
||||
secret = cc[section].get('aws_secret_access_key', None)
|
||||
|
||||
if access_key is not None and secret is not None:
|
||||
return (access_key, secret)
|
||||
else:
|
||||
Logger.warn('Unable to find section {} in AWS credential file {}'.format(section, path))
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
def _credential_paths(self, default_cred_location):
|
||||
results = []
|
||||
if self._conf_credpath is not None and len(self._conf_credpath) > 0:
|
||||
credpath = os.path.expanduser(self._conf_credpath)
|
||||
if os.path.isfile(credpath):
|
||||
results.append((credpath, self._conf_credsect))
|
||||
else:
|
||||
msg = 'AWS CredentialPath {} not found; falling back to default file {} and section {}'
|
||||
Logger.warn(msg.format(credpath, default_cred_location[0], default_cred_location[1]))
|
||||
results.append(default_cred_location)
|
||||
else:
|
||||
results.append(default_cred_location)
|
||||
return results
|
||||
|
||||
def sanity_check(self):
|
||||
creds = self._get_credentials()
|
||||
if creds is None:
|
||||
Logger.info("can't find aws credetials, assuming envirment variables or iam role")
|
||||
# self._aws_creds = creds
|
||||
|
||||
|
||||
class DataCloudError(DvcException):
|
||||
""" Data Cloud exception """
|
||||
def __init__(self, msg):
|
||||
|
||||
@@ -4,7 +4,7 @@ import uuid
|
||||
from dvc.config import ConfigError
|
||||
from dvc.data_cloud import DataCloud, DataCloudAWS, DataCloudLOCAL
|
||||
from dvc.data_cloud import STATUS_UNKNOWN, STATUS_OK, STATUS_MODIFIED, STATUS_NEW, STATUS_DELETED
|
||||
from dvc.data_cloud import CloudSettings
|
||||
from dvc.cloud.instance_manager import CloudSettings
|
||||
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
Reference in New Issue
Block a user