cloud: add ssh driver

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2018-04-08 22:26:18 +03:00
parent bdccd420dd
commit bd6688ff57
6 changed files with 143 additions and 2 deletions
+3
View File
@@ -205,3 +205,6 @@ class DataCloudBase(object):
def connect(self):
pass
def disconnect(self):
pass
+7 -1
View File
@@ -9,6 +9,7 @@ from dvc.utils import map_progress
from dvc.cloud.aws import DataCloudAWS
from dvc.cloud.gcp import DataCloudGCP
from dvc.cloud.ssh import DataCloudSSH
from dvc.cloud.local import DataCloudLOCAL
from dvc.cloud.base import DataCloudBase
@@ -19,6 +20,7 @@ class DataCloud(object):
CLOUD_MAP = {
'aws' : DataCloudAWS,
'gcp' : DataCloudGCP,
'ssh' : DataCloudSSH,
'local' : DataCloudLOCAL,
}
@@ -127,7 +129,11 @@ class DataCloud(object):
if collect_cloud:
collected |= self._collect(cloud, targets, jobs, False)
return map_progress(getattr(cloud, func), list(collected), jobs)
ret = map_progress(getattr(cloud, func), list(collected), jobs)
cloud.disconnect()
return ret
def push(self, targets, jobs=1, remote=None):
"""
+120
View File
@@ -0,0 +1,120 @@
import os
import errno
import filecmp
import paramiko
import posixpath
from dvc.logger import Logger
from dvc.utils import copyfile, file_md5
from dvc.cloud.base import DataCloudError, DataCloudBase
from dvc.cloud.aws import create_cb
from dvc.progress import progress
class SSHKey(object):
def __init__(self, bucket, name):
self.name = name
self.bucket = bucket
@property
def path(self):
return os.path.join(self.bucket, self.name)
class DataCloudSSH(DataCloudBase):
"""
Driver for remote storage over ssh.
"""
#NOTE: ~/ paths are temporarily forbidden
REGEX = r'^(?P<user>.*)@(?P<host>.*):(?P<path>/+.*)$'
@property
def hostname(self):
return self.group('host')
@property
def username(self):
return self.group('user')
def connect(self):
self.ssh = paramiko.SSHClient()
self.ssh.load_system_host_keys()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh.connect(self.hostname, username=self.username)
def disconnect(self):
self.ssh.close()
def get_sftp(self):
#NOTE: SFTP doesn't seem to be thread safe
return self.ssh.open_sftp()
def cache_file_key(self, path):
relpath = os.path.relpath(os.path.abspath(path), self._cloud_settings.cache.cache_dir)
return relpath.replace('\\', '/')
def _isfile_remote(self, path):
try:
self.get_sftp().open(path, 'r')
return True
except IOError as exc:
if exc.errno != errno.ENOENT:
raise
return False
def _cmp_checksum(self, key, path):
#FIXME actually compare md5
path_exists = os.path.exists(path)
key_exists = self._isfile_remote(key.path)
if path_exists and key_exists:
return True
return False
def _get_key(self, path):
key_name = self.cache_file_key(path)
key = SSHKey(self.path, key_name)
if self._isfile_remote(key.path):
return key
return None
def _makedirs_remote(self, dname):
try:
self.get_sftp().chdir(dname)
except IOError as exc:
if exc.errno != errno.ENOENT:
raise
parent = posixpath.dirname(dname)
if len(parent):
self._makedirs_remote(parent)
self.get_sftp().mkdir(dname)
def _new_key(self, path):
key_name = self.cache_file_key(path)
key = SSHKey(self.path, key_name)
self._makedirs_remote(posixpath.dirname(key.path))
return key
def _push_key(self, key, path):
self.get_sftp().put(path, key.path, callback=create_cb(key.name))
progress.finish_target(key.name)
return path
def _pull_key(self, key, path, no_progress_bar=False):
self._makedirs(path)
tmp_file = self.tmp_file(path)
try:
self.get_sftp().get(key.path, tmp_file, callback=create_cb(key.name))
except Exception as exc:
Logger.error('Failed to copy "{}": {}'.format(key.path, exc))
return None
os.rename(tmp_file, path)
progress.finish_target(key.name)
return path
+11 -1
View File
@@ -74,6 +74,14 @@ def copyfile(src, dest, no_progress_bar=False):
fdest.close()
def wrap(func, t):
try:
return func(t)
except Exception as exc:
Logger.error('wrap', exc)
raise
def map_progress(func, targets, n_threads):
"""
Process targets in multi-threaded mode with progress bar
@@ -82,8 +90,10 @@ def map_progress(func, targets, n_threads):
pool = ThreadPool(processes=n_threads)
ret = []
wrapper = lambda t: wrap(func, t)
try:
ret = pool.map(func, targets)
ret = pool.map(wrapper, targets)
except Exception as exc:
raise
finally:
+1
View File
@@ -16,3 +16,4 @@ setuptools>=34.0.0
nanotime>=0.5.2
pyasn1>=0.4.1
schema>=0.6.7
paramiko>=2.4.1
+1
View File
@@ -22,6 +22,7 @@ install_requires = [
"nanotime>=0.5.2",
"pyasn1>=0.4.1",
"schema>=0.6.7",
"paramiko>=2.4.1",
]
setup(