diff --git a/dvc/cloud/base.py b/dvc/cloud/base.py index cce7625b9..3e41a5a2b 100644 --- a/dvc/cloud/base.py +++ b/dvc/cloud/base.py @@ -205,3 +205,6 @@ class DataCloudBase(object): def connect(self): pass + + def disconnect(self): + pass diff --git a/dvc/cloud/data_cloud.py b/dvc/cloud/data_cloud.py index 480cf8a36..319890aa4 100644 --- a/dvc/cloud/data_cloud.py +++ b/dvc/cloud/data_cloud.py @@ -9,6 +9,7 @@ from dvc.utils import map_progress from dvc.cloud.aws import DataCloudAWS from dvc.cloud.gcp import DataCloudGCP +from dvc.cloud.ssh import DataCloudSSH from dvc.cloud.local import DataCloudLOCAL from dvc.cloud.base import DataCloudBase @@ -19,6 +20,7 @@ class DataCloud(object): CLOUD_MAP = { 'aws' : DataCloudAWS, 'gcp' : DataCloudGCP, + 'ssh' : DataCloudSSH, 'local' : DataCloudLOCAL, } @@ -127,7 +129,11 @@ class DataCloud(object): if collect_cloud: collected |= self._collect(cloud, targets, jobs, False) - return map_progress(getattr(cloud, func), list(collected), jobs) + ret = map_progress(getattr(cloud, func), list(collected), jobs) + + cloud.disconnect() + + return ret def push(self, targets, jobs=1, remote=None): """ diff --git a/dvc/cloud/ssh.py b/dvc/cloud/ssh.py new file mode 100644 index 000000000..680317af9 --- /dev/null +++ b/dvc/cloud/ssh.py @@ -0,0 +1,120 @@ +import os +import errno +import filecmp +import paramiko +import posixpath + +from dvc.logger import Logger +from dvc.utils import copyfile, file_md5 +from dvc.cloud.base import DataCloudError, DataCloudBase +from dvc.cloud.aws import create_cb +from dvc.progress import progress + + +class SSHKey(object): + def __init__(self, bucket, name): + self.name = name + self.bucket = bucket + + @property + def path(self): + return os.path.join(self.bucket, self.name) + + +class DataCloudSSH(DataCloudBase): + """ + Driver for remote storage over ssh. + """ + #NOTE: ~/ paths are temporarily forbidden + REGEX = r'^(?P.*)@(?P.*):(?P/+.*)$' + + @property + def hostname(self): + return self.group('host') + + @property + def username(self): + return self.group('user') + + def connect(self): + self.ssh = paramiko.SSHClient() + + self.ssh.load_system_host_keys() + self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + self.ssh.connect(self.hostname, username=self.username) + + def disconnect(self): + self.ssh.close() + + def get_sftp(self): + #NOTE: SFTP doesn't seem to be thread safe + return self.ssh.open_sftp() + + def cache_file_key(self, path): + relpath = os.path.relpath(os.path.abspath(path), self._cloud_settings.cache.cache_dir) + return relpath.replace('\\', '/') + + def _isfile_remote(self, path): + try: + self.get_sftp().open(path, 'r') + return True + except IOError as exc: + if exc.errno != errno.ENOENT: + raise + return False + + def _cmp_checksum(self, key, path): + #FIXME actually compare md5 + path_exists = os.path.exists(path) + key_exists = self._isfile_remote(key.path) + + if path_exists and key_exists: + return True + + return False + + def _get_key(self, path): + key_name = self.cache_file_key(path) + key = SSHKey(self.path, key_name) + if self._isfile_remote(key.path): + return key + return None + + def _makedirs_remote(self, dname): + try: + self.get_sftp().chdir(dname) + except IOError as exc: + if exc.errno != errno.ENOENT: + raise + + parent = posixpath.dirname(dname) + if len(parent): + self._makedirs_remote(parent) + self.get_sftp().mkdir(dname) + + def _new_key(self, path): + key_name = self.cache_file_key(path) + key = SSHKey(self.path, key_name) + self._makedirs_remote(posixpath.dirname(key.path)) + return key + + def _push_key(self, key, path): + self.get_sftp().put(path, key.path, callback=create_cb(key.name)) + progress.finish_target(key.name) + return path + + def _pull_key(self, key, path, no_progress_bar=False): + self._makedirs(path) + + tmp_file = self.tmp_file(path) + try: + self.get_sftp().get(key.path, tmp_file, callback=create_cb(key.name)) + except Exception as exc: + Logger.error('Failed to copy "{}": {}'.format(key.path, exc)) + return None + + os.rename(tmp_file, path) + progress.finish_target(key.name) + + return path diff --git a/dvc/utils.py b/dvc/utils.py index b9aaf62a1..56270352f 100644 --- a/dvc/utils.py +++ b/dvc/utils.py @@ -74,6 +74,14 @@ def copyfile(src, dest, no_progress_bar=False): fdest.close() +def wrap(func, t): + try: + return func(t) + except Exception as exc: + Logger.error('wrap', exc) + raise + + def map_progress(func, targets, n_threads): """ Process targets in multi-threaded mode with progress bar @@ -82,8 +90,10 @@ def map_progress(func, targets, n_threads): pool = ThreadPool(processes=n_threads) ret = [] + wrapper = lambda t: wrap(func, t) + try: - ret = pool.map(func, targets) + ret = pool.map(wrapper, targets) except Exception as exc: raise finally: diff --git a/requirements.txt b/requirements.txt index a74e6217a..7914e1987 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ setuptools>=34.0.0 nanotime>=0.5.2 pyasn1>=0.4.1 schema>=0.6.7 +paramiko>=2.4.1 diff --git a/setup.py b/setup.py index 51921f1af..2e9289c2f 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ install_requires = [ "nanotime>=0.5.2", "pyasn1>=0.4.1", "schema>=0.6.7", + "paramiko>=2.4.1", ] setup(