dvc: misc changes for DataCloud, 'data sync' and 'remove'

This patch introduces DataCloud class that used config to
determine proper cloud class(DataCloudAWS and DataCloudGCP
are awailable) to use in "data_sync" and "remove" commands.

This patch focuses on introducing the common class itself
saving original code and definetly needs some RFC later.

It also adds simple DataCloudLOCAL class that uses directory
as a StoragePath and cp as a sync method. Which makes testing
so much easier and covers local scenarios.

And it also introduces ability to run data sync in parallel
by collecting files first and then running sync in a number
of threads.

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2017-05-28 22:28:58 +03:00
parent adcd9038ce
commit a01ef3197c
7 changed files with 417 additions and 402 deletions
+4 -21
View File
@@ -6,7 +6,7 @@ from dvc.command.base import CmdBase, DvcLock
from dvc.exceptions import DvcException
from dvc.logger import Logger
from dvc.runtime import Runtime
from dvc.data_cloud import DataCloud
class DataRemoveError(DvcException):
def __init__(self, msg):
@@ -30,6 +30,8 @@ class CmdDataRemove(CmdBase):
def run(self):
with DvcLock(self.is_locker, self.git):
self.cloud = DataCloud(self.settings)
if not self.remove_all_targets():
return 1
@@ -94,8 +96,7 @@ class CmdDataRemove(CmdBase):
def _remove_cloud_cache(self, data_item):
if not self.parsed_args.keep_in_cloud:
aws_key = self.cache_file_key(data_item.cache.dvc)
self.remove_from_cloud(aws_key)
self.cloud.remove_from_cloud(data_item)
def _remove_state_file(self, data_item):
if os.path.isfile(data_item.state.relative):
@@ -119,24 +120,6 @@ class CmdDataRemove(CmdBase):
self.remove_dir_if_empty(dvc_path.relative)
Logger.debug(u'[Cmd-Remove] Remove {}. Success.'.format(name))
def remove_from_cloud(self, aws_file_name):
Logger.debug(u'[Cmd-Remove] Remove from cloud {}.'.format(aws_file_name))
if not self.config.aws_access_key_id or not self.config.aws_secret_access_key:
Logger.debug('[Cmd-Remove] Unable to check cache file in the cloud')
return
conn = S3Connection(self.config.aws_access_key_id, self.config.aws_secret_access_key)
bucket_name = self.config.storage_bucket
bucket = conn.lookup(bucket_name)
if bucket:
key = bucket.get_key(aws_file_name)
if not key:
Logger.warn('[Cmd-Remove] S3 remove warning: file "{}" does not exist in S3'.format(aws_file_name))
else:
key.delete()
Logger.info('[Cmd-Remove] File "{}" was removed from S3'.format(aws_file_name))
pass
def remove_dir_file_by_file(self, target):
for f in os.listdir(target):
file = os.path.join(target, f)