mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 07:18:27 +01:00
dvc: misc changes for DataCloud, 'data sync' and 'remove'
This patch introduces DataCloud class that used config to determine proper cloud class(DataCloudAWS and DataCloudGCP are awailable) to use in "data_sync" and "remove" commands. This patch focuses on introducing the common class itself saving original code and definetly needs some RFC later. It also adds simple DataCloudLOCAL class that uses directory as a StoragePath and cp as a sync method. Which makes testing so much easier and covers local scenarios. And it also introduces ability to run data sync in parallel by collecting files first and then running sync in a number of threads. Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
@@ -68,9 +68,6 @@ class CmdBase(object):
|
||||
def config(self):
|
||||
return self._settings.config
|
||||
|
||||
def cache_file_key(self, file):
|
||||
return '{}/{}'.format(self.config.storage_prefix, file).strip('/')
|
||||
|
||||
@property
|
||||
def dvc_home(self):
|
||||
return self._settings.dvc_home
|
||||
@@ -147,12 +144,6 @@ class CmdBase(object):
|
||||
def run(self):
|
||||
pass
|
||||
|
||||
def get_cache_file_s3_name(self, cache_file):
|
||||
cache_prefix_file_name = os.path.relpath(System.realpath(cache_file), System.realpath(self.git.git_dir))
|
||||
file_name = os.path.relpath(cache_prefix_file_name, self.config.cache_dir)
|
||||
dvc_file_path_trim = file_name.replace(os.sep, '/').strip('/')
|
||||
return self.config.aws_storage_prefix + '/' + dvc_file_path_trim
|
||||
|
||||
@staticmethod
|
||||
def warning_dvc_is_busy():
|
||||
Logger.warn('Cannot perform the cmd since DVC is busy and locked. Please retry the cmd later.')
|
||||
|
||||
Reference in New Issue
Block a user