From afd46e1a8a9033b6580ed0a99907979b59bcb42d Mon Sep 17 00:00:00 2001 From: Clemens Wolff Date: Mon, 9 Jul 2018 16:02:30 -0400 Subject: [PATCH] Fix CodeClimate inspections --- dvc/remote/azure.py | 29 +++++++---------------------- dvc/remote/base.py | 14 ++++++++++++++ dvc/remote/gs.py | 18 ++---------------- dvc/remote/hdfs.py | 18 ++---------------- dvc/remote/local.py | 18 ++---------------- dvc/remote/s3.py | 18 ++---------------- dvc/remote/ssh.py | 18 ++---------------- 7 files changed, 31 insertions(+), 102 deletions(-) diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index 6f6b09f63..6134716df 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -153,14 +153,7 @@ class RemoteAzure(RemoteBase): return ret def upload(self, paths, path_infos, names=None): - assert isinstance(paths, list) - assert isinstance(path_infos, list) - assert len(paths) == len(path_infos) - if not names: - names = len(paths) * [None] - else: - assert isinstance(names, list) - assert len(names) == len(paths) + names = self._verify_path_args(path_infos, paths, names) for path, path_info, name in zip(paths, path_infos, names): if path_info['scheme'] != self.scheme: @@ -186,14 +179,7 @@ class RemoteAzure(RemoteBase): progress.finish_target(name) def download(self, path_infos, fnames, no_progress_bar=False, names=None): - assert isinstance(fnames, list) - assert isinstance(path_infos, list) - assert len(fnames) == len(path_infos) - if not names: - names = len(fnames) * [None] - else: - assert isinstance(names, list) - assert len(names) == len(fnames) + names = self._verify_path_args(path_infos, fnames, names) for fname, path_info, name in zip(fnames, path_infos, names): if path_info['scheme'] != self.scheme: @@ -218,13 +204,12 @@ class RemoteAzure(RemoteBase): bucket, key, tmp_file, progress_callback=cb) except Exception as exc: Logger.error("Failed to download '{}/{}'".format( - path_info['bucket'], path_info['key']), exc) - return + bucket, key), exc) + else: + os.rename(tmp_file, fname) - os.rename(tmp_file, fname) - - if not no_progress_bar: - progress.finish_target(name) + if not no_progress_bar: + progress.finish_target(name) def gc(self, checksum_infos): used_etags = [info[self.PARAM_ETAG] for info in checksum_infos] diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 94ad18cc2..da101a808 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -97,3 +97,17 @@ class RemoteBase(object): def exists(self, path_infos): raise NotImplementedError + + @classmethod + def _verify_path_args(cls, path_infos, fnames, names=None): + assert isinstance(fnames, list) + assert isinstance(path_infos, list) + assert len(fnames) == len(path_infos) + + if not names: + names = len(fnames) * [None] + else: + assert isinstance(names, list) + assert len(names) == len(fnames) + + return names diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index 5285f4b5d..5a0c19206 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -116,14 +116,7 @@ class RemoteGS(RemoteBase): return ret def upload(self, paths, path_infos, names=None): - assert isinstance(paths, list) - assert isinstance(path_infos, list) - assert len(paths) == len(path_infos) - if not names: - names = len(paths) * [None] - else: - assert isinstance(names, list) - assert len(names) == len(paths) + names = self._verify_path_args(path_infos, paths, names) gs = self.gs @@ -151,14 +144,7 @@ class RemoteGS(RemoteBase): progress.finish_target(name) def download(self, path_infos, paths, no_progress_bar=False, names=None): - assert isinstance(paths, list) - assert isinstance(path_infos, list) - assert len(paths) == len(path_infos) - if not names: - names = len(paths) * [None] - else: - assert isinstance(names, list) - assert len(names) == len(paths) + names = self._verify_path_args(path_infos, paths, names) gs = self.gs diff --git a/dvc/remote/hdfs.py b/dvc/remote/hdfs.py index 773a8dbe6..ea1e3cf19 100644 --- a/dvc/remote/hdfs.py +++ b/dvc/remote/hdfs.py @@ -136,14 +136,7 @@ class RemoteHDFS(RemoteBase): return ret def upload(self, paths, path_infos, names=None): - assert isinstance(paths, list) - assert isinstance(path_infos, list) - assert len(paths) == len(path_infos) - if not names: - names = len(paths) * [None] - else: - assert isinstance(names, list) - assert len(names) == len(paths) + names = self._verify_path_args(path_infos, paths, names) for path, path_info, name in zip(paths, path_infos, names): if path_info['scheme'] != 'hdfs': @@ -153,14 +146,7 @@ class RemoteHDFS(RemoteBase): self.hadoop_fs('copyFromLocal {} {}'.format(path, path_info['url']), user=path_info['user']) def download(self, path_infos, paths, no_progress_bar=False, names=None): - assert isinstance(paths, list) - assert isinstance(path_infos, list) - assert len(paths) == len(path_infos) - if not names: - names = len(paths) * [None] - else: - assert isinstance(names, list) - assert len(names) == len(paths) + names = self._verify_path_args(path_infos, paths, names) for path, path_info, name in zip(paths, path_infos, names): if path_info['scheme'] != 'hdfs': diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 1b8d79483..07bf7fb10 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -271,14 +271,7 @@ class RemoteLOCAL(RemoteBase): return ret def upload(self, paths, path_infos, names=None): - assert isinstance(paths, list) - assert isinstance(path_infos, list) - assert len(paths) == len(path_infos) - if not names: - names = len(paths) * [None] - else: - assert isinstance(names, list) - assert len(names) == len(paths) + names = self._verify_path_args(path_infos, paths, names) for path, path_info, name in zip(paths, path_infos, names): if path_info['scheme'] != 'local': @@ -297,14 +290,7 @@ class RemoteLOCAL(RemoteBase): Logger.error("Failed to upload '{}' tp '{}'".format(path, path_info['path'])) def download(self, path_infos, paths, no_progress_bar=False, names=None): - assert isinstance(paths, list) - assert isinstance(path_infos, list) - assert len(paths) == len(path_infos) - if not names: - names = len(paths) * [None] - else: - assert isinstance(names, list) - assert len(names) == len(paths) + names = self._verify_path_args(path_infos, paths, names) for path, path_info, name in zip(paths, path_infos, names): if path_info['scheme'] != 'local': diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index c57e34906..03c699eb2 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -150,14 +150,7 @@ class RemoteS3(RemoteBase): return ret def upload(self, paths, path_infos, names=None): - assert isinstance(paths, list) - assert isinstance(path_infos, list) - assert len(paths) == len(path_infos) - if not names: - names = len(paths) * [None] - else: - assert isinstance(names, list) - assert len(names) == len(paths) + names = self._verify_path_args(path_infos, paths, names) session = boto3.session.Session() s3 = session.client('s3') @@ -185,14 +178,7 @@ class RemoteS3(RemoteBase): progress.finish_target(name) def download(self, path_infos, fnames, no_progress_bar=False, names=None): - assert isinstance(fnames, list) - assert isinstance(path_infos, list) - assert len(fnames) == len(path_infos) - if not names: - names = len(fnames) * [None] - else: - assert isinstance(names, list) - assert len(names) == len(fnames) + names = self._verify_path_args(path_infos, fnames, names) session = boto3.session.Session() s3 = session.client('s3') diff --git a/dvc/remote/ssh.py b/dvc/remote/ssh.py index d3f941db8..e0539d206 100644 --- a/dvc/remote/ssh.py +++ b/dvc/remote/ssh.py @@ -180,14 +180,7 @@ class RemoteSSH(RemoteBase): ssh.close() def download(self, path_infos, paths, no_progress_bar=False, names=None): - assert isinstance(paths, list) - assert isinstance(path_infos, list) - assert len(paths) == len(path_infos) - if not names: - names = len(paths) * [None] - else: - assert isinstance(names, list) - assert len(names) == len(paths) + names = self._verify_path_args(path_infos, paths, names) ssh = self.ssh(host=path_infos[0]['host'], user=path_infos[0]['user']) @@ -217,14 +210,7 @@ class RemoteSSH(RemoteBase): ssh.close() def upload(self, paths, path_infos, names=None): - assert isinstance(paths, list) - assert isinstance(path_infos, list) - assert len(paths) == len(path_infos) - if not names: - names = len(paths) * [None] - else: - assert isinstance(names, list) - assert len(names) == len(paths) + names = self._verify_path_args(path_infos, paths, names) ssh = self.ssh(host=path_infos[0]['host'], user=path_infos[0]['user']) sftp = ssh.open_sftp()