Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-09 23:10:24 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix CodeClimate inspections

Browse Source
This commit is contained in:
Clemens Wolff
2018-07-09 16:06:10 -04:00
parent 1bcd7408ec
commit afd46e1a8a
7 changed files with 31 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/remote/azure.py
+7 -22
View File
@@ -153,14 +153,7 @@ class RemoteAzure(RemoteBase):
return ret
def upload(self, paths, path_infos, names=None):
assert isinstance(paths, list)
assert isinstance(path_infos, list)
assert len(paths) == len(path_infos)
if not names:
names = len(paths) * [None]
else:
assert isinstance(names, list)
assert len(names) == len(paths)
names = self._verify_path_args(path_infos, paths, names)
for path, path_info, name in zip(paths, path_infos, names):
if path_info['scheme'] != self.scheme:
@@ -186,14 +179,7 @@ class RemoteAzure(RemoteBase):
progress.finish_target(name)
def download(self, path_infos, fnames, no_progress_bar=False, names=None):
assert isinstance(fnames, list)
assert isinstance(path_infos, list)
assert len(fnames) == len(path_infos)
if not names:
names = len(fnames) * [None]
else:
assert isinstance(names, list)
assert len(names) == len(fnames)
names = self._verify_path_args(path_infos, fnames, names)
for fname, path_info, name in zip(fnames, path_infos, names):
if path_info['scheme'] != self.scheme:
@@ -218,13 +204,12 @@ class RemoteAzure(RemoteBase):
bucket, key, tmp_file, progress_callback=cb)
except Exception as exc:
Logger.error("Failed to download '{}/{}'".format(
path_info['bucket'], path_info['key']), exc)
return
bucket, key), exc)
else:
os.rename(tmp_file, fname)
os.rename(tmp_file, fname)
if not no_progress_bar:
progress.finish_target(name)
if not no_progress_bar:
progress.finish_target(name)
def gc(self, checksum_infos):
used_etags = [info[self.PARAM_ETAG] for info in checksum_infos]
dvc/remote/base.py
+14
View File
@@ -97,3 +97,17 @@ class RemoteBase(object):
def exists(self, path_infos):
raise NotImplementedError
@classmethod
def _verify_path_args(cls, path_infos, fnames, names=None):
assert isinstance(fnames, list)
assert isinstance(path_infos, list)
assert len(fnames) == len(path_infos)
if not names:
names = len(fnames) * [None]
else:
assert isinstance(names, list)
assert len(names) == len(fnames)
return names
dvc/remote/gs.py
+2 -16
View File
@@ -116,14 +116,7 @@ class RemoteGS(RemoteBase):
return ret
def upload(self, paths, path_infos, names=None):
assert isinstance(paths, list)
assert isinstance(path_infos, list)
assert len(paths) == len(path_infos)
if not names:
names = len(paths) * [None]
else:
assert isinstance(names, list)
assert len(names) == len(paths)
names = self._verify_path_args(path_infos, paths, names)
gs = self.gs
@@ -151,14 +144,7 @@ class RemoteGS(RemoteBase):
progress.finish_target(name)
def download(self, path_infos, paths, no_progress_bar=False, names=None):
assert isinstance(paths, list)
assert isinstance(path_infos, list)
assert len(paths) == len(path_infos)
if not names:
names = len(paths) * [None]
else:
assert isinstance(names, list)
assert len(names) == len(paths)
names = self._verify_path_args(path_infos, paths, names)
gs = self.gs
dvc/remote/hdfs.py
+2 -16
View File
@@ -136,14 +136,7 @@ class RemoteHDFS(RemoteBase):
return ret
def upload(self, paths, path_infos, names=None):
assert isinstance(paths, list)
assert isinstance(path_infos, list)
assert len(paths) == len(path_infos)
if not names:
names = len(paths) * [None]
else:
assert isinstance(names, list)
assert len(names) == len(paths)
names = self._verify_path_args(path_infos, paths, names)
for path, path_info, name in zip(paths, path_infos, names):
if path_info['scheme'] != 'hdfs':
@@ -153,14 +146,7 @@ class RemoteHDFS(RemoteBase):
self.hadoop_fs('copyFromLocal {} {}'.format(path, path_info['url']), user=path_info['user'])
def download(self, path_infos, paths, no_progress_bar=False, names=None):
assert isinstance(paths, list)
assert isinstance(path_infos, list)
assert len(paths) == len(path_infos)
if not names:
names = len(paths) * [None]
else:
assert isinstance(names, list)
assert len(names) == len(paths)
names = self._verify_path_args(path_infos, paths, names)
for path, path_info, name in zip(paths, path_infos, names):
if path_info['scheme'] != 'hdfs':
dvc/remote/local.py
+2 -16
View File
@@ -271,14 +271,7 @@ class RemoteLOCAL(RemoteBase):
return ret
def upload(self, paths, path_infos, names=None):
assert isinstance(paths, list)
assert isinstance(path_infos, list)
assert len(paths) == len(path_infos)
if not names:
names = len(paths) * [None]
else:
assert isinstance(names, list)
assert len(names) == len(paths)
names = self._verify_path_args(path_infos, paths, names)
for path, path_info, name in zip(paths, path_infos, names):
if path_info['scheme'] != 'local':
@@ -297,14 +290,7 @@ class RemoteLOCAL(RemoteBase):
Logger.error("Failed to upload '{}' tp '{}'".format(path, path_info['path']))
def download(self, path_infos, paths, no_progress_bar=False, names=None):
assert isinstance(paths, list)
assert isinstance(path_infos, list)
assert len(paths) == len(path_infos)
if not names:
names = len(paths) * [None]
else:
assert isinstance(names, list)
assert len(names) == len(paths)
names = self._verify_path_args(path_infos, paths, names)
for path, path_info, name in zip(paths, path_infos, names):
if path_info['scheme'] != 'local':
dvc/remote/s3.py
+2 -16
View File
@@ -150,14 +150,7 @@ class RemoteS3(RemoteBase):
return ret
def upload(self, paths, path_infos, names=None):
assert isinstance(paths, list)
assert isinstance(path_infos, list)
assert len(paths) == len(path_infos)
if not names:
names = len(paths) * [None]
else:
assert isinstance(names, list)
assert len(names) == len(paths)
names = self._verify_path_args(path_infos, paths, names)
session = boto3.session.Session()
s3 = session.client('s3')
@@ -185,14 +178,7 @@ class RemoteS3(RemoteBase):
progress.finish_target(name)
def download(self, path_infos, fnames, no_progress_bar=False, names=None):
assert isinstance(fnames, list)
assert isinstance(path_infos, list)
assert len(fnames) == len(path_infos)
if not names:
names = len(fnames) * [None]
else:
assert isinstance(names, list)
assert len(names) == len(fnames)
names = self._verify_path_args(path_infos, fnames, names)
session = boto3.session.Session()
s3 = session.client('s3')
dvc/remote/ssh.py
+2 -16
View File
@@ -180,14 +180,7 @@ class RemoteSSH(RemoteBase):
ssh.close()
def download(self, path_infos, paths, no_progress_bar=False, names=None):
assert isinstance(paths, list)
assert isinstance(path_infos, list)
assert len(paths) == len(path_infos)
if not names:
names = len(paths) * [None]
else:
assert isinstance(names, list)
assert len(names) == len(paths)
names = self._verify_path_args(path_infos, paths, names)
ssh = self.ssh(host=path_infos[0]['host'], user=path_infos[0]['user'])
@@ -217,14 +210,7 @@ class RemoteSSH(RemoteBase):
ssh.close()
def upload(self, paths, path_infos, names=None):
assert isinstance(paths, list)
assert isinstance(path_infos, list)
assert len(paths) == len(path_infos)
if not names:
names = len(paths) * [None]
else:
assert isinstance(names, list)
assert len(names) == len(paths)
names = self._verify_path_args(path_infos, paths, names)
ssh = self.ssh(host=path_infos[0]['host'], user=path_infos[0]['user'])
sftp = ssh.open_sftp()