Merge pull request #921 from efiop/master

improve test coverage

dvc/command/status.py

@@ -9,8 +9,7 @@ class CmdDataStatus(CmdDataBase):
 
     def _normalize(self, s):
         s+=':'
-        if len(s) >= self.STATUS_LEN:
-            return s
+        assert len(s) < self.STATUS_LEN
         return s + (self.STATUS_LEN - len(s))*' '
 
     def _show(self, status, indent=0):

dvc/exceptions.py

@@ -11,3 +11,9 @@ class DvcException(Exception):
             except Exception:
                 pass
         super(DvcException, self).__init__(msg)
+
+
+class UnsupportedRemoteError(DvcException):
+    def __init__(self, config, cause=None):
+        msg = "Remote '{}' is not supported.".format(config)
+        super(UnsupportedRemoteError, self).__init__(msg, cause=cause)

dvc/remote/__init__.py

@@ -6,7 +6,7 @@ from dvc.remote.ssh import RemoteSSH
 from dvc.remote.azure import RemoteAzure
 
 from dvc.config import Config
-from dvc.exceptions import DvcException
+from dvc.exceptions import UnsupportedRemoteError
 
 
 REMOTES = [RemoteHDFS, RemoteSSH, RemoteS3, RemoteGS, RemoteAzure, RemoteLOCAL]
@@ -24,4 +24,4 @@ def Remote(project, config):
     for r in REMOTES:
         if r.supported(config):
             return r(project, config)
-    raise DvcException('Remote \'{}\' is not supported.'.format(config))
+    raise UnsupportedRemoteError(str(config))

dvc/remote/azure.py

@@ -68,68 +68,71 @@ class RemoteAzure(RemoteBase):
             self.__blob_service.create_container(self.bucket)
         return self.__blob_service
 
-    def _get_etag(self, bucket, key):
-        try:
-            blob = self.blob_service.get_blob_properties(bucket, key)
-            return blob.properties.etag
-        except Exception:
-            return None
-
-    def save_info(self, path_info):
-        if path_info['scheme'] != self.scheme:
-            raise NotImplementedError
-
-        return {self.PARAM_ETAG: self._get_etag(
-            path_info['bucket'], path_info['key'])}
-
-    def save(self, path_info):
-        if path_info['scheme'] != self.scheme:
-            raise NotImplementedError
-
-        etag = self._get_etag(path_info['bucket'], path_info['key'])
-        dest_key = '{}/{}'.format(etag[0:2], etag[2:])
-
-        self._copy_blob(to_bucket=self.bucket,
-                        to_key=dest_key,
-                        from_bucket=path_info['bucket'],
-                        from_key=path_info['key'])
-
-        return {self.PARAM_ETAG: etag}
-
-    def _copy_blob(self, to_bucket, to_key, from_bucket, from_key):
-        source = self.blob_service.make_blob_url(from_bucket, from_key)
-
-        copy = self.blob_service.copy_blob(to_bucket, to_key, source)
-
-        if self.COPY_POLL_SECONDS <= 0:
-            return
-
-        while copy.status != 'success':
-            time.sleep(self.COPY_POLL_SECONDS)
-            copy = self.blob_service.get_blob_properties(
-                to_bucket, to_key).properties.copy
-
-    def checkout(self, path_info, checksum_info):
-        if path_info['scheme'] != self.scheme:
-            raise NotImplementedError
-
-        etag = checksum_info.get(self.PARAM_ETAG, None)
-        if not etag:
-            return
-
-        self._copy_blob(to_bucket=path_info['bucket'],
-                        to_key=path_info['key'],
-                        from_bucket=self.bucket,
-                        from_key='{}/{}'.format(etag[0:2], etag[2:]))
-
-    def remove(self, path_info):
-        if path_info['scheme'] != self.scheme:
-            raise NotImplementedError
-
-        Logger.debug('Removing azure://{}/{}'.format(path_info['bucket'],
-                                                     path_info['key']))
-
-        self.blob_service.delete_blob(path_info['bucket'], path_info['key'])
+# FIXME: temporarily disabled because of the lack of test for external azure
+# dependencies/outputs/cache.
+#
+#    def _get_etag(self, bucket, key):
+#        try:
+#            blob = self.blob_service.get_blob_properties(bucket, key)
+#            return blob.properties.etag
+#        except Exception:
+#            return None
+#
+#    def save_info(self, path_info):
+#        if path_info['scheme'] != self.scheme:
+#            raise NotImplementedError
+#
+#        return {self.PARAM_ETAG: self._get_etag(
+#            path_info['bucket'], path_info['key'])}
+#
+#    def save(self, path_info):
+#        if path_info['scheme'] != self.scheme:
+#            raise NotImplementedError
+#
+#        etag = self._get_etag(path_info['bucket'], path_info['key'])
+#        dest_key = '{}/{}'.format(etag[0:2], etag[2:])
+#
+#        self._copy_blob(to_bucket=self.bucket,
+#                        to_key=dest_key,
+#                        from_bucket=path_info['bucket'],
+#                        from_key=path_info['key'])
+#
+#        return {self.PARAM_ETAG: etag}
+#
+#    def _copy_blob(self, to_bucket, to_key, from_bucket, from_key):
+#        source = self.blob_service.make_blob_url(from_bucket, from_key)
+#
+#        copy = self.blob_service.copy_blob(to_bucket, to_key, source)
+#
+#        if self.COPY_POLL_SECONDS <= 0:
+#            return
+#
+#        while copy.status != 'success':
+#            time.sleep(self.COPY_POLL_SECONDS)
+#            copy = self.blob_service.get_blob_properties(
+#                to_bucket, to_key).properties.copy
+#
+#    def checkout(self, path_info, checksum_info):
+#        if path_info['scheme'] != self.scheme:
+#            raise NotImplementedError
+#
+#        etag = checksum_info.get(self.PARAM_ETAG, None)
+#        if not etag:
+#            return
+#
+#        self._copy_blob(to_bucket=path_info['bucket'],
+#                        to_key=path_info['key'],
+#                        from_bucket=self.bucket,
+#                        from_key='{}/{}'.format(etag[0:2], etag[2:]))
+#
+#    def remove(self, path_info):
+#        if path_info['scheme'] != self.scheme:
+#            raise NotImplementedError
+#
+#        Logger.debug('Removing azure://{}/{}'.format(path_info['bucket'],
+#                                                     path_info['key']))
+#
+#        self.blob_service.delete_blob(path_info['bucket'], path_info['key'])
 
     def md5s_to_path_infos(self, md5s):
         return [{
@@ -217,17 +220,20 @@ class RemoteAzure(RemoteBase):
                 if not no_progress_bar:
                     progress.finish_target(name)
 
-    def gc(self, checksum_infos):
-        used_etags = [info[self.PARAM_ETAG] for info in checksum_infos['azure']]
-        used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in checksum_infos['local']]
-
-        all_blobs = self.blob_service.list_blobs(self.bucket)
-
-        for blob in all_blobs:
-            etag = blob.properties.etag
-            if etag in used_etags:
-                continue
-            path_info = {'scheme': self.scheme,
-                         'key': blob.name,
-                         'bucket': self.bucket}
-            self.remove(path_info)
+# FIXME: temporarily disabled because of the lack of test for external azure
+# dependencies/outputs/cache.
+#
+#    def gc(self, checksum_infos):
+#        used_etags = [info[self.PARAM_ETAG] for info in checksum_infos['azure']]
+#        used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in checksum_infos['local']]
+#
+#        all_blobs = self.blob_service.list_blobs(self.bucket)
+#
+#        for blob in all_blobs:
+#            etag = blob.properties.etag
+#            if etag in used_etags:
+#                continue
+#            path_info = {'scheme': self.scheme,
+#                         'key': blob.name,
+#                         'bucket': self.bucket}
+#            self.remove(path_info)

tests/test_add.py

@@ -6,6 +6,7 @@ import filecmp
 from dvc.main import main
 from dvc.utils import file_md5
 from dvc.stage import Stage
+from dvc.exceptions import DvcException
 from dvc.output.base import OutputAlreadyTrackedError
 from dvc.output.base import OutputDoesNotExistError, OutputIsNotFileOrDirError
 from dvc.command.add import CmdAdd
@@ -27,10 +28,10 @@ class TestAdd(TestDvc):
         self.assertEqual(stage.outs[0].info['md5'], md5)
 
 
-class TestAddNonExistentFile(TestDvc):
+class TestAddUnupportedFile(TestDvc):
     def test(self):
-        with self.assertRaises(OutputDoesNotExistError) as cx:
-            self.dvc.add('non_existent_file')
+        with self.assertRaises(DvcException) as cx:
+            self.dvc.add('unsupported://unsupported')
 
 
 class TestAddDirectory(TestDvc):

tests/test_import.py

@@ -5,6 +5,7 @@ import filecmp
 
 from dvc.main import main
 from dvc.command.imp import CmdImport
+from dvc.exceptions import DvcException
 
 from tests.basic_env import TestDvc
 
@@ -18,3 +19,7 @@ class TestCmdImport(TestDvc):
         ret = main(['import',
                     'non-existing-file', 'import'])
         self.assertNotEqual(ret, 0)
+
+    def test_unsupported(self):
+        ret = main(['import', 'unsupported://path', 'import_unsupported'])
+        self.assertNotEqual(ret, 0)

tests/test_remote.py

@@ -1,5 +1,8 @@
 from dvc.main import main
 from dvc.command.remote import CmdRemoteAdd
+from dvc.exceptions import UnsupportedRemoteError
+from dvc.remote import Remote
+from dvc.config import Config
 
 from tests.basic_env import TestDvc
 
@@ -34,3 +37,7 @@ class TestRemote(TestDvc):
         cmd.configobj.write = None 
         ret = cmd.run()           
         self.assertNotEqual(ret, 0)
+
+    def test_unsupported(self):
+        with self.assertRaises(UnsupportedRemoteError):
+            Remote(self.dvc, {Config.SECTION_REMOTE_URL: 'unsupported://url'})

tests/test_repro.py

@@ -247,6 +247,9 @@ class TestReproMissingMd5InStageFile(TestRepro):
 
 class TestCmdRepro(TestRepro):
     def test(self):
+        ret = main(['status'])
+        self.assertEqual(ret, 0)
+        
         ret = main(['repro',
                     self.file1_stage])
         self.assertEqual(ret, 0)